AWS: Query for location of S3 bucket, use that

Previously we would rely on the s3 bucket's region being configured correctly, at least for the existence check. By querying for the bucket region and then going direct to the correct region, we avoid errors and we avoid potential eventual consistency problems. May be related to issue: #12109

AWS: Query for location of S3 bucket, use that
230de787 · Justin Santa Barbara · 6129d3d4 · 230de787
Commit 230de787 authored Aug 01, 2015 by Justin Santa Barbara
Hide whitespace changes
Inline Side-by-side

Showing with 7 additions and 5 deletions

util.sh cluster/aws/util.sh +7 -5

No files found.
--- a/cluster/aws/util.sh
+++ b/cluster/aws/util.sh
@@ -487,7 +487,8 @@ function upload-server-tars() {
  fi

  echo "Uploading to Amazon S3"
-  if ! aws s3 ls "s3://${AWS_S3_BUCKET}" > /dev/null 2>&1 ; then
+
+  if ! aws s3api get-bucket-location --bucket ${AWS_S3_BUCKET} > /dev/null 2>&1 ; then
    echo "Creating ${AWS_S3_BUCKET}"

    # Buckets must be globally uniquely named, so always create in a known region
@@ -497,7 +498,7 @@ function upload-server-tars() {

    local attempt=0
    while true; do
-      if ! aws s3 ls "s3://${AWS_S3_BUCKET}" > /dev/null 2>&1; then
+      if ! aws s3 ls --region ${AWS_S3_REGION} "s3://${AWS_S3_BUCKET}" > /dev/null 2>&1; then
        if (( attempt > 5 )); then
          echo
          echo -e "${color_red}Unable to confirm bucket creation." >&2
@@ -518,6 +519,7 @@ function upload-server-tars() {
  if [[ "${s3_bucket_location}" == "None" ]]; then
    # "US Classic" does not follow the pattern
    s3_url_base=https://s3.amazonaws.com
+    s3_bucket_location=us-east-1
  fi

  local -r staging_path="devel"
@@ -530,13 +532,13 @@ function upload-server-tars() {
  cp -a "${SERVER_BINARY_TAR}" ${local_dir}
  cp -a "${SALT_TAR}" ${local_dir}

-  aws s3 sync --exact-timestamps ${local_dir} "s3://${AWS_S3_BUCKET}/${staging_path}/"
+  aws s3 sync --region ${s3_bucket_location} --exact-timestamps ${local_dir} "s3://${AWS_S3_BUCKET}/${staging_path}/"

-  aws s3api put-object-acl --bucket ${AWS_S3_BUCKET} --key "${server_binary_path}" --grant-read 'uri="http://acs.amazonaws.com/groups/global/AllUsers"'
+  aws s3api put-object-acl --region ${s3_bucket_location} --bucket ${AWS_S3_BUCKET} --key "${server_binary_path}" --grant-read 'uri="http://acs.amazonaws.com/groups/global/AllUsers"'
  SERVER_BINARY_TAR_URL="${s3_url_base}/${AWS_S3_BUCKET}/${server_binary_path}"

  local salt_tar_path="${staging_path}/${SALT_TAR##*/}"
-  aws s3api put-object-acl --bucket ${AWS_S3_BUCKET} --key "${salt_tar_path}" --grant-read 'uri="http://acs.amazonaws.com/groups/global/AllUsers"'
+  aws s3api put-object-acl --region ${s3_bucket_location} --bucket ${AWS_S3_BUCKET} --key "${salt_tar_path}" --grant-read 'uri="http://acs.amazonaws.com/groups/global/AllUsers"'
  SALT_TAR_URL="${s3_url_base}/${AWS_S3_BUCKET}/${salt_tar_path}"
 }