Merge pull request #22234 from zmerlynn/regional-buckets

cluster/gce/config-default.sh

@@ -18,6 +18,7 @@
 # gcloud multiplexing for shared GCE/GKE tests.
 GCLOUD=gcloud
 ZONE=${KUBE_GCE_ZONE:-us-central1-b}
+RELEASE_REGION_FALLBACK=${RELEASE_REGION_FALLBACK:-false}
 MASTER_SIZE=${MASTER_SIZE:-n1-standard-2}
 NODE_SIZE=${NODE_SIZE:-n1-standard-2}
 NUM_NODES=${NUM_NODES:-3}

cluster/gce/config-test.sh

@@ -18,6 +18,7 @@
 # gcloud multiplexing for shared GCE/GKE tests.
 GCLOUD=gcloud
 ZONE=${KUBE_GCE_ZONE:-us-central1-b}
+RELEASE_REGION_FALLBACK=${RELEASE_REGION_FALLBACK:-false}
 MASTER_SIZE=${MASTER_SIZE:-n1-standard-2}
 NODE_SIZE=${NODE_SIZE:-n1-standard-2}
 NUM_NODES=${NUM_NODES:-3}

cluster/gce/configure-vm.sh

@@ -140,15 +140,32 @@ function remove-docker-artifacts() {
   echo "== Finished deleting docker0 =="
 }
 
-# Retry a download until we get it.
+# Retry a download until we get it. Takes a hash and a set of URLs.
 #
-# $1 is the URL to download
+# $1 is the sha1 of the URL. Can be "" if the sha1 is unknown.
+# $2+ are the URLs to download.
 download-or-bust() {
-  local -r url="$1"
-  local -r file="${url##*/}"
-  rm -f "$file"
-  until curl --ipv4 -Lo "$file" --connect-timeout 20 --retry 6 --retry-delay 10 "${url}"; do
-    echo "Failed to download file (${url}). Retrying."
+  local -r hash="$1"
+  shift 1
+
+  urls=( $* )
+  while true; do
+    for url in "${urls[@]}"; do
+      local file="${url##*/}"
+      rm -f "${file}"
+      if ! curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10 "${url}"; then
+        echo "== Failed to download ${url}. Retrying. =="
+      elif [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then
+        echo "== Hash validation of ${url} failed. Retrying. =="
+      else
+        if [[ -n "${hash}" ]]; then
+          echo "== Downloaded ${url} (SHA1 = ${hash}) =="
+        else
+          echo "== Downloaded ${url} =="
+        fi
+        return
+      fi
+    done
   done
 }
 
@@ -603,39 +620,43 @@ EOF
   fi
 }
 
+function split-commas() {
+  echo $1 | tr "," "\n"
+}
+
 function try-download-release() {
   # TODO(zmerlynn): Now we REALLy have no excuse not to do the reboot
   # optimization.
 
-  # TODO(zmerlynn): This may not be set yet by everyone (GKE).
-  if [[ -z "${SERVER_BINARY_TAR_HASH:-}" ]]; then
+  local -r server_binary_tar_urls=( $(split-commas "${SERVER_BINARY_TAR_URL}") )
+  local -r server_binary_tar="${server_binary_tar_urls[0]##*/}"
+  if [[ -n "${SERVER_BINARY_TAR_HASH:-}" ]]; then
+    local -r server_binary_tar_hash="${SERVER_BINARY_TAR_HASH}"
+  else
     echo "Downloading binary release sha1 (not found in env)"
-    download-or-bust "${SERVER_BINARY_TAR_URL}.sha1"
-    SERVER_BINARY_TAR_HASH=$(cat "${SERVER_BINARY_TAR_URL##*/}.sha1")
+    download-or-bust "" "${server_binary_tar_urls[@]/.tar.gz/.tar.gz.sha1}"
+    local -r server_binary_tar_hash=$(cat "${server_binary_tar}.sha1")
   fi
 
-  echo "Downloading binary release tar (${SERVER_BINARY_TAR_URL})"
-  download-or-bust "${SERVER_BINARY_TAR_URL}"
+  echo "Downloading binary release tar (${server_binary_tar_urls[@]})"
+  download-or-bust "${server_binary_tar_hash}" "${server_binary_tar_urls[@]}"
 
-  validate-hash "${SERVER_BINARY_TAR_URL##*/}" "${SERVER_BINARY_TAR_HASH}"
-  echo "Validated ${SERVER_BINARY_TAR_URL} SHA1 = ${SERVER_BINARY_TAR_HASH}"
-
-  # TODO(zmerlynn): This may not be set yet by everyone (GKE).
-  if [[ -z "${SALT_TAR_HASH:-}" ]]; then
+  local -r salt_tar_urls=( $(split-commas "${SALT_TAR_URL}") )
+  local -r salt_tar="${salt_tar_urls[0]##*/}"
+  if [[ -n "${SALT_TAR_HASH:-}" ]]; then
+    local -r salt_tar_hash="${SALT_TAR_HASH}"
+  else
     echo "Downloading Salt tar sha1 (not found in env)"
-    download-or-bust "${SALT_TAR_URL}.sha1"
-    SALT_TAR_HASH=$(cat "${SALT_TAR_URL##*/}.sha1")
+    download-or-bust "" "${salt_tar_urls[@]/.tar.gz/.tar.gz.sha1}"
+    local -r salt_tar_hash=$(cat "${salt_tar}.sha1")
   fi
 
-  echo "Downloading Salt tar ($SALT_TAR_URL)"
-  download-or-bust "$SALT_TAR_URL"
-
-  validate-hash "${SALT_TAR_URL##*/}" "${SALT_TAR_HASH}"
-  echo "Validated ${SALT_TAR_URL} SHA1 = ${SALT_TAR_HASH}"
+  echo "Downloading Salt tar (${salt_tar_urls[@]})"
+  download-or-bust "${salt_tar_hash}" "${salt_tar_urls[@]}"
 
   echo "Unpacking Salt tree and checking integrity of binary release tar"
   rm -rf kubernetes
-  tar xzf "${SALT_TAR_URL##*/}" && tar tzf "${SERVER_BINARY_TAR_URL##*/}" > /dev/null
+  tar xzf "${salt_tar}" && tar tzf "${server_binary_tar}" > /dev/null
 }
 
 function download-release() {

cluster/gce/util.sh

@@ -115,33 +115,43 @@ function detect-project () {
   fi
 }
 
-function already-staged() {
-  local -r file=$1
-  local -r newsum=$2
-
-  [[ -e "${file}.uploaded.sha1" ]] || return 1
-
-  local oldsum
-  oldsum=$(cat "${file}.uploaded.sha1")
-
-  [[ "${oldsum}" == "${newsum}" ]]
-}
-
-# Copy a release tar, if we don't already think it's staged in GCS
-function copy-if-not-staged() {
+# Copy a release tar and its accompanying hash.
+function copy-to-staging() {
   local -r staging_path=$1
   local -r gs_url=$2
   local -r tar=$3
   local -r hash=$4
 
-  if already-staged "${tar}" "${hash}"; then
-    echo "+++ $(basename ${tar}) already staged ('rm ${tar}.uploaded.sha1' to force)"
-  else
-    echo "${hash}" > "${tar}.sha1"
-    gsutil -m -q -h "Cache-Control:private, max-age=0" cp "${tar}" "${tar}.sha1" "${staging_path}"
-    gsutil -m acl ch -g all:R "${gs_url}" "${gs_url}.sha1" >/dev/null 2>&1
-    echo "${hash}" > "${tar}.uploaded.sha1"
-    echo "+++ $(basename ${tar}) uploaded (sha1 = ${hash})"
+  echo "${hash}" > "${tar}.sha1"
+  gsutil -m -q -h "Cache-Control:private, max-age=0" cp "${tar}" "${tar}.sha1" "${staging_path}"
+  gsutil -m acl ch -g all:R "${gs_url}" "${gs_url}.sha1" >/dev/null 2>&1
+  echo "+++ $(basename ${tar}) uploaded (sha1 = ${hash})"
+}
+
+# Given the cluster zone, return the list of regional GCS release
+# bucket suffixes for the release in preference order. GCS doesn't
+# give us an API for this, so we hardcode it.
+#
+# Assumed vars:
+#   REGIONAL_RELEASE
+#   ZONE
+# Vars set:
+#   PREFERRED_REGION
+function set-preferred-region() {
+  case ${ZONE} in
+    asia-*)
+      PREFERRED_REGION=("asia" "us" "eu")
+      ;;
+    europe-*)
+      PREFERRED_REGION=("eu" "us" "asia")
+      ;;
+    *)
+      PREFERRED_REGION=("us" "eu" "asia")
+      ;;
+  esac
+
+  if [[ "${RELEASE_REGION_FALLBACK}" != "true" ]]; then
+    PREFERRED_REGION=( "${PREFERRED_REGION[0]}" )
   fi
 }
 
@@ -155,6 +165,7 @@ function copy-if-not-staged() {
 #   SERVER_BINARY_TAR
 #   SALT_TAR
 #   KUBE_MANIFESTS_TAR
+#   ZONE
 # Vars set:
 #   SERVER_BINARY_TAR_URL
 #   SERVER_BINARY_TAR_HASH
@@ -181,35 +192,59 @@ function upload-server-tars() {
   # that's probably good enough for now :P
   project_hash=${project_hash:0:10}
 
-  local -r staging_bucket="gs://kubernetes-staging-${project_hash}"
+  set-preferred-region
 
-  # Ensure the bucket is created
-  if ! gsutil ls "$staging_bucket" > /dev/null 2>&1 ; then
-    echo "Creating $staging_bucket"
-    gsutil mb "${staging_bucket}"
+  SERVER_BINARY_TAR_HASH=$(sha1sum-file "${SERVER_BINARY_TAR}")
+  SALT_TAR_HASH=$(sha1sum-file "${SALT_TAR}")
+  if [[ "${OS_DISTRIBUTION}" == "trusty" || "${OS_DISTRIBUTION}" == "coreos" ]]; then
+    KUBE_MANIFESTS_TAR_HASH=$(sha1sum-file "${KUBE_MANIFESTS_TAR}")
   fi
 
-  local -r staging_path="${staging_bucket}/${INSTANCE_PREFIX}-devel"
+  local server_binary_tar_urls=()
+  local salt_tar_urls=()
+  local kube_manifest_tar_urls=()
 
-  SERVER_BINARY_TAR_HASH=$(sha1sum-file "${SERVER_BINARY_TAR}")
-  SALT_TAR_HASH=$(sha1sum-file "${SALT_TAR}")
+  for region in "${PREFERRED_REGION[@]}"; do
+    suffix="-${region}"
+    if [[ "${suffix}" == "-us" ]]; then
+      suffix=""
+    fi
+    local staging_bucket="gs://kubernetes-staging-${project_hash}${suffix}"
 
-  echo "+++ Staging server tars to Google Storage: ${staging_path}"
-  local server_binary_gs_url="${staging_path}/${SERVER_BINARY_TAR##*/}"
-  local salt_gs_url="${staging_path}/${SALT_TAR##*/}"
-  copy-if-not-staged "${staging_path}" "${server_binary_gs_url}" "${SERVER_BINARY_TAR}" "${SERVER_BINARY_TAR_HASH}"
-  copy-if-not-staged "${staging_path}" "${salt_gs_url}" "${SALT_TAR}" "${SALT_TAR_HASH}"
+    # Ensure the buckets are created
+    if ! gsutil ls "${staging_bucket}" > /dev/null 2>&1 ; then
+      echo "Creating ${staging_bucket}"
+      gsutil mb -l "${region}" "${staging_bucket}"
+    fi
 
-  # Convert from gs:// URL to an https:// URL
-  SERVER_BINARY_TAR_URL="${server_binary_gs_url/gs:\/\//https://storage.googleapis.com/}"
-  SALT_TAR_URL="${salt_gs_url/gs:\/\//https://storage.googleapis.com/}"
+    local staging_path="${staging_bucket}/${INSTANCE_PREFIX}-devel"
+
+    echo "+++ Staging server tars to Google Storage: ${staging_path}"
+    local server_binary_gs_url="${staging_path}/${SERVER_BINARY_TAR##*/}"
+    local salt_gs_url="${staging_path}/${SALT_TAR##*/}"
+    copy-to-staging "${staging_path}" "${server_binary_gs_url}" "${SERVER_BINARY_TAR}" "${SERVER_BINARY_TAR_HASH}"
+    copy-to-staging "${staging_path}" "${salt_gs_url}" "${SALT_TAR}" "${SALT_TAR_HASH}"
 
-  if [[ "${OS_DISTRIBUTION}" == "trusty" || "${OS_DISTRIBUTION}" == "coreos" ]]; then
-    local kube_manifests_gs_url="${staging_path}/${KUBE_MANIFESTS_TAR##*/}"
-    KUBE_MANIFESTS_TAR_HASH=$(sha1sum-file "${KUBE_MANIFESTS_TAR}")
-    copy-if-not-staged "${staging_path}" "${kube_manifests_gs_url}" "${KUBE_MANIFESTS_TAR}" "${KUBE_MANIFESTS_TAR_HASH}"
     # Convert from gs:// URL to an https:// URL
-    KUBE_MANIFESTS_TAR_URL="${kube_manifests_gs_url/gs:\/\//https://storage.googleapis.com/}"
+    server_binary_tar_urls+=("${server_binary_gs_url/gs:\/\//https://storage.googleapis.com/}")
+    salt_tar_urls+=("${salt_gs_url/gs:\/\//https://storage.googleapis.com/}")
+
+    if [[ "${OS_DISTRIBUTION}" == "trusty" || "${OS_DISTRIBUTION}" == "coreos" ]]; then
+      local kube_manifests_gs_url="${staging_path}/${KUBE_MANIFESTS_TAR##*/}"
+      copy-to-staging "${staging_path}" "${kube_manifests_gs_url}" "${KUBE_MANIFESTS_TAR}" "${KUBE_MANIFESTS_TAR_HASH}"
+      # Convert from gs:// URL to an https:// URL
+      kube_manifests_tar_urls+=("${kube_manifests_gs_url/gs:\/\//https://storage.googleapis.com/}")
+    fi
+  done
+
+  if [[ "${OS_DISTRIBUTION}" == "trusty" || "${OS_DISTRIBUTION}" == "coreos" ]]; then
+    # TODO: Support fallback .tar.gz settings on CoreOS/Trusty
+    SERVER_BINARY_TAR_URL="${server_binary_tar_urls[0]}"
+    SALT_TAR_URL="${salt_tar_urls[0]}"
+    KUBE_MANIFESTS_TAR_URL="${kube_manifests_tar_urls[0]}"
+  else
+    SERVER_BINARY_TAR_URL=$(join_csv "${server_binary_tar_urls[@]}")
+    SALT_TAR_URL=$(join_csv "${salt_tar_urls[@]}")
   fi
 }