Merge pull request #10050 from zmerlynn/checkbinaries

cluster/common.sh

@@ -199,7 +199,9 @@ function set_binary_version() {
 #   PROJECT
 # Vars set:
 #   SERVER_BINARY_TAR_URL
+#   SERVER_BINARY_TAR_HASH
 #   SALT_TAR_URL
+#   SALT_TAR_HASH
 function tars_from_version() {
   if [[ -z "${KUBE_VERSION-}" ]]; then
     find-release-tars
@@ -214,9 +216,19 @@ function tars_from_version() {
     echo "Version doesn't match regexp" >&2
     exit 1
   fi
-
-  if ! curl -Ss --range 0-1 ${SERVER_BINARY_TAR_URL} >&/dev/null; then
+  until SERVER_BINARY_TAR_HASH=$(curl --fail --silent "${SERVER_BINARY_TAR_URL}.sha1"); do
+    echo "Failure trying to curl release .sha1"
+  done
+  until SALT_TAR_HASH=$(curl --fail --silent "${SALT_TAR_URL}.sha1"); do
+    echo "Failure trying to curl Salt tar .sha1"
+  done
+
+  if ! curl -Ss --range 0-1 "${SERVER_BINARY_TAR_URL}" >&/dev/null; then
     echo "Can't find release at ${SERVER_BINARY_TAR_URL}" >&2
     exit 1
   fi
+  if ! curl -Ss --range 0-1 "${SALT_TAR_URL}" >&/dev/null; then
+    echo "Can't find Salt tar at ${SALT_TAR_URL}" >&2
+    exit 1
+  fi
 }

cluster/gce/configure-vm.sh

@@ -107,11 +107,23 @@ download-or-bust() {
   local -r url="$1"
   local -r file="${url##*/}"
   rm -f "$file"
-  until curl --ipv4 -Lo "$file" --connect-timeout 20 --retry 6 --retry-delay 10 "$1"; do
-    echo "Failed to download file ($1). Retrying."
+  until curl --ipv4 -Lo "$file" --connect-timeout 20 --retry 6 --retry-delay 10 "${url}"; do
+    echo "Failed to download file (${url}). Retrying."
   done
 }
 
+validate-hash() {
+  local -r file="$1"
+  local -r expected="$2"
+  local actual
+
+  actual=$(sha1sum ${file} | awk '{ print $1 }') || true
+  if [[ "${actual}" != "${expected}" ]]; then
+    echo "== ${file} corrupted, sha1 ${actual} doesn't match expected ${expected} =="
+    return 1
+  fi
+}
+
 # Install salt from GCS.  See README.md for instructions on how to update these
 # debs.
 install-salt() {
@@ -430,31 +442,46 @@ EOF
   fi
 }
 
+function try-download-release() {
+  # TODO(zmerlynn): Now we REALLy have no excuse not to do the reboot
+  # optimization.
+
+  # TODO(zmerlynn): This may not be set yet by everyone (GKE).
+  if [[ -z "${SERVER_BINARY_TAR_HASH:-}" ]]; then
+    echo "Downloading binary release sha1 (not found in env)"
+    download-or-bust "${SERVER_BINARY_TAR_URL}.sha1"
+    SERVER_BINARY_TAR_HASH=$(cat "${SERVER_BINARY_TAR_URL##*/}.sha1")
+  fi
+
+  echo "Downloading binary release tar (${SERVER_BINARY_TAR_URL})"
+  download-or-bust "${SERVER_BINARY_TAR_URL}"
+
+  validate-hash "${SERVER_BINARY_TAR_URL##*/}" "${SERVER_BINARY_TAR_HASH}"
+  echo "Validated ${SERVER_BINARY_TAR_URL} SHA1 = ${SERVER_BINARY_TAR_HASH}"
+
+  # TODO(zmerlynn): This may not be set yet by everyone (GKE).
+  if [[ -z "${SALT_TAR_HASH:-}" ]]; then
+    echo "Downloading Salt tar sha1 (not found in env)"
+    download-or-bust "${SALT_TAR_URL}.sha1"
+    SALT_TAR_HASH=$(cat "${SALT_TAR_URL##*/}.sha1")
+  fi
+
+  echo "Downloading Salt tar ($SALT_TAR_URL)"
+  download-or-bust "$SALT_TAR_URL"
+
+  validate-hash "${SALT_TAR_URL##*/}" "${SALT_TAR_HASH}"
+  echo "Validated ${SALT_TAR_URL} SHA1 = ${SALT_TAR_HASH}"
+
+  echo "Unpacking Salt tree and checking integrity of binary release tar"
+  rm -rf kubernetes
+  tar xzf "${SALT_TAR_URL##*/}" && tar tzf "${SERVER_BINARY_TAR_URL##*/}" > /dev/null
+}
+
 function download-release() {
-  # TODO(zmerlynn): We should optimize for the reboot case here, but
-  # unlike the .debs, we don't have version information in the
-  # filenames here, nor do the URLs even provide useful information in
-  # the dev environment case (because they're just a project
-  # bucket). We should probably push a hash into the kube-env, and
-  # store it when we download, and then when it's different infer that
-  # a push occurred (otherwise it's a simple reboot).
-
-  # In case of failure of unpacking Salt tree or checking integrity of
-  # binary release tar (the last command in the "until" block) retry
-  # downloading both release and Salt tars.
-  until
-    echo "Downloading binary release tar ($SERVER_BINARY_TAR_URL)"
-    download-or-bust "$SERVER_BINARY_TAR_URL"
-
-    echo "Downloading Salt tar ($SALT_TAR_URL)"
-    download-or-bust "$SALT_TAR_URL"
-
-    echo "Unpacking Salt tree and checking integrity of binary release tar"
-    rm -rf kubernetes
-    tar xzf "${SALT_TAR_URL##*/}" && tar tzf "${SERVER_BINARY_TAR_URL##*/}" > /dev/null
-  do
+  # In case of failure checking integrity of release, retry.
+  until try-download-release; do
     sleep 15
-    echo "Couldn't unpack Salt tree. Retrying..."
+    echo "Couldn't download release. Retrying..."
   done
 
   echo "Running release install script"

cluster/gce/coreos/helper.sh

@@ -31,7 +31,9 @@ INSTANCE_PREFIX: $(yaml-quote ${INSTANCE_PREFIX})
 NODE_INSTANCE_PREFIX: $(yaml-quote ${NODE_INSTANCE_PREFIX})
 CLUSTER_IP_RANGE: $(yaml-quote ${CLUSTER_IP_RANGE:-10.244.0.0/16})
 SERVER_BINARY_TAR_URL: $(yaml-quote ${SERVER_BINARY_TAR_URL})
+SERVER_BINARY_TAR_HASH: $(yaml-quote ${SERVER_BINARY_TAR_HASH})
 SALT_TAR_URL: $(yaml-quote ${SALT_TAR_URL})
+SALT_TAR_HASH: $(yaml-quote ${SALT_TAR_HASH})
 SERVICE_CLUSTER_IP_RANGE: $(yaml-quote ${SERVICE_CLUSTER_IP_RANGE})
 ALLOCATE_NODE_CIDRS: $(yaml-quote ${ALLOCATE_NODE_CIDRS:-false})
 ENABLE_CLUSTER_MONITORING: $(yaml-quote ${ENABLE_CLUSTER_MONITORING:-none})

cluster/gce/debian/helper.sh

@@ -28,7 +28,9 @@ INSTANCE_PREFIX: $(yaml-quote ${INSTANCE_PREFIX})
 NODE_INSTANCE_PREFIX: $(yaml-quote ${NODE_INSTANCE_PREFIX})
 CLUSTER_IP_RANGE: $(yaml-quote ${CLUSTER_IP_RANGE:-10.244.0.0/16})
 SERVER_BINARY_TAR_URL: $(yaml-quote ${SERVER_BINARY_TAR_URL})
+SERVER_BINARY_TAR_HASH: $(yaml-quote ${SERVER_BINARY_TAR_HASH})
 SALT_TAR_URL: $(yaml-quote ${SALT_TAR_URL})
+SALT_TAR_HASH: $(yaml-quote ${SALT_TAR_HASH})
 SERVICE_CLUSTER_IP_RANGE: $(yaml-quote ${SERVICE_CLUSTER_IP_RANGE})
 KUBERNETES_MASTER_NAME: $(yaml-quote ${MASTER_NAME})
 ALLOCATE_NODE_CIDRS: $(yaml-quote ${ALLOCATE_NODE_CIDRS:-false})

cluster/gce/util.sh

@@ -178,10 +178,11 @@ function copy-if-not-staged() {
   if already-staged "${tar}" "${hash}"; then
     echo "+++ $(basename ${tar}) already staged ('rm ${tar}.sha1' to force)"
   else
-    echo "${server_hash}" > "${tar}.sha1"
+    echo "${hash}" > "${tar}.sha1"
     gsutil -m -q -h "Cache-Control:private, max-age=0" cp "${tar}" "${tar}.sha1" "${staging_path}"
     gsutil -m acl ch -g all:R "${gs_url}" "${gs_url}.sha1" >/dev/null 2>&1
-    echo "${server_hash}" > "${tar}.uploaded.sha1"
+    echo "${hash}" > "${tar}.uploaded.sha1"
+    echo "+++ $(basename ${tar}) uploaded (sha1 = ${hash})"
   fi
 }
 
@@ -194,10 +195,14 @@ function copy-if-not-staged() {
 #   SALT_TAR
 # Vars set:
 #   SERVER_BINARY_TAR_URL
+#   SERVER_BINARY_TAR_HASH
 #   SALT_TAR_URL
+#   SALT_TAR_HASH
 function upload-server-tars() {
   SERVER_BINARY_TAR_URL=
+  SERVER_BINARY_TAR_HASH=
   SALT_TAR_URL=
+  SALT_TAR_HASH=
 
   local project_hash
   if which md5 > /dev/null 2>&1; then
@@ -220,16 +225,14 @@ function upload-server-tars() {
 
   local -r staging_path="${staging_bucket}/devel${KUBE_GCS_STAGING_PATH_SUFFIX}"
 
-  local server_hash
-  local salt_hash
-  server_hash=$(sha1sum-file "${SERVER_BINARY_TAR}")
-  salt_hash=$(sha1sum-file "${SALT_TAR}")
+  SERVER_BINARY_TAR_HASH=$(sha1sum-file "${SERVER_BINARY_TAR}")
+  SALT_TAR_HASH=$(sha1sum-file "${SALT_TAR}")
 
   echo "+++ Staging server tars to Google Storage: ${staging_path}"
   local server_binary_gs_url="${staging_path}/${SERVER_BINARY_TAR##*/}"
   local salt_gs_url="${staging_path}/${SALT_TAR##*/}"
-  copy-if-not-staged "${staging_path}" "${server_binary_gs_url}" "${SERVER_BINARY_TAR}" "${server_hash}"
-  copy-if-not-staged "${staging_path}" "${salt_gs_url}" "${SALT_TAR}" "${salt_hash}"
+  copy-if-not-staged "${staging_path}" "${server_binary_gs_url}" "${SERVER_BINARY_TAR}" "${SERVER_BINARY_TAR_HASH}"
+  copy-if-not-staged "${staging_path}" "${salt_gs_url}" "${SALT_TAR}" "${SALT_TAR_HASH}"
 
   # Convert from gs:// URL to an https:// URL
   SERVER_BINARY_TAR_URL="${server_binary_gs_url/gs:\/\//https://storage.googleapis.com/}"