Skip to content

K
k3s
Commit 5a031bf5 authored by Zach Loafman's avatar Zach Loafman
Browse files
Options

GCE startup: Regional release .tar.gz support

* In kube-up.sh, create a staging bucket with a location nearest the zone being created. If new variable RELEASE_REGION_FALLBACK is set (default false), create multiple buckets and stage to fallback URLs. (In open source, this path is primarily for testing.) * In configure-vm.sh, split the URL env variables by comma (if any extra are present) and retry on the fallback URLs. Also factor the hash checking into this path rather than outside, since a corrupt release in a particular geo can be retried in a different geo. * Remove the local already-staged .tar.gz checks. They've caused several issues along the way, and with this code path become virtually unmaintainable. (I could add a sentinel for each bucket it's possibly staged to, but ew.)
parent cbf2b269
Hide whitespace changes
Inline Side-by-side
Showing with 127 additions and 69 deletions
cluster/gce/config-default.sh
View file @ 5a031bf5
...@@ -18,6 +18,7 @@ ...@@ -18,6 +18,7 @@
# gcloud multiplexing for shared GCE/GKE tests. # gcloud multiplexing for shared GCE/GKE tests.
GCLOUD=gcloud GCLOUD=gcloud
ZONE=${KUBE_GCE_ZONE:-us-central1-b} ZONE=${KUBE_GCE_ZONE:-us-central1-b}
RELEASE_REGION_FALLBACK=${RELEASE_REGION_FALLBACK:-false}
MASTER_SIZE=${MASTER_SIZE:-n1-standard-2} MASTER_SIZE=${MASTER_SIZE:-n1-standard-2}
NODE_SIZE=${NODE_SIZE:-n1-standard-2} NODE_SIZE=${NODE_SIZE:-n1-standard-2}
NUM_NODES=${NUM_NODES:-3} NUM_NODES=${NUM_NODES:-3}
......
cluster/gce/config-test.sh
View file @ 5a031bf5
...@@ -18,6 +18,7 @@ ...@@ -18,6 +18,7 @@
# gcloud multiplexing for shared GCE/GKE tests. # gcloud multiplexing for shared GCE/GKE tests.
GCLOUD=gcloud GCLOUD=gcloud
ZONE=${KUBE_GCE_ZONE:-us-central1-b} ZONE=${KUBE_GCE_ZONE:-us-central1-b}
RELEASE_REGION_FALLBACK=${RELEASE_REGION_FALLBACK:-false}
MASTER_SIZE=${MASTER_SIZE:-n1-standard-2} MASTER_SIZE=${MASTER_SIZE:-n1-standard-2}
NODE_SIZE=${NODE_SIZE:-n1-standard-2} NODE_SIZE=${NODE_SIZE:-n1-standard-2}
NUM_NODES=${NUM_NODES:-3} NUM_NODES=${NUM_NODES:-3}
......
cluster/gce/configure-vm.sh
View file @ 5a031bf5
...@@ -140,15 +140,32 @@ function remove-docker-artifacts() { ...@@ -140,15 +140,32 @@ function remove-docker-artifacts() {
echo "== Finished deleting docker0 ==" echo "== Finished deleting docker0 =="
} }
# Retry a download until we get it. # Retry a download until we get it. Takes a hash and a set of URLs.
# #
# $1 is the URL to download # $1 is the sha1 of the URL. Can be "" if the sha1 is unknown.
# $2+ are the URLs to download.
download-or-bust() { download-or-bust() {
local -r url="$1" local -r hash="$1"
local -r file="${url##*/}" shift 1
rm -f "$file"
until curl --ipv4 -Lo "$file" --connect-timeout 20 --retry 6 --retry-delay 10 "${url}"; do urls=( $* )
echo "Failed to download file (${url}). Retrying." while true; do
for url in "${urls[@]}"; do
local file="${url##*/}"
rm -f "${file}"
if ! curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10 "${url}"; then
echo "== Failed to download ${url}. Retrying. =="
elif [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then
echo "== Hash validation of ${url} failed. Retrying. =="
else
if [[ -n "${hash}" ]]; then
echo "== Downloaded ${url} (SHA1 = ${hash}) =="
else
echo "== Downloaded ${url} =="
fi
return
fi
done
done done
} }
...@@ -603,39 +620,43 @@ EOF ...@@ -603,39 +620,43 @@ EOF
fi fi
} }
function split-commas() {
echo $1 | tr "," "\n"
}
function try-download-release() { function try-download-release() {
# TODO(zmerlynn): Now we REALLy have no excuse not to do the reboot # TODO(zmerlynn): Now we REALLy have no excuse not to do the reboot
# optimization. # optimization.
# TODO(zmerlynn): This may not be set yet by everyone (GKE). local -r server_binary_tar_urls=( $(split-commas "${SERVER_BINARY_TAR_URL}") )
if [[ -z "${SERVER_BINARY_TAR_HASH:-}" ]]; then local -r server_binary_tar="${server_binary_tar_urls[0]##*/}"
if [[ -n "${SERVER_BINARY_TAR_HASH:-}" ]]; then
local -r server_binary_tar_hash="${SERVER_BINARY_TAR_HASH}"
else
echo "Downloading binary release sha1 (not found in env)" echo "Downloading binary release sha1 (not found in env)"
download-or-bust "${SERVER_BINARY_TAR_URL}.sha1" download-or-bust "" "${server_binary_tar_urls[@]/.tar.gz/.tar.gz.sha1}"
SERVER_BINARY_TAR_HASH=$(cat "${SERVER_BINARY_TAR_URL##*/}.sha1") local -r server_binary_tar_hash=$(cat "${server_binary_tar}.sha1")
fi fi
echo "Downloading binary release tar (${SERVER_BINARY_TAR_URL})" echo "Downloading binary release tar (${server_binary_tar_urls[@]})"
download-or-bust "${SERVER_BINARY_TAR_URL}" download-or-bust "${server_binary_tar_hash}" "${server_binary_tar_urls[@]}"
validate-hash "${SERVER_BINARY_TAR_URL##*/}" "${SERVER_BINARY_TAR_HASH}" local -r salt_tar_urls=( $(split-commas "${SALT_TAR_URL}") )
echo "Validated ${SERVER_BINARY_TAR_URL} SHA1 = ${SERVER_BINARY_TAR_HASH}" local -r salt_tar="${salt_tar_urls[0]##*/}"
if [[ -n "${SALT_TAR_HASH:-}" ]]; then
# TODO(zmerlynn): This may not be set yet by everyone (GKE). local -r salt_tar_hash="${SALT_TAR_HASH}"
if [[ -z "${SALT_TAR_HASH:-}" ]]; then else
echo "Downloading Salt tar sha1 (not found in env)" echo "Downloading Salt tar sha1 (not found in env)"
download-or-bust "${SALT_TAR_URL}.sha1" download-or-bust "" "${salt_tar_urls[@]/.tar.gz/.tar.gz.sha1}"
SALT_TAR_HASH=$(cat "${SALT_TAR_URL##*/}.sha1") local -r salt_tar_hash=$(cat "${salt_tar}.sha1")
fi fi
echo "Downloading Salt tar ($SALT_TAR_URL)" echo "Downloading Salt tar (${salt_tar_urls[@]})"
download-or-bust "$SALT_TAR_URL" download-or-bust "${salt_tar_hash}" "${salt_tar_urls[@]}"
validate-hash "${SALT_TAR_URL##*/}" "${SALT_TAR_HASH}"
echo "Validated ${SALT_TAR_URL} SHA1 = ${SALT_TAR_HASH}"
echo "Unpacking Salt tree and checking integrity of binary release tar" echo "Unpacking Salt tree and checking integrity of binary release tar"
rm -rf kubernetes rm -rf kubernetes
tar xzf "${SALT_TAR_URL##*/}" && tar tzf "${SERVER_BINARY_TAR_URL##*/}" > /dev/null tar xzf "${salt_tar}" && tar tzf "${server_binary_tar}" > /dev/null
} }
function download-release() { function download-release() {
......
cluster/gce/util.sh
View file @ 5a031bf5
...@@ -115,33 +115,43 @@ function detect-project () { ...@@ -115,33 +115,43 @@ function detect-project () {
fi fi
} }
function already-staged() { # Copy a release tar and its accompanying hash.
local -r file=$1 function copy-to-staging() {
local -r newsum=$2
[[ -e "${file}.uploaded.sha1" ]] || return 1
local oldsum
oldsum=$(cat "${file}.uploaded.sha1")
[[ "${oldsum}" == "${newsum}" ]]
}
# Copy a release tar, if we don't already think it's staged in GCS
function copy-if-not-staged() {
local -r staging_path=$1 local -r staging_path=$1
local -r gs_url=$2 local -r gs_url=$2
local -r tar=$3 local -r tar=$3
local -r hash=$4 local -r hash=$4
if already-staged "${tar}" "${hash}"; then echo "${hash}" > "${tar}.sha1"
echo "+++ $(basename ${tar}) already staged ('rm ${tar}.uploaded.sha1' to force)" gsutil -m -q -h "Cache-Control:private, max-age=0" cp "${tar}" "${tar}.sha1" "${staging_path}"
else gsutil -m acl ch -g all:R "${gs_url}" "${gs_url}.sha1" >/dev/null 2>&1
echo "${hash}" > "${tar}.sha1" echo "+++ $(basename ${tar}) uploaded (sha1 = ${hash})"
gsutil -m -q -h "Cache-Control:private, max-age=0" cp "${tar}" "${tar}.sha1" "${staging_path}" }
gsutil -m acl ch -g all:R "${gs_url}" "${gs_url}.sha1" >/dev/null 2>&1
echo "${hash}" > "${tar}.uploaded.sha1" # Given the cluster zone, return the list of regional GCS release
echo "+++ $(basename ${tar}) uploaded (sha1 = ${hash})" # bucket suffixes for the release in preference order. GCS doesn't
# give us an API for this, so we hardcode it.
#
# Assumed vars:
# REGIONAL_RELEASE
# ZONE
# Vars set:
# PREFERRED_REGION
function set-preferred-region() {
case ${ZONE} in
asia-*)
PREFERRED_REGION=("asia" "us" "eu")
;;
europe-*)
PREFERRED_REGION=("eu" "us" "asia")
;;
*)
PREFERRED_REGION=("us" "eu" "asia")
;;
esac
if [[ "${RELEASE_REGION_FALLBACK}" != "true" ]]; then
PREFERRED_REGION=( "${PREFERRED_REGION[0]}" )
fi fi
} }
...@@ -155,6 +165,7 @@ function copy-if-not-staged() { ...@@ -155,6 +165,7 @@ function copy-if-not-staged() {
# SERVER_BINARY_TAR # SERVER_BINARY_TAR
# SALT_TAR # SALT_TAR
# KUBE_MANIFESTS_TAR # KUBE_MANIFESTS_TAR
# ZONE
# Vars set: # Vars set:
# SERVER_BINARY_TAR_URL # SERVER_BINARY_TAR_URL
# SERVER_BINARY_TAR_HASH # SERVER_BINARY_TAR_HASH
...@@ -181,35 +192,59 @@ function upload-server-tars() { ...@@ -181,35 +192,59 @@ function upload-server-tars() {
# that's probably good enough for now :P # that's probably good enough for now :P
project_hash=${project_hash:0:10} project_hash=${project_hash:0:10}
local -r staging_bucket="gs://kubernetes-staging-${project_hash}" set-preferred-region
# Ensure the bucket is created SERVER_BINARY_TAR_HASH=$(sha1sum-file "${SERVER_BINARY_TAR}")
if ! gsutil ls "$staging_bucket" > /dev/null 2>&1 ; then SALT_TAR_HASH=$(sha1sum-file "${SALT_TAR}")
echo "Creating $staging_bucket" if [[ "${OS_DISTRIBUTION}" == "trusty" || "${OS_DISTRIBUTION}" == "coreos" ]]; then
gsutil mb "${staging_bucket}" KUBE_MANIFESTS_TAR_HASH=$(sha1sum-file "${KUBE_MANIFESTS_TAR}")
fi fi
local -r staging_path="${staging_bucket}/${INSTANCE_PREFIX}-devel" local server_binary_tar_urls=()
local salt_tar_urls=()
local kube_manifest_tar_urls=()
SERVER_BINARY_TAR_HASH=$(sha1sum-file "${SERVER_BINARY_TAR}") for region in "${PREFERRED_REGION[@]}"; do
SALT_TAR_HASH=$(sha1sum-file "${SALT_TAR}") suffix="-${region}"
if [[ "${suffix}" == "-us" ]]; then
suffix=""
fi
local staging_bucket="gs://kubernetes-staging-${project_hash}${suffix}"
echo "+++ Staging server tars to Google Storage: ${staging_path}" # Ensure the buckets are created
local server_binary_gs_url="${staging_path}/${SERVER_BINARY_TAR##*/}" if ! gsutil ls "${staging_bucket}" > /dev/null 2>&1 ; then
local salt_gs_url="${staging_path}/${SALT_TAR##*/}" echo "Creating ${staging_bucket}"
copy-if-not-staged "${staging_path}" "${server_binary_gs_url}" "${SERVER_BINARY_TAR}" "${SERVER_BINARY_TAR_HASH}" gsutil mb -l "${region}" "${staging_bucket}"
copy-if-not-staged "${staging_path}" "${salt_gs_url}" "${SALT_TAR}" "${SALT_TAR_HASH}" fi
# Convert from gs:// URL to an https:// URL local staging_path="${staging_bucket}/${INSTANCE_PREFIX}-devel"
SERVER_BINARY_TAR_URL="${server_binary_gs_url/gs:\/\//https://storage.googleapis.com/}"
SALT_TAR_URL="${salt_gs_url/gs:\/\//https://storage.googleapis.com/}" echo "+++ Staging server tars to Google Storage: ${staging_path}"
local server_binary_gs_url="${staging_path}/${SERVER_BINARY_TAR##*/}"
local salt_gs_url="${staging_path}/${SALT_TAR##*/}"
copy-to-staging "${staging_path}" "${server_binary_gs_url}" "${SERVER_BINARY_TAR}" "${SERVER_BINARY_TAR_HASH}"
copy-to-staging "${staging_path}" "${salt_gs_url}" "${SALT_TAR}" "${SALT_TAR_HASH}"
if [[ "${OS_DISTRIBUTION}" == "trusty" || "${OS_DISTRIBUTION}" == "coreos" ]]; then
local kube_manifests_gs_url="${staging_path}/${KUBE_MANIFESTS_TAR##*/}"
KUBE_MANIFESTS_TAR_HASH=$(sha1sum-file "${KUBE_MANIFESTS_TAR}")
copy-if-not-staged "${staging_path}" "${kube_manifests_gs_url}" "${KUBE_MANIFESTS_TAR}" "${KUBE_MANIFESTS_TAR_HASH}"
# Convert from gs:// URL to an https:// URL # Convert from gs:// URL to an https:// URL
KUBE_MANIFESTS_TAR_URL="${kube_manifests_gs_url/gs:\/\//https://storage.googleapis.com/}" server_binary_tar_urls+=("${server_binary_gs_url/gs:\/\//https://storage.googleapis.com/}")
salt_tar_urls+=("${salt_gs_url/gs:\/\//https://storage.googleapis.com/}")
if [[ "${OS_DISTRIBUTION}" == "trusty" || "${OS_DISTRIBUTION}" == "coreos" ]]; then
local kube_manifests_gs_url="${staging_path}/${KUBE_MANIFESTS_TAR##*/}"
copy-to-staging "${staging_path}" "${kube_manifests_gs_url}" "${KUBE_MANIFESTS_TAR}" "${KUBE_MANIFESTS_TAR_HASH}"
# Convert from gs:// URL to an https:// URL
kube_manifests_tar_urls+=("${kube_manifests_gs_url/gs:\/\//https://storage.googleapis.com/}")
fi
done
if [[ "${OS_DISTRIBUTION}" == "trusty" || "${OS_DISTRIBUTION}" == "coreos" ]]; then
# TODO: Support fallback .tar.gz settings on CoreOS/Trusty
SERVER_BINARY_TAR_URL="${server_binary_tar_urls[0]}"
SALT_TAR_URL="${salt_tar_urls[0]}"
KUBE_MANIFESTS_TAR_URL="${kube_manifests_tar_urls[0]}"
else
SERVER_BINARY_TAR_URL=$(join_csv "${server_binary_tar_urls[@]}")
SALT_TAR_URL=$(join_csv "${salt_tar_urls[@]}")
fi fi
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment