Skip to content

K
k3s
Commit 816b2953 authored by Andy Zheng's avatar Andy Zheng
Browse files
Options

Prepare kube-system pods manifest for trusty nodes.

This change refactors the code of preparing kube-system manifests for trusty based cluster. The manifests used by nodes do not contain salt configuration, so we can simply copy them from the directory cluster/saltbase/salt, make a tarball, and upload to Google Storage.
parent 224aebd2
Hide whitespace changes
Inline Side-by-side
Showing with 80 additions and 26 deletions
cluster/gce/trusty/configure.sh
View file @ 816b2953
...@@ -144,8 +144,19 @@ install_additional_packages() { ...@@ -144,8 +144,19 @@ install_additional_packages() {
fi fi
} }
# Downloads kubernetes binaries and salt tarball, unpacks them, and places them # Retry a download until we get it.
# to suitable directories. #
# $1 is the file to create
# $2 is the URL to download
download_or_bust() {
rm -f $1 > /dev/null
until curl --ipv4 -Lo "$1" --connect-timeout 20 --retry 6 --retry-delay 10 "$2"; do
echo "Failed to download file ($2). Retrying."
done
}
# Downloads kubernetes binaries and kube-system manifest tarball, unpacks them,
# and places them into suitable directories.
install_kube_binary_config() { install_kube_binary_config() {
. /etc/kube-env . /etc/kube-env
# For a testing cluster, we pull kubelet, kube-proxy, and kubectl binaries, # For a testing cluster, we pull kubelet, kube-proxy, and kubectl binaries,
...@@ -160,10 +171,10 @@ install_kube_binary_config() { ...@@ -160,10 +171,10 @@ install_kube_binary_config() {
cd /tmp cd /tmp
k8s_sha1="${SERVER_BINARY_TAR_URL##*/}.sha1" k8s_sha1="${SERVER_BINARY_TAR_URL##*/}.sha1"
echo "Downloading k8s tar sha1 file ${k8s_sha1}" echo "Downloading k8s tar sha1 file ${k8s_sha1}"
curl -Lo "${k8s_sha1}" --connect-timeout 20 --retry 6 --retry-delay 2 "${SERVER_BINARY_TAR_URL}.sha1" download_or_bust "${k8s_sha1}" "${SERVER_BINARY_TAR_URL}.sha1"
k8s_tar="${SERVER_BINARY_TAR_URL##*/}" k8s_tar="${SERVER_BINARY_TAR_URL##*/}"
echo "Downloading k8s tar file ${k8s_tar}" echo "Downloading k8s tar file ${k8s_tar}"
curl -Lo "${k8s_tar}" --connect-timeout 20 --retry 6 --retry-delay 2 "${SERVER_BINARY_TAR_URL}" download_or_bust "${k8s_tar}" "${SERVER_BINARY_TAR_URL}"
# Validate hash. # Validate hash.
actual=$(sha1sum ${k8s_tar} | awk '{ print $1 }') || true actual=$(sha1sum ${k8s_tar} | awk '{ print $1 }') || true
if [ "${actual}" != "${SERVER_BINARY_TAR_HASH}" ]; then if [ "${actual}" != "${SERVER_BINARY_TAR_HASH}" ]; then
...@@ -178,27 +189,26 @@ install_kube_binary_config() { ...@@ -178,27 +189,26 @@ install_kube_binary_config() {
rm -rf "/tmp/kubernetes" rm -rf "/tmp/kubernetes"
rm "/tmp/${k8s_tar}" rm "/tmp/${k8s_tar}"
rm "/tmp/${k8s_sha1}" rm "/tmp/${k8s_sha1}"
fi fi
# Put saltbase configuration files in /etc/saltbase. We will use the add-on yaml files. # Put kube-system pods manifests in /etc/kube-manifests/.
mkdir -p /etc/saltbase cd /etc
cd /etc/saltbase manifests_sha1="${KUBE_MANIFESTS_TAR_URL##*/}.sha1"
salt_sha1="${SALT_TAR_URL##*/}.sha1" echo "Downloading kube-manifests tar sha1 file ${manifests_sha1}"
echo "Downloading Salt tar sha1 file ${salt_sha1}" download_or_bust "${manifests_sha1}" "${KUBE_MANIFESTS_TAR_URL}.sha1"
curl -Lo "${salt_sha1}" --connect-timeout 20 --retry 6 --retry-delay 2 "${SALT_TAR_URL}.sha1" manifests_tar="${KUBE_MANIFESTS_TAR_URL##*/}"
salt_tar="${SALT_TAR_URL##*/}" echo "Downloading kube-manifest tar file ${manifests_tar}"
echo "Downloading Salt tar file ${salt_tar}" download_or_bust "${manifests_tar}" "${KUBE_MANIFESTS_TAR_URL}"
curl -Lo "${salt_tar}" --connect-timeout 20 --retry 6 --retry-delay 2 "${SALT_TAR_URL}"
# Validate hash. # Validate hash.
actual=$(sha1sum ${salt_tar} | awk '{ print $1 }') || true actual=$(sha1sum ${manifests_tar} | awk '{ print $1 }') || true
if [ "${actual}" != "${SALT_TAR_HASH}" ]; then if [ "${actual}" != "${KUBE_MANIFESTS_TAR_HASH}" ]; then
echo "== ${salt_tar} corrupted, sha1 ${actual} doesn't match expected ${SALT_TAR_HASH} ==" echo "== ${manifests_tar} corrupted, sha1 ${actual} doesn't match expected ${KUBE_MANIFESTS_TAR_HASH} =="
else else
echo "Validated ${SALT_TAR_URL} SHA1 = ${SALT_TAR_HASH}" echo "Validated ${KUBE_MANIFESTS_TAR_URL} SHA1 = ${KUBE_MANIFESTS_TAR_HASH}"
fi fi
tar xzf "/etc/saltbase/${salt_tar}" -C /etc/saltbase/ --overwrite tar xzf "/etc/${manifests_tar}" -C /etc/ --overwrite
rm "/etc/saltbase/${salt_sha1}" rm "/etc/${manifests_sha1}"
rm "/etc/saltbase/${salt_tar}" rm "/etc/${manifests_tar}"
} }
restart_docker_daemon() { restart_docker_daemon() {
...@@ -208,7 +218,7 @@ restart_docker_daemon() { ...@@ -208,7 +218,7 @@ restart_docker_daemon() {
if [ "${TEST_CLUSTER:-}" = "true" ]; then if [ "${TEST_CLUSTER:-}" = "true" ]; then
DOCKER_OPTS="${DOCKER_OPTS} --log-level=debug" DOCKER_OPTS="${DOCKER_OPTS} --log-level=debug"
fi fi
echo "DOCKER_OPTS=\"${DOCKER_OPTS} ${EXTRA_DOCKER_OPTS}\"" > /etc/default/docker echo "DOCKER_OPTS=\"${DOCKER_OPTS} ${EXTRA_DOCKER_OPTS:-}\"" > /etc/default/docker
# Make sure the network interface cbr0 is created before restarting docker daemon # Make sure the network interface cbr0 is created before restarting docker daemon
while ! [ -L /sys/class/net/cbr0 ]; do while ! [ -L /sys/class/net/cbr0 ]; do
echo "Sleep 1 second to wait for cbr0" echo "Sleep 1 second to wait for cbr0"
......
cluster/gce/trusty/node.yaml
View file @ 816b2953
...@@ -218,19 +218,19 @@ script ...@@ -218,19 +218,19 @@ script
set -o errexit set -o errexit
set -o nounset set -o nounset
# Configuration files are located at /etc/saltbase. # Kube-system pod manifest files are located at /etc/kube-manifests.
. /etc/kube-env . /etc/kube-env
# Fluentd # Fluentd
if [ "${ENABLE_NODE_LOGGING:-}" = "true" ]; then if [ "${ENABLE_NODE_LOGGING:-}" = "true" ]; then
if [ "${LOGGING_DESTINATION:-}" = "gcp" ]; then if [ "${LOGGING_DESTINATION:-}" = "gcp" ]; then
cp /etc/saltbase/kubernetes/saltbase/salt/fluentd-gcp/fluentd-gcp.yaml /etc/kubernetes/manifests/ cp /etc/kube-manifests/fluentd-gcp.yaml /etc/kubernetes/manifests/
elif [ "${LOGGING_DESTINATION:-}" = "elasticsearch" ]; then elif [ "${LOGGING_DESTINATION:-}" = "elasticsearch" ]; then
cp /etc/saltbase/kubernetes/saltbase/salt/fluentd-es/fluentd-es.yaml /etc/kubernetes/manifests/ cp /etc/kube-manifests/fluentd-es.yaml /etc/kubernetes/manifests/
fi fi
fi fi
# Kube-registry-proxy # Kube-registry-proxy
if [ "${ENABLE_CLUSTER_REGISTRY:-}" = "true" ]; then if [ "${ENABLE_CLUSTER_REGISTRY:-}" = "true" ]; then
cp /etc/saltbase/kubernetes/saltbase/salt/kube-registry-proxy/kube-registry-proxy.yaml /etc/kubernetes/manifests/ cp /etc/kube-manifests/kube-registry-proxy.yaml /etc/kubernetes/manifests/
fi fi
end script end script
......
cluster/gce/util.sh
View file @ 816b2953
...@@ -156,8 +156,40 @@ function copy-if-not-staged() { ...@@ -156,8 +156,40 @@ function copy-if-not-staged() {
fi fi
} }
# Prepare a tarball of kube-system manifests for trusty based cluster.
#
# Vars set:
# KUBE_MANIFESTS_TAR_URL
# KUBE_MANIFESTS_TAR_HASH
function prepare-manifests-tar() {
KUBE_MANIFESTS_TAR_URL=
KUBE_MANIFESTS_TAR_HASH=
if [[ "${OS_DISTRIBUTION}" != "trusty" ]]; then
return
fi
local tmp_dir="${KUBE_TEMP}/kube-manifests"
mkdir -p ${tmp_dir}
# The manifests used by nodes can be directly used on non-salt system.
# We simply copy them from cluster/saltbase/salt.
local salt_dir="${KUBE_ROOT}/cluster/saltbase/salt"
cp -f "${salt_dir}/fluentd-es/fluentd-es.yaml" "${tmp_dir}"
cp -f "${salt_dir}/fluentd-gcp/fluentd-gcp.yaml" "${tmp_dir}"
cp -f "${salt_dir}/kube-registry-proxy/kube-registry-proxy.yaml" "${tmp_dir}"
local kube_manifests_tar="${KUBE_TEMP}/kube-manifests.tar.gz"
tar czf "${kube_manifests_tar}" -C "${KUBE_TEMP}" kube-manifests
KUBE_MANIFESTS_TAR_HASH=$(sha1sum-file "${kube_manifests_tar}")
local kube_manifests_gs_url="${staging_path}/${kube_manifests_tar##*/}"
copy-if-not-staged "${staging_path}" "${kube_manifests_gs_url}" "${kube_manifests_tar}" "${KUBE_MANIFESTS_TAR_HASH}"
# Convert from gs:// URL to an https:// URL
KUBE_MANIFESTS_TAR_URL="${kube_manifests_gs_url/gs:\/\//https://storage.googleapis.com/}"
}
# Take the local tar files and upload them to Google Storage. They will then be # Take the local tar files and upload them to Google Storage. They will then be
# downloaded by the master as part of the start up script for the master. # downloaded by the master as part of the start up script for the master.
# If running on Ubuntu trusty, we also pack the dir cluster/gce/trusty/kube-manifest
# and upload it to Google Storage.
# #
# Assumed vars: # Assumed vars:
# PROJECT # PROJECT
...@@ -207,6 +239,12 @@ function upload-server-tars() { ...@@ -207,6 +239,12 @@ function upload-server-tars() {
# Convert from gs:// URL to an https:// URL # Convert from gs:// URL to an https:// URL
SERVER_BINARY_TAR_URL="${server_binary_gs_url/gs:\/\//https://storage.googleapis.com/}" SERVER_BINARY_TAR_URL="${server_binary_gs_url/gs:\/\//https://storage.googleapis.com/}"
SALT_TAR_URL="${salt_gs_url/gs:\/\//https://storage.googleapis.com/}" SALT_TAR_URL="${salt_gs_url/gs:\/\//https://storage.googleapis.com/}"
# Create a tar for kube-system manifests files and stage it.
# TODO(andyzheng0831): After finishing k8s master on trusty (issue #16702),
# we will not need to stage the salt tar for trusty anymore.
# TODO(andyzheng0831): Add release support for this tar, in case GKE will it.
prepare-manifests-tar
} }
# Detect minions created in the minion group # Detect minions created in the minion group
...@@ -1313,6 +1351,12 @@ EOF ...@@ -1313,6 +1351,12 @@ EOF
TERMINATED_POD_GC_THRESHOLD: $(yaml-quote ${TERMINATED_POD_GC_THRESHOLD}) TERMINATED_POD_GC_THRESHOLD: $(yaml-quote ${TERMINATED_POD_GC_THRESHOLD})
EOF EOF
fi fi
if [[ "${OS_DISTRIBUTION}" == "trusty" ]]; then
cat >>$file <<EOF
KUBE_MANIFESTS_TAR_URL: $(yaml-quote ${KUBE_MANIFESTS_TAR_URL})
KUBE_MANIFESTS_TAR_HASH: $(yaml-quote ${KUBE_MANIFESTS_TAR_HASH})
EOF
fi
if [ -n "${TEST_CLUSTER:-}" ]; then if [ -n "${TEST_CLUSTER:-}" ]; then
cat >>$file <<EOF cat >>$file <<EOF
TEST_CLUSTER: $(yaml-quote ${TEST_CLUSTER}) TEST_CLUSTER: $(yaml-quote ${TEST_CLUSTER})
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment