Run kube-proxy in Trusty in a static pod.

build/common.sh

@@ -864,9 +864,9 @@ function kube::release::package_salt_tarball() {
 # such as Ubuntu Trusty.
 #
 # There are two sources of manifests files: (1) some manifests in the directory
-# cluster/saltbase/salt can be directly used on instances without salt, so we copy
-# them from there; (2) for the ones containing salt config, we cannot directly
-# use them. Therefore, we will maintain separate copies in cluster/gce/kube-manifests.
+# cluster/saltbase/salt can be used directly or after minor revision, so we copy
+# them from there; (2) otherwise, we will maintain separate copies in
+# cluster/gce/kube-manifests.
 function kube::release::package_kube_manifests_tarball() {
   kube::log::status "Building tarball: manifests"
 
@@ -880,9 +880,11 @@ function kube::release::package_kube_manifests_tarball() {
   cp "${salt_dir}/fluentd-es/fluentd-es.yaml" "${release_stage}/"
   cp "${salt_dir}/fluentd-gcp/fluentd-gcp.yaml" "${release_stage}/"
   cp "${salt_dir}/kube-registry-proxy/kube-registry-proxy.yaml" "${release_stage}/"
+  cp "${salt_dir}/kube-proxy/kube-proxy.manifest" "${release_stage}/"
+
   # Source 2: manifests from cluster/gce/kube-manifests.
   # TODO(andyzheng0831): Enable the following line after finishing issue #16702.
-  # cp "${KUBE_ROOT}/cluster/gce/kube-manifests/*" "${release_stage}/"
+  # cp "${KUBE_ROOT}/cluster/gce/kube-manifests/"* "${release_stage}/"
 
   kube::release::clean_cruft
 

cluster/gce/trusty/configure.sh

@@ -45,8 +45,10 @@ config_ip_firewall() {
 create_dirs() {
   # Create required directories.
   mkdir -p /var/lib/kubelet
-  mkdir -p /var/lib/kube-proxy
   mkdir -p /etc/kubernetes/manifests
+  if [ "${KUBERNETES_MASTER:-}" = "false" ]; then
+    mkdir -p /var/lib/kube-proxy
+  fi
 }
 
 download_kube_env() {
@@ -65,7 +67,6 @@ for k,v in yaml.load(sys.stdin).iteritems():
 
 create_kubelet_kubeconfig() {
   # Create the kubelet kubeconfig file.
-  . /etc/kube-env
   if [ -z "${KUBELET_CA_CERT:-}" ]; then
     KUBELET_CA_CERT="${CA_CERT}"
   fi
@@ -158,38 +159,45 @@ download_or_bust() {
 # Downloads kubernetes binaries and kube-system manifest tarball, unpacks them,
 # and places them into suitable directories.
 install_kube_binary_config() {
-  . /etc/kube-env
-  # For a testing cluster, we pull kubelet, kube-proxy, and kubectl binaries,
-  # and place them in /usr/local/bin. For a non-test cluster, we use the binaries
-  # pre-installed in the image, or pull and place them in /usr/bin if they are
-  # not pre-installed.
+  # In anyway we have to download the release tarball as docker_tag files and
+  # kube-proxy image file are there.
+  cd /tmp
+  k8s_sha1="${SERVER_BINARY_TAR_URL##*/}.sha1"
+  echo "Downloading k8s tar sha1 file ${k8s_sha1}"
+  download_or_bust "${k8s_sha1}" "${SERVER_BINARY_TAR_URL}.sha1"
+  k8s_tar="${SERVER_BINARY_TAR_URL##*/}"
+  echo "Downloading k8s tar file ${k8s_tar}"
+  download_or_bust "${k8s_tar}" "${SERVER_BINARY_TAR_URL}"
+  # Validate hash.
+  actual=$(sha1sum ${k8s_tar} | awk '{ print $1 }') || true
+  if [ "${actual}" != "${SERVER_BINARY_TAR_HASH}" ]; then
+    echo "== ${k8s_tar} corrupted, sha1 ${actual} doesn't match expected ${SERVER_BINARY_TAR_HASH} =="
+  else
+    echo "Validated ${SERVER_BINARY_TAR_URL} SHA1 = ${SERVER_BINARY_TAR_HASH}"
+  fi
+  tar xzf "/tmp/${k8s_tar}" -C /tmp/ --overwrite
+  # Copy docker_tag and image files to /run/kube-docker-files.
+  mkdir -p /run/kube-docker-files
+  cp /tmp/kubernetes/server/bin/*.docker_tag /run/kube-docker-files/
+  if [ "${KUBERNETES_MASTER:-}" = "false" ]; then
+    cp /tmp/kubernetes/server/bin/kube-proxy.tar /run/kube-docker-files/
+  fi
+  # For a testing cluster, we use kubelet, kube-proxy, and kubectl binaries
+  # from the release tarball and place them in /usr/local/bin. For a non-test
+  # cluster, we use the binaries pre-installed in the image, or pull and place
+  # them in /usr/bin if they are not pre-installed.
   BINARY_PATH="/usr/bin/"
   if [ "${TEST_CLUSTER:-}" = "true" ]; then
     BINARY_PATH="/usr/local/bin/"
   fi
   if ! which kubelet > /dev/null || ! which kube-proxy > /dev/null || [ "${TEST_CLUSTER:-}" = "true" ]; then
-    cd /tmp
-    k8s_sha1="${SERVER_BINARY_TAR_URL##*/}.sha1"
-    echo "Downloading k8s tar sha1 file ${k8s_sha1}"
-    download_or_bust "${k8s_sha1}" "${SERVER_BINARY_TAR_URL}.sha1"
-    k8s_tar="${SERVER_BINARY_TAR_URL##*/}"
-    echo "Downloading k8s tar file ${k8s_tar}"
-    download_or_bust "${k8s_tar}" "${SERVER_BINARY_TAR_URL}"
-    # Validate hash.
-    actual=$(sha1sum ${k8s_tar} | awk '{ print $1 }') || true
-    if [ "${actual}" != "${SERVER_BINARY_TAR_HASH}" ]; then
-      echo "== ${k8s_tar} corrupted, sha1 ${actual} doesn't match expected ${SERVER_BINARY_TAR_HASH} =="
-    else
-      echo "Validated ${SERVER_BINARY_TAR_URL} SHA1 = ${SERVER_BINARY_TAR_HASH}"
-    fi
-    tar xzf "/tmp/${k8s_tar}" -C /tmp/ --overwrite
     cp /tmp/kubernetes/server/bin/kubelet ${BINARY_PATH}
-    cp /tmp/kubernetes/server/bin/kube-proxy ${BINARY_PATH}
     cp /tmp/kubernetes/server/bin/kubectl ${BINARY_PATH}
-    rm -rf "/tmp/kubernetes"
-    rm "/tmp/${k8s_tar}"
-    rm "/tmp/${k8s_sha1}"
   fi
+  # Clean up.
+  rm -rf "/tmp/kubernetes"
+  rm "/tmp/${k8s_tar}"
+  rm "/tmp/${k8s_sha1}"
 
   # Put kube-system pods manifests in /etc/kube-manifests/.
   mkdir -p /run/kube-manifests
@@ -213,7 +221,6 @@ install_kube_binary_config() {
 }
 
 restart_docker_daemon() {
-  . /etc/kube-env
   # Assemble docker deamon options
   DOCKER_OPTS="-p /var/run/docker.pid --bridge=cbr0 --iptables=false --ip-masq=false"
   if [ "${TEST_CLUSTER:-}" = "true" ]; then
@@ -230,3 +237,12 @@ restart_docker_daemon() {
   ifconfig docker0 down
   brctl delbr docker0
 }
+
+# Create the log file and set its properties.
+#
+# $1 is the file to create
+prepare_log_file() {
+  touch $1
+  chmod 644 $1
+  chown root:root $1
+}

cluster/gce/trusty/node.yaml

-From nobody Tue Aug 11 10:13:54 2015
+From nobody Tue Dec 22 10:13:54 2015
 Content-Type: multipart/mixed; boundary="===================================="
 MIME-Version: 1.0
 
@@ -29,10 +29,11 @@ script
 	config_hostname
 	echo "Configuring IP firewall rules"
 	config_ip_firewall
-	echo "Creating required directories"
-	create_dirs
 	echo "Downloading kube-env file"
 	download_kube_env
+	echo "Creating required directories"
+	. /etc/kube-env
+	create_dirs
 	echo "Creating kubelet kubeconfig file"
 	create_kubelet_kubeconfig
 	echo "Creating kube-proxy kubeconfig file"
@@ -96,6 +97,7 @@ script
 	set -o nounset
 
 	. /etc/kube-configure.sh
+	. /etc/kube-env
 	install_kube_binary_config
 end script
 
@@ -148,59 +150,77 @@ post-stop exec sleep 10
 MIME-Version: 1.0
 Content-Type: text/upstart-job; charset="us-ascii"
 Content-Transfer-Encoding: 7bit
-Content-Disposition: attachment; filename="kube-proxy.conf"
+Content-Disposition: attachment; filename="kube-docker.conf"
 
 #upstart-job
 
-description "Start kube-proxy service"
-
-start on stopped kube-install-minion and stopped kube-install-packages
+description "Restart docker daemon"
 
-respawn
+# The condition "stopped kube-install-additional-packages" is to avoid
+# breaking nsenter installation, which is through a docker container.
+# It can be removed if we find a better way to install nsenter.
+start on started kubelet and stopped kube-install-additional-packages
 
 script
 	set -o errexit
 	set -o nounset
 
+	. /etc/kube-configure.sh
 	. /etc/kube-env
-	ARGS="--v=2"
-	if [ -n "${KUBEPROXY_TEST_ARGS:-}" ]; then
-		ARGS="${KUBEPROXY_TEST_ARGS}"
-	fi
-	BINARY_PATH="/usr/bin/kube-proxy"
-	if [ "${TEST_CLUSTER:-}" = "true" ]; then
-		BINARY_PATH="/usr/local/bin/kube-proxy"
-	fi
-	${BINARY_PATH} \
-		--master=https://${KUBERNETES_MASTER_NAME} \
-		--kubeconfig=/var/lib/kube-proxy/kubeconfig \
-		${ARGS}
+	restart_docker_daemon
 end script
 
-# Wait for 10s to start kube-proxy again.
-post-stop exec sleep 10
-
 --====================================
 MIME-Version: 1.0
 Content-Type: text/upstart-job; charset="us-ascii"
 Content-Transfer-Encoding: 7bit
-Content-Disposition: attachment; filename="kube-docker.conf"
+Content-Disposition: attachment; filename="kube-proxy.conf"
 
 #upstart-job
 
-description "Restart docker daemon"
+description "Start kube-proxy static pod"
 
-# The condition "stopped kube-install-additional-packages" is to avoid
-# breaking nsenter installation, which is through a docker container.
-# It can be removed if we find a better way to install nsenter.
-start on started kubelet and stopped kube-install-additional-packages
+start on stopped kube-docker
 
 script
 	set -o errexit
 	set -o nounset
 
 	. /etc/kube-configure.sh
-	restart_docker_daemon
+	. /etc/kube-env
+	prepare_log_file "/var/log/kube-proxy.log"
+	# Load the docker image from file /run/kube-docker-files/kube-proxy.tar.
+	echo "Try to load docker image file kube-proxy.tar"
+	timeout 30 docker load -i /run/kube-docker-files/kube-proxy.tar
+	# Copy the manifest to /tmp to manipulate
+	tmp_file="/tmp/kube-proxy.manifest"
+	cp -f /run/kube-manifests/kubernetes/kube-proxy.manifest ${tmp_file}
+	# Remove the lines of salt configuration and replace variables with values.
+	# NOTE: Changes to variable names in cluster/saltbase/salt/kube-proxy/kube-proxy.manifest
+	# may break this upstart job.
+	sed -i "/^ *{%/d" ${tmp_file}
+	kubeconfig="--kubeconfig=\/var\/lib\/kube-proxy\/kubeconfig"
+	kube_docker_registry="gcr.io\/google_containers"
+	if [ -n "${KUBE_DOCKER_REGISTRY:-}" ]; then
+		kube_docker_registry=${KUBE_DOCKER_REGISTRY}
+	fi
+	kube_proxy_docker_tag=$(cat /run/kube-docker-files/kube-proxy.docker_tag)
+	test_args=""
+	log_level="--v=2"
+	if [ -n "${KUBEPROXY_TEST_ARGS:-}" ]; then
+		test_args="${KUBEPROXY_TEST_ARGS}"
+		# test_args should already contain log level setting.
+		log_level=""
+	fi
+	api_servers="--master=https:\/\/${KUBERNETES_MASTER_NAME}"
+	sed -i -e "s/{{kubeconfig}}/${kubeconfig}/g" ${tmp_file}
+	sed -i -e "s/{{pillar\['kube_docker_registry'\]}}/${kube_docker_registry}/g" ${tmp_file}
+	sed -i -e "s/{{pillar\['kube-proxy_docker_tag'\]}}/${kube_proxy_docker_tag}/g" ${tmp_file}
+	sed -i -e "s/{{test_args}}/${test_args}/g" ${tmp_file}
+	sed -i -e "s/{{log_level}}/${log_level}/g" ${tmp_file}
+	sed -i -e "s/{{api_servers_with_port}}/${api_servers}/g" ${tmp_file}
+
+	mv -f ${tmp_file} /etc/kubernetes/manifests/
 end script
 
 --====================================
@@ -243,7 +263,8 @@ Content-Disposition: attachment; filename="kube-node-health-monitoring.conf"
 
 description "Kubenetes node health monitoring"
 
-start on stopped kube-docker and started kube-proxy
+# The termination of kube-proxy upstart job indicates that kubelet and docker are ready.
+start on stopped kube-proxy
 
 respawn
 
@@ -254,6 +275,7 @@ script
 	# TODO(andyzheng0831): replace it with a more reliable method if possible.
 	sleep 60
 
+	. /etc/kube-env
 	sleep_seconds=10
 	max_seconds=10
 	# We simply kill the process when there is a failure. Another upstart job will automatically
@@ -263,14 +285,15 @@ script
 			echo "Docker daemon failed!"
 			pkill docker
 		fi
-		. /etc/kube-env
 		if ! curl --insecure -m ${max_seconds} -f -s https://127.0.0.1:${KUBELET_PORT:-10250}/healthz > /dev/null; then
 			echo "Kubelet is unhealthy!"
 			pkill kubelet
 		fi
 		if ! curl -m ${max_seconds} -f -s http://127.0.0.1:10249/healthz > /dev/null; then
 			echo "Kube-proxy is unhealthy!"
-			pkill kube-proxy
+			# Get the ID of kube-proxy container and then kill it.
+			container=$(docker ps -q --filter name='k8s_kube-proxy')
+			docker kill ${container}
 		fi
 		sleep ${sleep_seconds}
 	done