Merge pull request #20463 from andyzheng0831/trusty-master

build/common.sh

@@ -883,27 +883,36 @@ function kube::release::package_salt_tarball() {
 # such as Ubuntu Trusty.
 #
 # There are two sources of manifests files: (1) some manifests in the directory
-# cluster/saltbase/salt can be used directly or after minor revision, so we copy
-# them from there; (2) otherwise, we will maintain separate copies in
-# cluster/gce/kube-manifests.
+# cluster/saltbase/salt and cluster/addons can be used directly or after minor
+# revision, so we copy them from there; (2) otherwise, we will maintain separate
+# copies in cluster/gce/<distro>/kube-manifests.
 function kube::release::package_kube_manifests_tarball() {
   kube::log::status "Building tarball: manifests"
 
   local release_stage="${RELEASE_STAGE}/manifests/kubernetes"
   rm -rf "${release_stage}"
-  mkdir -p "${release_stage}"
+  mkdir -p "${release_stage}/trusty"
 
-  # Source 1: manifests from cluster/saltbase/salt.
-  # TODO(andyzheng0831): Add more manifests when supporting master on trusty.
+  # Source 1: manifests from cluster/saltbase/salt and cluster/addons
   local salt_dir="${KUBE_ROOT}/cluster/saltbase/salt"
   cp "${salt_dir}/fluentd-es/fluentd-es.yaml" "${release_stage}/"
   cp "${salt_dir}/fluentd-gcp/fluentd-gcp.yaml" "${release_stage}/"
   cp "${salt_dir}/kube-registry-proxy/kube-registry-proxy.yaml" "${release_stage}/"
   cp "${salt_dir}/kube-proxy/kube-proxy.manifest" "${release_stage}/"
+  cp "${salt_dir}/etcd/etcd.manifest" "${release_stage}/trusty"
+  cp "${salt_dir}/kube-scheduler/kube-scheduler.manifest" "${release_stage}/trusty"
+  cp "${salt_dir}/kube-addons/namespace.yaml" "${release_stage}/trusty"
+  cp "${salt_dir}/kube-addons/kube-addons.sh" "${release_stage}/trusty"
+  cp "${salt_dir}/kube-addons/kube-addon-update.sh" "${release_stage}/trusty"
+  cp -r "${salt_dir}/kube-admission-controls/limit-range" "${release_stage}/trusty"
+  local objects
+  objects=$(cd "${KUBE_ROOT}/cluster/addons" && find . \( -name \*.yaml -or -name \*.yaml.in -or -name \*.json \) | grep -v demo)
+  tar c -C "${KUBE_ROOT}/cluster/addons" ${objects} | tar x -C "${release_stage}/trusty"
 
-  # Source 2: manifests from cluster/gce/kube-manifests.
-  # TODO(andyzheng0831): Enable the following line after finishing issue #16702.
-  # cp "${KUBE_ROOT}/cluster/gce/kube-manifests/*" "${release_stage}/"
+  # Source 2: manifests from cluster/gce/<distro>/kube-manifests.
+  # TODO(andyzheng0831): Avoid using separate copies for trusty. We should use whatever
+  # from cluster/saltbase/salt to minimize maintenance cost.
+  cp "${KUBE_ROOT}/cluster/gce/trusty/kube-manifests/"* "${release_stage}/trusty"
   cp -r "${KUBE_ROOT}/cluster/gce/coreos/kube-manifests"/* "${release_stage}/"
 
   kube::release::clean_cruft
@@ -1094,10 +1103,11 @@ function kube::release::gcs::copy_release_artifacts() {
   # Stage everything in release directory
   kube::release::gcs::stage_and_hash "${RELEASE_DIR}"/* . || return 1
 
-  # Having the configure-vm.sh and trusty/node.yaml scripts from the GCE cluster
+  # Having the configure-vm.sh script and and trusty code from the GCE cluster
   # deploy hosted with the release is useful for GKE.
   kube::release::gcs::stage_and_hash "${RELEASE_STAGE}/full/kubernetes/cluster/gce/configure-vm.sh" extra/gce || return 1
   kube::release::gcs::stage_and_hash "${RELEASE_STAGE}/full/kubernetes/cluster/gce/trusty/node.yaml" extra/gce || return 1
+  kube::release::gcs::stage_and_hash "${RELEASE_STAGE}/full/kubernetes/cluster/gce/trusty/master.yaml" extra/gce || return 1
   kube::release::gcs::stage_and_hash "${RELEASE_STAGE}/full/kubernetes/cluster/gce/trusty/configure.sh" extra/gce || return 1
 
   # Upload the "naked" binaries to GCS.  This is useful for install scripts that

cluster/gce/trusty/helper.sh

@@ -22,10 +22,6 @@
 # replaced upstart with systemd as the init system. Consequently, the
 # configuration cannot work on these images.
 
-# By sourcing debian's helper.sh, we use the same  create-master-instance
-# functions as debian. But we overwrite the create-node-instance-template
-# function to use Ubuntu.
-source "${KUBE_ROOT}/cluster/gce/debian/helper.sh"
 
 # $1: template name (required)
 function create-node-instance-template {
@@ -35,3 +31,37 @@ function create-node-instance-template {
     "user-data=${KUBE_ROOT}/cluster/gce/trusty/node.yaml" \
     "configure-sh=${KUBE_ROOT}/cluster/gce/trusty/configure.sh"
 }
+
+# create-master-instance creates the master instance. If called with
+# an argument, the argument is used as the name to a reserved IP
+# address for the master. (In the case of upgrade/repair, we re-use
+# the same IP.)
+#
+# It requires a whole slew of assumed variables, partially due to to
+# the call to write-master-env. Listing them would be rather
+# futile. Instead, we list the required calls to ensure any additional
+# variables are set:
+#   ensure-temp-dir
+#   detect-project
+#   get-bearer-token
+#
+function create-master-instance {
+  local address_opt=""
+  [[ -n ${1:-} ]] && address_opt="--address ${1}"
+
+  write-master-env
+  gcloud compute instances create "${MASTER_NAME}" \
+    ${address_opt} \
+    --project "${PROJECT}" \
+    --zone "${ZONE}" \
+    --machine-type "${MASTER_SIZE}" \
+    --image-project="${MASTER_IMAGE_PROJECT}" \
+    --image "${MASTER_IMAGE}" \
+    --tags "${MASTER_TAG}" \
+    --network "${NETWORK}" \
+    --scopes "storage-ro,compute-rw,monitoring,logging-write" \
+    --can-ip-forward \
+    --metadata-from-file \
+      "kube-env=${KUBE_TEMP}/master-kube-env.yaml,user-data=${KUBE_ROOT}/cluster/gce/trusty/master.yaml,configure-sh=${KUBE_ROOT}/cluster/gce/trusty/configure.sh" \
+    --disk "name=${MASTER_NAME}-pd,device-name=master-pd,mode=rw,boot=no,auto-delete=no"
+}

cluster/gce/trusty/kube-manifests/kube-apiserver.manifest

+{
+"apiVersion": "v1",
+"kind": "Pod",
+"metadata": {
+  "name":"kube-apiserver",
+  "namespace": "kube-system"
+},
+"spec":{
+"hostNetwork": true,
+"containers":[
+    {
+    "name": "kube-apiserver",
+    "image": "{{kube_docker_registry}}/kube-apiserver:{{kube-apiserver_docker_tag}}",
+    "resources": {
+      "limits": {
+        "cpu": "250m"
+      }
+    },
+    "command": [
+                 "/bin/sh",
+                 "-c",
+                 "/usr/local/bin/kube-apiserver {{params}} 1>>/var/log/kube-apiserver.log 2>&1"
+               ],
+    "livenessProbe": {
+      "httpGet": {
+        "host": "127.0.0.1",
+        "port": 8080,
+        "path": "/healthz"
+      },
+      "initialDelaySeconds": 15,
+      "timeoutSeconds": 15
+    },
+    "ports":[
+      { "name": "https",
+        "containerPort": 443,
+        "hostPort": 443},{
+       "name": "local",
+        "containerPort": 8080,
+        "hostPort": 8080}
+        ],
+    "volumeMounts": [
+        {{cloud_config_mount}}
+        { "name": "srvkube",
+        "mountPath": "/etc/srv/kubernetes",
+        "readOnly": true},
+        { "name": "logfile",
+        "mountPath": "/var/log/kube-apiserver.log",
+        "readOnly": false},
+        { "name": "etcssl",
+        "mountPath": "/etc/ssl",
+        "readOnly": true},
+        { "name": "varssl",
+        "mountPath": "/var/ssl",
+        "readOnly": true},
+        { "name": "etcopenssl",
+        "mountPath": "/etc/openssl",
+        "readOnly": true},
+        { "name": "etcpkitls",
+        "mountPath": "/etc/pki/tls",
+        "readOnly": true},
+        { "name": "srvsshproxy",
+        "mountPath": "/etc/srv/sshproxy",
+        "readOnly": false}
+      ]
+    }
+],
+"volumes":[
+  {{cloud_config_volume}}
+  { "name": "srvkube",
+    "hostPath": {
+        "path": "/etc/srv/kubernetes"}
+  },
+  { "name": "logfile",
+    "hostPath": {
+        "path": "/var/log/kube-apiserver.log"}
+  },
+  { "name": "etcssl",
+    "hostPath": {
+        "path": "/etc/ssl"}
+  },
+  { "name": "varssl",
+    "hostPath": {
+        "path": "/var/ssl"}
+  },
+  { "name": "etcopenssl",
+    "hostPath": {
+        "path": "/etc/openssl"}
+  },
+  { "name": "etcpkitls",
+    "hostPath": {
+        "path": "/etc/pki/tls"}
+  },
+  { "name": "srvsshproxy",
+    "hostPath": {
+        "path": "/etc/srv/sshproxy"}
+  }
+]
+}}

cluster/gce/trusty/kube-manifests/kube-controller-manager.manifest

+{
+"apiVersion": "v1",
+"kind": "Pod",
+"metadata": {
+  "name":"kube-controller-manager",
+  "namespace": "kube-system"
+},
+"spec":{
+"hostNetwork": true,
+"containers":[
+    {
+    "name": "kube-controller-manager",
+    "image": "{{kube_docker_registry}}/kube-controller-manager:{{kube-controller-manager_docker_tag}}",
+    "resources": {
+      "limits": {
+        "cpu": "200m"
+      }
+    },
+    "command": [
+                 "/bin/sh",
+                 "-c",
+                 "/usr/local/bin/kube-controller-manager {{params}} 1>>/var/log/kube-controller-manager.log 2>&1"
+               ],
+    "livenessProbe": {
+      "httpGet": {
+        "host": "127.0.0.1",
+        "port": 10252,
+        "path": "/healthz"
+      },
+      "initialDelaySeconds": 15,
+      "timeoutSeconds": 15
+    },
+    "volumeMounts": [
+        {{cloud_config_mount}}
+        { "name": "srvkube",
+        "mountPath": "/etc/srv/kubernetes",
+        "readOnly": true},
+        { "name": "logfile",
+        "mountPath": "/var/log/kube-controller-manager.log",
+        "readOnly": false},
+        { "name": "etcssl",
+        "mountPath": "/etc/ssl",
+        "readOnly": true},
+        { "name": "varssl",
+        "mountPath": "/var/ssl",
+        "readOnly": true},
+        { "name": "etcopenssl",
+        "mountPath": "/etc/openssl",
+        "readOnly": true},
+        { "name": "etcpkitls",
+        "mountPath": "/etc/pki/tls",
+        "readOnly": true}
+      ]
+    }
+],
+"volumes":[
+  {{cloud_config_volume}}
+  { "name": "srvkube",
+    "hostPath": {
+        "path": "/etc/srv/kubernetes"}
+  },
+  { "name": "logfile",
+    "hostPath": {
+        "path": "/var/log/kube-controller-manager.log"}
+  },
+  { "name": "etcssl",
+    "hostPath": {
+        "path": "/etc/ssl"}
+  },
+  { "name": "varssl",
+    "hostPath": {
+        "path": "/var/ssl"}
+  },
+  { "name": "etcopenssl",
+    "hostPath": {
+        "path": "/etc/openssl"}
+  },
+  { "name": "etcpkitls",
+    "hostPath": {
+        "path": "/etc/pki/tls"}
+  }
+]
+}}

cluster/gce/trusty/master.yaml

+From nobody Tue Feb 1 11:33:00 2016
+Content-Type: multipart/mixed; boundary="===================================="
+MIME-Version: 1.0
+
+--====================================
+MIME-Version: 1.0
+Content-Type: text/upstart-job; charset="us-ascii"
+Content-Transfer-Encoding: 7bit
+Content-Disposition: attachment; filename="kube-env.conf"
+
+#upstart-job
+
+description "Prepare kube master environment"
+
+start on cloud-config
+
+script
+	set -o errexit
+	set -o nounset
+
+	# Fetch the script for configuring the instance.
+	curl --fail --silent --show-error \
+		-H "X-Google-Metadata-Request: True" \
+		-o /etc/kube-configure.sh \
+		http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-sh
+	. /etc/kube-configure.sh
+
+	echo "Configuring hostname"
+	config_hostname
+	echo "Configuring IP firewall rules"
+	config_ip_firewall
+	echo "Downloading kube-env file"
+	download_kube_env
+	. /etc/kube-env
+	echo "Creating required directories"
+	create_dirs
+	echo "Mount master PD"
+	mount_master_pd
+	echo "Creating kuberntes master auth file"
+	create_master_auth
+	echo "Creating master instance kubelet auth file"
+  create_master_kubelet_auth
+end script
+
+--====================================
+MIME-Version: 1.0
+Content-Type: text/upstart-job; charset="us-ascii"
+Content-Transfer-Encoding: 7bit
+Content-Disposition: attachment; filename="kube-install-packages.conf"
+
+#upstart-job
+
+description "Install packages needed to run kubernetes"
+
+start on stopped kube-env
+
+script
+	set -o errexit
+	set -o nounset
+
+	. /etc/kube-configure.sh
+	install_critical_packages
+end script
+
+--====================================
+MIME-Version: 1.0
+Content-Type: text/upstart-job; charset="us-ascii"
+Content-Transfer-Encoding: 7bit
+Content-Disposition: attachment; filename="kube-install-additional-packages.conf"
+
+#upstart-job
+
+description "Install additional packages used by kubernetes"
+
+start on stopped kube-install-packages
+
+script
+	set -o errexit
+	set -o nounset
+
+	. /etc/kube-configure.sh
+	install_additional_packages
+end script
+
+--====================================
+MIME-Version: 1.0
+Content-Type: text/upstart-job; charset="us-ascii"
+Content-Transfer-Encoding: 7bit
+Content-Disposition: attachment; filename="kube-install-master.conf"
+
+#upstart-job
+
+description "Download and install k8s binaries and configurations"
+
+start on stopped kube-env
+
+script
+	set -o errexit
+	set -o nounset
+
+	. /etc/kube-configure.sh
+	. /etc/kube-env
+	install_kube_binary_config
+end script
+
+--====================================
+MIME-Version: 1.0
+Content-Type: text/upstart-job; charset="us-ascii"
+Content-Transfer-Encoding: 7bit
+Content-Disposition: attachment; filename="kubelet.conf"
+
+#upstart-job
+
+description "Run kubelet service"
+
+start on stopped kube-install-packages and stopped kube-install-master
+
+respawn
+
+script
+	set -o errexit
+	set -o nounset
+
+	echo "Start kubelet upstart job"
+
+	. /etc/kube-env
+	BINARY_PATH="/usr/bin/kubelet"
+	if [ "${TEST_CLUSTER:-}" = "true" ]; then
+		BINARY_PATH="/usr/local/bin/kubelet"
+	fi
+	# Assemble command line flags based on env variables.
+	ARGS="--v=2"
+	if [ -n "${KUBELET_TEST_ARGS:-}" ]; then
+		ARGS="${KUBELET_TEST_ARGS}"
+	fi
+	if [ ! -z "${KUBELET_APISERVER:-}" ] && [ ! -z "${KUBELET_CERT:-}" ] && [ ! -z "${KUBELET_KEY:-}" ]; then
+		ARGS="${ARGS} --api-servers=https://${KUBELET_APISERVER}"
+		ARGS="${ARGS} --register-schedulable=false --reconcile-cidr=false"
+		ARGS="${ARGS} --pod-cidr=10.123.45.0/30"
+	else
+		ARGS="${ARGS} --pod-cidr=${MASTER_IP_RANGE}"
+	fi
+	if [ "${ENABLE_MANIFEST_URL:-}" = "true" ]; then
+		ARGS="${ARGS} --manifest-url=${MANIFEST_URL} --manifest-url-header=${MANIFEST_URL_HEADER}"
+	fi
+	
+	${BINARY_PATH} \
+		--enable-debugging-handlers=false \
+		--cloud-provider=gce \
+		--config=/etc/kubernetes/manifests \
+		--allow-privileged=true \
+		--cluster-dns=${DNS_SERVER_IP} \
+		--cluster-domain=${DNS_DOMAIN} \
+		--configure-cbr0=${ALLOCATE_NODE_CIDRS} \
+		--cgroup-root=/ \
+		--system-container=/system \
+		--nosystemd=true \
+		${ARGS}
+end script
+
+# Wait for 10s to start kubelet again.
+post-stop exec sleep 10
+
+--====================================
+MIME-Version: 1.0
+Content-Type: text/upstart-job; charset="us-ascii"
+Content-Transfer-Encoding: 7bit
+Content-Disposition: attachment; filename="kube-docker.conf"
+
+#upstart-job
+
+description "Restart docker daemon"
+
+# The condition "stopped kube-install-additional-packages" is to avoid
+# breaking nsenter installation, which is through a docker container.
+# It can be removed if we find a better way to install nsenter.
+start on started kubelet and stopped kube-install-additional-packages
+
+script
+	set -o errexit
+	set -o nounset
+
+	. /etc/kube-configure.sh
+	. /etc/kube-env
+	restart_docker_daemon
+end script
+
+--====================================
+MIME-Version: 1.0
+Content-Type: text/upstart-job; charset="us-ascii"
+Content-Transfer-Encoding: 7bit
+Content-Disposition: attachment; filename="kube-master-components.conf"
+
+#upstart-job
+
+description "Start kube-master components and addons pods"
+
+start on stopped kube-docker
+
+script
+	set -o errexit
+	set -o nounset
+
+	. /etc/kube-configure.sh
+	. /etc/kube-env
+	start_etcd_servers
+	compute_master_manifest_variables
+	start_kube_apiserver
+	start_kube_controller_manager
+	start_kube_scheduler
+	start_kube_addons
+end script
+
+--====================================
+MIME-Version: 1.0
+Content-Type: text/upstart-job; charset="us-ascii"
+Content-Transfer-Encoding: 7bit
+Content-Disposition: attachment; filename="kube-master-health-monitoring.conf"
+
+#upstart-job
+
+description "Kubenetes master health monitoring"
+
+start on stopped kube-docker
+
+script
+	set -o errexit
+	set -o nounset
+
+	. /etc/kube-configure.sh
+	. /etc/kube-env
+	health_monitoring
+end script
+
+--====================================--

cluster/gce/trusty/node.yaml

-From nobody Tue Dec 22 10:13:54 2015
+From nobody Tue Feb 1 11:33:00 2016
 Content-Type: multipart/mixed; boundary="===================================="
 MIME-Version: 1.0
 
@@ -240,15 +240,10 @@ script
 	set -o nounset
 
 	# Kube-system pod manifest files are located at /run/kube-manifests/kubernetes.
+	. /etc/kube-configure.sh
 	. /etc/kube-env
 	# Fluentd
-	if [ "${ENABLE_NODE_LOGGING:-}" = "true" ]; then
-		if [ "${LOGGING_DESTINATION:-}" = "gcp" ]; then
-			cp /run/kube-manifests/kubernetes/fluentd-gcp.yaml /etc/kubernetes/manifests/
-		elif [ "${LOGGING_DESTINATION:-}" = "elasticsearch" ]; then
-			cp /run/kube-manifests/kubernetes/fluentd-es.yaml /etc/kubernetes/manifests/
-		fi
-	fi
+	start_fluentd
 	# Kube-registry-proxy
 	if [ "${ENABLE_CLUSTER_REGISTRY:-}" = "true" ]; then
 		cp /run/kube-manifests/kubernetes/kube-registry-proxy.yaml /etc/kubernetes/manifests/
@@ -275,28 +270,9 @@ script
 	# TODO(andyzheng0831): replace it with a more reliable method if possible.
 	sleep 60
 
+	. /etc/kube-configure.sh
 	. /etc/kube-env
-	sleep_seconds=10
-	max_seconds=10
-	# We simply kill the process when there is a failure. Another upstart job will automatically
-	# restart the process.
-	while [ 1 ]; do
-		if ! timeout 20 docker ps > /dev/null; then
-			echo "Docker daemon failed!"
-			pkill docker
-		fi
-		if ! curl --insecure -m ${max_seconds} -f -s https://127.0.0.1:${KUBELET_PORT:-10250}/healthz > /dev/null; then
-			echo "Kubelet is unhealthy!"
-			pkill kubelet
-		fi
-		if ! curl -m ${max_seconds} -f -s http://127.0.0.1:10249/healthz > /dev/null; then
-			echo "Kube-proxy is unhealthy!"
-			# Get the ID of kube-proxy container and then kill it.
-			container=$(docker ps -q --filter name='k8s_kube-proxy')
-			docker kill ${container}
-		fi
-		sleep ${sleep_seconds}
-	done
+	health_monitoring
 end script
 
 --====================================--

cluster/saltbase/salt/kube-addons/kube-addons.sh

@@ -23,6 +23,7 @@ ADDON_CHECK_INTERVAL_SEC=${TEST_ADDON_CHECK_INTERVAL_SEC:-600}
 
 SYSTEM_NAMESPACE=kube-system
 token_dir=${TOKEN_DIR:-/srv/kubernetes}
+trusty_master=${TRUSTY_MASTER:-false}
 
 function ensure_python() {
   if ! python --version > /dev/null 2>&1; then    
@@ -162,8 +163,11 @@ function load-docker-images() {
 # managed result is of that. Start everything below that directory.
 echo "== Kubernetes addon manager started at $(date -Is) with ADDON_CHECK_INTERVAL_SEC=${ADDON_CHECK_INTERVAL_SEC} =="
 
-# Load any images that we may need
-load-docker-images /srv/salt/kube-addons-images
+# Load any images that we may need. This is not needed for trusty master and
+# the way it restarts docker daemon does not work for trusty.
+if [[ "${trusty_master}" == "false" ]]; then
+  load-docker-images /srv/salt/kube-addons-images
+fi
 
 ensure_python