Merge pull request #33965 from euank/coreos-kube-up-now-with-less-cloud-init

build/lib/release.sh

@@ -331,14 +331,12 @@ function kube::release::package_kube_manifests_tarball() {
   cp "${KUBE_ROOT}/cluster/gce/gci/configure-helper.sh" "${dst_dir}/gci-configure-helper.sh"
   cp "${KUBE_ROOT}/cluster/gce/gci/mounter/mounter" "${dst_dir}/gci-mounter"
   cp "${KUBE_ROOT}/cluster/gce/gci/health-monitor.sh" "${dst_dir}/health-monitor.sh"
+  cp "${KUBE_ROOT}/cluster/gce/coreos/configure-helper.sh" "${dst_dir}/coreos-configure-helper.sh"
   cp -r "${salt_dir}/kube-admission-controls/limit-range" "${dst_dir}"
   local objects
   objects=$(cd "${KUBE_ROOT}/cluster/addons" && find . \( -name \*.yaml -or -name \*.yaml.in -or -name \*.json \) | grep -v demo)
   tar c -C "${KUBE_ROOT}/cluster/addons" ${objects} | tar x -C "${dst_dir}"
 
-  # This is for coreos only. ContainerVM, GCI, or Trusty does not use it.
-  cp -r "${KUBE_ROOT}/cluster/gce/coreos/kube-manifests"/* "${release_stage}/"
-
   kube::release::clean_cruft
 
   local package_name="${RELEASE_DIR}/kubernetes-manifests.tar.gz"

cluster/addons/dashboard/dashboard-controller.yaml

-# This file should be kept in sync with cluster/gce/coreos/kube-manifests/addons/dashboard/dashboard-controller.yaml
 apiVersion: extensions/v1beta1
 kind: Deployment
 metadata:

cluster/addons/dashboard/dashboard-service.yaml

-# This file should be kept in sync with cluster/gce/coreos/kube-manifests/addons/dashboard/dashboard-service.yaml
 apiVersion: v1
 kind: Service
 metadata:

cluster/gce/coreos/README.md

+# CoreOS image
+
+The [CoreOS operating system](https://coreos.com/why/) is a Linux distribution optimized for running containers securely at scale.
+CoreOS provides [an image](https://coreos.com/os/docs/latest/booting-on-google-compute-engine.html) for Google Cloud Platform (GCP).
+
+This folder contains configuration and tooling to allow kube-up to create a Kubernetes cluster on Google Cloud Platform running on the official CoreOS image.
+
+[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/cluster/gce/coreos/README.md?pixel)]()

cluster/gce/coreos/configure.sh

+#!/bin/bash
+
+# Copyright 2016 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+function download-kube-env {
+  # Fetch kube-env from GCE metadata server.
+  local -r tmp_kube_env="/tmp/kube-env.yaml"
+  curl --fail --retry 5 --retry-delay 3 --silent --show-error \
+    -H "X-Google-Metadata-Request: True" \
+    -o "${tmp_kube_env}" \
+    http://metadata.google.internal/computeMetadata/v1/instance/attributes/kube-env
+  # Convert the yaml format file into a shell-style file.
+  sed 's/: /=/' < "${tmp_kube_env}" > "${KUBE_HOME}/kube-env"
+  rm -f "${tmp_kube_env}"
+}
+
+function validate-hash {
+  local -r file="$1"
+  local -r expected="$2"
+
+  actual=$(sha1sum ${file} | awk '{ print $1 }') || true
+  if [[ "${actual}" != "${expected}" ]]; then
+    echo "== ${file} corrupted, sha1 ${actual} doesn't match expected ${expected} =="
+    return 1
+  fi
+}
+
+
+# Retry a download until we get it. Takes a hash and a set of URLs.
+#
+# $1 is the sha1 of the URL. Can be "" if the sha1 is unknown.
+# $2+ are the URLs to download.
+function download-or-bust {
+  local -r hash="$1"
+  shift 1
+
+  local -r urls=( $* )
+  while true; do
+    for url in "${urls[@]}"; do
+      local file="${url##*/}"
+      rm -f "${file}"
+      if ! curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --max-time 300 --retry 6 --retry-delay 10 "${url}"; then
+        echo "== Failed to download ${url}. Retrying. =="
+      elif [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then
+        echo "== Hash validation of ${url} failed. Retrying. =="
+      else
+        if [[ -n "${hash}" ]]; then
+          echo "== Downloaded ${url} (SHA1 = ${hash}) =="
+        else
+          echo "== Downloaded ${url} =="
+        fi
+        return
+      fi
+    done
+  done
+}
+
+function split-commas {
+  echo $1 | tr "," "\n"
+}
+
+# Downloads kubernetes binaries and kube-system manifest tarball, unpacks them,
+# and places them into suitable directories. Files are placed in /opt/kubernetes.
+function install-kube-binary-config {
+  cd "${KUBE_HOME}"
+  local -r server_binary_tar_urls=( $(split-commas "${SERVER_BINARY_TAR_URL}") )
+  local -r server_binary_tar="${server_binary_tar_urls[0]##*/}"
+  if [[ -n "${SERVER_BINARY_TAR_HASH:-}" ]]; then
+    local -r server_binary_tar_hash="${SERVER_BINARY_TAR_HASH}"
+  else
+    echo "Downloading binary release sha1 (not found in env)"
+    download-or-bust "" "${server_binary_tar_urls[@]/.tar.gz/.tar.gz.sha1}"
+    local -r server_binary_tar_hash=$(cat "${server_binary_tar}.sha1")
+  fi
+  echo "Downloading binary release tar"
+  download-or-bust "${server_binary_tar_hash}" "${server_binary_tar_urls[@]}"
+  tar xzf "${KUBE_HOME}/${server_binary_tar}" -C "${KUBE_HOME}" --overwrite
+  # Copy docker_tag and image files to ${KUBE_HOME}/kube-docker-files.
+  src_dir="${KUBE_HOME}/kubernetes/server/bin"
+  dst_dir="${KUBE_HOME}/kube-docker-files"
+  mkdir -p "${dst_dir}"
+  cp "${src_dir}/"*.docker_tag "${dst_dir}"
+  if [[ "${KUBERNETES_MASTER:-}" == "false" ]]; then
+    cp "${src_dir}/kube-proxy.tar" "${dst_dir}"
+  else
+    cp "${src_dir}/kube-apiserver.tar" "${dst_dir}"
+    cp "${src_dir}/kube-controller-manager.tar" "${dst_dir}"
+    cp "${src_dir}/kube-scheduler.tar" "${dst_dir}"
+    cp -r "${KUBE_HOME}/kubernetes/addons" "${dst_dir}"
+  fi
+  local -r kube_bin="${KUBE_HOME}/bin"
+  mv "${src_dir}/kubelet" "${kube_bin}"
+  mv "${src_dir}/kubectl" "${kube_bin}"
+
+  if [[ "${NETWORK_PROVIDER:-}" == "kubenet" ]] || \
+     [[ "${NETWORK_PROVIDER:-}" == "cni" ]]; then
+    #TODO(andyzheng0831): We should make the cni version number as a k8s env variable.
+    local -r cni_tar="cni-07a8a28637e97b22eb8dfe710eeae1344f69d16e.tar.gz"
+    local -r cni_sha1="19d49f7b2b99cd2493d5ae0ace896c64e289ccbb"
+    download-or-bust "${cni_sha1}" "https://storage.googleapis.com/kubernetes-release/network-plugins/${cni_tar}"
+    local -r cni_dir="${KUBE_HOME}/cni"
+    mkdir -p "${cni_dir}"
+    tar xzf "${KUBE_HOME}/${cni_tar}" -C "${cni_dir}" --overwrite
+    mv "${cni_dir}/bin"/* "${kube_bin}"
+    rmdir "${cni_dir}/bin"
+    rm -f "${KUBE_HOME}/${cni_tar}"
+  fi
+
+  mv "${KUBE_HOME}/kubernetes/LICENSES" "${KUBE_HOME}"
+  mv "${KUBE_HOME}/kubernetes/kubernetes-src.tar.gz" "${KUBE_HOME}"
+
+  # Put kube-system pods manifests in ${KUBE_HOME}/kube-manifests/.
+  dst_dir="${KUBE_HOME}/kube-manifests"
+  mkdir -p "${dst_dir}"
+  local -r manifests_tar_urls=( $(split-commas "${KUBE_MANIFESTS_TAR_URL}") )
+  local -r manifests_tar="${manifests_tar_urls[0]##*/}"
+  if [ -n "${KUBE_MANIFESTS_TAR_HASH:-}" ]; then
+    local -r manifests_tar_hash="${KUBE_MANIFESTS_TAR_HASH}"
+  else
+    echo "Downloading k8s manifests sha1 (not found in env)"
+    download-or-bust "" "${manifests_tar_urls[@]/.tar.gz/.tar.gz.sha1}"
+    local -r manifests_tar_hash=$(cat "${manifests_tar}.sha1")
+  fi
+  echo "Downloading k8s manifests tar"
+  download-or-bust "${manifests_tar_hash}" "${manifests_tar_urls[@]}"
+  tar xzf "${KUBE_HOME}/${manifests_tar}" -C "${dst_dir}" --overwrite
+  local -r kube_addon_registry="${KUBE_ADDON_REGISTRY:-gcr.io/google_containers}"
+  if [[ "${kube_addon_registry}" != "gcr.io/google_containers" ]]; then
+    find "${dst_dir}" -name \*.yaml -or -name \*.yaml.in | \
+      xargs sed -ri "s@(image:\s.*)gcr.io/google_containers@\1${kube_addon_registry}@"
+    find "${dst_dir}" -name \*.manifest -or -name \*.json | \
+      xargs sed -ri "s@(image\":\s+\")gcr.io/google_containers@\1${kube_addon_registry}@"
+  fi
+  cp "${dst_dir}/kubernetes/gci-trusty/coreos-configure-helper.sh" "${KUBE_HOME}/bin/configure-helper.sh"
+  chmod -R 755 "${kube_bin}"
+
+  # Clean up.
+  rm -rf "${KUBE_HOME}/kubernetes"
+  rm -f "${KUBE_HOME}/${server_binary_tar}"
+  rm -f "${KUBE_HOME}/${server_binary_tar}.sha1"
+  rm -f "${KUBE_HOME}/${manifests_tar}"
+  rm -f "${KUBE_HOME}/${manifests_tar}.sha1"
+}
+
+######### Main Function ##########
+echo "Start to install kubernetes files"
+KUBE_HOME="/opt/kubernetes"
+mkdir -p "${KUBE_HOME}"
+download-kube-env
+source "${KUBE_HOME}/kube-env"
+install-kube-binary-config
+echo "Done for installing kubernetes files"
+
+# On CoreOS, the hosts is in /usr/share/baselayout/hosts
+# So we need to manually populdate the hosts file here on gce.
+echo "127.0.0.1 localhost" >> /etc/hosts
+echo "::1 localhost" >> /etc/hosts
+
+echo "Configuring hostname"
+hostnamectl set-hostname $(hostname | cut -f1 -d.)

cluster/gce/coreos/health-monitor.sh

+#!/bin/bash
+
+# Copyright 2016 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script is for master and node instance health monitoring, which is
+# packed in kube-manifest tarball. It is executed through a systemd service
+# in cluster/gce/gci/<master/node>.yaml. The env variables come from an env
+# file provided by the systemd service.
+
+set -o nounset
+set -o pipefail
+
+# We simply kill the process when there is a failure. Another systemd service will
+# automatically restart the process.
+function docker_monitoring {
+  while [ 1 ]; do
+    if ! timeout 10 docker ps > /dev/null; then
+      echo "Docker daemon failed!"
+      pkill docker
+      # Wait for a while, as we don't want to kill it again before it is really up.
+      sleep 30
+    else
+      sleep "${SLEEP_SECONDS}"
+    fi
+  done
+}
+
+function kubelet_monitoring {
+  echo "Wait for 2 minutes for kubelet to be fuctional"
+  # TODO(andyzheng0831): replace it with a more reliable method if possible.
+  sleep 120
+  local -r max_seconds=10
+  while [ 1 ]; do
+    if ! curl --insecure -m "${max_seconds}" -f -s https://127.0.0.1:${KUBELET_PORT:-10250}/healthz > /dev/null; then
+      echo "Kubelet is unhealthy!"
+      curl --insecure https://127.0.0.1:${KUBELET_PORT:-10250}/healthz
+      pkill kubelet
+      # Wait for a while, as we don't want to kill it again before it is really up.
+      sleep 60
+    else
+      sleep "${SLEEP_SECONDS}"
+    fi
+  done
+}
+
+
+############## Main Function ################
+if [[ "$#" -ne 1 ]]; then
+  echo "Usage: health-monitor.sh <docker/kubelet>"
+  exit 1
+fi
+
+KUBE_ENV="/home/kubernetes/kube-env"
+if [[ ! -e "${KUBE_ENV}" ]]; then
+  echo "The ${KUBE_ENV} file does not exist!! Terminate health monitoring"
+  exit 1
+fi
+
+SLEEP_SECONDS=10
+component=$1
+echo "Start kubernetes health monitoring for ${component}"
+source "${KUBE_ENV}"
+if [[ "${component}" == "docker" ]]; then
+  docker_monitoring 
+elif [[ "${component}" == "kubelet" ]]; then
+  kubelet_monitoring
+else
+  echo "Health monitoring for component "${component}" is not supported!"
+fi

cluster/gce/coreos/configure-kubelet.sh → cluster/gce/coreos/helper.sh

 #!/bin/bash
 
-# Copyright 2015 The Kubernetes Authors.
+# Copyright 2016 The Kubernetes Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,20 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-set -o errexit
-set -o nounset
-set -o pipefail
+# A library of helper functions and constants for the CoreOS distro
 
-MANIFESTS_DIR=/opt/kube-manifests/kubernetes
-
-echo "Configuring hostname"
-hostnamectl set-hostname $(hostname | cut -f1 -d.)
-
-echo "Configuring kubelet"
-mkdir -p /var/lib/kubelet
-mkdir -p /etc/kubernetes/manifests
-src=${MANIFESTS_DIR}/kubelet-config.yaml
-dst=/var/lib/kubelet/kubeconfig
-cp ${src} ${dst}
-sed -i 's/\"/\\\"/g' ${dst} # eval will remove the double quotes if they are not escaped
-eval "echo \"$(< ${dst})\"" > ${dst}
+# This file intentionally left blank

cluster/gce/coreos/kube-manifests/addons/cluster-loadbalancing/glbc/default-svc.yaml

-apiVersion: v1
-kind: Service
-metadata:
-  # This must match the --default-backend-service argument of the l7 lb
-  # controller and is required because GCE mandates a default backend.
-  name: default-http-backend
-  namespace: kube-system
-  labels:
-    k8s-app: glbc
-    kubernetes.io/cluster-service: "true"
-    kubernetes.io/name: "GLBCDefaultBackend"
-spec:
-  # The default backend must be of type NodePort.
-  type: NodePort
-  ports:
-  - port: 80
-    targetPort: 8080
-    protocol: TCP
-    name: http
-  selector:
-    k8s-app: glbc
\ No newline at end of file

cluster/gce/coreos/kube-manifests/addons/cluster-loadbalancing/glbc/glbc-controller.yaml

-apiVersion: v1
-kind: ReplicationController
-metadata:
-  name: l7-lb-controller-v0.6.0
-  namespace: kube-system
-  labels:
-    k8s-app: glbc
-    version: v0.6.0
-    kubernetes.io/cluster-service: "true"
-    kubernetes.io/name: "GLBC"
-spec:
-  # There should never be more than 1 controller alive simultaneously.
-  replicas: 1
-  selector:
-    k8s-app: glbc
-    version: v0.6.0
-  template:
-    metadata:
-      labels:
-        k8s-app: glbc
-        version: v0.6.0
-        name: glbc
-        kubernetes.io/cluster-service: "true"
-    spec:
-      terminationGracePeriodSeconds: 600
-      containers:
-      - name: default-http-backend
-        # Any image is permissible as long as:
-        # 1. It serves a 404 page at /
-        # 2. It serves 200 on a /healthz endpoint
-        image: gcr.io/google_containers/defaultbackend:1.0
-        livenessProbe:
-          httpGet:
-            path: /healthz
-            port: 8080
-            scheme: HTTP
-          initialDelaySeconds: 30
-          timeoutSeconds: 5
-        ports:
-        - containerPort: 8080
-        resources:
-          limits:
-            cpu: 10m
-            memory: 20Mi
-          requests:
-            cpu: 10m
-            memory: 20Mi
-      - image: gcr.io/google_containers/glbc:0.6.0
-        livenessProbe:
-          httpGet:
-            path: /healthz
-            port: 8081
-            scheme: HTTP
-          initialDelaySeconds: 30
-          # healthz reaches out to GCE
-          periodSeconds: 30
-          timeoutSeconds: 5
-        name: l7-lb-controller
-        resources:
-          limits:
-            cpu: 100m
-            memory: 100Mi
-          requests:
-            cpu: 100m
-            memory: 50Mi
-        args:
-        - --default-backend-service=kube-system/default-http-backend
-        - --sync-period=60s
-        - --cluster-uid=${KUBE_UID}

cluster/gce/coreos/kube-manifests/addons/cluster-monitoring/google/heapster-controller.yaml

-apiVersion: extensions/v1beta1
-kind: Deployment
-metadata:
-  name: heapster-v1.2.0
-  namespace: kube-system
-  labels:
-    k8s-app: heapster
-    kubernetes.io/cluster-service: "true"
-    version: v1.2.0
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      k8s-app: heapster
-      version: v1.2.0
-  template:
-    metadata:
-      labels:
-        k8s-app: heapster
-        version: v1.2.0
-    spec:
-      containers:
-        - image: gcr.io/google_containers/heapster:v1.2.0
-          name: heapster
-          resources:
-            # keep request = limit to keep this container in guaranteed class
-            limits:
-              cpu: 100m
-              memory: 300Mi
-            requests:
-              cpu: 100m
-              memory: 300Mi
-          command:
-            - /heapster
-            - --source=kubernetes.summary_api:''
-            - --sink=gcm
-          volumeMounts:
-            - name: ssl-certs
-              mountPath: /etc/ssl/certs
-              readOnly: true
-            - name: usrsharecacerts
-              mountPath: /usr/share/ca-certificates
-              readOnly: true
-        - image: gcr.io/google_containers/heapster:v1.2.0
-          name: eventer
-          resources:
-            # keep request = limit to keep this container in guaranteed class
-            limits:
-              cpu: 100m
-              memory: 300Mi
-            requests:
-              cpu: 100m
-              memory: 300Mi
-          command:
-            - /eventer
-            - --source=kubernetes:''
-            - --sink=gcl
-          volumeMounts:
-            - name: ssl-certs
-              mountPath: /etc/ssl/certs
-              readOnly: true
-            - name: usrsharecacerts
-              mountPath: /usr/share/ca-certificates
-              readOnly: true
-        - image: gcr.io/google_containers/addon-resizer:1.0
-          name: heapster-nanny
-          resources:
-            limits:
-              cpu: 50m
-              memory: 100Mi
-            requests:
-              cpu: 50m
-              memory: 100Mi
-          env:
-            - name: MY_POD_NAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.name
-            - name: MY_POD_NAMESPACE
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.namespace
-          command:
-            - /pod_nanny
-            - --cpu=100m
-            - --extra-cpu=0m
-            - --memory=300Mi
-            - --extra-memory=4Mi
-            - --threshold=5
-            - --deployment=heapster-v1.2.0
-            - --container=heapster
-            - --poll-period=300000
-        - image: gcr.io/google_containers/addon-resizer:1.0
-          name: eventer-nanny
-          resources:
-            limits:
-              cpu: 50m
-              memory: 100Mi
-            requests:
-              cpu: 50m
-              memory: 100Mi
-          env:
-            - name: MY_POD_NAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.name
-            - name: MY_POD_NAMESPACE
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.namespace
-          command:
-            - /pod_nanny
-            - --cpu=100m
-            - --extra-cpu=0m
-            - --memory=300Mi
-            - --extra-memory=307200Ki
-            - --threshold=5
-            - --deployment=heapster-v1.2.0
-            - --container=eventer
-            - --poll-period=300000
-      volumes:
-        - name: ssl-certs
-          hostPath:
-            path: "/etc/ssl/certs"
-        - name: usrsharecacerts
-          hostPath:
-            path: "/usr/share/ca-certificates"

cluster/gce/coreos/kube-manifests/addons/cluster-monitoring/google/heapster-service.yaml

-kind: Service
-apiVersion: v1
-metadata: 
-  name: heapster
-  namespace: kube-system
-  labels: 
-    kubernetes.io/cluster-service: "true"
-    kubernetes.io/name: "Heapster"
-spec: 
-  ports: 
-    - port: 80
-      targetPort: 8082
-  selector: 
-    k8s-app: heapster

cluster/gce/coreos/kube-manifests/addons/cluster-monitoring/googleinfluxdb/heapster-controller-combined.yaml

-apiVersion: extensions/v1beta1
-kind: Deployment
-metadata:
-  name: heapster-v1.1.0
-  namespace: kube-system
-  labels:
-    k8s-app: heapster
-    kubernetes.io/cluster-service: "true"
-    version: v1.1.0
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      k8s-app: heapster
-      version: v1.1.0
-  template:
-    metadata:
-      labels:
-        k8s-app: heapster
-        version: v1.1.0
-    spec:
-      containers:
-        - image: gcr.io/google_containers/heapster:v1.2.0
-          name: heapster
-          resources:
-            # keep request = limit to keep this container in guaranteed class
-            limits:
-              cpu: 100m
-              memory: 300Mi
-            requests:
-              cpu: 100m
-              memory: 300Mi
-          command:
-            - /heapster
-            - --source=kubernetes.summary_api:''
-            - --sink=influxdb:http://monitoring-influxdb:8086
-            - --sink=gcm:?metrics=autoscaling
-          volumeMounts:
-            - name: ssl-certs
-              mountPath: /etc/ssl/certs
-              readOnly: true
-            - name: usrsharecacerts
-              mountPath: /usr/share/ca-certificates
-              readOnly: true
-        - image: gcr.io/google_containers/heapster:v1.2.0
-          name: eventer
-          resources:
-            # keep request = limit to keep this container in guaranteed class
-            limits:
-              cpu: 100m
-              memory: 300Mi
-            requests:
-              cpu: 100m
-              memory: 300Mi
-          command:
-            - /eventer
-            - --source=kubernetes:''
-            - --sink=gcl
-          volumeMounts:
-            - name: ssl-certs
-              mountPath: /etc/ssl/certs
-              readOnly: true
-            - name: usrsharecacerts
-              mountPath: /usr/share/ca-certificates
-              readOnly: true
-        - image: gcr.io/google_containers/addon-resizer:1.0
-          name: heapster-nanny
-          resources:
-            limits:
-              cpu: 50m
-              memory: 100Mi
-            requests:
-              cpu: 50m
-              memory: 100Mi
-          env:
-            - name: MY_POD_NAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.name
-            - name: MY_POD_NAMESPACE
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.namespace
-          command:
-            - /pod_nanny
-            - --cpu=100m
-            - --extra-cpu=0m
-            - --memory=300Mi
-            - --extra-memory=4Mi
-            - --threshold=5
-            - --deployment=heapster-v1.1.0
-            - --container=heapster
-            - --poll-period=300000
-        - image: gcr.io/google_containers/addon-resizer:1.0
-          name: eventer-nanny
-          resources:
-            limits:
-              cpu: 50m
-              memory: 100Mi
-            requests:
-              cpu: 50m
-              memory: 100Mi
-          env:
-            - name: MY_POD_NAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.name
-            - name: MY_POD_NAMESPACE
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.namespace
-          command:
-            - /pod_nanny
-            - --cpu=100m
-            - --extra-cpu=0m
-            - --memory=300Mi
-            - --extra-memory=307200Ki
-            - --threshold=5
-            - --deployment=heapster-v1.1.0
-            - --container=eventer
-            - --poll-period=300000
-      volumes:
-        - name: ssl-certs
-          hostPath:
-            path: "/etc/ssl/certs"
-        - name: usrsharecacerts
-          hostPath:
-            path: "/usr/share/ca-certificates"

cluster/gce/coreos/kube-manifests/addons/cluster-monitoring/influxdb/grafana-service.yaml

-apiVersion: v1
-kind: Service
-metadata:
-  name: monitoring-grafana
-  namespace: kube-system
-  labels: 
-    kubernetes.io/cluster-service: "true"
-    kubernetes.io/name: "Grafana"
-spec:
-  # On production clusters, consider setting up auth for grafana, and
-  # exposing Grafana either using a LoadBalancer or a public IP.
-  # type: LoadBalancer
-  ports: 
-    - port: 80
-      targetPort: 3000
-  selector: 
-    k8s-app: influxGrafana
-

cluster/gce/coreos/kube-manifests/addons/cluster-monitoring/influxdb/heapster-controller.yaml

-apiVersion: extensions/v1beta1
-kind: Deployment
-metadata:
-  name: heapster-v1.2.0
-  namespace: kube-system
-  labels:
-    k8s-app: heapster
-    kubernetes.io/cluster-service: "true"
-    version: v1.2.0
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      k8s-app: heapster
-      version: v1.2.0
-  template:
-    metadata:
-      labels:
-        k8s-app: heapster
-        version: v1.2.0
-    spec:
-      containers:
-        - image: gcr.io/google_containers/heapster:v1.2.0
-          name: heapster
-          resources:
-            # keep request = limit to keep this container in guaranteed class
-            limits:
-              cpu: 100m
-              memory: 300Mi
-            requests:
-              cpu: 100m
-              memory: 300Mi
-          command:
-            - /heapster
-            - --source=kubernetes.summary_api:''
-            - --sink=influxdb:http://monitoring-influxdb:8086
-        - image: gcr.io/google_containers/heapster:v1.2.0
-          name: eventer
-          resources:
-            # keep request = limit to keep this container in guaranteed class
-            limits:
-              cpu: 100m
-              memory: 300Mi
-            requests:
-              cpu: 100m
-              memory: 300Mi
-          command:
-            - /eventer
-            - --source=kubernetes:''
-            - --sink=influxdb:http://monitoring-influxdb:8086
-        - image: gcr.io/google_containers/addon-resizer:1.0
-          name: heapster-nanny
-          resources:
-            limits:
-              cpu: 50m
-              memory: 100Mi
-            requests:
-              cpu: 50m
-              memory: 100Mi
-          env:
-            - name: MY_POD_NAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.name
-            - name: MY_POD_NAMESPACE
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.namespace
-          command:
-            - /pod_nanny
-            - --cpu=100m
-            - --extra-cpu=0m
-            - --memory=300Mi
-            - --extra-memory=4Mi
-            - --threshold=5
-            - --deployment=heapster-v1.2.0
-            - --container=heapster
-            - --poll-period=300000
-        - image: gcr.io/google_containers/addon-resizer:1.0
-          name: eventer-nanny
-          resources:
-            limits:
-              cpu: 50m
-              memory: 100Mi
-            requests:
-              cpu: 50m
-              memory: 100Mi
-          env:
-            - name: MY_POD_NAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.name
-            - name: MY_POD_NAMESPACE
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.namespace
-          command:
-            - /pod_nanny
-            - --cpu=100m
-            - --extra-cpu=0m
-            - --memory=300Mi
-            - --extra-memory=307200Ki
-            - --threshold=5
-            - --deployment=heapster-v1.2.0
-            - --container=eventer
-            - --poll-period=300000

cluster/gce/coreos/kube-manifests/addons/cluster-monitoring/influxdb/heapster-service.yaml

-kind: Service
-apiVersion: v1
-metadata:
-  name: heapster
-  namespace: kube-system
-  labels:
-    kubernetes.io/cluster-service: "true"
-    kubernetes.io/name: "Heapster"
-spec: 
-  ports: 
-    - port: 80
-      targetPort: 8082
-  selector: 
-    k8s-app: heapster

cluster/gce/coreos/kube-manifests/addons/cluster-monitoring/influxdb/influxdb-grafana-controller.yaml

-apiVersion: v1
-kind: ReplicationController
-metadata:
-  name: monitoring-influxdb-grafana-v3
-  namespace: kube-system
-  labels: 
-    k8s-app: influxGrafana
-    version: v3
-    kubernetes.io/cluster-service: "true"
-spec: 
-  replicas: 1
-  selector: 
-    k8s-app: influxGrafana
-    version: v3
-  template: 
-    metadata: 
-      labels: 
-        k8s-app: influxGrafana
-        version: v3
-        kubernetes.io/cluster-service: "true"
-    spec: 
-      containers: 
-        - image: gcr.io/google_containers/heapster_influxdb:v0.7
-          name: influxdb
-          resources:
-            # keep request = limit to keep this container in guaranteed class
-            limits:
-              cpu: 100m
-              memory: 500Mi
-            requests:
-              cpu: 100m
-              memory: 500Mi
-          ports: 
-            - containerPort: 8083
-            - containerPort: 8086
-          volumeMounts:
-          - name: influxdb-persistent-storage
-            mountPath: /data
-        - image: gcr.io/google_containers/heapster_grafana:v3.1.1
-          name: grafana
-          env:
-          resources:
-            # keep request = limit to keep this container in guaranteed class
-            limits:
-              cpu: 100m
-              memory: 100Mi
-            requests:
-              cpu: 100m
-              memory: 100Mi
-          env:
-            # This variable is required to setup templates in Grafana.
-            - name: INFLUXDB_SERVICE_URL
-              value: http://monitoring-influxdb:8086
-              # The following env variables are required to make Grafana accessible via
-              # the kubernetes api-server proxy. On production clusters, we recommend
-              # removing these env variables, setup auth for grafana, and expose the grafana
-              # service using a LoadBalancer or a public IP.
-            - name: GF_AUTH_BASIC_ENABLED
-              value: "false"
-            - name: GF_AUTH_ANONYMOUS_ENABLED
-              value: "true"
-            - name: GF_AUTH_ANONYMOUS_ORG_ROLE
-              value: Admin
-            - name: GF_SERVER_ROOT_URL
-              value: /api/v1/proxy/namespaces/kube-system/services/monitoring-grafana/
-          volumeMounts:
-          - name: grafana-persistent-storage
-            mountPath: /var
-      volumes:
-      - name: influxdb-persistent-storage
-        emptyDir: {}
-      - name: grafana-persistent-storage
-        emptyDir: {}
-

cluster/gce/coreos/kube-manifests/addons/cluster-monitoring/influxdb/influxdb-service.yaml

-apiVersion: v1
-kind: Service
-metadata:
-  name: monitoring-influxdb
-  namespace: kube-system
-  labels: 
-    kubernetes.io/cluster-service: "true"
-    kubernetes.io/name: "InfluxDB"
-spec: 
-  ports: 
-    - name: http
-      port: 8083
-      targetPort: 8083
-    - name: api
-      port: 8086
-      targetPort: 8086
-  selector: 
-    k8s-app: influxGrafana
-

cluster/gce/coreos/kube-manifests/addons/cluster-monitoring/standalone/heapster-controller.yaml

-apiVersion: extensions/v1beta1
-kind: Deployment
-metadata:
-  name: heapster-v1.2.0
-  namespace: kube-system
-  labels:
-    k8s-app: heapster
-    kubernetes.io/cluster-service: "true"
-    version: v1.2.0
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      k8s-app: heapster
-      version: v1.2.0
-  template:
-    metadata:
-      labels:
-        k8s-app: heapster
-        version: v1.2.0
-    spec:
-      containers:
-        - image: gcr.io/google_containers/heapster:v1.2.0
-          name: heapster
-          resources:
-            # keep request = limit to keep this container in guaranteed class
-            limits:
-              cpu: 100m
-              memory: 300Mi
-            requests:
-              cpu: 100m
-              memory: 300Mi
-          command:
-            - /heapster
-            - --source=kubernetes.summary_api:''
-        - image: gcr.io/google_containers/addon-resizer:1.0
-          name: heapster-nanny
-          resources:
-            limits:
-              cpu: 50m
-              memory: 100Mi
-            requests:
-              cpu: 50m
-              memory: 100Mi
-          env:
-            - name: MY_POD_NAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.name
-            - name: MY_POD_NAMESPACE
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.namespace
-          command:
-            - /pod_nanny
-            - --cpu=100m
-            - --extra-cpu=0m
-            - --memory=300Mi
-            - --extra-memory=4Mi
-            - --threshold=5
-            - --deployment=heapster-v1.2.0
-            - --container=heapster
-            - --poll-period=300000

cluster/gce/coreos/kube-manifests/addons/cluster-monitoring/standalone/heapster-service.yaml

-kind: Service
-apiVersion: v1
-metadata: 
-  name: heapster
-  namespace: kube-system
-  labels: 
-    kubernetes.io/cluster-service: "true"
-    kubernetes.io/name: "Heapster"
-spec: 
-  ports: 
-    - port: 80
-      targetPort: 8082
-  selector: 
-    k8s-app: heapster

cluster/gce/coreos/kube-manifests/addons/dashboard/dashboard-controller.yaml

-# Keep this file in sync with addons/dashboard/dashboard-controller.yaml
-apiVersion: extensions/v1beta1
-kind: Deployment
-metadata:
-  name: kubernetes-dashboard
-  namespace: kube-system
-  labels:
-    k8s-app: kubernetes-dashboard
-    kubernetes.io/cluster-service: "true"
-spec:
-  selector:
-    matchLabels:
-      k8s-app: kubernetes-dashboard
-  template:
-    metadata:
-      labels:
-        k8s-app: kubernetes-dashboard
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
-        scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]'
-    spec:
-      containers:
-      - name: kubernetes-dashboard
-        image: gcr.io/google_containers/kubernetes-dashboard-amd64:v1.5.0
-        resources:
-          # keep request = limit to keep this container in guaranteed class
-          limits:
-            cpu: 100m
-            memory: 50Mi
-          requests:
-            cpu: 100m
-            memory: 50Mi
-        ports:
-        - containerPort: 9090
-        livenessProbe:
-          httpGet:
-            path: /
-            port: 9090
-          initialDelaySeconds: 30
-          timeoutSeconds: 30

cluster/gce/coreos/kube-manifests/addons/dashboard/dashboard-service.yaml

-apiVersion: v1
-kind: Service
-metadata:
-  name: kubernetes-dashboard
-  namespace: kube-system
-  labels:
-    k8s-app: kubernetes-dashboard
-    kubernetes.io/cluster-service: "true"
-spec:
-  selector:
-    k8s-app: kubernetes-dashboard
-  ports:
-  - port: 80
-    targetPort: 9090

cluster/gce/coreos/kube-manifests/addons/dns/skydns-rc.yaml

-apiVersion: extensions/v1beta1
-kind: Deployment
-metadata:
-  name: kube-dns
-  namespace: kube-system
-  labels:
-    k8s-app: kube-dns
-    kubernetes.io/cluster-service: "true"
-spec:
-  # replicas: not specified here:
-  # 1. In order to make Addon Manager do not reconcile this replicas parameter.
-  # 2. Default is 1.
-  # 3. Will be tuned in real time if DNS horizontal auto-scaling is turned on.
-  strategy:
-    rollingUpdate:
-      maxSurge: 10%
-      maxUnavailable: 0
-  selector:
-    matchLabels:
-      k8s-app: kube-dns
-  template:
-    metadata:
-      labels:
-        k8s-app: kube-dns
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
-        scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]'
-    spec:
-      containers:
-      - name: kubedns
-        image: gcr.io/google_containers/kubedns-amd64:1.9
-        resources:
-          # TODO: Set memory limits when we've profiled the container for large
-          # clusters, then set request = limit to keep this container in
-          # guaranteed class. Currently, this container falls into the
-          # "burstable" category so the kubelet doesn't backoff from restarting it.
-          limits:
-            memory: 200Mi
-          requests:
-            cpu: 100m
-            memory: 100Mi
-        livenessProbe:
-          httpGet:
-            path: /healthz-kubedns
-            port: 8080
-            scheme: HTTP
-          initialDelaySeconds: 60
-          timeoutSeconds: 5
-          successThreshold: 1
-          failureThreshold: 5
-        readinessProbe:
-          httpGet:
-            path: /readiness
-            port: 8081
-            scheme: HTTP
-          # we poll on pod startup for the Kubernetes master service and
-          # only setup the /readiness HTTP server once that's available.
-          initialDelaySeconds: 3
-          timeoutSeconds: 5
-        args:
-        # command = "/kube-dns"
-        - --domain=${DNS_DOMAIN}.
-        - --dns-port=10053
-        - --config-map=kube-dns
-        - --v=2
-        env:
-          - name: PROMETHEUS_PORT
-            value: "10055"
-        ports:
-        - containerPort: 10053
-          name: dns-local
-          protocol: UDP
-        - containerPort: 10053
-          name: dns-tcp-local
-          protocol: TCP
-        - containerPort: 10055
-          name: metrics
-          protocol: TCP
-      - name: dnsmasq
-        image: gcr.io/google_containers/kube-dnsmasq-amd64:1.4
-        livenessProbe:
-          httpGet:
-            path: /healthz-dnsmasq
-            port: 8080
-            scheme: HTTP
-          initialDelaySeconds: 60
-          timeoutSeconds: 5
-          successThreshold: 1
-          failureThreshold: 5
-        args:
-        - --cache-size=1000
-        - --no-resolv
-        - --server=127.0.0.1#10053
-        - --log-facility=-
-        ports:
-        - containerPort: 53
-          name: dns
-          protocol: UDP
-        - containerPort: 53
-          name: dns-tcp
-          protocol: TCP
-      - name: dnsmasq-metrics
-        image: gcr.io/google_containers/dnsmasq-metrics-amd64:1.0
-        livenessProbe:
-          httpGet:
-            path: /metrics
-            port: 10054
-            scheme: HTTP
-          initialDelaySeconds: 60
-          timeoutSeconds: 5
-          successThreshold: 1
-          failureThreshold: 5
-        args:
-        - --v=2
-        - --logtostderr
-        ports:
-        - containerPort: 10054
-          name: metrics
-          protocol: TCP
-        resources:
-          requests:
-            memory: 10Mi
-      - name: healthz
-        image: gcr.io/google_containers/exechealthz-amd64:1.2
-        resources:
-          limits:
-            memory: 50Mi
-          requests:
-            cpu: 10m
-            # Note that this container shouldn't really need 50Mi of memory. The
-            # limits are set higher than expected pending investigation on #29688.
-            # The extra memory was stolen from the kubedns container to keep the
-            # net memory requested by the pod constant.
-            memory: 50Mi
-        args:
-        - --cmd=nslookup kubernetes.default.svc.${DNS_DOMAIN} 127.0.0.1 >/dev/null
-        - --url=/healthz-dnsmasq
-        - --cmd=nslookup kubernetes.default.svc.${DNS_DOMAIN} 127.0.0.1:10053 >/dev/null
-        - --url=/healthz-kubedns
-        - --port=8080
-        - --quiet
-        ports:
-        - containerPort: 8080
-          protocol: TCP
-      dnsPolicy: Default  # Don't use cluster DNS.

cluster/gce/coreos/kube-manifests/addons/dns/skydns-svc.yaml

-apiVersion: v1
-kind: Service
-metadata:
-  name: kube-dns
-  namespace: kube-system
-  labels:
-    k8s-app: kube-dns
-    kubernetes.io/cluster-service: "true"
-    kubernetes.io/name: "KubeDNS"
-spec:
-  selector:
-    k8s-app: kube-dns
-  clusterIP:  ${DNS_SERVER_IP}
-  ports:
-  - name: dns
-    port: 53
-    protocol: UDP
-  - name: dns-tcp
-    port: 53
-    protocol: TCP

cluster/gce/coreos/kube-manifests/addons/fluentd-elasticsearch/es-controller.yaml

-apiVersion: v1
-kind: ReplicationController
-metadata:
-  name: elasticsearch-logging-v1
-  namespace: kube-system
-  labels:
-    k8s-app: elasticsearch-logging
-    version: v1
-    kubernetes.io/cluster-service: "true"
-spec:
-  replicas: 2
-  selector:
-    k8s-app: elasticsearch-logging
-    version: v1
-  template:
-    metadata:
-      labels:
-        k8s-app: elasticsearch-logging
-        version: v1
-        kubernetes.io/cluster-service: "true"
-    spec:
-      containers:
-      - image: gcr.io/google_containers/elasticsearch:v2.4.1
-        name: elasticsearch-logging
-        resources:
-          # need more cpu upon initialization, therefore burstable class
-          limits:
-            cpu: 1000m
-          requests:
-            cpu: 100m
-        ports:
-        - containerPort: 9200
-          name: db
-          protocol: TCP
-        - containerPort: 9300
-          name: transport
-          protocol: TCP
-        volumeMounts:
-        - name: es-persistent-storage
-          mountPath: /data
-      volumes:
-      - name: es-persistent-storage
-        emptyDir: {}

cluster/gce/coreos/kube-manifests/addons/fluentd-elasticsearch/es-service.yaml

-apiVersion: v1
-kind: Service
-metadata:
-  name: elasticsearch-logging
-  namespace: kube-system
-  labels:
-    k8s-app: elasticsearch-logging
-    kubernetes.io/cluster-service: "true"
-    kubernetes.io/name: "Elasticsearch"
-spec:
-  ports:
-  - port: 9200
-    protocol: TCP
-    targetPort: db
-  selector:
-    k8s-app: elasticsearch-logging

cluster/gce/coreos/kube-manifests/addons/fluentd-elasticsearch/kibana-controller.yaml

-apiVersion: v1
-kind: ReplicationController
-metadata:
-  name: kibana-logging-v1
-  namespace: kube-system
-  labels:
-    k8s-app: kibana-logging
-    version: v1
-    kubernetes.io/cluster-service: "true"
-spec:
-  replicas: 1
-  selector:
-    k8s-app: kibana-logging
-    version: v1
-  template:
-    metadata:
-      labels:
-        k8s-app: kibana-logging
-        version: v1
-        kubernetes.io/cluster-service: "true"
-    spec:
-      containers:
-      - name: kibana-logging
-        image: gcr.io/google_containers/kibana:v4.6.1
-        resources:
-          # keep request = limit to keep this container in guaranteed class
-          limits:
-            cpu: 100m
-          requests:
-            cpu: 100m
-        env:
-          - name: "ELASTICSEARCH_URL"
-            value: "http://elasticsearch-logging:9200"
-        ports:
-        - containerPort: 5601
-          name: ui
-          protocol: TCP
\ No newline at end of file

cluster/gce/coreos/kube-manifests/addons/fluentd-elasticsearch/kibana-service.yaml

-apiVersion: v1
-kind: Service
-metadata:
-  name: kibana-logging
-  namespace: kube-system
-  labels:
-    k8s-app: kibana-logging
-    kubernetes.io/cluster-service: "true"
-    kubernetes.io/name: "Kibana"
-spec:
-  ports:
-  - port: 5601
-    protocol: TCP
-    targetPort: ui
-  selector:
-    k8s-app: kibana-logging

cluster/gce/coreos/kube-manifests/addons/namespace.yaml

-apiVersion: v1
-kind: Namespace
-metadata:
-  name: kube-system

cluster/gce/coreos/kube-manifests/addons/node-problem-detector/node-problem-detector.yaml

-apiVersion: extensions/v1beta1
-kind: DaemonSet
-metadata:
-  name: node-problem-detector-v0.1
-  namespace: kube-system
-  labels:
-    k8s-app: node-problem-detector
-    version: v0.1
-    kubernetes.io/cluster-service: "true"
-spec:
-  template:
-    metadata:
-      labels:
-        k8s-app: node-problem-detector
-        version: v0.1
-        kubernetes.io/cluster-service: "true"
-    spec:
-      hostNetwork: true
-      containers:
-      - name: node-problem-detector
-        image: gcr.io/google_containers/node-problem-detector:v0.1
-        env:
-        # Config the host ip and port of apiserver.
-        - name: "KUBERNETES_SERVICE_HOST"
-          value: ${INSTANCE_PREFIX}-master
-        - name: "KUBERNETES_SERVICE_PORT"
-          value: "443"
-        securityContext:
-          privileged: true
-        resources:
-          limits:
-            cpu: "200m"
-            memory: "100Mi"
-          requests:
-            cpu: "20m"
-            memory: "20Mi"
-        volumeMounts:
-        - name: log
-          mountPath: /log
-          readOnly: true
-      volumes:
-      - name: log
-        hostPath:
-          path: /var/log/

cluster/gce/coreos/kube-manifests/addons/registry/registry-pv.yaml

-kind: PersistentVolume
-apiVersion: v1
-metadata:
-  name: kube-system-kube-registry-pv
-  labels:
-    kubernetes.io/cluster-service: "true"
-spec:
-  capacity:
-    storage: ${CLUSTER_REGISTRY_DISK_SIZE}
-  accessModes:
-    - ReadWriteOnce
-  gcePersistentDisk:
-    pdName: ${CLUSTER_REGISTRY_DISK}
-    fsType: "ext4"

cluster/gce/coreos/kube-manifests/addons/registry/registry-pvc.yaml

-kind: PersistentVolumeClaim
-apiVersion: v1
-metadata:
-  name: kube-registry-pvc
-  namespace: kube-system
-  labels:
-    kubernetes.io/cluster-service: "true"
-spec:
-  accessModes:
-    - ReadWriteOnce
-  resources:
-    requests:
-      storage: ${CLUSTER_REGISTRY_DISK_SIZE}

cluster/gce/coreos/kube-manifests/addons/registry/registry-rc.yaml

-apiVersion: v1
-kind: ReplicationController
-metadata:
-  name: kube-registry-v0
-  namespace: kube-system
-  labels:
-    k8s-app: kube-registry
-    version: v0
-    kubernetes.io/cluster-service: "true"
-spec:
-  replicas: 1
-  selector:
-    k8s-app: kube-registry
-    version: v0
-  template:
-    metadata:
-      labels:
-        k8s-app: kube-registry
-        version: v0
-        kubernetes.io/cluster-service: "true"
-    spec:
-      containers:
-      - name: registry
-        image: registry:2
-        resources:
-          limits:
-            cpu: 100m
-            memory: 100Mi
-        env:
-        - name: REGISTRY_HTTP_ADDR
-          value: :5000
-        - name: REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY
-          value: /var/lib/registry
-        volumeMounts:
-        - name: image-store
-          mountPath: /var/lib/registry
-        ports:
-        - containerPort: 5000
-          name: registry
-          protocol: TCP
-      volumes:
-      - name: image-store
-        persistentVolumeClaim:
-          claimName: kube-registry-pvc

cluster/gce/coreos/kube-manifests/addons/registry/registry-svc.yaml

-apiVersion: v1
-kind: Service
-metadata:
-  name: kube-registry
-  namespace: kube-system
-  labels:
-    k8s-app: kube-registry
-    kubernetes.io/cluster-service: "true"
-    kubernetes.io/name: "KubeRegistry"
-spec:
-  selector:
-    k8s-app: kube-registry
-  ports:
-  - name: registry
-    port: 5000
-    protocol: TCP

cluster/gce/coreos/kube-manifests/etcd-events.yaml

-apiVersion: v1
-kind: Pod
-metadata:
-  name: etcd-server-events
-  namespace: kube-system
-spec:
-  containers:
-  - command:
-    - /bin/sh
-    - -c
-    - /usr/local/bin/etcd
-      --listen-peer-urls=http://127.0.0.1:2381
-      --advertise-client-urls=http://127.0.0.1:4002
-      --listen-client-urls=http://127.0.0.1:4002
-      --data-dir=/var/etcd/data-events
-      1>>/var/log/etcd-events.log 2>&1
-    image: gcr.io/google_containers/etcd:2.2.1
-    imagePullPolicy: IfNotPresent
-    livenessProbe:
-      httpGet:
-        host: 127.0.0.1
-        path: /health
-        port: 4002
-        scheme: HTTP
-      initialDelaySeconds: 15
-      timeoutSeconds: 15
-    name: etcd-container
-    ports:
-    - containerPort: 2381
-      hostPort: 2381
-      name: serverport
-      protocol: TCP
-    - containerPort: 4002
-      hostPort: 4002
-      name: clientport
-      protocol: TCP
-    resources:
-      limits:
-        cpu: 100m
-      requests:
-        cpu: 100m
-    volumeMounts:
-    - mountPath: /var/etcd
-      name: varetcd
-    - mountPath: /var/log/etcd-events.log
-      name: varlogetcd
-  dnsPolicy: ClusterFirst
-  hostNetwork: true
-  nodeName: ${INSTANCE_PREFIX}-master
-  restartPolicy: Always
-  terminationGracePeriodSeconds: 30
-  volumes:
-  - hostPath:
-      path: /mnt/master-pd/var/etcd
-    name: varetcd
-  - hostPath:
-      path: /var/log/etcd-events.log
-    name: varlogetcd

cluster/gce/coreos/kube-manifests/etcd.yaml

-apiVersion: v1
-kind: Pod
-metadata:
-  name: etcd-server
-  namespace: kube-system
-spec:
-  containers:
-  - command:
-    - /bin/sh
-    - -c
-    - /usr/local/bin/etcd
-      --listen-peer-urls=http://127.0.0.1:2380
-      --advertise-client-urls=http://127.0.0.1:2379
-      --listen-client-urls=http://127.0.0.1:2379
-      --data-dir=/var/etcd/data
-      1>>/var/log/etcd.log 2>&1
-    image: gcr.io/google_containers/etcd:2.2.1
-    imagePullPolicy: IfNotPresent
-    livenessProbe:
-      httpGet:
-        host: 127.0.0.1
-        path: /health
-        port: 2379
-        scheme: HTTP
-      initialDelaySeconds: 15
-      timeoutSeconds: 15
-    name: etcd-container
-    ports:
-    - containerPort: 2380
-      hostPort: 2380
-      name: serverport
-      protocol: TCP
-    - containerPort: 2379
-      hostPort: 2379
-      name: clientport
-      protocol: TCP
-    resources:
-      requests:
-        cpu: 200m
-    volumeMounts:
-    - mountPath: /var/etcd
-      name: varetcd
-    - mountPath: /var/log/etcd.log
-      name: varlogetcd
-  dnsPolicy: ClusterFirst
-  hostNetwork: true
-  restartPolicy: Always
-  terminationGracePeriodSeconds: 30
-  volumes:
-  - hostPath:
-      path: /mnt/master-pd/var/etcd
-    name: varetcd
-  - hostPath:
-      path: /var/log/etcd.log
-    name: varlogetcd

cluster/gce/coreos/kube-manifests/kube-addon-manager.yaml

-apiVersion: v1
-kind: Pod
-metadata:
-  name: kube-addon-manager
-  namespace: kube-system
-  labels:
-    component: kube-addon-manager
-spec:
-  hostNetwork: true
-  containers:
-  - name: kube-addon-manager
-    image: gcr.io/google-containers/kube-addon-manager:v6.1
-    command:
-    - /bin/bash
-    - -c
-    - /opt/kube-addons.sh 1>>/var/log/kube-addon-manager.log 2>&1
-    resources:
-      requests:
-        cpu: 5m
-        memory: 50Mi
-    volumeMounts:
-    - mountPath: /etc/kubernetes/
-      name: addons
-      readOnly: true
-    - mountPath: /var/log
-      name: varlog
-      readOnly: false
-  volumes:
-  - hostPath:
-      path: /etc/kubernetes/
-    name: addons
-  - hostPath:
-      path: /var/log
-    name: varlog

cluster/gce/coreos/kube-manifests/kube-apiserver.yaml

-apiVersion: v1
-kind: Pod
-metadata:
-  name: kube-apiserver
-  namespace: kube-system
-spec:
-  containers:
-  - command:
-    - /bin/sh
-    - -c
-    - /usr/local/bin/kube-apiserver
-      --address=127.0.0.1
-      --etcd-servers=http://127.0.0.1:2379
-      --etcd-servers-overrides=/events#http://127.0.0.1:4002 
-      --cloud-provider=gce
-      --admission-control=${ADMISSION_CONTROL}
-      --service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE}
-      --client-ca-file=/srv/kubernetes/ca.crt
-      --basic-auth-file=/srv/kubernetes/basic_auth.csv
-      --tls-cert-file=/srv/kubernetes/server.cert
-      --tls-private-key-file=/srv/kubernetes/server.key
-      --secure-port=443
-      --token-auth-file=/srv/kubernetes/known_tokens.csv
-      --v=2
-      --allow-privileged=True
-      1>>/var/log/kube-apiserver.log 2>&1
-    image: gcr.io/google_containers/kube-apiserver:${KUBE_APISERVER_DOCKER_TAG}
-    imagePullPolicy: IfNotPresent
-    livenessProbe:
-      httpGet:
-        host: 127.0.0.1
-        path: /healthz
-        port: 8080
-        scheme: HTTP
-      initialDelaySeconds: 15
-      timeoutSeconds: 15
-    name: kube-apiserver
-    ports:
-    - containerPort: 443
-      hostPort: 443
-      name: https
-      protocol: TCP
-    - containerPort: 8080
-      hostPort: 8080
-      name: local
-      protocol: TCP
-    resources:
-      requests:
-        cpu: 250m
-    volumeMounts:
-    - mountPath: /srv/kubernetes
-      name: srvkube
-      readOnly: true
-    - mountPath: /var/log/kube-apiserver.log
-      name: logfile
-    - mountPath: /etc/ssl
-      name: etcssl
-      readOnly: true
-    - mountPath: /usr/share/ca-certificates
-      name: usrsharecacerts
-      readOnly: true
-    - mountPath: /srv/sshproxy
-      name: srvsshproxy
-  dnsPolicy: ClusterFirst
-  hostNetwork: true
-  restartPolicy: Always
-  terminationGracePeriodSeconds: 30
-  volumes:
-  - hostPath:
-      path: /srv/kubernetes
-    name: srvkube
-  - hostPath:
-      path: /var/log/kube-apiserver.log
-    name: logfile
-  - hostPath:
-      path: /etc/ssl
-    name: etcssl
-  - hostPath:
-      path: /usr/share/ca-certificates
-    name: usrsharecacerts
-  - hostPath:
-      path: /srv/sshproxy
-    name: srvsshproxy

cluster/gce/coreos/kube-manifests/kube-controller-manager.yaml

-apiVersion: v1
-kind: Pod
-metadata:
-  name: kube-controller-manager
-  namespace: kube-system
-spec:
-  containers:
-  - command:
-    - /bin/sh
-    - -c
-    - /usr/local/bin/kube-controller-manager
-      --master=127.0.0.1:8080
-      --cluster-name=${INSTANCE_PREFIX}
-      --cluster-cidr=${CLUSTER_IP_RANGE}
-      --service-cluster-ip-range="${SERVICE_CLUSTER_IP_RANGE}"
-      --allocate-node-cidrs=true
-      --cloud-provider=gce
-      --service-account-private-key-file=/srv/kubernetes/server.key
-      --v=2
-      --root-ca-file=/srv/kubernetes/ca.crt
-      1>>/var/log/kube-controller-manager.log 2>&1
-    image: gcr.io/google_containers/kube-controller-manager:${KUBE_CONTROLLER_MANAGER_DOCKER_TAG}
-    imagePullPolicy: IfNotPresent
-    livenessProbe:
-      httpGet:
-        host: 127.0.0.1
-        path: /healthz
-        port: 10252
-        scheme: HTTP
-      initialDelaySeconds: 15
-      timeoutSeconds: 15
-    name: kube-controller-manager
-    resources:
-      limits:
-        cpu: 200m
-      requests:
-        cpu: 200m
-    volumeMounts:
-    - mountPath: /srv/kubernetes
-      name: srvkube
-      readOnly: true
-    - mountPath: /var/log/kube-controller-manager.log
-      name: logfile
-    - mountPath: /etc/ssl
-      name: etcssl
-      readOnly: true
-    - mountPath: /usr/share/ca-certificates
-      name: usrsharecacerts
-      readOnly: true
-  dnsPolicy: ClusterFirst
-  hostNetwork: true
-  restartPolicy: Always
-  terminationGracePeriodSeconds: 30
-  volumes:
-  - hostPath:
-      path: /srv/kubernetes
-    name: srvkube
-  - hostPath:
-      path: /var/log/kube-controller-manager.log
-    name: logfile
-  - hostPath:
-      path: /etc/ssl
-    name: etcssl
-  - hostPath:
-      path: /usr/share/ca-certificates
-    name: usrsharecacerts

cluster/gce/coreos/kube-manifests/kube-scheduler.yaml

-apiVersion: v1
-kind: Pod
-metadata:
-  name: kube-scheduler
-  namespace: kube-system
-spec:
-  containers:
-  - command:
-    - /bin/sh
-    - -c
-    - /usr/local/bin/kube-scheduler
-      --master=127.0.0.1:8080
-      --v=2
-      1>>/var/log/kube-scheduler.log 2>&1
-    image: gcr.io/google_containers/kube-scheduler:${KUBE_SCHEDULER_DOCKER_TAG}
-    imagePullPolicy: IfNotPresent
-    livenessProbe:
-      httpGet:
-        host: 127.0.0.1
-        path: /healthz
-        port: 10251
-        scheme: HTTP
-      initialDelaySeconds: 15
-      timeoutSeconds: 15
-    name: kube-scheduler
-    resources:
-      requests:
-        cpu: 100m
-    volumeMounts:
-    - mountPath: /var/log/kube-scheduler.log
-      name: logfile
-  dnsPolicy: ClusterFirst
-  hostNetwork: true
-  nodeName: ${INSTANCE_PREFIX}-master
-  restartPolicy: Always
-  terminationGracePeriodSeconds: 30
-  volumes:
-  - hostPath:
-      path: /var/log/kube-scheduler.log
-    name: logfile

cluster/gce/coreos/kube-manifests/kube-system.json

-{
-  "apiVersion": "v1",
-  "kind": "Namespace",
-  "metadata": {
-    "name": "kube-system"
-  }
-}

cluster/gce/coreos/kube-manifests/kubelet-config.yaml

-apiVersion: v1
-kind: Config
-users:
-- name: kubelet
-  user:
-    client-certificate-data: ${KUBELET_CERT}
-    client-key-data: ${KUBELET_KEY}
-clusters:
-- name: local
-  cluster:
-    certificate-authority-data: ${CA_CERT}
-contexts:
-- context:
-    cluster: local
-    user: kubelet
-  name: service-account-context
-current-context: service-account-context

cluster/gce/coreos/kube-manifests/kubeproxy-config.yaml

-apiVersion: v1
-kind: Config
-users:
-- name: kube-proxy
-  user:
-    token: ${KUBE_PROXY_TOKEN}
-clusters:
-- name: local
-  cluster:
-    certificate-authority-data: ${CA_CERT}
-contexts:
-- context:
-    cluster: local
-    user: kube-proxy
-  name: service-account-context
-current-context: service-account-context

cluster/gce/coreos/master-docker.yaml

-#cloud-config
-
-coreos:
-  update:
-    reboot-strategy: off
-  units:
-    - name: locksmithd.service
-      mask: true
-    - name: kube-env.service
-      command: start
-      content: |
-        [Unit]
-        Description=Fetch kubernetes-node-environment
-        Requires=network-online.target
-        After=network-online.target
-        [Service]
-        Type=oneshot
-        RemainAfterExit=yes
-        ExecStartPre=/usr/bin/curl --fail --silent --show-error \
-        -H "X-Google-Metadata-Request: True" \
-        -o /etc/kube-env.yaml \
-        http://metadata.google.internal/computeMetadata/v1/instance/attributes/kube-env
-        # Transform the yaml to env file.
-        ExecStartPre=/usr/bin/mv /etc/kube-env.yaml /etc/kube-env
-        ExecStart=/usr/bin/sed -i "s/: '/=/;s/'$//" /etc/kube-env
-
-    - name: kubernetes-install-cni.service
-      command: start
-      content: |
-        [Unit]
-        Description=Download cni
-        Requires=network-online.target
-        After=network-online.target
-        [Service]
-        Type=oneshot
-        RemainAfterExit=yes
-        ExecStartPre=/usr/bin/mkdir -p /opt/cni
-        ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output /opt/downloads/cni.tar.gz https://storage.googleapis.com/kubernetes-release/network-plugins/cni-07a8a28637e97b22eb8dfe710eeae1344f69d16e.tar.gz
-        ExecStart=/usr/bin/tar xf /opt/downloads/cni.tar.gz -C /opt/cni/        
-        
-    - name: kubernetes-download-salt.service
-      command: start
-      content: |
-        [Unit]
-        Description=Download salt
-        Requires=network-online.target
-        After=network-online.target
-        Requires=kube-env.service
-        After=kube-env.service
-        [Service]
-        Type=oneshot
-        RemainAfterExit=yes
-        EnvironmentFile=/etc/kube-env
-        ExecStartPre=/usr/bin/mkdir -p /opt/downloads
-        ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output \
-        /opt/downloads/kubernetes-salt.tar.gz ${SALT_TAR_URL}
-        # TODO(yifan): Check hash.
-        ExecStart=/usr/bin/tar xf /opt/downloads/kubernetes-salt.tar.gz -C /opt --overwrite
-
-    - name: kubernetes-download-manifests.service
-      command: start
-      content: |
-        [Unit]
-        Description=Download manifests
-        Requires=network-online.target
-        After=network-online.target
-        Requires=kube-env.service
-        After=kube-env.service
-        [Service]
-        Type=oneshot
-        RemainAfterExit=yes
-        EnvironmentFile=/etc/kube-env
-        ExecStartPre=/usr/bin/mkdir -p /opt/downloads
-        ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output \
-        /opt/downloads/kubernetes-manifests.tar.gz ${KUBE_MANIFESTS_TAR_URL}
-        # TODO(yifan): Check hash.
-        ExecStartPre=/usr/bin/mkdir -p /opt/kube-manifests
-        ExecStart=/usr/bin/tar xf /opt/downloads/kubernetes-manifests.tar.gz -C /opt/kube-manifests --overwrite
-
-    - name: kubernetes-install-node.service
-      command: start
-      content: |
-        [Unit]
-        Description=Install Kubernetes Server
-        Requires=network-online.target
-        After=network-online.target
-        Requires=kube-env.service
-        After=kube-env.service
-        [Service]
-        Type=oneshot
-        RemainAfterExit=yes
-        EnvironmentFile=/etc/kube-env
-        ExecStartPre=/usr/bin/mkdir -p /opt/downloads
-        ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output \
-        /opt/downloads/kubernetes-server-linux-amd64.tar.gz ${SERVER_BINARY_TAR_URL}
-        # TODO(yifan): Check hash.
-        ExecStart=/usr/bin/tar xf /opt/downloads/kubernetes-server-linux-amd64.tar.gz -C /opt --overwrite
-
-    - name: kubelet.service
-      command: start
-      content: |
-        [Unit]
-        Description=Run Kubelet service
-        Requires=network-online.target kube-env.service kubernetes-download-manifests.service kubernetes-install-cni.service
-        After=network-online.target kube-env.service kubernetes-download-manifests.service kubernetes-install-cni.service
-        [Service]
-        EnvironmentFile=/etc/kube-env
-        ExecStartPre=/usr/bin/curl --fail --silent --show-error \
-        -H "X-Google-Metadata-Request: True" \
-        -o /run/configure-kubelet.sh \
-        http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-kubelet
-        ExecStartPre=/usr/bin/chmod 0755 /run/configure-kubelet.sh
-        ExecStartPre=/run/configure-kubelet.sh
-        ExecStart=/opt/kubernetes/server/bin/kubelet \
-        --api-servers=https://${INSTANCE_PREFIX}-master \
-        --enable-debugging-handlers=false \
-        --cloud-provider=gce \
-        --config=/etc/kubernetes/manifests \
-        --allow-privileged=true \
-        --v=2 \
-        --cluster-dns=${DNS_SERVER_IP} \
-        --cluster-domain=${DNS_DOMAIN} \
-        --logtostderr=true \
-        --container-runtime=docker \
-        --pod-cidr=${MASTER_IP_RANGE} \
-        --register-schedulable=false
-        Restart=always
-        RestartSec=10
-        KillMode=process
-
-    - name: docker.service
-      drop-ins:
-        - name: 50-docker-opts.conf
-          content: |
-            [Service]
-            Environment='DOCKER_OPTS=--bridge=cbr0 --iptables=false --ip-masq=false'
-            MountFlags=slave
-            LimitNOFILE=1048576
-            LimitNPROC=1048576
-            LimitCORE=infinity
-            Restart=always
-            RestartSec=2s
-            StartLimitInterval=0
-
-    - name: kubernetes-configure-node.service
-      command: start
-      content: |
-        [Unit]
-        Description=Configure Node For Kubernetes service
-        Requires=kubernetes-install-node.service
-        After=kubernetes-install-node.service
-        Requires=kubernetes-download-salt.service
-        After=kubernetes-download-salt.service
-        Requires=kubernetes-download-manifests.service
-        After=kubernetes-download-manifests.service
-        # Need the kubelet/docker running because we will use docker load for docker images.
-        Requires=kubelet.service
-        After=kubelet.service
-        [Service]
-        Type=oneshot
-        RemainAfterExit=yes
-        EnvironmentFile=/etc/kube-env
-        ExecStartPre=/usr/bin/curl --fail --silent --show-error \
-        -H "X-Google-Metadata-Request: True" \
-        -o /run/configure-node.sh \
-        http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-node
-        ExecStartPre=/usr/bin/chmod 0755 /run/configure-node.sh
-        ExecStart=/run/configure-node.sh

cluster/gce/coreos/master-helper.sh

 #!/bin/bash
 
-# Copyright 2015 The Kubernetes Authors.
+# Copyright 2016 The Kubernetes Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,6 +15,7 @@
 # limitations under the License.
 
 # A library of helper functions and constant for coreos os distro
+source "${KUBE_ROOT}/cluster/gce/coreos/helper.sh"
 
 # create-master-instance creates the master instance. If called with
 # an argument, the argument is used as the name to a reserved IP
@@ -24,23 +25,56 @@
 # It requires a whole slew of assumed variables, partially due to to
 # the call to write-master-env. Listing them would be rather
 # futile. Instead, we list the required calls to ensure any additional
+#
 # variables are set:
 #   ensure-temp-dir
 #   detect-project
 #   get-bearer-token
-#
-function create-master-instance() {
+function create-master-instance {
   local address_opt=""
   [[ -n ${1:-} ]] && address_opt="--address ${1}"
 
+  write-master-env
+  create-master-instance-internal "${MASTER_NAME}" "${address_opt}"
+}
+
+function replicate-master-instance() {
+  local existing_master_zone="${1}"
+  local existing_master_name="${2}"
+  local existing_master_replicas="${3}"
+
+  local kube_env="$(get-metadata "${existing_master_zone}" "${existing_master_name}" kube-env)"
+  # Substitute INITIAL_ETCD_CLUSTER to enable etcd clustering.
+  kube_env="$(echo "${kube_env}" | grep -v "INITIAL_ETCD_CLUSTER")"
+  kube_env="$(echo -e "${kube_env}\nINITIAL_ETCD_CLUSTER: '${existing_master_replicas},${REPLICA_NAME}'")"
+  ETCD_CA_KEY="$(echo "${kube_env}" | grep "ETCD_CA_KEY" |  sed "s/^.*: '//" | sed "s/'$//")"
+  ETCD_CA_CERT="$(echo "${kube_env}" | grep "ETCD_CA_CERT" |  sed "s/^.*: '//" | sed "s/'$//")"
+
+  create-etcd-certs "${REPLICA_NAME}" "${ETCD_CA_CERT}" "${ETCD_CA_KEY}"
+
+  kube_env="$(echo "${kube_env}" | grep -v "ETCD_PEER_KEY")"
+  kube_env="$(echo -e "${kube_env}\nETCD_PEER_KEY: '${ETCD_PEER_KEY_BASE64}'")"
+  kube_env="$(echo "${kube_env}" | grep -v "ETCD_PEER_CERT")"
+  kube_env="$(echo -e "${kube_env}\nETCD_PEER_CERT: '${ETCD_PEER_CERT_BASE64}'")"
+
+  echo "${kube_env}" > ${KUBE_TEMP}/master-kube-env.yaml
+  get-metadata "${existing_master_zone}" "${existing_master_name}" cluster-name > "${KUBE_TEMP}/cluster-name.txt"
+
+  create-master-instance-internal "${REPLICA_NAME}"
+}
+
+
+function create-master-instance-internal() {
+  local -r master_name="${1}"
+  local -r address_option="${2:-}"
+
   local preemptible_master=""
   if [[ "${PREEMPTIBLE_MASTER:-}" == "true" ]]; then
     preemptible_master="--preemptible --maintenance-policy TERMINATE"
   fi
 
-  write-master-env
-  gcloud compute instances create "${MASTER_NAME}" \
-    ${address_opt} \
+  gcloud compute instances create "${master_name}" \
+    ${address_option} \
     --project "${PROJECT}" \
     --zone "${ZONE}" \
     --machine-type "${MASTER_SIZE}" \
@@ -51,8 +85,18 @@ function create-master-instance() {
     --scopes "storage-ro,compute-rw,monitoring,logging-write" \
     --can-ip-forward \
     --metadata-from-file \
-      "kube-env=${KUBE_TEMP}/master-kube-env.yaml,user-data=${KUBE_ROOT}/cluster/gce/coreos/master-${CONTAINER_RUNTIME}.yaml,configure-node=${KUBE_ROOT}/cluster/gce/coreos/configure-node.sh,configure-kubelet=${KUBE_ROOT}/cluster/gce/coreos/configure-kubelet.sh,cluster-name=${KUBE_TEMP}/cluster-name.txt" \
-    --disk "name=${MASTER_NAME}-pd,device-name=master-pd,mode=rw,boot=no,auto-delete=no" \
-    --boot-disk-size "${MASTER_ROOT_DISK_SIZE:-10}" \
-      ${preemptible_master}
+      "kube-env=${KUBE_TEMP}/master-kube-env.yaml,user-data=${KUBE_ROOT}/cluster/gce/coreos/master.yaml,configure-sh=${KUBE_ROOT}/cluster/gce/coreos/configure.sh,cluster-name=${KUBE_TEMP}/cluster-name.txt" \
+    --disk "name=${master_name}-pd,device-name=master-pd,mode=rw,boot=no,auto-delete=no" \
+    --boot-disk-size "${MASTER_ROOT_DISK_SIZE:-30}" \
+    ${preemptible_master}
+}
+
+function get-metadata() {
+  local zone="${1}"
+  local name="${2}"
+  local key="${3}"
+  gcloud compute ssh "${name}" \
+    --project "${PROJECT}" \
+    --zone "${zone}" \
+    --command "curl \"http://metadata.google.internal/computeMetadata/v1/instance/attributes/${key}\" -H \"Metadata-Flavor: Google\"" 2>/dev/null
 }

cluster/gce/coreos/master-rkt.yaml

-#cloud-config
-
-coreos:
-  update:
-    reboot-strategy: off
-  units:
-    - name: locksmithd.service
-      mask: true
-    - name: kube-env.service
-      command: start
-      content: |
-        [Unit]
-        Description=Fetch kubernetes-node-environment
-        Requires=network-online.target
-        After=network-online.target
-        [Service]
-        Type=oneshot
-        RemainAfterExit=yes
-        ExecStartPre=/usr/bin/curl --fail --silent --show-error \
-        -H "X-Google-Metadata-Request: True" \
-        -o /etc/kube-env.yaml \
-        http://metadata.google.internal/computeMetadata/v1/instance/attributes/kube-env
-        # Transform the yaml to env file.
-        ExecStartPre=/usr/bin/mv /etc/kube-env.yaml /etc/kube-env
-        ExecStart=/usr/bin/sed -i "s/: '/=/;s/'$//" /etc/kube-env
-
-    - name: kubernetes-install-cni.service
-      command: start
-      content: |
-        [Unit]
-        Description=Download cni
-        Requires=network-online.target
-        After=network-online.target
-        [Service]
-        Type=oneshot
-        RemainAfterExit=yes
-        ExecStartPre=/usr/bin/mkdir -p /opt/cni
-        ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output /opt/downloads/cni.tar.gz https://storage.googleapis.com/kubernetes-release/network-plugins/cni-07a8a28637e97b22eb8dfe710eeae1344f69d16e.tar.gz
-        ExecStart=/usr/bin/tar xf /opt/downloads/cni.tar.gz -C /opt/cni/
-
-    - name: kubernetes-install-docker2aci.service
-      command: start
-      content: |
-        [Unit]
-        Description=Download docker2aci
-        Requires=network-online.target
-        After=network-online.target
-        [Service]
-        Type=oneshot
-        RemainAfterExit=yes
-        ExecStartPre=/usr/bin/mkdir -p /opt/docker2aci
-        ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output /opt/downloads/docker2aci.tar.gz https://github.com/appc/docker2aci/releases/download/v0.11.1/docker2aci-v0.11.1.tar.gz
-        ExecStart=/usr/bin/tar --strip-components=1 -xf /opt/downloads/docker2aci.tar.gz -C /opt/docker2aci/ --overwrite
-
-    - name: kubernetes-install-rkt.service
-      command: start
-      content: |
-        [Unit]
-        Description=Fetch rkt
-        Documentation=http://github.com/coreos/rkt
-        Requires=network-online.target
-        After=network-online.target
-        Requires=kube-env.service
-        After=kube-env.service
-        [Service]
-        Type=oneshot
-        RemainAfterExit=yes
-        EnvironmentFile=/etc/kube-env
-        ExecStartPre=/usr/bin/mkdir -p /etc/rkt /opt/downloads /opt/rkt/
-        ExecStartPre=/usr/bin/curl --fail --silent --location --create-dirs --output /opt/downloads/rkt.tar.gz https://github.com/coreos/rkt/releases/download/v${RKT_VERSION}/rkt-v${RKT_VERSION}.tar.gz
-        ExecStart=/usr/bin/tar --strip-components=1 -xf /opt/downloads/rkt.tar.gz -C /opt/rkt/ --overwrite
-
-    - name: kubernetes-load-rkt-stage1.service
-      command: start
-      content: |
-        [Unit]
-        Description=Load rkt stage1 images
-        Documentation=http://github.com/coreos/rkt
-        Requires=network-online.target
-        After=network-online.target
-        Requires=kubernetes-install-rkt.service
-        After=kubernetes-install-rkt.service
-        [Service]
-        Type=oneshot
-        ExecStart=/opt/rkt/rkt fetch /opt/rkt/stage1-coreos.aci /opt/rkt/stage1-kvm.aci /opt/rkt/stage1-fly.aci  --insecure-options=image
-
-    - name: kubernetes-download-salt.service
-      command: start
-      content: |
-        [Unit]
-        Description=Download salt
-        Requires=network-online.target
-        After=network-online.target
-        Requires=kube-env.service
-        After=kube-env.service
-        [Service]
-        Type=oneshot
-        RemainAfterExit=yes
-        EnvironmentFile=/etc/kube-env
-        ExecStartPre=/usr/bin/mkdir -p /opt/downloads
-        ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output \
-        /opt/downloads/kubernetes-salt.tar.gz ${SALT_TAR_URL}
-        # TODO(yifan): Check hash.
-        ExecStart=/usr/bin/tar xf /opt/downloads/kubernetes-salt.tar.gz -C /opt --overwrite
-
-    - name: kubernetes-download-manifests.service
-      command: start
-      content: |
-        [Unit]
-        Description=Download manifests
-        Requires=network-online.target
-        After=network-online.target
-        Requires=kube-env.service
-        After=kube-env.service
-        [Service]
-        Type=oneshot
-        RemainAfterExit=yes
-        EnvironmentFile=/etc/kube-env
-        ExecStartPre=/usr/bin/mkdir -p /opt/downloads
-        ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output \
-        /opt/downloads/kubernetes-manifests.tar.gz ${KUBE_MANIFESTS_TAR_URL}
-        # TODO(yifan): Check hash.
-        ExecStartPre=/usr/bin/mkdir -p /opt/kube-manifests
-        ExecStart=/usr/bin/tar xf /opt/downloads/kubernetes-manifests.tar.gz -C /opt/kube-manifests --overwrite
-
-    - name: kubernetes-install-node.service
-      command: start
-      content: |
-        [Unit]
-        Description=Install Kubernetes Server
-        Requires=network-online.target
-        After=network-online.target
-        Requires=kube-env.service
-        After=kube-env.service
-        [Service]
-        Type=oneshot
-        RemainAfterExit=yes
-        EnvironmentFile=/etc/kube-env
-        ExecStartPre=/usr/bin/mkdir -p /opt/downloads
-        ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output \
-        /opt/downloads/kubernetes-server-linux-amd64.tar.gz ${SERVER_BINARY_TAR_URL}
-        # TODO(yifan): Check hash.
-        ExecStart=/usr/bin/tar xf /opt/downloads/kubernetes-server-linux-amd64.tar.gz -C /opt --overwrite
-
-    - name: rkt-api-service.service
-      command: start
-      content: |
-        [Unit]
-        Description=Start rkt API service as Daemon
-        Requires=kubernetes-install-rkt.service
-        After=kubernetes-install-rkt.service
-        [Service]
-        ExecStart=/opt/rkt/rkt api-service
-        Restart=always
-        RestartSec=10
-
-    - name: kubelet.service
-      command: start
-      content: |
-        [Unit]
-        Description=Run Kubelet service
-        Requires=network-online.target kube-env.service kubernetes-download-manifests.service kubernetes-install-cni.service kubernetes-load-rkt-stage1.service
-        After=network-online.target kube-env.service kubernetes-download-manifests.service kubernetes-install-cni.service kubernetes-load-rkt-stage1.service
-        [Service]
-        EnvironmentFile=/etc/kube-env
-        ExecStartPre=/usr/bin/curl --fail --silent --show-error \
-        -H "X-Google-Metadata-Request: True" \
-        -o /run/configure-kubelet.sh \
-        http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-kubelet
-        ExecStartPre=/usr/bin/chmod 0755 /run/configure-kubelet.sh
-        ExecStartPre=/run/configure-kubelet.sh
-        ExecStart=/opt/kubernetes/server/bin/kubelet \
-        --api-servers=https://${INSTANCE_PREFIX}-master \
-        --enable-debugging-handlers=false \
-        --cloud-provider=gce \
-        --config=/etc/kubernetes/manifests \
-        --allow-privileged=true \
-        --v=2 \
-        --cluster-dns=${DNS_SERVER_IP} \
-        --cluster-domain=${DNS_DOMAIN} \
-        --logtostderr=true \
-        --container-runtime=rkt \
-        --rkt-path=/opt/rkt/rkt \
-        --rkt-stage1-image=${RKT_STAGE_IMAGE} \
-        --pod-cidr=${MASTER_IP_RANGE} \
-        --register-schedulable=false
-        Restart=always
-        RestartSec=10
-        KillMode=process
-
-    - name: docker.service
-      command: stop
-
-    - name: kubernetes-configure-node.service
-      command: start
-      content: |
-        [Unit]
-        Description=Configure Node For Kubernetes service
-        Requires=kubernetes-install-node.service
-        After=kubernetes-install-node.service
-        Requires=kubernetes-install-rkt.service
-        After=kubernetes-install-rkt.service
-        Requires=kubernetes-download-salt.service
-        After=kubernetes-download-salt.service
-        Requires=kubernetes-download-manifests.service
-        After=kubernetes-download-manifests.service
-        Requires=kubernetes-install-docker2aci.service
-        After=kubernetes-install-docker2aci.service
-        Requires=kubelet.service
-        After=kubelet.service
-        [Service]
-        Type=oneshot
-        RemainAfterExit=yes
-        EnvironmentFile=/etc/kube-env
-        ExecStartPre=/usr/bin/curl --fail --silent --show-error \
-        -H "X-Google-Metadata-Request: True" \
-        -o /run/configure-node.sh \
-        http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-node
-        ExecStartPre=/usr/bin/chmod 0755 /run/configure-node.sh
-        ExecStart=/run/configure-node.sh

cluster/gce/coreos/master.yaml

+#cloud-config
+
+coreos:
+  update:
+    reboot-strategy: off
+  units:
+    - name: locksmithd.service
+      mask: true
+    - name: kube-master-installation.service
+      command: start
+      content: |
+        [Unit]
+        Description=Download and install k8s binaries and configurations
+        After=network-online.target
+
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        ExecStartPre=/bin/mkdir -p /opt/kubernetes/bin
+        ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error	-H "X-Google-Metadata-Request: True" -o /opt/kubernetes/bin/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-sh
+        ExecStartPre=/bin/chmod 544 /opt/kubernetes/bin/configure.sh
+        ExecStart=/opt/kubernetes/bin/configure.sh
+
+        [Install]
+        WantedBy=kubernetes.target
+    - name: kube-master-configuration.service
+      command: start
+      content: |
+        [Unit]
+        Description=Configure kubernetes master
+        After=kube-master-installation.service
+
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        ExecStartPre=/bin/chmod 544 /opt/kubernetes/bin/configure-helper.sh
+        ExecStart=/opt/kubernetes/bin/configure-helper.sh
+
+        [Install]
+        WantedBy=kubernetes.target
+    - name: kubernetes.target
+      enable: true
+      command: start
+      content: |
+        [Unit]
+        Description=Kubernetes
+
+        [Install]
+        WantedBy=multi-user.target
+    - name: docker.service
+      drop-ins:
+        - name: "use-cgroupfs-driver.conf"
+          # This is required for setting cgroup parent in the current ~1.4 per-pod cgroup impl
+          content: |
+            [Service]
+            Environment="DOCKER_CGROUPS=--exec-opt native.cgroupdriver="

cluster/gce/coreos/node-docker.yaml

-#cloud-config
-
-coreos:
-  update:
-    reboot-strategy: off
-  units:
-    - name: locksmithd.service
-      mask: true
-    - name: kube-env.service
-      command: start
-      content: |
-        [Unit]
-        Description=Fetch kubernetes-node-environment
-        Requires=network-online.target
-        After=network-online.target
-        [Service]
-        Type=oneshot
-        RemainAfterExit=yes
-        ExecStartPre=/usr/bin/curl --fail --silent --show-error \
-        -H "X-Google-Metadata-Request: True" \
-        -o /etc/kube-env.yaml \
-        http://metadata.google.internal/computeMetadata/v1/instance/attributes/kube-env
-        # Transform the yaml to env file.
-        ExecStartPre=/usr/bin/mv /etc/kube-env.yaml /etc/kube-env
-        ExecStart=/usr/bin/sed -i "s/: '/=/;s/'$//" /etc/kube-env
-
-    - name: kubernetes-install-cni.service
-      command: start
-      content: |
-        [Unit]
-        Description=Download cni
-        Requires=network-online.target
-        After=network-online.target
-        [Service]
-        Type=oneshot
-        RemainAfterExit=yes
-        ExecStartPre=/usr/bin/mkdir -p /opt/cni
-        ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output /opt/downloads/cni.tar.gz https://storage.googleapis.com/kubernetes-release/network-plugins/cni-07a8a28637e97b22eb8dfe710eeae1344f69d16e.tar.gz
-        ExecStart=/usr/bin/tar xf /opt/downloads/cni.tar.gz -C /opt/cni/        
-
-    - name: kubernetes-download-manifests.service
-      command: start
-      content: |
-        [Unit]
-        Description=Download manifests
-        Requires=network-online.target
-        After=network-online.target
-        Requires=kube-env.service
-        After=kube-env.service
-        [Service]
-        Type=oneshot
-        RemainAfterExit=yes
-        EnvironmentFile=/etc/kube-env
-        ExecStartPre=/usr/bin/mkdir -p /opt/downloads
-        ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output \
-        /opt/downloads/kubernetes-manifests.tar.gz ${KUBE_MANIFESTS_TAR_URL}
-        # TODO(yifan): Check hash.
-        ExecStartPre=/usr/bin/mkdir -p /opt/kube-manifests
-        ExecStart=/usr/bin/tar xf /opt/downloads/kubernetes-manifests.tar.gz -C /opt/kube-manifests --overwrite
-
-    - name: kubernetes-install-node.service
-      command: start
-      content: |
-        [Unit]
-        Description=Install Kubernetes Server
-        Requires=network-online.target
-        After=network-online.target
-        Requires=kube-env.service
-        After=kube-env.service
-        [Service]
-        Type=oneshot
-        RemainAfterExit=yes
-        EnvironmentFile=/etc/kube-env
-        ExecStartPre=/usr/bin/mkdir -p /opt/kubernetes/pkg
-        ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output \
-        /opt/kubernetes/pkg/kubernetes-server-linux-amd64.tar.gz ${SERVER_BINARY_TAR_URL}
-        ExecStart=/usr/bin/tar xf /opt/kubernetes/pkg/kubernetes-server-linux-amd64.tar.gz -C /opt --overwrite
-
-    - name: kubelet.service
-      command: start
-      content: |
-        [Unit]
-        Description=Run Kubelet service
-        Requires=network-online.target kube-env.service kubernetes-download-manifests.service kubernetes-install-cni.service
-        After=network-online.target kube-env.service kubernetes-download-manifests.service kubernetes-install-cni.service
-        [Service]
-        EnvironmentFile=/etc/kube-env
-        ExecStartPre=/usr/bin/curl --fail --silent --show-error \
-        -H "X-Google-Metadata-Request: True" \
-        -o /run/configure-kubelet.sh \
-        http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-kubelet
-        ExecStartPre=/usr/bin/chmod 0755 /run/configure-kubelet.sh
-        ExecStartPre=/run/configure-kubelet.sh
-        ExecStart=/opt/kubernetes/server/bin/kubelet \
-        --api-servers=https://${INSTANCE_PREFIX}-master \
-        --enable-debugging-handlers=true \
-        --cloud-provider=gce \
-        --config=/etc/kubernetes/manifests \
-        --allow-privileged=true \
-        --v=2 \
-        --cluster-dns=${DNS_SERVER_IP} \
-        --cluster-domain=${DNS_DOMAIN} \
-        --logtostderr=true \
-        --container-runtime=docker \
-        --network-plugin=${NETWORK_PROVIDER}
-        Restart=always
-        RestartSec=10
-        KillMode=process
-
-    - name: kube-proxy.service
-      command: start
-      content: |
-        [Unit]
-        Description=Start Kube-proxy service as Daemon
-        Requires=kubernetes-configure-node.service
-        After=kubernetes-configure-node.service
-        [Service]
-        EnvironmentFile=/etc/kube-env
-        ExecStart=/opt/kubernetes/server/bin/kube-proxy \
-        --master=https://${KUBERNETES_MASTER_NAME} \
-        --kubeconfig=/var/lib/kube-proxy/kubeconfig \
-        --v=2 \
-        --logtostderr=true
-        Restart=always
-        RestartSec=10
-
-    - name: docker.service
-      drop-ins:
-        - name: 50-docker-opts.conf
-          content: |
-            [Service]
-            Environment='DOCKER_OPTS= --iptables=false --ip-masq=false'
-            MountFlags=slave
-            LimitNOFILE=1048576
-            LimitNPROC=1048576
-            LimitCORE=infinity
-            Restart=always
-            RestartSec=2s
-            StartLimitInterval=0
-
-    - name: kubernetes-configure-node.service
-      command: start
-      content: |
-        [Unit]
-        Description=Configure Node For Kubernetes service
-        Requires=kubernetes-install-node.service
-        After=kubernetes-install-node.service
-        Requires=kubernetes-download-manifests.service
-        After=kubernetes-download-manifests.service
-        [Service]
-        Type=oneshot
-        RemainAfterExit=yes
-        EnvironmentFile=/etc/kube-env
-        ExecStartPre=/usr/bin/curl --fail --silent --show-error \
-        -H "X-Google-Metadata-Request: True" \
-        -o /run/configure-node.sh \
-        http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-node
-        ExecStartPre=/usr/bin/chmod 0755 /run/configure-node.sh
-        ExecStart=/run/configure-node.sh

cluster/gce/coreos/node-helper.sh

 #!/bin/bash
 
-# Copyright 2015 The Kubernetes Authors.
+# Copyright 2016 The Kubernetes Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,15 +14,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# A library of helper functions and constant for coreos os distro
+# A library of helper functions and constant for the CoreOS distro
+source "${KUBE_ROOT}/cluster/gce/coreos/helper.sh"
 
-# $1: template name (required)
-function create-node-instance-template() {
+# $1: template name (required).
+function create-node-instance-template {
   local template_name="$1"
-  create-node-template "$template_name" "${scope_flags}" \
+
+  create-node-template "$template_name" "${scope_flags[*]}" \
     "kube-env=${KUBE_TEMP}/node-kube-env.yaml" \
-    "user-data=${KUBE_ROOT}/cluster/gce/coreos/node-${CONTAINER_RUNTIME}.yaml" \
-    "configure-node=${KUBE_ROOT}/cluster/gce/coreos/configure-node.sh" \
-    "configure-kubelet=${KUBE_ROOT}/cluster/gce/coreos/configure-kubelet.sh" \
+    "user-data=${KUBE_ROOT}/cluster/gce/coreos/node.yaml" \
+    "configure-sh=${KUBE_ROOT}/cluster/gce/coreos/configure.sh" \
     "cluster-name=${KUBE_TEMP}/cluster-name.txt"
+  # TODO(euank): We should include update-strategy here. We should also switch to ignition
 }

cluster/gce/coreos/node-rkt.yaml

-#cloud-config
-
-coreos:
-  update:
-    reboot-strategy: off
-  units:
-    - name: locksmithd.service
-      mask: true
-    - name: kube-env.service
-      command: start
-      content: |
-        [Unit]
-        Description=Fetch kubernetes-node-environment
-        Requires=network-online.target
-        After=network-online.target
-        [Service]
-        Type=oneshot
-        RemainAfterExit=yes
-        ExecStartPre=/usr/bin/curl --fail --silent --show-error \
-        -H "X-Google-Metadata-Request: True" \
-        -o /etc/kube-env.yaml \
-        http://metadata.google.internal/computeMetadata/v1/instance/attributes/kube-env
-        # Transform the yaml to env file.
-        ExecStartPre=/usr/bin/mv /etc/kube-env.yaml /etc/kube-env
-        ExecStart=/usr/bin/sed -i "s/: '/=/;s/'$//" /etc/kube-env
-
-    - name: kubernetes-install-cni.service
-      command: start
-      content: |
-        [Unit]
-        Description=Download cni
-        Requires=network-online.target
-        After=network-online.target
-        [Service]
-        Type=oneshot
-        RemainAfterExit=yes
-        ExecStartPre=/usr/bin/mkdir -p /opt/cni
-        ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output /opt/downloads/cni.tar.gz https://storage.googleapis.com/kubernetes-release/network-plugins/cni-07a8a28637e97b22eb8dfe710eeae1344f69d16e.tar.gz
-        ExecStart=/usr/bin/tar xf /opt/downloads/cni.tar.gz -C /opt/cni/        
-
-    - name: kubernetes-install-rkt.service
-      command: start
-      content: |
-        [Unit]
-        Description=Fetch rkt
-        Documentation=http://github.com/coreos/rkt
-        Requires=network-online.target
-        After=network-online.target
-        Requires=kube-env.service
-        After=kube-env.service
-        [Service]
-        Type=oneshot
-        RemainAfterExit=yes
-        EnvironmentFile=/etc/kube-env
-        ExecStartPre=/usr/bin/mkdir -p /etc/rkt /opt/downloads /opt/rkt/
-        ExecStartPre=/usr/bin/curl --fail --silent --location --create-dirs --output /opt/downloads/rkt.tar.gz https://github.com/coreos/rkt/releases/download/v${RKT_VERSION}/rkt-v${RKT_VERSION}.tar.gz
-        ExecStart=/usr/bin/tar --strip-components=1 -xf /opt/downloads/rkt.tar.gz -C /opt/rkt/ --overwrite
-
-    - name: kubernetes-load-rkt-stage1.service
-      command: start
-      content: |
-        [Unit]
-        Description=Load rkt stage1 images
-        Documentation=http://github.com/coreos/rkt
-        Requires=network-online.target
-        After=network-online.target
-        Requires=kubernetes-install-rkt.service
-        After=kubernetes-install-rkt.service
-        [Service]
-        Type=oneshot
-        ExecStart=/opt/rkt/rkt fetch /opt/rkt/stage1-coreos.aci /opt/rkt/stage1-kvm.aci /opt/rkt/stage1-fly.aci  --insecure-options=image
-
-    - name: kubernetes-download-manifests.service
-      command: start
-      content: |
-        [Unit]
-        Description=Download manifests
-        Requires=network-online.target
-        After=network-online.target
-        Requires=kube-env.service
-        After=kube-env.service
-        [Service]
-        Type=oneshot
-        RemainAfterExit=yes
-        EnvironmentFile=/etc/kube-env
-        ExecStartPre=/usr/bin/mkdir -p /opt/downloads
-        ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output \
-        /opt/downloads/kubernetes-manifests.tar.gz ${KUBE_MANIFESTS_TAR_URL}
-        # TODO(yifan): Check hash.
-        ExecStartPre=/usr/bin/mkdir -p /opt/kube-manifests
-        ExecStart=/usr/bin/tar xf /opt/downloads/kubernetes-manifests.tar.gz -C /opt/kube-manifests --overwrite
-
-    - name: kubernetes-install-node.service
-      command: start
-      content: |
-        [Unit]
-        Description=Install Kubernetes Server
-        Requires=network-online.target
-        After=network-online.target
-        Requires=kube-env.service
-        After=kube-env.service
-        [Service]
-        Type=oneshot
-        RemainAfterExit=yes
-        EnvironmentFile=/etc/kube-env
-        ExecStartPre=/usr/bin/mkdir -p /opt/kubernetes/pkg
-        ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output \
-        /opt/kubernetes/pkg/kubernetes-server-linux-amd64.tar.gz ${SERVER_BINARY_TAR_URL}
-        ExecStart=/usr/bin/tar xf /opt/kubernetes/pkg/kubernetes-server-linux-amd64.tar.gz -C /opt --overwrite
-
-    - name: rkt-api-service.service
-      command: start
-      content: |
-        [Unit]
-        Description=Start rkt API service as Daemon
-        Requires=kubernetes-install-rkt.service
-        After=kubernetes-install-rkt.service
-        [Service]
-        ExecStart=/opt/rkt/rkt api-service
-        Restart=always
-        RestartSec=10
-
-    - name: kubelet.service
-      command: start
-      content: |
-        [Unit]
-        Description=Run Kubelet service
-        Requires=network-online.target kube-env.service kubernetes-download-manifests.service kubernetes-install-cni.service kubernetes-load-rkt-stage1.service
-        After=network-online.target kube-env.service kubernetes-download-manifests.service kubernetes-install-cni.service kubernetes-load-rkt-stage1.service
-        [Service]
-        EnvironmentFile=/etc/kube-env
-        ExecStartPre=/usr/bin/curl --fail --silent --show-error \
-        -H "X-Google-Metadata-Request: True" \
-        -o /run/configure-kubelet.sh \
-        http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-kubelet
-        ExecStartPre=/usr/bin/chmod 0755 /run/configure-kubelet.sh
-        ExecStartPre=/run/configure-kubelet.sh
-        ExecStart=/opt/kubernetes/server/bin/kubelet \
-        --api-servers=https://${INSTANCE_PREFIX}-master \
-        --enable-debugging-handlers=true \
-        --cloud-provider=gce \
-        --config=/etc/kubernetes/manifests \
-        --allow-privileged=true \
-        --v=2 \
-        --cluster-dns=${DNS_SERVER_IP} \
-        --cluster-domain=${DNS_DOMAIN} \
-        --logtostderr=true \
-        --container-runtime=rkt \
-        --rkt-path=/opt/rkt/rkt \
-        --rkt-stage1-image=${RKT_STAGE1_IMAGE} \
-        --network-plugin=kubenet
-        Restart=always
-        RestartSec=10
-        KillMode=process
-
-    - name: kube-proxy.service
-      command: start
-      content: |
-        [Unit]
-        Description=Start Kube-proxy service as Daemon
-        Requires=kubernetes-configure-node.service
-        After=kubernetes-configure-node.service
-        [Service]
-        EnvironmentFile=/etc/kube-env
-        ExecStart=/opt/kubernetes/server/bin/kube-proxy \
-        --master=https://${KUBERNETES_MASTER_NAME} \
-        --kubeconfig=/var/lib/kube-proxy/kubeconfig \
-        --v=2 \
-        --logtostderr=true
-        Restart=always
-        RestartSec=10
-
-    - name: docker.service
-      command: stop
-
-    - name: kubernetes-configure-node.service
-      command: start
-      content: |
-        [Unit]
-        Description=Configure Node For Kubernetes service
-        Requires=kubernetes-install-node.service
-        After=kubernetes-install-node.service
-        Requires=kubernetes-install-rkt.service
-        After=kubernetes-install-rkt.service
-        Requires=kubernetes-download-manifests.service
-        After=kubernetes-download-manifests.service
-        [Service]
-        Type=oneshot
-        RemainAfterExit=yes
-        EnvironmentFile=/etc/kube-env
-        ExecStartPre=/usr/bin/curl --fail --silent --show-error \
-        -H "X-Google-Metadata-Request: True" \
-        -o /run/configure-node.sh \
-        http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-node
-        ExecStartPre=/usr/bin/chmod 0755 /run/configure-node.sh
-        ExecStart=/run/configure-node.sh

cluster/gce/coreos/node.yaml

+#cloud-config
+
+coreos:
+  update:
+    reboot-strategy: off
+  units:
+    - name: locksmithd.service
+      mask: true
+    - name: kube-node-installation.service
+      command: start
+      content: |
+        [Unit]
+        Description=Download and install k8s binaries and configurations
+        After=network-online.target
+
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        ExecStartPre=/bin/mkdir -p /opt/kubernetes/bin
+        ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error	-H "X-Google-Metadata-Request: True" -o /opt/kubernetes/bin/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-sh
+        ExecStartPre=/bin/chmod 544 /opt/kubernetes/bin/configure.sh
+        ExecStart=/opt/kubernetes/bin/configure.sh
+
+        [Install]
+        WantedBy=kubernetes.target
+    - name: kube-node-configuration.service
+      command: start
+      content: |
+        [Unit]
+        Description=Configure kubernetes master
+        After=kube-node-installation.service
+
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        ExecStartPre=/bin/chmod 544 /opt/kubernetes/bin/configure-helper.sh
+        ExecStart=/opt/kubernetes/bin/configure-helper.sh
+
+        [Install]
+        WantedBy=kubernetes.target
+    - name: kubernetes.target
+      enable: true
+      command: start
+      content: |
+        [Unit]
+        Description=Kubernetes
+
+        [Install]
+        WantedBy=multi-user.target
+    - name: docker.service
+      drop-ins:
+        - name: "use-cgroupfs-driver.conf"
+          # This is required for setting cgroup parent in the current ~1.4 per-pod cgroup impl
+          content: |
+            [Service]
+            Environment="DOCKER_CGROUPS=--exec-opt native.cgroupdriver="

cluster/gce/gci/configure.sh

@@ -202,7 +202,6 @@ function install-kube-binary-config {
   rm -f "${KUBE_HOME}/${manifests_tar}.sha1"
 }
 
-
 ######### Main Function ##########
 echo "Start to install kubernetes files"
 set-broken-motd
@@ -211,3 +210,4 @@ download-kube-env
 source "${KUBE_HOME}/kube-env"
 install-kube-binary-config
 echo "Done for installing kubernetes files"
+

cluster/gce/gci/master-helper.sh

@@ -15,7 +15,6 @@
 # limitations under the License.
 
 # A library of helper functions and constant for GCI distro
-
 source "${KUBE_ROOT}/cluster/gce/gci/helper.sh"
 
 # create-master-instance creates the master instance. If called with
@@ -72,6 +71,12 @@ function replicate-master-instance() {
 function create-master-instance-internal() {
   local -r master_name="${1}"
   local -r address_option="${2:-}"
+
+  local preemptible_master=""
+  if [[ "${PREEMPTIBLE_MASTER:-}" == "true" ]]; then
+    preemptible_master="--preemptible --maintenance-policy TERMINATE"
+  fi
+
   gcloud compute instances create "${master_name}" \
     ${address_option} \
     --project "${PROJECT}" \
@@ -86,7 +91,8 @@ function create-master-instance-internal() {
     --metadata-from-file \
       "kube-env=${KUBE_TEMP}/master-kube-env.yaml,user-data=${KUBE_ROOT}/cluster/gce/gci/master.yaml,configure-sh=${KUBE_ROOT}/cluster/gce/gci/configure.sh,cluster-name=${KUBE_TEMP}/cluster-name.txt,gci-update-strategy=${KUBE_TEMP}/gci-update.txt,gci-ensure-gke-docker=${KUBE_TEMP}/gci-ensure-gke-docker.txt,gci-docker-version=${KUBE_TEMP}/gci-docker-version.txt" \
     --disk "name=${master_name}-pd,device-name=master-pd,mode=rw,boot=no,auto-delete=no" \
-    --boot-disk-size "${MASTER_ROOT_DISK_SIZE:-10}"
+    --boot-disk-size "${MASTER_ROOT_DISK_SIZE:-10}" \
+    ${preemptible_master}
 }
 
 function get-metadata() {

hack/verify-flags/exceptions.txt

@@ -12,6 +12,9 @@ cluster/gce/configure-vm.sh:  cloud_config: ${CLOUD_CONFIG}
 cluster/gce/configure-vm.sh:  env-to-grains "feature_gates"
 cluster/gce/configure-vm.sh:  env-to-grains "runtime_config"
 cluster/gce/configure-vm.sh:  kubelet_api_servers: '${KUBELET_APISERVER}'
+cluster/gce/coreos/configure-helper.sh:  local api_servers="--master=https://${KUBERNETES_MASTER_NAME}"
+cluster/gce/coreos/configure-helper.sh:  sed -i -e "s@{{ *storage_backend *}}@${STORAGE_BACKEND:-}@g" "${temp_file}"
+cluster/gce/coreos/configure-helper.sh:  sed -i -e "s@{{pillar\['allow_privileged'\]}}@true@g" "${src_file}"
 cluster/gce/gci/configure-helper.sh:  local api_servers="--master=https://${KUBERNETES_MASTER_NAME}"
 cluster/gce/gci/configure-helper.sh:  sed -i -e "s@{{ *storage_backend *}}@${STORAGE_BACKEND:-}@g" "${temp_file}"
 cluster/gce/gci/configure-helper.sh:  sed -i -e "s@{{pillar\['allow_privileged'\]}}@true@g" "${src_file}"