Merge pull request #27220 from yifan-gu/kube-up-master

cluster/gce/coreos/configure-node.sh

@@ -119,7 +119,7 @@ function wait-url-up() {
   done
 }
 
-# Configure addon yamls, and run salt/kube-addons/kube-addon.sh
+# Configure addon yamls, and run salt/kube-addons/kube-addons.sh
 function configure-master-addons() {
   echo "Configuring master addons"
 
@@ -160,6 +160,10 @@ function configure-master-addons() {
     CLUSTER_REGISTRY_DISK_SIZE=$(convert-bytes-gce-kube "${CLUSTER_REGISTRY_DISK_SIZE}")
     evaluate-manifests-dir ${MANIFESTS_DIR}/addons/registry  ${addon_dir}/registry
   fi
+
+  if [[ "${ENABLE_NODE_PROBLEM_DETECTOR}" == "true" ]]; then
+    evaluate-manifests-dir ${MANIFESTS_DIR}/addons/node-problem-detector  ${addon_dir}/node-problem-detector
+  fi
 }
 
 function configure-master-components() {
@@ -169,8 +173,8 @@ function configure-master-components() {
   configure-kube-apiserver
   configure-kube-scheduler
   configure-kube-controller-manager
-  configure-addon-manager
   configure-master-addons
+  configure-addon-manager
 }
 
 # TODO(yifan): Merge this with mount-master-pd() in configure-vm.sh
@@ -297,9 +301,10 @@ function load-docker-images() {
 }
 
 
-# TODO(yifan): Making this function more generic for other runtimes.
 function load-master-components-images() {
   echo "Loading docker images for master components"
+  export RKT_BIN=/opt/rkt/rkt
+  export DOCKER2ACI_BIN=/opt/docker2aci/docker2aci
   ${SALT_DIR}/install.sh ${KUBE_BIN_TAR}
   ${SALT_DIR}/salt/kube-master-addons/kube-master-addons.sh
 

cluster/gce/coreos/helper.sh

@@ -23,7 +23,7 @@ function create-node-instance-template() {
   local template_name="$1"
   create-node-template "$template_name" "${scope_flags}" \
     "kube-env=${KUBE_TEMP}/node-kube-env.yaml" \
-    "user-data=${KUBE_ROOT}/cluster/gce/coreos/node.yaml" \
+    "user-data=${KUBE_ROOT}/cluster/gce/coreos/node-${CONTAINER_RUNTIME}.yaml" \
     "configure-node=${KUBE_ROOT}/cluster/gce/coreos/configure-node.sh" \
     "configure-kubelet=${KUBE_ROOT}/cluster/gce/coreos/configure-kubelet.sh" \
     "cluster-name=${KUBE_TEMP}/cluster-name.txt"
@@ -65,7 +65,7 @@ function create-master-instance() {
     --scopes "storage-ro,compute-rw,monitoring,logging-write" \
     --can-ip-forward \
     --metadata-from-file \
-      "kube-env=${KUBE_TEMP}/master-kube-env.yaml,user-data=${KUBE_ROOT}/cluster/gce/coreos/master.yaml,configure-node=${KUBE_ROOT}/cluster/gce/coreos/configure-node.sh,configure-kubelet=${KUBE_ROOT}/cluster/gce/coreos/configure-kubelet.sh,cluster-name=${KUBE_TEMP}/cluster-name.txt" \
+      "kube-env=${KUBE_TEMP}/master-kube-env.yaml,user-data=${KUBE_ROOT}/cluster/gce/coreos/master-${CONTAINER_RUNTIME}.yaml,configure-node=${KUBE_ROOT}/cluster/gce/coreos/configure-node.sh,configure-kubelet=${KUBE_ROOT}/cluster/gce/coreos/configure-kubelet.sh,cluster-name=${KUBE_TEMP}/cluster-name.txt" \
     --disk "name=${MASTER_NAME}-pd,device-name=master-pd,mode=rw,boot=no,auto-delete=no" \
     --boot-disk-size "${MASTER_ROOT_DISK_SIZE:-10}" \
       ${preemptible_master}

cluster/gce/coreos/kube-manifests/addons/dns/skydns-rc.yaml

 apiVersion: v1
 kind: ReplicationController
 metadata:
-  name: kube-dns-v11
+  name: kube-dns-v14
   namespace: kube-system
   labels:
     k8s-app: kube-dns
-    version: v11
+    version: v14
     kubernetes.io/cluster-service: "true"
 spec:
   replicas: ${DNS_REPLICAS}
   selector:
     k8s-app: kube-dns
-    version: v11
+    version: v14
   template:
     metadata:
       labels:
         k8s-app: kube-dns
-        version: v11
+        version: v14
         kubernetes.io/cluster-service: "true"
     spec:
       containers:
-      - name: etcd
-        image: gcr.io/google_containers/etcd-amd64:2.2.1
+      - name: kubedns
+        image: gcr.io/google_containers/kubedns-amd64:1.3
         resources:
           # TODO: Set memory limits when we've profiled the container for large
           # clusters, then set request = limit to keep this container in
@@ -29,33 +29,6 @@ spec:
           # "burstable" category so the kubelet doesn't backoff from restarting it.
           limits:
             cpu: 100m
-            memory: 500Mi
-          requests:
-            cpu: 100m
-            memory: 50Mi
-        command:
-        - /usr/local/bin/etcd
-        - -data-dir
-        - /var/etcd/data
-        - -listen-client-urls
-        - http://127.0.0.1:2379,http://127.0.0.1:4001
-        - -advertise-client-urls
-        - http://127.0.0.1:2379,http://127.0.0.1:4001
-        - -initial-cluster-token
-        - skydns-etcd
-        volumeMounts:
-        - name: etcd-storage
-          mountPath: /var/etcd/data
-      - name: kube2sky
-        image: gcr.io/google_containers/kube2sky:1.15
-        resources:
-          # TODO: Set memory limits when we've profiled the container for large
-          # clusters, then set request = limit to keep this container in
-          # guaranteed class. Currently, this container falls into the
-          # "burstable" category so the kubelet doesn't backoff from restarting it.
-          limits:
-            cpu: 100m
-            # Kube2sky watches all pods.
             memory: 200Mi
           requests:
             cpu: 100m
@@ -78,32 +51,23 @@ spec:
           # only setup the /readiness HTTP server once that's available.
           initialDelaySeconds: 30
           timeoutSeconds: 5
-        command:
-        - /kube2sky
         args:
-        # command = "/kube2sky"
-        - --domain=${DNS_DOMAIN}
-      - name: skydns
-        image: gcr.io/google_containers/skydns:2015-10-13-8c72f8c
-        resources:
-          # TODO: Set memory limits when we've profiled the container for large
-          # clusters, then set request = limit to keep this container in
-          # guaranteed class. Currently, this container falls into the
-          # "burstable" category so the kubelet doesn't backoff from restarting it.
-          limits:
-            cpu: 100m
-            memory: 200Mi
-          requests:
-            cpu: 100m
-            memory: 50Mi
-        command:
-        - /skydns
+        # command = "/kube-dns"
+        - --domain=${DNS_DOMAIN}.
+        - --dns-port=10053
+        ports:
+        - containerPort: 10053
+          name: dns-local
+          protocol: UDP
+        - containerPort: 10053
+          name: dns-tcp-local
+          protocol: TCP
+      - name: dnsmasq
+        image: gcr.io/google_containers/dnsmasq:1.1
         args:
-        # command = "/skydns"
-        - -machines=http://127.0.0.1:4001
-        - -addr=0.0.0.0:53
-        - -ns-rotate=false
-        - -domain=${DNS_DOMAIN}.
+        - --cache-size=1000
+        - --no-resolv
+        - --server=127.0.0.1#10053
         ports:
         - containerPort: 53
           name: dns
@@ -112,7 +76,7 @@ spec:
           name: dns-tcp
           protocol: TCP
       - name: healthz
-        image: gcr.io/google_containers/exechealthz:1.0
+        image: gcr.io/google_containers/exechealthz-amd64:1.0
         resources:
           # keep request = limit to keep this container in guaranteed class
           limits:
@@ -121,15 +85,10 @@ spec:
           requests:
             cpu: 10m
             memory: 20Mi
-        command:
-        - /exechealthz
         args:
         - -cmd=nslookup kubernetes.default.svc.${DNS_DOMAIN} 127.0.0.1 >/dev/null
         - -port=8080
         ports:
         - containerPort: 8080
           protocol: TCP
-      volumes:
-      - name: etcd-storage
-        emptyDir: {}
-      dnsPolicy: Default  # Don't use cluster DNS.
+      dnsPolicy: Default  # Don't use cluster DNS.
\ No newline at end of file

cluster/gce/coreos/kube-manifests/addons/fluentd-elasticsearch/es-controller.yaml

@@ -20,7 +20,7 @@ spec:
         kubernetes.io/cluster-service: "true"
     spec:
       containers:
-      - image: gcr.io/google_containers/elasticsearch:1.8
+      - image: gcr.io/google_containers/elasticsearch:1.9
         name: elasticsearch-logging         
         resources:
           # keep request = limit to keep this container in guaranteed class
@@ -40,4 +40,4 @@ spec:
           mountPath: /data
       volumes:
       - name: es-persistent-storage
-        emptyDir: {}
+        emptyDir: {}
\ No newline at end of file

cluster/gce/coreos/kube-manifests/addons/node-problem-detector/node-problem-detector.yaml

+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: node-problem-detector-v0.1
+  namespace: kube-system
+  labels:
+    k8s-app: node-problem-detector
+    version: v0.1
+    kubernetes.io/cluster-service: "true"
+spec:
+  template:
+    metadata:
+      labels:
+        k8s-app: node-problem-detector
+        version: v0.1
+        kubernetes.io/cluster-service: "true"
+    spec:
+      hostNetwork: true
+      containers:
+      - name: node-problem-detector
+        image: gcr.io/google_containers/node-problem-detector:v0.1
+        env:
+        # Config the host ip and port of apiserver.
+        - name: "KUBERNETES_SERVICE_HOST"
+          value: ${INSTANCE_PREFIX}-master
+        - name: "KUBERNETES_SERVICE_PORT"
+          value: "443"
+        securityContext:
+          privileged: true
+        resources:
+          limits:
+            cpu: "200m"
+            memory: "100Mi"
+          requests:
+            cpu: "20m"
+            memory: "20Mi"
+        volumeMounts:
+        - name: log
+          mountPath: /log
+          readOnly: true
+      volumes:
+      - name: log
+        hostPath:
+          path: /var/log/

cluster/gce/coreos/kube-manifests/etcd-events.yaml

@@ -14,7 +14,7 @@ spec:
       --bind-addr=127.0.0.1:4002
       --data-dir=/var/etcd/data-events
       1>>/var/log/etcd-events.log 2>&1
-    image: gcr.io/google_containers/etcd:2.0.12
+    image: gcr.io/google_containers/etcd:2.2.1
     imagePullPolicy: IfNotPresent
     livenessProbe:
       httpGet:

cluster/gce/coreos/kube-manifests/kube-addon-manager.yaml

@@ -12,7 +12,9 @@ spec:
     resources:
       requests:
         cpu: 5m
-        memory: 50Mi
+        # TODO(yifan): Figure out what's the memory usage should be here.
+        # See https://github.com/kubernetes/kubernetes/issues/23641.
+        memory: 100Mi
     volumeMounts:
     - mountPath: /etc/kubernetes/
       name: addons

cluster/gce/coreos/master-docker.yaml

+#cloud-config
+
+coreos:
+  update:
+    reboot-strategy: off
+  units:
+    - name: locksmithd.service
+      mask: true
+    - name: kube-env.service
+      command: start
+      content: |
+        [Unit]
+        Description=Fetch kubernetes-node-environment
+        Requires=network-online.target
+        After=network-online.target
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        ExecStartPre=/usr/bin/curl --fail --silent --show-error \
+        -H "X-Google-Metadata-Request: True" \
+        -o /etc/kube-env.yaml \
+        http://metadata.google.internal/computeMetadata/v1/instance/attributes/kube-env
+        # Transform the yaml to env file.
+        ExecStartPre=/usr/bin/mv /etc/kube-env.yaml /etc/kube-env
+        ExecStart=/usr/bin/sed -i "s/: '/=/;s/'$//" /etc/kube-env
+
+    - name: kubernetes-install-cni.service
+      command: start
+      content: |
+        [Unit]
+        Description=Download cni
+        Requires=network-online.target
+        After=network-online.target
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        ExecStartPre=/usr/bin/mkdir -p /opt/cni
+        ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output /opt/downloads/cni.tar.gz https://storage.googleapis.com/kubernetes-release/network-plugins/cni-c864f0e1ea73719b8f4582402b0847064f9883b0.tar.gz
+        ExecStart=/usr/bin/tar xf /opt/downloads/cni.tar.gz -C /opt/cni/        
+        
+    - name: kubernetes-download-salt.service
+      command: start
+      content: |
+        [Unit]
+        Description=Download salt
+        Requires=network-online.target
+        After=network-online.target
+        Requires=kube-env.service
+        After=kube-env.service
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        EnvironmentFile=/etc/kube-env
+        ExecStartPre=/usr/bin/mkdir -p /opt/downloads
+        ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output \
+        /opt/downloads/kubernetes-salt.tar.gz ${SALT_TAR_URL}
+        # TODO(yifan): Check hash.
+        ExecStart=/usr/bin/tar xf /opt/downloads/kubernetes-salt.tar.gz -C /opt --overwrite
+
+    - name: kubernetes-download-manifests.service
+      command: start
+      content: |
+        [Unit]
+        Description=Download manifests
+        Requires=network-online.target
+        After=network-online.target
+        Requires=kube-env.service
+        After=kube-env.service
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        EnvironmentFile=/etc/kube-env
+        ExecStartPre=/usr/bin/mkdir -p /opt/downloads
+        ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output \
+        /opt/downloads/kubernetes-manifests.tar.gz ${KUBE_MANIFESTS_TAR_URL}
+        # TODO(yifan): Check hash.
+        ExecStartPre=/usr/bin/mkdir -p /opt/kube-manifests
+        ExecStart=/usr/bin/tar xf /opt/downloads/kubernetes-manifests.tar.gz -C /opt/kube-manifests --overwrite
+
+    - name: kubernetes-install-node.service
+      command: start
+      content: |
+        [Unit]
+        Description=Install Kubernetes Server
+        Requires=network-online.target
+        After=network-online.target
+        Requires=kube-env.service
+        After=kube-env.service
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        EnvironmentFile=/etc/kube-env
+        ExecStartPre=/usr/bin/mkdir -p /opt/downloads
+        ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output \
+        /opt/downloads/kubernetes-server-linux-amd64.tar.gz ${SERVER_BINARY_TAR_URL}
+        # TODO(yifan): Check hash.
+        ExecStart=/usr/bin/tar xf /opt/downloads/kubernetes-server-linux-amd64.tar.gz -C /opt --overwrite
+
+    - name: kubelet.service
+      command: start
+      content: |
+        [Unit]
+        Description=Run Kubelet service
+        Requires=network-online.target kube-env.service kubernetes-download-manifests.service kubernetes-install-cni.service
+        After=network-online.target kube-env.service kubernetes-download-manifests.service kubernetes-install-cni.service
+        [Service]
+        EnvironmentFile=/etc/kube-env
+        ExecStartPre=/usr/bin/curl --fail --silent --show-error \
+        -H "X-Google-Metadata-Request: True" \
+        -o /run/configure-kubelet.sh \
+        http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-kubelet
+        ExecStartPre=/usr/bin/chmod 0755 /run/configure-kubelet.sh
+        ExecStartPre=/run/configure-kubelet.sh
+        ExecStart=/opt/kubernetes/server/bin/kubelet \
+        --api-servers=https://${INSTANCE_PREFIX}-master \
+        --enable-debugging-handlers=false \
+        --cloud-provider=gce \
+        --config=/etc/kubernetes/manifests \
+        --allow-privileged=true \
+        --v=2 \
+        --cluster-dns=${DNS_SERVER_IP} \
+        --cluster-domain=${DNS_DOMAIN} \
+        --logtostderr=true \
+        --container-runtime=docker \
+        --rkt-path=/opt/rkt/rkt \
+        --rkt-stage1-image=/opt/rkt/stage1-coreos.aci \
+        --configure-cbr0=${KUBERNETES_CONFIGURE_CBR0} \
+        --pod-cidr=${MASTER_IP_RANGE} \
+        --register-schedulable=false \
+        --reconcile-cidr=false
+        Restart=always
+        RestartSec=10
+        KillMode=process
+
+    - name: docker.service
+      drop-ins:
+        - name: 50-docker-opts.conf
+          content: |
+            [Service]
+            Environment='DOCKER_OPTS=--bridge=cbr0 --iptables=false --ip-masq=false'
+            MountFlags=slave
+            LimitNOFILE=1048576
+            LimitNPROC=1048576
+            LimitCORE=infinity
+            Restart=always
+            RestartSec=2s
+            StartLimitInterval=0
+
+    - name: kubernetes-configure-node.service
+      command: start
+      content: |
+        [Unit]
+        Description=Configure Node For Kubernetes service
+        Requires=kubernetes-install-node.service
+        After=kubernetes-install-node.service
+        Requires=kubernetes-download-salt.service
+        After=kubernetes-download-salt.service
+        Requires=kubernetes-download-manifests.service
+        After=kubernetes-download-manifests.service
+        # Need the kubelet/docker running because we will use docker load for docker images.
+        Requires=kubelet.service
+        After=kubelet.service
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        EnvironmentFile=/etc/kube-env
+        ExecStartPre=/usr/bin/curl --fail --silent --show-error \
+        -H "X-Google-Metadata-Request: True" \
+        -o /run/configure-node.sh \
+        http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-node
+        ExecStartPre=/usr/bin/chmod 0755 /run/configure-node.sh
+        ExecStart=/run/configure-node.sh

cluster/gce/coreos/master.yaml → cluster/gce/coreos/master-rkt.yaml

@@ -35,9 +35,23 @@ coreos:
         Type=oneshot
         RemainAfterExit=yes
         ExecStartPre=/usr/bin/mkdir -p /opt/cni
-        ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output /opt/downloads/cni.tar.gz https://storage.googleapis.com/kubernetes-release/network-plugins/cni-42c4cb842dad606a84e93aad5a4484ded48e3046.tar.gz
-        ExecStart=/usr/bin/tar xf /opt/downloads/cni.tar.gz -C /opt/cni/        
-        
+        ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output /opt/downloads/cni.tar.gz https://storage.googleapis.com/kubernetes-release/network-plugins/cni-c864f0e1ea73719b8f4582402b0847064f9883b0.tar.gz
+        ExecStart=/usr/bin/tar xf /opt/downloads/cni.tar.gz -C /opt/cni/
+
+    - name: kubernetes-install-docker2aci.service
+      command: start
+      content: |
+        [Unit]
+        Description=Download docker2aci
+        Requires=network-online.target
+        After=network-online.target
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        ExecStartPre=/usr/bin/mkdir -p /opt/docker2aci
+        ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output /opt/downloads/docker2aci.tar.gz https://github.com/appc/docker2aci/releases/download/v0.11.1/docker2aci-v0.11.1.tar.gz
+        ExecStart=/usr/bin/tar --strip-components=1 -xf /opt/downloads/docker2aci.tar.gz -C /opt/docker2aci/ --overwrite
+
     - name: kubernetes-install-rkt.service
       command: start
       content: |
@@ -114,6 +128,18 @@ coreos:
         # TODO(yifan): Check hash.
         ExecStart=/usr/bin/tar xf /opt/downloads/kubernetes-server-linux-amd64.tar.gz -C /opt --overwrite
 
+    - name: rkt-api-service.service
+      command: start
+      content: |
+        [Unit]
+        Description=Start rkt API service as Daemon
+        Requires=kubernetes-install-rkt.service
+        After=kubernetes-install-rkt.service
+        [Service]
+        ExecStart=/opt/rkt/rkt api-service
+        Restart=always
+        RestartSec=10
+
     - name: kubelet.service
       command: start
       content: |
@@ -139,7 +165,7 @@ coreos:
         --cluster-dns=${DNS_SERVER_IP} \
         --cluster-domain=${DNS_DOMAIN} \
         --logtostderr=true \
-        --container-runtime=${KUBERNETES_CONTAINER_RUNTIME} \
+        --container-runtime=rkt \
         --rkt-path=/opt/rkt/rkt \
         --rkt-stage1-image=/opt/rkt/stage1-coreos.aci \
         --configure-cbr0=${KUBERNETES_CONFIGURE_CBR0} \
@@ -149,20 +175,9 @@ coreos:
         Restart=always
         RestartSec=10
         KillMode=process
-        
+
     - name: docker.service
-      drop-ins:
-        - name: 50-docker-opts.conf
-          content: |
-            [Service]
-            Environment='DOCKER_OPTS=--bridge=cbr0 --iptables=false --ip-masq=false'
-            MountFlags=slave
-            LimitNOFILE=1048576
-            LimitNPROC=1048576
-            LimitCORE=infinity
-            Restart=always
-            RestartSec=2s
-            StartLimitInterval=0
+      command: stop
 
     - name: kubernetes-configure-node.service
       command: start
@@ -177,6 +192,8 @@ coreos:
         After=kubernetes-download-salt.service
         Requires=kubernetes-download-manifests.service
         After=kubernetes-download-manifests.service
+        Requires=kubernetes-install-docker2aci.service
+        After=kubernetes-install-docker2aci.service
         # Need the kubelet/docker running because we will use docker load for docker images.
         Requires=kubelet.service
         After=kubelet.service

cluster/gce/coreos/node-docker.yaml

+#cloud-config
+
+coreos:
+  update:
+    reboot-strategy: off
+  units:
+    - name: locksmithd.service
+      mask: true
+    - name: kube-env.service
+      command: start
+      content: |
+        [Unit]
+        Description=Fetch kubernetes-node-environment
+        Requires=network-online.target
+        After=network-online.target
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        ExecStartPre=/usr/bin/curl --fail --silent --show-error \
+        -H "X-Google-Metadata-Request: True" \
+        -o /etc/kube-env.yaml \
+        http://metadata.google.internal/computeMetadata/v1/instance/attributes/kube-env
+        # Transform the yaml to env file.
+        ExecStartPre=/usr/bin/mv /etc/kube-env.yaml /etc/kube-env
+        ExecStart=/usr/bin/sed -i "s/: '/=/;s/'$//" /etc/kube-env
+
+    - name: kubernetes-install-cni.service
+      command: start
+      content: |
+        [Unit]
+        Description=Download cni
+        Requires=network-online.target
+        After=network-online.target
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        ExecStartPre=/usr/bin/mkdir -p /opt/cni
+        ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output /opt/downloads/cni.tar.gz https://storage.googleapis.com/kubernetes-release/network-plugins/cni-c864f0e1ea73719b8f4582402b0847064f9883b0.tar.gz
+        ExecStart=/usr/bin/tar xf /opt/downloads/cni.tar.gz -C /opt/cni/        
+
+    - name: kubernetes-download-manifests.service
+      command: start
+      content: |
+        [Unit]
+        Description=Download manifests
+        Requires=network-online.target
+        After=network-online.target
+        Requires=kube-env.service
+        After=kube-env.service
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        EnvironmentFile=/etc/kube-env
+        ExecStartPre=/usr/bin/mkdir -p /opt/downloads
+        ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output \
+        /opt/downloads/kubernetes-manifests.tar.gz ${KUBE_MANIFESTS_TAR_URL}
+        # TODO(yifan): Check hash.
+        ExecStartPre=/usr/bin/mkdir -p /opt/kube-manifests
+        ExecStart=/usr/bin/tar xf /opt/downloads/kubernetes-manifests.tar.gz -C /opt/kube-manifests --overwrite
+
+    - name: kubernetes-install-node.service
+      command: start
+      content: |
+        [Unit]
+        Description=Install Kubernetes Server
+        Requires=network-online.target
+        After=network-online.target
+        Requires=kube-env.service
+        After=kube-env.service
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        EnvironmentFile=/etc/kube-env
+        ExecStartPre=/usr/bin/mkdir -p /opt/kubernetes/pkg
+        ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output \
+        /opt/kubernetes/pkg/kubernetes-server-linux-amd64.tar.gz ${SERVER_BINARY_TAR_URL}
+        ExecStart=/usr/bin/tar xf /opt/kubernetes/pkg/kubernetes-server-linux-amd64.tar.gz -C /opt --overwrite
+
+    - name: kubelet.service
+      command: start
+      content: |
+        [Unit]
+        Description=Run Kubelet service
+        Requires=network-online.target kube-env.service kubernetes-download-manifests.service kubernetes-install-cni.service
+        After=network-online.target kube-env.service kubernetes-download-manifests.service kubernetes-install-cni.service
+        [Service]
+        EnvironmentFile=/etc/kube-env
+        ExecStartPre=/usr/bin/curl --fail --silent --show-error \
+        -H "X-Google-Metadata-Request: True" \
+        -o /run/configure-kubelet.sh \
+        http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-kubelet
+        ExecStartPre=/usr/bin/chmod 0755 /run/configure-kubelet.sh
+        ExecStartPre=/run/configure-kubelet.sh
+        ExecStart=/opt/kubernetes/server/bin/kubelet \
+        --api-servers=https://${INSTANCE_PREFIX}-master \
+        --enable-debugging-handlers=true \
+        --cloud-provider=gce \
+        --config=/etc/kubernetes/manifests \
+        --allow-privileged=true \
+        --v=2 \
+        --cluster-dns=${DNS_SERVER_IP} \
+        --cluster-domain=${DNS_DOMAIN} \
+        --logtostderr=true \
+        --container-runtime=docker \
+        --configure-cbr0=${KUBERNETES_CONFIGURE_CBR0} \
+        --network-plugin=${NETWORK_PROVIDER} \
+        --reconcile-cidr=true
+        Restart=always
+        RestartSec=10
+        KillMode=process
+
+    - name: kube-proxy.service
+      command: start
+      content: |
+        [Unit]
+        Description=Start Kube-proxy service as Daemon
+        Requires=kubernetes-configure-node.service
+        After=kubernetes-configure-node.service
+        [Service]
+        EnvironmentFile=/etc/kube-env
+        ExecStart=/opt/kubernetes/server/bin/kube-proxy \
+        --master=https://${KUBERNETES_MASTER_NAME} \
+        --kubeconfig=/var/lib/kube-proxy/kubeconfig \
+        --v=2 \
+        --logtostderr=true
+        Restart=always
+        RestartSec=10
+
+    - name: docker.service
+      drop-ins:
+        - name: 50-docker-opts.conf
+          content: |
+            [Service]
+            Environment='DOCKER_OPTS= --iptables=false --ip-masq=false'
+            MountFlags=slave
+            LimitNOFILE=1048576
+            LimitNPROC=1048576
+            LimitCORE=infinity
+            Restart=always
+            RestartSec=2s
+            StartLimitInterval=0
+
+    - name: kubernetes-configure-node.service
+      command: start
+      content: |
+        [Unit]
+        Description=Configure Node For Kubernetes service
+        Requires=kubernetes-install-node.service
+        After=kubernetes-install-node.service
+        Requires=kubernetes-download-manifests.service
+        After=kubernetes-download-manifests.service
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        EnvironmentFile=/etc/kube-env
+        ExecStartPre=/usr/bin/curl --fail --silent --show-error \
+        -H "X-Google-Metadata-Request: True" \
+        -o /run/configure-node.sh \
+        http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-node
+        ExecStartPre=/usr/bin/chmod 0755 /run/configure-node.sh
+        ExecStart=/run/configure-node.sh

cluster/gce/coreos/node.yaml → cluster/gce/coreos/node-rkt.yaml

@@ -35,7 +35,7 @@ coreos:
         Type=oneshot
         RemainAfterExit=yes
         ExecStartPre=/usr/bin/mkdir -p /opt/cni
-        ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output /opt/downloads/cni.tar.gz https://storage.googleapis.com/kubernetes-release/network-plugins/cni-42c4cb842dad606a84e93aad5a4484ded48e3046.tar.gz
+        ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output /opt/downloads/cni.tar.gz https://storage.googleapis.com/kubernetes-release/network-plugins/cni-c864f0e1ea73719b8f4582402b0847064f9883b0.tar.gz
         ExecStart=/usr/bin/tar xf /opt/downloads/cni.tar.gz -C /opt/cni/        
 
     - name: kubernetes-install-rkt.service
@@ -131,10 +131,12 @@ coreos:
         --cluster-dns=${DNS_SERVER_IP} \
         --cluster-domain=${DNS_DOMAIN} \
         --logtostderr=true \
-        --container-runtime=${KUBERNETES_CONTAINER_RUNTIME} \
+        --container-runtime=rkt \
         --rkt-path=/opt/rkt/rkt \
         --rkt-stage1-image=/opt/rkt/stage1-coreos.aci \
-        --configure-cbr0=${KUBERNETES_CONFIGURE_CBR0}
+        --configure-cbr0=${KUBERNETES_CONFIGURE_CBR0} \
+        --network-plugin=kubenet \
+        --reconcile-cidr=true
         Restart=always
         RestartSec=10
         KillMode=process
@@ -157,18 +159,7 @@ coreos:
         RestartSec=10
 
     - name: docker.service
-      drop-ins:
-        - name: 50-docker-opts.conf
-          content: |
-            [Service]
-            Environment='DOCKER_OPTS=--bridge=cbr0 --iptables=false --ip-masq=false'
-            MountFlags=slave
-            LimitNOFILE=1048576
-            LimitNPROC=1048576
-            LimitCORE=infinity
-            Restart=always
-            RestartSec=2s
-            StartLimitInterval=0
+      command: stop
 
     - name: kubernetes-configure-node.service
       command: start

cluster/saltbase/salt/kube-master-addons/kube-master-addons.sh

@@ -15,52 +15,76 @@
 # limitations under the License.
 
 # loadedImageFlags is a bit-flag to track which docker images loaded successfully.
-let loadedImageFlags=0
 
-while true; do
-  restart_docker=false
-
-  if which docker 1>/dev/null 2>&1; then
-
-    timeout 30 docker load -i /srv/salt/kube-bins/kube-apiserver.tar 1>/dev/null 2>&1
-    rc=$?
-    if [[ $rc == 0 ]]; then
-      let loadedImageFlags="$loadedImageFlags|1"
-    elif [[ $rc == 124 ]]; then
-      restart_docker=true
-    fi
-
-    timeout 30 docker load -i /srv/salt/kube-bins/kube-scheduler.tar 1>/dev/null 2>&1
-    rc=$?
-    if [[ $rc == 0 ]]; then
-      let loadedImageFlags="$loadedImageFlags|2"
-    elif [[ $rc == 124 ]]; then
-      restart_docker=true
+function load-docker-images() {
+  let loadedImageFlags=0
+  
+  while true; do
+    restart_docker=false
+  
+    if which docker 1>/dev/null 2>&1; then
+  
+      timeout 30 docker load -i /srv/salt/kube-bins/kube-apiserver.tar 1>/dev/null 2>&1
+      rc=$?
+      if [[ $rc == 0 ]]; then
+        let loadedImageFlags="$loadedImageFlags|1"
+      elif [[ $rc == 124 ]]; then
+        restart_docker=true
+      fi
+  
+      timeout 30 docker load -i /srv/salt/kube-bins/kube-scheduler.tar 1>/dev/null 2>&1
+      rc=$?
+      if [[ $rc == 0 ]]; then
+        let loadedImageFlags="$loadedImageFlags|2"
+      elif [[ $rc == 124 ]]; then
+        restart_docker=true
+      fi
+  
+      timeout 30 docker load -i /srv/salt/kube-bins/kube-controller-manager.tar 1>/dev/null 2>&1
+      rc=$?
+      if [[ $rc == 0 ]]; then
+        let loadedImageFlags="$loadedImageFlags|4"
+      elif [[ $rc == 124 ]]; then
+        restart_docker=true
+      fi
     fi
-
-    timeout 30 docker load -i /srv/salt/kube-bins/kube-controller-manager.tar 1>/dev/null 2>&1
-    rc=$?
-    if [[ $rc == 0 ]]; then
-      let loadedImageFlags="$loadedImageFlags|4"
-    elif [[ $rc == 124 ]]; then
-      restart_docker=true
+  
+    # required docker images got installed. exit while loop.
+    if [[ $loadedImageFlags == 7 ]]; then break; fi
+  
+    # Sometimes docker load hang, restart docker daemon resolve the issue
+    if [[ $restart_docker ]]; then
+      if ! service docker restart; then # Try systemctl if there's no service command.
+        systemctl restart docker      
+      fi
     fi
-  fi
+  
+    # sleep for 15 seconds before attempting to load docker images again
+    sleep 15
+  
+  done
+}
 
-  # required docker images got installed. exit while loop.
-  if [[ $loadedImageFlags == 7 ]]; then break; fi
+function convert-rkt-image() {
+  (cd /tmp; ${DOCKER2ACI_BIN} $1)
+}
 
-  # Sometimes docker load hang, restart docker daemon resolve the issue
-  if [[ $restart_docker ]]; then
-    if ! service docker restart; then # Try systemctl if there's no service command.
-      systemctl restart docker      
-    fi
-  fi
+function load-rkt-images() {
+  convert-rkt-image /srv/salt/kube-bins/kube-apiserver.tar
+  convert-rkt-image /srv/salt/kube-bins/kube-scheduler.tar
+  convert-rkt-image /srv/salt/kube-bins/kube-controller-manager.tar
 
-  # sleep for 15 seconds before attempting to load docker images again
-  sleep 15
+  # Currently, we can't run docker image tarballs directly,
+  # So we use 'rkt fetch' to load the docker images into rkt image stores.
+  # see https://github.com/coreos/rkt/issues/2392.
+  ${RKT_BIN} fetch /tmp/*.aci --insecure-options=image
+}
 
-done
+if [[ "${KUBERNETES_CONTAINER_RUNTIME}" == "rkt" ]]; then
+  load-rkt-images
+else
+  load-docker-images
+fi
 
 # Now exit. After kube-push, salt will notice that the service is down and it
 # will start it and new docker images will be loaded.