Large changes to the docker deployment. Added kube-addon-manager as a static…

cluster/addons/dashboard/dashboard-controller.yaml

+# This file should be kept in sync with cluster/images/hyperkube/dashboard-rc.yaml
+# and cluster/gce/coreos/kube-manifests/addons/dashboard/dashboard-controller.yaml
 apiVersion: v1
 kind: ReplicationController
 metadata:
-  # Keep the name in sync with image version and
-  # gce/coreos/kube-manifests/addons/dashboard counterparts
   name: kubernetes-dashboard-v1.0.1
   namespace: kube-system
   labels:

cluster/addons/dashboard/dashboard-service.yaml

+# This file should be kept in sync with cluster/images/hyperkube/dashboard-svc.yaml
+# and cluster/gce/coreos/kube-manifests/addons/dashboard/dashboard-service.yaml
 apiVersion: v1
 kind: Service
 metadata:

cluster/addons/dns/skydns-rc.yaml.in

+# This file should be kept in sync with cluster/images/hyperkube/dns-rc.yaml
 apiVersion: v1
 kind: ReplicationController
 metadata:

cluster/addons/dns/skydns-svc.yaml.in

+# This file should be kept in sync with cluster/images/hyperkube/dns-svc.yaml
 apiVersion: v1
 kind: Service
 metadata:

cluster/images/hyperkube/Dockerfile

@@ -38,25 +38,28 @@ RUN cp /usr/bin/nsenter /nsenter
 COPY hyperkube /hyperkube
 
 # Manifests for the docker guide
-COPY master.json /etc/kubernetes/manifests/master.json
-COPY etcd.json /etc/kubernetes/manifests/etcd.json
-COPY kube-proxy.json /etc/kubernetes/manifests/kube-proxy.json
+COPY static-pods/master.json /etc/kubernetes/manifests/
+COPY static-pods/etcd.json /etc/kubernetes/manifests/
+COPY static-pods/addon-manager.json /etc/kubernetes/manifests/
 
 # Manifests for the docker-multinode guide
-COPY master-multi.json /etc/kubernetes/manifests-multi/master.json
-COPY kube-proxy.json /etc/kubernetes/manifests-multi/kube-proxy.json
+COPY static-pods/master-multi.json /etc/kubernetes/manifests-multi/
+COPY static-pods/addon-manager.json /etc/kubernetes/manifests-multi/
+
+# Copy over all addons
+COPY addons /etc/kubernetes/addons
 
 # Other required scripts for the setup
 COPY safe_format_and_mount /usr/share/google/safe_format_and_mount
 COPY setup-files.sh /setup-files.sh
 COPY make-ca-cert.sh /make-ca-cert.sh
+COPY copy-addons.sh /copy-addons.sh
 
 # easy-rsa package required by make-ca-cert
 ADD https://storage.googleapis.com/kubernetes-release/easy-rsa/easy-rsa.tar.gz /root/kube/
 
-RUN mkdir -p /opt/cni
-RUN curl https://storage.googleapis.com/kubernetes-release/network-plugins/cni-c864f0e1ea73719b8f4582402b0847064f9883b0.tar.gz \
-  | tar xzv -C /opt/cni
+# Copy the cni folder into /opt/
+COPY cni /opt/cni
 
 # Create symlinks for each hyperkube server
 # TODO: this is unreliable for now (e.g. running "/kubelet" panics)

cluster/images/hyperkube/Makefile

@@ -20,6 +20,7 @@
 REGISTRY?="gcr.io/google_containers"
 ARCH?=amd64
 TEMP_DIR:=$(shell mktemp -d)
+CNI_RELEASE=c864f0e1ea73719b8f4582402b0847064f9883b0
 
 UNAME_S:=$(shell uname -s)
 ifeq ($(UNAME_S),Darwin)
@@ -28,6 +29,7 @@ endif
 ifeq ($(UNAME_S),Linux)
 	SED_CMD?=sed -i
 endif
+
 ifeq ($(ARCH),amd64)
 	BASEIMAGE?=debian:jessie
 endif
@@ -51,27 +53,32 @@ build:
 ifndef VERSION
     $(error VERSION is undefined)
 endif
-	cp ./* ${TEMP_DIR}
+	cp -r ./* ${TEMP_DIR}
+	mkdir -p ${TEMP_DIR}/cni
 	cp ../../saltbase/salt/helpers/safe_format_and_mount ${TEMP_DIR}
 	cp ../../saltbase/salt/generate-cert/make-ca-cert.sh ${TEMP_DIR}
 	cp ../../../_output/dockerized/bin/linux/${ARCH}/hyperkube ${TEMP_DIR}
-	cd ${TEMP_DIR} && sed -i.back "s|VERSION|${VERSION}|g" master-multi.json master.json kube-proxy.json
-	cd ${TEMP_DIR} && sed -i.back "s|ARCH|${ARCH}|g" master-multi.json master.json kube-proxy.json etcd.json
+
+	cd ${TEMP_DIR} && sed -i.back "s|VERSION|${VERSION}|g" addons/*.yaml static-pods/*.json
+	cd ${TEMP_DIR} && sed -i.back "s|ARCH|${ARCH}|g" addons/*.yaml static-pods/*.json
 	cd ${TEMP_DIR} && sed -i.back "s|ARCH|${QEMUARCH}|g" Dockerfile
 	cd ${TEMP_DIR} && sed -i.back "s|BASEIMAGE|${BASEIMAGE}|g" Dockerfile
-	rm ${TEMP_DIR}/*.back
+	rm ${TEMP_DIR}/addons/*.back
 
 	# Make scripts executable before they are copied into the Docker image. If we make them executable later, in another layer
 	# they'll take up twice the space because the new executable binary differs from the old one, but everything is cached in layers.
 	cd ${TEMP_DIR} && chmod a+rx \
-        hyperkube \
-        safe_format_and_mount \
-        setup-files.sh \
-        make-ca-cert.sh
+		hyperkube \
+		safe_format_and_mount \
+		setup-files.sh \
+		make-ca-cert.sh \
+		copy-addons.sh
 
 ifeq ($(ARCH),amd64)
 	# When building "normally" for amd64, remove the whole line, it has no part in the amd64 image
 	cd ${TEMP_DIR} && ${SED_CMD} "/CROSS_BUILD_/d" Dockerfile
+	# Download CNI
+	curl -sSL --retry 5 https://storage.googleapis.com/kubernetes-release/network-plugins/cni-${CNI_RELEASE}.tar.gz | tar -xz -C ${TEMP_DIR}/cni
 else
 	# When cross-building, only the placeholder "CROSS_BUILD_" should be removed
 	# Register /usr/bin/qemu-ARCH-static as the handler for ARM binaries in the kernel

cluster/images/hyperkube/addons/dashboard-rc.yaml

+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file should be kept in sync with cluster/addons/dashboard/dashboard-controller.yaml
+apiVersion: v1
+kind: ReplicationController
+metadata:
+  name: kubernetes-dashboard
+  namespace: kube-system
+  labels:
+    app: kubernetes-dashboard
+    version: v1.0.1
+    kubernetes.io/cluster-service: "true"
+spec:
+  replicas: 1
+  selector:
+    app: kubernetes-dashboard
+    version: v1.0.1
+    kubernetes.io/cluster-service: "true"
+  template:
+    metadata:
+      labels:
+        app: kubernetes-dashboard
+        version: v1.0.1
+        kubernetes.io/cluster-service: "true"
+    spec:
+      containers:
+      - name: kubernetes-dashboard
+        # ARCH will be replaced with the architecture it's built for. Check out the Makefile for more details
+        image: gcr.io/google_containers/kubernetes-dashboard-ARCH:v1.0.1
+        imagePullPolicy: Always
+        ports:
+        - containerPort: 9090
+          protocol: TCP
+        livenessProbe:
+          httpGet:
+            path: /
+            port: 9090
+          initialDelaySeconds: 30
+          timeoutSeconds: 30

cluster/images/hyperkube/addons/dashboard-svc.yaml

+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file should be kept in sync with cluster/addons/dashboard/dashboard-service.yaml
+kind: Service
+apiVersion: v1
+metadata:
+  name: kubernetes-dashboard
+  namespace: kube-system
+  labels:
+    app: kubernetes-dashboard
+    kubernetes.io/cluster-service: "true"
+spec:
+  ports:
+  - port: 80
+    targetPort: 9090
+  selector:
+    app: kubernetes-dashboard

cluster/images/hyperkube/addons/dns-rc.yaml

+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file should be kept in sync with cluster/addons/dns/skydns-rc.yaml.in
+apiVersion: v1
+kind: ReplicationController
+metadata:
+  name: kube-dns-v11
+  namespace: kube-system
+  labels:
+    k8s-app: kube-dns
+    version: v11
+    kubernetes.io/cluster-service: "true"
+spec:
+  replicas: 1
+  selector:
+    k8s-app: kube-dns
+    version: v11
+  template:
+    metadata:
+      labels:
+        k8s-app: kube-dns
+        version: v11
+        kubernetes.io/cluster-service: "true"
+    spec:
+      containers:
+      - name: etcd
+        # ARCH will be replaced with the architecture it's built for. Check out the Makefile for more details
+        image: gcr.io/google_containers/etcd-ARCH:2.2.5
+        resources:
+          # TODO: Set memory limits when we've profiled the container for large
+          # clusters, then set request = limit to keep this container in
+          # guaranteed class. Currently, this container falls into the
+          # "burstable" category so the kubelet doesn't backoff from restarting it.
+          limits:
+            cpu: 100m
+            memory: 500Mi
+          requests:
+            cpu: 100m
+            memory: 50Mi
+        command:
+        - /usr/local/bin/etcd
+        - -data-dir
+        - /var/etcd/data
+        - -listen-client-urls
+        - http://127.0.0.1:2379,http://127.0.0.1:4001
+        - -advertise-client-urls
+        - http://127.0.0.1:2379,http://127.0.0.1:4001
+        - -initial-cluster-token
+        - skydns-etcd
+        volumeMounts:
+        - name: etcd-storage
+          mountPath: /var/etcd/data
+      - name: kube2sky
+        image: gcr.io/google_containers/kube2sky-ARCH:1.15
+        resources:
+          # TODO: Set memory limits when we've profiled the container for large
+          # clusters, then set request = limit to keep this container in
+          # guaranteed class. Currently, this container falls into the
+          # "burstable" category so the kubelet doesn't backoff from restarting it.
+          limits:
+            cpu: 100m
+            # Kube2sky watches all pods.
+            memory: 200Mi
+          requests:
+            cpu: 100m
+            memory: 50Mi
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 8080
+            scheme: HTTP
+          initialDelaySeconds: 60
+          timeoutSeconds: 5
+          successThreshold: 1
+          failureThreshold: 5
+        readinessProbe:
+          httpGet:
+            path: /readiness
+            port: 8081
+            scheme: HTTP
+          # we poll on pod startup for the Kubernetes master service and
+          # only setup the /readiness HTTP server once that's available.
+          initialDelaySeconds: 30
+          timeoutSeconds: 5
+        args:
+        # command = "/kube2sky"
+        - --domain=cluster.local
+      - name: skydns
+        image: gcr.io/google_containers/skydns-ARCH:1.0
+        resources:
+          # TODO: Set memory limits when we've profiled the container for large
+          # clusters, then set request = limit to keep this container in
+          # guaranteed class. Currently, this container falls into the
+          # "burstable" category so the kubelet doesn't backoff from restarting it.
+          limits:
+            cpu: 100m
+            memory: 200Mi
+          requests:
+            cpu: 100m
+            memory: 50Mi
+        args:
+        - -machines=http://127.0.0.1:4001
+        - -addr=0.0.0.0:53
+        - -ns-rotate=false
+        - -domain=cluster.local.
+        ports:
+        - containerPort: 53
+          name: dns
+          protocol: UDP
+        - containerPort: 53
+          name: dns-tcp
+          protocol: TCP
+      - name: healthz
+        image: gcr.io/google_containers/exechealthz-ARCH:1.0
+        resources:
+          # keep request = limit to keep this container in guaranteed class
+          limits:
+            cpu: 10m
+            memory: 20Mi
+          requests:
+            cpu: 10m
+            memory: 20Mi
+        args:
+        - -cmd=nslookup kubernetes.default.svc.cluster.local 127.0.0.1 >/dev/null
+        - -port=8080
+        ports:
+        - containerPort: 8080
+          protocol: TCP
+      volumes:
+      - name: etcd-storage
+        emptyDir: {}
+      dnsPolicy: Default  # Don't use cluster DNS.

cluster/images/hyperkube/addons/dns-svc.yaml

+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file should be kept in sync with cluster/addons/dns/skydns-svc.yaml.in
+apiVersion: v1
+kind: Service
+metadata:
+  name: kube-dns
+  namespace: kube-system
+  labels:
+    k8s-app: kube-dns
+    kubernetes.io/cluster-service: "true"
+    kubernetes.io/name: "KubeDNS"
+spec:
+  selector:
+    k8s-app: kube-dns
+  clusterIP: 10.0.0.10
+  ports:
+  - name: dns
+    port: 53
+    protocol: UDP
+  - name: dns-tcp
+    port: 53
+    protocol: TCP

cluster/images/hyperkube/addons/kube-proxy.yaml

+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: k8s-proxy
+  namespace: kube-system
+  labels:
+    k8s-app: k8s-proxy
+    version: v1
+    kubernetes.io/cluster-service: "true"
+spec:
+  template:
+    metadata:
+      labels:
+        k8s-app: k8s-proxy
+        version: v1
+        kubernetes.io/cluster-service: "true"
+    spec:
+      hostNetwork: true
+      containers:
+      - name: kube-proxy
+        image: gcr.io/google_containers/hyperkube-ARCH:VERSION
+        command:
+        - /hyperkube
+        - proxy
+        - --master=http://127.0.0.1:8080
+        - --v=2
+        - --resource-container=""
+        securityContext:
+          privileged: true

cluster/images/hyperkube/copy-addons.sh

+#!/bin/bash
+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Now we're running in the sidecar container
+# /etc/kubernetes/addons holds the data in the hyperkube container
+# /srv/kubernetes is an emptyDir that maps to /etc/kubernetes in the addon-manager container
+# This way we're using the latest manifests from hyperkube without updating
+# kube-addon-manager which is used for other deployments too
+
+# While there is no data copied over to the emptyDir, try to copy it.
+while [[ ! -d /srv/kubernetes/addons ]]; do
+	cp -r /etc/kubernetes/* /srv/kubernetes/
+done
+
+# Then sleep forever
+while true; do
+	sleep 3600;
+done

cluster/images/hyperkube/kube-proxy.json

-{
-  "apiVersion": "v1",
-  "kind": "Pod",
-  "metadata": {
-    "name": "k8s-proxy",
-    "namespace": "kube-system"
-  },
-  "spec": {
-    "hostNetwork": true,
-    "containers": [
-      {
-        "name": "kube-proxy",
-        "image": "gcr.io/google_containers/hyperkube-ARCH:VERSION",
-        "command": [
-                "/hyperkube",
-                "proxy",
-                "--master=http://127.0.0.1:8080",
-                "--v=2",
-                "--resource-container=\"\""
-        ],
-        "securityContext": {
-          "privileged": true
-        }
-      }
-    ]
-  }
-}

cluster/images/hyperkube/setup-files.sh

 #!/bin/bash
-
 # Copyright 2015 The Kubernetes Authors All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

cluster/images/hyperkube/static-pods/addon-manager.json

+{
+  "apiVersion": "v1",
+  "kind": "Pod",
+  "metadata": {
+    "name": "kube-addon-manager",
+    "namespace": "kube-system",
+    "version": "v1"
+  },
+  "spec": {
+    "hostNetwork": true,
+    "containers": [
+      {
+        "name": "kube-addon-manager",
+        "image": "gcr.io/google-containers/kube-addon-manager-ARCH:v2",
+        "resources": {
+          "requests": {
+            "cpu": "5m",
+            "memory": "50Mi"
+          }
+        },
+        "volumeMounts": [
+          {
+            "name": "addons",
+            "mountPath": "/etc/kubernetes/",
+            "readOnly": true
+          }
+        ]
+      },
+      {
+        "name": "kube-addon-manager-data",
+        "image": "gcr.io/google_containers/hyperkube-ARCH:VERSION",
+        "command": [
+          "/copy-addons.sh"
+        ],
+        "volumeMounts": [
+          {
+            "name": "addons",
+            "mountPath": "/srv/kubernetes/",
+            "readOnly": false
+          }
+        ]
+      }
+    ],
+    "volumes":[
+      {
+        "name": "addons",
+        "emptyDir": {}
+      }
+    ]
+  }
+}

cluster/images/hyperkube/master-multi.json → cluster/images/hyperkube/static-pods/master-multi.json

@@ -36,7 +36,7 @@
               "--service-cluster-ip-range=10.0.0.1/24",
               "--insecure-bind-address=0.0.0.0",
               "--etcd-servers=http://127.0.0.1:4001",
-              "--admission-control=NamespaceLifecycle,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota",
+              "--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota",
               "--client-ca-file=/srv/kubernetes/ca.crt",
               "--basic-auth-file=/srv/kubernetes/basic_auth.csv",
               "--min-request-timeout=300",
@@ -44,7 +44,7 @@
               "--tls-private-key-file=/srv/kubernetes/server.key",
               "--token-auth-file=/srv/kubernetes/known_tokens.csv",
               "--allow-privileged=true",
-              "--v=4"
+              "--v=2"
       ],
       "volumeMounts": [
         {

cluster/images/hyperkube/master.json → cluster/images/hyperkube/static-pods/master.json

@@ -36,7 +36,7 @@
               "--service-cluster-ip-range=10.0.0.1/24",
               "--insecure-bind-address=127.0.0.1",
               "--etcd-servers=http://127.0.0.1:4001",
-              "--admission-control=NamespaceLifecycle,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota",
+              "--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota",
               "--client-ca-file=/srv/kubernetes/ca.crt",
               "--basic-auth-file=/srv/kubernetes/basic_auth.csv",
               "--min-request-timeout=300",
@@ -44,7 +44,7 @@
               "--tls-private-key-file=/srv/kubernetes/server.key",
               "--token-auth-file=/srv/kubernetes/known_tokens.csv",
               "--allow-privileged=true",
-              "--v=4"
+              "--v=2"
       ],
       "volumeMounts": [
         {

cluster/images/hyperkube/teardown.sh

-#!/bin/bash
-
-# Copyright 2015 The Kubernetes Authors All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Tears down an existing cluster.  Warning destroys _all_ docker containers on the machine
-
-set -o errexit
-set -o nounset
-set -o pipefail
-
-echo "Warning, this will delete all Docker containers on this machine."
-echo "Proceed? [Y/n]"
-
-read resp
-if [[ $resp == "n" || $resp == "N" ]]; then
-  exit 0
-fi
-
-docker ps -aq | xargs docker rm -f

cluster/images/hyperkube/turnup.sh

-#!/bin/bash
-
-# Copyright 2015 The Kubernetes Authors All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Useful for testing images and changes, turns up a fresh single node cluster
-
-set -o errexit
-set -o nounset
-set -o pipefail
-
-K8S_VERSION=${K8S_VERSION:-"1.2.0"}
-
-docker run \
-  --volume=/:/rootfs:ro \
-  --volume=/sys:/sys:ro \
-  --volume=/var/lib/docker/:/var/lib/docker:rw \
-  --volume=/var/lib/kubelet/:/var/lib/kubelet:rw \
-  --volume=/var/run:/var/run:rw \
-  --net=host \
-  --pid=host \
-  --privileged=true \
-  -d gcr.io/google_containers/hyperkube-amd64:v${K8S_VERSION} \
-  /hyperkube kubelet \
-    --containerized \
-    --hostname-override="127.0.0.1" \
-    --address="0.0.0.0" \
-    --api-servers=http://localhost:8080 \
-    --config=/etc/kubernetes/manifests \
-    --cluster-dns=10.0.0.10 \
-    --cluster-domain=cluster.local \
-    --allow-privileged=true --v=2
-
-until $(kubectl cluster-info &> /dev/null); do
-    sleep 1
-done
-
-kubectl create ns kube-system