On kubemark master, kubelet now runs as a supervisord process and all master components as pods

test/kubemark/resources/heapster_template.json

@@ -37,14 +37,14 @@
 					"resources": {
 						"requests": {
 							"cpu": "100m",
-							"memory": "##METRICS_MEM##Mi"
+							"memory": "{{METRICS_MEM}}Mi"
 						}
 					},
 					"command": [
 						"/heapster"
 					],
 					"args": [
-						"--source=kubernetes:https://##MASTER_IP##:443?inClusterConfig=0&useServiceAccount=0&auth=/kubeconfig/kubeconfig"
+						"--source=kubernetes:https://{{MASTER_IP}}:443?inClusterConfig=0&useServiceAccount=0&auth=/kubeconfig/kubeconfig"
 					],
 					"volumeMounts": [
 						{
@@ -59,7 +59,7 @@
 					"resources": {
 						"requests": {
 							"cpu": "100m",
-							"memory": "##EVENTER_MEM##Ki"
+							"memory": "{{EVENTER_MEM}}Ki"
 						}
 					},
 					"command": [

test/kubemark/resources/hollow-node_template.json

@@ -8,7 +8,7 @@
 		}
 	},
 	"spec": {
-		"replicas": ##numreplicas##,
+		"replicas": {{numreplicas}},
 		"selector": {
 			"name": "hollow-node"
 		},
@@ -36,7 +36,7 @@
 				"containers": [
 				{
 					"name": "hollow-kubelet",
-					"image": "gcr.io/##project##/kubemark:latest",
+					"image": "gcr.io/{{project}}/kubemark:latest",
 					"ports": [
 						{"containerPort": 4194},
 						{"containerPort": 10250},
@@ -89,7 +89,7 @@
 				},
 				{
 					"name": "hollow-proxy",
-					"image": "gcr.io/##project##/kubemark:latest",
+					"image": "gcr.io/{{project}}/kubemark:latest",
 					"env": [
 						{
 							"name": "CONTENT_TYPE",

test/kubemark/resources/manifests/etcd-events.yaml

+apiVersion: v1
+kind: Pod
+metadata:
+  name: etcd-server-events
+  namespace: kube-system
+spec:
+  hostNetwork: true
+  nodeName: {{instance_prefix}}-master
+  containers:
+  - name: etcd-container
+    image: {{kube_docker_registry}}/etcd:{{etcd_image}}
+    imagePullPolicy: IfNotPresent
+    resources:
+      requests:
+        cpu: 100m
+    command:
+    - /bin/sh
+    - -c
+    - /usr/local/bin/etcd
+      {{params}}
+      1>>/var/log/etcd-events.log 2>&1
+    livenessProbe:
+      httpGet:
+        host: 127.0.0.1
+        path: /health
+        port: 4002
+        scheme: HTTP
+      initialDelaySeconds: 15
+      timeoutSeconds: 15
+    ports:
+    - name: serverport
+      containerPort: 2381
+      hostPort: 2381
+      protocol: TCP
+    - name: clientport
+      containerPort: 4002
+      hostPort: 4002
+      protocol: TCP
+    volumeMounts:
+    - name: varetcd
+      mountPath: /var/etcd
+    - name: varlogetcd
+      mountPath: /var/log/etcd-events.log
+  volumes:
+  - name: varetcd
+    hostPath:
+      path: /var/etcd/events
+  - name: varlogetcd
+    hostPath:
+      path: /var/log/etcd-events.log

test/kubemark/resources/manifests/etcd.yaml

+apiVersion: v1
+kind: Pod
+metadata:
+  name: etcd-server
+  namespace: kube-system
+spec:
+  hostNetwork: true
+  containers:
+  - name: etcd-container
+    image: {{kube_docker_registry}}/etcd:{{etcd_image}}
+    imagePullPolicy: IfNotPresent
+    resources:
+      requests:
+        cpu: 200m
+    command:
+    - /bin/sh
+    - -c
+    - /usr/local/bin/etcd
+      {{params}}
+      1>>/var/log/etcd.log 2>&1
+    livenessProbe:
+      httpGet:
+        host: 127.0.0.1
+        path: /health
+        port: 2379
+        scheme: HTTP
+      initialDelaySeconds: 15
+      timeoutSeconds: 15
+    ports:
+    - name: serverport
+      containerPort: 2380
+      hostPort: 2380
+      protocol: TCP
+    - name: clientport
+      containerPort: 2379
+      hostPort: 2379
+      protocol: TCP
+    volumeMounts:
+    - name: varetcd
+      mountPath: /var/etcd
+    - name: varlogetcd
+      mountPath: /var/log/etcd.log
+  volumes:
+  - name: varetcd
+    hostPath:
+      path: /var/etcd
+  - name: varlogetcd
+    hostPath:
+      path: /var/log/etcd.log

test/kubemark/resources/manifests/kube-apiserver.yaml

+apiVersion: v1
+kind: Pod
+metadata:
+  name: kube-apiserver
+  namespace: kube-system
+spec:
+  hostNetwork: true
+  containers:
+  - name: kube-apiserver
+    image: {{kube_docker_registry}}/kube-apiserver:{{kube-apiserver_docker_tag}}
+    imagePullPolicy: IfNotPresent
+    resources:
+      requests:
+        cpu: 250m
+    command:
+    - /bin/sh
+    - -c
+    - /usr/local/bin/kube-apiserver
+      {{params}}
+      1>>/var/log/kube-apiserver.log 2>&1
+    livenessProbe:
+      httpGet:
+        host: 127.0.0.1
+        path: /healthz
+        port: 8080
+        scheme: HTTP
+      initialDelaySeconds: 15
+      timeoutSeconds: 15
+    ports:
+    - name: https
+      containerPort: 443
+      hostPort: 443
+      protocol: TCP
+    - name: local
+      containerPort: 8080
+      hostPort: 8080
+      protocol: TCP
+    volumeMounts:
+    - name: srvkube
+      mountPath: /srv/kubernetes
+      readOnly: true
+    - name: logfile
+      mountPath: /var/log/kube-apiserver.log
+    - name: etcssl
+      mountPath: /etc/ssl
+      readOnly: true
+    - name: usrsharecacerts
+      mountPath: /usr/share/ca-certificates
+      readOnly: true
+    - name: srvsshproxy
+      mountPath: /srv/sshproxy
+  volumes:
+  - name: srvkube
+    hostPath:
+      path: /srv/kubernetes
+  - name: logfile
+    hostPath:
+      path: /var/log/kube-apiserver.log
+  - name: etcssl
+    hostPath:
+      path: /etc/ssl
+  - name: usrsharecacerts
+    hostPath:
+      path: /usr/share/ca-certificates
+  - name: srvsshproxy
+    hostPath:
+      path: /srv/sshproxy

test/kubemark/resources/manifests/kube-controller-manager.yaml

+apiVersion: v1
+kind: Pod
+metadata:
+  name: kube-controller-manager
+  namespace: kube-system
+spec:
+  hostNetwork: true
+  containers:
+  - name: kube-controller-manager
+    image: {{kube_docker_registry}}/kube-controller-manager:{{kube-controller-manager_docker_tag}}
+    imagePullPolicy: IfNotPresent
+    resources:
+      requests:
+        cpu: 200m
+    command:
+    - /bin/sh
+    - -c
+    - /usr/local/bin/kube-controller-manager
+      {{params}}
+      1>>/var/log/kube-controller-manager.log 2>&1
+    livenessProbe:
+      httpGet:
+        host: 127.0.0.1
+        path: /healthz
+        port: 10252
+        scheme: HTTP
+      initialDelaySeconds: 15
+      timeoutSeconds: 15
+    volumeMounts:
+    - name: srvkube
+      mountPath: /srv/kubernetes
+      readOnly: true
+    - name: logfile
+      mountPath: /var/log/kube-controller-manager.log
+    - name: etcssl
+      mountPath: /etc/ssl
+      readOnly: true
+    - name: usrsharecacerts
+      mountPath: /usr/share/ca-certificates
+      readOnly: true
+  volumes:
+  - name: srvkube
+    hostPath:
+      path: /srv/kubernetes
+  - name: logfile
+    hostPath:
+      path: /var/log/kube-controller-manager.log
+  - name: etcssl
+    hostPath:
+      path: /etc/ssl
+  - name: usrsharecacerts
+    hostPath:
+      path: /usr/share/ca-certificates

test/kubemark/resources/manifests/kube-scheduler.yaml

+apiVersion: v1
+kind: Pod
+metadata:
+  name: kube-scheduler
+  namespace: kube-system
+spec:
+  hostNetwork: true
+  nodeName: {{instance_prefix}}-master
+  containers:
+  - name: kube-scheduler
+    image: {{kube_docker_registry}}/kube-scheduler:{{kube-scheduler_docker_tag}}
+    imagePullPolicy: IfNotPresent
+    resources:
+      requests:
+        cpu: 100m
+    command:
+    - /bin/sh
+    - -c
+    - /usr/local/bin/kube-scheduler
+      {{params}}
+      1>>/var/log/kube-scheduler.log 2>&1
+    livenessProbe:
+      httpGet:
+        host: 127.0.0.1
+        path: /healthz
+        port: 10251
+        scheme: HTTP
+      initialDelaySeconds: 15
+      timeoutSeconds: 15
+    volumeMounts:
+    - name: logfile
+      mountPath: /var/log/kube-scheduler.log
+  volumes:
+  - name: logfile
+    hostPath:
+      path: /var/log/kube-scheduler.log

test/kubemark/start-kubemark.sh

@@ -22,36 +22,38 @@ KUBE_ROOT=$(readlink -e ${TMP_ROOT} 2> /dev/null || perl -MCwd -e 'print Cwd::ab
 
 source "${KUBE_ROOT}/test/kubemark/common.sh"
 
-function writeEnvironmentFiles() {
-  cat > "${RESOURCE_DIRECTORY}/apiserver_flags" <<EOF
-${APISERVER_TEST_ARGS}
---storage-backend=${STORAGE_BACKEND}
---service-cluster-ip-range="${SERVICE_CLUSTER_IP_RANGE}"
+function writeEnvironmentFile() {
+  cat > "${RESOURCE_DIRECTORY}/kubemark-master-env.sh" <<EOF
+# Generic variables.
+INSTANCE_PREFIX=${INSTANCE_PREFIX}
+SERVICE_CLUSTER_IP_RANGE=${SERVICE_CLUSTER_IP_RANGE}
+
+# Etcd related variables.
+ETCD_IMAGE=2.2.1
+
+# Controller-manager related variables.
+CONTROLLER_MANAGER_TEST_ARGS="${CONTROLLER_MANAGER_TEST_ARGS}"
+ALLOCATE_NODE_CIDRS=${ALLOCATE_NODE_CIDRS}
+CLUSTER_IP_RANGE=${CLUSTER_IP_RANGE}
+TERMINATED_POD_GC_THRESHOLD=${TERMINATED_POD_GC_THRESHOLD}
+
+# Scheduler related variables.
+SCHEDULER_TEST_ARGS="${SCHEDULER_TEST_ARGS}"
+
+# Apiserver related variables.
+APISERVER_TEST_ARGS="${APISERVER_TEST_ARGS}"
+STORAGE_BACKEND=${STORAGE_BACKEND}
+NUM_NODES=${NUM_NODES}
 EOF
 if [ -z "${CUSTOM_ADMISSION_PLUGINS:-}" ]; then
- cat >> "${RESOURCE_DIRECTORY}/apiserver_flags" <<EOF
---admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota
+  cat >> "${RESOURCE_DIRECTORY}/kubemark-master-env.sh" <<EOF
+CUSTOM_ADMISSION_PLUGINS=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota
 EOF
 else
-  cat >> "${RESOURCE_DIRECTORY}/apiserver_flags" <<EOF
---admission-control=${CUSTOM_ADMISSION_PLUGINS}
+  cat >> "${RESOURCE_DIRECTORY}/kubemark-master-env.sh" <<EOF
+CUSTOM_ADMISSION_PLUGINS=${CUSTOM_ADMISSION_PLUGINS}
 EOF
 fi
-sed -i'' -e "s/\"//g" "${RESOURCE_DIRECTORY}/apiserver_flags"
-
-  cat > "${RESOURCE_DIRECTORY}/scheduler_flags" <<EOF
-${SCHEDULER_TEST_ARGS}
-EOF
-sed -i'' -e "s/\"//g" "${RESOURCE_DIRECTORY}/scheduler_flags"
-
-  cat > "${RESOURCE_DIRECTORY}/controllers_flags" <<EOF
-${CONTROLLER_MANAGER_TEST_ARGS}
---allocate-node-cidrs="${ALLOCATE_NODE_CIDRS}"
---cluster-cidr="${CLUSTER_IP_RANGE}"
---service-cluster-ip-range="${SERVICE_CLUSTER_IP_RANGE}"
---terminated-pod-gc-threshold="${TERMINATED_POD_GC_THRESHOLD}"
-EOF
-sed -i'' -e "s/\"//g" "${RESOURCE_DIRECTORY}/controllers_flags"
 }
 
 MAKE_DIR="${KUBE_ROOT}/cluster/images/kubemark"
@@ -148,7 +150,7 @@ done
 password=$(python -c 'import string,random; print("".join(random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(16)))')
 
 gcloud compute ssh --zone="${ZONE}" --project="${PROJECT}" "${MASTER_NAME}" \
-  --command="sudo mkdir /srv/kubernetes -p && \
+  --command="sudo mkdir /home/kubernetes -p && sudo mkdir /srv/kubernetes -p && \
     sudo bash -c \"echo ${MASTER_CERT_BASE64} | base64 --decode > /srv/kubernetes/server.cert\" && \
     sudo bash -c \"echo ${MASTER_KEY_BASE64} | base64 --decode > /srv/kubernetes/server.key\" && \
     sudo bash -c \"echo ${CA_CERT_BASE64} | base64 --decode > /srv/kubernetes/ca.crt\" && \
@@ -159,21 +161,24 @@ gcloud compute ssh --zone="${ZONE}" --project="${PROJECT}" "${MASTER_NAME}" \
     sudo bash -c \"echo \"${KUBE_PROXY_TOKEN},kube_proxy,kube_proxy\" >> /srv/kubernetes/known_tokens.csv\" && \
     sudo bash -c \"echo ${password},admin,admin > /srv/kubernetes/basic_auth.csv\""
 
-writeEnvironmentFiles
+writeEnvironmentFile
 
 gcloud compute copy-files --zone="${ZONE}" --project="${PROJECT}" \
   "${SERVER_BINARY_TAR}" \
+  "${RESOURCE_DIRECTORY}/kubemark-master-env.sh" \
   "${RESOURCE_DIRECTORY}/start-kubemark-master.sh" \
   "${KUBEMARK_DIRECTORY}/configure-kubectl.sh" \
-  "${RESOURCE_DIRECTORY}/apiserver_flags" \
-  "${RESOURCE_DIRECTORY}/scheduler_flags" \
-  "${RESOURCE_DIRECTORY}/controllers_flags" \
-  "root@${MASTER_NAME}":/
-
+  "${RESOURCE_DIRECTORY}/manifests/etcd.yaml" \
+  "${RESOURCE_DIRECTORY}/manifests/etcd-events.yaml" \
+  "${RESOURCE_DIRECTORY}/manifests/kube-apiserver.yaml" \
+  "${RESOURCE_DIRECTORY}/manifests/kube-scheduler.yaml" \
+  "${RESOURCE_DIRECTORY}/manifests/kube-controller-manager.yaml" \
+  "root@${MASTER_NAME}":/home/kubernetes/
 
 gcloud compute ssh "${MASTER_NAME}" --zone="${ZONE}" --project="${PROJECT}" \
-  --command="sudo chmod a+x /configure-kubectl.sh && sudo chmod a+x /start-kubemark-master.sh && \
-             sudo /start-kubemark-master.sh ${EVENT_STORE_IP:-127.0.0.1} ${NUM_NODES:-0} ${EVENT_PD:-false} ${ETCD_IMAGE:-}"
+  --command="sudo chmod a+x /home/kubernetes/configure-kubectl.sh && \
+    sudo chmod a+x /home/kubernetes/start-kubemark-master.sh && \
+    sudo /home/kubernetes/start-kubemark-master.sh"
 
 # create kubeconfig for Kubelet:
 KUBECONFIG_CONTENTS=$(echo "apiVersion: v1
@@ -248,18 +253,18 @@ contexts:
 current-context: kubemark-context
 EOF
 
-sed "s/##numreplicas##/${NUM_NODES:-10}/g" "${RESOURCE_DIRECTORY}/hollow-node_template.json" > "${RESOURCE_DIRECTORY}/hollow-node.json"
-sed -i'' -e "s/##project##/${PROJECT}/g" "${RESOURCE_DIRECTORY}/hollow-node.json"
+sed "s/{{numreplicas}}/${NUM_NODES:-10}/g" "${RESOURCE_DIRECTORY}/hollow-node_template.json" > "${RESOURCE_DIRECTORY}/hollow-node.json"
+sed -i'' -e "s/{{project}}/${PROJECT}/g" "${RESOURCE_DIRECTORY}/hollow-node.json"
 
 mkdir "${RESOURCE_DIRECTORY}/addons" || true
 
-sed "s/##MASTER_IP##/${MASTER_IP}/g" "${RESOURCE_DIRECTORY}/heapster_template.json" > "${RESOURCE_DIRECTORY}/addons/heapster.json"
+sed "s/{{MASTER_IP}}/${MASTER_IP}/g" "${RESOURCE_DIRECTORY}/heapster_template.json" > "${RESOURCE_DIRECTORY}/addons/heapster.json"
 metrics_mem_per_node=4
 metrics_mem=$((200 + ${metrics_mem_per_node}*${NUM_NODES:-10}))
-sed -i'' -e "s/##METRICS_MEM##/${metrics_mem}/g" "${RESOURCE_DIRECTORY}/addons/heapster.json"
+sed -i'' -e "s/{{METRICS_MEM}}/${metrics_mem}/g" "${RESOURCE_DIRECTORY}/addons/heapster.json"
 eventer_mem_per_node=500
 eventer_mem=$((200 * 1024 + ${eventer_mem_per_node}*${NUM_NODES:-10}))
-sed -i'' -e "s/##EVENTER_MEM##/${eventer_mem}/g" "${RESOURCE_DIRECTORY}/addons/heapster.json"
+sed -i'' -e "s/{{EVENTER_MEM}}/${eventer_mem}/g" "${RESOURCE_DIRECTORY}/addons/heapster.json"
 
 "${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/kubemark-ns.json"
 "${KUBECTL}" create -f "${KUBECONFIG_SECRET}" --namespace="kubemark"