Run etcd for events on separate machine in Kubemark

test/kubemark/common.sh

+#!/bin/bash
+
+# Copyright 2015 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+source "${KUBE_ROOT}/cluster/kubemark/util.sh"
+source "${KUBE_ROOT}/cluster/kubemark/config-default.sh"
+
+detect-project &> /dev/null
+export PROJECT
+
+MASTER_NAME="${INSTANCE_PREFIX}-kubemark-master"
+MASTER_TAG="kubemark-master"
+EVENT_STORE_NAME="${INSTANCE_PREFIX}-event-store"

test/kubemark/start-kubemark-master.sh

@@ -15,18 +15,23 @@
 # limitations under the License.
 
 # TODO: figure out how to get etcd tag from some real configuration and put it here.
+
+EVENT_STORE_IP=$1
+EVENT_STORE_URL="http://${EVENT_STORE_IP}:4002"
+if [ "${EVENT_STORE_IP}" == "127.0.0.1" ]; then
+	sudo docker run --net=host -d gcr.io/google_containers/etcd:2.0.12 /usr/local/bin/etcd \
+		--listen-peer-urls http://127.0.0.1:2381 \
+		--addr=127.0.0.1:4002 \
+		--bind-addr=0.0.0.0:4002 \
+		--data-dir=/var/etcd/data
+fi
+
 sudo docker run --net=host -d gcr.io/google_containers/etcd:2.0.12 /usr/local/bin/etcd \
 	--listen-peer-urls http://127.0.0.1:2380 \
 	--addr=127.0.0.1:4001 \
 	--bind-addr=0.0.0.0:4001 \
 	--data-dir=/var/etcd/data
 
-sudo docker run --net=host -d gcr.io/google_containers/etcd:2.0.12 /usr/local/bin/etcd \
-	--listen-peer-urls http://127.0.0.1:2381 \
-	--addr=127.0.0.1:4002 \
-	--bind-addr=0.0.0.0:4002 \
-	--data-dir=/var/etcd/data_events
-
 # Increase the allowed number of open file descriptors
 ulimit -n 65536
 
@@ -38,7 +43,7 @@ kubernetes/server/bin/kube-apiserver \
 	--portal-net=10.0.0.1/24 \
 	--address=0.0.0.0 \
 	--etcd-servers=http://127.0.0.1:4001 \
-	--etcd-servers-overrides=/events#http://127.0.0.1:4002 \
+	--etcd-servers-overrides=/events#${EVENT_STORE_URL} \
 	--cluster-name=hollow-kubernetes \
 	--v=4 \
 	--tls-cert-file=/srv/kubernetes/server.cert \

test/kubemark/start-kubemark.sh

@@ -18,11 +18,7 @@
 
 KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
 
-source "${KUBE_ROOT}/cluster/kubemark/config-default.sh"
-source "${KUBE_ROOT}/cluster/kubemark/util.sh"
-
-detect-project &> /dev/null
-export PROJECT
+source "${KUBE_ROOT}/test/kubemark/common.sh"
 
 RUN_FROM_DISTRO=${RUN_FROM_DISTRO:-false}
 MAKE_DIR="${KUBE_ROOT}/cluster/images/kubemark"
@@ -43,18 +39,15 @@ make
 rm kubemark
 cd $CURR_DIR
 
-MASTER_NAME="${INSTANCE_PREFIX}-kubemark-master"
-MASTER_TAG="kubemark-master"
+GCLOUD_COMMON_ARGS="--project ${PROJECT} --zone ${ZONE}"
 
 gcloud compute disks create "${MASTER_NAME}-pd" \
-    --project "${PROJECT}" \
-    --zone "${ZONE}" \
+    ${GCLOUD_COMMON_ARGS} \
     --type "${MASTER_DISK_TYPE}" \
     --size "${MASTER_DISK_SIZE}"
 
 gcloud compute instances create "${MASTER_NAME}" \
-    --project "${PROJECT}" \
-    --zone "${ZONE}" \
+    ${GCLOUD_COMMON_ARGS} \
     --machine-type "${MASTER_SIZE}" \
     --image-project="${MASTER_IMAGE_PROJECT}" \
     --image "${MASTER_IMAGE}" \
@@ -68,11 +61,43 @@ gcloud compute firewall-rules create "${INSTANCE_PREFIX}-kubemark-master-https"
     --network "${NETWORK}" \
     --source-ranges "0.0.0.0/0" \
     --target-tags "${MASTER_TAG}" \
-    --allow "tcp:443" || true
+    --allow "tcp:443"
 
 MASTER_IP=$(gcloud compute instances describe ${MASTER_NAME} \
   --zone="${ZONE}" --project="${PROJECT}" | grep natIP: | cut -f2 -d":" | sed "s/ //g")
 
+if [ "${SEPARATE_EVENT_MACHINE:-false}" == "true" ]; then
+  EVENT_STORE_NAME="${INSTANCE_PREFIX}-event-store"
+  gcloud compute disks create "${EVENT_STORE_NAME}-pd" \
+      ${GCLOUD_COMMON_ARGS} \
+      --type "${MASTER_DISK_TYPE}" \
+      --size "${MASTER_DISK_SIZE}"
+
+  gcloud compute instances create "${EVENT_STORE_NAME}" \
+      ${GCLOUD_COMMON_ARGS} \
+      --machine-type "${MASTER_SIZE}" \
+      --image-project="${MASTER_IMAGE_PROJECT}" \
+      --image "${MASTER_IMAGE}" \
+      --tags "${EVENT_STORE_NAME}" \
+      --network "${NETWORK}" \
+      --scopes "storage-ro,compute-rw,logging-write" \
+      --disk "name=${EVENT_STORE_NAME}-pd,device-name=master-pd,mode=rw,boot=no,auto-delete=no"
+
+  EVENT_STORE_IP=$(gcloud compute instances describe ${EVENT_STORE_NAME} \
+  --zone="${ZONE}" --project="${PROJECT}" | grep networkIP: | cut -f2 -d":" | sed "s/ //g")
+
+  until gcloud compute ssh --zone="${ZONE}" --project="${PROJECT}" "${EVENT_STORE_NAME}" --command="ls" &> /dev/null; do
+    sleep 1
+  done
+
+  gcloud compute ssh ${EVENT_STORE_NAME} --zone=${ZONE} --project="${PROJECT}" \
+    --command="sudo docker run --net=host -d gcr.io/google_containers/etcd:2.0.12 /usr/local/bin/etcd \
+      --listen-peer-urls http://127.0.0.1:2380 \
+      --addr=127.0.0.1:4002 \
+      --bind-addr=0.0.0.0:4002 \
+      --data-dir=/var/etcd/data"
+fi
+
 ensure-temp-dir
 gen-kube-bearertoken
 create-certs ${MASTER_IP}
@@ -83,7 +108,7 @@ echo "${CA_CERT_BASE64}" | base64 -d > ca.crt
 echo "${KUBECFG_CERT_BASE64}" | base64 -d > kubecfg.crt
 echo "${KUBECFG_KEY_BASE64}" | base64 -d > kubecfg.key
 
-until gcloud compute ssh --zone="${ZONE}" --project="${PROJECT}" ${MASTER_NAME} --command="ls" &> /dev/null; do
+until gcloud compute ssh --zone="${ZONE}" --project="${PROJECT}" "${MASTER_NAME}" --command="ls" &> /dev/null; do
   sleep 1
 done
 
@@ -114,7 +139,7 @@ else
 fi
 
 gcloud compute ssh ${MASTER_NAME} --zone=${ZONE} --project="${PROJECT}" \
-  --command="chmod a+x configure-kubectl.sh && chmod a+x start-kubemark-master.sh && sudo ./start-kubemark-master.sh"
+  --command="chmod a+x configure-kubectl.sh && chmod a+x start-kubemark-master.sh && sudo ./start-kubemark-master.sh ${EVENT_STORE_IP:-127.0.0.1}"
 
 # create kubeconfig for Kubelet:
 KUBECONFIG_CONTENTS=$(echo "apiVersion: v1

test/kubemark/stop-kubemark.sh

@@ -17,29 +17,29 @@
 # Script that destroys Kubemark clusters and deletes all GCE resources created for Master
 KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
 
-source "${KUBE_ROOT}/cluster/kubemark/config-default.sh"
-source "${KUBE_ROOT}/cluster/kubemark/util.sh"
-source "${KUBE_ROOT}/cluster/kube-env.sh"
-
-detect-project &> /dev/null
-
-MASTER_NAME="${INSTANCE_PREFIX}-kubemark-master"
+source "${KUBE_ROOT}/test/kubemark/common.sh"
 
 kubectl delete -f ${KUBE_ROOT}/test/kubemark/hollow-kubelet.json &> /dev/null || true
 kubectl delete -f ${KUBE_ROOT}/test/kubemark/kubemark-ns.json &> /dev/null || true
 
+GCLOUD_COMMON_ARGS="--project ${PROJECT} --zone ${ZONE} --quiet"
+
 gcloud compute instances delete "${MASTER_NAME}" \
-    --project "${PROJECT}" \
-    --quiet \
-    --zone "${ZONE}" || true
+    ${GCLOUD_COMMON_ARGS} || true
 
-gcloud compute disks delete "${MASTER_NAME}"-pd \
-      --project "${PROJECT}" \
-      --quiet \
-      --zone "${ZONE}" || true
+gcloud compute disks delete "${MASTER_NAME}-pd" \
+    ${GCLOUD_COMMON_ARGS} || true
 
 gcloud compute firewall-rules delete "${INSTANCE_PREFIX}-kubemark-master-https" \
-	  --project "${PROJECT}" \
-	  --quiet || true
+	--project "${PROJECT}" \
+	--quiet || true
+
+if [ "${SEPARATE_EVENT_MACHINE:-false}" == "true" ]; then
+	gcloud compute instances delete "${EVENT_STORE_NAME}" \
+    	${GCLOUD_COMMON_ARGS} || true
+
+	gcloud compute disks delete "${EVENT_STORE_NAME}-pd" \
+    	${GCLOUD_COMMON_ARGS} || true
+fi
 
 rm -rf "${KUBE_ROOT}/test/kubemark/kubeconfig.loc" &> /dev/null || true