Pipe variables from kubemark config to master binaries

cluster/kubemark/config-default.sh

@@ -31,7 +31,7 @@ REGISTER_MASTER_KUBELET=${REGISTER_MASTER:-false}
 PREEMPTIBLE_NODE=${PREEMPTIBLE_NODE:-false}
 
 OS_DISTRIBUTION=${KUBE_OS_DISTRIBUTION:-debian}
-MASTER_IMAGE=${KUBE_GCE_MASTER_IMAGE:-container-vm-v20151103}
+MASTER_IMAGE=${KUBE_GCE_MASTER_IMAGE:-container-vm-v20160321}
 MASTER_IMAGE_PROJECT=${KUBE_GCE_MASTER_PROJECT:-google-containers}
 
 NETWORK=${KUBE_GCE_NETWORK:-default}
@@ -39,12 +39,20 @@ INSTANCE_PREFIX="${INSTANCE_PREFIX:-"default"}"
 MASTER_NAME="${INSTANCE_PREFIX}-kubemark-master"
 MASTER_TAG="kubemark-master"
 MASTER_IP_RANGE="${MASTER_IP_RANGE:-10.246.0.0/24}"
-CLUSTER_IP_RANGE="${CLUSTER_IP_RANGE:-10.244.0.0/16}"
+CLUSTER_IP_RANGE="${CLUSTER_IP_RANGE:-10.240.0.0/13}"
 RUNTIME_CONFIG="${KUBE_RUNTIME_CONFIG:-}"
 TERMINATED_POD_GC_THRESHOLD=${TERMINATED_POD_GC_THRESHOLD:-100}
 
+# Default Log level for all components in test clusters and variables to override it in specific components.
 TEST_CLUSTER_LOG_LEVEL="${TEST_CLUSTER_LOG_LEVEL:---v=2}"
-TEST_CLUSTER_RESYNC_PERIOD="${TEST_CLUSTER_RESYNC_PERIOD:---min-resync-period=12h}"
+KUBELET_TEST_LOG_LEVEL="${KUBELET_TEST_LOG_LEVEL:-$TEST_CLUSTER_LOG_LEVEL}"
+API_SERVER_TEST_LOG_LEVEL="${API_SERVER_TEST_LOG_LEVEL:-$TEST_CLUSTER_LOG_LEVEL}"
+CONTROLLER_MANAGER_TEST_LOG_LEVEL="${CONTROLLER_MANAGER_TEST_LOG_LEVEL:-$TEST_CLUSTER_LOG_LEVEL}"
+SCHEDULER_TEST_LOG_LEVEL="${SCHEDULER_TEST_LOG_LEVEL:-$TEST_CLUSTER_LOG_LEVEL}"
+KUBEPROXY_TEST_LOG_LEVEL="${KUBEPROXY_TEST_LOG_LEVEL:-$TEST_CLUSTER_LOG_LEVEL}"
+
+TEST_CLUSTER_DELETE_COLLECTION_WORKERS="${TEST_CLUSTER_DELETE_COLLECTION_WORKERS:---delete-collection-workers=1}"
+TEST_CLUSTER_RESYNC_PERIOD="${TEST_CLUSTER_RESYNC_PERIOD:---min-resync-period=3m}"
 
 # ContentType used by all components to communicate with apiserver.
 TEST_CLUSTER_API_CONTENT_TYPE="${TEST_CLUSTER_API_CONTENT_TYPE:-}"
@@ -52,34 +60,10 @@ TEST_CLUSTER_API_CONTENT_TYPE="${TEST_CLUSTER_API_CONTENT_TYPE:-}"
 TEST_CLUSTER_STORAGE_CONTENT_TYPE="${TEST_CLUSTER_STORAGE_CONTENT_TYPE:-}"
 
 KUBELET_TEST_ARGS="--max-pods=100 $TEST_CLUSTER_LOG_LEVEL ${TEST_CLUSTER_API_CONTENT_TYPE}"
-APISERVER_TEST_ARGS="--runtime-config=extensions/v1beta1 ${TEST_CLUSTER_LOG_LEVEL} ${TEST_CLUSTER_STORAGE_CONTENT_TYPE}"
-CONTROLLER_MANAGER_TEST_ARGS="${TEST_CLUSTER_LOG_LEVEL} ${TEST_CLUSTER_RESYNC_PERIOD} ${TEST_CLUSTER_API_CONTENT_TYPE}"
-SCHEDULER_TEST_ARGS="${TEST_CLUSTER_LOG_LEVEL} ${TEST_CLUSTER_API_CONTENT_TYPE}"
-KUBEPROXY_TEST_ARGS="${TEST_CLUSTER_LOG_LEVEL} ${TEST_CLUSTER_API_CONTENT_TYPE}"
-
-# Extra docker options for nodes.
-EXTRA_DOCKER_OPTS="${EXTRA_DOCKER_OPTS:-}"
+APISERVER_TEST_ARGS="--runtime-config=extensions/v1beta1 ${API_SERVER_TEST_LOG_LEVEL} ${TEST_CLUSTER_STORAGE_CONTENT_TYPE} ${TEST_CLUSTER_DELETE_COLLECTION_WORKERS}"
+CONTROLLER_MANAGER_TEST_ARGS="${CONTROLLER_MANAGER_TEST_LOG_LEVEL} ${TEST_CLUSTER_RESYNC_PERIOD} ${TEST_CLUSTER_API_CONTENT_TYPE}"
+SCHEDULER_TEST_ARGS="${SCHEDULER_TEST_LOG_LEVEL} ${TEST_CLUSTER_API_CONTENT_TYPE}"
+KUBEPROXY_TEST_ARGS="${KUBEPROXY_TEST_LOG_LEVEL} ${TEST_CLUSTER_API_CONTENT_TYPE}"
 
-# Increase the sleep interval value if concerned about API rate limits. 3, in seconds, is the default.
-POLL_SLEEP_INTERVAL=3
 SERVICE_CLUSTER_IP_RANGE="10.0.0.0/16"  # formerly PORTAL_NET
 ALLOCATE_NODE_CIDRS=true
-
-ENABLE_CLUSTER_MONITORING="${KUBE_ENABLE_CLUSTER_MONITORING:-none}"
-ENABLE_NODE_LOGGING="${KUBE_ENABLE_NODE_LOGGING:-false}"
-ENABLE_CLUSTER_LOGGING="${KUBE_ENABLE_CLUSTER_LOGGING:-false}"
-# Optional: Don't require https for registries in our local RFC1918 network
-if [[ ${KUBE_ENABLE_INSECURE_REGISTRY:-false} == "true" ]]; then
-  EXTRA_DOCKER_OPTS="${EXTRA_DOCKER_OPTS} --insecure-registry 10.0.0.0/8"
-fi
-
-ENABLE_CLUSTER_DNS="${KUBE_ENABLE_CLUSTER_DNS:-false}"
-ENABLE_CLUSTER_REGISTRY="${KUBE_ENABLE_CLUSTER_REGISTRY:-false}"
-ENABLE_CLUSTER_UI="${KUBE_ENABLE_CLUSTER_UI:-false}"
-ENABLE_NODE_AUTOSCALER="${KUBE_ENABLE_NODE_AUTOSCALER:-false}"
-
-# Admission Controllers to invoke prior to persisting objects in cluster
-ADMISSION_CONTROL=NamespaceLifecycle,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota
-
-# Optional: if set to true kube-up will automatically check for existing resources and clean them up.
-KUBE_UP_AUTOMATIC_CLEANUP=${KUBE_UP_AUTOMATIC_CLEANUP:-false}

test/kubemark/start-kubemark-master.sh

@@ -37,22 +37,19 @@ ulimit -n 65536
 
 tar xzf kubernetes-server-linux-amd64.tar.gz
 
-kubernetes/server/bin/kube-scheduler --master=127.0.0.1:8080 --v=2 &> /var/log/kube-scheduler.log &
+kubernetes/server/bin/kube-scheduler --master=127.0.0.1:8080 $(cat scheduler_flags) &> /var/log/kube-scheduler.log &
 
 kubernetes/server/bin/kube-apiserver \
-	--portal-net=10.0.0.1/24 \
 	--address=0.0.0.0 \
 	--etcd-servers=http://127.0.0.1:4001 \
 	--etcd-servers-overrides=/events#${EVENT_STORE_URL} \
-	--v=4 \
 	--tls-cert-file=/srv/kubernetes/server.cert \
 	--tls-private-key-file=/srv/kubernetes/server.key \
 	--client-ca-file=/srv/kubernetes/ca.crt \
 	--token-auth-file=/srv/kubernetes/known_tokens.csv \
 	--secure-port=443 \
 	--basic-auth-file=/srv/kubernetes/basic_auth.csv \
-	--service-cluster-ip-range=10.0.0.0/16 \
-	--delete-collection-workers=16 &> /var/log/kube-apiserver.log &
+	$(cat apiserver_flags) &> /var/log/kube-apiserver.log &
 
 # kube-contoller-manager now needs running kube-api server to actually start
 until [ "$(curl 127.0.0.1:8080/healthz 2> /dev/null)" == "ok" ]; do
@@ -62,6 +59,4 @@ kubernetes/server/bin/kube-controller-manager \
   --master=127.0.0.1:8080 \
   --service-account-private-key-file=/srv/kubernetes/server.key \
   --root-ca-file=/srv/kubernetes/ca.crt \
-  --v=2 \
-  --allocate-node-cidrs=true \
-  --cluster-cidr=10.180.0.0/13 &> /var/log/kube-controller-manager.log &
+  $(cat controllers_flags) &> /var/log/kube-controller-manager.log &

test/kubemark/start-kubemark.sh

@@ -20,6 +20,27 @@ KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
 
 source "${KUBE_ROOT}/test/kubemark/common.sh"
 
+function writeEnvironmentFiles() {
+  cat > "${RESOURCE_DIRECTORY}/apiserver_flags" <<EOF
+${APISERVER_TEST_ARGS}
+--service-cluster-ip-range="${SERVICE_CLUSTER_IP_RANGE}"
+EOF
+sed -i'' -e "s/\"//g" "${RESOURCE_DIRECTORY}/apiserver_flags"
+
+  cat > "${RESOURCE_DIRECTORY}/scheduler_flags" <<EOF
+${SCHEDULER_TEST_ARGS}
+EOF
+sed -i'' -e "s/\"//g" "${RESOURCE_DIRECTORY}/scheduler_flags"
+
+  cat > "${RESOURCE_DIRECTORY}/controllers_flags" <<EOF
+${CONTROLLER_MANAGER_TEST_ARGS}
+--allocate-node-cidrs="${ALLOCATE_NODE_CIDRS}"
+--cluster-cidr="${CLUSTER_IP_RANGE}"
+--terminated-pod-gc-threshold="${TERMINATED_POD_GC_THRESHOLD}"
+EOF
+sed -i'' -e "s/\"//g" "${RESOURCE_DIRECTORY}/controllers_flags"
+}
+
 RUN_FROM_DISTRO=${RUN_FROM_DISTRO:-false}
 MAKE_DIR="${KUBE_ROOT}/cluster/images/kubemark"
 
@@ -126,17 +147,25 @@ gcloud compute ssh --zone="${ZONE}" --project="${PROJECT}" "${MASTER_NAME}" \
     sudo bash -c \"echo \"${KUBE_PROXY_TOKEN},kube_proxy,kube_proxy\" >> /srv/kubernetes/known_tokens.csv\" && \
     sudo bash -c \"echo ${password},admin,admin > /srv/kubernetes/basic_auth.csv\""
 
+writeEnvironmentFiles
+
 if [ "${RUN_FROM_DISTRO}" == "false" ]; then
   gcloud compute copy-files --zone="${ZONE}" --project="${PROJECT}" \
     "${KUBE_ROOT}/_output/release-tars/kubernetes-server-linux-amd64.tar.gz" \
     "${KUBEMARK_DIRECTORY}/start-kubemark-master.sh" \
     "${KUBEMARK_DIRECTORY}/configure-kubectl.sh" \
+    "${RESOURCE_DIRECTORY}/apiserver_flags" \
+    "${RESOURCE_DIRECTORY}/scheduler_flags" \
+    "${RESOURCE_DIRECTORY}/controllers_flags" \
     "${MASTER_NAME}":~
 else
   gcloud compute copy-files --zone="${ZONE}" --project="${PROJECT}" \
     "${KUBE_ROOT}/server/kubernetes-server-linux-amd64.tar.gz" \
     "${KUBEMARK_DIRECTORY}/start-kubemark-master.sh" \
     "${KUBEMARK_DIRECTORY}/configure-kubectl.sh" \
+    "${RESOURCE_DIRECTORY}/apiserver_flags" \
+    "${RESOURCE_DIRECTORY}/scheduler_flags" \
+    "${RESOURCE_DIRECTORY}/controllers_flags" \
     "${MASTER_NAME}":~
 fi
 

test/kubemark/stop-kubemark.sh

@@ -45,4 +45,10 @@ if [ "${SEPARATE_EVENT_MACHINE:-false}" == "true" ]; then
 fi
 
 rm -rf "${RESOURCE_DIRECTORY}/addons" "${RESOURCE_DIRECTORY}/kubeconfig.loc" &> /dev/null || true
-rm "${RESOURCE_DIRECTORY}/ca.crt" "${RESOURCE_DIRECTORY}/kubecfg.crt" "${RESOURCE_DIRECTORY}/kubecfg.key" "${RESOURCE_DIRECTORY}/hollow-node.json" &> /dev/null || true
+rm "${RESOURCE_DIRECTORY}/ca.crt" \
+	"${RESOURCE_DIRECTORY}/kubecfg.crt" \
+	"${RESOURCE_DIRECTORY}/kubecfg.key" \
+	"${RESOURCE_DIRECTORY}/hollow-node.json" \
+	"${RESOURCE_DIRECTORY}/apiserver_flags" \
+	"${RESOURCE_DIRECTORY}/controllers_flags" \
+	"${RESOURCE_DIRECTORY}/scheduler_flags" &> /dev/null || true