vsphere support

cluster/saltbase/salt/docker/docker.list

+deb https://apt.dockerproject.org/repo debian-{{ salt['grains.get']('oscodename') }} main

cluster/saltbase/salt/docker/init.sls

 {% if pillar.get('is_systemd') %}
-{% set environment_file = '/etc/sysconfig/docker' %}
+  {% set environment_file = '/etc/sysconfig/docker' %}
 {% else %}
-{% set environment_file = '/etc/default/docker' %}
+  {% set environment_file = '/etc/default/docker' %}
 {% endif %}
 
 bridge-utils:
@@ -47,6 +47,96 @@ docker:
       - pkg: docker-io
 
 {% endif %}
+{% elif grains.cloud is defined and grains.cloud == 'vsphere' and grains.os == 'Debian' and grains.osrelease_info[0] >=8 %}
+
+{% if pillar.get('is_systemd') %}
+
+{{ pillar.get('systemd_system_path') }}/docker.service:
+  file.managed:
+    - source: salt://docker/docker.service
+    - template: jinja
+    - user: root
+    - group: root
+    - mode: 644
+    - defaults:
+        environment_file: {{ environment_file }}
+
+# The docker service.running block below doesn't work reliably
+# Instead we run our script which e.g. does a systemd daemon-reload
+# But we keep the service block below, so it can be used by dependencies
+# TODO: Fix this
+fix-service-docker:
+  cmd.wait:
+    - name: /opt/kubernetes/helpers/services bounce docker
+    - watch:
+      - file: {{ pillar.get('systemd_system_path') }}/docker.service
+      - file: {{ environment_file }}
+{% endif %}
+
+{{ environment_file }}:
+  file.managed:
+    - source: salt://docker/docker-defaults
+    - template: jinja
+    - user: root
+    - group: root
+    - mode: 644
+    - makedirs: true
+    - require:
+      - pkg: docker-engine
+
+'apt-key':
+   cmd.run:
+     - name: 'apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D'
+     - unless: 'apt-key finger | grep "5811 8E89"'
+
+'apt-update':
+  cmd.wait:
+    - name: '/usr/bin/apt-get update -y'
+    - require:
+       - cmd : 'apt-key'
+
+lxc-docker:
+  pkg:
+    - purged
+
+docker-io:
+  pkg:
+    - purged
+
+cbr0:
+  network.managed:
+    - enabled: True
+    - type: bridge
+    - proto: dhcp
+    - ports: none
+    - bridge: cbr0
+    - delay: 0
+    - bypassfirewall: True
+    - require_in:
+      - service: docker
+
+/etc/apt/sources.list.d/docker.list:
+  file.managed:
+    - source: salt://docker/docker.list
+    - template: jinja
+    - user: root
+    - group: root
+    - mode: 644
+    - require:
+      - cmd: 'apt-update'
+
+docker-engine:
+   pkg:
+     - installed
+     - require:
+       - file: /etc/apt/sources.list.d/docker.list
+docker:
+   service.running:
+     - enable: True
+     - require:
+       - file: {{ environment_file }}
+     - watch:
+       - file: {{ environment_file }}
 
 {% else %}
 
@@ -216,3 +306,4 @@ docker:
       - pkg: docker-upgrade
 {% endif %}
 {% endif %} # end grains.os_family != 'RedHat'
+

cluster/saltbase/salt/kube-addons/init.sls

@@ -133,7 +133,7 @@ addon-dir-create:
 {% endif %}
 
 {% if pillar.get('enable_node_logging', '').lower() == 'true'
-   and pillar.get('logging_destination').lower() == 'elasticsearch'
+   and pillar.get('logging_destination', '').lower() == 'elasticsearch'
    and pillar.get('enable_cluster_logging', '').lower() == 'true' %}
 /etc/kubernetes/addons/fluentd-elasticsearch:
   file.recurse:

cluster/saltbase/salt/kube-apiserver/init.sls

 {% if grains.cloud is defined %}
-{% if grains.cloud in ['aws', 'gce', 'vagrant'] %}
+{% if grains.cloud in ['aws', 'gce', 'vagrant', 'vsphere'] %}
 # TODO: generate and distribute tokens on other cloud providers.
 /srv/kubernetes/known_tokens.csv:
   file.managed:
@@ -9,7 +9,7 @@
 {% endif %}
 {% endif %}
 
-{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant' ]  %}
+{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant' ,'vsphere']  %}
 /srv/kubernetes/basic_auth.csv:
   file.managed:
     - source: salt://kube-apiserver/basic_auth.csv

cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest

@@ -9,7 +9,7 @@
 {% set cloud_config_volume = "" -%}
 
 {% if grains.cloud is defined -%}
-  {% if grains.cloud != 'vagrant' -%}
+  {% if grains.cloud not in ['vagrant', 'vsphere'] -%}
     {% set cloud_provider = "--cloud-provider=" + grains.cloud -%}
   {% endif -%}
 
@@ -57,7 +57,7 @@
 {% set client_ca_file = "" -%}
 
 {% set secure_port = "6443" -%}
-{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant' ]    %}
+{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere' ]    %}
   {% set secure_port = "443" -%}
   {% set client_ca_file = "--client-ca-file=/srv/kubernetes/ca.crt" -%}
 {% endif -%}
@@ -71,12 +71,12 @@
 {% endif -%}
 
 {% if grains.cloud is defined -%}
-{% if grains.cloud in [ 'aws', 'gce', 'vagrant' ] -%}
+{% if grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere' ] -%}
     {% set token_auth_file = "--token-auth-file=/srv/kubernetes/known_tokens.csv" -%}
 {% endif -%}
 {% endif -%}
 
-{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant']  %}
+{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere']  %}
   {% set basic_auth_file = "--basic-auth-file=/srv/kubernetes/basic_auth.csv" -%}
 {% endif -%}
 

cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest

@@ -27,7 +27,7 @@
 {% set cloud_config_volume = "" -%}
 
 {% if grains.cloud is defined -%}
-  {% if grains.cloud != 'vagrant' -%}
+  {% if grains.cloud not in ['vagrant', 'vsphere'] -%}
     {% set cloud_provider = "--cloud-provider=" + grains.cloud -%}
   {% endif -%}
   {% set service_account_key = "--service-account-private-key-file=/srv/kubernetes/server.key" -%}
@@ -41,7 +41,7 @@
 
 {% set root_ca_file = "" -%}
 
-{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant' ]    %}
+{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere' ]    %}
    {% set root_ca_file = "--root-ca-file=/srv/kubernetes/ca.crt" -%}
 {% endif -%}
 

cluster/saltbase/salt/kube-proxy/kube-proxy.manifest

@@ -5,7 +5,7 @@
   {% set ips = salt['mine.get']('roles:kubernetes-master', 'network.ip_addrs', 'grain').values() -%}
   {% set api_servers = "--master=https://" + ips[0][0] -%}
 {% endif -%}
-{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant' ]  %}
+{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere' ]  %}
   {% set api_servers_with_port = api_servers -%}
 {% else -%}
   {% set api_servers_with_port = api_servers + ":6443" -%}

cluster/saltbase/salt/kubelet/default

@@ -16,7 +16,7 @@
 {% endif -%}
 
 # TODO: remove nginx for other cloud providers.
-{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant' ]  %}
+{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere' ]  %}
   {% set api_servers_with_port = api_servers -%}
 {% else -%}
   {% set api_servers_with_port = api_servers + ":6443" -%}
@@ -27,7 +27,7 @@
 {% set debugging_handlers = "--enable-debugging-handlers=true" -%}
 
 {% if grains['roles'][0] == 'kubernetes-master' -%}
-  {% if grains.cloud in ['aws', 'gce', 'vagrant'] -%}
+  {% if grains.cloud in ['aws', 'gce', 'vagrant', 'vsphere'] -%}
 
     # Unless given a specific directive, disable registration for the kubelet
     # running on the master.
@@ -46,7 +46,7 @@
 {% endif -%}
 
 {% set cloud_provider = "" -%}
-{% if grains.cloud is defined and grains.cloud != 'vagrant' -%}
+{% if grains.cloud is defined and grains.cloud not in ['vagrant', 'vsphere'] -%}
   {% set cloud_provider = "--cloud-provider=" + grains.cloud -%}
 {% endif -%}
 
@@ -100,7 +100,11 @@
 {% set cgroup_root = "" -%}
 {% if grains['os_family'] == 'Debian' -%}
   {% set system_container = "--system-container=/system" -%}
-  {% set cgroup_root = "--cgroup-root=/" -%}
+  {% if pillar.get('is_systemd') %}
+    {% set cgroup_root = "--cgroup-root=docker" -%}
+  {% else %}
+    {% set cgroup_root = "--cgroup-root=/" -%}
+  {% endif %}
 {% endif -%}
 {% if grains['oscodename'] == 'vivid' -%}
   {% set cgroup_root = "--cgroup-root=docker" -%}

cluster/saltbase/salt/static-routes/if-down

@@ -3,8 +3,14 @@
 [ "$IFACE" == "eth0" ] || exit 0
 
 {% for host, ip_addrs in salt['mine.get']('roles:kubernetes-pool', 'network.ip_addrs', 'grain').items() %}
-{% if ip_addrs[0] != salt['network.ip_addrs']('eth0')[0] %}
-{% set cidr = salt['mine.get'](host, 'grains.items')[host]['cbr-cidr'] %}
-route del -net {{ cidr }}
-{% endif %}
+  {% set network_ipaddr = None %}
+  {% if salt['network.ip_addrs']('eth0') is defined %}
+    {% set network_ipaddr = salt['network.ip_addrs']('eth0')[0] %}
+  {% elif salt['network.ip_addrs']('local') is defined %}
+    {% set network_ipaddr = salt['network.ip_addrs']('local')[0] %}
+  {% endif %}
+  {% if network_ipaddr is defined and ip_addrs[0] != network_ipaddr %}
+    {% set cidr = salt['mine.get'](host, 'grains.items')[host]['cbr-cidr'] %}
+      route del -net {{ cidr }}
+  {% endif %}
 {% endfor %}

cluster/saltbase/salt/static-routes/if-up

@@ -3,8 +3,14 @@
 [ "$IFACE" == "eth0" ] || exit 0
 
 {% for host, ip_addrs in salt['mine.get']('roles:kubernetes-pool', 'network.ip_addrs', 'grain').items() %}
-{% if ip_addrs[0] != salt['network.ip_addrs']('eth0')[0] %}
-{% set cidr = salt['mine.get'](host, 'grains.items')[host]['cbr-cidr'] %}
-route add -net {{ cidr }} gw {{ ip_addrs[0] }}
-{% endif %}
+  {% set network_ipaddr = None %}
+  {% if salt['network.ip_addrs']('eth0') %}
+    {% set network_ipaddr = salt['network.ip_addrs']('eth0')[0] %}
+  {% elif salt['network.ip_addrs']('local') %}
+    {% set network_ipaddr = salt['network.ip_addrs']('local').first %}
+  {% endif %}
+  {% if network_ipaddr and ip_addrs[0] != network_ipaddr %}
+    {% set cidr = salt['mine.get'](host, 'grains.items')[host]['cbr-cidr'] %}
+      route add -net {{ cidr }} gw {{ ip_addrs[0] }}
+  {% endif %}
 {% endfor %}

cluster/saltbase/salt/top.sls

@@ -51,7 +51,7 @@ base:
     - kube-controller-manager
     - kube-scheduler
     - supervisor
-{% if grains['cloud'] is defined and not grains.cloud in [ 'aws', 'gce', 'vagrant' ] %}
+{% if grains['cloud'] is defined and not grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere'] %}
     - nginx
 {% endif %}
     - cadvisor
@@ -69,7 +69,7 @@ base:
     - logrotate
 {% endif %}
     - kube-addons
-{% if grains['cloud'] is defined and grains['cloud'] in [ 'vagrant', 'gce', 'aws' ] %}
+{% if grains['cloud'] is defined and grains['cloud'] in [ 'vagrant', 'gce', 'aws', 'vsphere' ] %}
     - docker
     - kubelet
 {% endif %}

cluster/vsphere/config-common.sh

@@ -20,9 +20,8 @@ SSH_OPTS="-oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -oLogLevel=E
 #export GOVC_URL=
 #export GOVC_DATACENTER=
 #export GOVC_DATASTORE=
-#export GOVC_RESOURCE_POOL=
 #export GOVC_NETWORK=
-#export GOVC_GUEST_LOGIN='kube:kube'
+#export GOVC_GUEST_LOGIN=
 
 # Set GOVC_INSECURE if the host in GOVC_URL is using a certificate that cannot
 # be verified (i.e. a self-signed certificate), but IS trusted.

cluster/vsphere/templates/create-dynamic-salt-files.sh

@@ -14,6 +14,93 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+#generate token files
+
+KUBELET_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
+KUBE_PROXY_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
+known_tokens_file="/srv/salt-overlay/salt/kube-apiserver/known_tokens.csv"
+if [[ ! -f "${known_tokens_file}" ]]; then
+
+  mkdir -p /srv/salt-overlay/salt/kube-apiserver
+  known_tokens_file="/srv/salt-overlay/salt/kube-apiserver/known_tokens.csv"
+  (umask u=rw,go= ;
+   echo "$KUBELET_TOKEN,kubelet,kubelet" > $known_tokens_file;
+   echo "$KUBE_PROXY_TOKEN,kube_proxy,kube_proxy" >> $known_tokens_file)
+
+  mkdir -p /srv/salt-overlay/salt/kubelet
+  kubelet_auth_file="/srv/salt-overlay/salt/kubelet/kubernetes_auth"
+  (umask u=rw,go= ; echo "{\"BearerToken\": \"$KUBELET_TOKEN\", \"Insecure\": true }" > $kubelet_auth_file)
+  kubelet_kubeconfig_file="/srv/salt-overlay/salt/kubelet/kubeconfig"
+
+  mkdir -p /srv/salt-overlay/salt/kubelet
+  (umask 077;
+  cat > "${kubelet_kubeconfig_file}" << EOF
+apiVersion: v1
+kind: Config
+clusters:
+- cluster:
+    insecure-skip-tls-verify: true
+  name: local
+contexts:
+- context:
+    cluster: local
+    user: kubelet
+  name: service-account-context
+current-context: service-account-context
+users:
+- name: kubelet
+  user:
+    token: ${KUBELET_TOKEN}
+EOF
+)
+
+
+  mkdir -p /srv/salt-overlay/salt/kube-proxy
+  kube_proxy_kubeconfig_file="/srv/salt-overlay/salt/kube-proxy/kubeconfig"
+  # Make a kubeconfig file with the token.
+  # TODO(etune): put apiserver certs into secret too, and reference from authfile,
+  # so that "Insecure" is not needed.
+  (umask 077;
+  cat > "${kube_proxy_kubeconfig_file}" << EOF
+apiVersion: v1
+kind: Config
+clusters:
+- cluster:
+    insecure-skip-tls-verify: true
+  name: local
+contexts:
+- context:
+    cluster: local
+    user: kube-proxy
+  name: service-account-context
+current-context: service-account-context
+users:
+- name: kube-proxy
+  user:
+    token: ${KUBE_PROXY_TOKEN}
+EOF
+)
+
+  # Generate tokens for other "service accounts".  Append to known_tokens.
+  #
+  # NB: If this list ever changes, this script actually has to
+  # change to detect the existence of this file, kill any deleted
+  # old tokens and add any new tokens (to handle the upgrade case).
+  service_accounts=("system:scheduler" "system:controller_manager" "system:logging" "system:monitoring" "system:dns")
+  for account in "${service_accounts[@]}"; do
+    token=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
+    echo "${token},${account},${account}" >> "${known_tokens_file}"
+  done
+fi
+
+readonly BASIC_AUTH_FILE="/srv/salt-overlay/salt/kube-apiserver/basic_auth.csv"
+if [ ! -e "${BASIC_AUTH_FILE}" ]; then
+  mkdir -p /srv/salt-overlay/salt/kube-apiserver
+  (umask 077;
+    echo "${KUBE_PASSWORD},${KUBE_USER},admin" > "${BASIC_AUTH_FILE}")
+fi
+
+
 # Create the overlay files for the salt tree.  We create these in a separate
 # place so that we can blow away the rest of the salt configs on a kube-push and
 # re-apply these.
@@ -23,17 +110,17 @@ cat <<EOF >/srv/salt-overlay/pillar/cluster-params.sls
 instance_prefix: '$(echo "$INSTANCE_PREFIX" | sed -e "s/'/''/g")'
 node_instance_prefix: $NODE_INSTANCE_PREFIX
 service_cluster_ip_range: $SERVICE_CLUSTER_IP_RANGE
-enable_cluster_monitoring: $ENABLE_CLUSTER_MONITORING
-enable_cluster_logging: $ENABLE_CLUSTER_LOGGING
-enable_cluster_ui: $ENABLE_CLUSTER_UI
-enable_node_logging: $ENABLE_NODE_LOGGING
+enable_cluster_monitoring: "${ENABLE_CLUSTER_MONITORING:-none}"
+enable_cluster_logging: "${ENABLE_CLUSTER_LOGGING:false}"
+enable_cluster_ui: "${ENABLE_CLUSTER_UI:false}"
+enable_node_logging: "${ENABLE_NODE_LOGGING:false}"
 logging_destination: $LOGGING_DESTINATION
 elasticsearch_replicas: $ELASTICSEARCH_LOGGING_REPLICAS
-enable_cluster_dns: $ENABLE_CLUSTER_DNS
-dns_replicas: $DNS_REPLICAS
+enable_cluster_dns: "${ENABLE_CLUSTER_DNS:-false}"
+dns_replicas: ${DNS_REPLICAS:-1}
 dns_server: $DNS_SERVER_IP
 dns_domain: $DNS_DOMAIN
-e2e_storage_test_environment: $E2E_STORAGE_TEST_ENVIRONMENT
+e2e_storage_test_environment: "${E2E_STORAGE_TEST_ENVIRONMENT:-false}"
 
 EOF
 

cluster/vsphere/templates/salt-minion.sh

@@ -42,6 +42,7 @@ grains:
     - kubernetes-pool
     - kubernetes-pool-vsphere
   cbr-cidr: $NODE_IP_RANGE
+  cloud: vsphere
 EOF
 
 # Install Salt

cluster/vsphere/util.sh

@@ -18,6 +18,7 @@
 
 # Use the config file specified in $KUBE_CONFIG_FILE, or default to
 # config-default.sh.
+
 KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
 source "${KUBE_ROOT}/cluster/vsphere/config-common.sh"
 source "${KUBE_ROOT}/cluster/vsphere/${KUBE_CONFIG_FILE-"config-default.sh"}"
@@ -30,6 +31,7 @@ source "${KUBE_ROOT}/cluster/common.sh"
 # Vars set:
 #   KUBE_MASTER
 #   KUBE_MASTER_IP
+
 function detect-master {
   KUBE_MASTER=${MASTER_NAME}
   if [[ -z "${KUBE_MASTER_IP-}" ]]; then
@@ -42,7 +44,7 @@ function detect-master {
   echo "Using master: $KUBE_MASTER (external IP: $KUBE_MASTER_IP)"
 }
 
-# Detect the information about the minions
+# Detect the information about the nodes
 #
 # Assumed vars:
 #   NODE_NAMES
@@ -51,16 +53,16 @@ function detect-master {
 function detect-nodes {
   KUBE_NODE_IP_ADDRESSES=()
   for (( i=0; i<${#NODE_NAMES[@]}; i++)); do
-    local minion_ip=$(govc vm.ip ${NODE_NAMES[$i]})
-    if [[ -z "${minion_ip-}" ]] ; then
+    local nodeip=$(govc vm.ip ${NODE_NAMES[$i]})
+    if [[ -z "${nodeip-}" ]] ; then
       echo "Did not find ${NODE_NAMES[$i]}" >&2
     else
-      echo "Found ${NODE_NAMES[$i]} at ${minion_ip}"
-      KUBE_NODE_IP_ADDRESSES+=("${minion_ip}")
+      echo "Found ${NODE_NAMES[$i]} at ${nodeip}"
+      KUBE_NODE_IP_ADDRESSES+=("${nodeip}")
     fi
   done
   if [[ -z "${KUBE_NODE_IP_ADDRESSES-}" ]]; then
-    echo "Could not detect Kubernetes minion nodes. Make sure you've launched a cluster with 'kube-up.sh'" >&2
+    echo "Could not detect Kubernetes nodes. Make sure you've launched a cluster with 'kube-up.sh'" >&2
     exit 1
   fi
 }
@@ -159,7 +161,7 @@ function kube-scp {
   scp ${SSH_OPTS-} "${src}" "kube@${host}:${dst}"
 }
 
-# Instantiate a generic kubernetes virtual machine (master or minion)
+# Instantiate a generic kubernetes virtual machine (master or node)
 #
 # Usage:
 #   kube-up-vm VM_NAME [options to pass to govc vm.create]
@@ -199,7 +201,7 @@ function kube-up-vm {
     /home/kube/.ssh/authorized_keys
 }
 
-# Kick off a local script on a kubernetes virtual machine (master or minion)
+# Kick off a local script on a kubernetes virtual machine (master or node)
 #
 # Usage:
 #   kube-run VM_NAME LOCAL_FILE
@@ -208,12 +210,100 @@ function kube-run {
   local file="$2"
   local dst="/tmp/$(basename "${file}")"
   govc guest.upload -vm="${vm_name}" -f -perm=0755 "${file}" "${dst}"
-
+  echo "uploaded ${file} to ${dst}"
   local vm_ip
   vm_ip=$(govc vm.ip "${vm_name}")
   kube-ssh ${vm_ip} "nohup sudo ${dst} < /dev/null 1> ${dst}.out 2> ${dst}.err &"
 }
 
+#
+# run the command remotely and check if the specific kube artifact is running or not.
+# keep checking till the you hit the timeout. default timeout 300s
+#
+# Usage:
+#   kube_check 10.0.0.1 cmd timeout
+function kube-check {
+  nodeip=$1
+  cmd=$2
+  sleepstep=5
+  if [[ $# -lt 3 || -z $3 ]]; then
+    timeout=300
+  else
+    timeout=$3
+  fi
+  let effective_timeout=($timeout/$sleepstep)
+  attempt=0
+  echo
+  printf "This may take several minutes. Bound to $effective_timeout attemmps"
+  while true; do
+    local rc=0
+    output=$(kube-ssh ${nodeip} "${cmd}") || rc=1
+    if [[ $rc != 0 ]]; then
+      if (( $attempt == $effective_timeout )); then
+        echo
+        echo "(Failed) rc: $rc Output: ${output}"
+        echo
+        echo -e "${cmd} failed to start on ${nodeip}. Your cluster is unlikely" >&2
+        echo "to work correctly. You may have to debug it by logging in." >&2
+        echo
+        exit 1
+      fi
+    else
+      echo
+      echo -e "[${cmd}] passed"
+      echo
+      break
+    fi
+    printf "."
+    attempt=$(($attempt+1))
+    sleep $sleepstep
+  done
+}
+
+
+
+
+#
+# verify if salt master is up. check 30 times and then echo out bad output and return 0
+#
+# Usage:
+#   remote-pgrep 10.0.0.1 salt-master
+#
+function remote-pgrep {
+  nodeip=$1
+  regex=$2
+
+  max_attempt=60
+
+  printf "This may take several minutes. Bound to $max_attempt attemmps"
+
+  attempt=0
+  while true; do
+    local rc=0
+    output=$(kube-ssh ${nodeip} pgrep ${regex}) || rc=1
+
+    if [[ $rc != 0 ]]; then
+      if (( $attempt == $max_attempt )); then
+        echo
+        echo "(Failed) rc: $rc, output:${output}"
+        echo
+        echo -e "${regex} failed to start on ${nodeip} after checking for $attempt attempts. Your cluster is unlikely" >&2
+        echo "to work correctly. You may have to debug it by logging in." >&2
+        echo
+        exit 1
+      fi
+    else
+      echo
+      echo -e "[${regex} running]"
+      echo
+      break
+    fi
+    printf "."
+    attempt=$(($attempt+1))
+    sleep 10
+  done
+}
+
 # Instantiate a kubernetes cluster
 #
 # Assumed vars:
@@ -240,13 +330,15 @@ function kube-up {
     echo "cd /home/kube/cache/kubernetes-install"
     echo "readonly MASTER_NAME='${MASTER_NAME}'"
     echo "readonly INSTANCE_PREFIX='${INSTANCE_PREFIX}'"
-    echo "readonly NODE_INSTANCE_PREFIX='${INSTANCE_PREFIX}-minion'"
+    echo "readonly NODE_INSTANCE_PREFIX='${INSTANCE_PREFIX}-node'"
     echo "readonly SERVICE_CLUSTER_IP_RANGE='${SERVICE_CLUSTER_IP_RANGE}'"
     echo "readonly ENABLE_NODE_LOGGING='${ENABLE_NODE_LOGGING:-false}'"
     echo "readonly LOGGING_DESTINATION='${LOGGING_DESTINATION:-}'"
     echo "readonly ENABLE_CLUSTER_DNS='${ENABLE_CLUSTER_DNS:-false}'"
     echo "readonly DNS_SERVER_IP='${DNS_SERVER_IP:-}'"
     echo "readonly DNS_DOMAIN='${DNS_DOMAIN:-}'"
+    echo "readonly KUBE_USER='${KUBE_USER:-}'"
+    echo "readonly KUBE_PASSWORD='${KUBE_PASSWORD:-}'"
     echo "readonly SERVER_BINARY_TAR='${SERVER_BINARY_TAR##*/}'"
     echo "readonly SALT_TAR='${SALT_TAR##*/}'"
     echo "readonly MASTER_HTPASSWD='${htpasswd}'"
@@ -259,13 +351,11 @@ function kube-up {
   kube-up-vm ${MASTER_NAME} -c ${MASTER_CPU-1} -m ${MASTER_MEMORY_MB-1024}
   upload-server-tars
   kube-run ${MASTER_NAME} "${KUBE_TEMP}/master-start.sh"
-
   # Print master IP, so user can log in for debugging.
   detect-master
   echo
 
-  echo "Starting minion VMs (this can take a minute)..."
-
+  echo "Starting node VMs (this can take a minute)..."
   for (( i=0; i<${#NODE_NAMES[@]}; i++)); do
     (
       echo "#! /bin/bash"
@@ -275,11 +365,11 @@ function kube-up {
       echo "KUBE_MASTER_IP=${KUBE_MASTER_IP}"
       echo "NODE_IP_RANGE=${NODE_IP_RANGES[$i]}"
       grep -v "^#" "${KUBE_ROOT}/cluster/vsphere/templates/salt-minion.sh"
-    ) > "${KUBE_TEMP}/minion-start-${i}.sh"
+    ) > "${KUBE_TEMP}/node-start-${i}.sh"
 
     (
       kube-up-vm "${NODE_NAMES[$i]}" -c ${NODE_CPU-1} -m ${NODE_MEMORY_MB-1024}
-      kube-run "${NODE_NAMES[$i]}" "${KUBE_TEMP}/minion-start-${i}.sh"
+      kube-run "${NODE_NAMES[$i]}" "${KUBE_TEMP}/node-start-${i}.sh"
     ) &
   done
 
@@ -293,17 +383,32 @@ function kube-up {
     exit 2
   fi
 
-  # Print minion IPs, so user can log in for debugging.
+  # Print node IPs, so user can log in for debugging.
   detect-nodes
+
+  printf "Waiting for salt-master to be up on ${KUBE_MASTER} ...\n"
+  remote-pgrep ${KUBE_MASTER_IP} "salt-master"
+
+  printf "Waiting for all packages to be installed on ${KUBE_MASTER} ...\n"
+  kube-check  ${KUBE_MASTER_IP} 'sudo salt "kubernetes-master" state.highstate -t 30 | grep -E "Failed:[[:space:]]+0"'
+
+  local i
+  for (( i=0; i<${#NODE_NAMES[@]}; i++)); do
+    printf "Waiting for salt-minion to be up on ${NODE_NAMES[$i]} ....\n"
+    remote-pgrep ${KUBE_NODE_IP_ADDRESSES[$i]} "salt-minion"
+    printf "Waiting for all salt packages to be installed on ${NODE_NAMES[$i]} .... \n"
+    kube-check  ${KUBE_MASTER_IP} 'sudo salt '"${NODE_NAMES[$i]}"' state.highstate -t 30 | grep -E "Failed:[[:space:]]+0"'
+    printf " OK\n"
+  done
+
   echo
 
-  echo "Waiting for master and minion initialization."
+  echo "Waiting for master and node initialization."
   echo
   echo "  This will continually check to see if the API for kubernetes is reachable."
   echo "  This might loop forever if there was some uncaught error during start up."
   echo
 
-  printf "Waiting for ${KUBE_MASTER} to become available..."
   until curl --insecure --user "${KUBE_USER}:${KUBE_PASSWORD}" --max-time 5 \
           --fail --output /dev/null --silent "https://${KUBE_MASTER_IP}/healthz"; do
       printf "."
@@ -311,7 +416,6 @@ function kube-up {
   done
   printf " OK\n"
 
-  local i
   for (( i=0; i<${#NODE_NAMES[@]}; i++)); do
     printf "Waiting for ${NODE_NAMES[$i]} to become available..."
     until curl --max-time 5 \
@@ -321,7 +425,6 @@ function kube-up {
     done
     printf " OK\n"
   done
-
   echo "Kubernetes cluster created."
 
   # TODO use token instead of basic auth

docs/getting-started-guides/vsphere.md

@@ -68,10 +68,10 @@ convenient).
 
 ### Setup
 
-Download a prebuilt Debian 7.7 VMDK that we'll use as a base image:
+Download a prebuilt Debian 8.2 VMDK that we'll use as a base image:
 
 ```sh
-curl --remote-name-all https://storage.googleapis.com/govmomi/vmdk/2014-11-11/kube.vmdk.gz{,.md5}
+curl --remote-name-all https://storage.googleapis.com/govmomi/vmdk/2016-01-08/kube.vmdk.gz{,.md5}
 md5sum -c kube.vmdk.gz.md5
 gzip -d kube.vmdk.gz
 ```
@@ -79,7 +79,10 @@ gzip -d kube.vmdk.gz
 Import this VMDK into your vSphere datastore:
 
 ```sh
-export GOVC_URL='user:pass@hostname'
+export GOVC_URL='hostname' # hostname of the vc
+export GOVC_USER='username' # username for logging into the vsphere.
+export GOVC_PASSWORD='password' # password for the above username
+export GOVC_NETWORK='Network Name' # Name of the network the vms should join. Many times it could be "VM Network"
 export GOVC_INSECURE=1 # If the host above uses a self-signed cert
 export GOVC_DATASTORE='target datastore'
 export GOVC_RESOURCE_POOL='resource pool or cluster with access to datastore'
@@ -99,7 +102,7 @@ parameters. The guest login for the image that you imported is `kube:kube`.
 ### Starting a cluster
 
 Now, let's continue with deploying Kubernetes.
-This process takes about ~10 minutes.
+This process takes about ~20-30 minutes depending on your network.
 
 ```sh
 cd kubernetes # Extracted binary release OR repository root