kube-up: Finish refactor of AWS to reuse GCE script

cluster/aws/templates/common.sh

-#!/bin/bash
-
-# Copyright 2015 The Kubernetes Authors All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-echo "== Refreshing package database =="
-until apt-get update; do
-  echo "== apt-get update failed, retrying =="
-  sleep 5
-done
-
-function apt-get-install {
-  # Forcibly install packages (options borrowed from Salt logs).
-  until apt-get -q -y -o DPkg::Options::=--force-confold -o DPkg::Options::=--force-confdef install $@; do
-    echo "== install of packages $@ failed, retrying =="
-    sleep 5
-  done
-}
-
-apt-get-install curl
-
-# Retry a download until we get it.
-#
-# $1 is the URL to download
-download-or-bust() {
-  local -r url="$1"
-  local -r file="${url##*/}"
-  rm -f "$file"
-  until [[ -e "${1##*/}" ]]; do
-    echo "Downloading file ($1)"
-    curl --ipv4 -Lo "$file" --connect-timeout 20 --retry 6 --retry-delay 10 "$1"
-    md5sum "$file"
-  done
-}
-
-
-
-# Install salt from GCS.  See README.md for instructions on how to update these
-# debs.
-install-salt() {
-  local salt_mode="$1"
-
-  if dpkg -s salt-minion &>/dev/null; then
-    echo "== SaltStack already installed, skipping install step =="
-    return
-  fi
-
-  mkdir -p /var/cache/salt-install
-  cd /var/cache/salt-install
-
-  DEBS=(
-    libzmq3_3.2.3+dfsg-1~bpo70~dst+1_amd64.deb
-    python-zmq_13.1.0-1~bpo70~dst+1_amd64.deb
-    salt-common_2014.1.13+ds-1~bpo70+1_all.deb
-  )
-  if [[ "${salt_mode}" == "master" ]]; then
-    DEBS+=( salt-master_2014.1.13+ds-1~bpo70+1_all.deb )
-  fi
-  DEBS+=( salt-minion_2014.1.13+ds-1~bpo70+1_all.deb )
-  URL_BASE="https://storage.googleapis.com/kubernetes-release/salt"
-
-  for deb in "${DEBS[@]}"; do
-    if [ ! -e "${deb}" ]; then
-      download-or-bust "${URL_BASE}/${deb}"
-    fi
-  done
-
-  for deb in "${DEBS[@]}"; do
-    echo "== Installing ${deb}, ignore dependency complaints (will fix later) =="
-    dpkg --skip-same-version --force-depends -i "${deb}"
-  done
-
-  # This will install any of the unmet dependencies from above.
-  echo "== Installing unmet dependencies =="
-  until apt-get install -f -y; do
-    echo "== apt-get install failed, retrying =="
-    sleep 5
-  done
-
-  # Log a timestamp
-  echo "== Finished installing Salt =="
-}

cluster/aws/templates/configure-vm-aws.sh

+#!/bin/bash
+
+# Copyright 2015 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Note: these functions override functions in the GCE configure-vm script
+# We include the GCE script first, and this one second.
+
+ensure-basic-networking() {
+  :
+}
+
+ensure-packages() {
+  apt-get-install curl
+  # For reading kube_env.yaml
+  apt-get-install python-yaml
+
+  # TODO: Where to get safe_format_and_mount?
+  mkdir -p /usr/share/google
+  cd /usr/share/google
+  download-or-bust https://raw.githubusercontent.com/GoogleCloudPlatform/compute-image-packages/82b75f314528b90485d5239ab5d5495cc22d775f/google-startup-scripts/usr/share/google/safe_format_and_mount
+  chmod +x safe_format_and_mount
+}
+
+set-kube-env() {
+  local kube_env_yaml="${INSTALL_DIR}/kube_env.yaml"
+
+  # kube-env has all the environment variables we care about, in a flat yaml format
+  eval "$(python -c '
+import pipes,sys,yaml
+
+for k,v in yaml.load(sys.stdin).iteritems():
+  print("""readonly {var}={value}""".format(var = k, value = pipes.quote(str(v))))
+  print("""export {var}""".format(var = k))
+  ' < """${kube_env_yaml}""")"
+}
+
+remove-docker-artifacts() {
+  :
+}
+
+# Finds the master PD device
+find-master-pd() {
+  echo "Waiting for master pd to be attached"
+  attempt=0
+  while true; do
+    echo Attempt "$(($attempt+1))" to check for /dev/xvdb
+    if [[ -e /dev/xvdb ]]; then
+      echo "Found /dev/xvdb"
+      MASTER_PD_DEVICE="/dev/xvdb"
+      break
+    fi
+    attempt=$(($attempt+1))
+    sleep 1
+  done
+}
+
+fix-apt-sources() {
+  :
+}
+
+salt-master-role() {
+  cat <<EOF >/etc/salt/minion.d/grains.conf
+grains:
+  roles:
+    - kubernetes-master
+  cloud: aws
+EOF
+
+  # If the kubelet on the master is enabled, give it the same CIDR range
+  # as a generic node.
+  if [[ ! -z "${KUBELET_APISERVER:-}" ]] && [[ ! -z "${KUBELET_CERT:-}" ]] && [[ ! -z "${KUBELET_KEY:-}" ]]; then
+    cat <<EOF >>/etc/salt/minion.d/grains.conf
+  kubelet_api_servers: '${KUBELET_APISERVER}'
+  cbr-cidr: 10.123.45.0/30
+EOF
+  else
+    # If the kubelet is running disconnected from a master, give it a fixed
+    # CIDR range.
+    cat <<EOF >>/etc/salt/minion.d/grains.conf
+  cbr-cidr: ${MASTER_IP_RANGE}
+EOF
+  fi
+
+  env-to-grains "runtime_config"
+}
+
+salt-node-role() {
+  cat <<EOF >/etc/salt/minion.d/grains.conf
+grains:
+  roles:
+    - kubernetes-pool
+  cbr-cidr: 10.123.45.0/30
+  cloud: aws
+  api_servers: '${API_SERVERS}'
+EOF
+
+  # We set the hostname_override to the full EC2 private dns name
+  # we'd like to use EC2 instance-id, but currently the kubelet health-check assumes the name
+  # is resolvable, although that check should be going away entirely (#7092)
+  if [[ -z "${HOSTNAME_OVERRIDE:-}" ]]; then
+    HOSTNAME_OVERRIDE=`curl --silent curl http://169.254.169.254/2007-01-19/meta-data/local-hostname`
+  fi
+
+  env-to-grains "hostname_override"
+}
+
+function run-user-script() {
+  # TODO(justinsb): Support user scripts on AWS
+  # AWS doesn't have as rich a metadata service as GCE does
+  # Maybe specify an env var that is the path to a script?
+  :
+}
+

cluster/aws/templates/create-dynamic-salt-files.sh

-#!/bin/bash
-
-# Copyright 2014 The Kubernetes Authors All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Create the overlay files for the salt tree.  We create these in a separate
-# place so that we can blow away the rest of the salt configs on a kube-push and
-# re-apply these.
-
-mkdir -p /srv/salt-overlay/pillar
-cat <<EOF >/srv/salt-overlay/pillar/cluster-params.sls
-instance_prefix: '$(echo "$INSTANCE_PREFIX" | sed -e "s/'/''/g")'
-node_instance_prefix: '$(echo "$NODE_INSTANCE_PREFIX" | sed -e "s/'/''/g")'
-cluster_cidr: '$(echo "$CLUSTER_IP_RANGE" | sed -e "s/'/''/g")'
-allocate_node_cidrs: '$(echo "$ALLOCATE_NODE_CIDRS" | sed -e "s/'/''/g")'
-service_cluster_ip_range: '$(echo "$SERVICE_CLUSTER_IP_RANGE" | sed -e "s/'/''/g")'
-enable_cluster_monitoring: '$(echo "$ENABLE_CLUSTER_MONITORING" | sed -e "s/'/''/g")'
-enable_cluster_logging: '$(echo "$ENABLE_CLUSTER_LOGGING" | sed -e "s/'/''/g")'
-enable_cluster_ui: '$(echo "$ENABLE_CLUSTER_UI" | sed -e "s/'/''/g")'
-enable_node_logging: '$(echo "$ENABLE_NODE_LOGGING" | sed -e "s/'/''/g")'
-logging_destination: '$(echo "$LOGGING_DESTINATION" | sed -e "s/'/''/g")'
-elasticsearch_replicas: '$(echo "$ELASTICSEARCH_LOGGING_REPLICAS" | sed -e "s/'/''/g")'
-enable_cluster_dns: '$(echo "$ENABLE_CLUSTER_DNS" | sed -e "s/'/''/g")'
-dns_replicas: '$(echo "$DNS_REPLICAS" | sed -e "s/'/''/g")'
-dns_server: '$(echo "$DNS_SERVER_IP" | sed -e "s/'/''/g")'
-dns_domain: '$(echo "$DNS_DOMAIN" | sed -e "s/'/''/g")'
-admission_control: '$(echo "$ADMISSION_CONTROL" | sed -e "s/'/''/g")'
-network_provider: '$(echo "$NETWORK_PROVIDER")'
-opencontrail_tag: '$(echo "$OPENCONTRAIL_TAG")'
-opencontrail_kubernetes_tag: '$(echo "$OPENCONTRAIL_KUBERNETES_TAG")'
-opencontrail_public_subnet: '$(echo "$OPENCONTRAIL_PUBLIC_SUBNET")'
-num_nodes: $(echo "${NUM_NODES}")
-e2e_storage_test_environment: '$(echo "$E2E_STORAGE_TEST_ENVIRONMENT" | sed -e "s/'/''/g")'
-EOF
-
-readonly BASIC_AUTH_FILE="/srv/salt-overlay/salt/kube-apiserver/basic_auth.csv"
-if [ ! -e "${BASIC_AUTH_FILE}" ]; then
-  mkdir -p /srv/salt-overlay/salt/kube-apiserver
-  (umask 077;
-    echo "${KUBE_PASSWORD},${KUBE_USER},admin" > "${BASIC_AUTH_FILE}")
-fi
-
-# Generate and distribute a shared secret (bearer token) to
-# apiserver and the nodes so that kubelet and kube-proxy can
-# authenticate to apiserver.
-kubelet_token=$KUBELET_TOKEN
-kube_proxy_token=$KUBE_PROXY_TOKEN
-
-# Make a list of tokens and usernames to be pushed to the apiserver
-mkdir -p /srv/salt-overlay/salt/kube-apiserver
-readonly KNOWN_TOKENS_FILE="/srv/salt-overlay/salt/kube-apiserver/known_tokens.csv"
-(umask u=rw,go= ; echo "$kubelet_token,kubelet,kubelet" > $KNOWN_TOKENS_FILE ;
-echo "$kube_proxy_token,kube_proxy,kube_proxy" >> $KNOWN_TOKENS_FILE)
-
-mkdir -p /srv/salt-overlay/salt/kubelet
-kubelet_auth_file="/srv/salt-overlay/salt/kubelet/kubernetes_auth"
-(umask u=rw,go= ; echo "{\"BearerToken\": \"$kubelet_token\", \"Insecure\": true }" > $kubelet_auth_file)
-
-mkdir -p /srv/salt-overlay/salt/kube-proxy
-kube_proxy_kubeconfig_file="/srv/salt-overlay/salt/kube-proxy/kubeconfig"
-cat > "${kube_proxy_kubeconfig_file}" <<EOF
-apiVersion: v1
-kind: Config
-users:
-- name: kube-proxy
-  user:
-    token: ${kube_proxy_token}
-clusters:
-- name: local
-  cluster:
-     insecure-skip-tls-verify: true
-contexts:
-- context:
-    cluster: local
-    user: kube-proxy
-  name: service-account-context
-current-context: service-account-context
-EOF
-
-mkdir -p /srv/salt-overlay/salt/kubelet
-kubelet_kubeconfig_file="/srv/salt-overlay/salt/kubelet/kubeconfig"
-cat > "${kubelet_kubeconfig_file}" <<EOF
-apiVersion: v1
-kind: Config
-users:
-- name: kubelet
-  user:
-    token: ${kubelet_token}
-clusters:
-- name: local
-  cluster:
-     insecure-skip-tls-verify: true
-contexts:
-- context:
-    cluster: local
-    user: kubelet
-  name: service-account-context
-current-context: service-account-context
-EOF
-
-# Generate tokens for other "service accounts".  Append to known_tokens.
-#
-# NB: If this list ever changes, this script actually has to
-# change to detect the existence of this file, kill any deleted
-# old tokens and add any new tokens (to handle the upgrade case).
-service_accounts=("system:scheduler" "system:controller_manager" "system:logging" "system:monitoring" "system:dns")
-for account in "${service_accounts[@]}"; do
-  token=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
-  echo "${token},${account},${account}" >> "${KNOWN_TOKENS_FILE}"
-done

cluster/aws/templates/download-release.sh

-#!/bin/bash
-
-# Copyright 2014 The Kubernetes Authors All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Download and install release
-
-# This script assumes that the environment variable MASTER_RELEASE_TAR contains
-# the release tar to download and unpack.  It is meant to be pushed to the
-# master and run.
-
-echo "Downloading binary release tar ($SERVER_BINARY_TAR_URL)"
-download-or-bust "$SERVER_BINARY_TAR_URL"
-
-echo "Downloading binary release tar ($SALT_TAR_URL)"
-download-or-bust "$SALT_TAR_URL"
-
-echo "Unpacking Salt tree"
-rm -rf kubernetes
-tar xzf "${SALT_TAR_URL##*/}"
-
-echo "Running release install script"
-sudo kubernetes/saltbase/install.sh "${SERVER_BINARY_TAR_URL##*/}"

cluster/aws/templates/extract-kube-env.sh

-#!/bin/bash
-
-# Copyright 2016 The Kubernetes Authors All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-apt-get-install python-yaml
-
-# kube-env has all the environment variables we care about, in a flat yaml format
-eval "$(python -c '
-import pipes,sys,yaml
-
-for k,v in yaml.load(sys.stdin).iteritems():
-  print("""readonly {var}={value}""".format(var = k, value = pipes.quote(str(v))))
-  print("""export {var}""".format(var = k))
-  ' < kube-env.yaml)"
-

cluster/aws/templates/format-disks.sh

@@ -16,6 +16,8 @@
 
 # Discover all the ephemeral disks
 
+function ensure-local-disks() {
+
 block_devices=()
 
 ephemeral_devices=$(curl --silent http://169.254.169.254/2014-11-05/meta-data/block-device-mapping/ | grep ephemeral)
@@ -119,7 +121,7 @@ else
       # 80% goes to the docker thin-pool; we want to leave some space for host-volumes
       lvcreate -l 80%VG --thinpool docker-thinpool vg-ephemeral
 
-      DOCKER_OPTS="${DOCKER_OPTS} --storage-opt dm.thinpooldev=/dev/mapper/vg--ephemeral-docker--thinpool"
+      DOCKER_OPTS="${DOCKER_OPTS:-} --storage-opt dm.thinpooldev=/dev/mapper/vg--ephemeral-docker--thinpool"
       # Note that we don't move docker; docker goes direct to the thinpool
 
       # Remaining space (20%) is for kubernetes data
@@ -159,7 +161,7 @@ fi
 
 
 if [[ ${docker_storage} == "btrfs" ]]; then
-  DOCKER_OPTS="${DOCKER_OPTS} -s btrfs"
+  DOCKER_OPTS="${DOCKER_OPTS:-} -s btrfs"
 elif [[ ${docker_storage} == "aufs-nolvm" || ${docker_storage} == "aufs" ]]; then
   # Install aufs kernel module
   # Fix issue #14162 with extra-virtual
@@ -168,9 +170,9 @@ elif [[ ${docker_storage} == "aufs-nolvm" || ${docker_storage} == "aufs" ]]; the
   # Install aufs tools
   apt-get-install aufs-tools
 
-  DOCKER_OPTS="${DOCKER_OPTS} -s aufs"
+  DOCKER_OPTS="${DOCKER_OPTS:-} -s aufs"
 elif [[ ${docker_storage} == "devicemapper" ]]; then
-  DOCKER_OPTS="${DOCKER_OPTS} -s devicemapper"
+  DOCKER_OPTS="${DOCKER_OPTS:-} -s devicemapper"
 else
   echo "Ignoring unknown DOCKER_STORAGE: ${docker_storage}"
 fi
@@ -183,7 +185,7 @@ if [[ -n "${move_docker}" ]]; then
   mkdir -p ${move_docker}/docker
   ln -s ${move_docker}/docker /var/lib/docker
   DOCKER_ROOT="${move_docker}/docker"
-  DOCKER_OPTS="${DOCKER_OPTS} -g ${DOCKER_ROOT}"
+  DOCKER_OPTS="${DOCKER_OPTS:-} -g ${DOCKER_ROOT}"
 fi
 
 if [[ -n "${move_kubelet}" ]]; then
@@ -197,3 +199,4 @@ if [[ -n "${move_kubelet}" ]]; then
   KUBELET_ROOT="${move_kubelet}/kubelet"
 fi
 
+}

cluster/aws/templates/salt-master.sh

-#!/bin/bash
-
-# Copyright 2014 The Kubernetes Authors All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Prepopulate the name of the Master
-mkdir -p /etc/salt/minion.d
-echo "master: $SALT_MASTER" > /etc/salt/minion.d/master.conf
-
-cat <<EOF >/etc/salt/minion.d/grains.conf
-grains:
-  roles:
-    - kubernetes-master
-  cloud: aws
-  cbr-cidr: "${MASTER_IP_RANGE}"
-EOF
-
-# Helper that sets a salt grain in grains.conf, if the upper-cased key is a non-empty env
-function env_to_salt {
-  local key=$1
-  local env_key=`echo $key | tr '[:lower:]' '[:upper:]'`
-  local value=${!env_key}
-  if [[ -n "${value}" ]]; then
-    # Note this is yaml, so indentation matters
-    cat <<EOF >>/etc/salt/minion.d/grains.conf
-  ${key}: '$(echo "${value}" | sed -e "s/'/''/g")'
-EOF
-  fi
-}
-
-env_to_salt docker_opts
-env_to_salt docker_root
-env_to_salt kubelet_root
-env_to_salt master_extra_sans
-env_to_salt runtime_config
-env_to_salt non_masquerade_cidr
-
-# Auto accept all keys from minions that try to join
-mkdir -p /etc/salt/master.d
-cat <<EOF >/etc/salt/master.d/auto-accept.conf
-auto_accept: True
-EOF
-
-cat <<EOF >/etc/salt/master.d/reactor.conf
-# React to new minions starting by running highstate on them.
-reactor:
-  - 'salt/minion/*/start':
-    - /srv/reactor/highstate-new.sls
-EOF
-
-install-salt master
-
-service salt-master start
-service salt-minion start

cluster/aws/templates/salt-minion.sh

-#!/bin/bash
-
-# Copyright 2014 The Kubernetes Authors All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Prepopulate the name of the Master
-mkdir -p /etc/salt/minion.d
-echo "master: $SALT_MASTER" > /etc/salt/minion.d/master.conf
-
-# Turn on debugging for salt-minion
-# echo "DAEMON_ARGS=\"\$DAEMON_ARGS --log-file-level=debug\"" > /etc/default/salt-minion
-
-# Our minions will have a pool role to distinguish them from the master.
-cat <<EOF >/etc/salt/minion.d/grains.conf
-grains:
-  roles:
-    - kubernetes-pool
-  cbr-cidr: 10.123.45.0/30
-  cloud: aws
-EOF
-
-# We set the hostname_override to the full EC2 private dns name
-# we'd like to use EC2 instance-id, but currently the kubelet health-check assumes the name
-# is resolvable, although that check should be going away entirely (#7092)
-if [[ -z "${HOSTNAME_OVERRIDE}" ]]; then
-  HOSTNAME_OVERRIDE=`curl --silent curl http://169.254.169.254/2007-01-19/meta-data/local-hostname`
-fi
-
-# Helper that sets a salt grain in grains.conf, if the upper-cased key is a non-empty env
-function env_to_salt {
-  local key=$1
-  local env_key=`echo $key | tr '[:lower:]' '[:upper:]'`
-  local value=${!env_key}
-  if [[ -n "${value}" ]]; then
-    # Note this is yaml, so indentation matters
-    cat <<EOF >>/etc/salt/minion.d/grains.conf
-  ${key}: '$(echo "${value}" | sed -e "s/'/''/g")'
-EOF
-  fi
-}
-
-env_to_salt hostname_override
-env_to_salt docker_opts
-env_to_salt docker_root
-env_to_salt kubelet_root
-env_to_salt non_masquerade_cidr
-
-install-salt
-
-service salt-minion start

cluster/aws/templates/setup-master-pd.sh

-#!/bin/bash
-
-# Copyright 2015 The Kubernetes Authors All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Format and mount the disk, create directories on it for all of the master's
-# persistent data, and link them to where they're used.
-
-echo "Waiting for master pd to be attached"
-attempt=0
-while true; do
-  echo Attempt "$(($attempt+1))" to check for /dev/xvdb
-  if [[ -e /dev/xvdb ]]; then
-    echo "Found /dev/xvdb"
-    break
-  fi
-  attempt=$(($attempt+1))
-  sleep 1
-done
-
-# Mount Master Persistent Disk
-echo "Mounting master-pd"
-mkdir -p /mnt/master-pd
-mkfs -t ext4 /dev/xvdb
-echo "/dev/xvdb  /mnt/master-pd  ext4  noatime  0 0" >> /etc/fstab
-mount /mnt/master-pd
-
-# Contains all the data stored in etcd
-mkdir -m 700 -p /mnt/master-pd/var/etcd
-# Contains the dynamically generated apiserver auth certs and keys
-mkdir -p /mnt/master-pd/srv/kubernetes
-# Contains the cluster's initial config parameters and auth tokens
-mkdir -p /mnt/master-pd/srv/salt-overlay
-# Directory for kube-apiserver to store SSH key (if necessary)
-mkdir -p /mnt/master-pd/srv/sshproxy
-
-ln -s -f /mnt/master-pd/var/etcd /var/etcd
-ln -s -f /mnt/master-pd/srv/kubernetes /srv/kubernetes
-ln -s -f /mnt/master-pd/srv/sshproxy /srv/sshproxy
-ln -s -f /mnt/master-pd/srv/salt-overlay /srv/salt-overlay
-
-# This is a bit of a hack to get around the fact that salt has to run after the
-# PD and mounted directory are already set up. We can't give ownership of the
-# directory to etcd until the etcd user and group exist, but they don't exist
-# until salt runs if we don't create them here. We could alternatively make the
-# permissions on the directory more permissive, but this seems less bad.
-if ! id etcd &>/dev/null; then
-  useradd -s /sbin/nologin -d /var/etcd etcd
-fi
-chown -R etcd /mnt/master-pd/var/etcd
-chgrp -R etcd /mnt/master-pd/var/etcd

cluster/common.sh

@@ -672,6 +672,8 @@ function create-certs {
   fi
   sans="${sans}IP:${service_ip},DNS:kubernetes,DNS:kubernetes.default,DNS:kubernetes.default.svc,DNS:kubernetes.default.svc.${DNS_DOMAIN},DNS:${MASTER_NAME}"
 
+  echo "Generating certs for alternate-names: ${sans}"
+
   local -r cert_create_debug_output=$(mktemp "${KUBE_TEMP}/cert_create_debug_output.XXX")
   # Note: This was heavily cribbed from make-ca-cert.sh
   (set -x

cluster/gce/configure-vm.sh

@@ -18,6 +18,10 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
+# Note that this script is also used by AWS; we include it and then override
+# functions with AWS equivalents.  Note `#+AWS_OVERRIDES_HERE` below.
+# TODO(justinsb): Refactor into common script & GCE specific script?
+
 # If we have any arguments at all, this is a push and not just setup.
 is_push=$@
 
@@ -44,6 +48,16 @@ function ensure-basic-networking() {
   echo "Networking functional on $(hostname) ($(hostname -i))"
 }
 
+# A hookpoint for installing any needed packages
+ensure-packages() {
+  :
+}
+
+# A hookpoint for setting up local devices
+ensure-local-disks() {
+  :
+}
+
 function ensure-install-dir() {
   INSTALL_DIR="/var/cache/kubernetes-install"
   mkdir -p ${INSTALL_DIR}
@@ -57,11 +71,11 @@ EOF
 }
 
 function set-broken-motd() {
-  echo -e '\nBroken (or in progress) GCE Kubernetes node setup! Suggested first step:\n  tail /var/log/startupscript.log\n' > /etc/motd
+  echo -e '\nBroken (or in progress) Kubernetes node setup! Suggested first step:\n  tail /var/log/startupscript.log\n' > /etc/motd
 }
 
 function set-good-motd() {
-  echo -e '\n=== GCE Kubernetes node setup complete ===\n' > /etc/motd
+  echo -e '\n=== Kubernetes node setup complete ===\n' > /etc/motd
 }
 
 function curl-metadata() {
@@ -88,11 +102,7 @@ for k,v in yaml.load(sys.stdin).iteritems():
 
 function remove-docker-artifacts() {
   echo "== Deleting docker0 =="
-  # Forcibly install bridge-utils (options borrowed from Salt logs).
-  until apt-get -q -y -o DPkg::Options::=--force-confold -o DPkg::Options::=--force-confdef install bridge-utils; do
-    echo "== install of bridge-utils failed, retrying =="
-    sleep 5
-  done
+  apt-get-install bridge-utils
 
   # Remove docker artifacts on minion nodes, if present
   iptables -t nat -F || true
@@ -125,6 +135,23 @@ validate-hash() {
   fi
 }
 
+apt-get-install() {
+  # Forcibly install packages (options borrowed from Salt logs).
+  until apt-get -q -y -o DPkg::Options::=--force-confold -o DPkg::Options::=--force-confdef install $@; do
+    echo "== install of packages $@ failed, retrying =="
+    sleep 5
+  done
+}
+
+apt-get-update() {
+  echo "== Refreshing package database =="
+  until apt-get update; do
+    echo "== apt-get update failed, retrying =="
+    echo sleep 5
+  done
+}
+
+#
 # Install salt from GCS.  See README.md for instructions on how to update these
 # debs.
 install-salt() {
@@ -205,6 +232,18 @@ stop-salt-minion() {
   done
 }
 
+# Finds the master PD device; returns it in MASTER_PD_DEVICE
+find-master-pd() {
+  MASTER_PD_DEVICE=""
+  # TODO(zmerlynn): GKE is still lagging in master-pd creation
+  if [[ ! -e /dev/disk/by-id/google-master-pd ]]; then
+    return
+  fi
+  device_info=$(ls -l /dev/disk/by-id/google-master-pd)
+  relative_path=${device_info##* }
+  MASTER_PD_DEVICE="/dev/disk/by-id/${relative_path}"
+}
+
 # Mounts a persistent disk (formatting if needed) to store the persistent data
 # on the master -- etcd's data, a few settings, and security certs/keys/tokens.
 #
@@ -213,19 +252,16 @@ stop-salt-minion() {
 # formats an unformatted disk, and mkdir -p will leave a directory be if it
 # already exists.
 mount-master-pd() {
-  # TODO(zmerlynn): GKE is still lagging in master-pd creation
-  if [[ ! -e /dev/disk/by-id/google-master-pd ]]; then
+  find-master-pd
+  if [[ -z "${MASTER_PD_DEVICE}" ]]; then
     return
   fi
-  device_info=$(ls -l /dev/disk/by-id/google-master-pd)
-  relative_path=${device_info##* }
-  device_path="/dev/disk/by-id/${relative_path}"
 
   # Format and mount the disk, create directories on it for all of the master's
   # persistent data, and link them to where they're used.
   echo "Mounting master-pd"
   mkdir -p /mnt/master-pd
-  /usr/share/google/safe_format_and_mount -m "mkfs.ext4 -F" "${device_path}" /mnt/master-pd &>/var/log/master-pd-mount.log || \
+  /usr/share/google/safe_format_and_mount -m "mkfs.ext4 -F" "${MASTER_PD_DEVICE}" /mnt/master-pd &>/var/log/master-pd-mount.log || \
     { echo "!!! master-pd mount failed, review /var/log/master-pd-mount.log !!!"; return 1; }
   # Contains all the data stored in etcd
   mkdir -m 700 -p /mnt/master-pd/var/etcd
@@ -636,11 +672,8 @@ EOF
   cbr-cidr: ${MASTER_IP_RANGE}
 EOF
   fi
-  if [[ ! -z "${RUNTIME_CONFIG:-}" ]]; then
-    cat <<EOF >>/etc/salt/minion.d/grains.conf
-  runtime_config: '$(echo "$RUNTIME_CONFIG" | sed -e "s/'/''/g")'
-EOF
-  fi
+
+  env-to-grains "runtime_config"
 }
 
 function salt-node-role() {
@@ -654,22 +687,31 @@ grains:
 EOF
 }
 
-function salt-docker-opts() {
-  DOCKER_OPTS=""
+function env-to-grains {
+  local key=$1
+  local env_key=`echo $key | tr '[:lower:]' '[:upper:]'`
+  local value=${!env_key:-}
+  if [[ -n "${value}" ]]; then
+    # Note this is yaml, so indentation matters
+    cat <<EOF >>/etc/salt/minion.d/grains.conf
+  ${key}: '$(echo "${value}" | sed -e "s/'/''/g")'
+EOF
+  fi
+}
 
+function node-docker-opts() {
   if [[ -n "${EXTRA_DOCKER_OPTS-}" ]]; then
-    DOCKER_OPTS="${EXTRA_DOCKER_OPTS}"
+    DOCKER_OPTS="${DOCKER_OPTS:-} ${EXTRA_DOCKER_OPTS}"
   fi
+}
 
-  if [[ -n "{DOCKER_OPTS}" ]]; then
-    cat <<EOF >>/etc/salt/minion.d/grains.conf
-  docker_opts: '$(echo "$DOCKER_OPTS" | sed -e "s/'/''/g")'
-EOF
-  fi
+function salt-grains() {
+  env-to-grains "docker_opts"
+  env-to-grains "docker_root"
+  env-to-grains "kubelet_root"
 }
 
 function configure-salt() {
-  fix-apt-sources
   mkdir -p /etc/salt/minion.d
   salt-run-local
   if [[ "${KUBERNETES_MASTER}" == "true" ]]; then
@@ -679,8 +721,9 @@ function configure-salt() {
     fi
   else
     salt-node-role
-    salt-docker-opts
+    node-docker-opts
   fi
+  salt-grains
   install-salt
   stop-salt-minion
 }
@@ -690,14 +733,33 @@ function run-salt() {
   salt-call --local state.highstate || true
 }
 
+function run-user-script() {
+  if curl-metadata k8s-user-startup-script > "${INSTALL_DIR}/k8s-user-script.sh"; then
+    user_script=$(cat "${INSTALL_DIR}/k8s-user-script.sh")
+  fi
+  if [[ ! -z ${user_script:-} ]]; then
+    chmod u+x "${INSTALL_DIR}/k8s-user-script.sh"
+    echo "== running user startup script =="
+    "${INSTALL_DIR}/k8s-user-script.sh"
+  fi
+}
+
+# This script is re-used on AWS.  Some of the above functions will be replaced.
+# The AWS kube-up script looks for this marker:
+#+AWS_OVERRIDES_HERE
+
 ####################################################################################
 
 if [[ -z "${is_push}" ]]; then
   echo "== kube-up node config starting =="
   set-broken-motd
   ensure-basic-networking
+  fix-apt-sources
+  apt-get-update
   ensure-install-dir
+  ensure-packages
   set-kube-env
+  ensure-local-disks
   [[ "${KUBERNETES_MASTER}" == "true" ]] && mount-master-pd
   create-salt-pillar
   if [[ "${KUBERNETES_MASTER}" == "true" ]]; then
@@ -713,14 +775,7 @@ if [[ -z "${is_push}" ]]; then
   run-salt
   set-good-motd
 
-  if curl-metadata k8s-user-startup-script > "${INSTALL_DIR}/k8s-user-script.sh"; then
-    user_script=$(cat "${INSTALL_DIR}/k8s-user-script.sh")
-  fi
-  if [[ ! -z ${user_script:-} ]]; then
-    chmod u+x "${INSTALL_DIR}/k8s-user-script.sh"
-    echo "== running user startup script =="
-    "${INSTALL_DIR}/k8s-user-script.sh"
-  fi
+  run-user-script
   echo "== kube-up node config done =="
 else
   echo "== kube-push node config starting =="