Merge remote-tracking branch 'upstream/master' into test-cmd-what

Godeps/Godeps.json

@@ -1555,8 +1555,8 @@
 		},
 		{
 			"ImportPath": "github.com/evanphx/json-patch",
-			"Comment": "v4.0.0-3-g36442dbdb58521",
-			"Rev": "36442dbdb585210f8d5a1b45e67aa323c197d5c4"
+			"Comment": "v4.1.0-11-gd4020504c68b6b",
+			"Rev": "d4020504c68b6bfa818032bedfb48e33e9638506"
 		},
 		{
 			"ImportPath": "github.com/exponent-io/jsonpath",
@@ -2101,123 +2101,127 @@
 		},
 		{
 			"ImportPath": "github.com/gophercloud/gophercloud",
-			"Rev": "781450b3c4fcb4f5182bcc5133adb4b2e4a09d1d"
+			"Rev": "c818fa66e4c88b30db28038fe3f18f2f4a0db9a8"
 		},
 		{
 			"ImportPath": "github.com/gophercloud/gophercloud/openstack",
-			"Rev": "781450b3c4fcb4f5182bcc5133adb4b2e4a09d1d"
+			"Rev": "c818fa66e4c88b30db28038fe3f18f2f4a0db9a8"
 		},
 		{
 			"ImportPath": "github.com/gophercloud/gophercloud/openstack/blockstorage/extensions/volumeactions",
-			"Rev": "781450b3c4fcb4f5182bcc5133adb4b2e4a09d1d"
+			"Rev": "c818fa66e4c88b30db28038fe3f18f2f4a0db9a8"
 		},
 		{
 			"ImportPath": "github.com/gophercloud/gophercloud/openstack/blockstorage/v1/volumes",
-			"Rev": "781450b3c4fcb4f5182bcc5133adb4b2e4a09d1d"
+			"Rev": "c818fa66e4c88b30db28038fe3f18f2f4a0db9a8"
 		},
 		{
 			"ImportPath": "github.com/gophercloud/gophercloud/openstack/blockstorage/v2/volumes",
-			"Rev": "781450b3c4fcb4f5182bcc5133adb4b2e4a09d1d"
+			"Rev": "c818fa66e4c88b30db28038fe3f18f2f4a0db9a8"
 		},
 		{
 			"ImportPath": "github.com/gophercloud/gophercloud/openstack/blockstorage/v3/volumes",
-			"Rev": "781450b3c4fcb4f5182bcc5133adb4b2e4a09d1d"
+			"Rev": "c818fa66e4c88b30db28038fe3f18f2f4a0db9a8"
 		},
 		{
 			"ImportPath": "github.com/gophercloud/gophercloud/openstack/common/extensions",
-			"Rev": "781450b3c4fcb4f5182bcc5133adb4b2e4a09d1d"
+			"Rev": "c818fa66e4c88b30db28038fe3f18f2f4a0db9a8"
 		},
 		{
 			"ImportPath": "github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/attachinterfaces",
-			"Rev": "781450b3c4fcb4f5182bcc5133adb4b2e4a09d1d"
+			"Rev": "c818fa66e4c88b30db28038fe3f18f2f4a0db9a8"
 		},
 		{
 			"ImportPath": "github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/volumeattach",
-			"Rev": "781450b3c4fcb4f5182bcc5133adb4b2e4a09d1d"
+			"Rev": "c818fa66e4c88b30db28038fe3f18f2f4a0db9a8"
 		},
 		{
 			"ImportPath": "github.com/gophercloud/gophercloud/openstack/compute/v2/flavors",
-			"Rev": "781450b3c4fcb4f5182bcc5133adb4b2e4a09d1d"
+			"Rev": "c818fa66e4c88b30db28038fe3f18f2f4a0db9a8"
 		},
 		{
 			"ImportPath": "github.com/gophercloud/gophercloud/openstack/compute/v2/images",
-			"Rev": "781450b3c4fcb4f5182bcc5133adb4b2e4a09d1d"
+			"Rev": "c818fa66e4c88b30db28038fe3f18f2f4a0db9a8"
 		},
 		{
 			"ImportPath": "github.com/gophercloud/gophercloud/openstack/compute/v2/servers",
-			"Rev": "781450b3c4fcb4f5182bcc5133adb4b2e4a09d1d"
+			"Rev": "c818fa66e4c88b30db28038fe3f18f2f4a0db9a8"
 		},
 		{
 			"ImportPath": "github.com/gophercloud/gophercloud/openstack/identity/v2/tenants",
-			"Rev": "781450b3c4fcb4f5182bcc5133adb4b2e4a09d1d"
+			"Rev": "c818fa66e4c88b30db28038fe3f18f2f4a0db9a8"
 		},
 		{
 			"ImportPath": "github.com/gophercloud/gophercloud/openstack/identity/v2/tokens",
-			"Rev": "781450b3c4fcb4f5182bcc5133adb4b2e4a09d1d"
+			"Rev": "c818fa66e4c88b30db28038fe3f18f2f4a0db9a8"
 		},
 		{
 			"ImportPath": "github.com/gophercloud/gophercloud/openstack/identity/v3/extensions/trusts",
-			"Rev": "781450b3c4fcb4f5182bcc5133adb4b2e4a09d1d"
+			"Rev": "c818fa66e4c88b30db28038fe3f18f2f4a0db9a8"
 		},
 		{
 			"ImportPath": "github.com/gophercloud/gophercloud/openstack/identity/v3/tokens",
-			"Rev": "781450b3c4fcb4f5182bcc5133adb4b2e4a09d1d"
+			"Rev": "c818fa66e4c88b30db28038fe3f18f2f4a0db9a8"
 		},
 		{
 			"ImportPath": "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions",
-			"Rev": "781450b3c4fcb4f5182bcc5133adb4b2e4a09d1d"
+			"Rev": "c818fa66e4c88b30db28038fe3f18f2f4a0db9a8"
 		},
 		{
 			"ImportPath": "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/external",
-			"Rev": "781450b3c4fcb4f5182bcc5133adb4b2e4a09d1d"
+			"Rev": "c818fa66e4c88b30db28038fe3f18f2f4a0db9a8"
 		},
 		{
 			"ImportPath": "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/layer3/floatingips",
-			"Rev": "781450b3c4fcb4f5182bcc5133adb4b2e4a09d1d"
+			"Rev": "c818fa66e4c88b30db28038fe3f18f2f4a0db9a8"
 		},
 		{
 			"ImportPath": "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/layer3/routers",
-			"Rev": "781450b3c4fcb4f5182bcc5133adb4b2e4a09d1d"
+			"Rev": "c818fa66e4c88b30db28038fe3f18f2f4a0db9a8"
+		},
+		{
+			"ImportPath": "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/lbaas_v2/l7policies",
+			"Rev": "c818fa66e4c88b30db28038fe3f18f2f4a0db9a8"
 		},
 		{
 			"ImportPath": "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/lbaas_v2/listeners",
-			"Rev": "781450b3c4fcb4f5182bcc5133adb4b2e4a09d1d"
+			"Rev": "c818fa66e4c88b30db28038fe3f18f2f4a0db9a8"
 		},
 		{
 			"ImportPath": "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/lbaas_v2/loadbalancers",
-			"Rev": "781450b3c4fcb4f5182bcc5133adb4b2e4a09d1d"
+			"Rev": "c818fa66e4c88b30db28038fe3f18f2f4a0db9a8"
 		},
 		{
 			"ImportPath": "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/lbaas_v2/monitors",
-			"Rev": "781450b3c4fcb4f5182bcc5133adb4b2e4a09d1d"
+			"Rev": "c818fa66e4c88b30db28038fe3f18f2f4a0db9a8"
 		},
 		{
 			"ImportPath": "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/lbaas_v2/pools",
-			"Rev": "781450b3c4fcb4f5182bcc5133adb4b2e4a09d1d"
+			"Rev": "c818fa66e4c88b30db28038fe3f18f2f4a0db9a8"
 		},
 		{
 			"ImportPath": "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/security/groups",
-			"Rev": "781450b3c4fcb4f5182bcc5133adb4b2e4a09d1d"
+			"Rev": "c818fa66e4c88b30db28038fe3f18f2f4a0db9a8"
 		},
 		{
 			"ImportPath": "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/security/rules",
-			"Rev": "781450b3c4fcb4f5182bcc5133adb4b2e4a09d1d"
+			"Rev": "c818fa66e4c88b30db28038fe3f18f2f4a0db9a8"
 		},
 		{
 			"ImportPath": "github.com/gophercloud/gophercloud/openstack/networking/v2/networks",
-			"Rev": "781450b3c4fcb4f5182bcc5133adb4b2e4a09d1d"
+			"Rev": "c818fa66e4c88b30db28038fe3f18f2f4a0db9a8"
 		},
 		{
 			"ImportPath": "github.com/gophercloud/gophercloud/openstack/networking/v2/ports",
-			"Rev": "781450b3c4fcb4f5182bcc5133adb4b2e4a09d1d"
+			"Rev": "c818fa66e4c88b30db28038fe3f18f2f4a0db9a8"
 		},
 		{
 			"ImportPath": "github.com/gophercloud/gophercloud/openstack/utils",
-			"Rev": "781450b3c4fcb4f5182bcc5133adb4b2e4a09d1d"
+			"Rev": "c818fa66e4c88b30db28038fe3f18f2f4a0db9a8"
 		},
 		{
 			"ImportPath": "github.com/gophercloud/gophercloud/pagination",
-			"Rev": "781450b3c4fcb4f5182bcc5133adb4b2e4a09d1d"
+			"Rev": "c818fa66e4c88b30db28038fe3f18f2f4a0db9a8"
 		},
 		{
 			"ImportPath": "github.com/gorilla/websocket",
@@ -4090,39 +4094,51 @@
 		},
 		{
 			"ImportPath": "k8s.io/utils/buffer",
-			"Rev": "8a87304934321b4b0ad72a7cb3cbc715d67d38c7"
+			"Rev": "ed37f7428a91fc2a81070808937195dcd46d320e"
 		},
 		{
 			"ImportPath": "k8s.io/utils/clock",
-			"Rev": "8a87304934321b4b0ad72a7cb3cbc715d67d38c7"
+			"Rev": "ed37f7428a91fc2a81070808937195dcd46d320e"
 		},
 		{
 			"ImportPath": "k8s.io/utils/exec",
-			"Rev": "8a87304934321b4b0ad72a7cb3cbc715d67d38c7"
+			"Rev": "ed37f7428a91fc2a81070808937195dcd46d320e"
 		},
 		{
 			"ImportPath": "k8s.io/utils/exec/testing",
-			"Rev": "8a87304934321b4b0ad72a7cb3cbc715d67d38c7"
+			"Rev": "ed37f7428a91fc2a81070808937195dcd46d320e"
 		},
 		{
 			"ImportPath": "k8s.io/utils/integer",
-			"Rev": "8a87304934321b4b0ad72a7cb3cbc715d67d38c7"
+			"Rev": "ed37f7428a91fc2a81070808937195dcd46d320e"
 		},
 		{
 			"ImportPath": "k8s.io/utils/io",
-			"Rev": "8a87304934321b4b0ad72a7cb3cbc715d67d38c7"
+			"Rev": "ed37f7428a91fc2a81070808937195dcd46d320e"
+		},
+		{
+			"ImportPath": "k8s.io/utils/keymutex",
+			"Rev": "ed37f7428a91fc2a81070808937195dcd46d320e"
 		},
 		{
 			"ImportPath": "k8s.io/utils/nsenter",
-			"Rev": "8a87304934321b4b0ad72a7cb3cbc715d67d38c7"
+			"Rev": "ed37f7428a91fc2a81070808937195dcd46d320e"
+		},
+		{
+			"ImportPath": "k8s.io/utils/path",
+			"Rev": "ed37f7428a91fc2a81070808937195dcd46d320e"
 		},
 		{
 			"ImportPath": "k8s.io/utils/pointer",
-			"Rev": "8a87304934321b4b0ad72a7cb3cbc715d67d38c7"
+			"Rev": "ed37f7428a91fc2a81070808937195dcd46d320e"
+		},
+		{
+			"ImportPath": "k8s.io/utils/strings",
+			"Rev": "ed37f7428a91fc2a81070808937195dcd46d320e"
 		},
 		{
 			"ImportPath": "k8s.io/utils/trace",
-			"Rev": "8a87304934321b4b0ad72a7cb3cbc715d67d38c7"
+			"Rev": "ed37f7428a91fc2a81070808937195dcd46d320e"
 		},
 		{
 			"ImportPath": "sigs.k8s.io/yaml",

api/openapi-spec/swagger.json

@@ -92006,6 +92006,10 @@
       "description": "singularName is the singular name of the resource.  This allows clients to handle plural and singular opaquely. The singularName is more correct for reporting status on a single item and both singular and plural are allowed from the kubectl CLI interface.",
       "type": "string"
      },
+     "storageVersionHash": {
+      "description": "The hash value of the storage version, the version this resource is converted to when written to the data store. Value must be treated as opaque by clients. Only equality comparison on the value is valid. This is an alpha feature and may change or be removed in the future. The field is populated by the apiserver only if the StorageVersionHash feature gate is enabled. This field will remain optional even if it graduates.",
+      "type": "string"
+     },
      "verbs": {
       "description": "verbs is a list of supported kube verbs (this includes get, list, watch, create, update, patch, delete, deletecollection, and proxy)",
       "type": "array",

build/common.sh

@@ -26,13 +26,13 @@ USER_ID=$(id -u)
 GROUP_ID=$(id -g)
 
 DOCKER_OPTS=${DOCKER_OPTS:-""}
-DOCKER=(docker ${DOCKER_OPTS})
+IFS=" " read -r -a DOCKER <<< "docker ${DOCKER_OPTS}"
 DOCKER_HOST=${DOCKER_HOST:-""}
 DOCKER_MACHINE_NAME=${DOCKER_MACHINE_NAME:-"kube-dev"}
 readonly DOCKER_MACHINE_DRIVER=${DOCKER_MACHINE_DRIVER:-"virtualbox --virtualbox-cpu-count -1"}
 
 # This will canonicalize the path
-KUBE_ROOT=$(cd "$(dirname "${BASH_SOURCE}")"/.. && pwd -P)
+KUBE_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd -P)
 
 source "${KUBE_ROOT}/hack/lib/init.sh"
 
@@ -137,7 +137,7 @@ function kube::build::verify_prereqs() {
     fi
     kube::util::ensure_docker_daemon_connectivity || return 1
 
-    if (( ${KUBE_VERBOSE} > 6 )); then
+    if (( KUBE_VERBOSE > 6 )); then
       kube::log::status "Docker Version:"
       "${DOCKER[@]}" version | kube::log::info_from_stdin
     fi
@@ -185,7 +185,8 @@ function kube::build::docker_available_on_osx() {
 function kube::build::prepare_docker_machine() {
   kube::log::status "docker-machine was found."
 
-  local available_memory_bytes=$(sysctl -n hw.memsize 2>/dev/null)
+  local available_memory_bytes
+  available_memory_bytes=$(sysctl -n hw.memsize 2>/dev/null)
 
   local bytes_in_mb=1048576
 
@@ -193,7 +194,7 @@ function kube::build::prepare_docker_machine() {
   # of multiple by .5, because bash can only multiply by ints.
   local memory_divisor=2
 
-  local virtualbox_memory_mb=$(( ${available_memory_bytes} / (${bytes_in_mb} * ${memory_divisor}) ))
+  local virtualbox_memory_mb=$(( available_memory_bytes / (bytes_in_mb * memory_divisor) ))
 
   docker-machine inspect "${DOCKER_MACHINE_NAME}" &> /dev/null || {
     kube::log::status "Creating a machine to build Kubernetes"
@@ -215,7 +216,7 @@ function kube::build::prepare_docker_machine() {
   while ! docker_machine_out=$(docker-machine env "${DOCKER_MACHINE_NAME}" 2>&1); do
     if [[ ${docker_machine_out} =~ "Error checking TLS connection" ]]; then
       echo "${docker_machine_out}"
-      docker-machine regenerate-certs ${DOCKER_MACHINE_NAME}
+      docker-machine regenerate-certs "${DOCKER_MACHINE_NAME}"
     else
       sleep 1
     fi
@@ -368,7 +369,7 @@ function kube::build::short_hash() {
   else
     short_hash=$(echo -n "$1" | md5sum)
   fi
-  echo ${short_hash:0:10}
+  echo "${short_hash:0:10}"
 }
 
 # Pedantically kill, wait-on and remove a container. The -f -v options
@@ -412,7 +413,7 @@ function kube::build::clean() {
 function kube::build::build_image() {
   mkdir -p "${LOCAL_OUTPUT_BUILD_CONTEXT}"
   # Make sure the context directory owned by the right user for syncing sources to container.
-  chown -R ${USER_ID}:${GROUP_ID} "${LOCAL_OUTPUT_BUILD_CONTEXT}"
+  chown -R "${USER_ID}":"${GROUP_ID}" "${LOCAL_OUTPUT_BUILD_CONTEXT}"
 
   cp /etc/localtime "${LOCAL_OUTPUT_BUILD_CONTEXT}/"
 
@@ -503,7 +504,7 @@ function kube::build::ensure_data_container() {
       --name "${KUBE_DATA_CONTAINER_NAME}"
       --hostname "${HOSTNAME}"
       "${KUBE_BUILD_IMAGE}"
-      chown -R ${USER_ID}:${GROUP_ID}
+      chown -R "${USER_ID}":"${GROUP_ID}"
         "${REMOTE_ROOT}"
         /usr/local/go/pkg/
     )
@@ -582,7 +583,7 @@ function kube::build::run_build_command_ex() {
   if [[ -t 0 ]]; then
     docker_run_opts+=(--interactive --tty)
   elif [[ "${detach}" == false ]]; then
-    docker_run_opts+=(--attach=stdout --attach=stderr)
+    docker_run_opts+=("--attach=stdout" "--attach=stderr")
   fi
 
   local -ra docker_cmd=(
@@ -599,13 +600,13 @@ function kube::build::run_build_command_ex() {
 function kube::build::rsync_probe {
   # Wait unil rsync is up and running.
   local tries=20
-  while (( ${tries} > 0 )) ; do
+  while (( tries > 0 )) ; do
     if rsync "rsync://k8s@${1}:${2}/" \
          --password-file="${LOCAL_OUTPUT_BUILD_CONTEXT}/rsyncd.password" \
          &> /dev/null ; then
       return 0
     fi
-    tries=$(( ${tries} - 1))
+    tries=$(( tries - 1))
     sleep 0.1
   done
 
@@ -625,7 +626,7 @@ function kube::build::start_rsyncd_container() {
   kube::build::stop_rsyncd_container
   V=3 kube::log::status "Starting rsyncd container"
   kube::build::run_build_command_ex \
-    "${KUBE_RSYNC_CONTAINER_NAME}" -p 127.0.0.1:${KUBE_RSYNC_PORT}:${KUBE_CONTAINER_RSYNC_PORT} -d \
+    "${KUBE_RSYNC_CONTAINER_NAME}" -p 127.0.0.1:"${KUBE_RSYNC_PORT}":"${KUBE_CONTAINER_RSYNC_PORT}" -d \
     -e ALLOW_HOST="$(${IPTOOL} | grep -Eo 'inet (addr:)?([0-9]*\.){3}[0-9]*' | grep -Eo '([0-9]*\.){3}[0-9]*' | grep -v '127.0.0.1')" \
     -- /rsyncd.sh >/dev/null
 
@@ -643,7 +644,7 @@ function kube::build::start_rsyncd_container() {
   # machines) we have to talk directly to the container IP.  There is no one
   # strategy that works in all cases so we test to figure out which situation we
   # are in.
-  if kube::build::rsync_probe 127.0.0.1 ${mapped_port}; then
+  if kube::build::rsync_probe 127.0.0.1 "${mapped_port}"; then
     KUBE_RSYNC_ADDR="127.0.0.1:${mapped_port}"
     return 0
   elif kube::build::rsync_probe "${container_ip}" ${KUBE_CONTAINER_RSYNC_PORT}; then
@@ -664,12 +665,12 @@ function kube::build::stop_rsyncd_container() {
 function kube::build::rsync {
   local -a rsync_opts=(
     --archive
-    --password-file="${LOCAL_OUTPUT_BUILD_CONTEXT}/rsyncd.password"
+    "--password-file=${LOCAL_OUTPUT_BUILD_CONTEXT}/rsyncd.password"
   )
-  if (( ${KUBE_VERBOSE} >= 6 )); then
+  if (( KUBE_VERBOSE >= 6 )); then
     rsync_opts+=("-iv")
   fi
-  if (( ${KUBE_RSYNC_COMPRESS} > 0 )); then
+  if (( KUBE_RSYNC_COMPRESS > 0 )); then
      rsync_opts+=("--compress-level=${KUBE_RSYNC_COMPRESS}")
   fi
   V=3 kube::log::status "Running rsync"
@@ -712,11 +713,6 @@ function kube::build::copy_output() {
 
   kube::build::start_rsyncd_container
 
-  local rsync_extra=""
-  if (( ${KUBE_VERBOSE} >= 6 )); then
-    rsync_extra="-iv"
-  fi
-
   # The filter syntax for rsync is a little obscure. It filters on files and
   # directories.  If you don't go in to a directory you won't find any files
   # there.  Rules are evaluated in order.  The last two rules are a little

build/debian-base/Makefile

@@ -18,7 +18,7 @@ REGISTRY ?= staging-k8s.gcr.io
 IMAGE ?= $(REGISTRY)/debian-base
 BUILD_IMAGE ?= debian-build
 
-TAG ?= 0.4.0
+TAG ?= 0.4.1
 
 TAR_FILE ?= rootfs.tar
 ARCH?=amd64

build/debian-hyperkube-base/Makefile

@@ -19,12 +19,12 @@
 
 REGISTRY?=staging-k8s.gcr.io
 IMAGE?=$(REGISTRY)/debian-hyperkube-base
-TAG=0.12.0
+TAG=0.12.1
 ARCH?=amd64
 ALL_ARCH = amd64 arm arm64 ppc64le s390x
 CACHEBUST?=1
 
-BASEIMAGE=k8s.gcr.io/debian-base-$(ARCH):0.4.0
+BASEIMAGE=k8s.gcr.io/debian-base-$(ARCH):0.4.1
 CNI_VERSION=v0.6.0
 
 TEMP_DIR:=$(shell mktemp -d)

build/debian-iptables/Makefile

@@ -16,12 +16,12 @@
 
 REGISTRY?="staging-k8s.gcr.io"
 IMAGE=$(REGISTRY)/debian-iptables
-TAG?=v11.0
+TAG?=v11.0.1
 ARCH?=amd64
 ALL_ARCH = amd64 arm arm64 ppc64le s390x
 TEMP_DIR:=$(shell mktemp -d)
 
-BASEIMAGE?=k8s.gcr.io/debian-base-$(ARCH):0.4.0
+BASEIMAGE?=k8s.gcr.io/debian-base-$(ARCH):0.4.1
 
 # This option is for running docker manifest command
 export DOCKER_CLI_EXPERIMENTAL := enabled

build/root/Makefile.generated_files

@@ -296,16 +296,23 @@ $(DEFAULTER_GEN): $(k8s.io/kubernetes/vendor/k8s.io/code-generator/cmd/defaulter
 
 
 # Conversion generation
+
+# Any package that wants conversion functions generated into it must
+# include one or more comment-tags in its `doc.go` file, of the form:
+#     // +k8s:conversion-gen=<INTERNAL_TYPES_DIR>
+#
+# The INTERNAL_TYPES_DIR is a project-local path to another directory
+# which should be considered when evaluating peer types for
+# conversions.  An optional additional comment of the form
+#     // +k8s:conversion-gen-external-types=<EXTERNAL_TYPES_DIR>
 #
-# Any package that wants conversion functions generated must include one or
-# more comment-tags in any .go file, in column 0, of the form:
-#     // +k8s:conversion-gen=<CONVERSION_TARGET_DIR>
+# identifies where to find the external types; if there is no such
+# comment then the external types are sought in the package where the
+# `k8s:conversion` tag is found.
 #
-# The CONVERSION_TARGET_DIR is a project-local path to another directory which
-# should be considered when evaluating peer types for conversions.  Types which
-# are found in the source package (where conversions are being generated)
-# but do not have a peer in one of the target directories will not have
-# conversions generated.
+# Conversions, in both directions, are generated for every type name
+# that is defined in both an internal types package and the external
+# types package.
 #
 # TODO: it might be better in the long term to make peer-types explicit in the
 # IDL.

cluster/addons/dns-horizontal-autoscaler/dns-horizontal-autoscaler.yaml

@@ -33,7 +33,7 @@ rules:
   - apiGroups: [""]
     resources: ["replicationcontrollers/scale"]
     verbs: ["get", "update"]
-  - apiGroups: ["extensions"]
+  - apiGroups: ["extensions","apps"]
     resources: ["deployments/scale", "replicasets/scale"]
     verbs: ["get", "update"]
 # Remove the configmaps rule once below issue is fixed:
@@ -85,7 +85,7 @@ spec:
         fsGroup: 65534
       containers:
       - name: autoscaler
-        image: k8s.gcr.io/cluster-proportional-autoscaler-amd64:1.3.0
+        image: k8s.gcr.io/cluster-proportional-autoscaler-amd64:1.4.0
         resources:
             requests:
                 cpu: "20m"

cluster/addons/dns/coredns/coredns.yaml.base

@@ -70,7 +70,7 @@ data:
             fallthrough in-addr.arpa ip6.arpa
         }
         prometheus :9153
-        proxy . /etc/resolv.conf
+        forward . /etc/resolv.conf
         cache 30
         loop
         reload
@@ -114,7 +114,7 @@ spec:
         beta.kubernetes.io/os: linux
       containers:
       - name: coredns
-        image: k8s.gcr.io/coredns:1.2.6
+        image: k8s.gcr.io/coredns:1.3.1
         imagePullPolicy: IfNotPresent
         resources:
           limits:

cluster/addons/dns/coredns/coredns.yaml.in

@@ -70,7 +70,7 @@ data:
             fallthrough in-addr.arpa ip6.arpa
         }
         prometheus :9153
-        proxy . /etc/resolv.conf
+        forward . /etc/resolv.conf
         cache 30
         loop
         reload
@@ -114,7 +114,7 @@ spec:
         beta.kubernetes.io/os: linux
       containers:
       - name: coredns
-        image: k8s.gcr.io/coredns:1.2.6
+        image: k8s.gcr.io/coredns:1.3.1
         imagePullPolicy: IfNotPresent
         resources:
           limits:

cluster/addons/dns/coredns/coredns.yaml.sed

@@ -70,7 +70,7 @@ data:
             fallthrough in-addr.arpa ip6.arpa
         }
         prometheus :9153
-        proxy . /etc/resolv.conf
+        forward . /etc/resolv.conf
         cache 30
         loop
         reload
@@ -114,7 +114,7 @@ spec:
         beta.kubernetes.io/os: linux
       containers:
       - name: coredns
-        image: k8s.gcr.io/coredns:1.2.6
+        image: k8s.gcr.io/coredns:1.3.1
         imagePullPolicy: IfNotPresent
         resources:
           limits:

cluster/addons/fluentd-gcp/fluentd-gcp-ds.yaml

@@ -104,6 +104,7 @@ spec:
       # END_PROMETHEUS_TO_SD
       nodeSelector:
         beta.kubernetes.io/fluentd-ds-ready: "true"
+        beta.kubernetes.io/os: linux
       terminationGracePeriodSeconds: 60
       tolerations:
       - operator: "Exists"

cluster/addons/metadata-proxy/gce/metadata-proxy.yaml

@@ -89,4 +89,5 @@ spec:
       # END_PROMETHEUS_TO_SD
       nodeSelector:
         beta.kubernetes.io/metadata-proxy-ready: "true"
+        beta.kubernetes.io/os: linux
       terminationGracePeriodSeconds: 30

cluster/common.sh

@@ -321,6 +321,9 @@ function find-tar() {
 #   KUBE_MANIFESTS_TAR
 function find-release-tars() {
   SERVER_BINARY_TAR=$(find-tar kubernetes-server-linux-amd64.tar.gz)
+  if [[ "${NUM_WINDOWS_NODES}" -gt "0" && "${USE_RELEASE_NODE_BINARIES:-false}" == "false" ]]; then
+    NODE_BINARY_TAR=$(find-tar kubernetes-node-windows-amd64.tar.gz)
+  fi
 
   # This tarball is used by GCI, Ubuntu Trusty, and Container Linux.
   KUBE_MANIFESTS_TAR=

cluster/gce/config-common.sh

@@ -14,26 +14,36 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# Returns the total number of Linux and Windows nodes in the cluster.
+#
+# Vars assumed:
+#   NUM_NODES
+#   NUM_WINDOWS_NODES
+function get-num-nodes {
+  echo "$((${NUM_NODES} + ${NUM_WINDOWS_NODES}))"
+}
+
 # Vars assumed:
 #   NUM_NODES
+#   NUM_WINDOWS_NODES
 function get-master-size {
   local suggested_master_size=1
-  if [[ "${NUM_NODES}" -gt "5" ]]; then
+  if [[ "$(get-num-nodes)" -gt "5" ]]; then
     suggested_master_size=2
   fi
-  if [[ "${NUM_NODES}" -gt "10" ]]; then
+  if [[ "$(get-num-nodes)" -gt "10" ]]; then
     suggested_master_size=4
   fi
-  if [[ "${NUM_NODES}" -gt "100" ]]; then
+  if [[ "$(get-num-nodes)" -gt "100" ]]; then
     suggested_master_size=8
   fi
-  if [[ "${NUM_NODES}" -gt "250" ]]; then
+  if [[ "$(get-num-nodes)" -gt "250" ]]; then
     suggested_master_size=16
   fi
-  if [[ "${NUM_NODES}" -gt "500" ]]; then
+  if [[ "$(get-num-nodes)" -gt "500" ]]; then
     suggested_master_size=32
   fi
-  if [[ "${NUM_NODES}" -gt "3000" ]]; then
+  if [[ "$(get-num-nodes)" -gt "3000" ]]; then
     suggested_master_size=64
   fi
   echo "${suggested_master_size}"
@@ -41,25 +51,27 @@ function get-master-size {
 
 # Vars assumed:
 #   NUM_NODES
+#   NUM_WINDOWS_NODES
 function get-master-root-disk-size() {
   local suggested_master_root_disk_size="20GB"
-  if [[ "${NUM_NODES}" -gt "1000" ]]; then
-    suggested_master_root_disk_size="50GB"
-  fi
-  if [[ "${NUM_NODES}" -gt "2000" ]]; then
+  if [[ "$(get-num-nodes)" -gt "500" ]]; then
     suggested_master_root_disk_size="100GB"
   fi
+  if [[ "$(get-num-nodes)" -gt "3000" ]]; then
+    suggested_master_root_disk_size="500GB"
+  fi
   echo "${suggested_master_root_disk_size}"
 }
 
 # Vars assumed:
 #   NUM_NODES
+#   NUM_WINDOWS_NODES
 function get-master-disk-size() {
   local suggested_master_disk_size="20GB"
-  if [[ "${NUM_NODES}" -gt "1000" ]]; then
+  if [[ "$(get-num-nodes)" -gt "500" ]]; then
     suggested_master_disk_size="100GB"
   fi
-  if [[ "${NUM_NODES}" -gt "2000" ]]; then
+  if [[ "$(get-num-nodes)" -gt "3000" ]]; then
     suggested_master_disk_size="200GB"
   fi
   echo "${suggested_master_disk_size}"
@@ -72,13 +84,13 @@ function get-node-ip-range {
     return
   fi
   local suggested_range="10.40.0.0/22"
-  if [[ "${NUM_NODES}" -gt 1000 ]]; then
+  if [[ "$(get-num-nodes)" -gt 1000 ]]; then
     suggested_range="10.40.0.0/21"
   fi
-  if [[ "${NUM_NODES}" -gt 2000 ]]; then
+  if [[ "$(get-num-nodes)" -gt 2000 ]]; then
     suggested_range="10.40.0.0/20"
   fi
-  if [[ "${NUM_NODES}" -gt 4000 ]]; then
+  if [[ "$(get-num-nodes)" -gt 4000 ]]; then
     suggested_range="10.40.0.0/19"
   fi
   echo "${suggested_range}"
@@ -86,13 +98,13 @@ function get-node-ip-range {
 
 function get-cluster-ip-range {
   local suggested_range="10.64.0.0/14"
-  if [[ "${NUM_NODES}" -gt 1000 ]]; then
+  if [[ "$(get-num-nodes)" -gt 1000 ]]; then
     suggested_range="10.64.0.0/13"
   fi
-  if [[ "${NUM_NODES}" -gt 2000 ]]; then
+  if [[ "$(get-num-nodes)" -gt 2000 ]]; then
     suggested_range="10.64.0.0/12"
   fi
-  if [[ "${NUM_NODES}" -gt 4000 ]]; then
+  if [[ "$(get-num-nodes)" -gt 4000 ]]; then
     suggested_range="10.64.0.0/11"
   fi
   echo "${suggested_range}"
@@ -114,3 +126,26 @@ function get-alias-range-size() {
 # NOTE: Avoid giving nodes empty scopes, because kubelet needs a service account
 # in order to initialize properly.
 NODE_SCOPES="${NODE_SCOPES:-monitoring,logging-write,storage-ro}"
+
+# Root directory for Kubernetes files on Windows nodes.
+WINDOWS_K8S_DIR="C:\etc\kubernetes"
+# Directory where Kubernetes binaries will be installed on Windows nodes.
+WINDOWS_NODE_DIR="${WINDOWS_K8S_DIR}\node\bin"
+# Directory where Kubernetes log files will be stored on Windows nodes.
+WINDOWS_LOGS_DIR="${WINDOWS_K8S_DIR}\logs"
+# Directory where CNI binaries will be stored on Windows nodes.
+WINDOWS_CNI_DIR="${WINDOWS_K8S_DIR}\cni"
+# Directory where CNI config files will be stored on Windows nodes.
+WINDOWS_CNI_CONFIG_DIR="${WINDOWS_K8S_DIR}\cni\config"
+# Pod manifests directory for Windows nodes on Windows nodes.
+WINDOWS_MANIFESTS_DIR="${WINDOWS_K8S_DIR}\manifests"
+# Directory where cert/key files will be stores on Windows nodes.
+WINDOWS_PKI_DIR="${WINDOWS_K8S_DIR}\pki"
+# Path for kubelet config file on Windows nodes.
+WINDOWS_KUBELET_CONFIG_FILE="${WINDOWS_K8S_DIR}\kubelet-config.yaml"
+# Path for kubeconfig file on Windows nodes.
+WINDOWS_KUBECONFIG_FILE="${WINDOWS_K8S_DIR}\kubelet.kubeconfig"
+# Path for bootstrap kubeconfig file on Windows nodes.
+WINDOWS_BOOTSTRAP_KUBECONFIG_FILE="${WINDOWS_K8S_DIR}\kubelet.bootstrap-kubeconfig"
+# Path for kube-proxy kubeconfig file on Windows nodes.
+WINDOWS_KUBEPROXY_KUBECONFIG_FILE="${WINDOWS_K8S_DIR}\kubeproxy.kubeconfig"

cluster/gce/config-default.sh

@@ -29,6 +29,7 @@ RELEASE_REGION_FALLBACK=${RELEASE_REGION_FALLBACK:-false}
 REGIONAL_KUBE_ADDONS=${REGIONAL_KUBE_ADDONS:-true}
 NODE_SIZE=${NODE_SIZE:-n1-standard-2}
 NUM_NODES=${NUM_NODES:-3}
+NUM_WINDOWS_NODES=${NUM_WINDOWS_NODES:-0}
 MASTER_SIZE=${MASTER_SIZE:-n1-standard-$(get-master-size)}
 MASTER_MIN_CPU_ARCHITECTURE=${MASTER_MIN_CPU_ARCHITECTURE:-} # To allow choosing better architectures.
 MASTER_DISK_TYPE=pd-ssd
@@ -44,6 +45,7 @@ NODE_LOCAL_SSDS=${NODE_LOCAL_SSDS:-0}
 # fluentd is not running as a manifest pod with appropriate label.
 # TODO(piosz): remove this in 1.8
 NODE_LABELS="${KUBE_NODE_LABELS:-beta.kubernetes.io/fluentd-ds-ready=true}"
+WINDOWS_NODE_LABELS="${WINDOWS_NODE_LABELS:-}"
 
 # An extension to local SSDs allowing users to specify block/fs and SCSI/NVMe devices
 # Format of this variable will be "#,scsi/nvme,block/fs" you can specify multiple
@@ -63,6 +65,7 @@ MIG_WAIT_UNTIL_STABLE_TIMEOUT=${MIG_WAIT_UNTIL_STABLE_TIMEOUT:-1800}
 
 MASTER_OS_DISTRIBUTION=${KUBE_MASTER_OS_DISTRIBUTION:-${KUBE_OS_DISTRIBUTION:-gci}}
 NODE_OS_DISTRIBUTION=${KUBE_NODE_OS_DISTRIBUTION:-${KUBE_OS_DISTRIBUTION:-gci}}
+WINDOWS_NODE_OS_DISTRIBUTION=${WINDOWS_NODE_OS_DISTRIBUTION:-win1803}
 
 if [[ "${MASTER_OS_DISTRIBUTION}" == "cos" ]]; then
     MASTER_OS_DISTRIBUTION="gci"
@@ -173,15 +176,19 @@ HEAPSTER_MACHINE_TYPE="${HEAPSTER_MACHINE_TYPE:-}"
 
 # NON_MASTER_NODE_LABELS are labels will only be applied on non-master nodes.
 NON_MASTER_NODE_LABELS="${KUBE_NON_MASTER_NODE_LABELS:-}"
+WINDOWS_NON_MASTER_NODE_LABELS="${WINDOWS_NON_MASTER_NODE_LABELS:-}"
 
 if [[ "${PREEMPTIBLE_MASTER}" == "true" ]]; then
     NODE_LABELS="${NODE_LABELS},cloud.google.com/gke-preemptible=true"
+    WINDOWS_NODE_LABELS="${WINDOWS_NODE_LABELS},cloud.google.com/gke-preemptible=true"
 elif [[ "${PREEMPTIBLE_NODE}" == "true" ]]; then
     NON_MASTER_NODE_LABELS="${NON_MASTER_NODE_LABELS},cloud.google.com/gke-preemptible=true"
+    WINDOWS_NON_MASTER_NODE_LABELS="${WINDOWS_NON_MASTER_NODE_LABELS},cloud.google.com/gke-preemptible=true"
 fi
 
 # To avoid running Calico on a node that is not configured appropriately,
 # label each Node so that the DaemonSet can run the Pods only on ready Nodes.
+# Windows nodes do not support Calico.
 if [[ ${NETWORK_POLICY_PROVIDER:-} == "calico" ]]; then
 	NON_MASTER_NODE_LABELS="${NON_MASTER_NODE_LABELS:+${NON_MASTER_NODE_LABELS},}projectcalico.org/ds-ready=true"
 fi
@@ -194,6 +201,7 @@ CUSTOM_TYPHA_DEPLOYMENT_YAML="${KUBE_CUSTOM_TYPHA_DEPLOYMENT_YAML:-}"
 
 # To avoid running netd on a node that is not configured appropriately,
 # label each Node so that the DaemonSet can run the Pods only on ready Nodes.
+# Windows nodes do not support netd.
 if [[ ${ENABLE_NETD:-} == "true" ]]; then
 	NON_MASTER_NODE_LABELS="${NON_MASTER_NODE_LABELS:+${NON_MASTER_NODE_LABELS},}cloud.google.com/gke-netd-ready=true"
 fi
@@ -467,3 +475,7 @@ ENABLE_NODE_TERMINATION_HANDLER="${ENABLE_NODE_TERMINATION_HANDLER:-false}"
 if [[ "${NODE_TERMINATION_HANDLER_IMAGE:-}" ]]; then
   PROVIDER_VARS="${PROVIDER_VARS:-} NODE_TERMINATION_HANDLER_IMAGE"
 fi
+
+# Taint Windows nodes by default to prevent Linux workloads from being
+# scheduled onto them.
+WINDOWS_NODE_TAINTS="${WINDOWS_NODE_TAINTS:-node.kubernetes.io/os=windows:NoSchedule}"

cluster/gce/config-test.sh

@@ -29,6 +29,7 @@ RELEASE_REGION_FALLBACK=${RELEASE_REGION_FALLBACK:-false}
 REGIONAL_KUBE_ADDONS=${REGIONAL_KUBE_ADDONS:-true}
 NODE_SIZE=${NODE_SIZE:-n1-standard-2}
 NUM_NODES=${NUM_NODES:-3}
+NUM_WINDOWS_NODES=${NUM_WINDOWS_NODES:-0}
 MASTER_SIZE=${MASTER_SIZE:-n1-standard-$(get-master-size)}
 MASTER_MIN_CPU_ARCHITECTURE=${MASTER_MIN_CPU_ARCHITECTURE:-} # To allow choosing better architectures.
 MASTER_DISK_TYPE=pd-ssd
@@ -44,6 +45,7 @@ NODE_LOCAL_SSDS=${NODE_LOCAL_SSDS:-0}
 # fluentd is not running as a manifest pod with appropriate label.
 # TODO(piosz): remove this in 1.8
 NODE_LABELS="${KUBE_NODE_LABELS:-beta.kubernetes.io/fluentd-ds-ready=true}"
+WINDOWS_NODE_LABELS="${WINDOWS_NODE_LABELS:-}"
 
 # An extension to local SSDs allowing users to specify block/fs and SCSI/NVMe devices
 # Format of this variable will be "#,scsi/nvme,block/fs" you can specify multiple
@@ -66,6 +68,8 @@ MIG_WAIT_UNTIL_STABLE_TIMEOUT=${MIG_WAIT_UNTIL_STABLE_TIMEOUT:-1800}
 
 MASTER_OS_DISTRIBUTION=${KUBE_MASTER_OS_DISTRIBUTION:-${KUBE_OS_DISTRIBUTION:-gci}}
 NODE_OS_DISTRIBUTION=${KUBE_NODE_OS_DISTRIBUTION:-${KUBE_OS_DISTRIBUTION:-gci}}
+WINDOWS_NODE_OS_DISTRIBUTION=${WINDOWS_NODE_OS_DISTRIBUTION:-win1803}
+
 if [[ "${MASTER_OS_DISTRIBUTION}" == "cos" ]]; then
     MASTER_OS_DISTRIBUTION="gci"
 fi
@@ -81,7 +85,7 @@ fi
 
 # To avoid failing large tests due to some flakes in starting nodes, allow
 # for a small percentage of nodes to not start during cluster startup.
-ALLOWED_NOTREADY_NODES="${ALLOWED_NOTREADY_NODES:-$((NUM_NODES / 100))}"
+ALLOWED_NOTREADY_NODES="${ALLOWED_NOTREADY_NODES:-$(($(get-num-nodes) / 100))}"
 
 # By default a cluster will be started with the master and nodes
 # on Container-optimized OS (cos, previously known as gci). If
@@ -208,18 +212,21 @@ fi
 if [[ "${MASTER_OS_DISTRIBUTION}" == "gci" ]] || [[ "${MASTER_OS_DISTRIBUTION}" == "ubuntu" ]]; then 
   MASTER_KUBELET_TEST_ARGS="${MASTER_KUBELET_TEST_ARGS:-} --experimental-kernel-memcg-notification=true"  
 fi
-APISERVER_TEST_ARGS="${APISERVER_TEST_ARGS:-} --vmodule=httplog=3 --runtime-config=extensions/v1beta1,scheduling.k8s.io/v1alpha1,settings.k8s.io/v1alpha1 ${TEST_CLUSTER_DELETE_COLLECTION_WORKERS} ${TEST_CLUSTER_MAX_REQUESTS_INFLIGHT}"
+APISERVER_TEST_ARGS="${APISERVER_TEST_ARGS:-} --runtime-config=extensions/v1beta1,scheduling.k8s.io/v1alpha1,settings.k8s.io/v1alpha1 ${TEST_CLUSTER_DELETE_COLLECTION_WORKERS} ${TEST_CLUSTER_MAX_REQUESTS_INFLIGHT}"
 CONTROLLER_MANAGER_TEST_ARGS="${CONTROLLER_MANAGER_TEST_ARGS:-} ${TEST_CLUSTER_RESYNC_PERIOD} ${TEST_CLUSTER_API_CONTENT_TYPE}"
 SCHEDULER_TEST_ARGS="${SCHEDULER_TEST_ARGS:-} ${TEST_CLUSTER_API_CONTENT_TYPE}"
 KUBEPROXY_TEST_ARGS="${KUBEPROXY_TEST_ARGS:-} ${TEST_CLUSTER_API_CONTENT_TYPE}"
 
 # NON_MASTER_NODE_LABELS are labels will only be applied on non-master nodes.
 NON_MASTER_NODE_LABELS="${KUBE_NON_MASTER_NODE_LABELS:-}"
+WINDOWS_NON_MASTER_NODE_LABELS="${WINDOWS_NON_MASTER_NODE_LABELS:-}"
 
 if [[ "${PREEMPTIBLE_MASTER}" == "true" ]]; then
     NODE_LABELS="${NODE_LABELS},cloud.google.com/gke-preemptible=true"
+    WINDOWS_NODE_LABELS="${WINDOWS_NODE_LABELS},cloud.google.com/gke-preemptible=true"
 elif [[ "${PREEMPTIBLE_NODE}" == "true" ]]; then
     NON_MASTER_NODE_LABELS="${NON_MASTER_NODE_LABELS},cloud.google.com/gke-preemptible=true"
+    WINDOWS_NON_MASTER_NODE_LABELS="${WINDOWS_NON_MASTER_NODE_LABELS},cloud.google.com/gke-preemptible=true"
 fi
 
 # Optional: Enable netd.
@@ -230,6 +237,7 @@ CUSTOM_TYPHA_DEPLOYMENT_YAML="${KUBE_CUSTOM_TYPHA_DEPLOYMENT_YAML:-}"
 
 # To avoid running netd on a node that is not configured appropriately,
 # label each Node so that the DaemonSet can run the Pods only on ready Nodes.
+# Windows nodes do not support netd.
 if [[ ${ENABLE_NETD:-} == "true" ]]; then
 	NON_MASTER_NODE_LABELS="${NON_MASTER_NODE_LABELS:+${NON_MASTER_NODE_LABELS},}cloud.google.com/gke-netd-ready=true"
 fi
@@ -238,6 +246,7 @@ ENABLE_NODELOCAL_DNS="${KUBE_ENABLE_NODELOCAL_DNS:-false}"
 
 # To avoid running Calico on a node that is not configured appropriately,
 # label each Node so that the DaemonSet can run the Pods only on ready Nodes.
+# Windows nodes do not support Calico.
 if [[ ${NETWORK_POLICY_PROVIDER:-} == "calico" ]]; then
 	NON_MASTER_NODE_LABELS="${NON_MASTER_NODE_LABELS:+${NON_MASTER_NODE_LABELS},}projectcalico.org/ds-ready=true"
 fi
@@ -486,3 +495,7 @@ ENABLE_NODE_TERMINATION_HANDLER="${ENABLE_NODE_TERMINATION_HANDLER:-false}"
 if [[ "${NODE_TERMINATION_HANDLER_IMAGE:-}" ]]; then
   PROVIDER_VARS="${PROVIDER_VARS:-} NODE_TERMINATION_HANDLER_IMAGE"
 fi
+
+# Taint Windows nodes by default to prevent Linux workloads from being
+# scheduled onto them.
+WINDOWS_NODE_TAINTS="${WINDOWS_NODE_TAINTS:-node.kubernetes.io/os=windows:NoSchedule}"

cluster/gce/gci/node-helper.sh

@@ -17,7 +17,7 @@
 # A library of helper functions and constant for GCI distro
 source "${KUBE_ROOT}/cluster/gce/gci/helper.sh"
 
-function get-node-instance-metadata {
+function get-node-instance-metadata-from-file {
   local metadata=""
   metadata+="kube-env=${KUBE_TEMP}/node-kube-env.yaml,"
   metadata+="kubelet-config=${KUBE_TEMP}/node-kubelet-config.yaml,"
@@ -34,8 +34,8 @@ function get-node-instance-metadata {
 }
 
 # $1: template name (required).
-function create-node-instance-template {
+function create-linux-node-instance-template {
   local template_name="$1"
   ensure-gci-metadata-files
-  create-node-template "$template_name" "${scope_flags[*]}" "$(get-node-instance-metadata)"
+  create-node-template "${template_name}" "${scope_flags[*]}" "$(get-node-instance-metadata-from-file)" "" "linux"
 }

cluster/gce/win1803/OWNERS

+approvers:
+- yujuhong

cluster/gce/win1803/README-GCE-Windows-kube-up.md

+# Starting a Windows Kubernetes cluster on GCE using kube-up
+
+## Bring up the cluster
+
+Prerequisites: a Google Cloud Platform project.
+
+### 0. Prepare your environment
+
+Clone this repository under your `$GOPATH/src` directory on a Linux machine.
+Then, optionally clean/prepare your environment using these commands:
+
+```
+# Remove files that interfere with get-kube / kube-up:
+rm -rf ./kubernetes/; rm -f kubernetes.tar.gz; rm -f ~/.kube/config
+
+# Set the default gcloud project for this shell. This is optional but convenient
+# if you're working with multiple projects and don't want to repeatedly switch
+# between gcloud config configurations.
+export CLOUDSDK_CORE_PROJECT=<your_project_name>
+```
+
+### 1. Build Kubernetes
+
+The most straightforward approach to build those binaries is to run `make
+release`. However, that builds binaries for all supported platforms, and can be
+slow. You can speed up the process by following the instructions below to only
+build the necessary binaries.
+```
+# Fetch the PR: https://github.com/pjh/kubernetes/pull/43
+git remote add pjh https://github.com/pjh/kubernetes
+git fetch pjh pull/43/head
+
+# Get the commit hash and cherry-pick the commit to your current branch
+BUILD_WIN_COMMIT=$(git ls-remote pjh | grep refs/pull/43/head | cut -f 1)
+git cherry-pick $BUILD_WIN_COMMIT
+
+# Build binaries for both Linux and Windows
+make quick-release
+```
+
+### 2 Create a Kubernetes cluster
+
+You can create a regular Kubernetes cluster or an end-to-end test cluster.
+Please make sure you set the environment variables properly following the
+instructions in the previous section.
+
+First, set the following environment variables which are required for
+controlling the number of Linux and Windows nodes in the cluster and for
+enabling IP aliases (which are required for Windows pod routing):
+
+```
+export NUM_NODES=2  # number of Linux nodes
+export NUM_WINDOWS_NODES=2
+export KUBE_GCE_ENABLE_IP_ALIASES=true
+```
+
+If you wish to use `netd` as the CNI plugin for Linux nodes, set these
+variables:
+
+```
+export KUBE_ENABLE_NETD=true
+export KUBE_CUSTOM_NETD_YAML=$(curl -s \
+  https://raw.githubusercontent.com/GoogleCloudPlatform/netd/master/netd.yaml \
+  | sed -e 's/^/ /')
+```
+
+Now bring up a cluster using one of the following two methods:
+
+#### 2.a Create a regular Kubernetes cluster
+
+```
+# Invoke kube-up.sh with these environment variables:
+#   PROJECT: text name of your GCP project.
+#   KUBERNETES_SKIP_CONFIRM: skips any kube-up prompts.
+PROJECT=${CLOUDSDK_CORE_PROJECT} KUBERNETES_SKIP_CONFIRM=y ./cluster/kube-up.sh
+```
+
+To teardown the cluster run:
+
+```
+PROJECT=${CLOUDSDK_CORE_PROJECT} KUBERNETES_SKIP_CONFIRM=y ./cluster/kube-down.sh
+```
+
+#### 2.b Create a Kubernetes end-to-end (E2E) test cluster
+
+```
+PROJECT=${CLOUDSDK_CORE_PROJECT} go run ./hack/e2e.go  -- --up
+```
+This command, by default, tears down the existing E2E cluster and create a new
+one.
+
+No matter what type of cluster you chose to create, the result should be a
+Kubernetes cluster with one Linux master node, `NUM_NODES` Linux worker nodes
+and `NUM_WINDOWS_NODES` Windows worker nodes.
+
+## Validating the cluster
+
+Invoke this script to run a smoke test that verifies that the cluster has been
+brought up correctly:
+
+```
+cluster/gce/win1803/smoke-test.sh
+```
+
+## Running tests against the cluster
+
+These steps are based on
+[kubernetes-sigs/windows-testing](https://github.com/kubernetes-sigs/windows-testing).
+
+*   TODO(pjh): use patched `cluster/local/util.sh` from
+    https://github.com/pjh/kubernetes/blob/windows-up/cluster/local/util.sh.
+
+*   If necessary run `alias kubectl=client/bin/kubectl` .
+
+*   Set the following environment variables (these values should make sense if
+    you built your cluster using the kube-up steps above):
+
+    ```
+    export KUBE_HOME=$(pwd)
+    export KUBECONFIG=~/.kube/config
+    export KUBE_MASTER=local
+    export KUBE_MASTER_NAME=kubernetes-master
+    export KUBE_MASTER_IP=$(kubectl get node ${KUBE_MASTER_NAME} -o jsonpath='{.status.addresses[?(@.type=="ExternalIP")].address}')
+    export KUBE_MASTER_URL=https://${KUBE_MASTER_IP}
+    export KUBE_MASTER_PORT=443
+    ```
+
+*   Download the list of Windows e2e tests:
+
+    ```
+    curl https://raw.githubusercontent.com/e2e-win/e2e-win-prow-deployment/master/repo-list.txt -o ${KUBE_HOME}/repo-list.yaml
+    export KUBE_TEST_REPO_LIST=${KUBE_HOME}/repo-list.yaml
+    ```
+
+*   Download and configure the list of tests to exclude:
+
+    ```
+    curl https://raw.githubusercontent.com/e2e-win/e2e-win-prow-deployment/master/exclude_conformance_test.txt -o ${KUBE_HOME}/exclude_conformance_test.txt
+    export EXCLUDED_TESTS=$(cat exclude_conformance_test.txt |
+      tr -d '\r' |                # remove Windows carriage returns
+      tr -s '\n' '|' |            # coalesce newlines into |
+      tr -s ' ' '.' |             # coalesce spaces into .
+      sed -e 's/[]\[()]/\\&/g' |  # escape brackets and parentheses
+      sed -e 's/.$//g')           # remove final | added by tr
+    ```
+
+*   Taint the Linux nodes so that test pods will not land on them:
+
+    ```
+    export LINUX_NODES=$(kubectl get nodes -l beta.kubernetes.io/os=linux,kubernetes.io/hostname!=${KUBE_MASTER_NAME} -o name)
+    export LINUX_NODE_COUNT=$(echo ${LINUX_NODES} | wc -w)
+    for node in $LINUX_NODES; do
+      kubectl taint node $node node-under-test=false:NoSchedule
+    done
+    ```
+
+*   Build necessary test binaries:
+
+    ```
+    make WHAT=test/e2e/e2e.test
+    ```
+
+*   Run the tests with flags that point at the "local" (already-running) cluster
+    and that permit the `NoSchedule` Linux nodes:
+
+    ```
+    export KUBETEST_ARGS="--ginkgo.noColor=true "\
+    "--report-dir=${KUBE_HOME}/e2e-reports "\
+    "--allowed-not-ready-nodes=${LINUX_NODE_COUNT} "\
+    "--ginkgo.dryRun=false "\
+    "--ginkgo.focus=\[Conformance\] "\
+    "--ginkgo.skip=${EXCLUDED_TESTS}"
+
+    go run ${KUBE_HOME}/hack/e2e.go -- --verbose-commands \
+      --ginkgo-parallel=4 \
+      --check-version-skew=false --test --provider=local \
+      --test_args="${KUBETEST_ARGS}" &> ${KUBE_HOME}/conformance.out
+    ```
+
+    TODO: copy log files from Windows nodes using some command like:
+
+    ```
+    scp -r -o PreferredAuthentications=keyboard-interactive,password \
+      -o PubkeyAuthentication=no \
+      user@kubernetes-minion-windows-group-mk0p:C:\\etc\\kubernetes\\logs \
+      kubetest-logs/
+    ```

cluster/gce/win1803/common.psm1

+# Copyright 2019 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+<#
+.SYNOPSIS
+  Library containing common variables and code used by other PowerShell modules
+  and scripts for configuring Windows nodes.
+#>
+
+# REDO_STEPS affects the behavior of a node that is rebooted after initial
+# bringup. When true, on a reboot the scripts will redo steps that were
+# determined to have already been completed once (e.g. to overwrite
+# already-existing config files). When false the scripts will perform the
+# minimum required steps to re-join this node to the cluster.
+$REDO_STEPS = $false
+Export-ModuleMember -Variable REDO_STEPS
+
+# Writes $Message to the console. Terminates the script if $Fatal is set.
+function Log-Output {
+  param (
+    [parameter(Mandatory=$true)] [string]$Message,
+    [switch]$Fatal
+  )
+  Write-Host "${Message}"
+  if (${Fatal}) {
+    Exit 1
+  }
+}
+
+# Checks if a file should be written or overwritten by testing if it already
+# exists and checking the value of the global $REDO_STEPS variable. Emits an
+# informative message if the file already exists.
+#
+# Returns $true if the file does not exist, or if it does but the global
+# $REDO_STEPS variable is set to $true. Returns $false if the file exists and
+# the caller should not overwrite it.
+function ShouldWrite-File {
+  param (
+    [parameter(Mandatory=$true)] [string]$Filename
+  )
+  if (Test-Path $Filename) {
+    if ($REDO_STEPS) {
+      Log-Output "Warning: $Filename already exists, will overwrite it"
+      return $true
+    }
+    Log-Output "Skip: $Filename already exists, not overwriting it"
+    return $false
+  }
+  return $true
+}
+
+# Returns the GCE instance metadata value for $Key. If the key is not present
+# in the instance metadata returns $Default if set, otherwise returns $null.
+function Get-InstanceMetadataValue {
+  param (
+    [parameter(Mandatory=$true)] [string]$Key,
+    [parameter(Mandatory=$false)] [string]$Default
+  )
+
+  $url = ("http://metadata.google.internal/computeMetadata/v1/instance/" +
+          "attributes/$Key")
+  try {
+    $client = New-Object Net.WebClient
+    $client.Headers.Add('Metadata-Flavor', 'Google')
+    return ($client.DownloadString($url)).Trim()
+  }
+  catch [System.Net.WebException] {
+    if ($Default) {
+      return $Default
+    }
+    else {
+      Log-Output "Failed to retrieve value for $Key."
+      return $null
+    }
+  }
+}
+
+# Export all public functions:
+Export-ModuleMember -Function *-*

cluster/gce/win1803/configure.ps1

+# Copyright 2019 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+<#
+.SYNOPSIS
+  Top-level script that runs on Windows nodes to join them to the K8s cluster.
+#>
+
+$ErrorActionPreference = 'Stop'
+
+# Turn on tracing to debug
+# Set-PSDebug -Trace 1
+
+# Update TLS setting to enable Github downloads and disable progress bar to
+# increase download speed.
+[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+$ProgressPreference = 'SilentlyContinue'
+
+# Returns the GCE instance metadata value for $Key. If the key is not present
+# in the instance metadata returns $Default if set, otherwise returns $null.
+function Get-InstanceMetadataValue {
+  param (
+    [parameter(Mandatory=$true)] [string]$Key,
+    [parameter(Mandatory=$false)] [string]$Default
+  )
+
+  $url = ("http://metadata.google.internal/computeMetadata/v1/instance/" +
+          "attributes/$Key")
+  try {
+    $client = New-Object Net.WebClient
+    $client.Headers.Add('Metadata-Flavor', 'Google')
+    return ($client.DownloadString($url)).Trim()
+  }
+  catch [System.Net.WebException] {
+    if ($Default) {
+      return $Default
+    }
+    else {
+      Write-Host "Failed to retrieve value for $Key."
+      return $null
+    }
+  }
+}
+
+# Fetches the value of $MetadataKey, saves it to C:\$Filename and imports it as
+# a PowerShell module.
+#
+# Note: this function depends on common.psm1.
+function FetchAndImport-ModuleFromMetadata {
+  param (
+    [parameter(Mandatory=$true)] [string]$MetadataKey,
+    [parameter(Mandatory=$true)] [string]$Filename
+  )
+
+  $module = Get-InstanceMetadataValue $MetadataKey
+  if (Test-Path C:\$Filename) {
+    if (-not $REDO_STEPS) {
+      Log-Output "Skip: C:\$Filename already exists, not overwriting"
+      Import-Module -Force C:\$Filename
+      return
+    }
+    Log-Output "Warning: C:\$Filename already exists, will overwrite it."
+  }
+  New-Item -ItemType file -Force C:\$Filename | Out-Null
+  Set-Content C:\$Filename $module
+  Import-Module -Force C:\$Filename
+}
+
+try {
+  # Don't use FetchAndImport-ModuleFromMetadata for common.psm1 - the common
+  # module includes variables and functions that any other function may depend
+  # on.
+  $module = Get-InstanceMetadataValue 'common-psm1'
+  New-Item -ItemType file -Force C:\common.psm1 | Out-Null
+  Set-Content C:\common.psm1 $module
+  Import-Module -Force C:\common.psm1
+
+  # TODO(pjh): update the function to set $Filename automatically from the key,
+  # then put these calls into a loop over a list of XYZ-psm1 keys.
+  FetchAndImport-ModuleFromMetadata 'k8s-node-setup-psm1' 'k8s-node-setup.psm1'
+
+  Set-PrerequisiteOptions
+  $kube_env = Fetch-KubeEnv
+  Set-EnvironmentVars
+  Create-Directories
+  Download-HelperScripts
+
+  Create-PauseImage
+  DownloadAndInstall-KubernetesBinaries
+  Create-NodePki
+  Create-KubeletKubeconfig
+  Create-KubeproxyKubeconfig
+  Set-PodCidr
+  Configure-HostNetworkingService
+  Configure-CniNetworking
+  Configure-Kubelet
+
+  Start-WorkerServices
+  Log-Output 'Waiting 15 seconds for node to join cluster.'
+  Start-Sleep 15
+  Verify-WorkerServices
+}
+catch {
+  Write-Host 'Exception caught in script:'
+  Write-Host $_.InvocationInfo.PositionMessage
+  Write-Host "Kubernetes Windows node setup failed: $($_.Exception.Message)"
+  exit 1
+}

cluster/gce/win1803/node-helper.sh

+#!/usr/bin/env bash
+
+# Copyright 2019 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# A library of helper functions and constants for Windows nodes.
+
+function get-windows-node-instance-metadata-from-file {
+  local metadata=""
+  metadata+="cluster-name=${KUBE_TEMP}/cluster-name.txt,"
+  metadata+="kube-env=${KUBE_TEMP}/windows-node-kube-env.yaml,"
+  metadata+="kubelet-config=${KUBE_TEMP}/windows-node-kubelet-config.yaml,"
+  # To get startup script output run "gcloud compute instances
+  # get-serial-port-output <instance>" from the location where you're running
+  # kube-up.
+  metadata+="windows-startup-script-ps1=${KUBE_ROOT}/cluster/gce/${WINDOWS_NODE_OS_DISTRIBUTION}/configure.ps1,"
+  metadata+="common-psm1=${KUBE_ROOT}/cluster/gce/${WINDOWS_NODE_OS_DISTRIBUTION}/common.psm1,"
+  metadata+="k8s-node-setup-psm1=${KUBE_ROOT}/cluster/gce/${WINDOWS_NODE_OS_DISTRIBUTION}/k8s-node-setup.psm1,"
+  metadata+="user-profile-psm1=${KUBE_ROOT}/cluster/gce/${WINDOWS_NODE_OS_DISTRIBUTION}/user-profile.psm1,"
+  metadata+="${NODE_EXTRA_METADATA}"
+  echo "${metadata}"
+}
+
+function get-windows-node-instance-metadata {
+  local metadata=""
+  metadata+="k8s-version=${KUBE_VERSION:-v1.13.2},"
+  metadata+="serial-port-enable=1,"
+  # This enables logging the serial port output.
+  # https://cloud.google.com/compute/docs/instances/viewing-serial-port-output
+  metadata+="serial-port-logging-enable=true,"
+  metadata+="win-version=${WINDOWS_NODE_OS_DISTRIBUTION}"
+  echo "${metadata}"
+}
+
+# $1: template name (required).
+# $2: scopes flag.
+function create-windows-node-instance-template {
+  local template_name="$1"
+  local scopes_flag="$2"
+  create-node-template "${template_name}" "${scopes_flag}" "$(get-windows-node-instance-metadata-from-file)" "$(get-windows-node-instance-metadata)" "windows"
+}

cluster/gce/win1803/user-profile.psm1

+
+<#
+.Synopsis
+   Rough PS functions to create new user profiles
+.DESCRIPTION
+   Call the Create-NewProfile function directly to create a new profile
+.EXAMPLE
+   Create-NewProfile -Username 'testUser1' -Password 'testUser1'
+.NOTES
+   Created by: Josh Rickard (@MS_dministrator) and Thom Schumacher (@driberif)
+   Forked by: @crshnbrn66, then @pjh (2018-11-08). See
+     https://gist.github.com/pjh/9753cd14400f4e3d4567f4553ba75f1d/revisions
+   Date: 24MAR2017
+   Location: https://gist.github.com/crshnbrn66/7e81bf20408c05ddb2b4fdf4498477d8
+
+   Contact: https://github.com/MSAdministrator
+            MSAdministrator.com
+            https://github.com/crshnbrn66
+            powershellposse.com
+#>
+
+
+#Function to create the new local user first
+function New-LocalUser
+{
+    [CmdletBinding()]
+    [Alias()]
+    [OutputType([int])]
+    Param
+    (
+        # Param1 help description
+        [Parameter(Mandatory=$true,
+                   ValueFromPipelineByPropertyName=$true,
+                   Position=0)]
+        $userName,
+        # Param2 help description
+        [string]
+        $password
+    )
+ 
+    $system = [ADSI]"WinNT://$env:COMPUTERNAME";
+    $user = $system.Create("user",$userName);
+    $user.SetPassword($password);
+    $user.SetInfo();
+ 
+    $flag=$user.UserFlags.value -bor 0x10000;
+    $user.put("userflags",$flag);
+    $user.SetInfo();
+ 
+    $group = [ADSI]("WinNT://$env:COMPUTERNAME/Users");
+    $group.PSBase.Invoke("Add", $user.PSBase.Path);
+}
+
+#function to register a native method
+function Register-NativeMethod
+{
+    [CmdletBinding()]
+    [Alias()]
+    [OutputType([int])]
+    Param
+    (
+        # Param1 help description
+        [Parameter(Mandatory=$true,
+                   ValueFromPipelineByPropertyName=$true,
+                   Position=0)]
+        [string]$dll,
+ 
+        # Param2 help description
+        [Parameter(Mandatory=$true,
+                   ValueFromPipelineByPropertyName=$true,
+                   Position=1)]
+        [string]
+        $methodSignature
+    )
+ 
+    $script:nativeMethods += [PSCustomObject]@{ Dll = $dll; Signature = $methodSignature; }
+}
+function Get-Win32LastError
+{
+    [CmdletBinding()]
+    [Alias()]
+    [OutputType([int])]
+    Param($typeName = 'LastError')
+ if (-not ([System.Management.Automation.PSTypeName]$typeName).Type)
+    {
+    $lasterrorCode = $script:lasterror | ForEach-Object{
+        '[DllImport("kernel32.dll", SetLastError = true)]
+         public static extern uint GetLastError();'
+    }
+        Add-Type @"
+        using System;
+        using System.Text;
+        using System.Runtime.InteropServices;
+        public static class $typeName {
+            $lasterrorCode
+        }
+"@
+    }
+}
+#function to add native method
+function Add-NativeMethods
+{
+    [CmdletBinding()]
+    [Alias()]
+    [OutputType([int])]
+    Param($typeName = 'NativeMethods')
+ 
+    $nativeMethodsCode = $script:nativeMethods | ForEach-Object { "
+        [DllImport(`"$($_.Dll)`")]
+        public static extern $($_.Signature);
+    " }
+ 
+    Add-Type @"
+        using System;
+        using System.Text;
+        using System.Runtime.InteropServices;
+        public static class $typeName {
+            $nativeMethodsCode
+        }
+"@
+}
+
+#Main function to create the new user profile
+function Create-NewProfile {
+ 
+    [CmdletBinding()]
+    [Alias()]
+    [OutputType([int])]
+    Param
+    (
+        # Param1 help description
+        [Parameter(Mandatory=$true,
+                   ValueFromPipelineByPropertyName=$true,
+                   Position=0)]
+        [string]$UserName,
+ 
+        # Param2 help description
+        [Parameter(Mandatory=$true,
+                   ValueFromPipelineByPropertyName=$true,
+                   Position=1)]
+        [string]
+        $Password
+    )
+  
+    Write-Verbose "Creating local user $Username";
+  
+    try
+    {
+        New-LocalUser -username $UserName -password $Password;
+    }
+    catch
+    {
+        Write-Error $_.Exception.Message;
+        break;
+    }
+    $methodName = 'UserEnvCP'
+    $script:nativeMethods = @();
+ 
+    if (-not ([System.Management.Automation.PSTypeName]$MethodName).Type)
+    {
+        Register-NativeMethod "userenv.dll" "int CreateProfile([MarshalAs(UnmanagedType.LPWStr)] string pszUserSid,`
+         [MarshalAs(UnmanagedType.LPWStr)] string pszUserName,`
+         [Out][MarshalAs(UnmanagedType.LPWStr)] StringBuilder pszProfilePath, uint cchProfilePath)";
+ 
+        Add-NativeMethods -typeName $MethodName;
+    }
+ 
+    $localUser = New-Object System.Security.Principal.NTAccount("$UserName");
+    $userSID = $localUser.Translate([System.Security.Principal.SecurityIdentifier]);
+    $sb = new-object System.Text.StringBuilder(260);
+    $pathLen = $sb.Capacity;
+ 
+    Write-Verbose "Creating user profile for $Username";
+ 
+    try
+    {
+        [UserEnvCP]::CreateProfile($userSID.Value, $Username, $sb, $pathLen) | Out-Null;
+    }
+    catch
+    {
+        Write-Error $_.Exception.Message;
+        break;
+    }
+}
+
+function New-ProfileFromSID {
+ 
+    [CmdletBinding()]
+    [Alias()]
+    [OutputType([int])]
+    Param
+    (
+        # Param1 help description
+        [Parameter(Mandatory=$true,
+                   ValueFromPipelineByPropertyName=$true,
+                   Position=0)]
+        [string]$UserName,
+        [string]$domain = 'PHCORP'
+    )
+    $methodname = 'UserEnvCP2'
+    $script:nativeMethods = @();
+    
+    if (-not ([System.Management.Automation.PSTypeName]$methodname).Type)
+    {
+        Register-NativeMethod "userenv.dll" "int CreateProfile([MarshalAs(UnmanagedType.LPWStr)] string pszUserSid,`
+         [MarshalAs(UnmanagedType.LPWStr)] string pszUserName,`
+         [Out][MarshalAs(UnmanagedType.LPWStr)] StringBuilder pszProfilePath, uint cchProfilePath)";
+ 
+        Add-NativeMethods -typeName $methodname;
+    }
+ 
+    $sb = new-object System.Text.StringBuilder(260);
+    $pathLen = $sb.Capacity;
+ 
+    Write-Verbose "Creating user profile for $Username";
+    #$SID= ((get-aduser -id $UserName -ErrorAction Stop).sid.value)
+  if($domain)
+   {
+        $objUser = New-Object System.Security.Principal.NTAccount($domain, $UserName)
+        $strSID = $objUser.Translate([System.Security.Principal.SecurityIdentifier])
+        $SID = $strSID.Value
+   }
+   else 
+   {
+       $objUser = New-Object System.Security.Principal.NTAccount($UserName)
+       $strSID = $objUser.Translate([System.Security.Principal.SecurityIdentifier])
+       $SID = $strSID.Value
+   }
+    Write-Verbose "$UserName SID: $SID"
+    try
+    {
+       $result = [UserEnvCP2]::CreateProfile($SID, $Username, $sb, $pathLen) 
+       if($result -eq '-2147024713')
+       {
+           $status = "$userName already exists"
+           write-verbose "$username Creation Result: $result"
+        }
+        elseif($result -eq '-2147024809')
+        {
+            $staus = "$username Not Found"
+            write-verbose "$username creation result: $result"
+        }
+       elseif($result -eq 0)
+       {
+           $status = "$username Profile has been created"
+           write-verbose "$username Creation Result: $result"
+       }
+       else
+       {
+          $status = "$UserName unknown return result: $result"
+       }
+    }
+    catch
+    {
+        Write-Error $_.Exception.Message;
+        break;
+    }
+    $status
+}
+Function Remove-Profile {
+ 
+    [CmdletBinding()]
+    [Alias()]
+    [OutputType([int])]
+    Param
+    (
+        # Param1 help description
+        [Parameter(Mandatory=$true,
+                   ValueFromPipelineByPropertyName=$true,
+                   Position=0)]
+        [string]$UserName,
+        [string]$ProfilePath,
+        [string]$domain = 'PHCORP'
+    )
+    $methodname = 'userenvDP'
+    $script:nativeMethods = @();
+ 
+    if (-not ([System.Management.Automation.PSTypeName]"$methodname.profile").Type)
+    {
+      add-type @"
+using System.Runtime.InteropServices;
+
+namespace $typename
+{
+    public static class UserEnv
+    {
+        [DllImport("userenv.dll", CharSet = CharSet.Unicode, ExactSpelling = false, SetLastError = true)]
+        public static extern bool DeleteProfile(string sidString, string profilePath, string computerName);
+
+        [DllImport("kernel32.dll")]
+        public static extern uint GetLastError();
+    }
+
+    public static class Profile
+    {
+        public static uint Delete(string sidString)
+        { //Profile path and computer name are optional
+            if (!UserEnv.DeleteProfile(sidString, null, null))
+            {
+                return UserEnv.GetLastError();
+            }
+
+            return 0;
+        }
+    }
+}
+"@
+    }
+
+   #$SID= ((get-aduser -id $UserName -ErrorAction Stop).sid.value)
+   if($domain)
+   {
+        $objUser = New-Object System.Security.Principal.NTAccount($domain, $UserName)
+        $strSID = $objUser.Translate([System.Security.Principal.SecurityIdentifier])
+        $SID = $strSID.Value
+   }
+   else 
+   {
+       $objUser = New-Object System.Security.Principal.NTAccount($UserName)
+       $strSID = $objUser.Translate([System.Security.Principal.SecurityIdentifier])
+       $SID = $strSID.Value
+   }
+    Write-Verbose "$UserName SID: $SID"
+    try
+    {
+        #http://stackoverflow.com/questions/31949002/c-sharp-delete-user-profile
+       $result = [userenvDP.Profile]::Delete($SID)
+    }
+    catch
+    {
+        Write-Error $_.Exception.Message;
+        break;
+    }
+    $LastError
+}
+
+Export-ModuleMember Create-NewProfile

cluster/kubemark/gce/config-default.sh

@@ -95,7 +95,7 @@ KUBEMARK_MASTER_COMPONENTS_QPS_LIMITS="${KUBEMARK_MASTER_COMPONENTS_QPS_LIMITS:-
 CUSTOM_ADMISSION_PLUGINS="${CUSTOM_ADMISSION_PLUGINS:-NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,Priority,StorageObjectInUseProtection,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota}"
 
 # Master components' test arguments.
-APISERVER_TEST_ARGS="${KUBEMARK_APISERVER_TEST_ARGS:-} --vmodule=httplog=3 --runtime-config=extensions/v1beta1,scheduling.k8s.io/v1alpha1 ${API_SERVER_TEST_LOG_LEVEL} ${TEST_CLUSTER_MAX_REQUESTS_INFLIGHT} ${TEST_CLUSTER_DELETE_COLLECTION_WORKERS}"
+APISERVER_TEST_ARGS="${KUBEMARK_APISERVER_TEST_ARGS:-} --runtime-config=extensions/v1beta1,scheduling.k8s.io/v1alpha1 ${API_SERVER_TEST_LOG_LEVEL} ${TEST_CLUSTER_MAX_REQUESTS_INFLIGHT} ${TEST_CLUSTER_DELETE_COLLECTION_WORKERS}"
 CONTROLLER_MANAGER_TEST_ARGS="${KUBEMARK_CONTROLLER_MANAGER_TEST_ARGS:-} ${CONTROLLER_MANAGER_TEST_LOG_LEVEL} ${TEST_CLUSTER_RESYNC_PERIOD} ${TEST_CLUSTER_API_CONTENT_TYPE} ${KUBEMARK_MASTER_COMPONENTS_QPS_LIMITS}"
 SCHEDULER_TEST_ARGS="${KUBEMARK_SCHEDULER_TEST_ARGS:-} ${SCHEDULER_TEST_LOG_LEVEL} ${TEST_CLUSTER_API_CONTENT_TYPE} ${KUBEMARK_MASTER_COMPONENTS_QPS_LIMITS}"
 

cluster/validate-cluster.sh

@@ -50,9 +50,8 @@ function kubectl_retry() {
 ALLOWED_NOTREADY_NODES="${ALLOWED_NOTREADY_NODES:-0}"
 CLUSTER_READY_ADDITIONAL_TIME_SECONDS="${CLUSTER_READY_ADDITIONAL_TIME_SECONDS:-30}"
 
-EXPECTED_NUM_NODES="${NUM_NODES}"
-
 if [[ "${KUBERNETES_PROVIDER:-}" == "gce" ]]; then
+  EXPECTED_NUM_NODES="$(get-num-nodes)"
   echo "Validating gce cluster, MULTIZONE=${MULTIZONE:-}"
   # In multizone mode we need to add instances for all nodes in the region.
   if [[ "${MULTIZONE:-}" == "true" ]]; then
@@ -60,6 +59,8 @@ if [[ "${KUBERNETES_PROVIDER:-}" == "gce" ]]; then
       --filter="name ~ '${NODE_INSTANCE_PREFIX}.*' AND zone:($(gcloud -q compute zones list --project="${PROJECT}" --filter=region=${REGION} --format=csv[no-heading]\(name\) | tr "\n" "," | sed  "s/,$//"))" | wc -l)
     echo "Computing number of nodes, NODE_INSTANCE_PREFIX=${NODE_INSTANCE_PREFIX}, REGION=${REGION}, EXPECTED_NUM_NODES=${EXPECTED_NUM_NODES}"
   fi
+else
+  EXPECTED_NUM_NODES="${NUM_NODES}"
 fi
 
 if [[ "${REGISTER_MASTER_KUBELET:-}" == "true" ]]; then

cmd/hyperkube/main.go

@@ -170,7 +170,7 @@ func makeSymlinks(targetName string, commandFns []func() *cobra.Command) error {
 	}
 
 	if errs {
-		return errors.New("Error creating one or more symlinks.")
+		return errors.New("Error creating one or more symlinks")
 	}
 	return nil
 }

cmd/kube-controller-manager/app/core.go

@@ -82,8 +82,8 @@ func startServiceController(ctx ControllerContext) (http.Handler, bool, error) {
 }
 
 func startNodeIpamController(ctx ControllerContext) (http.Handler, bool, error) {
-	var clusterCIDR *net.IPNet = nil
-	var serviceCIDR *net.IPNet = nil
+	var clusterCIDR *net.IPNet
+	var serviceCIDR *net.IPNet
 
 	if !ctx.ComponentConfig.KubeCloudShared.AllocateNodeCIDRs {
 		return nil, false, nil

cmd/kube-proxy/app/BUILD

@@ -22,6 +22,7 @@ go_library(
         "//pkg/kubelet/qos:go_default_library",
         "//pkg/master/ports:go_default_library",
         "//pkg/proxy:go_default_library",
+        "//pkg/proxy/apis:go_default_library",
         "//pkg/proxy/apis/config:go_default_library",
         "//pkg/proxy/apis/config/scheme:go_default_library",
         "//pkg/proxy/apis/config/validation:go_default_library",

cmd/kube-proxy/app/server.go

@@ -53,6 +53,7 @@ import (
 	"k8s.io/kubernetes/pkg/kubelet/qos"
 	"k8s.io/kubernetes/pkg/master/ports"
 	"k8s.io/kubernetes/pkg/proxy"
+	"k8s.io/kubernetes/pkg/proxy/apis"
 	kubeproxyconfig "k8s.io/kubernetes/pkg/proxy/apis/config"
 	"k8s.io/kubernetes/pkg/proxy/apis/config/scheme"
 	"k8s.io/kubernetes/pkg/proxy/apis/config/validation"
@@ -583,7 +584,7 @@ func (s *ProxyServer) Run() error {
 
 	informerFactory := informers.NewSharedInformerFactoryWithOptions(s.Client, s.ConfigSyncPeriod,
 		informers.WithTweakListOptions(func(options *v1meta.ListOptions) {
-			options.LabelSelector = "!service.kubernetes.io/service-proxy-name"
+			options.LabelSelector = "!" + apis.LabelServiceProxyName
 		}))
 
 	// Create configs (i.e. Watches for Services and Endpoints)

cmd/kubeadm/.import-restrictions

@@ -41,6 +41,7 @@
 			"AllowedPrefixes": [
 				"k8s.io/utils/exec",
 				"k8s.io/utils/integer",
+				"k8s.io/utils/path",
 				"k8s.io/utils/pointer"
 			]
 		},
@@ -77,7 +78,6 @@
 				"k8s.io/kubernetes/pkg/scheduler/util",
 				"k8s.io/kubernetes/pkg/security/apparmor",
 				"k8s.io/kubernetes/pkg/serviceaccount",
-				"k8s.io/kubernetes/pkg/util/file",
 				"k8s.io/kubernetes/pkg/util/hash",
 				"k8s.io/kubernetes/pkg/util/initsystem",
 				"k8s.io/kubernetes/pkg/util/ipvs",

cmd/kubeadm/app/cmd/alpha/selfhosting.go

@@ -125,7 +125,7 @@ func getSelfhostingSubCommand(in io.Reader) *cobra.Command {
 
 			// KubernetesVersion is not used, but we set it explicitly to avoid the lookup
 			// of the version from the internet when executing ConfigFileAndDefaultsToInternalConfig
-			phases.SetKubernetesVersion(cfg)
+			phases.SetKubernetesVersion(&cfg.ClusterConfiguration)
 
 			// This call returns the ready-to-use configuration based on the configuration file that might or might not exist and the default cfg populated by flags
 			internalcfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(cfgPath, cfg)

cmd/kubeadm/app/cmd/config.go

@@ -388,7 +388,7 @@ func RunConfigView(out io.Writer, client clientset.Interface) error {
 func uploadConfiguration(client clientset.Interface, cfgPath string, defaultcfg *kubeadmapiv1beta1.InitConfiguration) error {
 	// KubernetesVersion is not used, but we set it explicitly to avoid the lookup
 	// of the version from the internet when executing ConfigFileAndDefaultsToInternalConfig
-	phaseutil.SetKubernetesVersion(defaultcfg)
+	phaseutil.SetKubernetesVersion(&defaultcfg.ClusterConfiguration)
 
 	// Default both statically and dynamically, convert to internal API type, and validate everything
 	// First argument is unset here as we shouldn't load a config file from disk

cmd/kubeadm/app/cmd/init.go

@@ -148,6 +148,7 @@ func NewCmdInit(out io.Writer, initOptions *initOptions) *cobra.Command {
 			err = showJoinCommand(data, out)
 			kubeadmutil.CheckErr(err)
 		},
+		Args: cobra.NoArgs,
 	}
 
 	// adds flags to the init command
@@ -286,27 +287,27 @@ func newInitData(cmd *cobra.Command, args []string, options *initOptions, out io
 	// validated values to the public kubeadm config API when applicable
 	var err error
 	if options.externalcfg.FeatureGates, err = features.NewFeatureGate(&features.InitFeatureGates, options.featureGatesString); err != nil {
-		return &initData{}, err
+		return nil, err
 	}
 
 	ignorePreflightErrorsSet, err := validation.ValidateIgnorePreflightErrors(options.ignorePreflightErrors)
 	if err != nil {
-		return &initData{}, err
+		return nil, err
 	}
 
 	if err = validation.ValidateMixedArguments(cmd.Flags()); err != nil {
-		return &initData{}, err
+		return nil, err
 	}
 
 	if err = options.bto.ApplyTo(options.externalcfg); err != nil {
-		return &initData{}, err
+		return nil, err
 	}
 
 	// Either use the config file if specified, or convert public kubeadm API to the internal InitConfiguration
 	// and validates InitConfiguration
 	cfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(options.cfgPath, options.externalcfg)
 	if err != nil {
-		return &initData{}, err
+		return nil, err
 	}
 
 	// override node name and CRI socket from the command line options
@@ -318,29 +319,29 @@ func newInitData(cmd *cobra.Command, args []string, options *initOptions, out io
 	}
 
 	if err := configutil.VerifyAPIServerBindAddress(cfg.LocalAPIEndpoint.AdvertiseAddress); err != nil {
-		return &initData{}, err
+		return nil, err
 	}
 	if err := features.ValidateVersion(features.InitFeatureGates, cfg.FeatureGates, cfg.KubernetesVersion); err != nil {
-		return &initData{}, err
+		return nil, err
 	}
 
 	// if dry running creates a temporary folder for saving kubeadm generated files
 	dryRunDir := ""
 	if options.dryRun {
 		if dryRunDir, err = ioutil.TempDir("", "kubeadm-init-dryrun"); err != nil {
-			return &initData{}, errors.Wrap(err, "couldn't create a temporary directory")
+			return nil, errors.Wrap(err, "couldn't create a temporary directory")
 		}
 	}
 
 	// Checks if an external CA is provided by the user.
-	externalCA, _ := certsphase.UsingExternalCA(cfg)
+	externalCA, _ := certsphase.UsingExternalCA(&cfg.ClusterConfiguration)
 	if externalCA {
 		kubeconfigDir := kubeadmconstants.KubernetesDir
 		if options.dryRun {
 			kubeconfigDir = dryRunDir
 		}
 		if err := kubeconfigphase.ValidateKubeconfigsForExternalCA(kubeconfigDir, cfg); err != nil {
-			return &initData{}, err
+			return nil, err
 		}
 	}
 

cmd/kubeadm/app/cmd/phases/addons.go

@@ -95,7 +95,7 @@ func runCoreDNSAddon(c workflow.RunData) error {
 	if err != nil {
 		return err
 	}
-	return dnsaddon.EnsureDNSAddon(cfg, client)
+	return dnsaddon.EnsureDNSAddon(&cfg.ClusterConfiguration, client)
 }
 
 // runKubeProxyAddon installs KubeProxy addon to a Kubernetes cluster
@@ -104,7 +104,7 @@ func runKubeProxyAddon(c workflow.RunData) error {
 	if err != nil {
 		return err
 	}
-	return proxyaddon.EnsureProxyAddon(cfg, client)
+	return proxyaddon.EnsureProxyAddon(&cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, client)
 }
 
 func getAddonPhaseFlags(name string) []string {

cmd/kubeadm/app/cmd/phases/certs.go

@@ -204,13 +204,8 @@ func runCertsSa(c workflow.RunData) error {
 		return nil
 	}
 
-	// if dryrunning, write certificates to a temporary folder (and defer restore to the path originally specified by the user)
-	cfg := data.Cfg()
-	cfg.CertificatesDir = data.CertificateWriteDir()
-	defer func() { cfg.CertificatesDir = data.CertificateDir() }()
-
 	// create the new service account key (or use existing)
-	return certsphase.CreateServiceAccountKeyAndPublicKeyFiles(cfg)
+	return certsphase.CreateServiceAccountKeyAndPublicKeyFiles(data.CertificateWriteDir())
 }
 
 func runCerts(c workflow.RunData) error {

cmd/kubeadm/app/cmd/phases/controlplane.go

@@ -34,10 +34,10 @@ var (
 	controlPlaneExample = normalizer.Examples(`
 		# Generates all static Pod manifest files for control plane components, 
 		# functionally equivalent to what is generated by kubeadm init.
-		kubeadm init phase control-plane
+		kubeadm init phase control-plane all
 
 		# Generates all static Pod manifest files using options read from a configuration file.
-		kubeadm init phase control-plane --config config.yaml
+		kubeadm init phase control-plane all --config config.yaml
 		`)
 
 	controlPlanePhaseProperties = map[string]struct {
@@ -80,6 +80,7 @@ func NewControlPlanePhase() workflow.Phase {
 				Name:           "all",
 				Short:          "Generates all static Pod manifest files",
 				InheritFlags:   getControlPlanePhaseFlags("all"),
+				Example:        controlPlaneExample,
 				RunAllSiblings: true,
 			},
 			newControlPlaneSubPhase(kubeadmconstants.KubeAPIServer),
@@ -150,6 +151,6 @@ func runControlPlaneSubPhase(component string) func(c workflow.RunData) error {
 		cfg := data.Cfg()
 
 		fmt.Printf("[control-plane] Creating static Pod manifest for %q\n", component)
-		return controlplane.CreateStaticPodFiles(data.ManifestDir(), cfg, component)
+		return controlplane.CreateStaticPodFiles(data.ManifestDir(), &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, component)
 	}
 }

cmd/kubeadm/app/cmd/phases/etcd.go

@@ -90,7 +90,7 @@ func runEtcdPhaseLocal() func(c workflow.RunData) error {
 		// Add etcd static pod spec only if external etcd is not configured
 		if cfg.Etcd.External == nil {
 			fmt.Printf("[etcd] Creating static Pod manifest for local etcd in %q\n", data.ManifestDir())
-			if err := etcdphase.CreateLocalEtcdStaticPodManifestFile(data.ManifestDir(), cfg); err != nil {
+			if err := etcdphase.CreateLocalEtcdStaticPodManifestFile(data.ManifestDir(), cfg.NodeRegistration.Name, &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint); err != nil {
 				return errors.Wrap(err, "error creating local etcd static pod manifest file")
 			}
 		} else {

cmd/kubeadm/app/cmd/phases/kubelet.go

@@ -75,7 +75,7 @@ func runKubeletStart(c workflow.RunData) error {
 	// Write env file with flags for the kubelet to use. We do not need to write the --register-with-taints for the master,
 	// as we handle that ourselves in the markmaster phase
 	// TODO: Maybe we want to do that some time in the future, in order to remove some logic from the markmaster phase?
-	if err := kubeletphase.WriteKubeletDynamicEnvFile(data.Cfg(), false, data.KubeletDir()); err != nil {
+	if err := kubeletphase.WriteKubeletDynamicEnvFile(&data.Cfg().ClusterConfiguration, &data.Cfg().NodeRegistration, false, data.KubeletDir()); err != nil {
 		return errors.Wrap(err, "error writing a dynamic environment file for the kubelet")
 	}
 

cmd/kubeadm/app/cmd/phases/preflight.go

@@ -17,23 +17,43 @@ limitations under the License.
 package phases
 
 import (
+	"bytes"
 	"fmt"
+	"text/template"
 
+	"github.com/lithammer/dedent"
 	"github.com/pkg/errors"
 	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/klog"
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
 	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
+	"k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
 	"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
 	"k8s.io/kubernetes/pkg/util/normalizer"
 	utilsexec "k8s.io/utils/exec"
 )
 
 var (
-	masterPreflightExample = normalizer.Examples(`
+	initPreflightExample = normalizer.Examples(`
 		# Run master pre-flight checks using a config file.
 		kubeadm init phase preflight --config kubeadm-config.yml
 		`)
+	joinPreflightExample = normalizer.Examples(`
+		# Run join pre-flight checks using a config file.
+		kubeadm join phase preflight --config kubeadm-config.yml
+		`)
+
+	notReadyToJoinControPlaneTemp = template.Must(template.New("join").Parse(dedent.Dedent(`
+		One or more conditions for hosting a new control plane instance is not satisfied.
+
+		{{.Error}}
+
+		Please ensure that:
+		* The cluster has a stable controlPlaneEndpoint address.
+		* The certificates that must be shared among control plane instances are provided.
+
+		`)))
 )
 
 // preflightMasterData defines the behavior that a runtime data struct passed to the PreflightMaster master phase
@@ -45,13 +65,19 @@ type preflightMasterData interface {
 	IgnorePreflightErrors() sets.String
 }
 
+type preflightJoinData interface {
+	Cfg() *kubeadmapi.JoinConfiguration
+	InitCfg() (*kubeadmapi.InitConfiguration, error)
+	IgnorePreflightErrors() sets.String
+}
+
 // NewPreflightMasterPhase creates a kubeadm workflow phase that implements preflight checks for a new master node.
 func NewPreflightMasterPhase() workflow.Phase {
 	return workflow.Phase{
 		Name:    "preflight",
 		Short:   "Run master pre-flight checks",
 		Long:    "Run master pre-flight checks, functionally equivalent to what implemented by kubeadm init.",
-		Example: masterPreflightExample,
+		Example: initPreflightExample,
 		Run:     runPreflightMaster,
 		InheritFlags: []string{
 			options.CfgPath,
@@ -60,6 +86,7 @@ func NewPreflightMasterPhase() workflow.Phase {
 	}
 }
 
+// TODO(dmaiocchi):  rename all instances of master to controlPlane in this file.
 // runPreflightMaster executes preflight checks logic.
 func runPreflightMaster(c workflow.RunData) error {
 	data, ok := c.(preflightMasterData)
@@ -85,3 +112,96 @@ func runPreflightMaster(c workflow.RunData) error {
 
 	return nil
 }
+
+// NewPreflightJoinPhase creates a kubeadm workflow phase that implements preflight checks for a new node join
+func NewPreflightJoinPhase() workflow.Phase {
+	return workflow.Phase{
+		Name:    "preflight",
+		Short:   "Run join pre-flight checks",
+		Long:    "Run join pre-flight checks, functionally equivalent to what is implemented by kubeadm join.",
+		Example: joinPreflightExample,
+		Run:     runPreflightJoin,
+		InheritFlags: []string{
+			options.CfgPath,
+			options.IgnorePreflightErrors,
+			options.TLSBootstrapToken,
+			options.TokenStr,
+			options.ControlPlane,
+			options.APIServerAdvertiseAddress,
+			options.APIServerBindPort,
+			options.NodeCRISocket,
+			options.NodeName,
+			options.FileDiscovery,
+			options.TokenDiscovery,
+			options.TokenDiscoveryCAHash,
+			options.TokenDiscoverySkipCAHash,
+		},
+	}
+}
+
+// runPreflightJoin executes preflight checks logic.
+func runPreflightJoin(c workflow.RunData) error {
+	j, ok := c.(preflightJoinData)
+	if !ok {
+		return errors.New("preflight phase invoked with an invalid data struct")
+	}
+	// Start with general checks
+	klog.V(1).Infoln("[preflight] Running general checks")
+	if err := preflight.RunJoinNodeChecks(utilsexec.New(), j.Cfg(), j.IgnorePreflightErrors()); err != nil {
+		return err
+	}
+
+	initCfg, err := j.InitCfg()
+	if err != nil {
+		return err
+	}
+
+	// Continue with more specific checks based on the init configuration
+	klog.V(1).Infoln("[preflight] Running configuration dependant checks")
+	if err := preflight.RunOptionalJoinNodeChecks(utilsexec.New(), &initCfg.ClusterConfiguration, j.IgnorePreflightErrors()); err != nil {
+		return err
+	}
+
+	if j.Cfg().ControlPlane != nil {
+		// Checks if the cluster configuration supports
+		// joining a new control plane instance and if all the necessary certificates are provided
+		if err := checkIfReadyForAdditionalControlPlane(&initCfg.ClusterConfiguration); err != nil {
+			// outputs the not ready for hosting a new control plane instance message
+			ctx := map[string]string{
+				"Error": err.Error(),
+			}
+
+			var msg bytes.Buffer
+			notReadyToJoinControPlaneTemp.Execute(&msg, ctx)
+			return errors.New(msg.String())
+		}
+
+		// run kubeadm init preflight checks for checking all the prequisites
+		fmt.Println("[preflight] Running pre-flight checks before initializing the new control plane instance")
+		if err := preflight.RunInitMasterChecks(utilsexec.New(), initCfg, j.IgnorePreflightErrors()); err != nil {
+			return err
+		}
+
+		fmt.Println("[preflight] Pulling control-plane images")
+		if err := preflight.RunPullImagesCheck(utilsexec.New(), initCfg, j.IgnorePreflightErrors()); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// checkIfReadyForAdditionalControlPlane ensures that the cluster is in a state that supports
+// joining an additional control plane instance and if the node is ready to preflight
+func checkIfReadyForAdditionalControlPlane(initConfiguration *kubeadmapi.ClusterConfiguration) error {
+	// blocks if the cluster was created without a stable control plane endpoint
+	if initConfiguration.ControlPlaneEndpoint == "" {
+		return errors.New("unable to add a new control plane instance a cluster that doesn't have a stable controlPlaneEndpoint address")
+	}
+
+	// checks if the certificates that must be equal across contolplane instances are provided
+	if ret, err := certs.SharedCertificateExists(initConfiguration); !ret {
+		return err
+	}
+
+	return nil
+}

cmd/kubeadm/app/cmd/phases/uploadconfig.go

@@ -126,7 +126,7 @@ func runUploadKubeletConfig(c workflow.RunData) error {
 	}
 
 	klog.V(1).Infof("[upload-config] Uploading the kubelet component config to a ConfigMap")
-	if err = kubeletphase.CreateConfigMap(cfg, client); err != nil {
+	if err = kubeletphase.CreateConfigMap(cfg.ClusterConfiguration.ComponentConfigs.Kubelet, cfg.KubernetesVersion, client); err != nil {
 		return errors.Wrap(err, "error creating kubelet configuration ConfigMap")
 	}
 

cmd/kubeadm/app/cmd/phases/util.go

@@ -23,7 +23,7 @@ import (
 
 // SetKubernetesVersion gets the current Kubeadm version and sets it as KubeadmVersion in the config,
 // unless it's already set to a value different from the default.
-func SetKubernetesVersion(cfg *kubeadmapiv1beta1.InitConfiguration) {
+func SetKubernetesVersion(cfg *kubeadmapiv1beta1.ClusterConfiguration) {
 
 	if cfg.KubernetesVersion != kubeadmapiv1beta1.DefaultKubernetesVersion && cfg.KubernetesVersion != "" {
 		return

cmd/kubeadm/app/cmd/phases/util_test.go

@@ -51,7 +51,7 @@ func TestSetKubernetesVersion(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			cfg := &kubeadmapiv1beta1.InitConfiguration{ClusterConfiguration: kubeadmapiv1beta1.ClusterConfiguration{KubernetesVersion: test.input}}
+			cfg := &kubeadmapiv1beta1.ClusterConfiguration{KubernetesVersion: test.input}
 			SetKubernetesVersion(cfg)
 			if cfg.KubernetesVersion != test.output {
 				t.Fatalf("expected %q, got %q", test.output, cfg.KubernetesVersion)

cmd/kubeadm/app/cmd/phases/workflow/runner.go

@@ -309,7 +309,6 @@ func (e *Runner) BindToCommand(cmd *cobra.Command) {
 	phaseCommand := &cobra.Command{
 		Use:   "phase",
 		Short: fmt.Sprintf("use this command to invoke single phase of the %s workflow", cmd.Name()),
-		// TODO: this logic is currently lacking verification if a suphase name is valid!
 	}
 
 	cmd.AddCommand(phaseCommand)
@@ -352,7 +351,6 @@ func (e *Runner) BindToCommand(cmd *cobra.Command) {
 					os.Exit(1)
 				}
 			},
-			Args: cobra.NoArgs, // this forces cobra to fail if a wrong phase name is passed
 		}
 
 		// makes the new command inherits local flags from the parent command
@@ -371,6 +369,11 @@ func (e *Runner) BindToCommand(cmd *cobra.Command) {
 			})
 		}
 
+		// if this phase has children (not a leaf) it doesn't accept any args
+		if len(p.Phases) > 0 {
+			phaseCmd.Args = cobra.NoArgs
+		}
+
 		// adds the command to parent
 		if p.level == 0 {
 			phaseCommand.AddCommand(phaseCmd)

cmd/kubeadm/app/cmd/token.go

@@ -212,7 +212,7 @@ func NewCmdTokenGenerate(out io.Writer) *cobra.Command {
 func RunCreateToken(out io.Writer, client clientset.Interface, cfgPath string, cfg *kubeadmapiv1beta1.InitConfiguration, printJoinCommand bool, kubeConfigFile string) error {
 	// KubernetesVersion is not used, but we set it explicitly to avoid the lookup
 	// of the version from the internet when executing ConfigFileAndDefaultsToInternalConfig
-	phaseutil.SetKubernetesVersion(cfg)
+	phaseutil.SetKubernetesVersion(&cfg.ClusterConfiguration)
 
 	// This call returns the ready-to-use configuration based on the configuration file that might or might not exist and the default cfg populated by flags
 	klog.V(1).Infoln("[token] loading configurations")

cmd/kubeadm/app/cmd/upgrade/diff.go

@@ -105,7 +105,7 @@ func runDiff(flags *diffFlags, args []string) error {
 		return err
 	}
 
-	specs := controlplane.GetStaticPodSpecs(cfg, k8sVer)
+	specs := controlplane.GetStaticPodSpecs(&cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, k8sVer)
 	for spec, pod := range specs {
 		var path string
 		switch spec {

cmd/kubeadm/app/constants/constants.go

@@ -332,7 +332,7 @@ const (
 	KubeDNSVersion = "1.14.13"
 
 	// CoreDNSVersion is the version of CoreDNS to be deployed if it is used
-	CoreDNSVersion = "1.2.6"
+	CoreDNSVersion = "1.3.1"
 
 	// ClusterConfigurationKind is the string kind value for the ClusterConfiguration struct
 	ClusterConfigurationKind = "ClusterConfiguration"

cmd/kubeadm/app/phases/addons/dns/dns.go

@@ -74,14 +74,14 @@ func DeployedDNSAddon(client clientset.Interface) (kubeadmapi.DNSAddOnType, stri
 }
 
 // EnsureDNSAddon creates the kube-dns or CoreDNS addon
-func EnsureDNSAddon(cfg *kubeadmapi.InitConfiguration, client clientset.Interface) error {
+func EnsureDNSAddon(cfg *kubeadmapi.ClusterConfiguration, client clientset.Interface) error {
 	if cfg.DNS.Type == kubeadmapi.CoreDNS {
 		return coreDNSAddon(cfg, client)
 	}
 	return kubeDNSAddon(cfg, client)
 }
 
-func kubeDNSAddon(cfg *kubeadmapi.InitConfiguration, client clientset.Interface) error {
+func kubeDNSAddon(cfg *kubeadmapi.ClusterConfiguration, client clientset.Interface) error {
 	if err := CreateServiceAccount(client); err != nil {
 		return err
 	}
@@ -103,9 +103,9 @@ func kubeDNSAddon(cfg *kubeadmapi.InitConfiguration, client clientset.Interface)
 	dnsDeploymentBytes, err := kubeadmutil.ParseTemplate(KubeDNSDeployment,
 		struct{ DeploymentName, KubeDNSImage, DNSMasqImage, SidecarImage, DNSBindAddr, DNSProbeAddr, DNSDomain, MasterTaintKey string }{
 			DeploymentName: kubeadmconstants.KubeDNSDeploymentName,
-			KubeDNSImage:   images.GetDNSImage(&cfg.ClusterConfiguration, kubeadmconstants.KubeDNSKubeDNSImageName),
-			DNSMasqImage:   images.GetDNSImage(&cfg.ClusterConfiguration, kubeadmconstants.KubeDNSDnsMasqNannyImageName),
-			SidecarImage:   images.GetDNSImage(&cfg.ClusterConfiguration, kubeadmconstants.KubeDNSSidecarImageName),
+			KubeDNSImage:   images.GetDNSImage(cfg, kubeadmconstants.KubeDNSKubeDNSImageName),
+			DNSMasqImage:   images.GetDNSImage(cfg, kubeadmconstants.KubeDNSDnsMasqNannyImageName),
+			SidecarImage:   images.GetDNSImage(cfg, kubeadmconstants.KubeDNSSidecarImageName),
 			DNSBindAddr:    dnsBindAddr,
 			DNSProbeAddr:   dnsProbeAddr,
 			DNSDomain:      cfg.Networking.DNSDomain,
@@ -155,11 +155,11 @@ func createKubeDNSAddon(deploymentBytes, serviceBytes []byte, client clientset.I
 	return createDNSService(kubednsService, serviceBytes, client)
 }
 
-func coreDNSAddon(cfg *kubeadmapi.InitConfiguration, client clientset.Interface) error {
+func coreDNSAddon(cfg *kubeadmapi.ClusterConfiguration, client clientset.Interface) error {
 	// Get the YAML manifest
 	coreDNSDeploymentBytes, err := kubeadmutil.ParseTemplate(CoreDNSDeployment, struct{ DeploymentName, Image, MasterTaintKey string }{
 		DeploymentName: kubeadmconstants.CoreDNSDeploymentName,
-		Image:          images.GetDNSImage(&cfg.ClusterConfiguration, kubeadmconstants.CoreDNSImageName),
+		Image:          images.GetDNSImage(cfg, kubeadmconstants.CoreDNSImageName),
 		MasterTaintKey: kubeadmconstants.LabelNodeRoleMaster,
 	})
 	if err != nil {
@@ -172,7 +172,7 @@ func coreDNSAddon(cfg *kubeadmapi.InitConfiguration, client clientset.Interface)
 		return err
 	}
 
-	stubDomain, err := translateStubDomainOfKubeDNSToProxyCoreDNS(kubeDNSStubDomain, kubeDNSConfigMap)
+	stubDomain, err := translateStubDomainOfKubeDNSToForwardCoreDNS(kubeDNSStubDomain, kubeDNSConfigMap)
 	if err != nil {
 		return err
 	}
@@ -294,9 +294,9 @@ func createDNSService(dnsService *v1.Service, serviceBytes []byte, client client
 	return nil
 }
 
-// translateStubDomainOfKubeDNSToProxyCoreDNS translates StubDomain Data in kube-dns ConfigMap
+// translateStubDomainOfKubeDNSToForwardCoreDNS translates StubDomain Data in kube-dns ConfigMap
 // in the form of Proxy for the CoreDNS Corefile.
-func translateStubDomainOfKubeDNSToProxyCoreDNS(dataField string, kubeDNSConfigMap *v1.ConfigMap) (string, error) {
+func translateStubDomainOfKubeDNSToForwardCoreDNS(dataField string, kubeDNSConfigMap *v1.ConfigMap) (string, error) {
 	if kubeDNSConfigMap == nil {
 		return "", nil
 	}
@@ -316,7 +316,7 @@ func translateStubDomainOfKubeDNSToProxyCoreDNS(dataField string, kubeDNSConfigM
 				{"errors"},
 				{"cache", "30"},
 				{"loop"},
-				append([]string{"proxy", "."}, proxyIP...),
+				append([]string{"forward", "."}, proxyIP...),
 			}
 			proxyStanza = append(proxyStanza, pStanza)
 		}

cmd/kubeadm/app/phases/addons/dns/dns_test.go

@@ -206,28 +206,28 @@ func TestTranslateStubDomainKubeDNSToCoreDNS(t *testing.T) {
        errors
        cache 30
        loop
-       proxy . 1.2.3.4:5300 3.3.3.3
+       forward . 1.2.3.4:5300 3.3.3.3
     }
     
     my.cluster.local:53 {
        errors
        cache 30
        loop
-       proxy . 2.3.4.5
+       forward . 2.3.4.5
     }`,
 			expectTwo: `
     my.cluster.local:53 {
        errors
        cache 30
        loop
-       proxy . 2.3.4.5
+       forward . 2.3.4.5
     }
     
     foo.com:53 {
        errors
        cache 30
        loop
-       proxy . 1.2.3.4:5300 3.3.3.3
+       forward . 1.2.3.4:5300 3.3.3.3
     }`,
 		},
 		{
@@ -257,28 +257,28 @@ func TestTranslateStubDomainKubeDNSToCoreDNS(t *testing.T) {
        errors
        cache 30
        loop
-       proxy . 1.2.3.4:5300
+       forward . 1.2.3.4:5300
     }
     
     my.cluster.local:53 {
        errors
        cache 30
        loop
-       proxy . 2.3.4.5
+       forward . 2.3.4.5
     }`,
 			expectTwo: `
     my.cluster.local:53 {
        errors
        cache 30
        loop
-       proxy . 2.3.4.5
+       forward . 2.3.4.5
     }
     
     foo.com:53 {
        errors
        cache 30
        loop
-       proxy . 1.2.3.4:5300
+       forward . 1.2.3.4:5300
     }`,
 		},
 		{
@@ -296,7 +296,7 @@ func TestTranslateStubDomainKubeDNSToCoreDNS(t *testing.T) {
 		},
 	}
 	for _, testCase := range testCases {
-		out, err := translateStubDomainOfKubeDNSToProxyCoreDNS(kubeDNSStubDomain, testCase.configMap)
+		out, err := translateStubDomainOfKubeDNSToForwardCoreDNS(kubeDNSStubDomain, testCase.configMap)
 		if err != nil {
 			t.Errorf("unexpected error: %v", err)
 		}

cmd/kubeadm/app/phases/addons/dns/manifests.go

@@ -313,7 +313,7 @@ data:
            fallthrough in-addr.arpa ip6.arpa
         }{{ .Federation }}
         prometheus :9153
-        proxy . {{ .UpstreamNameserver }}
+        forward . {{ .UpstreamNameserver }}
         cache 30
         loop
         reload

cmd/kubeadm/app/phases/addons/proxy/proxy.go

@@ -47,13 +47,13 @@ const (
 )
 
 // EnsureProxyAddon creates the kube-proxy addons
-func EnsureProxyAddon(cfg *kubeadmapi.InitConfiguration, client clientset.Interface) error {
+func EnsureProxyAddon(cfg *kubeadmapi.ClusterConfiguration, localEndpoint *kubeadmapi.APIEndpoint, client clientset.Interface) error {
 	if err := CreateServiceAccount(client); err != nil {
 		return errors.Wrap(err, "error when creating kube-proxy service account")
 	}
 
 	// Generate Master Enpoint kubeconfig file
-	masterEndpoint, err := kubeadmutil.GetMasterEndpoint(cfg)
+	masterEndpoint, err := kubeadmutil.GetMasterEndpoint(cfg.ControlPlaneEndpoint, localEndpoint)
 	if err != nil {
 		return err
 	}
@@ -81,7 +81,7 @@ func EnsureProxyAddon(cfg *kubeadmapi.InitConfiguration, client clientset.Interf
 		return errors.Wrap(err, "error when parsing kube-proxy configmap template")
 	}
 	proxyDaemonSetBytes, err = kubeadmutil.ParseTemplate(KubeProxyDaemonSet19, struct{ Image, ProxyConfigMap, ProxyConfigMapKey string }{
-		Image:             images.GetKubernetesImage(constants.KubeProxy, &cfg.ClusterConfiguration),
+		Image:             images.GetKubernetesImage(constants.KubeProxy, cfg),
 		ProxyConfigMap:    constants.KubeProxyConfigMap,
 		ProxyConfigMapKey: constants.KubeProxyConfigMapKey,
 	})

cmd/kubeadm/app/phases/addons/proxy/proxy_test.go

@@ -223,7 +223,7 @@ func TestEnsureProxyAddon(t *testing.T) {
 			t.Errorf("test failed to set dynamic defaults: %v", err)
 			break
 		}
-		err = EnsureProxyAddon(intMaster, client)
+		err = EnsureProxyAddon(&intMaster.ClusterConfiguration, &intMaster.LocalAPIEndpoint, client)
 
 		// Compare actual to expected errors
 		actErr := "No error"

cmd/kubeadm/app/phases/certs/certs.go

@@ -27,8 +27,9 @@ import (
 	certutil "k8s.io/client-go/util/cert"
 	"k8s.io/klog"
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
+	pkiutil "k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil"
+
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
-	"k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil"
 )
 
 // CreatePKIAssets will create and write to disk all PKI assets necessary to establish the control plane.
@@ -59,12 +60,12 @@ func CreatePKIAssets(cfg *kubeadmapi.InitConfiguration) error {
 	fmt.Printf("[certs] valid certificates and keys now exist in %q\n", cfg.CertificatesDir)
 
 	// Service accounts are not x509 certs, so handled separately
-	return CreateServiceAccountKeyAndPublicKeyFiles(cfg)
+	return CreateServiceAccountKeyAndPublicKeyFiles(cfg.CertificatesDir)
 }
 
 // CreateServiceAccountKeyAndPublicKeyFiles create a new public/private key files for signing service account users.
 // If the sa public/private key files already exists in the target folder, they are used only if evaluated equals; otherwise an error is returned.
-func CreateServiceAccountKeyAndPublicKeyFiles(cfg *kubeadmapi.InitConfiguration) error {
+func CreateServiceAccountKeyAndPublicKeyFiles(certsDir string) error {
 	klog.V(1).Infoln("creating a new public/private key files for signing service account users")
 	saSigningKey, err := NewServiceAccountSigningKey()
 	if err != nil {
@@ -72,7 +73,7 @@ func CreateServiceAccountKeyAndPublicKeyFiles(cfg *kubeadmapi.InitConfiguration)
 	}
 
 	return writeKeyFilesIfNotExist(
-		cfg.CertificatesDir,
+		certsDir,
 		kubeadmconstants.ServiceAccountKeyBaseName,
 		saSigningKey,
 	)
@@ -81,7 +82,7 @@ func CreateServiceAccountKeyAndPublicKeyFiles(cfg *kubeadmapi.InitConfiguration)
 // NewServiceAccountSigningKey generate public/private key pairs for signing service account tokens.
 func NewServiceAccountSigningKey() (*rsa.PrivateKey, error) {
 	// The key does NOT exist, let's generate it now
-	saSigningKey, err := certutil.NewPrivateKey()
+	saSigningKey, err := pkiutil.NewPrivateKey()
 	if err != nil {
 		return nil, errors.Wrap(err, "failure while creating service account token signing key")
 	}
@@ -328,7 +329,7 @@ type certKeyLocation struct {
 
 // SharedCertificateExists verifies if the shared certificates - the certificates that must be
 // equal across masters: ca.key, ca.crt, sa.key, sa.pub + etcd/ca.key, etcd/ca.crt if local/stacked etcd
-func SharedCertificateExists(cfg *kubeadmapi.InitConfiguration) (bool, error) {
+func SharedCertificateExists(cfg *kubeadmapi.ClusterConfiguration) (bool, error) {
 
 	if err := validateCACertAndKey(certKeyLocation{cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName, "", "CA"}); err != nil {
 		return false, err
@@ -355,7 +356,7 @@ func SharedCertificateExists(cfg *kubeadmapi.InitConfiguration) (bool, error) {
 // UsingExternalCA determines whether the user is relying on an external CA.  We currently implicitly determine this is the case
 // when both the CA Cert and the front proxy CA Cert are present but the CA Key and front proxy CA Key are not.
 // This allows us to, e.g., skip generating certs or not start the csr signing controller.
-func UsingExternalCA(cfg *kubeadmapi.InitConfiguration) (bool, error) {
+func UsingExternalCA(cfg *kubeadmapi.ClusterConfiguration) (bool, error) {
 
 	if err := validateCACert(certKeyLocation{cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName, "", "CA"}); err != nil {
 		return false, err

cmd/kubeadm/app/phases/certs/certs_test.go

@@ -463,10 +463,8 @@ func TestSharedCertificateExists(t *testing.T) {
 			os.MkdirAll(tmpdir+"/etcd", os.ModePerm)
 			defer os.RemoveAll(tmpdir)
 
-			cfg := &kubeadmapi.InitConfiguration{
-				ClusterConfiguration: kubeadmapi.ClusterConfiguration{
-					CertificatesDir: tmpdir,
-				},
+			cfg := &kubeadmapi.ClusterConfiguration{
+				CertificatesDir: tmpdir,
 			}
 
 			// created expected keys
@@ -554,7 +552,7 @@ func TestUsingExternalCA(t *testing.T) {
 			}
 		}
 
-		if val, _ := UsingExternalCA(cfg); val != test.expected {
+		if val, _ := UsingExternalCA(&cfg.ClusterConfiguration); val != test.expected {
 			t.Errorf("UsingExternalCA did not match expected: %v", test.expected)
 		}
 	}

cmd/kubeadm/app/phases/certs/renewal/certsapi.go

@@ -31,6 +31,7 @@ import (
 	certstype "k8s.io/client-go/kubernetes/typed/certificates/v1beta1"
 	certutil "k8s.io/client-go/util/cert"
 	csrutil "k8s.io/client-go/util/certificate/csr"
+	pkiutil "k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil"
 )
 
 const certAPIPrefixName = "kubeadm-cert"
@@ -60,7 +61,7 @@ func (r *CertsAPIRenewal) Renew(cfg *certutil.Config) (*x509.Certificate, *rsa.P
 		IPAddresses: cfg.AltNames.IPs,
 	}
 
-	key, err := certutil.NewPrivateKey()
+	key, err := pkiutil.NewPrivateKey()
 	if err != nil {
 		return nil, nil, errors.Wrap(err, "couldn't create new private key")
 	}

cmd/kubeadm/app/phases/certs/renewal/renewal_test.go

@@ -136,7 +136,7 @@ func getCertReq(t *testing.T, caCert *x509.Certificate, caKey *rsa.PrivateKey) *
 					Type: certsapi.CertificateApproved,
 				},
 			},
-			Certificate: certutil.EncodeCertPEM(cert),
+			Certificate: pkiutil.EncodeCertPEM(cert),
 		},
 	}
 }

cmd/kubeadm/app/phases/controlplane/BUILD

@@ -41,6 +41,7 @@ go_library(
         "//cmd/kubeadm/app/util/staticpod:go_default_library",
         "//pkg/kubeapiserver/authorizer/modes:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/version:go_default_library",
         "//vendor/github.com/pkg/errors:go_default_library",

cmd/kubeadm/app/phases/controlplane/manifests.go

@@ -26,6 +26,7 @@ import (
 
 	"github.com/pkg/errors"
 	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/apimachinery/pkg/util/version"
 	"k8s.io/klog"
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
@@ -40,12 +41,12 @@ import (
 // CreateInitStaticPodManifestFiles will write all static pod manifest files needed to bring up the control plane.
 func CreateInitStaticPodManifestFiles(manifestDir string, cfg *kubeadmapi.InitConfiguration) error {
 	klog.V(1).Infoln("[control-plane] creating static Pod files")
-	return CreateStaticPodFiles(manifestDir, cfg, kubeadmconstants.KubeAPIServer, kubeadmconstants.KubeControllerManager, kubeadmconstants.KubeScheduler)
+	return CreateStaticPodFiles(manifestDir, &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, kubeadmconstants.KubeAPIServer, kubeadmconstants.KubeControllerManager, kubeadmconstants.KubeScheduler)
 }
 
-// GetStaticPodSpecs returns all staticPodSpecs actualized to the context of the current InitConfiguration
+// GetStaticPodSpecs returns all staticPodSpecs actualized to the context of the current configuration
 // NB. this methods holds the information about how kubeadm creates static pod manifests.
-func GetStaticPodSpecs(cfg *kubeadmapi.InitConfiguration, k8sVersion *version.Version) map[string]v1.Pod {
+func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint, k8sVersion *version.Version) map[string]v1.Pod {
 	// Get the required hostpath mounts
 	mounts := getHostPathVolumesForTheControlPlane(cfg)
 
@@ -53,31 +54,31 @@ func GetStaticPodSpecs(cfg *kubeadmapi.InitConfiguration, k8sVersion *version.Ve
 	staticPodSpecs := map[string]v1.Pod{
 		kubeadmconstants.KubeAPIServer: staticpodutil.ComponentPod(v1.Container{
 			Name:            kubeadmconstants.KubeAPIServer,
-			Image:           images.GetKubernetesImage(kubeadmconstants.KubeAPIServer, &cfg.ClusterConfiguration),
+			Image:           images.GetKubernetesImage(kubeadmconstants.KubeAPIServer, cfg),
 			ImagePullPolicy: v1.PullIfNotPresent,
-			Command:         getAPIServerCommand(cfg),
+			Command:         getAPIServerCommand(cfg, endpoint),
 			VolumeMounts:    staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeAPIServer)),
-			LivenessProbe:   staticpodutil.ComponentProbe(cfg, kubeadmconstants.KubeAPIServer, int(cfg.LocalAPIEndpoint.BindPort), "/healthz", v1.URISchemeHTTPS),
+			LivenessProbe:   livenessProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), int(endpoint.BindPort), v1.URISchemeHTTPS),
 			Resources:       staticpodutil.ComponentResources("250m"),
 			Env:             getProxyEnvVars(),
 		}, mounts.GetVolumes(kubeadmconstants.KubeAPIServer)),
 		kubeadmconstants.KubeControllerManager: staticpodutil.ComponentPod(v1.Container{
 			Name:            kubeadmconstants.KubeControllerManager,
-			Image:           images.GetKubernetesImage(kubeadmconstants.KubeControllerManager, &cfg.ClusterConfiguration),
+			Image:           images.GetKubernetesImage(kubeadmconstants.KubeControllerManager, cfg),
 			ImagePullPolicy: v1.PullIfNotPresent,
 			Command:         getControllerManagerCommand(cfg, k8sVersion),
 			VolumeMounts:    staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeControllerManager)),
-			LivenessProbe:   staticpodutil.ComponentProbe(cfg, kubeadmconstants.KubeControllerManager, 10252, "/healthz", v1.URISchemeHTTP),
+			LivenessProbe:   livenessProbe(staticpodutil.GetControllerManagerProbeAddress(cfg), 10252, v1.URISchemeHTTP),
 			Resources:       staticpodutil.ComponentResources("200m"),
 			Env:             getProxyEnvVars(),
 		}, mounts.GetVolumes(kubeadmconstants.KubeControllerManager)),
 		kubeadmconstants.KubeScheduler: staticpodutil.ComponentPod(v1.Container{
 			Name:            kubeadmconstants.KubeScheduler,
-			Image:           images.GetKubernetesImage(kubeadmconstants.KubeScheduler, &cfg.ClusterConfiguration),
+			Image:           images.GetKubernetesImage(kubeadmconstants.KubeScheduler, cfg),
 			ImagePullPolicy: v1.PullIfNotPresent,
 			Command:         getSchedulerCommand(cfg),
 			VolumeMounts:    staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeScheduler)),
-			LivenessProbe:   staticpodutil.ComponentProbe(cfg, kubeadmconstants.KubeScheduler, 10251, "/healthz", v1.URISchemeHTTP),
+			LivenessProbe:   livenessProbe(staticpodutil.GetSchedulerProbeAddress(cfg), 10251, v1.URISchemeHTTP),
 			Resources:       staticpodutil.ComponentResources("100m"),
 			Env:             getProxyEnvVars(),
 		}, mounts.GetVolumes(kubeadmconstants.KubeScheduler)),
@@ -85,17 +86,33 @@ func GetStaticPodSpecs(cfg *kubeadmapi.InitConfiguration, k8sVersion *version.Ve
 	return staticPodSpecs
 }
 
+func livenessProbe(host string, port int, scheme v1.URIScheme) *v1.Probe {
+	return &v1.Probe{
+		Handler: v1.Handler{
+			HTTPGet: &v1.HTTPGetAction{
+				Host:   host,
+				Path:   "/healthz",
+				Port:   intstr.FromInt(port),
+				Scheme: scheme,
+			},
+		},
+		InitialDelaySeconds: 15,
+		TimeoutSeconds:      15,
+		FailureThreshold:    8,
+	}
+}
+
 // CreateStaticPodFiles creates all the requested static pod files.
-func CreateStaticPodFiles(manifestDir string, cfg *kubeadmapi.InitConfiguration, componentNames ...string) error {
+func CreateStaticPodFiles(manifestDir string, cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint, componentNames ...string) error {
 	// TODO: Move the "pkg/util/version".Version object into the internal API instead of always parsing the string
 	k8sVersion, err := version.ParseSemantic(cfg.KubernetesVersion)
 	if err != nil {
 		return err
 	}
 
-	// gets the StaticPodSpecs, actualized for the current InitConfiguration
+	// gets the StaticPodSpecs, actualized for the current ClusterConfiguration
 	klog.V(1).Infoln("[control-plane] getting StaticPodSpecs")
-	specs := GetStaticPodSpecs(cfg, k8sVersion)
+	specs := GetStaticPodSpecs(cfg, endpoint, k8sVersion)
 
 	// creates required static pod specs
 	for _, componentName := range componentNames {
@@ -117,9 +134,9 @@ func CreateStaticPodFiles(manifestDir string, cfg *kubeadmapi.InitConfiguration,
 }
 
 // getAPIServerCommand builds the right API server command from the given config object and version
-func getAPIServerCommand(cfg *kubeadmapi.InitConfiguration) []string {
+func getAPIServerCommand(cfg *kubeadmapi.ClusterConfiguration, localAPIEndpoint *kubeadmapi.APIEndpoint) []string {
 	defaultArguments := map[string]string{
-		"advertise-address":               cfg.LocalAPIEndpoint.AdvertiseAddress,
+		"advertise-address":               localAPIEndpoint.AdvertiseAddress,
 		"enable-admission-plugins":        "NodeRestriction",
 		"service-cluster-ip-range":        cfg.Networking.ServiceSubnet,
 		"service-account-key-file":        filepath.Join(cfg.CertificatesDir, kubeadmconstants.ServiceAccountPublicKeyName),
@@ -129,7 +146,7 @@ func getAPIServerCommand(cfg *kubeadmapi.InitConfiguration) []string {
 		"kubelet-client-certificate":      filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerKubeletClientCertName),
 		"kubelet-client-key":              filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerKubeletClientKeyName),
 		"enable-bootstrap-token-auth":     "true",
-		"secure-port":                     fmt.Sprintf("%d", cfg.LocalAPIEndpoint.BindPort),
+		"secure-port":                     fmt.Sprintf("%d", localAPIEndpoint.BindPort),
 		"allow-privileged":                "true",
 		"kubelet-preferred-address-types": "InternalIP,ExternalIP,Hostname",
 		// add options to configure the front proxy.  Without the generated client cert, this will never be useable
@@ -243,7 +260,7 @@ func calcNodeCidrSize(podSubnet string) string {
 }
 
 // getControllerManagerCommand builds the right controller manager command from the given config object and version
-func getControllerManagerCommand(cfg *kubeadmapi.InitConfiguration, k8sVersion *version.Version) []string {
+func getControllerManagerCommand(cfg *kubeadmapi.ClusterConfiguration, k8sVersion *version.Version) []string {
 
 	kubeconfigFile := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.ControllerManagerKubeConfigFileName)
 	caFile := filepath.Join(cfg.CertificatesDir, kubeadmconstants.CACertName)
@@ -287,7 +304,7 @@ func getControllerManagerCommand(cfg *kubeadmapi.InitConfiguration, k8sVersion *
 }
 
 // getSchedulerCommand builds the right scheduler command from the given config object and version
-func getSchedulerCommand(cfg *kubeadmapi.InitConfiguration) []string {
+func getSchedulerCommand(cfg *kubeadmapi.ClusterConfiguration) []string {
 	defaultArguments := map[string]string{
 		"bind-address": "127.0.0.1",
 		"leader-elect": "true",

cmd/kubeadm/app/phases/controlplane/volumes.go

@@ -42,7 +42,7 @@ const (
 var caCertsExtraVolumePaths = []string{"/etc/pki", "/usr/share/ca-certificates", "/usr/local/share/ca-certificates", "/etc/ca-certificates"}
 
 // getHostPathVolumesForTheControlPlane gets the required hostPath volumes and mounts for the control plane
-func getHostPathVolumesForTheControlPlane(cfg *kubeadmapi.InitConfiguration) controlPlaneHostPathMounts {
+func getHostPathVolumesForTheControlPlane(cfg *kubeadmapi.ClusterConfiguration) controlPlaneHostPathMounts {
 	hostPathDirectoryOrCreate := v1.HostPathDirectoryOrCreate
 	hostPathFileOrCreate := v1.HostPathFileOrCreate
 	mounts := newControlPlaneHostPathMounts()

cmd/kubeadm/app/phases/controlplane/volumes_test.go

@@ -514,11 +514,7 @@ func TestGetHostPathVolumesForTheControlPlane(t *testing.T) {
 	defer func() { caCertsExtraVolumePaths = []string{"/etc/pki", "/usr/share/ca-certificates"} }()
 
 	for _, rt := range tests {
-		// TODO: Make getHostPathVolumesForTheControlPlane accept a ClusterConfiguration object instead of InitConfiguration
-		initcfg := &kubeadmapi.InitConfiguration{
-			ClusterConfiguration: *rt.cfg,
-		}
-		mounts := getHostPathVolumesForTheControlPlane(initcfg)
+		mounts := getHostPathVolumesForTheControlPlane(rt.cfg)
 
 		// Avoid unit test errors when the flexvolume is mounted
 		if _, ok := mounts.volumes[kubeadmconstants.KubeControllerManager][flexvolumeDirVolumeName]; ok {

cmd/kubeadm/app/phases/etcd/local.go

@@ -46,8 +46,8 @@ const (
 // CreateLocalEtcdStaticPodManifestFile will write local etcd static pod manifest file.
 // This function is used by init - when the etcd cluster is empty - or by kubeadm
 // upgrade - when the etcd cluster is already up and running (and the --initial-cluster flag have no impact)
-func CreateLocalEtcdStaticPodManifestFile(manifestDir string, cfg *kubeadmapi.InitConfiguration) error {
-	if cfg.ClusterConfiguration.Etcd.External != nil {
+func CreateLocalEtcdStaticPodManifestFile(manifestDir string, nodeName string, cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint) error {
+	if cfg.Etcd.External != nil {
 		return errors.New("etcd static pod manifest cannot be generated for cluster using external etcd")
 	}
 	// gets etcd StaticPodSpec
@@ -58,7 +58,7 @@ func CreateLocalEtcdStaticPodManifestFile(manifestDir string, cfg *kubeadmapi.In
 		return errors.Wrapf(err, "failed to create etcd directory %q", cfg.Etcd.Local.DataDir)
 	}
 
-	spec := GetEtcdPodSpec(cfg, emptyInitialCluster)
+	spec := GetEtcdPodSpec(cfg, endpoint, nodeName, emptyInitialCluster)
 	// writes etcd StaticPod to disk
 	if err := staticpodutil.WriteStaticPodToDisk(kubeadmconstants.Etcd, manifestDir, spec); err != nil {
 		return err
@@ -69,7 +69,7 @@ func CreateLocalEtcdStaticPodManifestFile(manifestDir string, cfg *kubeadmapi.In
 }
 
 // CheckLocalEtcdClusterStatus verifies health state of local/stacked etcd cluster before installing a new etcd member
-func CheckLocalEtcdClusterStatus(client clientset.Interface, cfg *kubeadmapi.InitConfiguration) error {
+func CheckLocalEtcdClusterStatus(client clientset.Interface, cfg *kubeadmapi.ClusterConfiguration) error {
 	fmt.Println("[etcd] Checking etcd cluster health")
 
 	// creates an etcd client that connects to all the local/stacked etcd members
@@ -91,7 +91,7 @@ func CheckLocalEtcdClusterStatus(client clientset.Interface, cfg *kubeadmapi.Ini
 // CreateStackedEtcdStaticPodManifestFile will write local etcd static pod manifest file
 // for an additional etcd member that is joining an existing local/stacked etcd cluster.
 // Other members of the etcd cluster will be notified of the joining node in beforehand as well.
-func CreateStackedEtcdStaticPodManifestFile(client clientset.Interface, manifestDir string, cfg *kubeadmapi.InitConfiguration) error {
+func CreateStackedEtcdStaticPodManifestFile(client clientset.Interface, manifestDir string, nodeName string, cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint) error {
 	// creates an etcd client that connects to all the local/stacked etcd members
 	klog.V(1).Info("creating etcd client that connects to etcd pods")
 	etcdClient, err := etcdutil.NewFromCluster(client, cfg.CertificatesDir)
@@ -100,10 +100,10 @@ func CreateStackedEtcdStaticPodManifestFile(client clientset.Interface, manifest
 	}
 
 	// notifies the other members of the etcd cluster about the joining member
-	etcdPeerAddress := etcdutil.GetPeerURL(cfg)
+	etcdPeerAddress := etcdutil.GetPeerURL(endpoint)
 
 	klog.V(1).Infof("Adding etcd member: %s", etcdPeerAddress)
-	initialCluster, err := etcdClient.AddMember(cfg.NodeRegistration.Name, etcdPeerAddress)
+	initialCluster, err := etcdClient.AddMember(nodeName, etcdPeerAddress)
 	if err != nil {
 		return err
 	}
@@ -117,7 +117,7 @@ func CreateStackedEtcdStaticPodManifestFile(client clientset.Interface, manifest
 
 	klog.V(1).Info("Creating local etcd static pod manifest file")
 	// gets etcd StaticPodSpec, actualized for the current InitConfiguration and the new list of etcd members
-	spec := GetEtcdPodSpec(cfg, initialCluster)
+	spec := GetEtcdPodSpec(cfg, endpoint, nodeName, initialCluster)
 	// writes etcd StaticPod to disk
 	if err := staticpodutil.WriteStaticPodToDisk(kubeadmconstants.Etcd, manifestDir, spec); err != nil {
 		return err
@@ -133,9 +133,9 @@ func CreateStackedEtcdStaticPodManifestFile(client clientset.Interface, manifest
 	return nil
 }
 
-// GetEtcdPodSpec returns the etcd static Pod actualized to the context of the current InitConfiguration
+// GetEtcdPodSpec returns the etcd static Pod actualized to the context of the current configuration
 // NB. GetEtcdPodSpec methods holds the information about how kubeadm creates etcd static pod manifests.
-func GetEtcdPodSpec(cfg *kubeadmapi.InitConfiguration, initialCluster []etcdutil.Member) v1.Pod {
+func GetEtcdPodSpec(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint, nodeName string, initialCluster []etcdutil.Member) v1.Pod {
 	pathType := v1.HostPathDirectoryOrCreate
 	etcdMounts := map[string]v1.Volume{
 		etcdVolumeName:  staticpodutil.NewVolume(etcdVolumeName, cfg.Etcd.Local.DataDir, &pathType),
@@ -143,8 +143,8 @@ func GetEtcdPodSpec(cfg *kubeadmapi.InitConfiguration, initialCluster []etcdutil
 	}
 	return staticpodutil.ComponentPod(v1.Container{
 		Name:            kubeadmconstants.Etcd,
-		Command:         getEtcdCommand(cfg, initialCluster),
-		Image:           images.GetEtcdImage(&cfg.ClusterConfiguration),
+		Command:         getEtcdCommand(cfg, endpoint, nodeName, initialCluster),
+		Image:           images.GetEtcdImage(cfg),
 		ImagePullPolicy: v1.PullIfNotPresent,
 		// Mount the etcd datadir path read-write so etcd can store data in a more persistent manner
 		VolumeMounts: []v1.VolumeMount{
@@ -152,20 +152,20 @@ func GetEtcdPodSpec(cfg *kubeadmapi.InitConfiguration, initialCluster []etcdutil
 			staticpodutil.NewVolumeMount(certsVolumeName, cfg.CertificatesDir+"/etcd", false),
 		},
 		LivenessProbe: staticpodutil.EtcdProbe(
-			cfg, kubeadmconstants.Etcd, kubeadmconstants.EtcdListenClientPort, cfg.CertificatesDir,
+			&cfg.Etcd, kubeadmconstants.EtcdListenClientPort, cfg.CertificatesDir,
 			kubeadmconstants.EtcdCACertName, kubeadmconstants.EtcdHealthcheckClientCertName, kubeadmconstants.EtcdHealthcheckClientKeyName,
 		),
 	}, etcdMounts)
 }
 
 // getEtcdCommand builds the right etcd command from the given config object
-func getEtcdCommand(cfg *kubeadmapi.InitConfiguration, initialCluster []etcdutil.Member) []string {
+func getEtcdCommand(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint, nodeName string, initialCluster []etcdutil.Member) []string {
 	defaultArguments := map[string]string{
-		"name":                        cfg.GetNodeName(),
-		"listen-client-urls":          fmt.Sprintf("%s,%s", etcdutil.GetClientURLByIP("127.0.0.1"), etcdutil.GetClientURL(cfg)),
-		"advertise-client-urls":       etcdutil.GetClientURL(cfg),
-		"listen-peer-urls":            etcdutil.GetPeerURL(cfg),
-		"initial-advertise-peer-urls": etcdutil.GetPeerURL(cfg),
+		"name":                        nodeName,
+		"listen-client-urls":          fmt.Sprintf("%s,%s", etcdutil.GetClientURLByIP("127.0.0.1"), etcdutil.GetClientURL(endpoint)),
+		"advertise-client-urls":       etcdutil.GetClientURL(endpoint),
+		"listen-peer-urls":            etcdutil.GetPeerURL(endpoint),
+		"initial-advertise-peer-urls": etcdutil.GetPeerURL(endpoint),
 		"data-dir":                    cfg.Etcd.Local.DataDir,
 		"cert-file":                   filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdServerCertName),
 		"key-file":                    filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdServerKeyName),
@@ -179,7 +179,7 @@ func getEtcdCommand(cfg *kubeadmapi.InitConfiguration, initialCluster []etcdutil
 	}
 
 	if len(initialCluster) == 0 {
-		defaultArguments["initial-cluster"] = fmt.Sprintf("%s=%s", cfg.GetNodeName(), etcdutil.GetPeerURL(cfg))
+		defaultArguments["initial-cluster"] = fmt.Sprintf("%s=%s", nodeName, etcdutil.GetPeerURL(endpoint))
 	} else {
 		// NB. the joining etcd member should be part of the initialCluster list
 		endpoints := []string{}

cmd/kubeadm/app/phases/etcd/local_test.go

@@ -32,19 +32,18 @@ import (
 
 func TestGetEtcdPodSpec(t *testing.T) {
 	// Creates a Master Configuration
-	cfg := &kubeadmapi.InitConfiguration{
-		ClusterConfiguration: kubeadmapi.ClusterConfiguration{
-			KubernetesVersion: "v1.7.0",
-			Etcd: kubeadmapi.Etcd{
-				Local: &kubeadmapi.LocalEtcd{
-					DataDir: "/var/lib/etcd",
-				},
+	cfg := &kubeadmapi.ClusterConfiguration{
+		KubernetesVersion: "v1.7.0",
+		Etcd: kubeadmapi.Etcd{
+			Local: &kubeadmapi.LocalEtcd{
+				DataDir: "/var/lib/etcd",
 			},
 		},
 	}
+	endpoint := &kubeadmapi.APIEndpoint{}
 
 	// Executes GetEtcdPodSpec
-	spec := GetEtcdPodSpec(cfg, []etcdutil.Member{})
+	spec := GetEtcdPodSpec(cfg, endpoint, "", []etcdutil.Member{})
 
 	// Assert each specs refers to the right pod
 	if spec.Spec.Containers[0].Name != kubeadmconstants.Etcd {
@@ -58,35 +57,31 @@ func TestCreateLocalEtcdStaticPodManifestFile(t *testing.T) {
 	defer os.RemoveAll(tmpdir)
 
 	var tests = []struct {
-		cfg           *kubeadmapi.InitConfiguration
+		cfg           *kubeadmapi.ClusterConfiguration
 		expectedError bool
 	}{
 		{
-			cfg: &kubeadmapi.InitConfiguration{
-				ClusterConfiguration: kubeadmapi.ClusterConfiguration{
-					KubernetesVersion: "v1.7.0",
-					Etcd: kubeadmapi.Etcd{
-						Local: &kubeadmapi.LocalEtcd{
-							DataDir: tmpdir + "/etcd",
-						},
+			cfg: &kubeadmapi.ClusterConfiguration{
+				KubernetesVersion: "v1.7.0",
+				Etcd: kubeadmapi.Etcd{
+					Local: &kubeadmapi.LocalEtcd{
+						DataDir: tmpdir + "/etcd",
 					},
 				},
 			},
 			expectedError: false,
 		},
 		{
-			cfg: &kubeadmapi.InitConfiguration{
-				ClusterConfiguration: kubeadmapi.ClusterConfiguration{
-					KubernetesVersion: "v1.7.0",
-					Etcd: kubeadmapi.Etcd{
-						External: &kubeadmapi.ExternalEtcd{
-							Endpoints: []string{
-								"https://etcd-instance:2379",
-							},
-							CAFile:   "/etc/kubernetes/pki/etcd/ca.crt",
-							CertFile: "/etc/kubernetes/pki/etcd/apiserver-etcd-client.crt",
-							KeyFile:  "/etc/kubernetes/pki/etcd/apiserver-etcd-client.key",
+			cfg: &kubeadmapi.ClusterConfiguration{
+				KubernetesVersion: "v1.7.0",
+				Etcd: kubeadmapi.Etcd{
+					External: &kubeadmapi.ExternalEtcd{
+						Endpoints: []string{
+							"https://etcd-instance:2379",
 						},
+						CAFile:   "/etc/kubernetes/pki/etcd/ca.crt",
+						CertFile: "/etc/kubernetes/pki/etcd/apiserver-etcd-client.crt",
+						KeyFile:  "/etc/kubernetes/pki/etcd/apiserver-etcd-client.key",
 					},
 				},
 			},
@@ -97,7 +92,7 @@ func TestCreateLocalEtcdStaticPodManifestFile(t *testing.T) {
 	for _, test := range tests {
 		// Execute createStaticPodFunction
 		manifestPath := filepath.Join(tmpdir, kubeadmconstants.ManifestsSubDirName)
-		err := CreateLocalEtcdStaticPodManifestFile(manifestPath, test.cfg)
+		err := CreateLocalEtcdStaticPodManifestFile(manifestPath, "", test.cfg, &kubeadmapi.APIEndpoint{})
 
 		if !test.expectedError {
 			if err != nil {
@@ -230,23 +225,18 @@ func TestGetEtcdCommand(t *testing.T) {
 
 	for _, rt := range tests {
 		t.Run(rt.name, func(t *testing.T) {
-			cfg := &kubeadmapi.InitConfiguration{
-				LocalAPIEndpoint: kubeadmapi.APIEndpoint{
-					AdvertiseAddress: rt.advertiseAddress,
-				},
-				NodeRegistration: kubeadmapi.NodeRegistrationOptions{
-					Name: rt.nodeName,
-				},
-				ClusterConfiguration: kubeadmapi.ClusterConfiguration{
-					Etcd: kubeadmapi.Etcd{
-						Local: &kubeadmapi.LocalEtcd{
-							DataDir:   "/var/lib/etcd",
-							ExtraArgs: rt.extraArgs,
-						},
+			endpoint := &kubeadmapi.APIEndpoint{
+				AdvertiseAddress: rt.advertiseAddress,
+			}
+			cfg := &kubeadmapi.ClusterConfiguration{
+				Etcd: kubeadmapi.Etcd{
+					Local: &kubeadmapi.LocalEtcd{
+						DataDir:   "/var/lib/etcd",
+						ExtraArgs: rt.extraArgs,
 					},
 				},
 			}
-			actual := getEtcdCommand(cfg, rt.initialCluster)
+			actual := getEtcdCommand(cfg, endpoint, rt.nodeName, rt.initialCluster)
 			sort.Strings(actual)
 			sort.Strings(rt.expected)
 			if !reflect.DeepEqual(actual, rt.expected) {

cmd/kubeadm/app/phases/kubeconfig/kubeconfig.go

@@ -33,8 +33,9 @@ import (
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
+	pkiutil "k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil"
+
 	kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
-	"k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil"
 )
 
 // clientCertAuth struct holds info required to build a client certificate to provide authentication info in a kubeconfig object
@@ -134,7 +135,7 @@ func getKubeConfigSpecs(cfg *kubeadmapi.InitConfiguration) (map[string]*kubeConf
 		return nil, errors.Wrap(err, "couldn't create a kubeconfig; the CA files couldn't be loaded")
 	}
 
-	masterEndpoint, err := kubeadmutil.GetMasterEndpoint(cfg)
+	masterEndpoint, err := kubeadmutil.GetMasterEndpoint(cfg.ControlPlaneEndpoint, &cfg.LocalAPIEndpoint)
 	if err != nil {
 		return nil, err
 	}
@@ -189,7 +190,7 @@ func buildKubeConfigFromSpec(spec *kubeConfigSpec, clustername string) (*clientc
 			spec.APIServer,
 			clustername,
 			spec.ClientName,
-			certutil.EncodeCertPEM(spec.CACert),
+			pkiutil.EncodeCertPEM(spec.CACert),
 			spec.TokenAuth.Token,
 		), nil
 	}
@@ -210,9 +211,9 @@ func buildKubeConfigFromSpec(spec *kubeConfigSpec, clustername string) (*clientc
 		spec.APIServer,
 		clustername,
 		spec.ClientName,
-		certutil.EncodeCertPEM(spec.CACert),
+		pkiutil.EncodeCertPEM(spec.CACert),
 		certutil.EncodePrivateKeyPEM(clientKey),
-		certutil.EncodeCertPEM(clientCert),
+		pkiutil.EncodeCertPEM(clientCert),
 	), nil
 }
 
@@ -284,7 +285,7 @@ func WriteKubeConfigWithClientCert(out io.Writer, cfg *kubeadmapi.InitConfigurat
 		return errors.Wrap(err, "couldn't create a kubeconfig; the CA files couldn't be loaded")
 	}
 
-	masterEndpoint, err := kubeadmutil.GetMasterEndpoint(cfg)
+	masterEndpoint, err := kubeadmutil.GetMasterEndpoint(cfg.ControlPlaneEndpoint, &cfg.LocalAPIEndpoint)
 	if err != nil {
 		return err
 	}
@@ -311,7 +312,7 @@ func WriteKubeConfigWithToken(out io.Writer, cfg *kubeadmapi.InitConfiguration,
 		return errors.Wrap(err, "couldn't create a kubeconfig; the CA files couldn't be loaded")
 	}
 
-	masterEndpoint, err := kubeadmutil.GetMasterEndpoint(cfg)
+	masterEndpoint, err := kubeadmutil.GetMasterEndpoint(cfg.ControlPlaneEndpoint, &cfg.LocalAPIEndpoint)
 	if err != nil {
 		return err
 	}

cmd/kubeadm/app/phases/kubeconfig/kubeconfig_test.go

@@ -162,7 +162,7 @@ func TestGetKubeConfigSpecs(t *testing.T) {
 			}
 
 			// Asserts InitConfiguration values injected into spec
-			masterEndpoint, err := kubeadmutil.GetMasterEndpoint(cfg)
+			masterEndpoint, err := kubeadmutil.GetMasterEndpoint(cfg.ControlPlaneEndpoint, &cfg.LocalAPIEndpoint)
 			if err != nil {
 				t.Error(err)
 			}

cmd/kubeadm/app/phases/kubelet/config.go

@@ -30,7 +30,6 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/version"
 	clientset "k8s.io/client-go/kubernetes"
-	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	"k8s.io/kubernetes/cmd/kubeadm/app/componentconfigs"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
@@ -51,9 +50,9 @@ func WriteConfigToDisk(kubeletConfig *kubeletconfig.KubeletConfiguration, kubele
 
 // CreateConfigMap creates a ConfigMap with the generic kubelet configuration.
 // Used at "kubeadm init" and "kubeadm upgrade" time
-func CreateConfigMap(cfg *kubeadmapi.InitConfiguration, client clientset.Interface) error {
+func CreateConfigMap(cfg *kubeletconfig.KubeletConfiguration, k8sVersionStr string, client clientset.Interface) error {
 
-	k8sVersion, err := version.ParseSemantic(cfg.KubernetesVersion)
+	k8sVersion, err := version.ParseSemantic(k8sVersionStr)
 	if err != nil {
 		return err
 	}
@@ -61,7 +60,7 @@ func CreateConfigMap(cfg *kubeadmapi.InitConfiguration, client clientset.Interfa
 	configMapName := kubeadmconstants.GetKubeletConfigMapName(k8sVersion)
 	fmt.Printf("[kubelet] Creating a ConfigMap %q in namespace %s with the configuration for the kubelets in the cluster\n", configMapName, metav1.NamespaceSystem)
 
-	kubeletBytes, err := getConfigBytes(cfg.ComponentConfigs.Kubelet)
+	kubeletBytes, err := getConfigBytes(cfg)
 	if err != nil {
 		return err
 	}

cmd/kubeadm/app/phases/kubelet/config_test.go

@@ -25,7 +25,6 @@ import (
 	"k8s.io/apimachinery/pkg/util/version"
 	"k8s.io/client-go/kubernetes/fake"
 	core "k8s.io/client-go/testing"
-	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
 )
@@ -33,15 +32,8 @@ import (
 func TestCreateConfigMap(t *testing.T) {
 	nodeName := "fake-node"
 	client := fake.NewSimpleClientset()
-	cfg := &kubeadmapi.InitConfiguration{
-		NodeRegistration: kubeadmapi.NodeRegistrationOptions{Name: nodeName},
-		ClusterConfiguration: kubeadmapi.ClusterConfiguration{
-			KubernetesVersion: constants.CurrentKubernetesVersion.String(),
-			ComponentConfigs: kubeadmapi.ComponentConfigs{
-				Kubelet: &kubeletconfig.KubeletConfiguration{},
-			},
-		},
-	}
+	k8sVersionStr := constants.CurrentKubernetesVersion.String()
+	cfg := &kubeletconfig.KubeletConfiguration{}
 
 	client.PrependReactor("get", "nodes", func(action core.Action) (bool, runtime.Object, error) {
 		return true, &v1.Node{
@@ -61,7 +53,7 @@ func TestCreateConfigMap(t *testing.T) {
 		return true, nil, nil
 	})
 
-	if err := CreateConfigMap(cfg, client); err != nil {
+	if err := CreateConfigMap(cfg, k8sVersionStr, client); err != nil {
 		t.Errorf("CreateConfigMap: unexpected error %v", err)
 	}
 }

cmd/kubeadm/app/phases/kubelet/flags.go

@@ -46,29 +46,29 @@ type kubeletFlagsOpts struct {
 
 // WriteKubeletDynamicEnvFile writes an environment file with dynamic flags to the kubelet.
 // Used at "kubeadm init" and "kubeadm join" time.
-func WriteKubeletDynamicEnvFile(cfg *kubeadmapi.InitConfiguration, registerTaintsUsingFlags bool, kubeletDir string) error {
+func WriteKubeletDynamicEnvFile(cfg *kubeadmapi.ClusterConfiguration, nodeReg *kubeadmapi.NodeRegistrationOptions, registerTaintsUsingFlags bool, kubeletDir string) error {
 	hostName, err := nodeutil.GetHostname("")
 	if err != nil {
 		return err
 	}
 
 	flagOpts := kubeletFlagsOpts{
-		nodeRegOpts:              &cfg.NodeRegistration,
+		nodeRegOpts:              nodeReg,
 		featureGates:             cfg.FeatureGates,
-		pauseImage:               images.GetPauseImage(&cfg.ClusterConfiguration),
+		pauseImage:               images.GetPauseImage(cfg),
 		registerTaintsUsingFlags: registerTaintsUsingFlags,
 		execer:                   utilsexec.New(),
 		pidOfFunc:                procfs.PidOf,
 		defaultHostname:          hostName,
 	}
 	stringMap := buildKubeletArgMap(flagOpts)
-	argList := kubeadmutil.BuildArgumentListFromMap(stringMap, cfg.NodeRegistration.KubeletExtraArgs)
+	argList := kubeadmutil.BuildArgumentListFromMap(stringMap, nodeReg.KubeletExtraArgs)
 	envFileContent := fmt.Sprintf("%s=%s\n", constants.KubeletEnvFileVariableName, strings.Join(argList, " "))
 
 	return writeKubeletFlagBytesToDisk([]byte(envFileContent), kubeletDir)
 }
 
-// buildKubeletArgMap takes a InitConfiguration object and builds based on that a string-string map with flags
+// buildKubeletArgMap takes a kubeletFlagsOpts object and builds based on that a string-string map with flags
 // that should be given to the local kubelet daemon.
 func buildKubeletArgMap(opts kubeletFlagsOpts) map[string]string {
 	kubeletFlags := map[string]string{}

cmd/kubeadm/app/phases/upgrade/compute_test.go

@@ -174,7 +174,7 @@ func TestGetAvailableUpgrades(t *testing.T) {
 						KubeVersion:    "v1.10.3",
 						KubeadmVersion: "v1.10.3",
 						DNSType:        kubeadmapi.CoreDNS,
-						DNSVersion:     "1.2.6",
+						DNSVersion:     "1.3.1",
 						EtcdVersion:    "3.1.12",
 					},
 				},
@@ -213,7 +213,7 @@ func TestGetAvailableUpgrades(t *testing.T) {
 						KubeVersion:    "v1.10.3",
 						KubeadmVersion: "v1.10.3",
 						DNSType:        kubeadmapi.CoreDNS,
-						DNSVersion:     "1.2.6",
+						DNSVersion:     "1.3.1",
 						EtcdVersion:    "3.1.12",
 					},
 				},
@@ -252,7 +252,7 @@ func TestGetAvailableUpgrades(t *testing.T) {
 						KubeVersion:    "v1.11.0",
 						KubeadmVersion: "v1.11.0",
 						DNSType:        kubeadmapi.CoreDNS,
-						DNSVersion:     "1.2.6",
+						DNSVersion:     "1.3.1",
 						EtcdVersion:    "3.2.18",
 					},
 				},
@@ -291,7 +291,7 @@ func TestGetAvailableUpgrades(t *testing.T) {
 						KubeVersion:    "v1.10.5",
 						KubeadmVersion: "v1.10.5", // Note: The kubeadm version mustn't be "downgraded" here
 						DNSType:        kubeadmapi.CoreDNS,
-						DNSVersion:     "1.2.6",
+						DNSVersion:     "1.3.1",
 						EtcdVersion:    "3.1.12",
 					},
 				},
@@ -311,7 +311,7 @@ func TestGetAvailableUpgrades(t *testing.T) {
 						KubeVersion:    "v1.11.1",
 						KubeadmVersion: "v1.11.1",
 						DNSType:        kubeadmapi.CoreDNS,
-						DNSVersion:     "1.2.6",
+						DNSVersion:     "1.3.1",
 						EtcdVersion:    "3.2.18",
 					},
 				},
@@ -370,7 +370,7 @@ func TestGetAvailableUpgrades(t *testing.T) {
 						KubeVersion:    "v1.11.0-alpha.2",
 						KubeadmVersion: "v1.11.0-alpha.2",
 						DNSType:        kubeadmapi.CoreDNS,
-						DNSVersion:     "1.2.6",
+						DNSVersion:     "1.3.1",
 						EtcdVersion:    "3.2.18",
 					},
 				},
@@ -410,7 +410,7 @@ func TestGetAvailableUpgrades(t *testing.T) {
 						KubeVersion:    "v1.11.0-alpha.2",
 						KubeadmVersion: "v1.11.0-alpha.2",
 						DNSType:        kubeadmapi.CoreDNS,
-						DNSVersion:     "1.2.6",
+						DNSVersion:     "1.3.1",
 						EtcdVersion:    "3.2.18",
 					},
 				},
@@ -451,7 +451,7 @@ func TestGetAvailableUpgrades(t *testing.T) {
 						KubeVersion:    "v1.11.0-beta.1",
 						KubeadmVersion: "v1.11.0-beta.1",
 						DNSType:        kubeadmapi.CoreDNS,
-						DNSVersion:     "1.2.6",
+						DNSVersion:     "1.3.1",
 						EtcdVersion:    "3.2.18",
 					},
 				},
@@ -492,7 +492,7 @@ func TestGetAvailableUpgrades(t *testing.T) {
 						KubeVersion:    "v1.11.0-rc.1",
 						KubeadmVersion: "v1.11.0-rc.1",
 						DNSType:        kubeadmapi.CoreDNS,
-						DNSVersion:     "1.2.6",
+						DNSVersion:     "1.3.1",
 						EtcdVersion:    "3.2.18",
 					},
 				},
@@ -533,7 +533,7 @@ func TestGetAvailableUpgrades(t *testing.T) {
 						KubeVersion:    "v1.11.6-rc.1",
 						KubeadmVersion: "v1.11.6-rc.1",
 						DNSType:        kubeadmapi.CoreDNS,
-						DNSVersion:     "1.2.6",
+						DNSVersion:     "1.3.1",
 						EtcdVersion:    "3.2.18",
 					},
 				},
@@ -574,7 +574,7 @@ func TestGetAvailableUpgrades(t *testing.T) {
 						KubeVersion:    "v1.11.0-rc.1",
 						KubeadmVersion: "v1.11.0-rc.1",
 						DNSType:        kubeadmapi.CoreDNS,
-						DNSVersion:     "1.2.6",
+						DNSVersion:     "1.3.1",
 						EtcdVersion:    "3.2.18",
 					},
 				},
@@ -594,7 +594,7 @@ func TestGetAvailableUpgrades(t *testing.T) {
 						KubeVersion:    constants.MinimumControlPlaneVersion.WithPreRelease("alpha.2").String(),
 						KubeadmVersion: constants.MinimumControlPlaneVersion.WithPreRelease("alpha.2").String(),
 						DNSType:        kubeadmapi.CoreDNS,
-						DNSVersion:     "1.2.6",
+						DNSVersion:     "1.3.1",
 						EtcdVersion:    "3.2.24",
 					},
 				},
@@ -647,7 +647,7 @@ func TestGetAvailableUpgrades(t *testing.T) {
 						KubeVersion:    constants.MinimumControlPlaneVersion.WithPatch(1).String(),
 						KubeadmVersion: constants.MinimumControlPlaneVersion.WithPatch(1).String(),
 						DNSType:        kubeadmapi.CoreDNS,
-						DNSVersion:     "1.2.6",
+						DNSVersion:     "1.3.1",
 						EtcdVersion:    "3.2.24",
 					},
 				},
@@ -684,7 +684,7 @@ func TestGetAvailableUpgrades(t *testing.T) {
 						KubeVersion:    constants.MinimumControlPlaneVersion.String(),
 						KubeadmVersion: constants.MinimumControlPlaneVersion.String(),
 						DNSType:        kubeadmapi.CoreDNS,
-						DNSVersion:     "1.2.6",
+						DNSVersion:     "1.3.1",
 						EtcdVersion:    "3.2.24",
 					},
 				},

cmd/kubeadm/app/phases/upgrade/postupgrade.go

@@ -61,7 +61,7 @@ func PerformPostUpgradeTasks(client clientset.Interface, cfg *kubeadmapi.InitCon
 	}
 
 	// Create the new, version-branched kubelet ComponentConfig ConfigMap
-	if err := kubeletphase.CreateConfigMap(cfg, client); err != nil {
+	if err := kubeletphase.CreateConfigMap(cfg.ClusterConfiguration.ComponentConfigs.Kubelet, cfg.KubernetesVersion, client); err != nil {
 		errs = append(errs, errors.Wrap(err, "error creating kubelet configuration ConfigMap"))
 	}
 
@@ -108,21 +108,21 @@ func PerformPostUpgradeTasks(client clientset.Interface, cfg *kubeadmapi.InitCon
 	}
 
 	// Upgrade kube-dns/CoreDNS and kube-proxy
-	if err := dns.EnsureDNSAddon(cfg, client); err != nil {
+	if err := dns.EnsureDNSAddon(&cfg.ClusterConfiguration, client); err != nil {
 		errs = append(errs, err)
 	}
 	// Remove the old DNS deployment if a new DNS service is now used (kube-dns to CoreDNS or vice versa)
-	if err := removeOldDNSDeploymentIfAnotherDNSIsUsed(cfg, client, dryRun); err != nil {
+	if err := removeOldDNSDeploymentIfAnotherDNSIsUsed(&cfg.ClusterConfiguration, client, dryRun); err != nil {
 		errs = append(errs, err)
 	}
 
-	if err := proxy.EnsureProxyAddon(cfg, client); err != nil {
+	if err := proxy.EnsureProxyAddon(&cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, client); err != nil {
 		errs = append(errs, err)
 	}
 	return errorsutil.NewAggregate(errs)
 }
 
-func removeOldDNSDeploymentIfAnotherDNSIsUsed(cfg *kubeadmapi.InitConfiguration, client clientset.Interface, dryRun bool) error {
+func removeOldDNSDeploymentIfAnotherDNSIsUsed(cfg *kubeadmapi.ClusterConfiguration, client clientset.Interface, dryRun bool) error {
 	return apiclient.TryRunCommand(func() error {
 		installedDeploymentName := kubeadmconstants.KubeDNSDeploymentName
 		deploymentToDelete := kubeadmconstants.CoreDNSDeploymentName
@@ -210,7 +210,7 @@ func writeKubeletConfigFiles(client clientset.Interface, cfg *kubeadmapi.InitCon
 		// Write env file with flags for the kubelet to use. We do not need to write the --register-with-taints for the master,
 		// as we handle that ourselves in the markmaster phase
 		// TODO: Maybe we want to do that some time in the future, in order to remove some logic from the markmaster phase?
-		if err := kubeletphase.WriteKubeletDynamicEnvFile(cfg, false, kubeletDir); err != nil {
+		if err := kubeletphase.WriteKubeletDynamicEnvFile(&cfg.ClusterConfiguration, &cfg.NodeRegistration, false, kubeletDir); err != nil {
 			errs = append(errs, errors.Wrap(err, "error writing a dynamic environment file for the kubelet"))
 		}
 

cmd/kubeadm/app/phases/upgrade/staticpods.go

@@ -267,7 +267,7 @@ func performEtcdStaticPodUpgrade(client clientset.Interface, waiter apiclient.Wa
 	if err != nil {
 		return true, errors.Wrap(err, "failed to retrieve the current etcd version")
 	}
-	currentEtcdVersionStr, ok := currentEtcdVersions[etcdutil.GetClientURL(cfg)]
+	currentEtcdVersionStr, ok := currentEtcdVersions[etcdutil.GetClientURL(&cfg.LocalAPIEndpoint)]
 	if !ok {
 		return true, errors.Wrap(err, "failed to retrieve the current etcd version")
 	}
@@ -293,7 +293,7 @@ func performEtcdStaticPodUpgrade(client clientset.Interface, waiter apiclient.Wa
 
 	// Write the updated etcd static Pod manifest into the temporary directory, at this point no etcd change
 	// has occurred in any aspects.
-	if err := etcdphase.CreateLocalEtcdStaticPodManifestFile(pathMgr.TempManifestDir(), cfg); err != nil {
+	if err := etcdphase.CreateLocalEtcdStaticPodManifestFile(pathMgr.TempManifestDir(), cfg.NodeRegistration.Name, &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint); err != nil {
 		return true, errors.Wrap(err, "error creating local etcd static pod manifest file")
 	}
 

cmd/kubeadm/app/phases/upgrade/staticpods_test.go

@@ -453,7 +453,7 @@ func TestStaticPodControlPlane(t *testing.T) {
 			if err != nil {
 				t.Fatalf("couldn't run CreateInitStaticPodManifestFiles: %v", err)
 			}
-			err = etcdphase.CreateLocalEtcdStaticPodManifestFile(pathMgr.RealManifestDir(), oldcfg)
+			err = etcdphase.CreateLocalEtcdStaticPodManifestFile(pathMgr.RealManifestDir(), oldcfg.NodeRegistration.Name, &oldcfg.ClusterConfiguration, &oldcfg.LocalAPIEndpoint)
 			if err != nil {
 				t.Fatalf("couldn't run CreateLocalEtcdStaticPodManifestFile: %v", err)
 			}

cmd/kubeadm/app/preflight/checks.go

@@ -973,11 +973,11 @@ func RunJoinNodeChecks(execer utilsexec.Interface, cfg *kubeadmapi.JoinConfigura
 }
 
 // RunOptionalJoinNodeChecks executes all individual, applicable to node configuration dependant checks
-func RunOptionalJoinNodeChecks(execer utilsexec.Interface, initCfg *kubeadmapi.InitConfiguration, ignorePreflightErrors sets.String) error {
+func RunOptionalJoinNodeChecks(execer utilsexec.Interface, cfg *kubeadmapi.ClusterConfiguration, ignorePreflightErrors sets.String) error {
 	checks := []Checker{}
 
 	// Check ipvs required kernel module if we use ipvs kube-proxy mode
-	if initCfg.ComponentConfigs.KubeProxy != nil && initCfg.ComponentConfigs.KubeProxy.Mode == ipvsutil.IPVSProxyMode {
+	if cfg.ComponentConfigs.KubeProxy != nil && cfg.ComponentConfigs.KubeProxy.Mode == ipvsutil.IPVSProxyMode {
 		checks = append(checks,
 			ipvsutil.RequiredIPVSKernelModulesAvailableCheck{Executor: execer},
 		)

cmd/kubeadm/app/preflight/utils.go

@@ -33,7 +33,7 @@ func GetKubeletVersion(execer utilsexec.Interface) (*version.Version, error) {
 	command := execer.Command("kubelet", "--version")
 	out, err := command.CombinedOutput()
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrap(err, "cannot execute 'kubelet --version'")
 	}
 
 	cleanOutput := strings.TrimSpace(string(out))

cmd/kubeadm/app/util/certs/util.go

@@ -24,7 +24,7 @@ import (
 	"testing"
 
 	certutil "k8s.io/client-go/util/cert"
-	"k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil"
+	pkiutil "k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil"
 )
 
 // SetupCertificateAuthorithy is a utility function for kubeadm testing that creates a
@@ -230,11 +230,11 @@ func WritePKIFiles(t *testing.T, dir string, files PKIFiles) {
 	for filename, body := range files {
 		switch body := body.(type) {
 		case *x509.Certificate:
-			if err := certutil.WriteCert(path.Join(dir, filename), certutil.EncodeCertPEM(body)); err != nil {
+			if err := certutil.WriteCert(path.Join(dir, filename), pkiutil.EncodeCertPEM(body)); err != nil {
 				t.Errorf("unable to write certificate to file %q: [%v]", dir, err)
 			}
 		case *rsa.PublicKey:
-			publicKeyBytes, err := certutil.EncodePublicKeyPEM(body)
+			publicKeyBytes, err := pkiutil.EncodePublicKeyPEM(body)
 			if err != nil {
 				t.Errorf("unable to write public key to file %q: [%v]", filename, err)
 			}

cmd/kubeadm/app/util/endpoint.go

@@ -29,35 +29,35 @@ import (
 )
 
 // GetMasterEndpoint returns a properly formatted endpoint for the control plane built according following rules:
-// - If the ControlPlaneEndpoint is defined, use it.
-// - if the ControlPlaneEndpoint is defined but without a port number, use the ControlPlaneEndpoint + api.BindPort is used.
-// - Otherwise, in case the ControlPlaneEndpoint is not defined, use the api.AdvertiseAddress + the api.BindPort.
-func GetMasterEndpoint(cfg *kubeadmapi.InitConfiguration) (string, error) {
+// - If the controlPlaneEndpoint is defined, use it.
+// - if the controlPlaneEndpoint is defined but without a port number, use the controlPlaneEndpoint + localEndpoint.BindPort is used.
+// - Otherwise, in case the controlPlaneEndpoint is not defined, use the localEndpoint.AdvertiseAddress + the localEndpoint.BindPort.
+func GetMasterEndpoint(controlPlaneEndpoint string, localEndpoint *kubeadmapi.APIEndpoint) (string, error) {
 	// parse the bind port
-	bindPortString := strconv.Itoa(int(cfg.LocalAPIEndpoint.BindPort))
+	bindPortString := strconv.Itoa(int(localEndpoint.BindPort))
 	if _, err := ParsePort(bindPortString); err != nil {
-		return "", errors.Wrapf(err, "invalid value %q given for api.bindPort", cfg.LocalAPIEndpoint.BindPort)
+		return "", errors.Wrapf(err, "invalid value %q given for api.bindPort", localEndpoint.BindPort)
 	}
 
 	// parse the AdvertiseAddress
-	var ip = net.ParseIP(cfg.LocalAPIEndpoint.AdvertiseAddress)
+	var ip = net.ParseIP(localEndpoint.AdvertiseAddress)
 	if ip == nil {
-		return "", errors.Errorf("invalid value `%s` given for api.advertiseAddress", cfg.LocalAPIEndpoint.AdvertiseAddress)
+		return "", errors.Errorf("invalid value `%s` given for api.advertiseAddress", localEndpoint.AdvertiseAddress)
 	}
 
-	// set the master url using cfg.API.AdvertiseAddress + the cfg.API.BindPort
+	// set the master url using localEndpoint.AdvertiseAddress + the localEndpoint.BindPort
 	masterURL := &url.URL{
 		Scheme: "https",
 		Host:   net.JoinHostPort(ip.String(), bindPortString),
 	}
 
 	// if the controlplane endpoint is defined
-	if len(cfg.ControlPlaneEndpoint) > 0 {
+	if len(controlPlaneEndpoint) > 0 {
 		// parse the controlplane endpoint
 		var host, port string
 		var err error
-		if host, port, err = ParseHostPort(cfg.ControlPlaneEndpoint); err != nil {
-			return "", errors.Wrapf(err, "invalid value %q given for controlPlaneEndpoint", cfg.ControlPlaneEndpoint)
+		if host, port, err = ParseHostPort(controlPlaneEndpoint); err != nil {
+			return "", errors.Wrapf(err, "invalid value %q given for controlPlaneEndpoint", controlPlaneEndpoint)
 		}
 
 		// if a port is provided within the controlPlaneAddress warn the users we are using it, else use the bindport

cmd/kubeadm/app/util/endpoint_test.go

@@ -198,7 +198,7 @@ func TestGetMasterEndpoint(t *testing.T) {
 
 	for _, rt := range tests {
 		t.Run(rt.name, func(t *testing.T) {
-			actualEndpoint, actualError := GetMasterEndpoint(rt.cfg)
+			actualEndpoint, actualError := GetMasterEndpoint(rt.cfg.ControlPlaneEndpoint, &rt.cfg.LocalAPIEndpoint)
 
 			if (actualError != nil) && !rt.expectedError {
 				t.Errorf("%s unexpected failure: %v", rt.name, actualError)

cmd/kubeadm/app/util/etcd/etcd.go

@@ -290,14 +290,14 @@ func CheckConfigurationIsHA(cfg *kubeadmapi.Etcd) bool {
 
 // GetClientURL creates an HTTPS URL that uses the configured advertise
 // address and client port for the API controller
-func GetClientURL(cfg *kubeadmapi.InitConfiguration) string {
-	return "https://" + net.JoinHostPort(cfg.LocalAPIEndpoint.AdvertiseAddress, strconv.Itoa(constants.EtcdListenClientPort))
+func GetClientURL(localEndpoint *kubeadmapi.APIEndpoint) string {
+	return "https://" + net.JoinHostPort(localEndpoint.AdvertiseAddress, strconv.Itoa(constants.EtcdListenClientPort))
 }
 
 // GetPeerURL creates an HTTPS URL that uses the configured advertise
 // address and peer port for the API controller
-func GetPeerURL(cfg *kubeadmapi.InitConfiguration) string {
-	return "https://" + net.JoinHostPort(cfg.LocalAPIEndpoint.AdvertiseAddress, strconv.Itoa(constants.EtcdListenPeerPort))
+func GetPeerURL(localEndpoint *kubeadmapi.APIEndpoint) string {
+	return "https://" + net.JoinHostPort(localEndpoint.AdvertiseAddress, strconv.Itoa(constants.EtcdListenPeerPort))
 }
 
 // GetClientURLByIP creates an HTTPS URL based on an IP address

cmd/kubeadm/app/util/etcd/etcd_test.go

@@ -72,7 +72,7 @@ func TestCheckConfigurationIsHA(t *testing.T) {
 	}
 }
 
-func testGetURL(t *testing.T, getURLFunc func(*kubeadmapi.InitConfiguration) string, port int) {
+func testGetURL(t *testing.T, getURLFunc func(*kubeadmapi.APIEndpoint) string, port int) {
 	portStr := strconv.Itoa(port)
 	var tests = []struct {
 		name             string
@@ -102,12 +102,7 @@ func testGetURL(t *testing.T, getURLFunc func(*kubeadmapi.InitConfiguration) str
 	}
 
 	for _, test := range tests {
-		cfg := &kubeadmapi.InitConfiguration{
-			LocalAPIEndpoint: kubeadmapi.APIEndpoint{
-				AdvertiseAddress: test.advertiseAddress,
-			},
-		}
-		url := getURLFunc(cfg)
+		url := getURLFunc(&kubeadmapi.APIEndpoint{AdvertiseAddress: test.advertiseAddress})
 		if url != test.expectedURL {
 			t.Errorf("expected %s, got %s", test.expectedURL, url)
 		}

cmd/kubeadm/app/util/pkiutil/pki_helpers.go

@@ -18,6 +18,7 @@ package pkiutil
 
 import (
 	"crypto"
+	"crypto/rand"
 	cryptorand "crypto/rand"
 	"crypto/rsa"
 	"crypto/x509"
@@ -25,6 +26,8 @@ import (
 	"encoding/pem"
 	"fmt"
 	"io/ioutil"
+	"math"
+	"math/big"
 	"net"
 	"os"
 	"path/filepath"
@@ -40,9 +43,22 @@ import (
 	"k8s.io/kubernetes/pkg/registry/core/service/ipallocator"
 )
 
+const (
+	// PrivateKeyBlockType is a possible value for pem.Block.Type.
+	PrivateKeyBlockType = "PRIVATE KEY"
+	// PublicKeyBlockType is a possible value for pem.Block.Type.
+	PublicKeyBlockType = "PUBLIC KEY"
+	// CertificateBlockType is a possible value for pem.Block.Type.
+	CertificateBlockType = "CERTIFICATE"
+	// RSAPrivateKeyBlockType is a possible value for pem.Block.Type.
+	RSAPrivateKeyBlockType = "RSA PRIVATE KEY"
+	rsaKeySize             = 2048
+	duration365d           = time.Hour * 24 * 365
+)
+
 // NewCertificateAuthority creates new certificate and private key for the certificate authority
 func NewCertificateAuthority(config *certutil.Config) (*x509.Certificate, *rsa.PrivateKey, error) {
-	key, err := certutil.NewPrivateKey()
+	key, err := NewPrivateKey()
 	if err != nil {
 		return nil, nil, errors.Wrap(err, "unable to create private key")
 	}
@@ -57,12 +73,12 @@ func NewCertificateAuthority(config *certutil.Config) (*x509.Certificate, *rsa.P
 
 // NewCertAndKey creates new certificate and key by passing the certificate authority certificate and key
 func NewCertAndKey(caCert *x509.Certificate, caKey *rsa.PrivateKey, config *certutil.Config) (*x509.Certificate, *rsa.PrivateKey, error) {
-	key, err := certutil.NewPrivateKey()
+	key, err := NewPrivateKey()
 	if err != nil {
 		return nil, nil, errors.Wrap(err, "unable to create private key")
 	}
 
-	cert, err := certutil.NewSignedCert(*config, key, caCert, caKey)
+	cert, err := NewSignedCert(config, key, caCert, caKey)
 	if err != nil {
 		return nil, nil, errors.Wrap(err, "unable to sign certificate")
 	}
@@ -72,7 +88,7 @@ func NewCertAndKey(caCert *x509.Certificate, caKey *rsa.PrivateKey, config *cert
 
 // NewCSRAndKey generates a new key and CSR and that could be signed to create the given certificate
 func NewCSRAndKey(config *certutil.Config) (*x509.CertificateRequest, *rsa.PrivateKey, error) {
-	key, err := certutil.NewPrivateKey()
+	key, err := NewPrivateKey()
 	if err != nil {
 		return nil, nil, errors.Wrap(err, "unable to create private key")
 	}
@@ -111,7 +127,7 @@ func WriteCert(pkiPath, name string, cert *x509.Certificate) error {
 	}
 
 	certificatePath := pathForCert(pkiPath, name)
-	if err := certutil.WriteCert(certificatePath, certutil.EncodeCertPEM(cert)); err != nil {
+	if err := certutil.WriteCert(certificatePath, EncodeCertPEM(cert)); err != nil {
 		return errors.Wrapf(err, "unable to write certificate to file %s", certificatePath)
 	}
 
@@ -159,7 +175,7 @@ func WritePublicKey(pkiPath, name string, key *rsa.PublicKey) error {
 		return errors.New("public key cannot be nil when writing to file")
 	}
 
-	publicKeyBytes, err := certutil.EncodePublicKeyPEM(key)
+	publicKeyBytes, err := EncodePublicKeyPEM(key)
 	if err != nil {
 		return err
 	}
@@ -504,3 +520,63 @@ func NewCSR(cfg certutil.Config, key crypto.Signer) (*x509.CertificateRequest, e
 
 	return x509.ParseCertificateRequest(csrBytes)
 }
+
+// EncodeCertPEM returns PEM-endcoded certificate data
+func EncodeCertPEM(cert *x509.Certificate) []byte {
+	block := pem.Block{
+		Type:  CertificateBlockType,
+		Bytes: cert.Raw,
+	}
+	return pem.EncodeToMemory(&block)
+}
+
+// EncodePublicKeyPEM returns PEM-encoded public data
+func EncodePublicKeyPEM(key *rsa.PublicKey) ([]byte, error) {
+	der, err := x509.MarshalPKIXPublicKey(key)
+	if err != nil {
+		return []byte{}, err
+	}
+	block := pem.Block{
+		Type:  PublicKeyBlockType,
+		Bytes: der,
+	}
+	return pem.EncodeToMemory(&block), nil
+}
+
+// NewPrivateKey creates an RSA private key
+func NewPrivateKey() (*rsa.PrivateKey, error) {
+	return rsa.GenerateKey(cryptorand.Reader, rsaKeySize)
+}
+
+// NewSignedCert creates a signed certificate using the given CA certificate and key
+func NewSignedCert(cfg *certutil.Config, key crypto.Signer, caCert *x509.Certificate, caKey crypto.Signer) (*x509.Certificate, error) {
+	serial, err := rand.Int(rand.Reader, new(big.Int).SetInt64(math.MaxInt64))
+	if err != nil {
+		return nil, err
+	}
+	if len(cfg.CommonName) == 0 {
+		return nil, errors.New("must specify a CommonName")
+	}
+	if len(cfg.Usages) == 0 {
+		return nil, errors.New("must specify at least one ExtKeyUsage")
+	}
+
+	certTmpl := x509.Certificate{
+		Subject: pkix.Name{
+			CommonName:   cfg.CommonName,
+			Organization: cfg.Organization,
+		},
+		DNSNames:     cfg.AltNames.DNSNames,
+		IPAddresses:  cfg.AltNames.IPs,
+		SerialNumber: serial,
+		NotBefore:    caCert.NotBefore,
+		NotAfter:     time.Now().Add(duration365d).UTC(),
+		KeyUsage:     x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
+		ExtKeyUsage:  cfg.Usages,
+	}
+	certDERBytes, err := x509.CreateCertificate(cryptorand.Reader, &certTmpl, caCert, key.Public(), caKey)
+	if err != nil {
+		return nil, err
+	}
+	return x509.ParseCertificate(certDERBytes)
+}

cmd/kubeadm/app/util/staticpod/BUILD

@@ -12,11 +12,9 @@ go_test(
     embed = [":go_default_library"],
     deps = [
         "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
-        "//cmd/kubeadm/app/constants:go_default_library",
         "//cmd/kubeadm/test:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
-        "//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
     ],
 )
 
@@ -32,7 +30,6 @@ go_library(
         "//staging/src/k8s.io/api/core/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
-        "//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
         "//vendor/github.com/pkg/errors:go_default_library",
     ],
 )

cmd/kubeadm/app/util/staticpod/utils.go

@@ -31,7 +31,6 @@ import (
 	"k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/util/intstr"
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/kubernetes/cmd/kubeadm/app/util"
@@ -82,28 +81,11 @@ func ComponentResources(cpu string) v1.ResourceRequirements {
 	}
 }
 
-// ComponentProbe is a helper function building a ready v1.Probe object from some simple parameters
-func ComponentProbe(cfg *kubeadmapi.InitConfiguration, componentName string, port int, path string, scheme v1.URIScheme) *v1.Probe {
-	return &v1.Probe{
-		Handler: v1.Handler{
-			HTTPGet: &v1.HTTPGetAction{
-				Host:   GetProbeAddress(cfg, componentName),
-				Path:   path,
-				Port:   intstr.FromInt(port),
-				Scheme: scheme,
-			},
-		},
-		InitialDelaySeconds: 15,
-		TimeoutSeconds:      15,
-		FailureThreshold:    8,
-	}
-}
-
 // EtcdProbe is a helper function for building a shell-based, etcdctl v1.Probe object to healthcheck etcd
-func EtcdProbe(cfg *kubeadmapi.InitConfiguration, componentName string, port int, certsDir string, CACertName string, CertName string, KeyName string) *v1.Probe {
+func EtcdProbe(cfg *kubeadmapi.Etcd, port int, certsDir string, CACertName string, CertName string, KeyName string) *v1.Probe {
 	tlsFlags := fmt.Sprintf("--cacert=%[1]s/%[2]s --cert=%[1]s/%[3]s --key=%[1]s/%[4]s", certsDir, CACertName, CertName, KeyName)
 	// etcd pod is alive if a linearizable get succeeds.
-	cmd := fmt.Sprintf("ETCDCTL_API=3 etcdctl --endpoints=https://[%s]:%d %s get foo", GetProbeAddress(cfg, componentName), port, tlsFlags)
+	cmd := fmt.Sprintf("ETCDCTL_API=3 etcdctl --endpoints=https://[%s]:%d %s get foo", GetEtcdProbeAddress(cfg), port, tlsFlags)
 
 	return &v1.Probe{
 		Handler: v1.Handler{
@@ -225,71 +207,80 @@ func ReadStaticPodFromDisk(manifestPath string) (*v1.Pod, error) {
 	return pod, nil
 }
 
-// GetProbeAddress returns an IP address or 127.0.0.1 to use for liveness probes
-// in static pod manifests.
-func GetProbeAddress(cfg *kubeadmapi.InitConfiguration, componentName string) string {
-	switch {
-	case componentName == kubeadmconstants.KubeAPIServer:
-		// In the case of a self-hosted deployment, the initial host on which kubeadm --init is run,
-		// will generate a DaemonSet with a nodeSelector such that all nodes with the label
-		// node-role.kubernetes.io/master='' will have the API server deployed to it. Since the init
-		// is run only once on an initial host, the API advertise address will be invalid for any
-		// future hosts that do not have the same address. Furthermore, since liveness and readiness
-		// probes do not support the Downward API we cannot dynamically set the advertise address to
-		// the node's IP. The only option then is to use localhost.
-		if cfg.LocalAPIEndpoint.AdvertiseAddress != "" {
-			return cfg.LocalAPIEndpoint.AdvertiseAddress
-		}
-	case componentName == kubeadmconstants.KubeControllerManager:
-		if addr, exists := cfg.ControllerManager.ExtraArgs[kubeControllerManagerAddressArg]; exists {
-			return addr
-		}
-	case componentName == kubeadmconstants.KubeScheduler:
-		if addr, exists := cfg.Scheduler.ExtraArgs[kubeSchedulerAddressArg]; exists {
-			return addr
-		}
-	case componentName == kubeadmconstants.Etcd:
-		if cfg.Etcd.Local != nil && cfg.Etcd.Local.ExtraArgs != nil {
-			if arg, exists := cfg.Etcd.Local.ExtraArgs[etcdListenClientURLsArg]; exists {
-				// Use the first url in the listen-client-urls if multiple url's are specified.
-				if strings.ContainsAny(arg, ",") {
-					arg = strings.Split(arg, ",")[0]
-				}
-				parsedURL, err := url.Parse(arg)
-				if err != nil || parsedURL.Hostname() == "" {
-					break
+// GetAPIServerProbeAddress returns the probe address for the API server
+func GetAPIServerProbeAddress(endpoint *kubeadmapi.APIEndpoint) string {
+	// In the case of a self-hosted deployment, the initial host on which kubeadm --init is run,
+	// will generate a DaemonSet with a nodeSelector such that all nodes with the label
+	// node-role.kubernetes.io/master='' will have the API server deployed to it. Since the init
+	// is run only once on an initial host, the API advertise address will be invalid for any
+	// future hosts that do not have the same address. Furthermore, since liveness and readiness
+	// probes do not support the Downward API we cannot dynamically set the advertise address to
+	// the node's IP. The only option then is to use localhost.
+	if endpoint != nil && endpoint.AdvertiseAddress != "" {
+		return endpoint.AdvertiseAddress
+	}
+
+	return "127.0.0.1"
+}
+
+// GetControllerManagerProbeAddress returns the kubernetes controller manager probe address
+func GetControllerManagerProbeAddress(cfg *kubeadmapi.ClusterConfiguration) string {
+	if addr, exists := cfg.ControllerManager.ExtraArgs[kubeControllerManagerAddressArg]; exists {
+		return addr
+	}
+	return "127.0.0.1"
+}
+
+// GetSchedulerProbeAddress returns the kubernetes scheduler probe address
+func GetSchedulerProbeAddress(cfg *kubeadmapi.ClusterConfiguration) string {
+	if addr, exists := cfg.Scheduler.ExtraArgs[kubeSchedulerAddressArg]; exists {
+		return addr
+	}
+	return "127.0.0.1"
+}
+
+// GetEtcdProbeAddress returns the etcd probe address
+func GetEtcdProbeAddress(cfg *kubeadmapi.Etcd) string {
+	if cfg.Local != nil && cfg.Local.ExtraArgs != nil {
+		if arg, exists := cfg.Local.ExtraArgs[etcdListenClientURLsArg]; exists {
+			// Use the first url in the listen-client-urls if multiple url's are specified.
+			if strings.ContainsAny(arg, ",") {
+				arg = strings.Split(arg, ",")[0]
+			}
+			parsedURL, err := url.Parse(arg)
+			if err != nil || parsedURL.Hostname() == "" {
+				return "127.0.0.1"
+			}
+			// Return the IP if the URL contains an address instead of a name.
+			if ip := net.ParseIP(parsedURL.Hostname()); ip != nil {
+				// etcdctl doesn't support auto-converting zero addresses into loopback addresses
+				if ip.Equal(net.IPv4zero) {
+					return "127.0.0.1"
 				}
-				// Return the IP if the URL contains an address instead of a name.
-				if ip := net.ParseIP(parsedURL.Hostname()); ip != nil {
-					// etcdctl doesn't support auto-converting zero addresses into loopback addresses
-					if ip.Equal(net.IPv4zero) {
-						return "127.0.0.1"
-					}
-					if ip.Equal(net.IPv6zero) {
-						return net.IPv6loopback.String()
-					}
-					return ip.String()
+				if ip.Equal(net.IPv6zero) {
+					return net.IPv6loopback.String()
 				}
-				// Use the local resolver to try resolving the name within the URL.
-				// If the name can not be resolved, return an IPv4 loopback address.
-				// Otherwise, select the first valid IPv4 address.
-				// If the name does not resolve to an IPv4 address, select the first valid IPv6 address.
-				addrs, err := net.LookupIP(parsedURL.Hostname())
-				if err != nil {
+				return ip.String()
+			}
+			// Use the local resolver to try resolving the name within the URL.
+			// If the name can not be resolved, return an IPv4 loopback address.
+			// Otherwise, select the first valid IPv4 address.
+			// If the name does not resolve to an IPv4 address, select the first valid IPv6 address.
+			addrs, err := net.LookupIP(parsedURL.Hostname())
+			if err != nil {
+				return "127.0.0.1"
+			}
+			var ip net.IP
+			for _, addr := range addrs {
+				if addr.To4() != nil {
+					ip = addr
 					break
 				}
-				var ip net.IP
-				for _, addr := range addrs {
-					if addr.To4() != nil {
-						ip = addr
-						break
-					}
-					if addr.To16() != nil && ip == nil {
-						ip = addr
-					}
+				if addr.To16() != nil && ip == nil {
+					ip = addr
 				}
-				return ip.String()
 			}
+			return ip.String()
 		}
 	}
 	return "127.0.0.1"

cmd/kubeadm/app/util/system/docker_validator_test.go

@@ -28,7 +28,7 @@ func TestValidateDockerInfo(t *testing.T) {
 		Reporter: DefaultReporter,
 	}
 	spec := &DockerSpec{
-		Version:     []string{`1\.1[1-3]\..*`, `17\.0[3,6,9]\..*`, `18\.06\..*`},
+		Version:     []string{`1\.1[1-3]\..*`, `17\.0[3,6,9]\..*`, `18\.0[6,9]\..*`},
 		GraphDriver: []string{"driver_1", "driver_2"},
 	}
 	for _, test := range []struct {

cmd/kubeadm/app/util/system/types_unix.go

@@ -62,7 +62,7 @@ var DefaultSysSpec = SysSpec{
 	Cgroups: []string{"cpu", "cpuacct", "cpuset", "devices", "freezer", "memory"},
 	RuntimeSpec: RuntimeSpec{
 		DockerSpec: &DockerSpec{
-			Version:     []string{`1\.1[1-3]\..*`, `17\.0[3,6,9]\..*`, `18\.06\..*`},
+			Version:     []string{`1\.1[1-3]\..*`, `17\.0[3,6,9]\..*`, `18\.0[6,9]\..*`},
 			GraphDriver: []string{"aufs", "overlay", "overlay2", "devicemapper", "zfs"},
 		},
 	},

cmd/kubeadm/app/util/system/types_windows.go

@@ -38,7 +38,7 @@ var DefaultSysSpec = SysSpec{
 	Cgroups: []string{},
 	RuntimeSpec: RuntimeSpec{
 		DockerSpec: &DockerSpec{
-			Version:     []string{`18\.06\..*`}, //Requires [18.06] or later
+			Version:     []string{`18\.0[6,9]\..*`},
 			GraphDriver: []string{"windowsfilter"},
 		},
 	},

cmd/kubelet/app/options/options.go

@@ -473,8 +473,6 @@ func AddKubeletConfigFlags(mainfs *pflag.FlagSet, c *kubeletconfig.KubeletConfig
 	}()
 
 	fs.BoolVar(&c.FailSwapOn, "fail-swap-on", c.FailSwapOn, "Makes the Kubelet fail to start if swap is enabled on the node. ")
-	fs.BoolVar(&c.FailSwapOn, "experimental-fail-swap-on", c.FailSwapOn, "DEPRECATED: please use --fail-swap-on instead.")
-
 	fs.StringVar(&c.StaticPodPath, "pod-manifest-path", c.StaticPodPath, "Path to the directory containing static pod files to run, or the path to a single static pod file. Files starting with dots will be ignored.")
 	fs.DurationVar(&c.SyncFrequency.Duration, "sync-frequency", c.SyncFrequency.Duration, "Max period between synchronizing running containers and config")
 	fs.DurationVar(&c.FileCheckFrequency.Duration, "file-check-frequency", c.FileCheckFrequency.Duration, "Duration between checking config files for new data")

cmd/kubelet/app/server.go

@@ -369,7 +369,10 @@ func UnsecuredDependencies(s *options.KubeletServer) (*kubelet.Dependencies, err
 		}
 		mounter = mount.NewNsenterMounter(s.RootDirectory, ne)
 		// an exec interface which can use nsenter for flex plugin calls
-		pluginRunner = nsenter.NewNsenterExecutor(nsenter.DefaultHostRootFsPath, exec.New())
+		pluginRunner, err = nsenter.NewNsenter(nsenter.DefaultHostRootFsPath, exec.New())
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	var dockerClientConfig *dockershim.ClientConfig

hack/.golint_failures

 cmd/cloud-controller-manager/app/apis/config/v1alpha1
-cmd/hyperkube
 cmd/kube-apiserver/app
 cmd/kube-controller-manager/app
 cmd/kube-proxy/app
@@ -100,7 +99,6 @@ pkg/controller/clusterroleaggregation
 pkg/controller/cronjob
 pkg/controller/daemon
 pkg/controller/deployment
-pkg/controller/deployment/util
 pkg/controller/disruption
 pkg/controller/endpoint
 pkg/controller/garbagecollector
@@ -193,8 +191,6 @@ pkg/kubelet/checkpointmanager/checksum
 pkg/kubelet/checkpointmanager/testing/example_checkpoint_formats/v1
 pkg/kubelet/client
 pkg/kubelet/cm
-pkg/kubelet/cm/devicemanager/checkpoint
-pkg/kubelet/cm/util
 pkg/kubelet/config
 pkg/kubelet/configmap
 pkg/kubelet/container
@@ -359,7 +355,6 @@ pkg/util/bandwidth
 pkg/util/config
 pkg/util/ebtables
 pkg/util/env
-pkg/util/file
 pkg/util/goroutinemap/exponentialbackoff
 pkg/util/initsystem
 pkg/util/iptables

hack/.shellcheck_failures

-./build/common.sh
 ./build/lib/release.sh
 ./cluster/addons/addon-manager/kube-addons.sh
 ./cluster/addons/fluentd-elasticsearch/es-image/run.sh
@@ -158,19 +157,6 @@
 ./pkg/kubectl/cmd/edit/testdata/record_testcase.sh
 ./pkg/util/verify-util-pkg.sh
 ./plugin/pkg/admission/imagepolicy/gencerts.sh
-./staging/src/k8s.io/apiextensions-apiserver/examples/client-go/hack/update-codegen.sh
-./staging/src/k8s.io/apiextensions-apiserver/examples/client-go/hack/verify-codegen.sh
-./staging/src/k8s.io/apiextensions-apiserver/hack/build-image.sh
-./staging/src/k8s.io/apiextensions-apiserver/hack/update-codegen.sh
-./staging/src/k8s.io/apiextensions-apiserver/hack/verify-codegen.sh
-./staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/testcerts/gencerts.sh
-./staging/src/k8s.io/apiserver/pkg/util/webhook/gencerts.sh
-./staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/oidc/testdata/gen.sh
-./staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/gencerts.sh
-./staging/src/k8s.io/code-generator/generate-groups.sh
-./staging/src/k8s.io/code-generator/generate-internal-groups.sh
-./staging/src/k8s.io/code-generator/hack/update-codegen.sh
-./staging/src/k8s.io/code-generator/hack/verify-codegen.sh
 ./staging/src/k8s.io/csi-api/hack/update-codegen.sh
 ./staging/src/k8s.io/csi-api/hack/verify-codegen.sh
 ./staging/src/k8s.io/kube-aggregator/hack/build-image.sh
@@ -182,11 +168,6 @@
 ./staging/src/k8s.io/metrics/hack/verify-codegen.sh
 ./staging/src/k8s.io/node-api/hack/update-codegen.sh
 ./staging/src/k8s.io/node-api/hack/verify-codegen.sh
-./staging/src/k8s.io/sample-apiserver/hack/build-image.sh
-./staging/src/k8s.io/sample-apiserver/hack/update-codegen.sh
-./staging/src/k8s.io/sample-apiserver/hack/verify-codegen.sh
-./staging/src/k8s.io/sample-controller/hack/update-codegen.sh
-./staging/src/k8s.io/sample-controller/hack/verify-codegen.sh
 ./test/cmd/apply.sh
 ./test/cmd/apps.sh
 ./test/cmd/authorization.sh

hack/lib/util.sh

@@ -697,6 +697,15 @@ function kube::util::ensure-cfssl {
     return 0
   fi
 
+  host_arch=$(kube::util::host_arch)
+
+  if [[ "${host_arch}" != "amd64" ]]; then
+    echo "Cannot download cfssl on non-amd64 hosts and cfssl does not appear to be installed."
+    echo "Please install cfssl and cfssljson and verify they are in \$PATH."
+    echo "Hint: export PATH=\$PATH:\$GOPATH/bin; go get -u github.com/cloudflare/cfssl/cmd/..."
+    exit 1
+  fi
+
   # Create a temp dir for cfssl if no directory was given
   local cfssldir=${1:-}
   if [[ -z "${cfssldir}" ]]; then

hack/local-up-cluster.sh

@@ -78,6 +78,7 @@ CLOUD_PROVIDER=${CLOUD_PROVIDER:-""}
 CLOUD_CONFIG=${CLOUD_CONFIG:-""}
 FEATURE_GATES=${FEATURE_GATES:-"AllAlpha=false"}
 STORAGE_BACKEND=${STORAGE_BACKEND:-"etcd3"}
+STORAGE_MEDIA_TYPE=${STORAGE_MEDIA_TYPE:-""}
 # preserve etcd data. you also need to set ETCD_DIR.
 PRESERVE_ETCD="${PRESERVE_ETCD:-false}"
 # enable Pod priority and preemption
@@ -574,6 +575,7 @@ function start_apiserver {
       --insecure-bind-address="${API_HOST_IP}" \
       --insecure-port="${API_PORT}" \
       --storage-backend=${STORAGE_BACKEND} \
+      --storage-media-type=${STORAGE_MEDIA_TYPE} \
       --etcd-servers="http://${ETCD_HOST}:${ETCD_PORT}" \
       --service-cluster-ip-range="${SERVICE_CLUSTER_IP_RANGE}" \
       --feature-gates="${FEATURE_GATES}" \

hack/verify-generated-files-remake.sh

@@ -37,7 +37,7 @@ function find_genfiles() {
 # $1 = filename pattern as in "zz_generated.$1.go"
 # $2 timestamp file
 function newer() {
-    find_genfiles "$1" | while read F; do
+    find_genfiles "$1" | while read -r F; do
         if [[ "${F}" -nt "$2" ]]; then
             echo "${F}"
         fi
@@ -47,7 +47,7 @@ function newer() {
 # $1 = filename pattern as in "zz_generated.$1.go"
 # $2 timestamp file
 function older() {
-    find_genfiles "$1" | while read F; do
+    find_genfiles "$1" | while read -r F; do
         if [[ "$2" -nt "${F}" ]]; then
             echo "${F}"
         fi
@@ -61,7 +61,7 @@ function assert_clean() {
     X=($(newer deepcopy "${STAMP}"))
     if [[ "${#X[*]}" != 0 ]]; then
         echo "Generated files changed on back-to-back 'make' runs:"
-        echo "  ${X[@]:-(none)}"
+        echo "  ${X[*]:-(none)}"
         return 1
     fi
     true
@@ -82,19 +82,19 @@ make generated_files >/dev/null
 X=($(newer deepcopy "${STAMP}"))
 if [[ "${#X[*]}" != 1 || ! ( "${X[0]}" =~ "${DIR}/zz_generated.deepcopy.go" ) ]]; then
     echo "Wrong generated deepcopy files changed after touching src file:"
-    echo "  ${X[@]:-(none)}"
+    echo "  ${X[*]:-(none)}"
     exit 1
 fi
 X=($(newer defaults "${STAMP}"))
 if [[ "${#X[*]}" != 1 || ! ( "${X[0]}" =~ "${DIR}/zz_generated.defaults.go" ) ]]; then
     echo "Wrong generated defaults files changed after touching src file:"
-    echo "  ${X[@]:-(none)}"
+    echo "  ${X[*]:-(none)}"
     exit 1
 fi
 X=($(newer conversion "${STAMP}"))
 if [[ "${#X[*]}" != 1 || ! ( "${X[0]}" =~ "${DIR}/zz_generated.conversion.go" ) ]]; then
     echo "Wrong generated conversion files changed after touching src file:"
-    echo "  ${X[@]:-(none)}"
+    echo "  ${X[*]:-(none)}"
     exit 1
 fi
 
@@ -110,7 +110,7 @@ make generated_files >/dev/null
 X=($(older deepcopy "${STAMP}"))
 if [[ "${#X[*]}" != 0 ]]; then
     echo "Generated deepcopy files did not change after touching code-generator file:"
-    echo "  ${X[@]:-(none)}"
+    echo "  ${X[*]:-(none)}"
     exit 1
 fi
 
@@ -122,7 +122,7 @@ make generated_files >/dev/null
 X=($(older deepcopy "${STAMP}"))
 if [[ "${#X[*]}" != 0 ]]; then
     echo "Generated deepcopy files did not change after touching code-generator dir:"
-    echo "  ${X[@]:-(none)}"
+    echo "  ${X[*]}:-(none)"
     exit 1
 fi
 
@@ -134,7 +134,7 @@ make generated_files >/dev/null
 X=($(older deepcopy "${STAMP}"))
 if [[ "${#X[*]}" != 0 ]]; then
     echo "Generated deepcopy files did not change after touching code-generator dep file:"
-    echo "  ${X[@]:-(none)}"
+    echo "  ${X[*]:-(none)}"
     exit 1
 fi
 
@@ -146,7 +146,7 @@ make generated_files >/dev/null
 X=($(older deepcopy "${STAMP}"))
 if [[ "${#X[*]}" != 0 ]]; then
     echo "Generated deepcopy files did not change after touching code-generator dep dir:"
-    echo "  ${X[@]:-(none)}"
+    echo "  ${X[*]:-(none)}"
     exit 1
 fi
 
@@ -162,7 +162,7 @@ make generated_files >/dev/null
 X=($(older defaults "${STAMP}"))
 if [[ "${#X[*]}" != 0 ]]; then
     echo "Generated defaults files did not change after touching code-generator file:"
-    echo "  ${X[@]:-(none)}"
+    echo "  ${X[*]:-(none)}"
     exit 1
 fi
 
@@ -174,7 +174,7 @@ make generated_files >/dev/null
 X=($(older defaults "${STAMP}"))
 if [[ "${#X[*]}" != 0 ]]; then
     echo "Generated defaults files did not change after touching code-generator dir:"
-    echo "  ${X[@]:-(none)}"
+    echo "  ${X[*]:-(none)}"
     exit 1
 fi
 
@@ -186,7 +186,7 @@ make generated_files >/dev/null
 X=($(older defaults "${STAMP}"))
 if [[ "${#X[*]}" != 0 ]]; then
     echo "Generated defaults files did not change after touching code-generator dep file:"
-    echo "  ${X[@]:-(none)}"
+    echo "  ${X[*]:-(none)}"
     exit 1
 fi
 
@@ -198,7 +198,7 @@ make generated_files >/dev/null
 X=($(older defaults "${STAMP}"))
 if [[ "${#X[*]}" != 0 ]]; then
     echo "Generated defaults files did not change after touching code-generator dep dir:"
-    echo "  ${X[@]:-(none)}"
+    echo "  ${X[*]:-(none)}"
     exit 1
 fi
 
@@ -214,7 +214,7 @@ make generated_files >/dev/null
 X=($(older conversion "${STAMP}"))
 if [[ "${#X[*]}" != 0 ]]; then
     echo "Generated conversion files did not change after touching code-generator file:"
-    echo "  ${X[@]:-(none)}"
+    echo "  ${X[*]:-(none)}"
     exit 1
 fi
 
@@ -226,7 +226,7 @@ make generated_files >/dev/null
 X=($(older conversion "${STAMP}"))
 if [[ "${#X[*]}" != 0 ]]; then
     echo "Generated conversion files did not change after touching code-generator dir:"
-    echo "  ${X[@]:-(none)}"
+    echo "  ${X[*]:-(none)}"
     exit 1
 fi
 
@@ -238,7 +238,7 @@ make generated_files >/dev/null
 X=($(older conversion "${STAMP}"))
 if [[ "${#X[*]}" != 0 ]]; then
     echo "Generated conversion files did not change after touching code-generator dep file:"
-    echo "  ${X[@]:-(none)}"
+    echo "  ${X[*]:-(none)}"
     exit 1
 fi
 
@@ -250,7 +250,7 @@ make generated_files >/dev/null
 X=($(older conversion "${STAMP}"))
 if [[ "${#X[*]}" != 0 ]]; then
     echo "Generated conversion files did not change after touching code-generator dep dir:"
-    echo "  ${X[@]:-(none)}"
+    echo "  ${X[*]:-(none)}"
     exit 1
 fi
 
@@ -266,7 +266,7 @@ make generated_files >/dev/null
 X=($(newer openapi "${STAMP}"))
 if [[ "${#X[*]}" != 1 || ! ( "${X[0]}" =~ "pkg/generated/openapi/zz_generated.openapi.go" ) ]]; then
     echo "Wrong generated openapi files changed after touching src file:"
-    echo "${X[@]:-(none)}"
+    echo "${X[*]:-(none)}"
     exit 1
 fi
 
@@ -282,7 +282,7 @@ make generated_files >/dev/null
 X=($(older openapi "${STAMP}"))
 if [[ "${#X[*]}" != 0 ]]; then
     echo "Generated openapi files did not change after touching code-generator file:"
-    echo "  ${X[@]:-(none)}"
+    echo "  ${X[*]:-(none)}"
     exit 1
 fi
 
@@ -294,7 +294,7 @@ make generated_files >/dev/null
 X=($(older openapi "${STAMP}"))
 if [[ "${#X[*]}" != 0 ]]; then
     echo "Generated openapi files did not change after touching code-generator dir:"
-    echo "  ${X[@]:-(none)}"
+    echo "  ${X[*]:-(none)}"
     exit 1
 fi
 
@@ -306,7 +306,7 @@ make generated_files >/dev/null
 X=($(older openapi "${STAMP}"))
 if [[ "${#X[*]}" != 0 ]]; then
     echo "Generated openapi files did not change after touching code-generator dep file:"
-    echo "  ${X[@]:-(none)}"
+    echo "  ${X[*]:-(none)}"
     exit 1
 fi
 
@@ -318,6 +318,6 @@ make generated_files >/dev/null
 X=($(older openapi "${STAMP}"))
 if [[ "${#X[*]}" != 0 ]]; then
     echo "Generated openapi files did not change after touching code-generator dep dir:"
-    echo "  ${X[@]:-(none)}"
+    echo "  ${X[*]:-(none)}"
     exit 1
 fi

pkg/cloudprovider/providers/.import-restrictions

+{
+	"Rules": [
+	  {
+		"SelectorRegexp": "k8s[.]io/kubernetes",
+		"AllowedPrefixes": [
+		    "k8s.io/kubernetes/pkg/api/legacyscheme",
+		    "k8s.io/kubernetes/pkg/api/service",
+		    "k8s.io/kubernetes/pkg/api/v1/service",
+		    "k8s.io/kubernetes/pkg/apis/core",
+		    "k8s.io/kubernetes/pkg/cloudprovider",
+		    "k8s.io/kubernetes/pkg/credentialprovider",
+		    "k8s.io/kubernetes/pkg/features",
+		    "k8s.io/kubernetes/pkg/kubelet/apis",
+		    "k8s.io/kubernetes/pkg/master/ports",
+		    "k8s.io/kubernetes/pkg/util/mount",
+		    "k8s.io/kubernetes/pkg/util/file",
+		    "k8s.io/kubernetes/pkg/util/net/sets",
+		    "k8s.io/kubernetes/pkg/util/resizefs",
+		    "k8s.io/kubernetes/pkg/util/strings",
+		    "k8s.io/kubernetes/pkg/version",
+		    "k8s.io/kubernetes/pkg/volume"
+		],
+		"ForbiddenPrefixes": [
+		]
+	  }
+	]
+}