Update cluster/gce scripts to support Windows nodes.

cluster/common.sh

@@ -321,6 +321,9 @@ function find-tar() {
 #   KUBE_MANIFESTS_TAR
 function find-release-tars() {
   SERVER_BINARY_TAR=$(find-tar kubernetes-server-linux-amd64.tar.gz)
+  if [[ "${NUM_WINDOWS_NODES}" -gt "0" && "${USE_RELEASE_NODE_BINARIES:-false}" == "false" ]]; then
+    NODE_BINARY_TAR=$(find-tar kubernetes-node-windows-amd64.tar.gz)
+  fi
 
   # This tarball is used by GCI, Ubuntu Trusty, and Container Linux.
   KUBE_MANIFESTS_TAR=

cluster/gce/config-common.sh

@@ -14,26 +14,36 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# Returns the total number of Linux and Windows nodes in the cluster.
+#
+# Vars assumed:
+#   NUM_NODES
+#   NUM_WINDOWS_NODES
+function get-num-nodes {
+  echo "$((${NUM_NODES} + ${NUM_WINDOWS_NODES}))"
+}
+
 # Vars assumed:
 #   NUM_NODES
+#   NUM_WINDOWS_NODES
 function get-master-size {
   local suggested_master_size=1
-  if [[ "${NUM_NODES}" -gt "5" ]]; then
+  if [[ "$(get-num-nodes)" -gt "5" ]]; then
     suggested_master_size=2
   fi
-  if [[ "${NUM_NODES}" -gt "10" ]]; then
+  if [[ "$(get-num-nodes)" -gt "10" ]]; then
     suggested_master_size=4
   fi
-  if [[ "${NUM_NODES}" -gt "100" ]]; then
+  if [[ "$(get-num-nodes)" -gt "100" ]]; then
     suggested_master_size=8
   fi
-  if [[ "${NUM_NODES}" -gt "250" ]]; then
+  if [[ "$(get-num-nodes)" -gt "250" ]]; then
     suggested_master_size=16
   fi
-  if [[ "${NUM_NODES}" -gt "500" ]]; then
+  if [[ "$(get-num-nodes)" -gt "500" ]]; then
     suggested_master_size=32
   fi
-  if [[ "${NUM_NODES}" -gt "3000" ]]; then
+  if [[ "$(get-num-nodes)" -gt "3000" ]]; then
     suggested_master_size=64
   fi
   echo "${suggested_master_size}"
@@ -41,12 +51,13 @@ function get-master-size {
 
 # Vars assumed:
 #   NUM_NODES
+#   NUM_WINDOWS_NODES
 function get-master-root-disk-size() {
   local suggested_master_root_disk_size="20GB"
-  if [[ "${NUM_NODES}" -gt "500" ]]; then
+  if [[ "$(get-num-nodes)" -gt "500" ]]; then
     suggested_master_root_disk_size="100GB"
   fi
-  if [[ "${NUM_NODES}" -gt "3000" ]]; then
+  if [[ "$(get-num-nodes)" -gt "3000" ]]; then
     suggested_master_root_disk_size="500GB"
   fi
   echo "${suggested_master_root_disk_size}"
@@ -54,12 +65,13 @@ function get-master-root-disk-size() {
 
 # Vars assumed:
 #   NUM_NODES
+#   NUM_WINDOWS_NODES
 function get-master-disk-size() {
   local suggested_master_disk_size="20GB"
-  if [[ "${NUM_NODES}" -gt "500" ]]; then
+  if [[ "$(get-num-nodes)" -gt "500" ]]; then
     suggested_master_disk_size="100GB"
   fi
-  if [[ "${NUM_NODES}" -gt "3000" ]]; then
+  if [[ "$(get-num-nodes)" -gt "3000" ]]; then
     suggested_master_disk_size="200GB"
   fi
   echo "${suggested_master_disk_size}"
@@ -72,13 +84,13 @@ function get-node-ip-range {
     return
   fi
   local suggested_range="10.40.0.0/22"
-  if [[ "${NUM_NODES}" -gt 1000 ]]; then
+  if [[ "$(get-num-nodes)" -gt 1000 ]]; then
     suggested_range="10.40.0.0/21"
   fi
-  if [[ "${NUM_NODES}" -gt 2000 ]]; then
+  if [[ "$(get-num-nodes)" -gt 2000 ]]; then
     suggested_range="10.40.0.0/20"
   fi
-  if [[ "${NUM_NODES}" -gt 4000 ]]; then
+  if [[ "$(get-num-nodes)" -gt 4000 ]]; then
     suggested_range="10.40.0.0/19"
   fi
   echo "${suggested_range}"
@@ -86,13 +98,13 @@ function get-node-ip-range {
 
 function get-cluster-ip-range {
   local suggested_range="10.64.0.0/14"
-  if [[ "${NUM_NODES}" -gt 1000 ]]; then
+  if [[ "$(get-num-nodes)" -gt 1000 ]]; then
     suggested_range="10.64.0.0/13"
   fi
-  if [[ "${NUM_NODES}" -gt 2000 ]]; then
+  if [[ "$(get-num-nodes)" -gt 2000 ]]; then
     suggested_range="10.64.0.0/12"
   fi
-  if [[ "${NUM_NODES}" -gt 4000 ]]; then
+  if [[ "$(get-num-nodes)" -gt 4000 ]]; then
     suggested_range="10.64.0.0/11"
   fi
   echo "${suggested_range}"
@@ -114,3 +126,26 @@ function get-alias-range-size() {
 # NOTE: Avoid giving nodes empty scopes, because kubelet needs a service account
 # in order to initialize properly.
 NODE_SCOPES="${NODE_SCOPES:-monitoring,logging-write,storage-ro}"
+
+# Root directory for Kubernetes files on Windows nodes.
+WINDOWS_K8S_DIR="C:\etc\kubernetes"
+# Directory where Kubernetes binaries will be installed on Windows nodes.
+WINDOWS_NODE_DIR="${WINDOWS_K8S_DIR}\node\bin"
+# Directory where Kubernetes log files will be stored on Windows nodes.
+WINDOWS_LOGS_DIR="${WINDOWS_K8S_DIR}\logs"
+# Directory where CNI binaries will be stored on Windows nodes.
+WINDOWS_CNI_DIR="${WINDOWS_K8S_DIR}\cni"
+# Directory where CNI config files will be stored on Windows nodes.
+WINDOWS_CNI_CONFIG_DIR="${WINDOWS_K8S_DIR}\cni\config"
+# Pod manifests directory for Windows nodes on Windows nodes.
+WINDOWS_MANIFESTS_DIR="${WINDOWS_K8S_DIR}\manifests"
+# Directory where cert/key files will be stores on Windows nodes.
+WINDOWS_PKI_DIR="${WINDOWS_K8S_DIR}\pki"
+# Path for kubelet config file on Windows nodes.
+WINDOWS_KUBELET_CONFIG_FILE="${WINDOWS_K8S_DIR}\kubelet-config.yaml"
+# Path for kubeconfig file on Windows nodes.
+WINDOWS_KUBECONFIG_FILE="${WINDOWS_K8S_DIR}\kubelet.kubeconfig"
+# Path for bootstrap kubeconfig file on Windows nodes.
+WINDOWS_BOOTSTRAP_KUBECONFIG_FILE="${WINDOWS_K8S_DIR}\kubelet.bootstrap-kubeconfig"
+# Path for kube-proxy kubeconfig file on Windows nodes.
+WINDOWS_KUBEPROXY_KUBECONFIG_FILE="${WINDOWS_K8S_DIR}\kubeproxy.kubeconfig"

cluster/gce/config-default.sh

@@ -29,6 +29,7 @@ RELEASE_REGION_FALLBACK=${RELEASE_REGION_FALLBACK:-false}
 REGIONAL_KUBE_ADDONS=${REGIONAL_KUBE_ADDONS:-true}
 NODE_SIZE=${NODE_SIZE:-n1-standard-2}
 NUM_NODES=${NUM_NODES:-3}
+NUM_WINDOWS_NODES=${NUM_WINDOWS_NODES:-0}
 MASTER_SIZE=${MASTER_SIZE:-n1-standard-$(get-master-size)}
 MASTER_MIN_CPU_ARCHITECTURE=${MASTER_MIN_CPU_ARCHITECTURE:-} # To allow choosing better architectures.
 MASTER_DISK_TYPE=pd-ssd
@@ -44,6 +45,7 @@ NODE_LOCAL_SSDS=${NODE_LOCAL_SSDS:-0}
 # fluentd is not running as a manifest pod with appropriate label.
 # TODO(piosz): remove this in 1.8
 NODE_LABELS="${KUBE_NODE_LABELS:-beta.kubernetes.io/fluentd-ds-ready=true}"
+WINDOWS_NODE_LABELS="${WINDOWS_NODE_LABELS:-}"
 
 # An extension to local SSDs allowing users to specify block/fs and SCSI/NVMe devices
 # Format of this variable will be "#,scsi/nvme,block/fs" you can specify multiple
@@ -63,6 +65,7 @@ MIG_WAIT_UNTIL_STABLE_TIMEOUT=${MIG_WAIT_UNTIL_STABLE_TIMEOUT:-1800}
 
 MASTER_OS_DISTRIBUTION=${KUBE_MASTER_OS_DISTRIBUTION:-${KUBE_OS_DISTRIBUTION:-gci}}
 NODE_OS_DISTRIBUTION=${KUBE_NODE_OS_DISTRIBUTION:-${KUBE_OS_DISTRIBUTION:-gci}}
+WINDOWS_NODE_OS_DISTRIBUTION=${WINDOWS_NODE_OS_DISTRIBUTION:-win1803}
 
 if [[ "${MASTER_OS_DISTRIBUTION}" == "cos" ]]; then
     MASTER_OS_DISTRIBUTION="gci"
@@ -173,15 +176,19 @@ HEAPSTER_MACHINE_TYPE="${HEAPSTER_MACHINE_TYPE:-}"
 
 # NON_MASTER_NODE_LABELS are labels will only be applied on non-master nodes.
 NON_MASTER_NODE_LABELS="${KUBE_NON_MASTER_NODE_LABELS:-}"
+WINDOWS_NON_MASTER_NODE_LABELS="${WINDOWS_NON_MASTER_NODE_LABELS:-}"
 
 if [[ "${PREEMPTIBLE_MASTER}" == "true" ]]; then
     NODE_LABELS="${NODE_LABELS},cloud.google.com/gke-preemptible=true"
+    WINDOWS_NODE_LABELS="${WINDOWS_NODE_LABELS},cloud.google.com/gke-preemptible=true"
 elif [[ "${PREEMPTIBLE_NODE}" == "true" ]]; then
     NON_MASTER_NODE_LABELS="${NON_MASTER_NODE_LABELS},cloud.google.com/gke-preemptible=true"
+    WINDOWS_NON_MASTER_NODE_LABELS="${WINDOWS_NON_MASTER_NODE_LABELS},cloud.google.com/gke-preemptible=true"
 fi
 
 # To avoid running Calico on a node that is not configured appropriately,
 # label each Node so that the DaemonSet can run the Pods only on ready Nodes.
+# Windows nodes do not support Calico.
 if [[ ${NETWORK_POLICY_PROVIDER:-} == "calico" ]]; then
 	NON_MASTER_NODE_LABELS="${NON_MASTER_NODE_LABELS:+${NON_MASTER_NODE_LABELS},}projectcalico.org/ds-ready=true"
 fi
@@ -194,6 +201,7 @@ CUSTOM_TYPHA_DEPLOYMENT_YAML="${KUBE_CUSTOM_TYPHA_DEPLOYMENT_YAML:-}"
 
 # To avoid running netd on a node that is not configured appropriately,
 # label each Node so that the DaemonSet can run the Pods only on ready Nodes.
+# Windows nodes do not support netd.
 if [[ ${ENABLE_NETD:-} == "true" ]]; then
 	NON_MASTER_NODE_LABELS="${NON_MASTER_NODE_LABELS:+${NON_MASTER_NODE_LABELS},}cloud.google.com/gke-netd-ready=true"
 fi
@@ -467,3 +475,7 @@ ENABLE_NODE_TERMINATION_HANDLER="${ENABLE_NODE_TERMINATION_HANDLER:-false}"
 if [[ "${NODE_TERMINATION_HANDLER_IMAGE:-}" ]]; then
   PROVIDER_VARS="${PROVIDER_VARS:-} NODE_TERMINATION_HANDLER_IMAGE"
 fi
+
+# Taint Windows nodes by default to prevent Linux workloads from being
+# scheduled onto them.
+WINDOWS_NODE_TAINTS="${WINDOWS_NODE_TAINTS:-node.kubernetes.io/os=windows:NoSchedule}"

cluster/gce/config-test.sh

@@ -29,6 +29,7 @@ RELEASE_REGION_FALLBACK=${RELEASE_REGION_FALLBACK:-false}
 REGIONAL_KUBE_ADDONS=${REGIONAL_KUBE_ADDONS:-true}
 NODE_SIZE=${NODE_SIZE:-n1-standard-2}
 NUM_NODES=${NUM_NODES:-3}
+NUM_WINDOWS_NODES=${NUM_WINDOWS_NODES:-0}
 MASTER_SIZE=${MASTER_SIZE:-n1-standard-$(get-master-size)}
 MASTER_MIN_CPU_ARCHITECTURE=${MASTER_MIN_CPU_ARCHITECTURE:-} # To allow choosing better architectures.
 MASTER_DISK_TYPE=pd-ssd
@@ -44,6 +45,7 @@ NODE_LOCAL_SSDS=${NODE_LOCAL_SSDS:-0}
 # fluentd is not running as a manifest pod with appropriate label.
 # TODO(piosz): remove this in 1.8
 NODE_LABELS="${KUBE_NODE_LABELS:-beta.kubernetes.io/fluentd-ds-ready=true}"
+WINDOWS_NODE_LABELS="${WINDOWS_NODE_LABELS:-}"
 
 # An extension to local SSDs allowing users to specify block/fs and SCSI/NVMe devices
 # Format of this variable will be "#,scsi/nvme,block/fs" you can specify multiple
@@ -66,6 +68,8 @@ MIG_WAIT_UNTIL_STABLE_TIMEOUT=${MIG_WAIT_UNTIL_STABLE_TIMEOUT:-1800}
 
 MASTER_OS_DISTRIBUTION=${KUBE_MASTER_OS_DISTRIBUTION:-${KUBE_OS_DISTRIBUTION:-gci}}
 NODE_OS_DISTRIBUTION=${KUBE_NODE_OS_DISTRIBUTION:-${KUBE_OS_DISTRIBUTION:-gci}}
+WINDOWS_NODE_OS_DISTRIBUTION=${WINDOWS_NODE_OS_DISTRIBUTION:-win1803}
+
 if [[ "${MASTER_OS_DISTRIBUTION}" == "cos" ]]; then
     MASTER_OS_DISTRIBUTION="gci"
 fi
@@ -81,7 +85,7 @@ fi
 
 # To avoid failing large tests due to some flakes in starting nodes, allow
 # for a small percentage of nodes to not start during cluster startup.
-ALLOWED_NOTREADY_NODES="${ALLOWED_NOTREADY_NODES:-$((NUM_NODES / 100))}"
+ALLOWED_NOTREADY_NODES="${ALLOWED_NOTREADY_NODES:-$(($(get-num-nodes) / 100))}"
 
 # By default a cluster will be started with the master and nodes
 # on Container-optimized OS (cos, previously known as gci). If
@@ -215,11 +219,14 @@ KUBEPROXY_TEST_ARGS="${KUBEPROXY_TEST_ARGS:-} ${TEST_CLUSTER_API_CONTENT_TYPE}"
 
 # NON_MASTER_NODE_LABELS are labels will only be applied on non-master nodes.
 NON_MASTER_NODE_LABELS="${KUBE_NON_MASTER_NODE_LABELS:-}"
+WINDOWS_NON_MASTER_NODE_LABELS="${WINDOWS_NON_MASTER_NODE_LABELS:-}"
 
 if [[ "${PREEMPTIBLE_MASTER}" == "true" ]]; then
     NODE_LABELS="${NODE_LABELS},cloud.google.com/gke-preemptible=true"
+    WINDOWS_NODE_LABELS="${WINDOWS_NODE_LABELS},cloud.google.com/gke-preemptible=true"
 elif [[ "${PREEMPTIBLE_NODE}" == "true" ]]; then
     NON_MASTER_NODE_LABELS="${NON_MASTER_NODE_LABELS},cloud.google.com/gke-preemptible=true"
+    WINDOWS_NON_MASTER_NODE_LABELS="${WINDOWS_NON_MASTER_NODE_LABELS},cloud.google.com/gke-preemptible=true"
 fi
 
 # Optional: Enable netd.
@@ -230,6 +237,7 @@ CUSTOM_TYPHA_DEPLOYMENT_YAML="${KUBE_CUSTOM_TYPHA_DEPLOYMENT_YAML:-}"
 
 # To avoid running netd on a node that is not configured appropriately,
 # label each Node so that the DaemonSet can run the Pods only on ready Nodes.
+# Windows nodes do not support netd.
 if [[ ${ENABLE_NETD:-} == "true" ]]; then
 	NON_MASTER_NODE_LABELS="${NON_MASTER_NODE_LABELS:+${NON_MASTER_NODE_LABELS},}cloud.google.com/gke-netd-ready=true"
 fi
@@ -238,6 +246,7 @@ ENABLE_NODELOCAL_DNS="${KUBE_ENABLE_NODELOCAL_DNS:-false}"
 
 # To avoid running Calico on a node that is not configured appropriately,
 # label each Node so that the DaemonSet can run the Pods only on ready Nodes.
+# Windows nodes do not support Calico.
 if [[ ${NETWORK_POLICY_PROVIDER:-} == "calico" ]]; then
 	NON_MASTER_NODE_LABELS="${NON_MASTER_NODE_LABELS:+${NON_MASTER_NODE_LABELS},}projectcalico.org/ds-ready=true"
 fi
@@ -486,3 +495,7 @@ ENABLE_NODE_TERMINATION_HANDLER="${ENABLE_NODE_TERMINATION_HANDLER:-false}"
 if [[ "${NODE_TERMINATION_HANDLER_IMAGE:-}" ]]; then
   PROVIDER_VARS="${PROVIDER_VARS:-} NODE_TERMINATION_HANDLER_IMAGE"
 fi
+
+# Taint Windows nodes by default to prevent Linux workloads from being
+# scheduled onto them.
+WINDOWS_NODE_TAINTS="${WINDOWS_NODE_TAINTS:-node.kubernetes.io/os=windows:NoSchedule}"

cluster/gce/gci/node-helper.sh

@@ -17,7 +17,7 @@
 # A library of helper functions and constant for GCI distro
 source "${KUBE_ROOT}/cluster/gce/gci/helper.sh"
 
-function get-node-instance-metadata {
+function get-node-instance-metadata-from-file {
   local metadata=""
   metadata+="kube-env=${KUBE_TEMP}/node-kube-env.yaml,"
   metadata+="kubelet-config=${KUBE_TEMP}/node-kubelet-config.yaml,"
@@ -34,8 +34,8 @@ function get-node-instance-metadata {
 }
 
 # $1: template name (required).
-function create-node-instance-template {
+function create-linux-node-instance-template {
   local template_name="$1"
   ensure-gci-metadata-files
-  create-node-template "$template_name" "${scope_flags[*]}" "$(get-node-instance-metadata)"
+  create-node-template "${template_name}" "${scope_flags[*]}" "$(get-node-instance-metadata-from-file)" "" "linux"
 }

cluster/gce/win1803/OWNERS

+approvers:
+- yujuhong

cluster/gce/win1803/README-GCE-Windows-kube-up.md

+# Starting a Windows Kubernetes cluster on GCE using kube-up
+
+## Bring up the cluster
+
+Prerequisites: a Google Cloud Platform project.
+
+### 0. Prepare your environment
+
+Clone this repository under your `$GOPATH/src` directory on a Linux machine.
+Then, optionally clean/prepare your environment using these commands:
+
+```
+# Remove files that interfere with get-kube / kube-up:
+rm -rf ./kubernetes/; rm -f kubernetes.tar.gz; rm -f ~/.kube/config
+
+# Set the default gcloud project for this shell. This is optional but convenient
+# if you're working with multiple projects and don't want to repeatedly switch
+# between gcloud config configurations.
+export CLOUDSDK_CORE_PROJECT=<your_project_name>
+```
+
+### 1. Build Kubernetes
+
+The most straightforward approach to build those binaries is to run `make
+release`. However, that builds binaries for all supported platforms, and can be
+slow. You can speed up the process by following the instructions below to only
+build the necessary binaries.
+```
+# Fetch the PR: https://github.com/pjh/kubernetes/pull/43
+git remote add pjh https://github.com/pjh/kubernetes
+git fetch pjh pull/43/head
+
+# Get the commit hash and cherry-pick the commit to your current branch
+BUILD_WIN_COMMIT=$(git ls-remote pjh | grep refs/pull/43/head | cut -f 1)
+git cherry-pick $BUILD_WIN_COMMIT
+
+# Build binaries for both Linux and Windows
+make quick-release
+```
+
+### 2 Create a Kubernetes cluster
+
+You can create a regular Kubernetes cluster or an end-to-end test cluster.
+Please make sure you set the environment variables properly following the
+instructions in the previous section.
+
+First, set the following environment variables which are required for
+controlling the number of Linux and Windows nodes in the cluster and for
+enabling IP aliases (which are required for Windows pod routing):
+
+```
+export NUM_NODES=2  # number of Linux nodes
+export NUM_WINDOWS_NODES=2
+export KUBE_GCE_ENABLE_IP_ALIASES=true
+```
+
+If you wish to use `netd` as the CNI plugin for Linux nodes, set these
+variables:
+
+```
+export KUBE_ENABLE_NETD=true
+export KUBE_CUSTOM_NETD_YAML=$(curl -s \
+  https://raw.githubusercontent.com/GoogleCloudPlatform/netd/master/netd.yaml \
+  | sed -e 's/^/ /')
+```
+
+Now bring up a cluster using one of the following two methods:
+
+#### 2.a Create a regular Kubernetes cluster
+
+```
+# Invoke kube-up.sh with these environment variables:
+#   PROJECT: text name of your GCP project.
+#   KUBERNETES_SKIP_CONFIRM: skips any kube-up prompts.
+PROJECT=${CLOUDSDK_CORE_PROJECT} KUBERNETES_SKIP_CONFIRM=y ./cluster/kube-up.sh
+```
+
+To teardown the cluster run:
+
+```
+PROJECT=${CLOUDSDK_CORE_PROJECT} KUBERNETES_SKIP_CONFIRM=y ./cluster/kube-down.sh
+```
+
+#### 2.b Create a Kubernetes end-to-end (E2E) test cluster
+
+```
+PROJECT=${CLOUDSDK_CORE_PROJECT} go run ./hack/e2e.go  -- --up
+```
+This command, by default, tears down the existing E2E cluster and create a new
+one.
+
+No matter what type of cluster you chose to create, the result should be a
+Kubernetes cluster with one Linux master node, `NUM_NODES` Linux worker nodes
+and `NUM_WINDOWS_NODES` Windows worker nodes.
+
+## Validating the cluster
+
+Invoke this script to run a smoke test that verifies that the cluster has been
+brought up correctly:
+
+```
+cluster/gce/win1803/smoke-test.sh
+```
+
+## Running tests against the cluster
+
+These steps are based on
+[kubernetes-sigs/windows-testing](https://github.com/kubernetes-sigs/windows-testing).
+
+*   TODO(pjh): use patched `cluster/local/util.sh` from
+    https://github.com/pjh/kubernetes/blob/windows-up/cluster/local/util.sh.
+
+*   If necessary run `alias kubectl=client/bin/kubectl` .
+
+*   Set the following environment variables (these values should make sense if
+    you built your cluster using the kube-up steps above):
+
+    ```
+    export KUBE_HOME=$(pwd)
+    export KUBECONFIG=~/.kube/config
+    export KUBE_MASTER=local
+    export KUBE_MASTER_NAME=kubernetes-master
+    export KUBE_MASTER_IP=$(kubectl get node ${KUBE_MASTER_NAME} -o jsonpath='{.status.addresses[?(@.type=="ExternalIP")].address}')
+    export KUBE_MASTER_URL=https://${KUBE_MASTER_IP}
+    export KUBE_MASTER_PORT=443
+    ```
+
+*   Download the list of Windows e2e tests:
+
+    ```
+    curl https://raw.githubusercontent.com/e2e-win/e2e-win-prow-deployment/master/repo-list.txt -o ${KUBE_HOME}/repo-list.yaml
+    export KUBE_TEST_REPO_LIST=${KUBE_HOME}/repo-list.yaml
+    ```
+
+*   Download and configure the list of tests to exclude:
+
+    ```
+    curl https://raw.githubusercontent.com/e2e-win/e2e-win-prow-deployment/master/exclude_conformance_test.txt -o ${KUBE_HOME}/exclude_conformance_test.txt
+    export EXCLUDED_TESTS=$(cat exclude_conformance_test.txt |
+      tr -d '\r' |                # remove Windows carriage returns
+      tr -s '\n' '|' |            # coalesce newlines into |
+      tr -s ' ' '.' |             # coalesce spaces into .
+      sed -e 's/[]\[()]/\\&/g' |  # escape brackets and parentheses
+      sed -e 's/.$//g')           # remove final | added by tr
+    ```
+
+*   Taint the Linux nodes so that test pods will not land on them:
+
+    ```
+    export LINUX_NODES=$(kubectl get nodes -l beta.kubernetes.io/os=linux,kubernetes.io/hostname!=${KUBE_MASTER_NAME} -o name)
+    export LINUX_NODE_COUNT=$(echo ${LINUX_NODES} | wc -w)
+    for node in $LINUX_NODES; do
+      kubectl taint node $node node-under-test=false:NoSchedule
+    done
+    ```
+
+*   Build necessary test binaries:
+
+    ```
+    make WHAT=test/e2e/e2e.test
+    ```
+
+*   Run the tests with flags that point at the "local" (already-running) cluster
+    and that permit the `NoSchedule` Linux nodes:
+
+    ```
+    export KUBETEST_ARGS="--ginkgo.noColor=true "\
+    "--report-dir=${KUBE_HOME}/e2e-reports "\
+    "--allowed-not-ready-nodes=${LINUX_NODE_COUNT} "\
+    "--ginkgo.dryRun=false "\
+    "--ginkgo.focus=\[Conformance\] "\
+    "--ginkgo.skip=${EXCLUDED_TESTS}"
+
+    go run ${KUBE_HOME}/hack/e2e.go -- --verbose-commands \
+      --ginkgo-parallel=4 \
+      --check-version-skew=false --test --provider=local \
+      --test_args="${KUBETEST_ARGS}" &> ${KUBE_HOME}/conformance.out
+    ```
+
+    TODO: copy log files from Windows nodes using some command like:
+
+    ```
+    scp -r -o PreferredAuthentications=keyboard-interactive,password \
+      -o PubkeyAuthentication=no \
+      user@kubernetes-minion-windows-group-mk0p:C:\\etc\\kubernetes\\logs \
+      kubetest-logs/
+    ```

cluster/gce/win1803/common.psm1

+# Copyright 2019 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+<#
+.SYNOPSIS
+  Library containing common variables and code used by other PowerShell modules
+  and scripts for configuring Windows nodes.
+#>
+
+# REDO_STEPS affects the behavior of a node that is rebooted after initial
+# bringup. When true, on a reboot the scripts will redo steps that were
+# determined to have already been completed once (e.g. to overwrite
+# already-existing config files). When false the scripts will perform the
+# minimum required steps to re-join this node to the cluster.
+$REDO_STEPS = $false
+Export-ModuleMember -Variable REDO_STEPS
+
+# Writes $Message to the console. Terminates the script if $Fatal is set.
+function Log-Output {
+  param (
+    [parameter(Mandatory=$true)] [string]$Message,
+    [switch]$Fatal
+  )
+  Write-Host "${Message}"
+  if (${Fatal}) {
+    Exit 1
+  }
+}
+
+# Checks if a file should be written or overwritten by testing if it already
+# exists and checking the value of the global $REDO_STEPS variable. Emits an
+# informative message if the file already exists.
+#
+# Returns $true if the file does not exist, or if it does but the global
+# $REDO_STEPS variable is set to $true. Returns $false if the file exists and
+# the caller should not overwrite it.
+function ShouldWrite-File {
+  param (
+    [parameter(Mandatory=$true)] [string]$Filename
+  )
+  if (Test-Path $Filename) {
+    if ($REDO_STEPS) {
+      Log-Output "Warning: $Filename already exists, will overwrite it"
+      return $true
+    }
+    Log-Output "Skip: $Filename already exists, not overwriting it"
+    return $false
+  }
+  return $true
+}
+
+# Returns the GCE instance metadata value for $Key. If the key is not present
+# in the instance metadata returns $Default if set, otherwise returns $null.
+function Get-InstanceMetadataValue {
+  param (
+    [parameter(Mandatory=$true)] [string]$Key,
+    [parameter(Mandatory=$false)] [string]$Default
+  )
+
+  $url = ("http://metadata.google.internal/computeMetadata/v1/instance/" +
+          "attributes/$Key")
+  try {
+    $client = New-Object Net.WebClient
+    $client.Headers.Add('Metadata-Flavor', 'Google')
+    return ($client.DownloadString($url)).Trim()
+  }
+  catch [System.Net.WebException] {
+    if ($Default) {
+      return $Default
+    }
+    else {
+      Log-Output "Failed to retrieve value for $Key."
+      return $null
+    }
+  }
+}
+
+# Export all public functions:
+Export-ModuleMember -Function *-*

cluster/gce/win1803/configure.ps1

+# Copyright 2019 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+<#
+.SYNOPSIS
+  Top-level script that runs on Windows nodes to join them to the K8s cluster.
+#>
+
+$ErrorActionPreference = 'Stop'
+
+# Turn on tracing to debug
+# Set-PSDebug -Trace 1
+
+# Update TLS setting to enable Github downloads and disable progress bar to
+# increase download speed.
+[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+$ProgressPreference = 'SilentlyContinue'
+
+# Returns the GCE instance metadata value for $Key. If the key is not present
+# in the instance metadata returns $Default if set, otherwise returns $null.
+function Get-InstanceMetadataValue {
+  param (
+    [parameter(Mandatory=$true)] [string]$Key,
+    [parameter(Mandatory=$false)] [string]$Default
+  )
+
+  $url = ("http://metadata.google.internal/computeMetadata/v1/instance/" +
+          "attributes/$Key")
+  try {
+    $client = New-Object Net.WebClient
+    $client.Headers.Add('Metadata-Flavor', 'Google')
+    return ($client.DownloadString($url)).Trim()
+  }
+  catch [System.Net.WebException] {
+    if ($Default) {
+      return $Default
+    }
+    else {
+      Write-Host "Failed to retrieve value for $Key."
+      return $null
+    }
+  }
+}
+
+# Fetches the value of $MetadataKey, saves it to C:\$Filename and imports it as
+# a PowerShell module.
+#
+# Note: this function depends on common.psm1.
+function FetchAndImport-ModuleFromMetadata {
+  param (
+    [parameter(Mandatory=$true)] [string]$MetadataKey,
+    [parameter(Mandatory=$true)] [string]$Filename
+  )
+
+  $module = Get-InstanceMetadataValue $MetadataKey
+  if (Test-Path C:\$Filename) {
+    if (-not $REDO_STEPS) {
+      Log-Output "Skip: C:\$Filename already exists, not overwriting"
+      Import-Module -Force C:\$Filename
+      return
+    }
+    Log-Output "Warning: C:\$Filename already exists, will overwrite it."
+  }
+  New-Item -ItemType file -Force C:\$Filename | Out-Null
+  Set-Content C:\$Filename $module
+  Import-Module -Force C:\$Filename
+}
+
+try {
+  # Don't use FetchAndImport-ModuleFromMetadata for common.psm1 - the common
+  # module includes variables and functions that any other function may depend
+  # on.
+  $module = Get-InstanceMetadataValue 'common-psm1'
+  New-Item -ItemType file -Force C:\common.psm1 | Out-Null
+  Set-Content C:\common.psm1 $module
+  Import-Module -Force C:\common.psm1
+
+  # TODO(pjh): update the function to set $Filename automatically from the key,
+  # then put these calls into a loop over a list of XYZ-psm1 keys.
+  FetchAndImport-ModuleFromMetadata 'k8s-node-setup-psm1' 'k8s-node-setup.psm1'
+
+  Set-PrerequisiteOptions
+  $kube_env = Fetch-KubeEnv
+  Set-EnvironmentVars
+  Create-Directories
+  Download-HelperScripts
+
+  Create-PauseImage
+  DownloadAndInstall-KubernetesBinaries
+  Create-NodePki
+  Create-KubeletKubeconfig
+  Create-KubeproxyKubeconfig
+  Set-PodCidr
+  Configure-HostNetworkingService
+  Configure-CniNetworking
+  Configure-Kubelet
+
+  Start-WorkerServices
+  Log-Output 'Waiting 15 seconds for node to join cluster.'
+  Start-Sleep 15
+  Verify-WorkerServices
+}
+catch {
+  Write-Host 'Exception caught in script:'
+  Write-Host $_.InvocationInfo.PositionMessage
+  Write-Host "Kubernetes Windows node setup failed: $($_.Exception.Message)"
+  exit 1
+}

cluster/gce/win1803/node-helper.sh

+#!/usr/bin/env bash
+
+# Copyright 2019 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# A library of helper functions and constants for Windows nodes.
+
+function get-windows-node-instance-metadata-from-file {
+  local metadata=""
+  metadata+="cluster-name=${KUBE_TEMP}/cluster-name.txt,"
+  metadata+="kube-env=${KUBE_TEMP}/windows-node-kube-env.yaml,"
+  metadata+="kubelet-config=${KUBE_TEMP}/windows-node-kubelet-config.yaml,"
+  # To get startup script output run "gcloud compute instances
+  # get-serial-port-output <instance>" from the location where you're running
+  # kube-up.
+  metadata+="windows-startup-script-ps1=${KUBE_ROOT}/cluster/gce/${WINDOWS_NODE_OS_DISTRIBUTION}/configure.ps1,"
+  metadata+="common-psm1=${KUBE_ROOT}/cluster/gce/${WINDOWS_NODE_OS_DISTRIBUTION}/common.psm1,"
+  metadata+="k8s-node-setup-psm1=${KUBE_ROOT}/cluster/gce/${WINDOWS_NODE_OS_DISTRIBUTION}/k8s-node-setup.psm1,"
+  metadata+="user-profile-psm1=${KUBE_ROOT}/cluster/gce/${WINDOWS_NODE_OS_DISTRIBUTION}/user-profile.psm1,"
+  metadata+="${NODE_EXTRA_METADATA}"
+  echo "${metadata}"
+}
+
+function get-windows-node-instance-metadata {
+  local metadata=""
+  metadata+="k8s-version=${KUBE_VERSION:-v1.13.2},"
+  metadata+="serial-port-enable=1,"
+  # This enables logging the serial port output.
+  # https://cloud.google.com/compute/docs/instances/viewing-serial-port-output
+  metadata+="serial-port-logging-enable=true,"
+  metadata+="win-version=${WINDOWS_NODE_OS_DISTRIBUTION}"
+  echo "${metadata}"
+}
+
+# $1: template name (required).
+# $2: scopes flag.
+function create-windows-node-instance-template {
+  local template_name="$1"
+  local scopes_flag="$2"
+  create-node-template "${template_name}" "${scopes_flag}" "$(get-windows-node-instance-metadata-from-file)" "$(get-windows-node-instance-metadata)" "windows"
+}

cluster/gce/win1803/user-profile.psm1

+
+<#
+.Synopsis
+   Rough PS functions to create new user profiles
+.DESCRIPTION
+   Call the Create-NewProfile function directly to create a new profile
+.EXAMPLE
+   Create-NewProfile -Username 'testUser1' -Password 'testUser1'
+.NOTES
+   Created by: Josh Rickard (@MS_dministrator) and Thom Schumacher (@driberif)
+   Forked by: @crshnbrn66, then @pjh (2018-11-08). See
+     https://gist.github.com/pjh/9753cd14400f4e3d4567f4553ba75f1d/revisions
+   Date: 24MAR2017
+   Location: https://gist.github.com/crshnbrn66/7e81bf20408c05ddb2b4fdf4498477d8
+
+   Contact: https://github.com/MSAdministrator
+            MSAdministrator.com
+            https://github.com/crshnbrn66
+            powershellposse.com
+#>
+
+
+#Function to create the new local user first
+function New-LocalUser
+{
+    [CmdletBinding()]
+    [Alias()]
+    [OutputType([int])]
+    Param
+    (
+        # Param1 help description
+        [Parameter(Mandatory=$true,
+                   ValueFromPipelineByPropertyName=$true,
+                   Position=0)]
+        $userName,
+        # Param2 help description
+        [string]
+        $password
+    )
+ 
+    $system = [ADSI]"WinNT://$env:COMPUTERNAME";
+    $user = $system.Create("user",$userName);
+    $user.SetPassword($password);
+    $user.SetInfo();
+ 
+    $flag=$user.UserFlags.value -bor 0x10000;
+    $user.put("userflags",$flag);
+    $user.SetInfo();
+ 
+    $group = [ADSI]("WinNT://$env:COMPUTERNAME/Users");
+    $group.PSBase.Invoke("Add", $user.PSBase.Path);
+}
+
+#function to register a native method
+function Register-NativeMethod
+{
+    [CmdletBinding()]
+    [Alias()]
+    [OutputType([int])]
+    Param
+    (
+        # Param1 help description
+        [Parameter(Mandatory=$true,
+                   ValueFromPipelineByPropertyName=$true,
+                   Position=0)]
+        [string]$dll,
+ 
+        # Param2 help description
+        [Parameter(Mandatory=$true,
+                   ValueFromPipelineByPropertyName=$true,
+                   Position=1)]
+        [string]
+        $methodSignature
+    )
+ 
+    $script:nativeMethods += [PSCustomObject]@{ Dll = $dll; Signature = $methodSignature; }
+}
+function Get-Win32LastError
+{
+    [CmdletBinding()]
+    [Alias()]
+    [OutputType([int])]
+    Param($typeName = 'LastError')
+ if (-not ([System.Management.Automation.PSTypeName]$typeName).Type)
+    {
+    $lasterrorCode = $script:lasterror | ForEach-Object{
+        '[DllImport("kernel32.dll", SetLastError = true)]
+         public static extern uint GetLastError();'
+    }
+        Add-Type @"
+        using System;
+        using System.Text;
+        using System.Runtime.InteropServices;
+        public static class $typeName {
+            $lasterrorCode
+        }
+"@
+    }
+}
+#function to add native method
+function Add-NativeMethods
+{
+    [CmdletBinding()]
+    [Alias()]
+    [OutputType([int])]
+    Param($typeName = 'NativeMethods')
+ 
+    $nativeMethodsCode = $script:nativeMethods | ForEach-Object { "
+        [DllImport(`"$($_.Dll)`")]
+        public static extern $($_.Signature);
+    " }
+ 
+    Add-Type @"
+        using System;
+        using System.Text;
+        using System.Runtime.InteropServices;
+        public static class $typeName {
+            $nativeMethodsCode
+        }
+"@
+}
+
+#Main function to create the new user profile
+function Create-NewProfile {
+ 
+    [CmdletBinding()]
+    [Alias()]
+    [OutputType([int])]
+    Param
+    (
+        # Param1 help description
+        [Parameter(Mandatory=$true,
+                   ValueFromPipelineByPropertyName=$true,
+                   Position=0)]
+        [string]$UserName,
+ 
+        # Param2 help description
+        [Parameter(Mandatory=$true,
+                   ValueFromPipelineByPropertyName=$true,
+                   Position=1)]
+        [string]
+        $Password
+    )
+  
+    Write-Verbose "Creating local user $Username";
+  
+    try
+    {
+        New-LocalUser -username $UserName -password $Password;
+    }
+    catch
+    {
+        Write-Error $_.Exception.Message;
+        break;
+    }
+    $methodName = 'UserEnvCP'
+    $script:nativeMethods = @();
+ 
+    if (-not ([System.Management.Automation.PSTypeName]$MethodName).Type)
+    {
+        Register-NativeMethod "userenv.dll" "int CreateProfile([MarshalAs(UnmanagedType.LPWStr)] string pszUserSid,`
+         [MarshalAs(UnmanagedType.LPWStr)] string pszUserName,`
+         [Out][MarshalAs(UnmanagedType.LPWStr)] StringBuilder pszProfilePath, uint cchProfilePath)";
+ 
+        Add-NativeMethods -typeName $MethodName;
+    }
+ 
+    $localUser = New-Object System.Security.Principal.NTAccount("$UserName");
+    $userSID = $localUser.Translate([System.Security.Principal.SecurityIdentifier]);
+    $sb = new-object System.Text.StringBuilder(260);
+    $pathLen = $sb.Capacity;
+ 
+    Write-Verbose "Creating user profile for $Username";
+ 
+    try
+    {
+        [UserEnvCP]::CreateProfile($userSID.Value, $Username, $sb, $pathLen) | Out-Null;
+    }
+    catch
+    {
+        Write-Error $_.Exception.Message;
+        break;
+    }
+}
+
+function New-ProfileFromSID {
+ 
+    [CmdletBinding()]
+    [Alias()]
+    [OutputType([int])]
+    Param
+    (
+        # Param1 help description
+        [Parameter(Mandatory=$true,
+                   ValueFromPipelineByPropertyName=$true,
+                   Position=0)]
+        [string]$UserName,
+        [string]$domain = 'PHCORP'
+    )
+    $methodname = 'UserEnvCP2'
+    $script:nativeMethods = @();
+    
+    if (-not ([System.Management.Automation.PSTypeName]$methodname).Type)
+    {
+        Register-NativeMethod "userenv.dll" "int CreateProfile([MarshalAs(UnmanagedType.LPWStr)] string pszUserSid,`
+         [MarshalAs(UnmanagedType.LPWStr)] string pszUserName,`
+         [Out][MarshalAs(UnmanagedType.LPWStr)] StringBuilder pszProfilePath, uint cchProfilePath)";
+ 
+        Add-NativeMethods -typeName $methodname;
+    }
+ 
+    $sb = new-object System.Text.StringBuilder(260);
+    $pathLen = $sb.Capacity;
+ 
+    Write-Verbose "Creating user profile for $Username";
+    #$SID= ((get-aduser -id $UserName -ErrorAction Stop).sid.value)
+  if($domain)
+   {
+        $objUser = New-Object System.Security.Principal.NTAccount($domain, $UserName)
+        $strSID = $objUser.Translate([System.Security.Principal.SecurityIdentifier])
+        $SID = $strSID.Value
+   }
+   else 
+   {
+       $objUser = New-Object System.Security.Principal.NTAccount($UserName)
+       $strSID = $objUser.Translate([System.Security.Principal.SecurityIdentifier])
+       $SID = $strSID.Value
+   }
+    Write-Verbose "$UserName SID: $SID"
+    try
+    {
+       $result = [UserEnvCP2]::CreateProfile($SID, $Username, $sb, $pathLen) 
+       if($result -eq '-2147024713')
+       {
+           $status = "$userName already exists"
+           write-verbose "$username Creation Result: $result"
+        }
+        elseif($result -eq '-2147024809')
+        {
+            $staus = "$username Not Found"
+            write-verbose "$username creation result: $result"
+        }
+       elseif($result -eq 0)
+       {
+           $status = "$username Profile has been created"
+           write-verbose "$username Creation Result: $result"
+       }
+       else
+       {
+          $status = "$UserName unknown return result: $result"
+       }
+    }
+    catch
+    {
+        Write-Error $_.Exception.Message;
+        break;
+    }
+    $status
+}
+Function Remove-Profile {
+ 
+    [CmdletBinding()]
+    [Alias()]
+    [OutputType([int])]
+    Param
+    (
+        # Param1 help description
+        [Parameter(Mandatory=$true,
+                   ValueFromPipelineByPropertyName=$true,
+                   Position=0)]
+        [string]$UserName,
+        [string]$ProfilePath,
+        [string]$domain = 'PHCORP'
+    )
+    $methodname = 'userenvDP'
+    $script:nativeMethods = @();
+ 
+    if (-not ([System.Management.Automation.PSTypeName]"$methodname.profile").Type)
+    {
+      add-type @"
+using System.Runtime.InteropServices;
+
+namespace $typename
+{
+    public static class UserEnv
+    {
+        [DllImport("userenv.dll", CharSet = CharSet.Unicode, ExactSpelling = false, SetLastError = true)]
+        public static extern bool DeleteProfile(string sidString, string profilePath, string computerName);
+
+        [DllImport("kernel32.dll")]
+        public static extern uint GetLastError();
+    }
+
+    public static class Profile
+    {
+        public static uint Delete(string sidString)
+        { //Profile path and computer name are optional
+            if (!UserEnv.DeleteProfile(sidString, null, null))
+            {
+                return UserEnv.GetLastError();
+            }
+
+            return 0;
+        }
+    }
+}
+"@
+    }
+
+   #$SID= ((get-aduser -id $UserName -ErrorAction Stop).sid.value)
+   if($domain)
+   {
+        $objUser = New-Object System.Security.Principal.NTAccount($domain, $UserName)
+        $strSID = $objUser.Translate([System.Security.Principal.SecurityIdentifier])
+        $SID = $strSID.Value
+   }
+   else 
+   {
+       $objUser = New-Object System.Security.Principal.NTAccount($UserName)
+       $strSID = $objUser.Translate([System.Security.Principal.SecurityIdentifier])
+       $SID = $strSID.Value
+   }
+    Write-Verbose "$UserName SID: $SID"
+    try
+    {
+        #http://stackoverflow.com/questions/31949002/c-sharp-delete-user-profile
+       $result = [userenvDP.Profile]::Delete($SID)
+    }
+    catch
+    {
+        Write-Error $_.Exception.Message;
+        break;
+    }
+    $LastError
+}
+
+Export-ModuleMember Create-NewProfile

cluster/validate-cluster.sh

@@ -50,9 +50,8 @@ function kubectl_retry() {
 ALLOWED_NOTREADY_NODES="${ALLOWED_NOTREADY_NODES:-0}"
 CLUSTER_READY_ADDITIONAL_TIME_SECONDS="${CLUSTER_READY_ADDITIONAL_TIME_SECONDS:-30}"
 
-EXPECTED_NUM_NODES="${NUM_NODES}"
-
 if [[ "${KUBERNETES_PROVIDER:-}" == "gce" ]]; then
+  EXPECTED_NUM_NODES="$(get-num-nodes)"
   echo "Validating gce cluster, MULTIZONE=${MULTIZONE:-}"
   # In multizone mode we need to add instances for all nodes in the region.
   if [[ "${MULTIZONE:-}" == "true" ]]; then
@@ -60,6 +59,8 @@ if [[ "${KUBERNETES_PROVIDER:-}" == "gce" ]]; then
       --filter="name ~ '${NODE_INSTANCE_PREFIX}.*' AND zone:($(gcloud -q compute zones list --project="${PROJECT}" --filter=region=${REGION} --format=csv[no-heading]\(name\) | tr "\n" "," | sed  "s/,$//"))" | wc -l)
     echo "Computing number of nodes, NODE_INSTANCE_PREFIX=${NODE_INSTANCE_PREFIX}, REGION=${REGION}, EXPECTED_NUM_NODES=${EXPECTED_NUM_NODES}"
   fi
+else
+  EXPECTED_NUM_NODES="${NUM_NODES}"
 fi
 
 if [[ "${REGISTER_MASTER_KUBELET:-}" == "true" ]]; then