Code review + resync VMSS changes

Godeps/Godeps.json

@@ -30,33 +30,38 @@
 		},
 		{
 			"ImportPath": "github.com/Azure/azure-sdk-for-go/arm/compute",
-			"Comment": "v12.1.0-beta",
-			"Rev": "934e2462aeb6e0c14186dcfeedd73a026d1b8eeb"
+			"Comment": "v12.4.0-beta",
+			"Rev": "f111fc2fa3861c5fdced76cae4c9c71821969577"
 		},
 		{
 			"ImportPath": "github.com/Azure/azure-sdk-for-go/arm/containerregistry",
-			"Comment": "v12.1.0-beta",
-			"Rev": "934e2462aeb6e0c14186dcfeedd73a026d1b8eeb"
+			"Comment": "v12.4.0-beta",
+			"Rev": "f111fc2fa3861c5fdced76cae4c9c71821969577"
 		},
 		{
 			"ImportPath": "github.com/Azure/azure-sdk-for-go/arm/disk",
-			"Comment": "v12.1.0-beta",
-			"Rev": "934e2462aeb6e0c14186dcfeedd73a026d1b8eeb"
+			"Comment": "v12.4.0-beta",
+			"Rev": "f111fc2fa3861c5fdced76cae4c9c71821969577"
 		},
 		{
 			"ImportPath": "github.com/Azure/azure-sdk-for-go/arm/network",
-			"Comment": "v12.1.0-beta",
-			"Rev": "934e2462aeb6e0c14186dcfeedd73a026d1b8eeb"
+			"Comment": "v12.4.0-beta",
+			"Rev": "f111fc2fa3861c5fdced76cae4c9c71821969577"
 		},
 		{
 			"ImportPath": "github.com/Azure/azure-sdk-for-go/arm/storage",
-			"Comment": "v12.1.0-beta",
-			"Rev": "934e2462aeb6e0c14186dcfeedd73a026d1b8eeb"
+			"Comment": "v12.4.0-beta",
+			"Rev": "f111fc2fa3861c5fdced76cae4c9c71821969577"
+		},
+		{
+			"ImportPath": "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2017-12-01/compute",
+			"Comment": "v12.4.0-beta",
+			"Rev": "f111fc2fa3861c5fdced76cae4c9c71821969577"
 		},
 		{
 			"ImportPath": "github.com/Azure/azure-sdk-for-go/storage",
-			"Comment": "v12.1.0-beta",
-			"Rev": "934e2462aeb6e0c14186dcfeedd73a026d1b8eeb"
+			"Comment": "v12.4.0-beta",
+			"Rev": "f111fc2fa3861c5fdced76cae4c9c71821969577"
 		},
 		{
 			"ImportPath": "github.com/Azure/go-ansiterm",
@@ -68,33 +73,33 @@
 		},
 		{
 			"ImportPath": "github.com/Azure/go-autorest/autorest",
-			"Comment": "v9.1.0",
-			"Rev": "e14a70c556c8e0db173358d1a903dca345a8e75e"
+			"Comment": "v9.9.0",
+			"Rev": "d4e6b95c12a08b4de2d48b45d5b4d594e5d32fab"
 		},
 		{
 			"ImportPath": "github.com/Azure/go-autorest/autorest/adal",
-			"Comment": "v9.1.0",
-			"Rev": "e14a70c556c8e0db173358d1a903dca345a8e75e"
+			"Comment": "v9.9.0",
+			"Rev": "d4e6b95c12a08b4de2d48b45d5b4d594e5d32fab"
 		},
 		{
 			"ImportPath": "github.com/Azure/go-autorest/autorest/azure",
-			"Comment": "v9.1.0",
-			"Rev": "e14a70c556c8e0db173358d1a903dca345a8e75e"
+			"Comment": "v9.9.0",
+			"Rev": "d4e6b95c12a08b4de2d48b45d5b4d594e5d32fab"
 		},
 		{
 			"ImportPath": "github.com/Azure/go-autorest/autorest/date",
-			"Comment": "v9.1.0",
-			"Rev": "e14a70c556c8e0db173358d1a903dca345a8e75e"
+			"Comment": "v9.9.0",
+			"Rev": "d4e6b95c12a08b4de2d48b45d5b4d594e5d32fab"
 		},
 		{
 			"ImportPath": "github.com/Azure/go-autorest/autorest/to",
-			"Comment": "v9.1.0",
-			"Rev": "e14a70c556c8e0db173358d1a903dca345a8e75e"
+			"Comment": "v9.9.0",
+			"Rev": "d4e6b95c12a08b4de2d48b45d5b4d594e5d32fab"
 		},
 		{
 			"ImportPath": "github.com/Azure/go-autorest/autorest/validation",
-			"Comment": "v9.1.0",
-			"Rev": "e14a70c556c8e0db173358d1a903dca345a8e75e"
+			"Comment": "v9.9.0",
+			"Rev": "d4e6b95c12a08b4de2d48b45d5b4d594e5d32fab"
 		},
 		{
 			"ImportPath": "github.com/JeffAshton/win_pdh",
@@ -1151,17 +1156,17 @@
 		},
 		{
 			"ImportPath": "github.com/docker/go-connections/nat",
-			"Comment": "v0.2.1-30-g3ede32e",
+			"Comment": "v0.3.0",
 			"Rev": "3ede32e2033de7505e6500d6c868c2b9ed9f169d"
 		},
 		{
 			"ImportPath": "github.com/docker/go-connections/sockets",
-			"Comment": "v0.2.1-30-g3ede32e",
+			"Comment": "v0.3.0",
 			"Rev": "3ede32e2033de7505e6500d6c868c2b9ed9f169d"
 		},
 		{
 			"ImportPath": "github.com/docker/go-connections/tlsconfig",
-			"Comment": "v0.2.1-30-g3ede32e",
+			"Comment": "v0.3.0",
 			"Rev": "3ede32e2033de7505e6500d6c868c2b9ed9f169d"
 		},
 		{
@@ -2089,6 +2094,11 @@
 			"Rev": "2f5df55504ebc322e4d52d34df6a1f5b503bf26d"
 		},
 		{
+			"ImportPath": "github.com/marstr/guid",
+			"Comment": "v1.1.0-2-g8bdf7d1",
+			"Rev": "8bdf7d1a087ccc975cf37dd6507da50698fd19ca"
+		},
+		{
 			"ImportPath": "github.com/matttproud/golang_protobuf_extensions/pbutil",
 			"Rev": "fc2b8d3a73c4867e51861bbdd5ae3c1f0869dd6a"
 		},
@@ -2983,6 +2993,10 @@
 			"Rev": "f51c12702a4d776e4c1fa9b0fabab841babae631"
 		},
 		{
+			"ImportPath": "golang.org/x/tools/benchmark/parse",
+			"Rev": "2382e3994d48b1d22acc2c86bcad0a2aff028e32"
+		},
+		{
 			"ImportPath": "golang.org/x/tools/container/intsets",
 			"Rev": "2382e3994d48b1d22acc2c86bcad0a2aff028e32"
 		},

Godeps/OWNERS

-reviewers:
-  - vendor-reviewers
-approvers:
-  - vendor-approvers

README.md

@@ -68,10 +68,10 @@ That said, if you have questions, reach out to us
 [announcement]: https://cncf.io/news/announcement/2015/07/new-cloud-native-computing-foundation-drive-alignment-among-container
 [Borg]: https://research.google.com/pubs/pub43438.html
 [CNCF]: https://www.cncf.io/about
-[communication]: https://github.com/kubernetes/community/blob/master/communication.md
-[community repository]: https://github.com/kubernetes/community
+[communication]: https://git.k8s.io/community/communication
+[community repository]: https://git.k8s.io/community
 [containerized applications]: https://kubernetes.io/docs/concepts/overview/what-is-kubernetes/
-[developer's documentation]: https://github.com/kubernetes/community/tree/master/contributors/devel#readme
+[developer's documentation]: https://git.k8s.io/community/contributors/devel#readme
 [Docker environment]: https://docs.docker.com/engine
 [Go environment]: https://golang.org/doc/install
 [GoDoc]: https://godoc.org/k8s.io/kubernetes

api/openapi-spec/swagger.json

@@ -2,7 +2,7 @@
   "swagger": "2.0",
   "info": {
    "title": "Kubernetes",
-   "version": "v1.10.0"
+   "version": "v1.11.0"
   },
   "paths": {
    "/api/": {

cluster/addons/fluentd-elasticsearch/fluentd-es-configmap.yaml

 kind: ConfigMap
 apiVersion: v1
 metadata:
-  name: fluentd-es-config-v0.1.3
+  name: fluentd-es-config-v0.1.4
   namespace: kube-system
   labels:
     addonmanager.kubernetes.io/mode: Reconcile
@@ -160,6 +160,7 @@ data:
     # Examples:
     # time="2016-02-04T06:51:03.053580605Z" level=info msg="GET /containers/json"
     # time="2016-02-04T07:53:57.505612354Z" level=error msg="HTTP Error" err="No such image: -f" statusCode=404
+    # TODO(random-liu): Remove this after cri container runtime rolls out.
     <source>
       @id docker.log
       @type tail
@@ -307,6 +308,7 @@ data:
     </source>
 
     # Logs from systemd-journal for interesting services.
+    # TODO(random-liu): Remove this after cri container runtime rolls out.
     <source>
       @id journald-docker
       @type systemd
@@ -320,6 +322,18 @@ data:
     </source>
 
     <source>
+      @id journald-container-runtime
+      @type systemd
+      filters [{ "_SYSTEMD_UNIT": "{{ container_runtime }}.service" }]
+      <storage>
+        @type local
+        persistent true
+      </storage>
+      read_from_head true
+      tag container-runtime
+    </source>
+
+    <source>
       @id journald-kubelet
       @type systemd
       filters [{ "_SYSTEMD_UNIT": "kubelet.service" }]

cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml

@@ -113,4 +113,4 @@ spec:
           path: /usr/lib64
       - name: config-volume
         configMap:
-          name: fluentd-es-config-v0.1.3
+          name: fluentd-es-config-v0.1.4

cluster/addons/fluentd-gcp/fluentd-gcp-configmap.yaml

@@ -102,6 +102,7 @@ data:
     # Examples:
     # time="2016-02-04T06:51:03.053580605Z" level=info msg="GET /containers/json"
     # time="2016-02-04T07:53:57.505612354Z" level=error msg="HTTP Error" err="No such image: -f" statusCode=404
+    # TODO(random-liu): Remove this after cri container runtime rolls out.
     <source>
       type tail
       format /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/
@@ -239,6 +240,8 @@ data:
     </source>
 
     # Logs from systemd-journal for interesting services.
+    # TODO(random-liu): Keep this for compatibility, remove this after
+    # cri container runtime rolls out.
     <source>
       type systemd
       filters [{ "_SYSTEMD_UNIT": "docker.service" }]
@@ -249,6 +252,14 @@ data:
 
     <source>
       type systemd
+      filters [{ "_SYSTEMD_UNIT": "{{ container_runtime }}.service" }]
+      pos_file /var/log/gcp-journald-container-runtime.pos
+      read_from_head true
+      tag container-runtime
+    </source>
+
+    <source>
+      type systemd
       filters [{ "_SYSTEMD_UNIT": "kubelet.service" }]
       pos_file /var/log/gcp-journald-kubelet.pos
       read_from_head true
@@ -387,7 +398,7 @@ data:
       num_threads 2
     </match>
 metadata:
-  name: fluentd-gcp-config-v1.2.3
+  name: fluentd-gcp-config-v1.2.4
   namespace: kube-system
   labels:
     addonmanager.kubernetes.io/mode: Reconcile

cluster/addons/fluentd-gcp/fluentd-gcp-ds.yaml

@@ -32,12 +32,6 @@ spec:
         env:
         - name: FLUENTD_ARGS
           value: --no-supervisor -q
-        resources:
-          limits:
-            memory: 300Mi
-          requests:
-            cpu: 100m
-            memory: 200Mi
         volumeMounts:
         - name: varlog
           mountPath: /var/log
@@ -123,4 +117,4 @@ spec:
           path: /usr/lib64
       - name: config-volume
         configMap:
-          name: fluentd-gcp-config-v1.2.3
+          name: fluentd-gcp-config-v1.2.4

cluster/addons/fluentd-gcp/scaler-deployment.yaml

+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: fluentd-gcp-scaler
+  namespace: kube-system
+  labels:
+    k8s-app: fluentd-gcp-scaler
+    version: v0.1.0
+    addonmanager.kubernetes.io/mode: Reconcile
+spec:
+  selector:
+    matchLabels:
+      k8s-app: fluentd-gcp-scaler
+  template:
+    metadata:
+      labels:
+        k8s-app: fluentd-gcp-scaler
+    spec:
+      serviceAccountName: fluentd-gcp-scaler
+      containers:
+      - name: fluentd-gcp-scaler
+        image: gcr.io/google-containers/fluentd-gcp-scaler:0.1
+        command:
+          - /scaler.sh
+          - --ds-name=fluentd-gcp-v2.0.13
+          - --scaling-policy=fluentd-gcp-scaling-policy
+        env:
+        # Defaults, used if no overrides are found in fluentd-gcp-scaling-policy
+        - name: CPU_REQUEST
+          value: 100m
+        - name: MEMORY_REQUEST
+          value: 200Mi
+        - name: MEMORY_LIMIT
+          value: 300Mi

cluster/addons/fluentd-gcp/scaler-policy.yaml

+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: scalingpolicies.scalingpolicy.kope.io
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile
+spec:
+  group: scalingpolicy.kope.io
+  version: v1alpha1
+  names:
+    kind: ScalingPolicy
+    plural: scalingpolicies
+  scope: Namespaced

cluster/addons/fluentd-gcp/scaler-rbac.yaml

+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: fluentd-gcp-scaler
+  namespace: kube-system
+  labels:
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: system:fluentd-gcp-scaler
+  namespace: kube-system
+  labels:
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+rules:
+- apiGroups:
+  - "extensions"
+  resources:
+  - daemonsets
+  verbs:
+  - get
+  - patch
+- apiGroups:
+  - "scalingpolicy.kope.io"
+  resources:
+  - scalingpolicies
+  verbs:
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: fluentd-gcp-scaler-binding
+  namespace: kube-system
+  labels:
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: system:fluentd-gcp-scaler
+subjects:
+- kind: ServiceAccount
+  name: fluentd-gcp-scaler
+  namespace: kube-system

cluster/common.sh

@@ -523,6 +523,7 @@ DNS_DOMAIN: $(yaml-quote ${DNS_DOMAIN:-})
 ENABLE_DNS_HORIZONTAL_AUTOSCALER: $(yaml-quote ${ENABLE_DNS_HORIZONTAL_AUTOSCALER:-false})
 KUBE_PROXY_DAEMONSET: $(yaml-quote ${KUBE_PROXY_DAEMONSET:-false})
 KUBE_PROXY_TOKEN: $(yaml-quote ${KUBE_PROXY_TOKEN:-})
+KUBE_PROXY_MODE: $(yaml-quote ${KUBE_PROXY_MODE:-iptables})
 NODE_PROBLEM_DETECTOR_TOKEN: $(yaml-quote ${NODE_PROBLEM_DETECTOR_TOKEN:-})
 ADMISSION_CONTROL: $(yaml-quote ${ADMISSION_CONTROL:-})
 ENABLE_POD_SECURITY_POLICY: $(yaml-quote ${ENABLE_POD_SECURITY_POLICY:-})
@@ -561,6 +562,7 @@ ENABLE_PROMETHEUS_TO_SD: $(yaml-quote ${ENABLE_PROMETHEUS_TO_SD:-false})
 ENABLE_POD_PRIORITY: $(yaml-quote ${ENABLE_POD_PRIORITY:-})
 CONTAINER_RUNTIME: $(yaml-quote ${CONTAINER_RUNTIME:-})
 CONTAINER_RUNTIME_ENDPOINT: $(yaml-quote ${CONTAINER_RUNTIME_ENDPOINT:-})
+CONTAINER_RUNTIME_NAME: $(yaml-quote ${CONTAINER_RUNTIME_NAME:-})
 NODE_LOCAL_SSDS_EXT: $(yaml-quote ${NODE_LOCAL_SSDS_EXT:-})
 LOAD_IMAGE_COMMAND: $(yaml-quote ${LOAD_IMAGE_COMMAND:-})
 EOF

cluster/gce/config-default.sh

@@ -82,7 +82,8 @@ NODE_IMAGE_PROJECT=${KUBE_GCE_NODE_PROJECT:-cos-cloud}
 NODE_SERVICE_ACCOUNT=${KUBE_GCE_NODE_SERVICE_ACCOUNT:-default}
 CONTAINER_RUNTIME=${KUBE_CONTAINER_RUNTIME:-docker}
 CONTAINER_RUNTIME_ENDPOINT=${KUBE_CONTAINER_RUNTIME_ENDPOINT:-}
-LOAD_IMAGE_COMMAND=${KUBE_LOAD_IMAGE_COMMAND:-docker load -i}
+CONTAINER_RUNTIME_NAME=${KUBE_CONTAINER_RUNTIME_NAME:-}
+LOAD_IMAGE_COMMAND=${KUBE_LOAD_IMAGE_COMMAND:-}
 RKT_VERSION=${KUBE_RKT_VERSION:-1.23.0}
 RKT_STAGE1_IMAGE=${KUBE_RKT_STAGE1_IMAGE:-coreos.com/rkt/stage1-coreos}
 # MASTER_EXTRA_METADATA is the extra instance metadata on master instance separated by commas.
@@ -353,9 +354,9 @@ if [[ -n "${LOGROTATE_MAX_SIZE:-}" ]]; then
 fi
 
 # Fluentd requirements
-FLUENTD_GCP_MEMORY_LIMIT="${FLUENTD_GCP_MEMORY_LIMIT:-300Mi}"
-FLUENTD_GCP_CPU_REQUEST="${FLUENTD_GCP_CPU_REQUEST:-100m}"
-FLUENTD_GCP_MEMORY_REQUEST="${FLUENTD_GCP_MEMORY_REQUEST:-200Mi}"
+FLUENTD_GCP_MEMORY_LIMIT="${FLUENTD_GCP_MEMORY_LIMIT:-}"
+FLUENTD_GCP_CPU_REQUEST="${FLUENTD_GCP_CPU_REQUEST:-}"
+FLUENTD_GCP_MEMORY_REQUEST="${FLUENTD_GCP_MEMORY_REQUEST:-}"
 
 # Heapster requirements
 HEAPSTER_GCP_BASE_MEMORY="${HEAPSTER_GCP_BASE_MEMORY:-140Mi}"

cluster/gce/config-test.sh

@@ -80,7 +80,8 @@ NODE_IMAGE_PROJECT=${KUBE_GCE_NODE_PROJECT:-cos-cloud}
 NODE_SERVICE_ACCOUNT=${KUBE_GCE_NODE_SERVICE_ACCOUNT:-default}
 CONTAINER_RUNTIME=${KUBE_CONTAINER_RUNTIME:-docker}
 CONTAINER_RUNTIME_ENDPOINT=${KUBE_CONTAINER_RUNTIME_ENDPOINT:-}
-LOAD_IMAGE_COMMAND=${KUBE_LOAD_IMAGE_COMMAND:-docker load -i}
+CONTAINER_RUNTIME_NAME=${KUBE_CONTAINER_RUNTIME_NAME:-}
+LOAD_IMAGE_COMMAND=${KUBE_LOAD_IMAGE_COMMAND:-}
 GCI_DOCKER_VERSION=${KUBE_GCI_DOCKER_VERSION:-}
 RKT_VERSION=${KUBE_RKT_VERSION:-1.23.0}
 RKT_STAGE1_IMAGE=${KUBE_RKT_STAGE1_IMAGE:-coreos.com/rkt/stage1-coreos}
@@ -388,9 +389,9 @@ if [[ -n "${LOGROTATE_MAX_SIZE:-}" ]]; then
 fi
 
 # Fluentd requirements
-FLUENTD_GCP_MEMORY_LIMIT="${FLUENTD_GCP_MEMORY_LIMIT:-300Mi}"
-FLUENTD_GCP_CPU_REQUEST="${FLUENTD_GCP_CPU_REQUEST:-100m}"
-FLUENTD_GCP_MEMORY_REQUEST="${FLUENTD_GCP_MEMORY_REQUEST:-200Mi}"
+FLUENTD_GCP_MEMORY_LIMIT="${FLUENTD_GCP_MEMORY_LIMIT:-}"
+FLUENTD_GCP_CPU_REQUEST="${FLUENTD_GCP_CPU_REQUEST:-}"
+FLUENTD_GCP_MEMORY_REQUEST="${FLUENTD_GCP_MEMORY_REQUEST:-}"
 
 # Heapster requirements
 HEAPSTER_GCP_BASE_MEMORY="${HEAPSTER_GCP_BASE_MEMORY:-140Mi}"
@@ -410,6 +411,9 @@ ENABLE_PROMETHEUS_TO_SD="${ENABLE_PROMETHEUS_TO_SD:-true}"
 # Optional: [Experiment Only] Run kube-proxy as a DaemonSet if set to true, run as static pods otherwise.
 KUBE_PROXY_DAEMONSET="${KUBE_PROXY_DAEMONSET:-false}" # true, false
 
+# Optional: Change the kube-proxy implementation. Choices are [iptables, ipvs].
+KUBE_PROXY_MODE="${KUBE_PROXY_MODE:-iptables}"
+
 # Optional: duration of cluster signed certificates.
 CLUSTER_SIGNING_DURATION="${CLUSTER_SIGNING_DURATION:-}"
 

cluster/gce/gci/configure-helper.sh

@@ -1298,6 +1298,9 @@ function prepare-kube-proxy-manifest-variables {
   if [[ -n "${FEATURE_GATES:-}" ]]; then
     params+=" --feature-gates=${FEATURE_GATES}"
   fi
+  if [[ "${KUBE_PROXY_MODE:-}" == "ipvs" ]];then
+    params+=" --proxy-mode=ipvs --feature-gates=SupportIPVSProxyMode=true"
+  fi
   params+=" --iptables-sync-period=1m --iptables-min-sync-period=10s --ipvs-sync-period=1m --ipvs-min-sync-period=10s"
   if [[ -n "${KUBEPROXY_TEST_ARGS:-}" ]]; then
     params+=" ${KUBEPROXY_TEST_ARGS}"
@@ -1979,50 +1982,63 @@ function copy-manifests {
   chmod 644 "${dst_dir}"/*
 }
 
-# Fluentd manifest is modified using kubectl, which may not be available at
-# this point. Run this as a background process.
+# Fluentd resources are modified using ScalingPolicy CR, which may not be
+# available at this point. Run this as a background process.
 function wait-for-apiserver-and-update-fluentd {
-  local -r fluentd_gcp_yaml="${1}"
-
-  local modifying_flags=""
+  local any_overrides=false
   if [[ -n "${FLUENTD_GCP_MEMORY_LIMIT:-}" ]]; then
-    modifying_flags="${modifying_flags} --limits=memory=${FLUENTD_GCP_MEMORY_LIMIT}"
+    any_overrides=true
   fi
-  local request_resources=""
   if [[ -n "${FLUENTD_GCP_CPU_REQUEST:-}" ]]; then
-    request_resources="cpu=${FLUENTD_GCP_CPU_REQUEST}"
+    any_overrides=true
   fi
   if [[ -n "${FLUENTD_GCP_MEMORY_REQUEST:-}" ]]; then
-    if [[ -n "${request_resources}" ]]; then
-      request_resources="${request_resources},"
-    fi
-    request_resources="memory=${FLUENTD_GCP_MEMORY_REQUEST}"
+    any_overrides=true
   fi
-  if [[ -n "${request_resources}" ]]; then
-    modifying_flags="${modifying_flags} --requests=${request_resources}"
+  if ! $any_overrides; then
+    # Nothing to do here.
+    exit
   fi
 
-  until kubectl get nodes
+  # Wait until ScalingPolicy CRD is in place.
+  until kubectl get scalingpolicies.scalingpolicy.kope.io
   do
     sleep 10
   done
 
-  local -r temp_fluentd_gcp_yaml="${fluentd_gcp_yaml}.tmp"
-  if kubectl set resources --dry-run --local -f ${fluentd_gcp_yaml} ${modifying_flags} \
-      --containers=fluentd-gcp -o yaml > ${temp_fluentd_gcp_yaml}; then
-    mv ${temp_fluentd_gcp_yaml} ${fluentd_gcp_yaml}
-  else
-    (echo "Failed to update fluentd resources. Used manifest:" && cat ${temp_fluentd_gcp_yaml}) >&2
-    rm ${temp_fluentd_gcp_yaml}
-  fi
+  # Single-shot, not managed by addon manager. Can be later modified or removed
+  # at will.
+  cat <<EOF | kubectl apply -f -
+apiVersion: scalingpolicy.kope.io/v1alpha1
+kind: ScalingPolicy
+metadata:
+  name: fluentd-gcp-scaling-policy
+  namespace: kube-system
+spec:
+  containers:
+  - name: fluentd-gcp
+    resources:
+      requests:
+      - resource: cpu
+        base: ${FLUENTD_GCP_CPU_REQUEST:-}
+      - resource: memory
+        base: ${FLUENTD_GCP_MEMORY_REQUEST:-}
+      limits:
+      - resource: memory
+        base: ${FLUENTD_GCP_MEMORY_LIMIT:-}
+EOF
 }
 
 # Trigger background process that will ultimately update fluentd resource
 # requirements.
 function start-fluentd-resource-update {
-  local -r fluentd_gcp_yaml="${1}"
+  wait-for-apiserver-and-update-fluentd &
+}
 
-  wait-for-apiserver-and-update-fluentd ${fluentd_gcp_yaml} &
+# Update {{ container-runtime }} with actual container runtime name.
+function update-container-runtime {
+  local -r configmap_yaml="$1"
+  sed -i -e "s@{{ *container_runtime *}}@${CONTAINER_RUNTIME_NAME:-docker}@g" "${configmap_yaml}"
 }
 
 # Updates parameters in yaml file for prometheus-to-sd configuration, or
@@ -2177,15 +2193,19 @@ EOF
      [[ "${LOGGING_DESTINATION:-}" == "elasticsearch" ]] && \
      [[ "${ENABLE_CLUSTER_LOGGING:-}" == "true" ]]; then
     setup-addon-manifests "addons" "fluentd-elasticsearch"
+    local -r fluentd_es_configmap_yaml="${dst_dir}/fluentd-elasticsearch/fluentd-es-configmap.yaml"
+    update-container-runtime ${fluentd_es_configmap_yaml}
   fi
   if [[ "${ENABLE_NODE_LOGGING:-}" == "true" ]] && \
      [[ "${LOGGING_DESTINATION:-}" == "gcp" ]]; then
     setup-addon-manifests "addons" "fluentd-gcp"
     local -r event_exporter_yaml="${dst_dir}/fluentd-gcp/event-exporter.yaml"
     local -r fluentd_gcp_yaml="${dst_dir}/fluentd-gcp/fluentd-gcp-ds.yaml"
+    local -r fluentd_gcp_configmap_yaml="${dst_dir}/fluentd-gcp/fluentd-gcp-configmap.yaml"
     update-prometheus-to-sd-parameters ${event_exporter_yaml}
     update-prometheus-to-sd-parameters ${fluentd_gcp_yaml}
     start-fluentd-resource-update ${fluentd_gcp_yaml}
+    update-container-runtime ${fluentd_gcp_configmap_yaml}
   fi
   if [[ "${ENABLE_CLUSTER_UI:-}" == "true" ]]; then
     setup-addon-manifests "addons" "dashboard"

cluster/juju/layers/kubernetes-master/reactive/kubernetes_master.py

@@ -270,7 +270,7 @@ def password_changed():
     elif password == "":
         # Password not initialised
         password = token_generator()
-    setup_basic_auth(password, "admin", "admin")
+    setup_basic_auth(password, "admin", "admin", "system:masters")
     set_state('reconfigure.authentication.setup')
     remove_state('authentication.setup')
     set_state('client.password.initialised')

cluster/juju/layers/kubernetes-worker/reactive/kubernetes_worker.py

@@ -365,7 +365,7 @@ def start_worker(kube_api, kube_control, auth_control, cni):
     set_state('kubernetes-worker.config.created')
     restart_unit_services()
     update_kubelet_status()
-    apply_node_labels()
+    set_state('kubernetes-worker.label-config-required')
     remove_state('kubernetes-worker.restart-needed')
 
 
@@ -412,37 +412,54 @@ def render_and_launch_ingress():
         hookenv.close_port(443)
 
 
-@when('config.changed.labels', 'kubernetes-worker.config.created')
-def apply_node_labels():
-    ''' Parse the labels configuration option and apply the labels to the node.
-    '''
-    # scrub and try to format an array from the configuration option
-    config = hookenv.config()
-    user_labels = _parse_labels(config.get('labels'))
+@when('config.changed.labels')
+def handle_labels_changed():
+    set_state('kubernetes-worker.label-config-required')
 
-    # For diffing sake, iterate the previous label set
-    if config.previous('labels'):
-        previous_labels = _parse_labels(config.previous('labels'))
-        hookenv.log('previous labels: {}'.format(previous_labels))
-    else:
-        # this handles first time run if there is no previous labels config
-        previous_labels = _parse_labels("")
 
-    # Calculate label removal
-    for label in previous_labels:
-        if label not in user_labels:
-            hookenv.log('Deleting node label {}'.format(label))
-            _apply_node_label(label, delete=True)
-        # if the label is in user labels we do nothing here, it will get set
-        # during the atomic update below.
-
-    # Atomically set a label
-    for label in user_labels:
-        _apply_node_label(label, overwrite=True)
-
-    # Set label for application name
-    _apply_node_label('juju-application={}'.format(hookenv.service_name()),
-                      overwrite=True)
+@when('kubernetes-worker.label-config-required',
+      'kubernetes-worker.config.created')
+def apply_node_labels():
+    ''' Parse the labels configuration option and apply the labels to the
+        node. '''
+    # Get the user's configured labels.
+    config = hookenv.config()
+    user_labels = {}
+    for item in config.get('labels').split(' '):
+        if '=' in item:
+            key, val = item.split('=')
+            user_labels[key] = val
+        else:
+            hookenv.log('Skipping malformed option: {}.'.format(item))
+    # Collect the current label state.
+    current_labels = db.get('current_labels') or {}
+    # Remove any labels that the user has removed from the config.
+    for key in list(current_labels.keys()):
+        if key not in user_labels:
+            try:
+                remove_label(key)
+                del current_labels[key]
+                db.set('current_labels', current_labels)
+            except ApplyNodeLabelFailed as e:
+                hookenv.log(str(e))
+                return
+    # Add any new labels.
+    for key, val in user_labels.items():
+        try:
+            set_label(key, val)
+            current_labels[key] = val
+            db.set('current_labels', current_labels)
+        except ApplyNodeLabelFailed as e:
+            hookenv.log(str(e))
+            return
+    # Set the juju-application label.
+    try:
+        set_label('juju-application', hookenv.service_name())
+    except ApplyNodeLabelFailed as e:
+        hookenv.log(str(e))
+        return
+    # Label configuration complete.
+    remove_state('kubernetes-worker.label-config-required')
 
 
 @when_any('config.changed.kubelet-extra-args',
@@ -881,8 +898,8 @@ def enable_gpu():
         return
 
     # Apply node labels
-    _apply_node_label('gpu=true', overwrite=True)
-    _apply_node_label('cuda=true', overwrite=True)
+    set_label('gpu', 'true')
+    set_label('cuda', 'true')
 
     set_state('kubernetes-worker.gpu.enabled')
     set_state('kubernetes-worker.restart-needed')
@@ -902,8 +919,8 @@ def disable_gpu():
     hookenv.log('Disabling gpu mode')
 
     # Remove node labels
-    _apply_node_label('gpu', delete=True)
-    _apply_node_label('cuda', delete=True)
+    remove_label('gpu')
+    remove_label('cuda')
 
     remove_state('kubernetes-worker.gpu.enabled')
     set_state('kubernetes-worker.restart-needed')
@@ -953,15 +970,14 @@ def catch_change_in_creds(kube_control):
     """Request a service restart in case credential updates were detected."""
     nodeuser = 'system:node:{}'.format(get_node_name().lower())
     creds = kube_control.get_auth_credentials(nodeuser)
-    if creds \
-            and data_changed('kube-control.creds', creds) \
-            and creds['user'] == nodeuser:
+    if creds and creds['user'] == nodeuser:
         # We need to cache the credentials here because if the
         # master changes (master leader dies and replaced by a new one)
         # the new master will have no recollection of our certs.
         db.set('credentials', creds)
         set_state('worker.auth.bootstrapped')
-        set_state('kubernetes-worker.restart-needed')
+        if data_changed('kube-control.creds', creds):
+            set_state('kubernetes-worker.restart-needed')
 
 
 @when_not('kube-control.connected')
@@ -1012,43 +1028,33 @@ class ApplyNodeLabelFailed(Exception):
     pass
 
 
-def _apply_node_label(label, delete=False, overwrite=False):
-    ''' Invoke kubectl to apply node label changes '''
-    nodename = get_node_name()
-
-    # TODO: Make this part of the kubectl calls instead of a special string
-    cmd_base = 'kubectl --kubeconfig={0} label node {1} {2}'
-
-    if delete is True:
-        label_key = label.split('=')[0]
-        cmd = cmd_base.format(kubeconfig_path, nodename, label_key)
-        cmd = cmd + '-'
-    else:
-        cmd = cmd_base.format(kubeconfig_path, nodename, label)
-        if overwrite:
-            cmd = '{} --overwrite'.format(cmd)
-    cmd = cmd.split()
-
+def persistent_call(cmd, retry_message):
     deadline = time.time() + 180
     while time.time() < deadline:
         code = subprocess.call(cmd)
         if code == 0:
-            break
-        hookenv.log('Failed to apply label %s, exit code %d. Will retry.' % (
-            label, code))
+            return True
+        hookenv.log(retry_message)
         time.sleep(1)
     else:
-        msg = 'Failed to apply label %s' % label
-        raise ApplyNodeLabelFailed(msg)
+        return False
 
 
-def _parse_labels(labels):
-    ''' Parse labels from a key=value string separated by space.'''
-    label_array = labels.split(' ')
-    sanitized_labels = []
-    for item in label_array:
-        if '=' in item:
-            sanitized_labels.append(item)
-        else:
-            hookenv.log('Skipping malformed option: {}'.format(item))
-    return sanitized_labels
+def set_label(label, value):
+    nodename = get_node_name()
+    cmd = 'kubectl --kubeconfig={0} label node {1} {2}={3} --overwrite'
+    cmd = cmd.format(kubeconfig_path, nodename, label, value)
+    cmd = cmd.split()
+    retry = 'Failed to apply label %s=%s. Will retry.' % (label, value)
+    if not persistent_call(cmd, retry):
+        raise ApplyNodeLabelFailed(retry)
+
+
+def remove_label(label):
+    nodename = get_node_name()
+    cmd = 'kubectl --kubeconfig={0} label node {1} {2}-'
+    cmd = cmd.format(kubeconfig_path, nodename, label)
+    cmd = cmd.split()
+    retry = 'Failed to remove label {0}. Will retry.'.format(label)
+    if not persistent_call(cmd, retry):
+        raise ApplyNodeLabelFailed(retry)

cmd/kube-apiserver/app/options/options_test.go

@@ -208,7 +208,6 @@ func TestAddFlags(t *testing.T) {
 				ConfigFile: "/token-webhook-config",
 			},
 			BootstrapToken: &kubeoptions.BootstrapTokenAuthenticationOptions{},
-			Keystone:       &kubeoptions.KeystoneAuthenticationOptions{},
 			OIDC: &kubeoptions.OIDCAuthenticationOptions{
 				UsernameClaim: "sub",
 			},

cmd/kube-controller-manager/app/plugins.go

@@ -107,9 +107,6 @@ func ProbeExpandableVolumePlugins(config componentconfig.VolumeConfiguration) []
 	allPlugins = append(allPlugins, scaleio.ProbeVolumePlugins()...)
 	allPlugins = append(allPlugins, storageos.ProbeVolumePlugins()...)
 	allPlugins = append(allPlugins, fc.ProbeVolumePlugins()...)
-	if !utilfeature.DefaultFeatureGate.Enabled(features.CSIPersistentVolume) {
-		allPlugins = append(allPlugins, csi.ProbeVolumePlugins()...)
-	}
 	return allPlugins
 }
 

cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go

@@ -587,7 +587,7 @@ func TestValidateKubeletConfiguration(t *testing.T) {
 			OOMScoreAdj:                 utilpointer.Int32Ptr(-999),
 			PodsPerCore:                 100,
 			Port:                        65535,
-			ReadOnlyPort:                utilpointer.Int32Ptr(0),
+			ReadOnlyPort:                0,
 			RegistryBurst:               10,
 			RegistryPullQPS:             utilpointer.Int32Ptr(5),
 			HairpinMode:                 "promiscuous-bridge",
@@ -617,7 +617,7 @@ func TestValidateKubeletConfiguration(t *testing.T) {
 			OOMScoreAdj:                 utilpointer.Int32Ptr(-1001),
 			PodsPerCore:                 -10,
 			Port:                        0,
-			ReadOnlyPort:                utilpointer.Int32Ptr(-10),
+			ReadOnlyPort:                -10,
 			RegistryBurst:               -10,
 			RegistryPullQPS:             utilpointer.Int32Ptr(-10),
 		},

cmd/kubelet/app/options/BUILD

@@ -61,6 +61,7 @@ go_library(
         "//pkg/kubelet/apis/kubeletconfig/validation:go_default_library",
         "//pkg/kubelet/config:go_default_library",
         "//pkg/kubelet/types:go_default_library",
+        "//pkg/master/ports:go_default_library",
         "//pkg/util/taints:go_default_library",
         "//pkg/version/verflag:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",

cmd/kubelet/app/options/options.go

@@ -39,6 +39,7 @@ import (
 	kubeletconfigvalidation "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/validation"
 	"k8s.io/kubernetes/pkg/kubelet/config"
 	kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
+	"k8s.io/kubernetes/pkg/master/ports"
 	utiltaints "k8s.io/kubernetes/pkg/util/taints"
 )
 
@@ -65,6 +66,9 @@ type KubeletFlags struct {
 	// run those in addition to the pods specified by the local manifest, and exit.
 	RunOnce bool
 
+	// enableServer enables the Kubelet's server
+	EnableServer bool
+
 	// HostnameOverride is the hostname used to identify the kubelet instead
 	// of the actual hostname.
 	HostnameOverride string
@@ -214,6 +218,7 @@ func NewKubeletFlags() *KubeletFlags {
 	}
 
 	return &KubeletFlags{
+		EnableServer:                        true,
 		ContainerRuntimeOptions:             *NewContainerRuntimeOptions(),
 		CertDirectory:                       "/var/lib/kubelet/pki",
 		RootDirectory:                       v1alpha1.DefaultRootDir,
@@ -264,9 +269,24 @@ func NewKubeletConfiguration() (*kubeletconfig.KubeletConfiguration, error) {
 	if err := scheme.Convert(versioned, config, nil); err != nil {
 		return nil, err
 	}
+	applyLegacyDefaults(config)
 	return config, nil
 }
 
+// applyLegacyDefaults applies legacy default values to the KubeletConfiguration in order to
+// preserve the command line API. This is used to construct the baseline default KubeletConfiguration
+// before the first round of flag parsing.
+func applyLegacyDefaults(kc *kubeletconfig.KubeletConfiguration) {
+	// --anonymous-auth
+	kc.Authentication.Anonymous.Enabled = true
+	// --authentication-token-webhook
+	kc.Authentication.Webhook.Enabled = false
+	// --authorization-mode
+	kc.Authorization.Mode = kubeletconfig.KubeletAuthorizationModeAlwaysAllow
+	// --read-only-port
+	kc.ReadOnlyPort = ports.KubeletReadOnlyPort
+}
+
 // KubeletServer encapsulates all of the parameters necessary for starting up
 // a kubelet. These can either be set via command line or directly.
 type KubeletServer struct {
@@ -322,6 +342,7 @@ func (f *KubeletFlags) AddFlags(fs *pflag.FlagSet) {
 	fs.Float64Var(&f.ChaosChance, "chaos-chance", f.ChaosChance, "If > 0.0, introduce random client errors and latency. Intended for testing.")
 
 	fs.BoolVar(&f.RunOnce, "runonce", f.RunOnce, "If true, exit after spawning pods from local manifests or remote urls. Exclusive with --enable-server")
+	fs.BoolVar(&f.EnableServer, "enable-server", f.EnableServer, "Enable the Kubelet's server")
 
 	fs.StringVar(&f.HostnameOverride, "hostname-override", f.HostnameOverride, "If non-empty, will use this string as identification instead of the actual hostname.")
 
@@ -407,7 +428,6 @@ func AddKubeletConfigFlags(fs *pflag.FlagSet, c *kubeletconfig.KubeletConfigurat
 	fs.DurationVar(&c.HTTPCheckFrequency.Duration, "http-check-frequency", c.HTTPCheckFrequency.Duration, "Duration between checking http for new data")
 	fs.StringVar(&c.ManifestURL, "manifest-url", c.ManifestURL, "URL for accessing the container manifest")
 	fs.Var(flag.NewColonSeparatedMultimapStringString(&c.ManifestURLHeader), "manifest-url-header", "Comma-separated list of HTTP headers to use when accessing the manifest URL. Multiple headers with the same name will be added in the same order provided. This flag can be repeatedly invoked. For example: `--manifest-url-header 'a:hello,b:again,c:world' --manifest-url-header 'b:beautiful'`")
-	fs.BoolVar(&c.EnableServer, "enable-server", c.EnableServer, "Enable the Kubelet's server")
 	fs.Var(componentconfig.IPVar{Val: &c.Address}, "address", "The IP address for the Kubelet to serve on (set to `0.0.0.0` for all IPv4 interfaces and `::` for all IPv6 interfaces)")
 	fs.Int32Var(&c.Port, "port", c.Port, "The port for the Kubelet to serve on.")
 	fs.Int32Var(&c.ReadOnlyPort, "read-only-port", c.ReadOnlyPort, "The read-only port for the Kubelet to serve on with no authentication/authorization (set to 0 to disable)")

cmd/kubelet/app/server.go

@@ -930,18 +930,18 @@ func RunKubelet(kubeFlags *options.KubeletFlags, kubeCfg *kubeletconfiginternal.
 		}
 		glog.Infof("Started kubelet as runonce")
 	} else {
-		startKubelet(k, podCfg, kubeCfg, kubeDeps)
+		startKubelet(k, podCfg, kubeCfg, kubeDeps, kubeFlags.EnableServer)
 		glog.Infof("Started kubelet")
 	}
 	return nil
 }
 
-func startKubelet(k kubelet.Bootstrap, podCfg *config.PodConfig, kubeCfg *kubeletconfiginternal.KubeletConfiguration, kubeDeps *kubelet.Dependencies) {
+func startKubelet(k kubelet.Bootstrap, podCfg *config.PodConfig, kubeCfg *kubeletconfiginternal.KubeletConfiguration, kubeDeps *kubelet.Dependencies, enableServer bool) {
 	// start the kubelet
 	go wait.Until(func() { k.Run(podCfg.Updates()) }, 0, wait.NeverStop)
 
 	// start the kubelet server
-	if kubeCfg.EnableServer {
+	if enableServer {
 		go wait.Until(func() {
 			k.ListenAndServe(net.ParseIP(kubeCfg.Address), uint(kubeCfg.Port), kubeDeps.TLSOptions, kubeDeps.Auth, kubeCfg.EnableDebuggingHandlers, kubeCfg.EnableContentionProfiling)
 		}, 0, wait.NeverStop)

cmd/kubemark/BUILD

@@ -28,11 +28,14 @@ go_library(
         "//pkg/util/iptables/testing:go_default_library",
         "//pkg/util/sysctl/testing:go_default_library",
         "//pkg/version/prometheus:go_default_library",
+        "//pkg/version/verflag:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/github.com/spf13/cobra:go_default_library",
         "//vendor/github.com/spf13/pflag:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/util/flag:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/util/logs:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes:go_default_library",
         "//vendor/k8s.io/client-go/rest:go_default_library",
         "//vendor/k8s.io/client-go/tools/clientcmd:go_default_library",

cmd/kubemark/hollow-node.go

@@ -17,11 +17,20 @@ limitations under the License.
 package main
 
 import (
+	goflag "flag"
 	"fmt"
+	"math/rand"
+	"os"
+	"time"
+
+	"github.com/golang/glog"
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
 
 	"k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/util/sets"
-	"k8s.io/apiserver/pkg/util/flag"
+	utilflag "k8s.io/apiserver/pkg/util/flag"
+	"k8s.io/apiserver/pkg/util/logs"
 	clientset "k8s.io/client-go/kubernetes"
 	restclient "k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
@@ -37,10 +46,8 @@ import (
 	fakeiptables "k8s.io/kubernetes/pkg/util/iptables/testing"
 	fakesysctl "k8s.io/kubernetes/pkg/util/sysctl/testing"
 	_ "k8s.io/kubernetes/pkg/version/prometheus" // for version metric registration
+	"k8s.io/kubernetes/pkg/version/verflag"
 	fakeexec "k8s.io/utils/exec/testing"
-
-	"github.com/golang/glog"
-	"github.com/spf13/pflag"
 )
 
 type HollowNodeConfig struct {
@@ -88,10 +95,43 @@ func (c *HollowNodeConfig) createClientConfigFromFile() (*restclient.Config, err
 }
 
 func main() {
-	config := HollowNodeConfig{}
-	config.addFlags(pflag.CommandLine)
-	flag.InitFlags()
+	rand.Seed(time.Now().UTC().UnixNano())
+
+	command := newHollowNodeCommand()
+
+	// TODO: once we switch everything over to Cobra commands, we can go back to calling
+	// utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the
+	// normalize func and add the go flag set by hand.
+	pflag.CommandLine.SetNormalizeFunc(utilflag.WordSepNormalizeFunc)
+	pflag.CommandLine.AddGoFlagSet(goflag.CommandLine)
+	// utilflag.InitFlags()
+	logs.InitLogs()
+	defer logs.FlushLogs()
+
+	if err := command.Execute(); err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		os.Exit(1)
+	}
+}
+
+// newControllerManagerCommand creates a *cobra.Command object with default parameters
+func newHollowNodeCommand() *cobra.Command {
+	s := &HollowNodeConfig{}
+
+	cmd := &cobra.Command{
+		Use:  "kubemark",
+		Long: "kubemark",
+		Run: func(cmd *cobra.Command, args []string) {
+			verflag.PrintAndExitIfRequested()
+			run(s)
+		},
+	}
+	s.addFlags(cmd.Flags())
+
+	return cmd
+}
 
+func run(config *HollowNodeConfig) {
 	if !knownMorphs.Has(config.Morph) {
 		glog.Fatalf("Unknown morph: %v. Allowed values: %v", config.Morph, knownMorphs.List())
 	}

hack/.golint_failures

@@ -163,6 +163,7 @@ pkg/kubelet/apis
 pkg/kubelet/apis/cri/runtime/v1alpha2
 pkg/kubelet/apis/cri/testing
 pkg/kubelet/apis/deviceplugin/v1alpha
+pkg/kubelet/apis/deviceplugin/v1beta1
 pkg/kubelet/apis/kubeletconfig
 pkg/kubelet/apis/kubeletconfig/v1alpha1
 pkg/kubelet/cadvisor
@@ -602,7 +603,6 @@ staging/src/k8s.io/apiserver/pkg/util/trace
 staging/src/k8s.io/apiserver/pkg/util/webhook
 staging/src/k8s.io/apiserver/pkg/util/wsstream
 staging/src/k8s.io/apiserver/plugin/pkg/audit/log
-staging/src/k8s.io/apiserver/plugin/pkg/authenticator/password/keystone
 staging/src/k8s.io/apiserver/plugin/pkg/authenticator/password/passwordfile
 staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/oidc
 staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/oidc/testing

hack/grab-profiles.sh

@@ -51,7 +51,7 @@ KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
 source "${KUBE_ROOT}/hack/lib/init.sh"
 
 server_addr=""
-kubelet_addreses=""
+kubelet_addresses=""
 kubelet_binary=""
 master_binary=""
 scheduler_binary=""
@@ -97,7 +97,7 @@ while true; do
     --master-binary)
       shift
       if [ -z "$1" ]; then
-        >&2 echo "empty argumet to --master-binary flag"
+        >&2 echo "empty argument to --master-binary flag"
         exit 1
       fi
       master_binary=$1
@@ -111,16 +111,16 @@ while true; do
       shift
       profile_components="kubelet ${profile_components}"
       if [ -z "$1" ]; then
-        >&2 echo "empty argumet to --kubelet flag"
+        >&2 echo "empty argument to --kubelet flag"
         exit 1
       fi
-      kubelet_addreses="$1 $kubelet_addreses"
+      kubelet_addresses="$1 $kubelet_addresses"
       shift
       ;;
     --kubelet-binary)
       shift
       if [ -z "$1" ]; then
-        >&2 echo "empty argumet to --kubelet-binary flag"
+        >&2 echo "empty argument to --kubelet-binary flag"
         exit 1
       fi
       kubelet_binary=$1
@@ -133,7 +133,7 @@ while true; do
     --scheduler-binary)
       shift
       if [ -z "$1" ]; then
-        >&2 echo "empty argumet to --scheduler-binary flag"
+        >&2 echo "empty argument to --scheduler-binary flag"
         exit 1
       fi
       scheduler_binary=$1
@@ -142,7 +142,7 @@ while true; do
     --scheduler-port)
       shift
       if [ -z "$1" ]; then
-        >&2 echo "empty argumet to --scheduler-port flag"
+        >&2 echo "empty argument to --scheduler-port flag"
         exit 1
       fi
       scheduler_port=$1
@@ -155,7 +155,7 @@ while true; do
     --controller-manager-binary)
       shift
       if [ -z "$1" ]; then
-        >&2 echo "empty argumet to --controller-manager-binary flag"
+        >&2 echo "empty argument to --controller-manager-binary flag"
         exit 1
       fi
       controller_manager_binary=$1
@@ -164,10 +164,10 @@ while true; do
     --controller-manager-port)
       shift
       if [ -z "$1" ]; then
-        >&2 echo "empty argumet to --controller-manager-port flag"
+        >&2 echo "empty argument to --controller-manager-port flag"
         exit 1
       fi
-      controller-managerr_port=$1
+      controller_manager_port=$1
       shift
       ;;
     -o|--output)
@@ -240,18 +240,18 @@ if [[ -z "${requested_profiles}" ]]; then
   exit 1
 fi
 
-gcloud compute ssh "${server_addr}" --ssh-flag=-nN --ssh-flag=-L${tunnel_port}:localhost:8080 &
+gcloud compute ssh "${server_addr}" --ssh-flag=-nN --ssh-flag=-L"${tunnel_port}":localhost:8080 &
 
 echo "Waiting for tunnel to be created..."
-kube::util::wait_for_url http://localhost:${tunnel_port}/healthz
+kube::util::wait_for_url http://localhost:"${tunnel_port}"/healthz
 
 SSH_PID=$(pgrep -f "/usr/bin/ssh.*${tunnel_port}:localhost:8080")
-kube::util::trap_add 'kill $SSH_PID' EXIT
-kube::util::trap_add 'kill $SSH_PID' SIGTERM
+kube::util::trap_add "kill $SSH_PID" EXIT
+kube::util::trap_add "kill $SSH_PID" SIGTERM
 
-requested_profiles=$(echo ${requested_profiles} | xargs -n1 | LC_ALL=C sort -u | xargs)
-profile_components=$(echo ${profile_components} | xargs -n1 | LC_ALL=C sort -u | xargs)
-kubelet_addreses=$(echo ${kubelet_addreses} | xargs -n1 | LC_ALL=C sort -u | xargs)
+requested_profiles=$(echo "${requested_profiles}" | xargs -n1 | LC_ALL=C sort -u | xargs)
+profile_components=$(echo "${profile_components}" | xargs -n1 | LC_ALL=C sort -u | xargs)
+kubelet_addresses=$(echo "${kubelet_addresses}" | xargs -n1 | LC_ALL=C sort -u | xargs)
 echo "requested profiles: ${requested_profiles}"
 echo "flags for heap profile: ${mem_pprof_flags}"
 
@@ -291,7 +291,7 @@ for component in ${profile_components}; do
   esac
 
   if [[ "${component}" == "kubelet" ]]; then
-    for node in $(echo ${kubelet_addreses} | sed 's/[,;]/\n/g'); do
+    for node in ${kubelet_addresses//[,;]/' '}; do
       grab_profiles_from_component "${requested_profiles}" "${mem_pprof_flags}" "${binary}" "${tunnel_port}" "${path}/${node}" "${output_dir}/${component}" "${timestamp}"
     done    
   else 

hack/jenkins/benchmark-dockerized.sh

@@ -48,4 +48,6 @@ cd /go/src/k8s.io/kubernetes
 
 # Run the benchmark tests and pretty-print the results into a separate file.
 make test-integration WHAT="$*" KUBE_TEST_ARGS="-run='XXX' -bench=. -benchmem" \
-  | tee >(prettybench -no-passthrough > ${ARTIFACTS_DIR}/BenchmarkResults.txt)
+  | tee \
+   >(prettybench -no-passthrough > ${ARTIFACTS_DIR}/BenchmarkResults.txt) \
+   >(go run test/integration/benchmark/jsonify/main.go ${ARTIFACTS_DIR}/BenchmarkResults_benchmark_$(date -u +%Y-%m-%dT%H:%M:%SZ).json || cat > /dev/null)

hack/lib/etcd.sh

@@ -22,13 +22,22 @@ ETCD_PORT=${ETCD_PORT:-2379}
 
 kube::etcd::validate() {
   # validate if in path
-  which etcd >/dev/null || {
+  command -v etcd >/dev/null || {
     kube::log::usage "etcd must be in your PATH"
     exit 1
   }
 
   # validate etcd port is free
-  if netstat -nat | grep "[\.:]${ETCD_PORT:?} .*LISTEN" >/dev/null 2>&1; then
+  local port_check_command
+  if command -v ss &> /dev/null && ss -Version | grep 'iproute2' &> /dev/null; then
+    port_check_command="ss"
+  elif command -v netstat &>/dev/null; then
+    port_check_command="netstat"
+  else
+    kube::log::usage "unable to identify if etcd is bound to port ${ETCD_PORT}. unable to find ss or netstat utilities."
+    exit 1
+  fi
+  if ${port_check_command} -nat | grep "LISTEN" | grep "[\.:]${ETCD_PORT:?}" >/dev/null 2>&1; then
     kube::log::usage "unable to start etcd as port ${ETCD_PORT} is in use. please stop the process listening on this port and retry."
     kube::log::usage "`netstat -nat | grep "[\.:]${ETCD_PORT:?} .*LISTEN"`"
     exit 1

hack/lib/version.sh

@@ -96,6 +96,13 @@ kube::version::get_version_vars() {
           KUBE_GIT_MINOR+="+"
         fi
       fi
+
+      # If KUBE_GIT_VERSION is not a valid Semantic Version, then refuse to build.
+      if ! [[ "${KUBE_GIT_VERSION}" =~ ^v([0-9]+)\.([0-9]+)(\.[0-9]+)?(-[0-9A-Za-z.-]+)?(\+[0-9A-Za-z.-]+)?$ ]]; then
+          echo "KUBE_GIT_VERSION should be a valid Semantic Version"
+          echo "Please see more details here: https://semver.org"
+          exit 1
+      fi
     fi
   fi
 }

hack/local-up-cluster.sh

@@ -57,7 +57,7 @@ EVICTION_PRESSURE_TRANSITION_PERIOD=${EVICTION_PRESSURE_TRANSITION_PERIOD:-"1m"}
 # and we don't know the IP of the DNS pod to pass in as --cluster-dns.
 # To set this up by hand, set this flag and change DNS_SERVER_IP.
 # Note also that you need API_HOST (defined above) for correct DNS.
-KUBEPROXY_MODE=${KUBEPROXY_MODE:-""}
+KUBE_PROXY_MODE=${KUBE_PROXY_MODE:-""}
 ENABLE_CLUSTER_DNS=${KUBE_ENABLE_CLUSTER_DNS:-true}
 DNS_SERVER_IP=${KUBE_DNS_SERVER_IP:-10.0.0.10}
 DNS_DOMAIN=${KUBE_DNS_NAME:-"cluster.local"}
@@ -121,7 +121,7 @@ if [ "${CLOUD_PROVIDER}" == "openstack" ]; then
 fi
 
 # set feature gates if using ipvs mode
-if [ "${KUBEPROXY_MODE}" == "ipvs" ]; then
+if [ "${KUBE_PROXY_MODE}" == "ipvs" ]; then
     FEATURE_GATES="$FEATURE_GATES,SupportIPVSProxyMode=true"
 fi
 
@@ -811,9 +811,9 @@ clientConnection:
   kubeconfig: ${CERT_DIR}/kube-proxy.kubeconfig
 hostnameOverride: ${HOSTNAME_OVERRIDE}
 featureGates: ${FEATURE_GATES}
-mode: ${KUBEPROXY_MODE}
+mode: ${KUBE_PROXY_MODE}
 EOF
-    if [ "${KUBEPROXY_MODE}" == "ipvs" ]; then
+    if [ "${KUBE_PROXY_MODE}" == "ipvs" ]; then
 	# Load kernel modules required by IPVS proxier
         sudo modprobe -a ip_vs ip_vs_rr ip_vs_wrr ip_vs_sh nf_conntrack_ipv4
     fi

hack/update-codegen.sh

@@ -42,25 +42,25 @@ GROUP_VERSIONS=(${KUBE_AVAILABLE_GROUP_VERSIONS})
 GV_DIRS=()
 INTERNAL_DIRS=()
 for gv in "${GROUP_VERSIONS[@]}"; do
-	# add items, but strip off any leading apis/ you find to match command expectations
-	api_dir=$(kube::util::group-version-to-pkg-path "${gv}")
-	nopkg_dir=${api_dir#pkg/}
-	nopkg_dir=${nopkg_dir#vendor/k8s.io/api/}
-	pkg_dir=${nopkg_dir#apis/}
+  # add items, but strip off any leading apis/ you find to match command expectations
+  api_dir=$(kube::util::group-version-to-pkg-path "${gv}")
+  nopkg_dir=${api_dir#pkg/}
+  nopkg_dir=${nopkg_dir#vendor/k8s.io/api/}
+  pkg_dir=${nopkg_dir#apis/}
 
 
-	# skip groups that aren't being served, clients for these don't matter
+  # skip groups that aren't being served, clients for these don't matter
     if [[ " ${KUBE_NONSERVER_GROUP_VERSIONS} " == *" ${gv} "* ]]; then
       continue
     fi
 
-	GV_DIRS+=("${pkg_dir}")
+  GV_DIRS+=("${pkg_dir}")
 
-	# collect internal groups
-	int_group="${pkg_dir%/*}/"
-	if [[ "${pkg_dir}" = core/* ]]; then
-	  int_group="api/"
-	fi
+  # collect internal groups
+  int_group="${pkg_dir%/*}/"
+  if [[ "${pkg_dir}" = core/* ]]; then
+    int_group="api/"
+  fi
     if ! [[ " ${INTERNAL_DIRS[@]:-} " =~ " ${int_group} " ]]; then
       INTERNAL_DIRS+=("${int_group}")
     fi

hack/update-generated-device-plugin-dockerized.sh

@@ -19,7 +19,9 @@ set -o nounset
 set -o pipefail
 
 KUBE_ROOT="$(cd "$(dirname "${BASH_SOURCE}")/../" && pwd -P)"
-DEVICE_PLUGIN_ROOT="${KUBE_ROOT}/pkg/kubelet/apis/deviceplugin/v1alpha/"
+DEVICE_PLUGIN_ALPHA="${KUBE_ROOT}/pkg/kubelet/apis/deviceplugin/v1alpha/"
+DEVICE_PLUGIN_V1BETA1="${KUBE_ROOT}/pkg/kubelet/apis/deviceplugin/v1beta1/"
 
 source "${KUBE_ROOT}/hack/lib/protoc.sh"
-kube::protoc::generate_proto ${DEVICE_PLUGIN_ROOT}
+kube::protoc::generate_proto ${DEVICE_PLUGIN_ALPHA}
+kube::protoc::generate_proto ${DEVICE_PLUGIN_V1BETA1}

hack/verify-bazel.sh

@@ -21,9 +21,9 @@ export KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
 source "${KUBE_ROOT}/hack/lib/init.sh"
 
 if [[ ! -f "${KUBE_ROOT}/vendor/BUILD" ]]; then
-  echo "${KUBE_ROOT}/vendor/BUILD does not exist."
-  echo
-  echo "Run ./hack/update-bazel.sh"
+  echo "${KUBE_ROOT}/vendor/BUILD does not exist." >&2
+  echo >&2
+  echo "Run ./hack/update-bazel.sh" >&2
   exit 1
 fi
 
@@ -45,8 +45,8 @@ GOPATH="${_tmp_gopath}" ./hack/update-bazel.sh
 diff=$(diff -Naupr "${KUBE_ROOT}" "${_tmp_kuberoot}" || true)
 
 if [[ -n "${diff}" ]]; then
-  echo "${diff}"
-  echo
-  echo "Run ./hack/update-bazel.sh"
+  echo "${diff}" >&2
+  echo >&2
+  echo "Run ./hack/update-bazel.sh" >&2
   exit 1
 fi

hack/verify-boilerplate.sh

@@ -28,7 +28,7 @@ files_need_boilerplate=($(${boiler} "$@"))
 # Run boilerplate check
 if [[ ${#files_need_boilerplate[@]} -gt 0 ]]; then
   for file in "${files_need_boilerplate[@]}"; do
-    echo "Boilerplate header is wrong for: ${file}"
+    echo "Boilerplate header is wrong for: ${file}" >&2
   done
 
   exit 1

hack/verify-generated-device-plugin.sh

@@ -20,20 +20,26 @@ set -o pipefail
 
 KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
 ERROR="Device plugin api is out of date. Please run hack/update-generated-device-plugin.sh"
-DEVICE_PLUGIN_ROOT="${KUBE_ROOT}/pkg/kubelet/apis/deviceplugin/v1alpha/"
+DEVICE_PLUGIN_ALPHA="${KUBE_ROOT}/pkg/kubelet/apis/deviceplugin/v1alpha/"
+DEVICE_PLUGIN_V1BETA1="${KUBE_ROOT}/pkg/kubelet/apis/deviceplugin/v1beta1/"
 
 source "${KUBE_ROOT}/hack/lib/protoc.sh"
 kube::golang::setup_env
 
 function cleanup {
-	rm -rf ${DEVICE_PLUGIN_ROOT}/_tmp/
+	rm -rf ${DEVICE_PLUGIN_ALPHA}/_tmp/
+	rm -rf ${DEVICE_PLUGIN_V1BETA1}/_tmp/
 }
 
 trap cleanup EXIT
 
-mkdir -p ${DEVICE_PLUGIN_ROOT}/_tmp
-cp ${DEVICE_PLUGIN_ROOT}/api.pb.go ${DEVICE_PLUGIN_ROOT}/_tmp/
+mkdir -p ${DEVICE_PLUGIN_ALPHA}/_tmp
+cp ${DEVICE_PLUGIN_ALPHA}/api.pb.go ${DEVICE_PLUGIN_ALPHA}/_tmp/
+mkdir -p ${DEVICE_PLUGIN_V1BETA1}/_tmp
+cp ${DEVICE_PLUGIN_V1BETA1}/api.pb.go ${DEVICE_PLUGIN_V1BETA1}/_tmp/
 
 KUBE_VERBOSE=3 "${KUBE_ROOT}/hack/update-generated-device-plugin.sh"
-kube::protoc::diff "${DEVICE_PLUGIN_ROOT}/api.pb.go" "${DEVICE_PLUGIN_ROOT}/_tmp/api.pb.go" ${ERROR}
-echo "Generated device plugin api is up to date."
+kube::protoc::diff "${DEVICE_PLUGIN_ALPHA}/api.pb.go" "${DEVICE_PLUGIN_ALPHA}/_tmp/api.pb.go" ${ERROR}
+echo "Generated device plugin alpha api is up to date."
+kube::protoc::diff "${DEVICE_PLUGIN_V1BETA1}/api.pb.go" "${DEVICE_PLUGIN_V1BETA1}/_tmp/api.pb.go" ${ERROR}
+echo "Generated device plugin beta api is up to date."

hack/verify-godep-licenses.sh

@@ -49,7 +49,7 @@ LICENSE_ROOT="${_tmpdir}" "${KUBE_ROOT}/hack/update-godep-licenses.sh"
 
 # Compare Godep Licenses
 if ! _out="$(diff -Naupr ${KUBE_ROOT}/Godeps/LICENSES ${_tmpdir}/Godeps/LICENSES)"; then
-  echo "Your godep licenses file is out of date. Run hack/update-godep-licenses.sh and commit the results."
-  echo "${_out}"
+  echo "Your godep licenses file is out of date. Run hack/update-godep-licenses.sh and commit the results." >&2
+  echo "${_out}" >&2
   exit 1
 fi

hack/verify-godeps.sh

@@ -85,14 +85,14 @@ ret=0
 pushd "${KUBE_ROOT}" 2>&1 > /dev/null
   # Test for diffs
   if ! _out="$(diff -Naupr --ignore-matching-lines='^\s*\"GoVersion\":' --ignore-matching-line='^\s*\"GodepVersion\":' --ignore-matching-lines='^\s*\"Comment\":' Godeps/Godeps.json ${_kubetmp}/Godeps/Godeps.json)"; then
-    echo "Your Godeps.json is different:"
-    echo "${_out}"
-    echo "Godeps Verify failed."
+    echo "Your Godeps.json is different:" >&2
+    echo "${_out}" >&2
+    echo "Godeps Verify failed." >&2
     echo "${_out}" > godepdiff.patch
-    echo "If you're seeing this locally, run the below command to fix your Godeps.json:"
-    echo "patch -p0 < godepdiff.patch"
-    echo "(The above output can be saved as godepdiff.patch if you're not running this locally)"
-    echo "(The patch file should also be exported as a build artifact if run through CI)"
+    echo "If you're seeing this locally, run the below command to fix your Godeps.json:" >&2
+    echo "patch -p0 < godepdiff.patch" >&2
+    echo "(The above output can be saved as godepdiff.patch if you're not running this locally)" >&2
+    echo "(The patch file should also be exported as a build artifact if run through CI)" >&2
     KEEP_TMP=true
     if [[ -f godepdiff.patch && -d "${ARTIFACTS_DIR:-}" ]]; then
       echo "Copying patch to artifacts.."
@@ -102,14 +102,14 @@ pushd "${KUBE_ROOT}" 2>&1 > /dev/null
   fi
 
   if ! _out="$(diff -Naupr -x "BUILD" -x "OWNERS" -x "AUTHORS*" -x "CONTRIBUTORS*" vendor ${_kubetmp}/vendor)"; then
-    echo "Your vendored results are different:"
-    echo "${_out}"
-    echo "Godeps Verify failed."
+    echo "Your vendored results are different:" >&2
+    echo "${_out}" >&2
+    echo "Godeps Verify failed." >&2
     echo "${_out}" > vendordiff.patch
-    echo "If you're seeing this locally, run the below command to fix your directories:"
-    echo "patch -p0 < vendordiff.patch"
-    echo "(The above output can be saved as godepdiff.patch if you're not running this locally)"
-    echo "(The patch file should also be exported as a build artifact if run through CI)"
+    echo "If you're seeing this locally, run the below command to fix your directories:" >&2
+    echo "patch -p0 < vendordiff.patch" >&2
+    echo "(The above output can be saved as godepdiff.patch if you're not running this locally)" >&2
+    echo "(The patch file should also be exported as a build artifact if run through CI)" >&2
     KEEP_TMP=true
     if [[ -f vendordiff.patch && -d "${ARTIFACTS_DIR:-}" ]]; then
       echo "Copying patch to artifacts.."

hack/verify-gofmt.sh

@@ -54,6 +54,8 @@ find_files() {
 # have failed before getting to the "echo" in the block below.
 diff=$(find_files | xargs ${gofmt} -d -s 2>&1) || true
 if [[ -n "${diff}" ]]; then
-  echo "${diff}"
+  echo "${diff}" >&2
+  echo >&2
+  echo "Run ./hack/update-gofmt.sh" >&2
   exit 1
 fi

pkg/cloudprovider/providers/azure/BUILD

@@ -14,7 +14,9 @@ go_library(
         "azure_blobDiskController.go",
         "azure_cache.go",
         "azure_client.go",
-        "azure_controllerCommon.go",
+        "azure_controller_common.go",
+        "azure_controller_standard.go",
+        "azure_controller_vmss.go",
         "azure_fakes.go",
         "azure_file.go",
         "azure_instance_metadata.go",
@@ -44,6 +46,7 @@ go_library(
         "//vendor/github.com/Azure/azure-sdk-for-go/arm/disk:go_default_library",
         "//vendor/github.com/Azure/azure-sdk-for-go/arm/network:go_default_library",
         "//vendor/github.com/Azure/azure-sdk-for-go/arm/storage:go_default_library",
+        "//vendor/github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2017-12-01/compute:go_default_library",
         "//vendor/github.com/Azure/azure-sdk-for-go/storage:go_default_library",
         "//vendor/github.com/Azure/go-autorest/autorest:go_default_library",
         "//vendor/github.com/Azure/go-autorest/autorest/adal:go_default_library",
@@ -90,6 +93,7 @@ go_test(
         "//vendor/github.com/Azure/azure-sdk-for-go/arm/compute:go_default_library",
         "//vendor/github.com/Azure/azure-sdk-for-go/arm/network:go_default_library",
         "//vendor/github.com/Azure/azure-sdk-for-go/arm/storage:go_default_library",
+        "//vendor/github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2017-12-01/compute:go_default_library",
         "//vendor/github.com/Azure/go-autorest/autorest:go_default_library",
         "//vendor/github.com/Azure/go-autorest/autorest/to:go_default_library",
         "//vendor/github.com/stretchr/testify/assert:go_default_library",

pkg/cloudprovider/providers/azure/azure_backoff.go

@@ -17,13 +17,17 @@ limitations under the License.
 package azure
 
 import (
-	"k8s.io/apimachinery/pkg/util/wait"
+	"context"
+	"net/http"
 
 	"github.com/Azure/azure-sdk-for-go/arm/compute"
 	"github.com/Azure/azure-sdk-for-go/arm/network"
+	computepreview "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2017-12-01/compute"
 	"github.com/Azure/go-autorest/autorest"
 	"github.com/golang/glog"
+
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/wait"
 )
 
 // requestBackoff if backoff is disabled in cloud provider it
@@ -345,6 +349,15 @@ func (az *Cloud) CreateOrUpdateVMWithRetry(vmName string, newVM compute.VirtualM
 	})
 }
 
+// UpdateVmssVMWithRetry invokes az.VirtualMachineScaleSetVMsClient.Update with exponential backoff retry
+func (az *Cloud) UpdateVmssVMWithRetry(ctx context.Context, resourceGroupName string, VMScaleSetName string, instanceID string, parameters computepreview.VirtualMachineScaleSetVM) error {
+	return wait.ExponentialBackoff(az.requestBackoff(), func() (bool, error) {
+		resp, err := az.VirtualMachineScaleSetVMsClient.Update(ctx, resourceGroupName, VMScaleSetName, instanceID, parameters)
+		glog.V(10).Infof("VirtualMachinesClient.CreateOrUpdate(%s,%s): end", VMScaleSetName, instanceID)
+		return processHTTPRetryResponse(resp, err)
+	})
+}
+
 // A wait.ConditionFunc function to deal with common HTTP backoff response conditions
 func processRetryResponse(resp autorest.Response, err error) (bool, error) {
 	if isSuccessHTTPResponse(resp) {
@@ -380,3 +393,36 @@ func isSuccessHTTPResponse(resp autorest.Response) bool {
 	}
 	return false
 }
+
+func shouldRetryHTTPRequest(resp *http.Response, err error) bool {
+	if err != nil {
+		return true
+	}
+
+	if resp != nil {
+		// HTTP 4xx or 5xx suggests we should retry
+		if 399 < resp.StatusCode && resp.StatusCode < 600 {
+			return true
+		}
+	}
+
+	return false
+}
+
+func processHTTPRetryResponse(resp *http.Response, err error) (bool, error) {
+	if resp != nil {
+		// HTTP 2xx suggests a successful response
+		if 199 < resp.StatusCode && resp.StatusCode < 300 {
+			return true, nil
+		}
+	}
+
+	if shouldRetryHTTPRequest(resp, err) {
+		glog.Errorf("backoff: failure, will retry, HTTP response=%d, err=%v", resp.StatusCode, err)
+		// suppress the error object so that backoff process continues
+		return false, nil
+	}
+
+	// Fall-through: stop periodic backoff
+	return true, nil
+}

pkg/cloudprovider/providers/azure/azure_controller_common.go

+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package azure
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/Azure/azure-sdk-for-go/arm/compute"
+
+	"k8s.io/apimachinery/pkg/types"
+	kwait "k8s.io/apimachinery/pkg/util/wait"
+)
+
+const (
+	storageAccountNameTemplate = "pvc%s"
+
+	// for limits check https://docs.microsoft.com/en-us/azure/azure-subscription-service-limits#storage-limits
+	maxStorageAccounts                     = 100 // max # is 200 (250 with special request). this allows 100 for everything else including stand alone disks
+	maxDisksPerStorageAccounts             = 60
+	storageAccountUtilizationBeforeGrowing = 0.5
+
+	maxLUN               = 64 // max number of LUNs per VM
+	errLeaseFailed       = "AcquireDiskLeaseFailed"
+	errLeaseIDMissing    = "LeaseIdMissing"
+	errContainerNotFound = "ContainerNotFound"
+	errDiskBlobNotFound  = "DiskBlobNotFound"
+)
+
+var defaultBackOff = kwait.Backoff{
+	Steps:    20,
+	Duration: 2 * time.Second,
+	Factor:   1.5,
+	Jitter:   0.0,
+}
+
+type controllerCommon struct {
+	subscriptionID        string
+	location              string
+	storageEndpointSuffix string
+	resourceGroup         string
+	cloud                 *Cloud
+}
+
+// AttachDisk attaches a vhd to vm. The vhd must exist, can be identified by diskName, diskURI, and lun.
+func (c *controllerCommon) AttachDisk(isManagedDisk bool, diskName, diskURI string, nodeName types.NodeName, lun int32, cachingMode compute.CachingTypes) error {
+	// 1. vmType is standard, attach with availabilitySet.AttachDisk.
+	if c.cloud.VMType == vmTypeStandard {
+		return c.cloud.vmSet.AttachDisk(isManagedDisk, diskName, diskURI, nodeName, lun, cachingMode)
+	}
+
+	// 2. vmType is Virtual Machine Scale Set (vmss), convert vmSet to scaleSet.
+	ss, ok := c.cloud.vmSet.(*scaleSet)
+	if !ok {
+		return fmt.Errorf("error of converting vmSet (%q) to scaleSet with vmType %q", c.cloud.vmSet, c.cloud.VMType)
+	}
+
+	// 3. If the node is managed by availability set, then attach with availabilitySet.AttachDisk.
+	managedByAS, err := ss.isNodeManagedByAvailabilitySet(mapNodeNameToVMName(nodeName))
+	if err != nil {
+		return err
+	}
+	if managedByAS {
+		// vm is managed by availability set.
+		return ss.availabilitySet.AttachDisk(isManagedDisk, diskName, diskURI, nodeName, lun, cachingMode)
+	}
+
+	// 4. Node is managed by vmss, attach with scaleSet.AttachDisk.
+	return ss.AttachDisk(isManagedDisk, diskName, diskURI, nodeName, lun, cachingMode)
+}
+
+// DetachDiskByName detaches a vhd from host. The vhd can be identified by diskName or diskURI.
+func (c *controllerCommon) DetachDiskByName(diskName, diskURI string, nodeName types.NodeName) error {
+	// 1. vmType is standard, detach with availabilitySet.DetachDiskByName.
+	if c.cloud.VMType == vmTypeStandard {
+		return c.cloud.vmSet.DetachDiskByName(diskName, diskURI, nodeName)
+	}
+
+	// 2. vmType is Virtual Machine Scale Set (vmss), convert vmSet to scaleSet.
+	ss, ok := c.cloud.vmSet.(*scaleSet)
+	if !ok {
+		return fmt.Errorf("error of converting vmSet (%q) to scaleSet with vmType %q", c.cloud.vmSet, c.cloud.VMType)
+	}
+
+	// 3. If the node is managed by availability set, then detach with availabilitySet.DetachDiskByName.
+	managedByAS, err := ss.isNodeManagedByAvailabilitySet(mapNodeNameToVMName(nodeName))
+	if err != nil {
+		return err
+	}
+	if managedByAS {
+		// vm is managed by availability set.
+		return ss.availabilitySet.DetachDiskByName(diskName, diskURI, nodeName)
+	}
+
+	// 4. Node is managed by vmss, detach with scaleSet.DetachDiskByName.
+	return ss.DetachDiskByName(diskName, diskURI, nodeName)
+}
+
+// GetDiskLun finds the lun on the host that the vhd is attached to, given a vhd's diskName and diskURI.
+func (c *controllerCommon) GetDiskLun(diskName, diskURI string, nodeName types.NodeName) (int32, error) {
+	// 1. vmType is standard, get with availabilitySet.GetDiskLun.
+	if c.cloud.VMType == vmTypeStandard {
+		return c.cloud.vmSet.GetDiskLun(diskName, diskURI, nodeName)
+	}
+
+	// 2. vmType is Virtual Machine Scale Set (vmss), convert vmSet to scaleSet.
+	ss, ok := c.cloud.vmSet.(*scaleSet)
+	if !ok {
+		return -1, fmt.Errorf("error of converting vmSet (%q) to scaleSet with vmType %q", c.cloud.vmSet, c.cloud.VMType)
+	}
+
+	// 3. If the node is managed by availability set, then get with availabilitySet.GetDiskLun.
+	managedByAS, err := ss.isNodeManagedByAvailabilitySet(mapNodeNameToVMName(nodeName))
+	if err != nil {
+		return -1, err
+	}
+	if managedByAS {
+		// vm is managed by availability set.
+		return ss.availabilitySet.GetDiskLun(diskName, diskURI, nodeName)
+	}
+
+	// 4. Node is managed by vmss, get with scaleSet.GetDiskLun.
+	return ss.GetDiskLun(diskName, diskURI, nodeName)
+}
+
+// GetNextDiskLun searches all vhd attachment on the host and find unused lun. Return -1 if all luns are used.
+func (c *controllerCommon) GetNextDiskLun(nodeName types.NodeName) (int32, error) {
+	// 1. vmType is standard, get with availabilitySet.GetNextDiskLun.
+	if c.cloud.VMType == vmTypeStandard {
+		return c.cloud.vmSet.GetNextDiskLun(nodeName)
+	}
+
+	// 2. vmType is Virtual Machine Scale Set (vmss), convert vmSet to scaleSet.
+	ss, ok := c.cloud.vmSet.(*scaleSet)
+	if !ok {
+		return -1, fmt.Errorf("error of converting vmSet (%q) to scaleSet with vmType %q", c.cloud.vmSet, c.cloud.VMType)
+	}
+
+	// 3. If the node is managed by availability set, then get with availabilitySet.GetNextDiskLun.
+	managedByAS, err := ss.isNodeManagedByAvailabilitySet(mapNodeNameToVMName(nodeName))
+	if err != nil {
+		return -1, err
+	}
+	if managedByAS {
+		// vm is managed by availability set.
+		return ss.availabilitySet.GetNextDiskLun(nodeName)
+	}
+
+	// 4. Node is managed by vmss, get with scaleSet.GetNextDiskLun.
+	return ss.GetNextDiskLun(nodeName)
+}
+
+// DisksAreAttached checks if a list of volumes are attached to the node with the specified NodeName.
+func (c *controllerCommon) DisksAreAttached(diskNames []string, nodeName types.NodeName) (map[string]bool, error) {
+	// 1. vmType is standard, check with availabilitySet.DisksAreAttached.
+	if c.cloud.VMType == vmTypeStandard {
+		return c.cloud.vmSet.DisksAreAttached(diskNames, nodeName)
+	}
+
+	// 2. vmType is Virtual Machine Scale Set (vmss), convert vmSet to scaleSet.
+	ss, ok := c.cloud.vmSet.(*scaleSet)
+	if !ok {
+		return nil, fmt.Errorf("error of converting vmSet (%q) to scaleSet with vmType %q", c.cloud.vmSet, c.cloud.VMType)
+	}
+
+	// 3. If the node is managed by availability set, then check with availabilitySet.DisksAreAttached.
+	managedByAS, err := ss.isNodeManagedByAvailabilitySet(mapNodeNameToVMName(nodeName))
+	if err != nil {
+		return nil, err
+	}
+	if managedByAS {
+		// vm is managed by availability set.
+		return ss.availabilitySet.DisksAreAttached(diskNames, nodeName)
+	}
+
+	// 4. Node is managed by vmss, check with scaleSet.DisksAreAttached.
+	return ss.DisksAreAttached(diskNames, nodeName)
+}

pkg/cloudprovider/providers/azure/azure_controllerCommon.go → pkg/cloudprovider/providers/azure/azure_controller_standard.go

 /*
-Copyright 2017 The Kubernetes Authors.
+Copyright 2018 The Kubernetes Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -19,50 +19,18 @@ package azure
 import (
 	"fmt"
 	"strings"
-	"time"
-
-	"k8s.io/apimachinery/pkg/types"
-	kwait "k8s.io/apimachinery/pkg/util/wait"
-	"k8s.io/kubernetes/pkg/cloudprovider"
 
 	"github.com/Azure/azure-sdk-for-go/arm/compute"
 	"github.com/golang/glog"
-)
-
-const (
-	storageAccountNameTemplate = "pvc%s"
-
-	// for limits check https://docs.microsoft.com/en-us/azure/azure-subscription-service-limits#storage-limits
-	maxStorageAccounts                     = 100 // max # is 200 (250 with special request). this allows 100 for everything else including stand alone disks
-	maxDisksPerStorageAccounts             = 60
-	storageAccountUtilizationBeforeGrowing = 0.5
 
-	maxLUN               = 64 // max number of LUNs per VM
-	errLeaseFailed       = "AcquireDiskLeaseFailed"
-	errLeaseIDMissing    = "LeaseIdMissing"
-	errContainerNotFound = "ContainerNotFound"
-	errDiskBlobNotFound  = "DiskBlobNotFound"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/kubernetes/pkg/cloudprovider"
 )
 
-var defaultBackOff = kwait.Backoff{
-	Steps:    20,
-	Duration: 2 * time.Second,
-	Factor:   1.5,
-	Jitter:   0.0,
-}
-
-type controllerCommon struct {
-	subscriptionID        string
-	location              string
-	storageEndpointSuffix string
-	resourceGroup         string
-	cloud                 *Cloud
-}
-
 // AttachDisk attaches a vhd to vm
 // the vhd must exist, can be identified by diskName, diskURI, and lun.
-func (c *controllerCommon) AttachDisk(isManagedDisk bool, diskName, diskURI string, nodeName types.NodeName, lun int32, cachingMode compute.CachingTypes) error {
-	vm, err := c.cloud.getVirtualMachine(nodeName)
+func (as *availabilitySet) AttachDisk(isManagedDisk bool, diskName, diskURI string, nodeName types.NodeName, lun int32, cachingMode compute.CachingTypes) error {
+	vm, err := as.getVirtualMachine(nodeName)
 	if err != nil {
 		return err
 	}
@@ -101,16 +69,16 @@ func (c *controllerCommon) AttachDisk(isManagedDisk bool, diskName, diskURI stri
 		},
 	}
 	vmName := mapNodeNameToVMName(nodeName)
-	glog.V(2).Infof("azureDisk - update(%s): vm(%s) - attach disk", c.resourceGroup, vmName)
-	respChan, errChan := c.cloud.VirtualMachinesClient.CreateOrUpdate(c.resourceGroup, vmName, newVM, nil)
+	glog.V(2).Infof("azureDisk - update(%s): vm(%s) - attach disk", as.resourceGroup, vmName)
+	respChan, errChan := as.VirtualMachinesClient.CreateOrUpdate(as.resourceGroup, vmName, newVM, nil)
 	resp := <-respChan
 	err = <-errChan
-	if c.cloud.CloudProviderBackoff && shouldRetryAPIRequest(resp.Response, err) {
-		glog.V(2).Infof("azureDisk - update(%s) backing off: vm(%s)", c.resourceGroup, vmName)
-		retryErr := c.cloud.CreateOrUpdateVMWithRetry(vmName, newVM)
+	if as.CloudProviderBackoff && shouldRetryAPIRequest(resp.Response, err) {
+		glog.V(2).Infof("azureDisk - update(%s) backing off: vm(%s)", as.resourceGroup, vmName)
+		retryErr := as.CreateOrUpdateVMWithRetry(vmName, newVM)
 		if retryErr != nil {
 			err = retryErr
-			glog.V(2).Infof("azureDisk - update(%s) abort backoff: vm(%s)", c.resourceGroup, vmName)
+			glog.V(2).Infof("azureDisk - update(%s) abort backoff: vm(%s)", as.resourceGroup, vmName)
 		}
 	}
 	if err != nil {
@@ -119,20 +87,20 @@ func (c *controllerCommon) AttachDisk(isManagedDisk bool, diskName, diskURI stri
 		if strings.Contains(detail, errLeaseFailed) || strings.Contains(detail, errDiskBlobNotFound) {
 			// if lease cannot be acquired or disk not found, immediately detach the disk and return the original error
 			glog.Infof("azureDisk - err %s, try detach", detail)
-			c.cloud.DetachDiskByName(diskName, diskURI, nodeName)
+			as.DetachDiskByName(diskName, diskURI, nodeName)
 		}
 	} else {
 		glog.V(4).Info("azureDisk - azure attach succeeded")
 		// Invalidate the cache right after updating
-		c.cloud.vmCache.Delete(vmName)
+		as.cloud.vmCache.Delete(vmName)
 	}
 	return err
 }
 
 // DetachDiskByName detaches a vhd from host
 // the vhd can be identified by diskName or diskURI
-func (c *controllerCommon) DetachDiskByName(diskName, diskURI string, nodeName types.NodeName) error {
-	vm, err := c.cloud.getVirtualMachine(nodeName)
+func (as *availabilitySet) DetachDiskByName(diskName, diskURI string, nodeName types.NodeName) error {
+	vm, err := as.getVirtualMachine(nodeName)
 	if err != nil {
 		// if host doesn't exist, no need to detach
 		glog.Warningf("azureDisk - cannot find node %s, skip detaching disk %s", nodeName, diskName)
@@ -166,16 +134,16 @@ func (c *controllerCommon) DetachDiskByName(diskName, diskURI string, nodeName t
 		},
 	}
 	vmName := mapNodeNameToVMName(nodeName)
-	glog.V(2).Infof("azureDisk - update(%s): vm(%s) - detach disk", c.resourceGroup, vmName)
-	respChan, errChan := c.cloud.VirtualMachinesClient.CreateOrUpdate(c.resourceGroup, vmName, newVM, nil)
+	glog.V(2).Infof("azureDisk - update(%s): vm(%s) - detach disk", as.resourceGroup, vmName)
+	respChan, errChan := as.VirtualMachinesClient.CreateOrUpdate(as.resourceGroup, vmName, newVM, nil)
 	resp := <-respChan
 	err = <-errChan
-	if c.cloud.CloudProviderBackoff && shouldRetryAPIRequest(resp.Response, err) {
-		glog.V(2).Infof("azureDisk - update(%s) backing off: vm(%s)", c.resourceGroup, vmName)
-		retryErr := c.cloud.CreateOrUpdateVMWithRetry(vmName, newVM)
+	if as.CloudProviderBackoff && shouldRetryAPIRequest(resp.Response, err) {
+		glog.V(2).Infof("azureDisk - update(%s) backing off: vm(%s)", as.resourceGroup, vmName)
+		retryErr := as.CreateOrUpdateVMWithRetry(vmName, newVM)
 		if retryErr != nil {
 			err = retryErr
-			glog.V(2).Infof("azureDisk - update(%s) abort backoff: vm(%s)", c.cloud.ResourceGroup, vmName)
+			glog.V(2).Infof("azureDisk - update(%s) abort backoff: vm(%s)", as.ResourceGroup, vmName)
 		}
 	}
 	if err != nil {
@@ -183,14 +151,14 @@ func (c *controllerCommon) DetachDiskByName(diskName, diskURI string, nodeName t
 	} else {
 		glog.V(4).Info("azureDisk - azure disk detach succeeded")
 		// Invalidate the cache right after updating
-		c.cloud.vmCache.Delete(vmName)
+		as.cloud.vmCache.Delete(vmName)
 	}
 	return err
 }
 
 // GetDiskLun finds the lun on the host that the vhd is attached to, given a vhd's diskName and diskURI
-func (c *controllerCommon) GetDiskLun(diskName, diskURI string, nodeName types.NodeName) (int32, error) {
-	vm, err := c.cloud.getVirtualMachine(nodeName)
+func (as *availabilitySet) GetDiskLun(diskName, diskURI string, nodeName types.NodeName) (int32, error) {
+	vm, err := as.getVirtualMachine(nodeName)
 	if err != nil {
 		return -1, err
 	}
@@ -209,8 +177,8 @@ func (c *controllerCommon) GetDiskLun(diskName, diskURI string, nodeName types.N
 
 // GetNextDiskLun searches all vhd attachment on the host and find unused lun
 // return -1 if all luns are used
-func (c *controllerCommon) GetNextDiskLun(nodeName types.NodeName) (int32, error) {
-	vm, err := c.cloud.getVirtualMachine(nodeName)
+func (as *availabilitySet) GetNextDiskLun(nodeName types.NodeName) (int32, error) {
+	vm, err := as.getVirtualMachine(nodeName)
 	if err != nil {
 		return -1, err
 	}
@@ -230,12 +198,12 @@ func (c *controllerCommon) GetNextDiskLun(nodeName types.NodeName) (int32, error
 }
 
 // DisksAreAttached checks if a list of volumes are attached to the node with the specified NodeName
-func (c *controllerCommon) DisksAreAttached(diskNames []string, nodeName types.NodeName) (map[string]bool, error) {
+func (as *availabilitySet) DisksAreAttached(diskNames []string, nodeName types.NodeName) (map[string]bool, error) {
 	attached := make(map[string]bool)
 	for _, diskName := range diskNames {
 		attached[diskName] = false
 	}
-	vm, err := c.cloud.getVirtualMachine(nodeName)
+	vm, err := as.getVirtualMachine(nodeName)
 	if err == cloudprovider.InstanceNotFound {
 		// if host doesn't exist, no need to detach
 		glog.Warningf("azureDisk - Cannot find node %q, DisksAreAttached will assume disks %v are not attached to it.",

pkg/cloudprovider/providers/azure/azure_controller_vmss.go

+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package azure
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/Azure/azure-sdk-for-go/arm/compute"
+	computepreview "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2017-12-01/compute"
+	"github.com/golang/glog"
+
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/kubernetes/pkg/cloudprovider"
+)
+
+// AttachDisk attaches a vhd to vm
+// the vhd must exist, can be identified by diskName, diskURI, and lun.
+func (ss *scaleSet) AttachDisk(isManagedDisk bool, diskName, diskURI string, nodeName types.NodeName, lun int32, cachingMode compute.CachingTypes) error {
+	ssName, instanceID, vm, err := ss.getVmssVM(string(nodeName))
+	if err != nil {
+		return err
+	}
+
+	disks := *vm.StorageProfile.DataDisks
+	if isManagedDisk {
+		disks = append(disks,
+			computepreview.DataDisk{
+				Name:         &diskName,
+				Lun:          &lun,
+				Caching:      computepreview.CachingTypes(cachingMode),
+				CreateOption: "attach",
+				ManagedDisk: &computepreview.ManagedDiskParameters{
+					ID: &diskURI,
+				},
+			})
+	} else {
+		disks = append(disks,
+			computepreview.DataDisk{
+				Name: &diskName,
+				Vhd: &computepreview.VirtualHardDisk{
+					URI: &diskURI,
+				},
+				Lun:          &lun,
+				Caching:      computepreview.CachingTypes(cachingMode),
+				CreateOption: "attach",
+			})
+	}
+	vm.StorageProfile.DataDisks = &disks
+
+	ctx, cancel := getContextWithCancel()
+	defer cancel()
+	glog.V(2).Infof("azureDisk - update(%s): vm(%s) - attach disk", ss.resourceGroup, nodeName)
+	resp, err := ss.VirtualMachineScaleSetVMsClient.Update(ctx, ss.resourceGroup, ssName, instanceID, vm)
+	if ss.CloudProviderBackoff && shouldRetryHTTPRequest(resp, err) {
+		glog.V(2).Infof("azureDisk - update(%s) backing off: vm(%s)", ss.resourceGroup, nodeName)
+		retryErr := ss.UpdateVmssVMWithRetry(ctx, ss.resourceGroup, ssName, instanceID, vm)
+		if retryErr != nil {
+			err = retryErr
+			glog.V(2).Infof("azureDisk - update(%s) abort backoff: vm(%s)", ss.resourceGroup, nodeName)
+		}
+	}
+	if err != nil {
+		detail := err.Error()
+		if strings.Contains(detail, errLeaseFailed) || strings.Contains(detail, errDiskBlobNotFound) {
+			// if lease cannot be acquired or disk not found, immediately detach the disk and return the original error
+			glog.Infof("azureDisk - err %s, try detach", detail)
+			ss.DetachDiskByName(diskName, diskURI, nodeName)
+		}
+	} else {
+		glog.V(4).Info("azureDisk - azure attach succeeded")
+		// Invalidate the cache right after updating
+		ss.vmssVMCache.Delete(ss.makeVmssVMName(ssName, instanceID))
+	}
+	return err
+}
+
+// DetachDiskByName detaches a vhd from host
+// the vhd can be identified by diskName or diskURI
+func (ss *scaleSet) DetachDiskByName(diskName, diskURI string, nodeName types.NodeName) error {
+	ssName, instanceID, vm, err := ss.getVmssVM(string(nodeName))
+	if err != nil {
+		return err
+	}
+
+	disks := *vm.StorageProfile.DataDisks
+	bFoundDisk := false
+	for i, disk := range disks {
+		if disk.Lun != nil && (disk.Name != nil && diskName != "" && *disk.Name == diskName) ||
+			(disk.Vhd != nil && disk.Vhd.URI != nil && diskURI != "" && *disk.Vhd.URI == diskURI) ||
+			(disk.ManagedDisk != nil && diskURI != "" && *disk.ManagedDisk.ID == diskURI) {
+			// found the disk
+			glog.V(4).Infof("azureDisk - detach disk: name %q uri %q", diskName, diskURI)
+			disks = append(disks[:i], disks[i+1:]...)
+			bFoundDisk = true
+			break
+		}
+	}
+
+	if !bFoundDisk {
+		return fmt.Errorf("detach azure disk failure, disk %s not found, diskURI: %s", diskName, diskURI)
+	}
+
+	vm.StorageProfile.DataDisks = &disks
+	ctx, cancel := getContextWithCancel()
+	defer cancel()
+	glog.V(2).Infof("azureDisk - update(%s): vm(%s) - detach disk", ss.resourceGroup, nodeName)
+	resp, err := ss.VirtualMachineScaleSetVMsClient.Update(ctx, ss.resourceGroup, ssName, instanceID, vm)
+	if ss.CloudProviderBackoff && shouldRetryHTTPRequest(resp, err) {
+		glog.V(2).Infof("azureDisk - update(%s) backing off: vm(%s)", ss.resourceGroup, nodeName)
+		retryErr := ss.UpdateVmssVMWithRetry(ctx, ss.resourceGroup, ssName, instanceID, vm)
+		if retryErr != nil {
+			err = retryErr
+			glog.V(2).Infof("azureDisk - update(%s) abort backoff: vm(%s)", ss.resourceGroup, nodeName)
+		}
+	}
+	if err != nil {
+		glog.Errorf("azureDisk - azure disk detach %q from %s failed, err: %v", diskName, nodeName, err)
+	} else {
+		glog.V(4).Info("azureDisk - azure detach succeeded")
+		// Invalidate the cache right after updating
+		ss.vmssVMCache.Delete(ss.makeVmssVMName(ssName, instanceID))
+	}
+
+	return err
+}
+
+// GetDiskLun finds the lun on the host that the vhd is attached to, given a vhd's diskName and diskURI
+func (ss *scaleSet) GetDiskLun(diskName, diskURI string, nodeName types.NodeName) (int32, error) {
+	_, _, vm, err := ss.getVmssVM(string(nodeName))
+	if err != nil {
+		return -1, err
+	}
+
+	disks := *vm.StorageProfile.DataDisks
+	for _, disk := range disks {
+		if disk.Lun != nil && (disk.Name != nil && diskName != "" && *disk.Name == diskName) ||
+			(disk.Vhd != nil && disk.Vhd.URI != nil && diskURI != "" && *disk.Vhd.URI == diskURI) ||
+			(disk.ManagedDisk != nil && *disk.ManagedDisk.ID == diskURI) {
+			// found the disk
+			glog.V(4).Infof("azureDisk - find disk: lun %d name %q uri %q", *disk.Lun, diskName, diskURI)
+			return *disk.Lun, nil
+		}
+	}
+	return -1, fmt.Errorf("Cannot find Lun for disk %s", diskName)
+}
+
+// GetNextDiskLun searches all vhd attachment on the host and find unused lun
+// return -1 if all luns are used
+func (ss *scaleSet) GetNextDiskLun(nodeName types.NodeName) (int32, error) {
+	_, _, vm, err := ss.getVmssVM(string(nodeName))
+	if err != nil {
+		return -1, err
+	}
+
+	used := make([]bool, maxLUN)
+	disks := *vm.StorageProfile.DataDisks
+	for _, disk := range disks {
+		if disk.Lun != nil {
+			used[*disk.Lun] = true
+		}
+	}
+	for k, v := range used {
+		if !v {
+			return int32(k), nil
+		}
+	}
+	return -1, fmt.Errorf("All Luns are used")
+}
+
+// DisksAreAttached checks if a list of volumes are attached to the node with the specified NodeName
+func (ss *scaleSet) DisksAreAttached(diskNames []string, nodeName types.NodeName) (map[string]bool, error) {
+	attached := make(map[string]bool)
+	for _, diskName := range diskNames {
+		attached[diskName] = false
+	}
+
+	_, _, vm, err := ss.getVmssVM(string(nodeName))
+	if err != nil {
+		if err == cloudprovider.InstanceNotFound {
+			// if host doesn't exist, no need to detach
+			glog.Warningf("azureDisk - Cannot find node %q, DisksAreAttached will assume disks %v are not attached to it.",
+				nodeName, diskNames)
+			return attached, nil
+		}
+
+		return attached, err
+	}
+
+	disks := *vm.StorageProfile.DataDisks
+	for _, disk := range disks {
+		for _, diskName := range diskNames {
+			if disk.Name != nil && diskName != "" && *disk.Name == diskName {
+				attached[diskName] = true
+			}
+		}
+	}
+
+	return attached, nil
+}

pkg/cloudprovider/providers/azure/azure_fakes.go

@@ -17,6 +17,7 @@ limitations under the License.
 package azure
 
 import (
+	"context"
 	"fmt"
 	"math/rand"
 	"net/http"
@@ -28,6 +29,7 @@ import (
 	"github.com/Azure/azure-sdk-for-go/arm/disk"
 	"github.com/Azure/azure-sdk-for-go/arm/network"
 	"github.com/Azure/azure-sdk-for-go/arm/storage"
+	computepreview "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2017-12-01/compute"
 	"github.com/Azure/go-autorest/autorest"
 	"github.com/Azure/go-autorest/autorest/to"
 )
@@ -629,48 +631,39 @@ func getRandomIPPtr() *string {
 
 type fakeVirtualMachineScaleSetVMsClient struct {
 	mutex     *sync.Mutex
-	FakeStore map[string]map[string]compute.VirtualMachineScaleSetVM
+	FakeStore map[string]map[string]computepreview.VirtualMachineScaleSetVM
 }
 
 func newFakeVirtualMachineScaleSetVMsClient() *fakeVirtualMachineScaleSetVMsClient {
 	fVMC := &fakeVirtualMachineScaleSetVMsClient{}
-	fVMC.FakeStore = make(map[string]map[string]compute.VirtualMachineScaleSetVM)
+	fVMC.FakeStore = make(map[string]map[string]computepreview.VirtualMachineScaleSetVM)
 	fVMC.mutex = &sync.Mutex{}
 
 	return fVMC
 }
 
-func (fVMC *fakeVirtualMachineScaleSetVMsClient) setFakeStore(store map[string]map[string]compute.VirtualMachineScaleSetVM) {
+func (fVMC *fakeVirtualMachineScaleSetVMsClient) setFakeStore(store map[string]map[string]computepreview.VirtualMachineScaleSetVM) {
 	fVMC.mutex.Lock()
 	defer fVMC.mutex.Unlock()
 
 	fVMC.FakeStore = store
 }
 
-func (fVMC *fakeVirtualMachineScaleSetVMsClient) List(resourceGroupName string, virtualMachineScaleSetName string, filter string, selectParameter string, expand string) (result compute.VirtualMachineScaleSetVMListResult, err error) {
+func (fVMC *fakeVirtualMachineScaleSetVMsClient) List(ctx context.Context, resourceGroupName string, virtualMachineScaleSetName string, filter string, selectParameter string, expand string) (result []computepreview.VirtualMachineScaleSetVM, err error) {
 	fVMC.mutex.Lock()
 	defer fVMC.mutex.Unlock()
 
-	value := []compute.VirtualMachineScaleSetVM{}
+	result = []computepreview.VirtualMachineScaleSetVM{}
 	if _, ok := fVMC.FakeStore[resourceGroupName]; ok {
 		for _, v := range fVMC.FakeStore[resourceGroupName] {
-			value = append(value, v)
+			result = append(result, v)
 		}
 	}
 
-	result.Response.Response = &http.Response{
-		StatusCode: http.StatusOK,
-	}
-	result.NextLink = nil
-	result.Value = &value
-	return result, nil
-}
-
-func (fVMC *fakeVirtualMachineScaleSetVMsClient) ListNextResults(resourceGroupName string, lastResults compute.VirtualMachineScaleSetVMListResult) (result compute.VirtualMachineScaleSetVMListResult, err error) {
 	return result, nil
 }
 
-func (fVMC *fakeVirtualMachineScaleSetVMsClient) Get(resourceGroupName string, VMScaleSetName string, instanceID string) (result compute.VirtualMachineScaleSetVM, err error) {
+func (fVMC *fakeVirtualMachineScaleSetVMsClient) Get(ctx context.Context, resourceGroupName string, VMScaleSetName string, instanceID string) (result computepreview.VirtualMachineScaleSetVM, err error) {
 	fVMC.mutex.Lock()
 	defer fVMC.mutex.Unlock()
 
@@ -687,8 +680,8 @@ func (fVMC *fakeVirtualMachineScaleSetVMsClient) Get(resourceGroupName string, V
 	}
 }
 
-func (fVMC *fakeVirtualMachineScaleSetVMsClient) GetInstanceView(resourceGroupName string, VMScaleSetName string, instanceID string) (result compute.VirtualMachineScaleSetVMInstanceView, err error) {
-	_, err = fVMC.Get(resourceGroupName, VMScaleSetName, instanceID)
+func (fVMC *fakeVirtualMachineScaleSetVMsClient) GetInstanceView(ctx context.Context, resourceGroupName string, VMScaleSetName string, instanceID string) (result computepreview.VirtualMachineScaleSetVMInstanceView, err error) {
+	_, err = fVMC.Get(ctx, resourceGroupName, VMScaleSetName, instanceID)
 	if err != nil {
 		return result, err
 	}
@@ -696,54 +689,53 @@ func (fVMC *fakeVirtualMachineScaleSetVMsClient) GetInstanceView(resourceGroupNa
 	return result, nil
 }
 
+func (fVMC *fakeVirtualMachineScaleSetVMsClient) Update(ctx context.Context, resourceGroupName string, VMScaleSetName string, instanceID string, parameters computepreview.VirtualMachineScaleSetVM) (resp *http.Response, err error) {
+	fVMC.mutex.Lock()
+	defer fVMC.mutex.Unlock()
+
+	vmKey := fmt.Sprintf("%s_%s", VMScaleSetName, instanceID)
+	if scaleSetMap, ok := fVMC.FakeStore[resourceGroupName]; ok {
+		if _, ok := scaleSetMap[vmKey]; ok {
+			scaleSetMap[vmKey] = parameters
+		}
+	}
+
+	return nil, nil
+}
+
 type fakeVirtualMachineScaleSetsClient struct {
 	mutex     *sync.Mutex
-	FakeStore map[string]map[string]compute.VirtualMachineScaleSet
+	FakeStore map[string]map[string]computepreview.VirtualMachineScaleSet
 }
 
 func newFakeVirtualMachineScaleSetsClient() *fakeVirtualMachineScaleSetsClient {
 	fVMSSC := &fakeVirtualMachineScaleSetsClient{}
-	fVMSSC.FakeStore = make(map[string]map[string]compute.VirtualMachineScaleSet)
+	fVMSSC.FakeStore = make(map[string]map[string]computepreview.VirtualMachineScaleSet)
 	fVMSSC.mutex = &sync.Mutex{}
 
 	return fVMSSC
 }
 
-func (fVMSSC *fakeVirtualMachineScaleSetsClient) setFakeStore(store map[string]map[string]compute.VirtualMachineScaleSet) {
+func (fVMSSC *fakeVirtualMachineScaleSetsClient) setFakeStore(store map[string]map[string]computepreview.VirtualMachineScaleSet) {
 	fVMSSC.mutex.Lock()
 	defer fVMSSC.mutex.Unlock()
 
 	fVMSSC.FakeStore = store
 }
 
-func (fVMSSC *fakeVirtualMachineScaleSetsClient) CreateOrUpdate(resourceGroupName string, VMScaleSetName string, parameters compute.VirtualMachineScaleSet, cancel <-chan struct{}) (<-chan compute.VirtualMachineScaleSet, <-chan error) {
+func (fVMSSC *fakeVirtualMachineScaleSetsClient) CreateOrUpdate(ctx context.Context, resourceGroupName string, VMScaleSetName string, parameters computepreview.VirtualMachineScaleSet) (resp *http.Response, err error) {
 	fVMSSC.mutex.Lock()
 	defer fVMSSC.mutex.Unlock()
 
-	resultChan := make(chan compute.VirtualMachineScaleSet, 1)
-	errChan := make(chan error, 1)
-	var result compute.VirtualMachineScaleSet
-	var err error
-	defer func() {
-		resultChan <- result
-		errChan <- err
-		close(resultChan)
-		close(errChan)
-	}()
-
 	if _, ok := fVMSSC.FakeStore[resourceGroupName]; !ok {
-		fVMSSC.FakeStore[resourceGroupName] = make(map[string]compute.VirtualMachineScaleSet)
+		fVMSSC.FakeStore[resourceGroupName] = make(map[string]computepreview.VirtualMachineScaleSet)
 	}
 	fVMSSC.FakeStore[resourceGroupName][VMScaleSetName] = parameters
-	result = fVMSSC.FakeStore[resourceGroupName][VMScaleSetName]
-	result.Response.Response = &http.Response{
-		StatusCode: http.StatusOK,
-	}
-	err = nil
-	return resultChan, errChan
+
+	return nil, nil
 }
 
-func (fVMSSC *fakeVirtualMachineScaleSetsClient) Get(resourceGroupName string, VMScaleSetName string) (result compute.VirtualMachineScaleSet, err error) {
+func (fVMSSC *fakeVirtualMachineScaleSetsClient) Get(ctx context.Context, resourceGroupName string, VMScaleSetName string) (result computepreview.VirtualMachineScaleSet, err error) {
 	fVMSSC.mutex.Lock()
 	defer fVMSSC.mutex.Unlock()
 
@@ -759,45 +751,22 @@ func (fVMSSC *fakeVirtualMachineScaleSetsClient) Get(resourceGroupName string, V
 	}
 }
 
-func (fVMSSC *fakeVirtualMachineScaleSetsClient) List(resourceGroupName string) (result compute.VirtualMachineScaleSetListResult, err error) {
+func (fVMSSC *fakeVirtualMachineScaleSetsClient) List(ctx context.Context, resourceGroupName string) (result []computepreview.VirtualMachineScaleSet, err error) {
 	fVMSSC.mutex.Lock()
 	defer fVMSSC.mutex.Unlock()
 
-	value := []compute.VirtualMachineScaleSet{}
+	result = []computepreview.VirtualMachineScaleSet{}
 	if _, ok := fVMSSC.FakeStore[resourceGroupName]; ok {
 		for _, v := range fVMSSC.FakeStore[resourceGroupName] {
-			value = append(value, v)
+			result = append(result, v)
 		}
 	}
-	result.Response.Response = &http.Response{
-		StatusCode: http.StatusOK,
-	}
-	result.NextLink = nil
-	result.Value = &value
-	return result, nil
-}
 
-func (fVMSSC *fakeVirtualMachineScaleSetsClient) ListNextResults(resourceGroupName string, lastResults compute.VirtualMachineScaleSetListResult) (result compute.VirtualMachineScaleSetListResult, err error) {
 	return result, nil
 }
 
-func (fVMSSC *fakeVirtualMachineScaleSetsClient) UpdateInstances(resourceGroupName string, VMScaleSetName string, VMInstanceIDs compute.VirtualMachineScaleSetVMInstanceRequiredIDs, cancel <-chan struct{}) (<-chan compute.OperationStatusResponse, <-chan error) {
-	resultChan := make(chan compute.OperationStatusResponse, 1)
-	errChan := make(chan error, 1)
-	var result compute.OperationStatusResponse
-	var err error
-	defer func() {
-		resultChan <- result
-		errChan <- err
-		close(resultChan)
-		close(errChan)
-	}()
-
-	result.Response.Response = &http.Response{
-		StatusCode: http.StatusOK,
-	}
-	err = nil
-	return resultChan, errChan
+func (fVMSSC *fakeVirtualMachineScaleSetsClient) UpdateInstances(ctx context.Context, resourceGroupName string, VMScaleSetName string, VMInstanceIDs computepreview.VirtualMachineScaleSetVMInstanceRequiredIDs) (resp *http.Response, err error) {
+	return nil, nil
 }
 
 type fakeRoutesClient struct {

pkg/cloudprovider/providers/azure/azure_vmsets.go

@@ -17,6 +17,7 @@ limitations under the License.
 package azure
 
 import (
+	"github.com/Azure/azure-sdk-for-go/arm/compute"
 	"github.com/Azure/azure-sdk-for-go/arm/network"
 
 	"k8s.io/api/core/v1"
@@ -56,4 +57,15 @@ type VMSet interface {
 	EnsureHostsInPool(serviceName string, nodes []*v1.Node, backendPoolID string, vmSetName string) error
 	// EnsureBackendPoolDeleted ensures the loadBalancer backendAddressPools deleted from the specified vmSet.
 	EnsureBackendPoolDeleted(poolID, vmSetName string) error
+
+	// AttachDisk attaches a vhd to vm. The vhd must exist, can be identified by diskName, diskURI, and lun.
+	AttachDisk(isManagedDisk bool, diskName, diskURI string, nodeName types.NodeName, lun int32, cachingMode compute.CachingTypes) error
+	// DetachDiskByName detaches a vhd from host. The vhd can be identified by diskName or diskURI.
+	DetachDiskByName(diskName, diskURI string, nodeName types.NodeName) error
+	// GetDiskLun finds the lun on the host that the vhd is attached to, given a vhd's diskName and diskURI.
+	GetDiskLun(diskName, diskURI string, nodeName types.NodeName) (int32, error)
+	// GetNextDiskLun searches all vhd attachment on the host and find unused lun. Return -1 if all luns are used.
+	GetNextDiskLun(nodeName types.NodeName) (int32, error)
+	// DisksAreAttached checks if a list of volumes are attached to the node with the specified NodeName.
+	DisksAreAttached(diskNames []string, nodeName types.NodeName) (map[string]bool, error)
 }

pkg/cloudprovider/providers/azure/azure_vmss_cache.go

@@ -58,7 +58,9 @@ func (ss *scaleSet) extractVmssVMName(name string) (string, string, error) {
 
 func (ss *scaleSet) newVmssCache() (*timedCache, error) {
 	getter := func(key string) (interface{}, error) {
-		result, err := ss.VirtualMachineScaleSetsClient.Get(ss.ResourceGroup, key)
+		ctx, cancel := getContextWithCancel()
+		defer cancel()
+		result, err := ss.VirtualMachineScaleSetsClient.Get(ctx, ss.ResourceGroup, key)
 		exists, realErr := checkResourceExistsFromError(err)
 		if realErr != nil {
 			return nil, realErr
@@ -76,14 +78,14 @@ func (ss *scaleSet) newVmssCache() (*timedCache, error) {
 
 func (ss *scaleSet) newNodeNameToScaleSetMappingCache() (*timedCache, error) {
 	getter := func(key string) (interface{}, error) {
-		scaleSetNames, err := ss.listScaleSetsWithRetry()
+		scaleSetNames, err := ss.listScaleSets()
 		if err != nil {
 			return nil, err
 		}
 
 		localCache := make(nodeNameToScaleSetMapping)
 		for _, ssName := range scaleSetNames {
-			vms, err := ss.listScaleSetVMsWithRetry(ssName)
+			vms, err := ss.listScaleSetVMs(ssName)
 			if err != nil {
 				return nil, err
 			}
@@ -136,7 +138,9 @@ func (ss *scaleSet) newVmssVMCache() (*timedCache, error) {
 			return nil, nil
 		}
 
-		result, err := ss.VirtualMachineScaleSetVMsClient.Get(ss.ResourceGroup, ssName, instanceID)
+		ctx, cancel := getContextWithCancel()
+		defer cancel()
+		result, err := ss.VirtualMachineScaleSetVMsClient.Get(ctx, ss.ResourceGroup, ssName, instanceID)
 		exists, realErr := checkResourceExistsFromError(err)
 		if realErr != nil {
 			return nil, realErr

pkg/cloudprovider/providers/azure/azure_vmss_test.go

@@ -20,7 +20,7 @@ import (
 	"fmt"
 	"testing"
 
-	"github.com/Azure/azure-sdk-for-go/arm/compute"
+	computepreview "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2017-12-01/compute"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -37,8 +37,8 @@ func newTestScaleSet(scaleSetName string, vmList []string) (*scaleSet, error) {
 
 func setTestVirtualMachineCloud(ss *Cloud, scaleSetName string, vmList []string) {
 	virtualMachineScaleSetsClient := newFakeVirtualMachineScaleSetsClient()
-	scaleSets := make(map[string]map[string]compute.VirtualMachineScaleSet)
-	scaleSets["rg"] = map[string]compute.VirtualMachineScaleSet{
+	scaleSets := make(map[string]map[string]computepreview.VirtualMachineScaleSet)
+	scaleSets["rg"] = map[string]computepreview.VirtualMachineScaleSet{
 		scaleSetName: {
 			Name: &scaleSetName,
 		},
@@ -46,24 +46,24 @@ func setTestVirtualMachineCloud(ss *Cloud, scaleSetName string, vmList []string)
 	virtualMachineScaleSetsClient.setFakeStore(scaleSets)
 
 	virtualMachineScaleSetVMsClient := newFakeVirtualMachineScaleSetVMsClient()
-	ssVMs := make(map[string]map[string]compute.VirtualMachineScaleSetVM)
-	ssVMs["rg"] = make(map[string]compute.VirtualMachineScaleSetVM)
+	ssVMs := make(map[string]map[string]computepreview.VirtualMachineScaleSetVM)
+	ssVMs["rg"] = make(map[string]computepreview.VirtualMachineScaleSetVM)
 	for i := range vmList {
 		ID := fmt.Sprintf("/subscriptions/script/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/%s/virtualMachines/%d", scaleSetName, i)
 		nodeName := vmList[i]
 		instanceID := fmt.Sprintf("%d", i)
 		vmName := fmt.Sprintf("%s_%s", scaleSetName, instanceID)
-		networkInterfaces := []compute.NetworkInterfaceReference{
+		networkInterfaces := []computepreview.NetworkInterfaceReference{
 			{
 				ID: &nodeName,
 			},
 		}
-		ssVMs["rg"][vmName] = compute.VirtualMachineScaleSetVM{
-			VirtualMachineScaleSetVMProperties: &compute.VirtualMachineScaleSetVMProperties{
-				OsProfile: &compute.OSProfile{
+		ssVMs["rg"][vmName] = computepreview.VirtualMachineScaleSetVM{
+			VirtualMachineScaleSetVMProperties: &computepreview.VirtualMachineScaleSetVMProperties{
+				OsProfile: &computepreview.OSProfile{
 					ComputerName: &nodeName,
 				},
-				NetworkProfile: &compute.NetworkProfile{
+				NetworkProfile: &computepreview.NetworkProfile{
 					NetworkInterfaces: &networkInterfaces,
 				},
 			},

pkg/cloudprovider/providers/openstack/openstack.go

@@ -23,6 +23,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"net"
 	"net/http"
 	"os"
 	"regexp"
@@ -511,7 +512,7 @@ func getAddressesByName(client *gophercloud.ServiceClient, name types.NodeName)
 	return nodeAddresses(srv)
 }
 
-func getAddressByName(client *gophercloud.ServiceClient, name types.NodeName) (string, error) {
+func getAddressByName(client *gophercloud.ServiceClient, name types.NodeName, needIPv6 bool) (string, error) {
 	addrs, err := getAddressesByName(client, name)
 	if err != nil {
 		return "", err
@@ -520,12 +521,20 @@ func getAddressByName(client *gophercloud.ServiceClient, name types.NodeName) (s
 	}
 
 	for _, addr := range addrs {
-		if addr.Type == v1.NodeInternalIP {
+		isIPv6 := net.ParseIP(addr.Address).To4() == nil
+		if (addr.Type == v1.NodeInternalIP) && (isIPv6 == needIPv6) {
 			return addr.Address, nil
 		}
 	}
 
-	return addrs[0].Address, nil
+	for _, addr := range addrs {
+		isIPv6 := net.ParseIP(addr.Address).To4() == nil
+		if (addr.Type == v1.NodeExternalIP) && (isIPv6 == needIPv6) {
+			return addr.Address, nil
+		}
+	}
+	// It should never return an address from a different IP Address family than the one needed
+	return "", ErrNoAddressFound
 }
 
 // getAttachedInterfacesByID returns the node interfaces of the specified instance.

pkg/cloudprovider/providers/openstack/openstack_routes.go

@@ -19,6 +19,7 @@ package openstack
 import (
 	"context"
 	"errors"
+	"net"
 
 	"github.com/gophercloud/gophercloud"
 	"github.com/gophercloud/gophercloud/openstack/compute/v2/servers"
@@ -146,7 +147,10 @@ func (r *Routes) CreateRoute(ctx context.Context, clusterName string, nameHint s
 
 	onFailure := newCaller()
 
-	addr, err := getAddressByName(r.compute, route.TargetNode)
+	ip, _, _ := net.ParseCIDR(route.DestinationCIDR)
+	isCIDRv6 := ip.To4() == nil
+	addr, err := getAddressByName(r.compute, route.TargetNode, isCIDRv6)
+
 	if err != nil {
 		return err
 	}
@@ -219,7 +223,10 @@ func (r *Routes) DeleteRoute(ctx context.Context, clusterName string, route *clo
 
 	onFailure := newCaller()
 
-	addr, err := getAddressByName(r.compute, route.TargetNode)
+	ip, _, _ := net.ParseCIDR(route.DestinationCIDR)
+	isCIDRv6 := ip.To4() == nil
+	addr, err := getAddressByName(r.compute, route.TargetNode, isCIDRv6)
+
 	if err != nil {
 		return err
 	}

pkg/cloudprovider/providers/vsphere/BUILD

@@ -27,7 +27,6 @@ go_library(
         "//vendor/gopkg.in/gcfg.v1:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
         "//vendor/k8s.io/client-go/informers:go_default_library",
         "//vendor/k8s.io/client-go/tools/cache:go_default_library",
     ],

pkg/cloudprovider/providers/vsphere/vsphere.go

@@ -35,7 +35,6 @@ import (
 	"golang.org/x/net/context"
 	"k8s.io/api/core/v1"
 	k8stypes "k8s.io/apimachinery/pkg/types"
-	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/informers"
 	"k8s.io/client-go/tools/cache"
 	v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper"
@@ -210,21 +209,23 @@ func init() {
 
 // Initialize passes a Kubernetes clientBuilder interface to the cloud provider
 func (vs *VSphere) Initialize(clientBuilder controller.ControllerClientBuilder) {
+}
+
+// Initialize Node Informers
+func (vs *VSphere) SetInformers(informerFactory informers.SharedInformerFactory) {
 	if vs.cfg == nil {
 		return
 	}
 
 	// Only on controller node it is required to register listeners.
 	// Register callbacks for node updates
-	client := clientBuilder.ClientOrDie("vsphere-cloud-provider")
-	factory := informers.NewSharedInformerFactory(client, 5*time.Minute)
-	nodeInformer := factory.Core().V1().Nodes()
-	nodeInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
+	glog.V(4).Infof("Setting up node informers for vSphere Cloud Provider")
+	nodeInformer := informerFactory.Core().V1().Nodes().Informer()
+	nodeInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
 		AddFunc:    vs.NodeAdded,
 		DeleteFunc: vs.NodeDeleted,
 	})
-	go nodeInformer.Informer().Run(wait.NeverStop)
-	glog.V(4).Infof("vSphere cloud provider initialized")
+	glog.V(4).Infof("Node informers in vSphere cloud provider initialized")
 }
 
 // Creates new worker node interface and returns

pkg/controller/bootstrap/jws.go

@@ -33,7 +33,7 @@ func computeDetachedSig(content, tokenID, tokenSecret string) (string, error) {
 	}
 
 	opts := &jose.SignerOptions{
-		// Since this is a symetric key, go-jose doesn't automatically include
+		// Since this is a symmetric key, go-jose doesn't automatically include
 		// the KeyID as part of the protected header. We have to pass it here
 		// explicitly.
 		ExtraHeaders: map[jose.HeaderKey]interface{}{

pkg/credentialprovider/azure/azure_credentials.go

@@ -173,7 +173,7 @@ func getLoginServer(registry containerregistry.Registry) string {
 }
 
 func getACRDockerEntryFromARMToken(a *acrProvider, loginServer string) (*credentialprovider.DockerConfigEntry, error) {
-	armAccessToken := a.servicePrincipalToken.AccessToken
+	armAccessToken := a.servicePrincipalToken.OAuthToken()
 
 	glog.V(4).Infof("discovering auth redirects for: %s", loginServer)
 	directive, err := receiveChallengeFromLoginServer(loginServer)

pkg/kubeapiserver/authenticator/BUILD

@@ -24,7 +24,6 @@ go_library(
         "//vendor/k8s.io/apiserver/pkg/authentication/token/cache:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/authentication/token/tokenfile:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/authentication/token/union:go_default_library",
-        "//vendor/k8s.io/apiserver/plugin/pkg/authenticator/password/keystone:go_default_library",
         "//vendor/k8s.io/apiserver/plugin/pkg/authenticator/password/passwordfile:go_default_library",
         "//vendor/k8s.io/apiserver/plugin/pkg/authenticator/request/basicauth:go_default_library",
         "//vendor/k8s.io/apiserver/plugin/pkg/authenticator/token/oidc:go_default_library",

pkg/kubeapiserver/authenticator/config.go

@@ -33,7 +33,6 @@ import (
 	tokencache "k8s.io/apiserver/pkg/authentication/token/cache"
 	"k8s.io/apiserver/pkg/authentication/token/tokenfile"
 	tokenunion "k8s.io/apiserver/pkg/authentication/token/union"
-	"k8s.io/apiserver/plugin/pkg/authenticator/password/keystone"
 	"k8s.io/apiserver/plugin/pkg/authenticator/password/passwordfile"
 	"k8s.io/apiserver/plugin/pkg/authenticator/request/basicauth"
 	"k8s.io/apiserver/plugin/pkg/authenticator/token/oidc"
@@ -61,8 +60,6 @@ type AuthenticatorConfig struct {
 	OIDCGroupsPrefix            string
 	ServiceAccountKeyFiles      []string
 	ServiceAccountLookup        bool
-	KeystoneURL                 string
-	KeystoneCAFile              string
 	WebhookTokenAuthnConfigFile string
 	WebhookTokenAuthnCacheTTL   time.Duration
 
@@ -108,14 +105,6 @@ func (config AuthenticatorConfig) New() (authenticator.Request, *spec.SecurityDe
 		authenticators = append(authenticators, basicAuth)
 		hasBasicAuth = true
 	}
-	if len(config.KeystoneURL) > 0 {
-		keystoneAuth, err := newAuthenticatorFromKeystoneURL(config.KeystoneURL, config.KeystoneCAFile)
-		if err != nil {
-			return nil, nil, err
-		}
-		authenticators = append(authenticators, keystoneAuth)
-		hasBasicAuth = true
-	}
 
 	// X509 methods
 	if len(config.ClientCAFile) > 0 {
@@ -306,16 +295,6 @@ func newAuthenticatorFromClientCAFile(clientCAFile string) (authenticator.Reques
 	return x509.New(opts, x509.CommonNameUserConversion), nil
 }
 
-// newAuthenticatorFromKeystoneURL returns an authenticator.Request or an error
-func newAuthenticatorFromKeystoneURL(keystoneURL string, keystoneCAFile string) (authenticator.Request, error) {
-	keystoneAuthenticator, err := keystone.NewKeystoneAuthenticator(keystoneURL, keystoneCAFile)
-	if err != nil {
-		return nil, err
-	}
-
-	return basicauth.New(keystoneAuthenticator), nil
-}
-
 func newWebhookTokenAuthenticator(webhookConfigFile string, ttl time.Duration) (authenticator.Token, error) {
 	webhookTokenAuthenticator, err := webhook.New(webhookConfigFile, ttl)
 	if err != nil {

pkg/kubeapiserver/options/authentication.go

@@ -34,7 +34,6 @@ type BuiltInAuthenticationOptions struct {
 	Anonymous       *AnonymousAuthenticationOptions
 	BootstrapToken  *BootstrapTokenAuthenticationOptions
 	ClientCert      *genericoptions.ClientCertAuthenticationOptions
-	Keystone        *KeystoneAuthenticationOptions
 	OIDC            *OIDCAuthenticationOptions
 	PasswordFile    *PasswordFileAuthenticationOptions
 	RequestHeader   *genericoptions.RequestHeaderAuthenticationOptions
@@ -54,11 +53,6 @@ type BootstrapTokenAuthenticationOptions struct {
 	Enable bool
 }
 
-type KeystoneAuthenticationOptions struct {
-	URL    string
-	CAFile string
-}
-
 type OIDCAuthenticationOptions struct {
 	CAFile         string
 	ClientID       string
@@ -99,7 +93,6 @@ func (s *BuiltInAuthenticationOptions) WithAll() *BuiltInAuthenticationOptions {
 		WithAnonymous().
 		WithBootstrapToken().
 		WithClientCert().
-		WithKeystone().
 		WithOIDC().
 		WithPasswordFile().
 		WithRequestHeader().
@@ -123,11 +116,6 @@ func (s *BuiltInAuthenticationOptions) WithClientCert() *BuiltInAuthenticationOp
 	return s
 }
 
-func (s *BuiltInAuthenticationOptions) WithKeystone() *BuiltInAuthenticationOptions {
-	s.Keystone = &KeystoneAuthenticationOptions{}
-	return s
-}
-
 func (s *BuiltInAuthenticationOptions) WithOIDC() *BuiltInAuthenticationOptions {
 	s.OIDC = &OIDCAuthenticationOptions{}
 	return s
@@ -189,15 +177,6 @@ func (s *BuiltInAuthenticationOptions) AddFlags(fs *pflag.FlagSet) {
 		s.ClientCert.AddFlags(fs)
 	}
 
-	if s.Keystone != nil {
-		fs.StringVar(&s.Keystone.URL, "experimental-keystone-url", s.Keystone.URL,
-			"If passed, activates the keystone authentication plugin.")
-
-		fs.StringVar(&s.Keystone.CAFile, "experimental-keystone-ca-file", s.Keystone.CAFile, ""+
-			"If set, the Keystone server's certificate will be verified by one of the authorities "+
-			"in the experimental-keystone-ca-file, otherwise the host's root CA set will be used.")
-	}
-
 	if s.OIDC != nil {
 		fs.StringVar(&s.OIDC.IssuerURL, "oidc-issuer-url", s.OIDC.IssuerURL, ""+
 			"The URL of the OpenID issuer, only HTTPS scheme will be accepted. "+
@@ -285,11 +264,6 @@ func (s *BuiltInAuthenticationOptions) ToAuthenticationConfig() authenticator.Au
 		ret.ClientCAFile = s.ClientCert.ClientCA
 	}
 
-	if s.Keystone != nil {
-		ret.KeystoneURL = s.Keystone.URL
-		ret.KeystoneCAFile = s.Keystone.CAFile
-	}
-
 	if s.OIDC != nil {
 		ret.OIDCCAFile = s.OIDC.CAFile
 		ret.OIDCClientID = s.OIDC.ClientID

pkg/kubelet/apis/BUILD

@@ -27,6 +27,7 @@ filegroup(
         ":package-srcs",
         "//pkg/kubelet/apis/cri:all-srcs",
         "//pkg/kubelet/apis/deviceplugin/v1alpha:all-srcs",
+        "//pkg/kubelet/apis/deviceplugin/v1beta1:all-srcs",
         "//pkg/kubelet/apis/kubeletconfig:all-srcs",
         "//pkg/kubelet/apis/stats/v1alpha1:all-srcs",
     ],

staging/src/k8s.io/apiserver/plugin/pkg/authenticator/password/keystone/BUILD → pkg/kubelet/apis/deviceplugin/v1beta1/BUILD

@@ -8,17 +8,16 @@ load(
 go_library(
     name = "go_default_library",
     srcs = [
-        "doc.go",
-        "keystone.go",
+        "api.pb.go",
+        "constants.go",
     ],
-    importpath = "k8s.io/apiserver/plugin/pkg/authenticator/password/keystone",
+    importpath = "k8s.io/kubernetes/pkg/kubelet/apis/deviceplugin/v1beta1",
     deps = [
-        "//vendor/github.com/golang/glog:go_default_library",
-        "//vendor/github.com/gophercloud/gophercloud:go_default_library",
-        "//vendor/github.com/gophercloud/gophercloud/openstack:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",
-        "//vendor/k8s.io/apiserver/pkg/authentication/user:go_default_library",
-        "//vendor/k8s.io/client-go/util/cert:go_default_library",
+        "//vendor/github.com/gogo/protobuf/gogoproto:go_default_library",
+        "//vendor/github.com/gogo/protobuf/proto:go_default_library",
+        "//vendor/github.com/gogo/protobuf/sortkeys:go_default_library",
+        "//vendor/golang.org/x/net/context:go_default_library",
+        "//vendor/google.golang.org/grpc:go_default_library",
     ],
 )
 
@@ -34,3 +33,9 @@ filegroup(
     srcs = [":package-srcs"],
     tags = ["automanaged"],
 )
+
+filegroup(
+    name = "go_default_library_protos",
+    srcs = ["api.proto"],
+    visibility = ["//visibility:public"],
+)

pkg/kubelet/apis/deviceplugin/v1beta1/api.proto

+// To regenerate api.pb.go run hack/update-device-plugin.sh
+syntax = 'proto3';
+
+package v1beta1;
+
+import "github.com/gogo/protobuf/gogoproto/gogo.proto";
+
+option (gogoproto.goproto_stringer_all) = false;
+option (gogoproto.stringer_all) =  true;
+option (gogoproto.goproto_getters_all) = true;
+option (gogoproto.marshaler_all) = true;
+option (gogoproto.sizer_all) = true;
+option (gogoproto.unmarshaler_all) = true;
+option (gogoproto.goproto_unrecognized_all) = false;
+
+
+// Registration is the service advertised by the Kubelet
+// Only when Kubelet answers with a success code to a Register Request
+// may Device Plugins start their service
+// Registration may fail when device plugin version is not supported by
+// Kubelet or the registered resourceName is already taken by another
+// active device plugin. Device plugin is expected to terminate upon registration failure
+service Registration {
+	rpc Register(RegisterRequest) returns (Empty) {}
+}
+
+message RegisterRequest {
+	// Version of the API the Device Plugin was built against
+	string version = 1;
+	// Name of the unix socket the device plugin is listening on
+	// PATH = path.Join(DevicePluginPath, endpoint)
+	string endpoint = 2;
+	// Schedulable resource name. As of now it's expected to be a DNS Label
+	string resource_name = 3;
+}
+
+message Empty {
+}
+
+// DevicePlugin is the service advertised by Device Plugins
+service DevicePlugin {
+	// ListAndWatch returns a stream of List of Devices
+	// Whenever a Device state change or a Device disapears, ListAndWatch
+	// returns the new list
+	rpc ListAndWatch(Empty) returns (stream ListAndWatchResponse) {}
+
+	// Allocate is called during container creation so that the Device
+	// Plugin can run device specific operations and instruct Kubelet
+	// of the steps to make the Device available in the container
+	rpc Allocate(AllocateRequest) returns (AllocateResponse) {}
+}
+
+// ListAndWatch returns a stream of List of Devices
+// Whenever a Device state change or a Device disapears, ListAndWatch
+// returns the new list
+message ListAndWatchResponse {
+	repeated Device devices = 1;
+}
+
+/* E.g:
+* struct Device {
+*    ID: "GPU-fef8089b-4820-abfc-e83e-94318197576e",
+*    State: "Healthy",
+*} */
+message Device {
+	// A unique ID assigned by the device plugin used
+	// to identify devices during the communication
+	// Max length of this field is 63 characters
+	string ID = 1;
+	// Health of the device, can be healthy or unhealthy, see constants.go
+	string health = 2;
+}
+
+// - Allocate is expected to be called during pod creation since allocation
+//   failures for any container would result in pod startup failure.
+// - Allocate allows kubelet to exposes additional artifacts in a pod's
+//   environment as directed by the plugin.
+// - Allocate allows Device Plugin to run device specific operations on
+//   the Devices requested
+message AllocateRequest {
+	repeated string devicesIDs = 1;
+}
+
+// AllocateResponse includes the artifacts that needs to be injected into
+// a container for accessing 'deviceIDs' that were mentioned as part of
+// 'AllocateRequest'.
+// Failure Handling:
+// if Kubelet sends an allocation request for dev1 and dev2.
+// Allocation on dev1 succeeds but allocation on dev2 fails.
+// The Device plugin should send a ListAndWatch update and fail the
+// Allocation request
+message AllocateResponse {
+  	// List of environment variable to be set in the container to access one of more devices.
+	map<string, string> envs = 1;
+	// Mounts for the container.
+	repeated Mount mounts = 2;
+	// Devices for the container.
+	repeated DeviceSpec devices = 3;
+	// Container annotations to pass to the container runtime
+	map<string, string> annotations = 4;
+}
+
+// Mount specifies a host volume to mount into a container.
+// where device library or tools are installed on host and container
+message Mount {
+	// Path of the mount within the container.
+	string container_path = 1;
+	// Path of the mount on the host.
+	string host_path = 2;
+	// If set, the mount is read-only.
+	bool read_only = 3;
+}
+
+// DeviceSpec specifies a host device to mount into a container.
+message DeviceSpec {
+    // Path of the device within the container.
+    string container_path = 1;
+    // Path of the device on the host.
+    string host_path = 2;
+    // Cgroups permissions of the device, candidates are one or more of
+    // * r - allows container to read from the specified device.
+    // * w - allows container to write to the specified device.
+    // * m - allows container to create device files that do not yet exist.
+    string permissions = 3;
+}

pkg/kubelet/apis/deviceplugin/v1beta1/constants.go

+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1beta1
+
+const (
+	// Healthy means that the device is healty
+	Healthy = "Healthy"
+	// UnHealthy means that the device is unhealty
+	Unhealthy = "Unhealthy"
+
+	// Current version of the API supported by kubelet
+	Version = "v1beta1"
+	// DevicePluginPath is the folder the Device Plugin is expecting sockets to be on
+	// Only privileged pods have access to this path
+	// Note: Placeholder until we find a "standard path"
+	DevicePluginPath = "/var/lib/kubelet/device-plugins/"
+	// KubeletSocket is the path of the Kubelet registry socket
+	KubeletSocket = DevicePluginPath + "kubelet.sock"
+)

pkg/kubelet/apis/kubeletconfig/fuzzer/fuzzer.go

@@ -50,7 +50,6 @@ func Funcs(codecs runtimeserializer.CodecFactory) []interface{} {
 			obj.EventRecordQPS = 5
 			obj.EnableControllerAttachDetach = true
 			obj.EnableDebuggingHandlers = true
-			obj.EnableServer = true
 			obj.FileCheckFrequency = metav1.Duration{Duration: 20 * time.Second}
 			obj.HealthzBindAddress = "127.0.0.1"
 			obj.HealthzPort = 10248

pkg/kubelet/apis/kubeletconfig/helpers_test.go

@@ -156,7 +156,6 @@ var (
 		"EnableContentionProfiling",
 		"EnableControllerAttachDetach",
 		"EnableDebuggingHandlers",
-		"EnableServer",
 		"EnforceNodeAllocatable[*]",
 		"EventBurst",
 		"EventRecordQPS",

pkg/kubelet/apis/kubeletconfig/types.go

@@ -63,8 +63,6 @@ type KubeletConfiguration struct {
 	// manifestURLHeader is the HTTP header to use when accessing the manifest
 	// URL, with the key separated from the value with a ':', as in 'key:value'
 	ManifestURLHeader map[string][]string
-	// enableServer enables the Kubelet's server
-	EnableServer bool
 	// address is the IP address for the Kubelet to serve on (set to 0.0.0.0
 	// for all interfaces)
 	Address string

pkg/kubelet/apis/kubeletconfig/v1alpha1/defaults.go

@@ -46,16 +46,16 @@ func addDefaultingFuncs(scheme *kruntime.Scheme) error {
 
 func SetDefaults_KubeletConfiguration(obj *KubeletConfiguration) {
 	if obj.Authentication.Anonymous.Enabled == nil {
-		obj.Authentication.Anonymous.Enabled = boolVar(true)
+		obj.Authentication.Anonymous.Enabled = boolVar(false)
 	}
 	if obj.Authentication.Webhook.Enabled == nil {
-		obj.Authentication.Webhook.Enabled = boolVar(false)
+		obj.Authentication.Webhook.Enabled = boolVar(true)
 	}
 	if obj.Authentication.Webhook.CacheTTL == zeroDuration {
 		obj.Authentication.Webhook.CacheTTL = metav1.Duration{Duration: 2 * time.Minute}
 	}
 	if obj.Authorization.Mode == "" {
-		obj.Authorization.Mode = KubeletAuthorizationModeAlwaysAllow
+		obj.Authorization.Mode = KubeletAuthorizationModeWebhook
 	}
 	if obj.Authorization.Webhook.CacheAuthorizedTTL == zeroDuration {
 		obj.Authorization.Webhook.CacheAuthorizedTTL = metav1.Duration{Duration: 5 * time.Minute}
@@ -88,9 +88,6 @@ func SetDefaults_KubeletConfiguration(obj *KubeletConfiguration) {
 	if obj.EnableDebuggingHandlers == nil {
 		obj.EnableDebuggingHandlers = boolVar(true)
 	}
-	if obj.EnableServer == nil {
-		obj.EnableServer = boolVar(true)
-	}
 	if obj.FileCheckFrequency == zeroDuration {
 		obj.FileCheckFrequency = metav1.Duration{Duration: 20 * time.Second}
 	}
@@ -141,9 +138,6 @@ func SetDefaults_KubeletConfiguration(obj *KubeletConfiguration) {
 	if obj.Port == 0 {
 		obj.Port = ports.KubeletPort
 	}
-	if obj.ReadOnlyPort == nil {
-		obj.ReadOnlyPort = utilpointer.Int32Ptr(ports.KubeletReadOnlyPort)
-	}
 	if obj.RegistryBurst == 0 {
 		obj.RegistryBurst = 10
 	}

pkg/kubelet/apis/kubeletconfig/v1alpha1/types.go

@@ -63,16 +63,14 @@ type KubeletConfiguration struct {
 	// manifestURLHeader is the HTTP header to use when accessing the manifest
 	// URL, with the key separated from the value with a ':', as in 'key:value'
 	ManifestURLHeader map[string][]string `json:"manifestURLHeader"`
-	// enableServer enables the Kubelet's server
-	EnableServer *bool `json:"enableServer"`
 	// address is the IP address for the Kubelet to serve on (set to 0.0.0.0
 	// for all interfaces)
 	Address string `json:"address"`
 	// port is the port for the Kubelet to serve on.
 	Port int32 `json:"port"`
 	// readOnlyPort is the read-only port for the Kubelet to serve on with
-	// no authentication/authorization (set to 0 to disable)
-	ReadOnlyPort *int32 `json:"readOnlyPort"`
+	// no authentication/authorization. Disabled (set to 0) by default.
+	ReadOnlyPort int32 `json:"readOnlyPort"`
 	// tlsCertFile is the file containing x509 Certificate for HTTPS.  (CA cert,
 	// if any, concatenated after server cert). If tlsCertFile and
 	// tlsPrivateKeyFile are not provided, a self-signed certificate

pkg/kubelet/apis/kubeletconfig/v1alpha1/zz_generated.conversion.go

@@ -147,14 +147,9 @@ func autoConvert_v1alpha1_KubeletConfiguration_To_kubeletconfig_KubeletConfigura
 	out.HTTPCheckFrequency = in.HTTPCheckFrequency
 	out.ManifestURL = in.ManifestURL
 	out.ManifestURLHeader = *(*map[string][]string)(unsafe.Pointer(&in.ManifestURLHeader))
-	if err := v1.Convert_Pointer_bool_To_bool(&in.EnableServer, &out.EnableServer, s); err != nil {
-		return err
-	}
 	out.Address = in.Address
 	out.Port = in.Port
-	if err := v1.Convert_Pointer_int32_To_int32(&in.ReadOnlyPort, &out.ReadOnlyPort, s); err != nil {
-		return err
-	}
+	out.ReadOnlyPort = in.ReadOnlyPort
 	out.TLSCertFile = in.TLSCertFile
 	out.TLSPrivateKeyFile = in.TLSPrivateKeyFile
 	out.TLSCipherSuites = *(*[]string)(unsafe.Pointer(&in.TLSCipherSuites))
@@ -269,14 +264,9 @@ func autoConvert_kubeletconfig_KubeletConfiguration_To_v1alpha1_KubeletConfigura
 	out.HTTPCheckFrequency = in.HTTPCheckFrequency
 	out.ManifestURL = in.ManifestURL
 	out.ManifestURLHeader = *(*map[string][]string)(unsafe.Pointer(&in.ManifestURLHeader))
-	if err := v1.Convert_bool_To_Pointer_bool(&in.EnableServer, &out.EnableServer, s); err != nil {
-		return err
-	}
 	out.Address = in.Address
 	out.Port = in.Port
-	if err := v1.Convert_int32_To_Pointer_int32(&in.ReadOnlyPort, &out.ReadOnlyPort, s); err != nil {
-		return err
-	}
+	out.ReadOnlyPort = in.ReadOnlyPort
 	out.TLSCertFile = in.TLSCertFile
 	out.TLSPrivateKeyFile = in.TLSPrivateKeyFile
 	out.TLSCipherSuites = *(*[]string)(unsafe.Pointer(&in.TLSCipherSuites))

pkg/kubelet/apis/kubeletconfig/v1alpha1/zz_generated.deepcopy.go

@@ -104,24 +104,6 @@ func (in *KubeletConfiguration) DeepCopyInto(out *KubeletConfiguration) {
 			}
 		}
 	}
-	if in.EnableServer != nil {
-		in, out := &in.EnableServer, &out.EnableServer
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new(bool)
-			**out = **in
-		}
-	}
-	if in.ReadOnlyPort != nil {
-		in, out := &in.ReadOnlyPort, &out.ReadOnlyPort
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new(int32)
-			**out = **in
-		}
-	}
 	if in.TLSCipherSuites != nil {
 		in, out := &in.TLSCipherSuites, &out.TLSCipherSuites
 		*out = make([]string, len(*in))

pkg/kubelet/cm/devicemanager/manager_test.go

@@ -24,6 +24,7 @@ import (
 	"reflect"
 	"sync/atomic"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -40,18 +41,32 @@ import (
 )
 
 const (
-	socketName       = "/tmp/device_plugin/server.sock"
-	pluginSocketName = "/tmp/device_plugin/device-plugin.sock"
 	testResourceName = "fake-domain/resource"
 )
 
+func tmpSocketDir() (socketDir, socketName, pluginSocketName string, err error) {
+	socketDir, err = ioutil.TempDir("", "device_plugin")
+	if err != nil {
+		return
+	}
+	socketName = socketDir + "/server.sock"
+	pluginSocketName = socketDir + "/device-plugin.sock"
+	return
+}
+
 func TestNewManagerImpl(t *testing.T) {
-	_, err := newManagerImpl(socketName)
+	socketDir, socketName, _, err := tmpSocketDir()
+	require.NoError(t, err)
+	defer os.RemoveAll(socketDir)
+	_, err = newManagerImpl(socketName)
 	require.NoError(t, err)
 }
 
 func TestNewManagerImplStart(t *testing.T) {
-	m, p := setup(t, []*pluginapi.Device{}, func(n string, a, u, r []pluginapi.Device) {})
+	socketDir, socketName, pluginSocketName, err := tmpSocketDir()
+	require.NoError(t, err)
+	defer os.RemoveAll(socketDir)
+	m, p := setup(t, []*pluginapi.Device{}, func(n string, a, u, r []pluginapi.Device) {}, socketName, pluginSocketName)
 	cleanup(t, m, p)
 }
 
@@ -59,6 +74,9 @@ func TestNewManagerImplStart(t *testing.T) {
 // making sure that after registration, devices are correctly updated and if a re-registration
 // happens, we will NOT delete devices; and no orphaned devices left.
 func TestDevicePluginReRegistration(t *testing.T) {
+	socketDir, socketName, pluginSocketName, err := tmpSocketDir()
+	require.NoError(t, err)
+	defer os.RemoveAll(socketDir)
 	devs := []*pluginapi.Device{
 		{ID: "Dev1", Health: pluginapi.Healthy},
 		{ID: "Dev2", Health: pluginapi.Healthy},
@@ -77,22 +95,32 @@ func TestDevicePluginReRegistration(t *testing.T) {
 		}
 		callbackChan <- callbackCount
 	}
-	m, p1 := setup(t, devs, callback)
+	m, p1 := setup(t, devs, callback, socketName, pluginSocketName)
 	atomic.StoreInt32(&expCallbackCount, 1)
 	p1.Register(socketName, testResourceName)
 	// Wait for the first callback to be issued.
 
-	<-callbackChan
+	select {
+	case <-callbackChan:
+		break
+	case <-time.After(time.Second):
+		t.FailNow()
+	}
 	devices := m.Devices()
 	require.Equal(t, 2, len(devices[testResourceName]), "Devices are not updated.")
 
 	p2 := NewDevicePluginStub(devs, pluginSocketName+".new")
-	err := p2.Start()
+	err = p2.Start()
 	require.NoError(t, err)
 	atomic.StoreInt32(&expCallbackCount, 2)
 	p2.Register(socketName, testResourceName)
 	// Wait for the second callback to be issued.
-	<-callbackChan
+	select {
+	case <-callbackChan:
+		break
+	case <-time.After(time.Second):
+		t.FailNow()
+	}
 
 	devices2 := m.Devices()
 	require.Equal(t, 2, len(devices2[testResourceName]), "Devices shouldn't change.")
@@ -104,7 +132,12 @@ func TestDevicePluginReRegistration(t *testing.T) {
 	atomic.StoreInt32(&expCallbackCount, 3)
 	p3.Register(socketName, testResourceName)
 	// Wait for the second callback to be issued.
-	<-callbackChan
+	select {
+	case <-callbackChan:
+		break
+	case <-time.After(time.Second):
+		t.FailNow()
+	}
 
 	devices3 := m.Devices()
 	require.Equal(t, 1, len(devices3[testResourceName]), "Devices of plugin previously registered should be removed.")
@@ -114,7 +147,7 @@ func TestDevicePluginReRegistration(t *testing.T) {
 	close(callbackChan)
 }
 
-func setup(t *testing.T, devs []*pluginapi.Device, callback monitorCallback) (Manager, *Stub) {
+func setup(t *testing.T, devs []*pluginapi.Device, callback monitorCallback, socketName string, pluginSocketName string) (Manager, *Stub) {
 	m, err := newManagerImpl(socketName)
 	require.NoError(t, err)
 
@@ -139,6 +172,9 @@ func cleanup(t *testing.T, m Manager, p *Stub) {
 }
 
 func TestUpdateCapacityAllocatable(t *testing.T) {
+	socketDir, socketName, _, err := tmpSocketDir()
+	require.NoError(t, err)
+	defer os.RemoveAll(socketDir)
 	testManager, err := newManagerImpl(socketName)
 	as := assert.New(t)
 	as.NotNil(testManager)

pkg/kubelet/dockershim/docker_sandbox_test.go

@@ -19,6 +19,7 @@ package dockershim
 import (
 	"errors"
 	"fmt"
+	"math/rand"
 	"net"
 	"testing"
 	"time"
@@ -96,10 +97,8 @@ func TestSandboxStatus(t *testing.T) {
 	labels := map[string]string{"label": "foobar1"}
 	annotations := map[string]string{"annotation": "abc"}
 	config := makeSandboxConfigWithLabelsAndAnnotations("foo", "bar", "1", 0, labels, annotations)
-
-	// TODO: The following variables depend on the internal
-	// implementation of FakeDockerClient, and should be fixed.
-	fakeIP := "2.3.4.5"
+	r := rand.New(rand.NewSource(0)).Uint32()
+	podIP := fmt.Sprintf("10.%d.%d.%d", byte(r>>16), byte(r>>8), byte(r))
 
 	state := runtimeapi.PodSandboxState_SANDBOX_READY
 	ct := int64(0)
@@ -107,7 +106,7 @@ func TestSandboxStatus(t *testing.T) {
 		State:     state,
 		CreatedAt: ct,
 		Metadata:  config.Metadata,
-		Network:   &runtimeapi.PodSandboxNetworkStatus{Ip: fakeIP},
+		Network:   &runtimeapi.PodSandboxNetworkStatus{Ip: podIP},
 		Linux: &runtimeapi.LinuxPodSandboxStatus{
 			Namespaces: &runtimeapi.Namespace{
 				Options: &runtimeapi.NamespaceOption{
@@ -160,10 +159,8 @@ func TestSandboxStatus(t *testing.T) {
 func TestSandboxStatusAfterRestart(t *testing.T) {
 	ds, _, fClock := newTestDockerService()
 	config := makeSandboxConfig("foo", "bar", "1", 0)
-
-	// TODO: The following variables depend on the internal
-	// implementation of FakeDockerClient, and should be fixed.
-	fakeIP := "2.3.4.5"
+	r := rand.New(rand.NewSource(0)).Uint32()
+	podIP := fmt.Sprintf("10.%d.%d.%d", byte(r>>16), byte(r>>8), byte(r))
 
 	state := runtimeapi.PodSandboxState_SANDBOX_READY
 	ct := int64(0)
@@ -171,7 +168,7 @@ func TestSandboxStatusAfterRestart(t *testing.T) {
 		State:     state,
 		CreatedAt: ct,
 		Metadata:  config.Metadata,
-		Network:   &runtimeapi.PodSandboxNetworkStatus{Ip: fakeIP},
+		Network:   &runtimeapi.PodSandboxNetworkStatus{Ip: podIP},
 		Linux: &runtimeapi.LinuxPodSandboxStatus{
 			Namespaces: &runtimeapi.Namespace{
 				Options: &runtimeapi.NamespaceOption{

pkg/kubelet/dockershim/docker_service_test.go

@@ -18,6 +18,7 @@ package dockershim
 
 import (
 	"errors"
+	"math/rand"
 	"testing"
 	"time"
 
@@ -44,7 +45,7 @@ func newTestNetworkPlugin(t *testing.T) *nettest.MockNetworkPlugin {
 
 func newTestDockerService() (*dockerService, *libdocker.FakeDockerClient, *clock.FakeClock) {
 	fakeClock := clock.NewFakeClock(time.Time{})
-	c := libdocker.NewFakeDockerClient().WithClock(fakeClock).WithVersion("1.11.2", "1.23")
+	c := libdocker.NewFakeDockerClient().WithClock(fakeClock).WithVersion("1.11.2", "1.23").WithRandSource(rand.NewSource(0))
 	pm := network.NewPluginManager(&network.NoopNetworkPlugin{})
 	return &dockerService{
 		client:            c,

pkg/kubelet/dockershim/libdocker/fake_client.go

@@ -61,6 +61,7 @@ type FakeDockerClient struct {
 	called               []calledDetail
 	pulled               []string
 	EnableTrace          bool
+	RandGenerator        *rand.Rand
 
 	// Created, Started, Stopped and Removed all contain container docker ID
 	Created []string
@@ -99,6 +100,7 @@ func NewFakeDockerClient() *FakeDockerClient {
 		EnableTrace:         true,
 		ImageInspects:       make(map[string]*dockertypes.ImageInspect),
 		ImageIDsNeedingAuth: make(map[string]dockertypes.AuthConfig),
+		RandGenerator:       rand.New(rand.NewSource(time.Now().UnixNano())),
 	}
 }
 
@@ -123,6 +125,13 @@ func (f *FakeDockerClient) WithTraceDisabled() *FakeDockerClient {
 	return f
 }
 
+func (f *FakeDockerClient) WithRandSource(source rand.Source) *FakeDockerClient {
+	f.Lock()
+	defer f.Unlock()
+	f.RandGenerator = rand.New(source)
+	return f
+}
+
 func (f *FakeDockerClient) appendCalled(callDetail calledDetail) {
 	if f.EnableTrace {
 		f.called = append(f.called, callDetail)
@@ -597,7 +606,8 @@ func (f *FakeDockerClient) StartContainer(id string) error {
 	container.State.Running = true
 	container.State.Pid = os.Getpid()
 	container.State.StartedAt = dockerTimestampToString(timestamp)
-	container.NetworkSettings.IPAddress = "2.3.4.5"
+	r := f.RandGenerator.Uint32()
+	container.NetworkSettings.IPAddress = fmt.Sprintf("10.%d.%d.%d", byte(r>>16), byte(r>>8), byte(r))
 	f.ContainerMap[id] = container
 	f.updateContainerStatus(id, StatusRunningPrefix)
 	f.normalSleep(200, 50, 50)

pkg/kubelet/kubelet_getters.go

@@ -142,7 +142,15 @@ func (kl *Kubelet) getPodContainerDir(podUID types.UID, ctrName string) string {
 // GetPods returns all pods bound to the kubelet and their spec, and the mirror
 // pods.
 func (kl *Kubelet) GetPods() []*v1.Pod {
-	return kl.podManager.GetPods()
+	pods := kl.podManager.GetPods()
+	// a kubelet running without apiserver requires an additional
+	// update of the static pod status. See #57106
+	for _, p := range pods {
+		if status, ok := kl.statusManager.GetPodStatus(p.UID); ok {
+			p.Status = status
+		}
+	}
+	return pods
 }
 
 // GetRunningPods returns all pods running on kubelet from looking at the

pkg/kubelet/server/portforward/websocket.go

@@ -45,7 +45,7 @@ const (
 
 // options contains details about which streams are required for
 // port forwarding.
-// All fields incldued in V4Options need to be expressed explicitly in the
+// All fields included in V4Options need to be expressed explicitly in the
 // CRI (pkg/kubelet/apis/cri/{version}/api.proto) PortForwardRequest.
 type V4Options struct {
 	Ports []int32

pkg/kubemark/hollow_kubelet.go

@@ -110,6 +110,7 @@ func GetHollowKubeletConfig(
 
 	// Flags struct
 	f := options.NewKubeletFlags()
+	f.EnableServer = true
 	f.RootDirectory = testRootDir
 	f.HostnameOverride = nodeName
 	f.MinimumGCAge = metav1.Duration{Duration: 1 * time.Minute}
@@ -144,7 +145,6 @@ func GetHollowKubeletConfig(
 	c.CPUCFSQuota = true
 	c.EnableControllerAttachDetach = false
 	c.EnableDebuggingHandlers = true
-	c.EnableServer = true
 	c.CgroupsPerQOS = false
 	// hairpin-veth is used to allow hairpin packets. Note that this deviates from
 	// what the "real" kubelet currently does, because there's no way to

pkg/master/BUILD

@@ -114,6 +114,7 @@ go_library(
 go_test(
     name = "go_default_test",
     size = "medium",
+    timeout = "long",
     srcs = [
         "client_ca_hook_test.go",
         "controller_test.go",

pkg/proxy/ipvs/README.md

@@ -18,7 +18,7 @@ Currently, local-up scripts and kubeadm support switching IPVS proxy mode via ex
 
 Kube-proxy will run in iptables mode by default in a [local-up cluster](https://github.com/kubernetes/community/blob/master/contributors/devel/running-locally.md). 
 
-Users should export the env `KUBEPROXY_MODE=ipvs` to specify the ipvs mode before deploying the cluster if want to run kube-proxy in ipvs mode.
+Users should export the env `KUBE_PROXY_MODE=ipvs` to specify the ipvs mode before deploying the cluster if want to run kube-proxy in ipvs mode.
 
 ### Cluster Created by Kubeadm
 

pkg/proxy/ipvs/proxier.go

@@ -95,7 +95,7 @@ var ipvsModules = []string{
 	"nf_conntrack_ipv4",
 }
 
-// In IPVS proxy mode, the following flags need to be setted
+// In IPVS proxy mode, the following flags need to be set
 const sysctlRouteLocalnet = "net/ipv4/conf/all/route_localnet"
 const sysctlBridgeCallIPTables = "net/bridge/bridge-nf-call-iptables"
 const sysctlVSConnTrack = "net/ipv4/vs/conntrack"

pkg/scheduler/factory/BUILD

@@ -16,6 +16,7 @@ go_library(
     deps = [
         "//pkg/api/v1/pod:go_default_library",
         "//pkg/apis/core/helper:go_default_library",
+        "//pkg/apis/core/v1/helper:go_default_library",
         "//pkg/features:go_default_library",
         "//pkg/kubelet/apis:go_default_library",
         "//pkg/scheduler:go_default_library",

pkg/scheduler/factory/factory.go

@@ -50,6 +50,7 @@ import (
 	"k8s.io/client-go/tools/cache"
 	podutil "k8s.io/kubernetes/pkg/api/v1/pod"
 	"k8s.io/kubernetes/pkg/apis/core/helper"
+	v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper"
 	"k8s.io/kubernetes/pkg/features"
 	kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
 	"k8s.io/kubernetes/pkg/scheduler"
@@ -258,6 +259,7 @@ func NewConfigFactory(
 		cache.ResourceEventHandlerFuncs{
 			// MaxPDVolumeCountPredicate: since it relies on the counts of PV.
 			AddFunc:    c.onPvAdd,
+			UpdateFunc: c.onPvUpdate,
 			DeleteFunc: c.onPvDelete,
 		},
 	)
@@ -354,6 +356,56 @@ func (c *configFactory) onPvAdd(obj interface{}) {
 	}
 }
 
+func (c *configFactory) onPvUpdate(old, new interface{}) {
+	if c.enableEquivalenceClassCache {
+		newPV, ok := new.(*v1.PersistentVolume)
+		if !ok {
+			glog.Errorf("cannot convert to *v1.PersistentVolume: %v", new)
+			return
+		}
+		oldPV, ok := old.(*v1.PersistentVolume)
+		if !ok {
+			glog.Errorf("cannot convert to *v1.PersistentVolume: %v", old)
+			return
+		}
+		c.invalidatePredicatesForPvUpdate(oldPV, newPV)
+	}
+}
+
+func (c *configFactory) invalidatePredicatesForPvUpdate(oldPV, newPV *v1.PersistentVolume) {
+	invalidPredicates := sets.NewString()
+	for k, v := range newPV.Labels {
+		// If PV update modifies the zone/region labels.
+		if isZoneRegionLabel(k) && !reflect.DeepEqual(v, oldPV.Labels[k]) {
+			invalidPredicates.Insert("NoVolumeZoneConflict")
+			break
+		}
+	}
+	if utilfeature.DefaultFeatureGate.Enabled(features.VolumeScheduling) {
+		oldAffinity, err := v1helper.GetStorageNodeAffinityFromAnnotation(oldPV.Annotations)
+		if err != nil {
+			glog.Errorf("cannot get node affinity fo *v1.PersistentVolume: %v", oldPV)
+			return
+		}
+		newAffinity, err := v1helper.GetStorageNodeAffinityFromAnnotation(newPV.Annotations)
+		if err != nil {
+			glog.Errorf("cannot get node affinity fo *v1.PersistentVolume: %v", newPV)
+			return
+		}
+
+		// If node affinity of PV is changed.
+		if !reflect.DeepEqual(oldAffinity, newAffinity) {
+			invalidPredicates.Insert(predicates.CheckVolumeBindingPred)
+		}
+	}
+	c.equivalencePodCache.InvalidateCachedPredicateItemOfAllNodes(invalidPredicates)
+}
+
+// isZoneRegionLabel check if given key of label is zone or region label.
+func isZoneRegionLabel(k string) bool {
+	return k == kubeletapis.LabelZoneFailureDomain || k == kubeletapis.LabelZoneRegion
+}
+
 func (c *configFactory) onPvDelete(obj interface{}) {
 	if c.enableEquivalenceClassCache {
 		var pv *v1.PersistentVolume
@@ -392,8 +444,8 @@ func (c *configFactory) invalidatePredicatesForPv(pv *v1.PersistentVolume) {
 	}
 
 	// If PV contains zone related label, it may impact cached NoVolumeZoneConflict
-	for k := range pv.ObjectMeta.Labels {
-		if k == kubeletapis.LabelZoneFailureDomain || k == kubeletapis.LabelZoneRegion {
+	for k := range pv.Labels {
+		if isZoneRegionLabel(k) {
 			invalidPredicates.Insert("NoVolumeZoneConflict")
 			break
 		}
@@ -487,8 +539,6 @@ func (c *configFactory) invalidatePredicatesForPvcUpdate(old, new *v1.Persistent
 		}
 		// The bound volume type may change
 		invalidPredicates.Insert(maxPDVolumeCountPredicateKeys...)
-		// The bound volume's label may change
-		invalidPredicates.Insert("NoVolumeZoneConflict")
 	}
 
 	c.equivalencePodCache.InvalidateCachedPredicateItemOfAllNodes(invalidPredicates)
@@ -739,7 +789,7 @@ func (c *configFactory) invalidateCachedPredicatesOnNodeUpdate(newNode *v1.Node,
 					invalidPredicates.Insert("MatchInterPodAffinity")
 				}
 				// NoVolumeZoneConflict will only be affected by zone related label change
-				if k == kubeletapis.LabelZoneFailureDomain || k == kubeletapis.LabelZoneRegion {
+				if isZoneRegionLabel(k) {
 					if v != newNode.GetLabels()[k] {
 						invalidPredicates.Insert("NoVolumeZoneConflict")
 					}

pkg/security/podsecuritypolicy/provider.go

@@ -179,8 +179,8 @@ func (s *simpleProvider) DefaultContainerSecurityContext(pod *api.Pod, container
 	return nil
 }
 
-// Ensure a pod's SecurityContext is in compliance with the given constraints.
-func (s *simpleProvider) ValidatePodSecurityContext(pod *api.Pod, fldPath *field.Path) field.ErrorList {
+// ValidatePod ensure a pod is in compliance with the given constraints.
+func (s *simpleProvider) ValidatePod(pod *api.Pod, fldPath *field.Path) field.ErrorList {
 	allErrs := field.ErrorList{}
 
 	sc := securitycontext.NewPodSecurityContextAccessor(pod.Spec.SecurityContext)
@@ -209,8 +209,6 @@ func (s *simpleProvider) ValidatePodSecurityContext(pod *api.Pod, fldPath *field
 
 	allErrs = append(allErrs, s.strategies.SysctlsStrategy.Validate(pod)...)
 
-	// TODO(tallclair): ValidatePodSecurityContext should be renamed to ValidatePod since its scope
-	// is not limited to the PodSecurityContext.
 	if len(pod.Spec.Volumes) > 0 {
 		allowsAllVolumeTypes := psputil.PSPAllowsAllVolumes(s.psp)
 		allowedVolumes := psputil.FSTypeToStringSet(s.psp.Spec.Volumes)

pkg/security/podsecuritypolicy/provider_test.go

@@ -369,7 +369,7 @@ func TestValidatePodSecurityContextFailures(t *testing.T) {
 		if err != nil {
 			t.Fatalf("unable to create provider %v", err)
 		}
-		errs := provider.ValidatePodSecurityContext(v.pod, field.NewPath(""))
+		errs := provider.ValidatePod(v.pod, field.NewPath(""))
 		if len(errs) == 0 {
 			t.Errorf("%s expected validation failure but did not receive errors", k)
 			continue
@@ -732,7 +732,7 @@ func TestValidatePodSecurityContextSuccess(t *testing.T) {
 		if err != nil {
 			t.Fatalf("unable to create provider %v", err)
 		}
-		errs := provider.ValidatePodSecurityContext(v.pod, field.NewPath(""))
+		errs := provider.ValidatePod(v.pod, field.NewPath(""))
 		if len(errs) != 0 {
 			t.Errorf("%s expected validation pass but received errors %v", k, errs)
 			continue
@@ -1094,7 +1094,7 @@ func TestValidateAllowedVolumes(t *testing.T) {
 		}
 
 		// expect a denial for this PSP and test the error message to ensure it's related to the volumesource
-		errs := provider.ValidatePodSecurityContext(pod, field.NewPath(""))
+		errs := provider.ValidatePod(pod, field.NewPath(""))
 		if len(errs) != 1 {
 			t.Errorf("expected exactly 1 error for %s but got %v", fieldVal.Name, errs)
 		} else {
@@ -1105,14 +1105,14 @@ func TestValidateAllowedVolumes(t *testing.T) {
 
 		// now add the fstype directly to the psp and it should validate
 		psp.Spec.Volumes = []extensions.FSType{fsType}
-		errs = provider.ValidatePodSecurityContext(pod, field.NewPath(""))
+		errs = provider.ValidatePod(pod, field.NewPath(""))
 		if len(errs) != 0 {
 			t.Errorf("directly allowing volume expected no errors for %s but got %v", fieldVal.Name, errs)
 		}
 
 		// now change the psp to allow any volumes and the pod should still validate
 		psp.Spec.Volumes = []extensions.FSType{extensions.All}
-		errs = provider.ValidatePodSecurityContext(pod, field.NewPath(""))
+		errs = provider.ValidatePod(pod, field.NewPath(""))
 		if len(errs) != 0 {
 			t.Errorf("wildcard volume expected no errors for %s but got %v", fieldVal.Name, errs)
 		}

pkg/security/podsecuritypolicy/types.go

@@ -38,8 +38,8 @@ type Provider interface {
 	// DefaultContainerSecurityContext sets the default values of the required but not filled fields.
 	// It modifies the SecurityContext of the container and annotations of the pod.
 	DefaultContainerSecurityContext(pod *api.Pod, container *api.Container) error
-	// Ensure a pod's SecurityContext is in compliance with the given constraints.
-	ValidatePodSecurityContext(pod *api.Pod, fldPath *field.Path) field.ErrorList
+	// Ensure a pod is in compliance with the given constraints.
+	ValidatePod(pod *api.Pod, fldPath *field.Path) field.ErrorList
 	// Ensure a container's SecurityContext is in compliance with the given constraints
 	ValidateContainerSecurityContext(pod *api.Pod, container *api.Container, fldPath *field.Path) field.ErrorList
 	// Get the name of the PSP that this provider was initialized with.

pkg/util/mount/mount_linux.go

@@ -266,7 +266,7 @@ func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) {
 	if err != nil {
 		return true, err
 	}
-	rootStat, err := os.Lstat(file + "/..")
+	rootStat, err := os.Lstat(filepath.Dir(strings.TrimSuffix(file, "/")))
 	if err != nil {
 		return true, err
 	}

pkg/volume/azure_dd/attacher.go

@@ -24,7 +24,6 @@ import (
 	"path/filepath"
 	"runtime"
 	"strconv"
-	"strings"
 	"time"
 
 	"github.com/Azure/azure-sdk-for-go/arm/compute"
@@ -70,9 +69,6 @@ func (a *azureDiskAttacher) Attach(spec *volume.Spec, nodeName types.NodeName) (
 		glog.Warningf("failed to get azure instance id (%v)", err)
 		return "", fmt.Errorf("failed to get azure instance id for node %q (%v)", nodeName, err)
 	}
-	if ind := strings.LastIndex(instanceid, "/"); ind >= 0 {
-		instanceid = instanceid[(ind + 1):]
-	}
 
 	diskController, err := getDiskController(a.plugin.host)
 	if err != nil {
@@ -272,14 +268,11 @@ func (d *azureDiskDetacher) Detach(diskURI string, nodeName types.NodeName) erro
 		return fmt.Errorf("invalid disk to detach: %q", diskURI)
 	}
 
-	instanceid, err := d.cloud.InstanceID(context.TODO(), nodeName)
+	_, err := d.cloud.InstanceID(context.TODO(), nodeName)
 	if err != nil {
 		glog.Warningf("no instance id for node %q, skip detaching (%v)", nodeName, err)
 		return nil
 	}
-	if ind := strings.LastIndex(instanceid, "/"); ind >= 0 {
-		instanceid = instanceid[(ind + 1):]
-	}
 
 	glog.V(4).Infof("detach %v from node %q", diskURI, nodeName)
 

pkg/volume/glusterfs/glusterfs.go

@@ -160,7 +160,7 @@ func (plugin *glusterfsPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, _ volu
 		glog.Errorf("failed to get endpoints %s[%v]", epName, err)
 		return nil, err
 	}
-	glog.V(1).Infof("glusterfs pv endpoint %v", ep)
+	glog.V(1).Infof("glusterfs pv endpoint: [%v]", ep)
 	return plugin.newMounterInternal(spec, ep, pod, plugin.host.GetMounter(plugin.GetPluginName()))
 }
 
@@ -251,7 +251,7 @@ func (b *glusterfsMounter) SetUp(fsGroup *int64) error {
 
 func (b *glusterfsMounter) SetUpAt(dir string, fsGroup *int64) error {
 	notMnt, err := b.mounter.IsLikelyNotMountPoint(dir)
-	glog.V(4).Infof("mount setup: %s %v %v", dir, !notMnt, err)
+	glog.V(4).Infof("mount setup: [%s %v %v]", dir, !notMnt, err)
 	if err != nil && !os.IsNotExist(err) {
 		return err
 	}
@@ -300,7 +300,7 @@ func (b *glusterfsMounter) setUpAtInternal(dir string) error {
 
 	p := path.Join(b.glusterfs.plugin.host.GetPluginDir(glusterfsPluginName), b.glusterfs.volName)
 	if err := os.MkdirAll(p, 0750); err != nil {
-		return fmt.Errorf("Error creating directory %v: %v", p, err)
+		return fmt.Errorf("Error creating directory [%v]: error: %v", p, err)
 	}
 
 	// adding log-level ERROR to remove noise
@@ -424,11 +424,11 @@ type glusterfsVolumeProvisioner struct {
 func convertGid(gidString string) (int, error) {
 	gid64, err := strconv.ParseInt(gidString, 10, 32)
 	if err != nil {
-		return 0, fmt.Errorf("failed to parse gid %v ", gidString)
+		return 0, fmt.Errorf("failed to parse gid [%v]", gidString)
 	}
 
 	if gid64 < 0 {
-		return 0, fmt.Errorf("negative GIDs are not allowed: %v", gidString)
+		return 0, fmt.Errorf("negative GIDs are not allowed: [%v]", gidString)
 	}
 
 	// ParseInt returns a int64, but since we parsed only
@@ -504,7 +504,7 @@ func (plugin *glusterfsPlugin) collectGids(className string, gidTable *MinMaxAll
 		gidStr, ok := pv.Annotations[volumehelper.VolumeGidAnnotationKey]
 
 		if !ok {
-			glog.Warningf("no GID found in pv '%v'", pvName)
+			glog.Warningf("no GID found in pv [%v]", pvName)
 			continue
 		}
 
@@ -516,9 +516,9 @@ func (plugin *glusterfsPlugin) collectGids(className string, gidTable *MinMaxAll
 
 		_, err = gidTable.Allocate(gid)
 		if err == ErrConflict {
-			glog.Warningf("GID %v found in pv %v was already allocated", gid)
+			glog.Warningf("GID [%v] found in pv [%v] was already allocated", gid)
 		} else if err != nil {
-			glog.Errorf("failed to store gid %v found in pv '%v': %v", gid, pvName, err)
+			glog.Errorf("failed to store gid [%v] found in pv [%v]: error:%v", gid, pvName, err)
 			return err
 		}
 	}
@@ -614,7 +614,7 @@ func (d *glusterfsVolumeDeleter) Delete() error {
 	}
 	d.provisionerConfig = *cfg
 
-	glog.V(4).Infof("deleting volume %q", volumeID)
+	glog.V(4).Infof("deleting volume [%q]", volumeID)
 
 	gid, exists, err := d.getGid()
 	if err != nil {
@@ -622,12 +622,12 @@ func (d *glusterfsVolumeDeleter) Delete() error {
 	} else if exists {
 		gidTable, err := d.plugin.getGidTable(class.Name, cfg.gidMin, cfg.gidMax)
 		if err != nil {
-			return fmt.Errorf("failed to get gidTable: %v", err)
+			return fmt.Errorf("failed to get gidTable: error: %v", err)
 		}
 
 		err = gidTable.Release(gid)
 		if err != nil {
-			return fmt.Errorf("failed to release gid %v: %v", gid, err)
+			return fmt.Errorf("failed to release gid [%v], error: %v", gid, err)
 		}
 	}
 
@@ -641,7 +641,7 @@ func (d *glusterfsVolumeDeleter) Delete() error {
 		glog.Errorf("error when deleting the volume[%s]: %v", volumeName, err)
 		return err
 	}
-	glog.V(2).Infof("volume %s deleted successfully", volumeName)
+	glog.V(2).Infof("volume [%s] deleted successfully", volumeName)
 
 	//Deleter takes endpoint and namespace from pv spec.
 	pvSpec := d.spec.Spec
@@ -670,14 +670,14 @@ func (d *glusterfsVolumeDeleter) Delete() error {
 
 func (p *glusterfsVolumeProvisioner) Provision() (*v1.PersistentVolume, error) {
 	if !volume.AccessModesContainedInAll(p.plugin.GetAccessModes(), p.options.PVC.Spec.AccessModes) {
-		return nil, fmt.Errorf("invalid AccessModes %v: only AccessModes %v are supported", p.options.PVC.Spec.AccessModes, p.plugin.GetAccessModes())
+		return nil, fmt.Errorf("invalid AccessModes [%v]: only AccessModes [%v] are supported", p.options.PVC.Spec.AccessModes, p.plugin.GetAccessModes())
 	}
 
 	if p.options.PVC.Spec.Selector != nil {
 		glog.V(4).Infof("not able to parse your claim Selector")
 		return nil, fmt.Errorf("not able to parse your claim Selector")
 	}
-	glog.V(4).Infof("Provision VolumeOptions %v", p.options)
+	glog.V(4).Infof("Provision VolumeOptions [%v]", p.options)
 	scName := v1helper.GetPersistentVolumeClaimClass(p.options.PVC)
 	cfg, err := parseClassParameters(p.options.Parameters, p.plugin.host.GetKubeClient())
 	if err != nil {
@@ -689,21 +689,21 @@ func (p *glusterfsVolumeProvisioner) Provision() (*v1.PersistentVolume, error) {
 
 	gidTable, err := p.plugin.getGidTable(scName, cfg.gidMin, cfg.gidMax)
 	if err != nil {
-		return nil, fmt.Errorf("failed to get gidTable: %v", err)
+		return nil, fmt.Errorf("failed to get gidTable, error: %v", err)
 	}
 
 	gid, _, err := gidTable.AllocateNext()
 	if err != nil {
-		glog.Errorf("failed to reserve GID from table: %v", err)
-		return nil, fmt.Errorf("failed to reserve GID from table: %v", err)
+		glog.Errorf("failed to reserve GID from table, error: %v", err)
+		return nil, fmt.Errorf("failed to reserve GID from table, error: %v", err)
 	}
 
-	glog.V(2).Infof("Allocated GID [%d] for PVC %s", gid, p.options.PVC.Name)
+	glog.V(2).Infof("Allocated GID [%d] for PVC [%s]", gid, p.options.PVC.Name)
 
 	glusterfs, sizeGiB, volID, err := p.CreateVolume(gid)
 	if err != nil {
 		if releaseErr := gidTable.Release(gid); releaseErr != nil {
-			glog.Errorf("error when releasing GID in storageclass: %s", scName)
+			glog.Errorf("error when releasing GID in storageclass: [%s]", scName)
 		}
 
 		glog.Errorf("create volume error: %v.", err)
@@ -755,7 +755,7 @@ func (p *glusterfsVolumeProvisioner) CreateVolume(gid int) (r *v1.GlusterfsVolum
 	}
 	if p.provisionerConfig.clusterID != "" {
 		clusterIDs = dstrings.Split(p.clusterID, ",")
-		glog.V(4).Infof("provided clusterIDs: %v", clusterIDs)
+		glog.V(4).Infof("provided clusterIDs: [%v]", clusterIDs)
 	}
 
 	if p.provisionerConfig.volumeNamePrefix != "" {
@@ -766,15 +766,15 @@ func (p *glusterfsVolumeProvisioner) CreateVolume(gid int) (r *v1.GlusterfsVolum
 	volumeReq := &gapi.VolumeCreateRequest{Size: sz, Name: customVolumeName, Clusters: clusterIDs, Gid: gid64, Durability: p.volumeType, GlusterVolumeOptions: p.volumeOptions}
 	volume, err := cli.VolumeCreate(volumeReq)
 	if err != nil {
-		glog.Errorf("error creating volume %v ", err)
+		glog.Errorf("error creating volume, error: %v ", err)
 		return nil, 0, "", fmt.Errorf("error creating volume %v", err)
 	}
 	glog.V(1).Infof("volume with size: %d and name: %s created", volume.Size, volume.Name)
 	volID = volume.Id
 	dynamicHostIps, err := getClusterNodes(cli, volume.Cluster)
 	if err != nil {
-		glog.Errorf("error [%v] when getting cluster nodes for volume %s", err, volume)
-		return nil, 0, "", fmt.Errorf("error [%v] when getting cluster nodes for volume %s", err, volume)
+		glog.Errorf("error [%v] when getting cluster nodes for volume [%s]", err, volume)
+		return nil, 0, "", fmt.Errorf("error [%v] when getting cluster nodes for volume [%s]", err, volume)
 	}
 
 	// The 'endpointname' is created in form of 'glusterfs-dynamic-<claimname>'.
@@ -902,15 +902,15 @@ func getClusterNodes(cli *gcli.Client, cluster string) (dynamicHostIps []string,
 	// of the cluster on which provisioned volume belongs to, as there can be multiple
 	// clusters.
 	for _, node := range clusterinfo.Nodes {
-		nodei, err := cli.NodeInfo(string(node))
+		nodeInfo, err := cli.NodeInfo(string(node))
 		if err != nil {
-			glog.Errorf("failed to get hostip: %v", err)
-			return nil, fmt.Errorf("failed to get hostip: %v", err)
+			glog.Errorf("failed to get host ipaddress: %v", err)
+			return nil, fmt.Errorf("failed to get host ipaddress: %v", err)
 		}
-		ipaddr := dstrings.Join(nodei.NodeAddRequest.Hostnames.Storage, "")
+		ipaddr := dstrings.Join(nodeInfo.NodeAddRequest.Hostnames.Storage, "")
 		dynamicHostIps = append(dynamicHostIps, ipaddr)
 	}
-	glog.V(3).Infof("hostlist :%v", dynamicHostIps)
+	glog.V(3).Infof("host list :%v", dynamicHostIps)
 	if len(dynamicHostIps) == 0 {
 		glog.Errorf("no hosts found: %v", err)
 		return nil, fmt.Errorf("no hosts found: %v", err)
@@ -1095,11 +1095,11 @@ func getVolumeID(pv *v1.PersistentVolume, volumeName string) (string, error) {
 func (plugin *glusterfsPlugin) ExpandVolumeDevice(spec *volume.Spec, newSize resource.Quantity, oldSize resource.Quantity) (resource.Quantity, error) {
 	pvSpec := spec.PersistentVolume.Spec
 	volumeName := pvSpec.Glusterfs.Path
-	glog.V(2).Infof("Request to expand volume: %s ", volumeName)
+	glog.V(2).Infof("Request to expand volume: [%s]", volumeName)
 	volumeID, err := getVolumeID(spec.PersistentVolume, volumeName)
 
 	if err != nil {
-		return oldSize, fmt.Errorf("failed to get volumeID for volume %s, err: %v", volumeName, err)
+		return oldSize, fmt.Errorf("failed to get volumeID for volume [%s], err: %v", volumeName, err)
 	}
 
 	//Get details of SC.
@@ -1117,8 +1117,8 @@ func (plugin *glusterfsPlugin) ExpandVolumeDevice(spec *volume.Spec, newSize res
 	//Create REST server connection
 	cli := gcli.NewClient(cfg.url, cfg.user, cfg.secretValue)
 	if cli == nil {
-		glog.Errorf("failed to create glusterfs rest client")
-		return oldSize, fmt.Errorf("failed to create glusterfs rest client, REST server authentication failed")
+		glog.Errorf("failed to create glusterfs REST client")
+		return oldSize, fmt.Errorf("failed to create glusterfs REST client, REST server authentication failed")
 	}
 
 	// Find out delta size

plugin/pkg/admission/security/podsecuritypolicy/admission.go

@@ -301,7 +301,7 @@ func assignSecurityContext(provider psp.Provider, pod *api.Pod, fldPath *field.P
 		errs = append(errs, field.Invalid(field.NewPath("spec", "securityContext"), pod.Spec.SecurityContext, err.Error()))
 	}
 
-	errs = append(errs, provider.ValidatePodSecurityContext(pod, field.NewPath("spec", "securityContext"))...)
+	errs = append(errs, provider.ValidatePod(pod, field.NewPath("spec", "securityContext"))...)
 
 	for i := range pod.Spec.InitContainers {
 		err := provider.DefaultContainerSecurityContext(pod, &pod.Spec.InitContainers[i])

staging/src/k8s.io/apiserver/Godeps/Godeps.json

@@ -491,34 +491,6 @@
 			"Rev": "0c5108395e2debce0d731cf0287ddf7242066aba"
 		},
 		{
-			"ImportPath": "github.com/gophercloud/gophercloud",
-			"Rev": "8e59687aa4b27ab22a0bf3295f1e165ff7bd5f97"
-		},
-		{
-			"ImportPath": "github.com/gophercloud/gophercloud/openstack",
-			"Rev": "8e59687aa4b27ab22a0bf3295f1e165ff7bd5f97"
-		},
-		{
-			"ImportPath": "github.com/gophercloud/gophercloud/openstack/identity/v2/tenants",
-			"Rev": "8e59687aa4b27ab22a0bf3295f1e165ff7bd5f97"
-		},
-		{
-			"ImportPath": "github.com/gophercloud/gophercloud/openstack/identity/v2/tokens",
-			"Rev": "8e59687aa4b27ab22a0bf3295f1e165ff7bd5f97"
-		},
-		{
-			"ImportPath": "github.com/gophercloud/gophercloud/openstack/identity/v3/tokens",
-			"Rev": "8e59687aa4b27ab22a0bf3295f1e165ff7bd5f97"
-		},
-		{
-			"ImportPath": "github.com/gophercloud/gophercloud/openstack/utils",
-			"Rev": "8e59687aa4b27ab22a0bf3295f1e165ff7bd5f97"
-		},
-		{
-			"ImportPath": "github.com/gophercloud/gophercloud/pagination",
-			"Rev": "8e59687aa4b27ab22a0bf3295f1e165ff7bd5f97"
-		},
-		{
 			"ImportPath": "github.com/grpc-ecosystem/go-grpc-prometheus",
 			"Rev": "2500245aa6110c562d17020fb31a2c133d737799"
 		},

staging/src/k8s.io/apiserver/pkg/server/config.go

@@ -85,16 +85,13 @@ type Config struct {
 	// Authentication is the configuration for authentication
 	Authentication AuthenticationInfo
 
-	// Authentication is the configuration for authentication
+	// Authorization is the configuration for authorization
 	Authorization AuthorizationInfo
 
 	// LoopbackClientConfig is a config for a privileged loopback connection to the API server
 	// This is required for proper functioning of the PostStartHooks on a GenericAPIServer
 	// TODO: move into SecureServing(WithLoopback) as soon as insecure serving is gone
 	LoopbackClientConfig *restclient.Config
-	// Authorizer determines whether the subject is allowed to make the request based only
-	// on the RequestURI
-	Authorizer authorizer.Authorizer
 	// RuleResolver is required to get the list of rules that apply to a given user
 	// in a given namespace
 	RuleResolver authorizer.RuleResolver

staging/src/k8s.io/apiserver/pkg/storage/cacher.go

@@ -130,7 +130,7 @@ func (i *indexedWatchers) terminateAll(objectType reflect.Type) {
 	}
 }
 
-type watchFilterFunc func(key string, l labels.Set, f fields.Set, uninitialized bool) bool
+type filterWithAttrsFunc func(key string, l labels.Set, f fields.Set, uninitialized bool) bool
 
 // Cacher is responsible for serving WATCH and LIST requests for a given
 // resource from its internal cache and updating its cache in the background
@@ -337,7 +337,7 @@ func (c *Cacher) Watch(ctx context.Context, key string, resourceVersion string,
 	c.Lock()
 	defer c.Unlock()
 	forget := forgetWatcher(c, c.watcherIdx, triggerValue, triggerSupported)
-	watcher := newCacheWatcher(watchRV, chanSize, initEvents, watchFilterFunction(key, pred), forget, c.versioner)
+	watcher := newCacheWatcher(watchRV, chanSize, initEvents, filterWithAttrsFunction(key, pred), forget, c.versioner)
 
 	c.watchers.addWatcher(watcher, c.watcherIdx, triggerValue, triggerSupported)
 	c.watcherIdx++
@@ -439,7 +439,7 @@ func (c *Cacher) GetToList(ctx context.Context, key string, resourceVersion stri
 	if err != nil || listVal.Kind() != reflect.Slice {
 		return fmt.Errorf("need a pointer to slice, got %v", listVal.Kind())
 	}
-	filter := filterFunction(key, pred)
+	filter := filterWithAttrsFunction(key, pred)
 
 	obj, exists, readResourceVersion, err := c.watchCache.WaitUntilFreshAndGet(listRV, key, trace)
 	if err != nil {
@@ -452,7 +452,7 @@ func (c *Cacher) GetToList(ctx context.Context, key string, resourceVersion stri
 		if !ok {
 			return fmt.Errorf("non *storeElement returned from storage: %v", obj)
 		}
-		if filter(elem.Key, elem.Object) {
+		if filter(elem.Key, elem.Labels, elem.Fields, elem.Uninitialized) {
 			listVal.Set(reflect.Append(listVal, reflect.ValueOf(elem.Object).Elem()))
 		}
 	}
@@ -508,7 +508,7 @@ func (c *Cacher) List(ctx context.Context, key string, resourceVersion string, p
 	if err != nil || listVal.Kind() != reflect.Slice {
 		return fmt.Errorf("need a pointer to slice, got %v", listVal.Kind())
 	}
-	filter := filterFunction(key, pred)
+	filter := filterWithAttrsFunction(key, pred)
 
 	objs, readResourceVersion, err := c.watchCache.WaitUntilFreshAndList(listRV, trace)
 	if err != nil {
@@ -526,7 +526,7 @@ func (c *Cacher) List(ctx context.Context, key string, resourceVersion string, p
 		if !ok {
 			return fmt.Errorf("non *storeElement returned from storage: %v", obj)
 		}
-		if filter(elem.Key, elem.Object) {
+		if filter(elem.Key, elem.Labels, elem.Fields, elem.Uninitialized) {
 			listVal.Set(reflect.Append(listVal, reflect.ValueOf(elem.Object).Elem()))
 		}
 	}
@@ -680,22 +680,7 @@ func forgetWatcher(c *Cacher, index int, triggerValue string, triggerSupported b
 	}
 }
 
-func filterFunction(key string, p SelectionPredicate) func(string, runtime.Object) bool {
-	filterFunc := func(objKey string, obj runtime.Object) bool {
-		if !hasPathPrefix(objKey, key) {
-			return false
-		}
-		matches, err := p.Matches(obj)
-		if err != nil {
-			glog.Errorf("invalid object for matching. Obj: %v. Err: %v", obj, err)
-			return false
-		}
-		return matches
-	}
-	return filterFunc
-}
-
-func watchFilterFunction(key string, p SelectionPredicate) watchFilterFunc {
+func filterWithAttrsFunction(key string, p SelectionPredicate) filterWithAttrsFunc {
 	filterFunc := func(objKey string, label labels.Set, field fields.Set, uninitialized bool) bool {
 		if !hasPathPrefix(objKey, key) {
 			return false
@@ -788,13 +773,13 @@ type cacheWatcher struct {
 	input     chan *watchCacheEvent
 	result    chan watch.Event
 	done      chan struct{}
-	filter    watchFilterFunc
+	filter    filterWithAttrsFunc
 	stopped   bool
 	forget    func(bool)
 	versioner Versioner
 }
 
-func newCacheWatcher(resourceVersion uint64, chanSize int, initEvents []*watchCacheEvent, filter watchFilterFunc, forget func(bool), versioner Versioner) *cacheWatcher {
+func newCacheWatcher(resourceVersion uint64, chanSize int, initEvents []*watchCacheEvent, filter filterWithAttrsFunc, forget func(bool), versioner Versioner) *cacheWatcher {
 	watcher := &cacheWatcher{
 		input:     make(chan *watchCacheEvent, chanSize),
 		result:    make(chan watch.Event, chanSize),

staging/src/k8s.io/apiserver/pkg/storage/watch_cache.go

@@ -61,12 +61,16 @@ type watchCacheEvent struct {
 }
 
 // Computing a key of an object is generally non-trivial (it performs
-// e.g. validation underneath). To avoid computing it multiple times
-// (to serve the event in different List/Watch requests), in the
-// underlying store we are keeping pair (key, object).
+// e.g. validation underneath). Similarly computing object fields and
+// labels. To avoid computing them multiple times (to serve the event
+// in different List/Watch requests), in the underlying store we are
+// keeping structs (key, object, labels, fields, uninitialized).
 type storeElement struct {
-	Key    string
-	Object runtime.Object
+	Key           string
+	Object        runtime.Object
+	Labels        labels.Set
+	Fields        fields.Set
+	Uninitialized bool
 }
 
 func storeElementKey(obj interface{}) (string, error) {
@@ -220,6 +224,20 @@ func (w *watchCache) processEvent(event watch.Event, resourceVersion uint64, upd
 		return fmt.Errorf("couldn't compute key: %v", err)
 	}
 	elem := &storeElement{Key: key, Object: event.Object}
+	elem.Labels, elem.Fields, elem.Uninitialized, err = w.getAttrsFunc(event.Object)
+	if err != nil {
+		return err
+	}
+
+	watchCacheEvent := &watchCacheEvent{
+		Type:             event.Type,
+		Object:           elem.Object,
+		ObjLabels:        elem.Labels,
+		ObjFields:        elem.Fields,
+		ObjUninitialized: elem.Uninitialized,
+		Key:              key,
+		ResourceVersion:  resourceVersion,
+	}
 
 	// TODO: We should consider moving this lock below after the watchCacheEvent
 	// is created. In such situation, the only problematic scenario is Replace(
@@ -231,34 +249,14 @@ func (w *watchCache) processEvent(event watch.Event, resourceVersion uint64, upd
 	if err != nil {
 		return err
 	}
-	objLabels, objFields, objUninitialized, err := w.getAttrsFunc(event.Object)
-	if err != nil {
-		return err
-	}
-	var prevObject runtime.Object
-	var prevObjLabels labels.Set
-	var prevObjFields fields.Set
-	var prevObjUninitialized bool
 	if exists {
-		prevObject = previous.(*storeElement).Object
-		prevObjLabels, prevObjFields, prevObjUninitialized, err = w.getAttrsFunc(prevObject)
-		if err != nil {
-			return err
-		}
-	}
-	watchCacheEvent := &watchCacheEvent{
-		Type:                 event.Type,
-		Object:               event.Object,
-		ObjLabels:            objLabels,
-		ObjFields:            objFields,
-		ObjUninitialized:     objUninitialized,
-		PrevObject:           prevObject,
-		PrevObjLabels:        prevObjLabels,
-		PrevObjFields:        prevObjFields,
-		PrevObjUninitialized: prevObjUninitialized,
-		Key:                  key,
-		ResourceVersion:      resourceVersion,
+		previousElem := previous.(*storeElement)
+		watchCacheEvent.PrevObject = previousElem.Object
+		watchCacheEvent.PrevObjLabels = previousElem.Labels
+		watchCacheEvent.PrevObjFields = previousElem.Fields
+		watchCacheEvent.PrevObjUninitialized = previousElem.Uninitialized
 	}
+
 	if w.onEvent != nil {
 		w.onEvent(watchCacheEvent)
 	}

staging/src/k8s.io/apiserver/plugin/pkg/authenticator/password/BUILD

@@ -23,7 +23,6 @@ filegroup(
     srcs = [
         ":package-srcs",
         "//staging/src/k8s.io/apiserver/plugin/pkg/authenticator/password/allow:all-srcs",
-        "//staging/src/k8s.io/apiserver/plugin/pkg/authenticator/password/keystone:all-srcs",
         "//staging/src/k8s.io/apiserver/plugin/pkg/authenticator/password/passwordfile:all-srcs",
     ],
     tags = ["automanaged"],

staging/src/k8s.io/apiserver/plugin/pkg/authenticator/password/keystone/doc.go

-/*
-Copyright 2014 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Package keystone provides authentication via keystone.
-// For details about keystone and how to use the plugin, refer to
-// https://github.com/kubernetes/kubernetes.github.io/blob/master/docs/admin/authentication.md
-package keystone // import "k8s.io/apiserver/plugin/pkg/authenticator/password/keystone"

staging/src/k8s.io/apiserver/plugin/pkg/authenticator/password/keystone/keystone.go

-/*
-Copyright 2015 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package keystone
-
-import (
-	"crypto/tls"
-	"errors"
-	"net/http"
-	"strings"
-
-	"github.com/golang/glog"
-	"github.com/gophercloud/gophercloud"
-	"github.com/gophercloud/gophercloud/openstack"
-
-	netutil "k8s.io/apimachinery/pkg/util/net"
-	"k8s.io/apiserver/pkg/authentication/user"
-	certutil "k8s.io/client-go/util/cert"
-)
-
-// KeystoneAuthenticator contacts openstack keystone to validate user's credentials passed in the request.
-// The keystone endpoint is passed during apiserver startup
-type KeystoneAuthenticator struct {
-	authURL   string
-	transport http.RoundTripper
-}
-
-// AuthenticatePassword checks the username, password via keystone call
-func (keystoneAuthenticator *KeystoneAuthenticator) AuthenticatePassword(username string, password string) (user.Info, bool, error) {
-	opts := gophercloud.AuthOptions{
-		IdentityEndpoint: keystoneAuthenticator.authURL,
-		Username:         username,
-		Password:         password,
-	}
-
-	_, err := keystoneAuthenticator.AuthenticatedClient(opts)
-	if err != nil {
-		glog.Info("Failed: Starting openstack authenticate client:" + err.Error())
-		return nil, false, errors.New("Failed to authenticate")
-	}
-
-	return &user.DefaultInfo{Name: username}, true, nil
-}
-
-// AuthenticatedClient logs in to an OpenStack cloud found at the identity endpoint specified by options, acquires a
-// token, and returns a Client instance that's ready to operate.
-func (keystoneAuthenticator *KeystoneAuthenticator) AuthenticatedClient(options gophercloud.AuthOptions) (*gophercloud.ProviderClient, error) {
-	client, err := openstack.NewClient(options.IdentityEndpoint)
-	if err != nil {
-		return nil, err
-	}
-
-	if keystoneAuthenticator.transport != nil {
-		client.HTTPClient.Transport = keystoneAuthenticator.transport
-	}
-
-	err = openstack.Authenticate(client, options)
-	return client, err
-}
-
-// NewKeystoneAuthenticator returns a password authenticator that validates credentials using openstack keystone
-func NewKeystoneAuthenticator(authURL string, caFile string) (*KeystoneAuthenticator, error) {
-	if !strings.HasPrefix(authURL, "https") {
-		return nil, errors.New("Auth URL should be secure and start with https")
-	}
-	if authURL == "" {
-		return nil, errors.New("Auth URL is empty")
-	}
-	if caFile != "" {
-		roots, err := certutil.NewPool(caFile)
-		if err != nil {
-			return nil, err
-		}
-		config := &tls.Config{}
-		config.RootCAs = roots
-		transport := netutil.SetOldTransportDefaults(&http.Transport{TLSClientConfig: config})
-		return &KeystoneAuthenticator{authURL, transport}, nil
-	}
-
-	return &KeystoneAuthenticator{authURL: authURL}, nil
-}

staging/src/k8s.io/client-go/Godeps/Godeps.json

@@ -16,19 +16,19 @@
 		},
 		{
 			"ImportPath": "github.com/Azure/go-autorest/autorest",
-			"Rev": "e14a70c556c8e0db173358d1a903dca345a8e75e"
+			"Rev": "d4e6b95c12a08b4de2d48b45d5b4d594e5d32fab"
 		},
 		{
 			"ImportPath": "github.com/Azure/go-autorest/autorest/adal",
-			"Rev": "e14a70c556c8e0db173358d1a903dca345a8e75e"
+			"Rev": "d4e6b95c12a08b4de2d48b45d5b4d594e5d32fab"
 		},
 		{
 			"ImportPath": "github.com/Azure/go-autorest/autorest/azure",
-			"Rev": "e14a70c556c8e0db173358d1a903dca345a8e75e"
+			"Rev": "d4e6b95c12a08b4de2d48b45d5b4d594e5d32fab"
 		},
 		{
 			"ImportPath": "github.com/Azure/go-autorest/autorest/date",
-			"Rev": "e14a70c556c8e0db173358d1a903dca345a8e75e"
+			"Rev": "d4e6b95c12a08b4de2d48b45d5b4d594e5d32fab"
 		},
 		{
 			"ImportPath": "github.com/coreos/go-oidc/http",

staging/src/k8s.io/client-go/dynamic/client.go

@@ -145,6 +145,19 @@ type ResourceClient struct {
 	parameterCodec runtime.ParameterCodec
 }
 
+func (rc *ResourceClient) parseResourceSubresourceName() (string, []string) {
+	var resourceName string
+	var subresourceName []string
+	if strings.Contains(rc.resource.Name, "/") {
+		resourceName = strings.Split(rc.resource.Name, "/")[0]
+		subresourceName = strings.Split(rc.resource.Name, "/")[1:]
+	} else {
+		resourceName = rc.resource.Name
+	}
+
+	return resourceName, subresourceName
+}
+
 // List returns a list of objects for this resource.
 func (rc *ResourceClient) List(opts metav1.ListOptions) (runtime.Object, error) {
 	parameterEncoder := rc.parameterCodec
@@ -166,9 +179,11 @@ func (rc *ResourceClient) Get(name string, opts metav1.GetOptions) (*unstructure
 		parameterEncoder = defaultParameterEncoder
 	}
 	result := new(unstructured.Unstructured)
+	resourceName, subresourceName := rc.parseResourceSubresourceName()
 	err := rc.cl.Get().
 		NamespaceIfScoped(rc.ns, rc.resource.Namespaced).
-		Resource(rc.resource.Name).
+		Resource(resourceName).
+		SubResource(subresourceName...).
 		VersionedParams(&opts, parameterEncoder).
 		Name(name).
 		Do().
@@ -205,11 +220,26 @@ func (rc *ResourceClient) DeleteCollection(deleteOptions *metav1.DeleteOptions,
 // Create creates the provided resource.
 func (rc *ResourceClient) Create(obj *unstructured.Unstructured) (*unstructured.Unstructured, error) {
 	result := new(unstructured.Unstructured)
-	err := rc.cl.Post().
+	resourceName, subresourceName := rc.parseResourceSubresourceName()
+	req := rc.cl.Post().
 		NamespaceIfScoped(rc.ns, rc.resource.Namespaced).
-		Resource(rc.resource.Name).
-		Body(obj).
-		Do().
+		Resource(resourceName).
+		Body(obj)
+	if len(subresourceName) > 0 {
+		// If the provided resource is a subresource, the POST request should contain
+		// object name. Examples of subresources that support Create operation:
+		//	core/v1/pods/{name}/binding
+		//	core/v1/pods/{name}/eviction
+		//	extensions/v1beta1/deployments/{name}/rollback
+		//	apps/v1beta1/deployments/{name}/rollback
+		// NOTE: Currently our system assumes every subresource object has the same
+		//	 name as the parent resource object. E.g. a pods/binding object having
+		//	 metadada.name "foo" means pod "foo" is being bound. We may need to
+		//	 change this if we break the assumption in the future.
+		req = req.SubResource(subresourceName...).
+			Name(obj.GetName())
+	}
+	err := req.Do().
 		Into(result)
 	return result, err
 }
@@ -220,9 +250,15 @@ func (rc *ResourceClient) Update(obj *unstructured.Unstructured) (*unstructured.
 	if len(obj.GetName()) == 0 {
 		return result, errors.New("object missing name")
 	}
+	resourceName, subresourceName := rc.parseResourceSubresourceName()
 	err := rc.cl.Put().
 		NamespaceIfScoped(rc.ns, rc.resource.Namespaced).
-		Resource(rc.resource.Name).
+		Resource(resourceName).
+		SubResource(subresourceName...).
+		// NOTE: Currently our system assumes every subresource object has the same
+		//	 name as the parent resource object. E.g. a pods/binding object having
+		//	 metadada.name "foo" means pod "foo" is being bound. We may need to
+		//	 change this if we break the assumption in the future.
 		Name(obj.GetName()).
 		Body(obj).
 		Do().
@@ -246,9 +282,11 @@ func (rc *ResourceClient) Watch(opts metav1.ListOptions) (watch.Interface, error
 
 func (rc *ResourceClient) Patch(name string, pt types.PatchType, data []byte) (*unstructured.Unstructured, error) {
 	result := new(unstructured.Unstructured)
+	resourceName, subresourceName := rc.parseResourceSubresourceName()
 	err := rc.cl.Patch(pt).
 		NamespaceIfScoped(rc.ns, rc.resource.Namespaced).
-		Resource(rc.resource.Name).
+		Resource(resourceName).
+		SubResource(subresourceName...).
 		Name(name).
 		Body(data).
 		Do().