Merge pull request #50705 from MrHohn/kube-proxy-ds

Automatic merge from submit-queue (batch tested with PRs 50932, 49610, 51312, 51415, 50705) Allow running kube-proxy as a DaemonSet when using kube-up.sh on GCE **What this PR does / why we need it**: From #23225, this PR adds an option for user to run kube-proxy as a DaemonSet instead of static pods using GCE startup scripts. By default, kube-proxy will run as static pods. This is the first step for moving kube-proxy into a DaemonSet in GCE, remaining tasks will be tracked on #23225. **Special notes for your reviewer**: The last commit are purely for testing out kube-proxy as daemonset via CIs. cc @kubernetes/sig-network-misc @kubernetes/sig-cluster-lifecycle-misc **Release note**: ```release-note When using kube-up.sh on GCE, user could set env `KUBE_PROXY_DAEMONSET=true` to run kube-proxy as a DaemonSet. kube-proxy is run as static pods by default. ```

Merge pull request #50705 from MrHohn/kube-proxy-ds
04b3ab9a · Kubernetes Submit Queue · GitHub · 0b0b2a22 · 6d35b94f · 04b3ab9a
Commit 04b3ab9a authored Aug 29, 2017 by Kubernetes Submit Queue Committed by GitHub Aug 29, 2017
11 changed files
--- a/cluster/addons/kube-proxy/kube-proxy-ds.yaml
+++ b/cluster/addons/kube-proxy/kube-proxy-ds.yaml
+# Please keep kube-proxy configuration in-sync with:
+# cluster/saltbase/salt/kube-proxy/kube-proxy.manifest
+
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  labels:
+    k8s-app: kube-proxy
+    addonmanager.kubernetes.io/mode: Reconcile
+  name: kube-proxy
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      k8s-app: kube-proxy
+  updateStrategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 10%
+  template:
+    metadata:
+      labels:
+        k8s-app: kube-proxy
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
+    spec:
+      hostNetwork: true
+      nodeSelector:
+        beta.kubernetes.io/kube-proxy-ds-ready: "true"
+      initContainers:
+      - name: touch-lock
+        image: busybox
+        command: ['/bin/touch', '/run/xtables.lock']
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - mountPath: /run
+          name: run
+          readOnly: false
+      containers:
+      - name: kube-proxy
+        image: {{pillar['kube_docker_registry']}}/kube-proxy:{{pillar['kube-proxy_docker_tag']}}
+        resources:
+          requests:
+            cpu: {{ cpurequest }}
+        command:
+        - /bin/sh
+        - -c
+        - echo -998 > /proc/$$$/oom_score_adj && kube-proxy {{kubeconfig}} {{cluster_cidr}} --resource-container="" {{params}} 1>>/var/log/kube-proxy.log 2>&1
+        {{container_env}}
+        {{kube_cache_mutation_detector_env_name}}
+          {{kube_cache_mutation_detector_env_value}}
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - mountPath: /var/log
+          name: varlog
+          readOnly: false
+        - mountPath: /var/lib/kube-proxy/kubeconfig
+          name: kubeconfig
+          readOnly: false
+        - mountPath: /run/xtables.lock
+          name: xtables-lock
+          readOnly: false
+      volumes:
+      - name: varlog
+        hostPath:
+          path: /var/log
+      - name: kubeconfig
+        hostPath:
+          path: /var/lib/kube-proxy/kubeconfig
+      - name: xtables-lock
+        hostPath:
+          path: /run/xtables.lock
+      - name: run
+        hostPath:
+          path: /run
+      serviceAccountName: kube-proxy
--- a/cluster/addons/kube-proxy/kube-proxy-rbac.yaml
+++ b/cluster/addons/kube-proxy/kube-proxy-rbac.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kube-proxy
+  namespace: kube-system
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: system:kube-proxy
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile
+subjects:
+  - kind: ServiceAccount
+    name: kube-proxy
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: system:node-proxier
+  apiGroup: rbac.authorization.k8s.io
--- a/cluster/common.sh
+++ b/cluster/common.sh
@@ -638,6 +638,7 @@ DNS_SERVER_IP: $(yaml-quote ${DNS_SERVER_IP:-})
 DNS_DOMAIN: $(yaml-quote ${DNS_DOMAIN:-})
 ENABLE_DNS_HORIZONTAL_AUTOSCALER: $(yaml-quote ${ENABLE_DNS_HORIZONTAL_AUTOSCALER:-false})
 KUBELET_TOKEN: $(yaml-quote ${KUBELET_TOKEN:-})
+KUBE_PROXY_DAEMONSET: $(yaml-quote ${KUBE_PROXY_DAEMONSET:-false})
 KUBE_PROXY_TOKEN: $(yaml-quote ${KUBE_PROXY_TOKEN:-})
 NODE_PROBLEM_DETECTOR_TOKEN: $(yaml-quote ${NODE_PROBLEM_DETECTOR_TOKEN:-})
 ADMISSION_CONTROL: $(yaml-quote ${ADMISSION_CONTROL:-})

--- a/cluster/gce/config-default.sh
+++ b/cluster/gce/config-default.sh
@@ -297,3 +297,7 @@ PROVIDER_VARS="${PROVIDER_VARS:-} FLUENTD_GCP_MEMORY_LIMIT FLUENTD_GCP_CPU_REQUE
 PROMETHEUS_TO_SD_ENDPOINT="${PROMETHEUS_TO_SD_ENDPOINT:-https://monitoring.googleapis.com/}"
 PROMETHEUS_TO_SD_PREFIX="${PROMETHEUS_TO_SD_PREFIX:-custom.googleapis.com}"
 ENABLE_PROMETHEUS_TO_SD="${ENABLE_PROMETHEUS_TO_SD:-false}"
+
+# TODO(#51292): Make kube-proxy Daemonset default and remove the configuration here.
+# Optional: Run kube-proxy as a DaemonSet if set to true, run as static pods otherwise.
+KUBE_PROXY_DAEMONSET="${KUBE_PROXY_DAEMONSET:-false}" # true, false
--- a/cluster/gce/config-test.sh
+++ b/cluster/gce/config-test.sh
@@ -356,3 +356,7 @@ PROVIDER_VARS="${PROVIDER_VARS:-} FLUENTD_GCP_MEMORY_LIMIT FLUENTD_GCP_CPU_REQUE
 PROMETHEUS_TO_SD_ENDPOINT="${PROMETHEUS_TO_SD_ENDPOINT:-https://monitoring.googleapis.com/}"
 PROMETHEUS_TO_SD_PREFIX="${PROMETHEUS_TO_SD_PREFIX:-custom.googleapis.com}"
 ENABLE_PROMETHEUS_TO_SD="${ENABLE_PROMETHEUS_TO_SD:-true}"
+
+# TODO(#51292): Make kube-proxy Daemonset default and remove the configuration here.
+# Optional: Run kube-proxy as a DaemonSet if set to true, run as static pods otherwise.
+KUBE_PROXY_DAEMONSET="${KUBE_PROXY_DAEMONSET:-false}" # true, false
--- a/cluster/gce/configure-vm.sh
+++ b/cluster/gce/configure-vm.sh
@@ -448,6 +448,7 @@ initial_etcd_cluster_state: '$(echo "${INITIAL_ETCD_CLUSTER_STATE:-}" | sed -e "
 ca_cert_bundle_path: '$(echo "${CA_CERT_BUNDLE_PATH:-}" | sed -e "s/'/''/g")'
 hostname: $(hostname -s)
 enable_default_storage_class: '$(echo "$ENABLE_DEFAULT_STORAGE_CLASS" | sed -e "s/'/''/g")'
+kube_proxy_daemonset: '$(echo "$KUBE_PROXY_DAEMONSET" | sed -e "s/'/''/g")'
 EOF
    if [ -n "${STORAGE_BACKEND:-}" ]; then
      cat <<EOF >>/srv/salt-overlay/pillar/cluster-params.sls
@@ -652,14 +653,15 @@ EOF

 # This should happen both on cluster initialization and node upgrades.
 #
-#  - Uses the CA_CERT and KUBE_PROXY_TOKEN to generate a kubeconfig file for
-#    the kube-proxy to securely connect to the apiserver.
+#  - When run as static pods, use the CA_CERT and KUBE_PROXY_TOKEN to generate a
+#    kubeconfig file for the kube-proxy to securely connect to the apiserver.
+#  - When run as a daemonset, generate a kubeconfig file specific to service account.
 function create-salt-kubeproxy-auth() {
  local -r kube_proxy_kubeconfig_file="/srv/salt-overlay/salt/kube-proxy/kubeconfig"
+  local kubeconfig_content=""
  if [ ! -e "${kube_proxy_kubeconfig_file}" ]; then
-    mkdir -p /srv/salt-overlay/salt/kube-proxy
-    (umask 077;
-        cat > "${kube_proxy_kubeconfig_file}" <<EOF
+    if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]]; then
+      kubeconfig_content="\
 apiVersion: v1
 kind: Config
 users:
@@ -675,7 +677,33 @@ contexts:
    cluster: local
    user: kube-proxy
  name: service-account-context
-current-context: service-account-context
+current-context: service-account-context"
+    else
+      # Generate kubeconfig specific to service account.
+      kubeconfig_content="\
+apiVersion: v1
+kind: Config
+clusters:
+- cluster:
+    certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+    server: https://${KUBERNETES_MASTER_NAME}
+  name: default
+contexts:
+- context:
+    cluster: default
+    namespace: default
+    user: default
+  name: default
+current-context: default
+users:
+- name: default
+  user:
+    tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token"
+    fi
+    mkdir -p /srv/salt-overlay/salt/kube-proxy
+    (umask 077;
+        cat > "${kube_proxy_kubeconfig_file}" <<EOF
+${kubeconfig_content}
 EOF
 )
  fi

--- a/cluster/gce/container-linux/configure-helper.sh
+++ b/cluster/gce/container-linux/configure-helper.sh
@@ -393,8 +393,8 @@ function create-master-kubelet-auth {
  fi
 }

-function create-kubeproxy-kubeconfig {
-  echo "Creating kube-proxy kubeconfig file"
+function create-kubeproxy-user-kubeconfig {
+  echo "Creating kube-proxy user kubeconfig file"
  cat <<EOF >/var/lib/kube-proxy/kubeconfig
 apiVersion: v1
 kind: Config
@@ -415,6 +415,30 @@ current-context: service-account-context
 EOF
 }

+function create-kubeproxy-serviceaccount-kubeconfig {
+  echo "Creating kube-proxy serviceaccount kubeconfig file"
+  cat <<EOF >/var/lib/kube-proxy/kubeconfig
+apiVersion: v1
+kind: Config
+clusters:
+- cluster:
+    certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+    server: https://${KUBERNETES_MASTER_NAME}
+  name: default
+contexts:
+- context:
+    cluster: default
+    namespace: default
+    user: default
+  name: default
+current-context: default
+users:
+- name: default
+  user:
+    tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+EOF
+}
+
 function create-kubecontrollermanager-kubeconfig {
  echo "Creating kube-controller-manager kubeconfig file"
  mkdir -p /etc/srv/kubernetes/kube-controller-manager
@@ -619,8 +643,17 @@ function start-kubelet {
  if [[ -n "${ENABLE_CUSTOM_METRICS:-}" ]]; then
    flags+=" --enable-custom-metrics=${ENABLE_CUSTOM_METRICS}"
  fi
+  local node_labels=""
+  if [[ "${KUBE_PROXY_DAEMONSET:-}" == "true" && "${KUBERNETES_MASTER:-}" != "true" ]]; then
+    # Add kube-proxy daemonset label to node to avoid situation during cluster
+    # upgrade/downgrade when there are two instances of kube-proxy running on a node.
+    node_labels="beta.kubernetes.io/kube-proxy-ds-ready=true"
+  fi
  if [[ -n "${NODE_LABELS:-}" ]]; then
-    flags+=" --node-labels=${NODE_LABELS}"
+    node_labels="${node_labels:+${node_labels},}${NODE_LABELS}"
+  fi
+  if [[ -n "${node_labels:-}" ]]; then
+    flags+=" --node-labels=${node_labels}"
  fi
  if [[ -n "${NODE_TAINTS:-}" ]]; then
    flags+=" --register-with-taints=${NODE_TAINTS}"
@@ -672,11 +705,11 @@ function prepare-log-file {
  chown root:root $1
 }

-# Starts kube-proxy pod.
-function start-kube-proxy {
-  echo "Start kube-proxy pod"
-  prepare-log-file /var/log/kube-proxy.log
-  local -r src_file="${KUBE_HOME}/kube-manifests/kubernetes/kube-proxy.manifest"
+# Prepares parameters for kube-proxy manifest.
+# $1 source path of kube-proxy manifest.
+function prepare-kube-proxy-manifest-variables {
+  local -r src_file=$1;
+
  remove-salt-config-comments "${src_file}"

  local -r kubeconfig="--kubeconfig=/var/lib/kube-proxy/kubeconfig"
@@ -695,14 +728,20 @@ function start-kube-proxy {
    params+=" ${KUBEPROXY_TEST_ARGS}"
  fi
  local container_env=""
+  local kube_cache_mutation_detector_env_name=""
+  local kube_cache_mutation_detector_env_value=""
  if [[ -n "${ENABLE_CACHE_MUTATION_DETECTOR:-}" ]]; then
-    container_env="env:\n    - name: KUBE_CACHE_MUTATION_DETECTOR\n    value: \"${ENABLE_CACHE_MUTATION_DETECTOR}\""
+    container_env="env:"
+    kube_cache_mutation_detector_env_name="- name: KUBE_CACHE_MUTATION_DETECTOR"
+    kube_cache_mutation_detector_env_value="value: \"${ENABLE_CACHE_MUTATION_DETECTOR}\""
  fi
  sed -i -e "s@{{kubeconfig}}@${kubeconfig}@g" ${src_file}
  sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${kube_docker_registry}@g" ${src_file}
  sed -i -e "s@{{pillar\['kube-proxy_docker_tag'\]}}@${kube_proxy_docker_tag}@g" ${src_file}
  sed -i -e "s@{{params}}@${params}@g" ${src_file}
  sed -i -e "s@{{container_env}}@${container_env}@g" ${src_file}
+  sed -i -e "s@{{kube_cache_mutation_detector_env_name}}@${kube_cache_mutation_detector_env_name}@g" ${src_file}
+  sed -i -e "s@{{kube_cache_mutation_detector_env_value}}@${kube_cache_mutation_detector_env_value}@g" ${src_file}
  sed -i -e "s@{{ cpurequest }}@100m@g" ${src_file}
  sed -i -e "s@{{api_servers_with_port}}@${api_servers}@g" ${src_file}
  if [[ -n "${CLUSTER_IP_RANGE:-}" ]]; then
@@ -719,6 +758,14 @@ function start-kube-proxy {
      mount -o remount,rw /sys; "
    sed -i -e "s@-\\s\\+kube-proxy@- ${extra_workaround_cmd} kube-proxy@g" "${src_file}"
  fi
+}
+
+# Starts kube-proxy static pod.
+function start-kube-proxy {
+  echo "Start kube-proxy static pod"
+  prepare-log-file /var/log/kube-proxy.log
+  local -r src_file="${KUBE_HOME}/kube-manifests/kubernetes/kube-proxy.manifest"
+  prepare-kube-proxy-manifest-variables "$src_file"

  cp "${src_file}" /etc/kubernetes/manifests
 }
@@ -1223,6 +1270,10 @@ function start-kube-addons {
  setup-addon-manifests "addons" "rbac"

  # Set up manifests of other addons.
+  if [[ "${KUBE_PROXY_DAEMONSET:-}" == "true" ]]; then
+    prepare-kube-proxy-manifest-variables "$src_dir/kube-proxy/kube-proxy-ds.yaml"
+    setup-addon-manifests "addons" "kube-proxy"
+  fi
  if [[ "${ENABLE_CLUSTER_MONITORING:-}" == "influxdb" ]] || \
     [[ "${ENABLE_CLUSTER_MONITORING:-}" == "google" ]] || \
     [[ "${ENABLE_CLUSTER_MONITORING:-}" == "stackdriver" ]] || \
@@ -1470,7 +1521,11 @@ if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then
  create-master-etcd-auth
 else
  create-kubelet-kubeconfig "https://${KUBERNETES_MASTER_NAME}"
-  create-kubeproxy-kubeconfig
+  if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]]; then
+    create-kubeproxy-user-kubeconfig
+  else
+    create-kubeproxy-serviceaccount-kubeconfig
+  fi
 fi

 if [[ "${CONTAINER_RUNTIME:-}" == "rkt" ]]; then
@@ -1498,7 +1553,9 @@ if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then
  start-lb-controller
  start-rescheduler
 else
-  start-kube-proxy
+  if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]]; then
+    start-kube-proxy
+  fi
  # Kube-registry-proxy.
  if [[ "${ENABLE_CLUSTER_REGISTRY:-}" == "true" ]]; then
    start-kube-registry-proxy

--- a/cluster/gce/gci/configure-helper.sh
+++ b/cluster/gce/gci/configure-helper.sh
@@ -80,7 +80,6 @@ function get-calico-typha-cpu {
  echo "${typha_cpu}"
 }

-
 function config-ip-firewall {
  echo "Configuring IP firewall rules"
  # The GCI image has host firewall which drop most inbound/forwarded packets.
@@ -681,8 +680,8 @@ function create-master-kubelet-auth {
  fi
 }

-function create-kubeproxy-kubeconfig {
-  echo "Creating kube-proxy kubeconfig file"
+function create-kubeproxy-user-kubeconfig {
+  echo "Creating kube-proxy user kubeconfig file"
  cat <<EOF >/var/lib/kube-proxy/kubeconfig
 apiVersion: v1
 kind: Config
@@ -703,6 +702,30 @@ current-context: service-account-context
 EOF
 }

+function create-kubeproxy-serviceaccount-kubeconfig {
+  echo "Creating kube-proxy serviceaccount kubeconfig file"
+  cat <<EOF >/var/lib/kube-proxy/kubeconfig
+apiVersion: v1
+kind: Config
+clusters:
+- cluster:
+    certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+    server: https://${KUBERNETES_MASTER_NAME}
+  name: default
+contexts:
+- context:
+    cluster: default
+    namespace: default
+    user: default
+  name: default
+current-context: default
+users:
+- name: default
+  user:
+    tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+EOF
+}
+
 function create-kubecontrollermanager-kubeconfig {
  echo "Creating kube-controller-manager kubeconfig file"
  mkdir -p /etc/srv/kubernetes/kube-controller-manager
@@ -927,8 +950,17 @@ function start-kubelet {
  if [[ -n "${ENABLE_CUSTOM_METRICS:-}" ]]; then
    flags+=" --enable-custom-metrics=${ENABLE_CUSTOM_METRICS}"
  fi
+  local node_labels=""
+  if [[ "${KUBE_PROXY_DAEMONSET:-}" == "true" && "${KUBERNETES_MASTER:-}" != "true" ]]; then
+    # Add kube-proxy daemonset label to node to avoid situation during cluster
+    # upgrade/downgrade when there are two instances of kube-proxy running on a node.
+    node_labels="beta.kubernetes.io/kube-proxy-ds-ready=true"
+  fi
  if [[ -n "${NODE_LABELS:-}" ]]; then
-    flags+=" --node-labels=${NODE_LABELS}"
+    node_labels="${node_labels:+${node_labels},}${NODE_LABELS}"
+  fi
+  if [[ -n "${node_labels:-}" ]]; then
+    flags+=" --node-labels=${node_labels}"
  fi
  if [[ -n "${NODE_TAINTS:-}" ]]; then
    flags+=" --register-with-taints=${NODE_TAINTS}"
@@ -1009,11 +1041,11 @@ function prepare-log-file {
  chown root:root $1
 }

-# Starts kube-proxy pod.
-function start-kube-proxy {
-  echo "Start kube-proxy pod"
-  prepare-log-file /var/log/kube-proxy.log
-  local -r src_file="${KUBE_HOME}/kube-manifests/kubernetes/kube-proxy.manifest"
+# Prepares parameters for kube-proxy manifest.
+# $1 source path of kube-proxy manifest.
+function prepare-kube-proxy-manifest-variables {
+  local -r src_file=$1;
+
  remove-salt-config-comments "${src_file}"

  local -r kubeconfig="--kubeconfig=/var/lib/kube-proxy/kubeconfig"
@@ -1032,19 +1064,34 @@ function start-kube-proxy {
    params+=" ${KUBEPROXY_TEST_ARGS}"
  fi
  local container_env=""
+  local kube_cache_mutation_detector_env_name=""
+  local kube_cache_mutation_detector_env_value=""
  if [[ -n "${ENABLE_CACHE_MUTATION_DETECTOR:-}" ]]; then
-    container_env="env:\n    - name: KUBE_CACHE_MUTATION_DETECTOR\n    value: \"${ENABLE_CACHE_MUTATION_DETECTOR}\""
+    container_env="env:"
+    kube_cache_mutation_detector_env_name="- name: KUBE_CACHE_MUTATION_DETECTOR"
+    kube_cache_mutation_detector_env_value="value: \"${ENABLE_CACHE_MUTATION_DETECTOR}\""
  fi
  sed -i -e "s@{{kubeconfig}}@${kubeconfig}@g" ${src_file}
  sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${kube_docker_registry}@g" ${src_file}
  sed -i -e "s@{{pillar\['kube-proxy_docker_tag'\]}}@${kube_proxy_docker_tag}@g" ${src_file}
  sed -i -e "s@{{params}}@${params}@g" ${src_file}
  sed -i -e "s@{{container_env}}@${container_env}@g" ${src_file}
+  sed -i -e "s@{{kube_cache_mutation_detector_env_name}}@${kube_cache_mutation_detector_env_name}@g" ${src_file}
+  sed -i -e "s@{{kube_cache_mutation_detector_env_value}}@${kube_cache_mutation_detector_env_value}@g" ${src_file}
  sed -i -e "s@{{ cpurequest }}@100m@g" ${src_file}
  sed -i -e "s@{{api_servers_with_port}}@${api_servers}@g" ${src_file}
  if [[ -n "${CLUSTER_IP_RANGE:-}" ]]; then
    sed -i -e "s@{{cluster_cidr}}@--cluster-cidr=${CLUSTER_IP_RANGE}@g" ${src_file}
  fi
+}
+
+# Starts kube-proxy static pod.
+function start-kube-proxy {
+  echo "Start kube-proxy static pod"
+  prepare-log-file /var/log/kube-proxy.log
+  local -r src_file="${KUBE_HOME}/kube-manifests/kubernetes/kube-proxy.manifest"
+  prepare-kube-proxy-manifest-variables "${src_file}"
+
  cp "${src_file}" /etc/kubernetes/manifests
 }

@@ -1647,6 +1694,10 @@ function start-kube-addons {
  setup-addon-manifests "addons" "rbac"

  # Set up manifests of other addons.
+  if [[ "${KUBE_PROXY_DAEMONSET:-}" == "true" ]]; then
+    prepare-kube-proxy-manifest-variables "$src_dir/kube-proxy/kube-proxy-ds.yaml"
+    setup-addon-manifests "addons" "kube-proxy"
+  fi
  if [[ "${ENABLE_CLUSTER_MONITORING:-}" == "influxdb" ]] || \
     [[ "${ENABLE_CLUSTER_MONITORING:-}" == "google" ]] || \
     [[ "${ENABLE_CLUSTER_MONITORING:-}" == "stackdriver" ]] || \
@@ -1895,7 +1946,11 @@ if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then
 else
  create-node-pki
  create-kubelet-kubeconfig ${KUBERNETES_MASTER_NAME}
-  create-kubeproxy-kubeconfig
+  if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]]; then
+    create-kubeproxy-user-kubeconfig
+  else
+    create-kubeproxy-serviceaccount-kubeconfig
+  fi
  if [[ "${ENABLE_NODE_PROBLEM_DETECTOR:-}" == "standalone" ]]; then
    create-node-problem-detector-kubeconfig
  fi
@@ -1918,7 +1973,9 @@ if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then
  start-lb-controller
  start-rescheduler
 else
-  start-kube-proxy
+  if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]]; then
+    start-kube-proxy
+  fi
  # Kube-registry-proxy.
  if [[ "${ENABLE_CLUSTER_REGISTRY:-}" == "true" ]]; then
    start-kube-registry-proxy

--- a/cluster/saltbase/salt/kube-proxy/init.sls
+++ b/cluster/saltbase/salt/kube-proxy/init.sls
@@ -7,6 +7,7 @@
    - makedirs: true

 # kube-proxy in a static pod
+{% if pillar.get('kube_proxy_daemonset', '').lower() != 'true' %}
 /etc/kubernetes/manifests/kube-proxy.manifest:
  file.managed:
    - source: salt://kube-proxy/kube-proxy.manifest
@@ -24,6 +25,7 @@
    - require:
      - service: docker
      - service: kubelet
+{% endif %}

 /var/log/kube-proxy.log:
  file.managed:

--- a/cluster/saltbase/salt/kube-proxy/kube-proxy.manifest
+++ b/cluster/saltbase/salt/kube-proxy/kube-proxy.manifest
+# Please keep kube-proxy configuration in-sync with:
+# cluster/addons/kube-proxy/kube-proxy-ds.yaml
+
 {% set kubeconfig = "--kubeconfig=/var/lib/kube-proxy/kubeconfig" -%}
 {% if grains.api_servers is defined -%}
  {% set api_servers = "--master=https://" + grains.api_servers -%}
@@ -35,6 +38,8 @@
 {% set params = log_level + " " + throttles + " " + feature_gates + " " + test_args -%}

 {% set container_env = "" -%}
+{% set kube_cache_mutation_detector_env_name = "" -%}
+{% set kube_cache_mutation_detector_env_value = "" -%}

 # kube-proxy podspec
 apiVersion: v1
@@ -75,6 +80,8 @@ spec:
    - -c
    - echo -998 > /proc/$$$/oom_score_adj && kube-proxy {{api_servers_with_port}} {{kubeconfig}} {{cluster_cidr}} --resource-container="" {{params}} 1>>/var/log/kube-proxy.log 2>&1
    {{container_env}}
+    {{kube_cache_mutation_detector_env_name}}
+      {{kube_cache_mutation_detector_env_value}}
    securityContext:
      privileged: true
    volumeMounts:

--- a/cluster/saltbase/salt/kubelet/default
+++ b/cluster/saltbase/salt/kubelet/default
@@ -164,10 +164,16 @@
  {% set enable_custom_metrics="--enable-custom-metrics=" + pillar['enable_custom_metrics'] %}
 {% endif -%}

-{% set node_labels = "" %}
-{% if pillar['node_labels'] is defined -%}
-  {% set node_labels="--node-labels=" + pillar['node_labels'] %}
-{% endif -%}
+{% set kube_proxy_ds_label = "" %}
+{% if grains['roles'][0] != 'kubernetes-master' and pillar.get('kube_proxy_daemonset', '').lower() == 'true' %}
+  # Add kube-proxy daemonset label to node to avoid situation during cluster
+  # upgrade/downgrade when there are two instances of kube-proxy running on a node.
+  {% set kube_proxy_ds_label = "beta.kubernetes.io/kube-proxy-ds-ready=true," %}
+{% endif %}
+{% set node_labels = kube_proxy_ds_label + pillar['node_labels'] %}
+{% if node_labels != "" %}
+  {% set node_labels="--node-labels=" + node_labels %}
+{% endif %}

 {% set node_taints = "" %}
 {% if pillar['node_taints'] is defined -%}