Merge pull request #77140 from dekkagaijin/glbc

cluster/addons/rbac/cluster-loadbalancing/glbc/roles.yaml

+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: system:controller:glbc
+  namespace: kube-system
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile
+rules:
+- apiGroups: [""]
+  resources: ["configmaps"]
+  verbs: ["get", "list", "watch", "update", "create", "patch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: system:controller:glbc
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile
+rules:
+- apiGroups: [""]
+  resources: ["secrets"]
+  verbs: ["get"]
+- apiGroups: [""]
+  resources: ["events"]
+  verbs: ["get", "list", "watch", "update", "create", "patch"]
+- apiGroups: [""]
+  resources: ["endpoints", "services", "pods", "nodes", "namespaces"]
+  verbs: ["get", "list", "watch"]
+# TODO: switch to patch services/status
+# https://github.com/kubernetes/ingress-gce/blob/4918eb2f0f484f09ac9e5a975907a9b16ed2b344/pkg/neg/controller.go#L339-L342
+# https://github.com/kubernetes/ingress-gce/blob/4918eb2f0f484f09ac9e5a975907a9b16ed2b344/pkg/neg/controller.go#L359-L361
+- apiGroups: [""]
+  resources: ["services"]
+  verbs: ["update", "patch"]
+- apiGroups: ["extensions", "networking.k8s.io"]
+  resources: ["ingresses"]
+  verbs: ["get", "list", "watch"]
+# For now, GLBC annotates ingress resources with various state and statuses: 
+# https://github.com/kubernetes/ingress-gce/blob/50d49b077d9ab4362a02fae05f94e433cd3f08dc/pkg/controller/controller.go#L579
+# TODO(rramkumar1): Remove unnecessary `update` permission once statuses are propagated through `ingresses/status`
+- apiGroups: ["extensions", "networking.k8s.io"]
+  resources: ["ingresses"]
+  verbs: ["update"]
+- apiGroups: ["extensions", "networking.k8s.io"]
+  resources: ["ingresses/status"]
+  verbs: ["update"]
+# GLBC ensures that the `cloud.google.com/backendconfigs` CRD exists in a desired state:
+# https://github.com/kubernetes/ingress-gce/blob/4918eb2f0f484f09ac9e5a975907a9b16ed2b344/cmd/glbc/main.go#L93
+# TODO(rramkumar1): https://github.com/kubernetes/ingress-gce/issues/744
+- apiGroups: ["apiextensions.k8s.io"]
+  resources: ["customresourcedefinitions"]
+  verbs: ["get", "list", "watch", "update", "create", "patch"]
+- apiGroups: ["cloud.google.com"]
+  resources: ["backendconfigs"]
+  verbs: ["get", "list", "watch", "update", "create", "patch"]

cluster/addons/rbac/cluster-loadbalancing/glbc/user-rolebindings.yaml

+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: system:controller:glbc
+  namespace: kube-system
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: system:controller:glbc
+subjects:
+- kind: User
+  name: system:controller:glbc
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: system:controller:glbc
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:controller:glbc
+subjects:
+- kind: User
+  name: system:controller:glbc

cluster/gce/gci/configure-helper.sh

@@ -594,6 +594,9 @@ function create-master-auth {
   if [[ -n "${NODE_PROBLEM_DETECTOR_TOKEN:-}" ]]; then
     append_or_replace_prefixed_line "${known_tokens_csv}" "${NODE_PROBLEM_DETECTOR_TOKEN},"   "system:node-problem-detector,uid:node-problem-detector"
   fi
+  if [[ -n "${GCE_GLBC_TOKEN:-}" ]]; then
+    append_or_replace_prefixed_line "${known_tokens_csv}" "${GCE_GLBC_TOKEN},"                "system:controller:glbc,uid:system:controller:glbc"
+  fi
   local use_cloud_config="false"
   cat <<EOF >/etc/gce.conf
 [global]
@@ -1048,6 +1051,30 @@ current-context: service-account-context
 EOF
 }
 
+function create-l7-lb-controller-kubeconfig {
+  echo "Creating l7-lb-controller kubeconfig file"
+  mkdir -p /etc/srv/kubernetes/l7-lb-controller
+  cat <<EOF >/etc/srv/kubernetes/l7-lb-controller/kubeconfig
+apiVersion: v1
+kind: Config
+users:
+- name: l7-lb-controller
+  user:
+    token: ${GCE_GLBC_TOKEN}
+clusters:
+- name: local
+  cluster:
+    insecure-skip-tls-verify: true
+    server: https://localhost:443
+contexts:
+- context:
+    cluster: local
+    user: l7-lb-controller
+  name: l7-lb-controller
+current-context: l7-lb-controller
+EOF
+}
+
 function create-kubescheduler-kubeconfig {
   echo "Creating kube-scheduler kubeconfig file"
   mkdir -p /etc/srv/kubernetes/kube-scheduler
@@ -2737,6 +2764,8 @@ function start-lb-controller {
     echo "Start GCE L7 pod"
     prepare-log-file /var/log/glbc.log
     setup-addon-manifests "addons" "cluster-loadbalancing/glbc"
+    setup-addon-manifests "addons" "rbac/cluster-loadbalancing/glbc"
+    create-l7-lb-controller-kubeconfig
 
     local -r src_manifest="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/glbc.manifest"
     local -r dest_manifest="/etc/kubernetes/manifests/glbc.manifest"
@@ -2906,6 +2935,9 @@ function main() {
   KUBE_CONTROLLER_MANAGER_TOKEN="$(secure_random 32)"
   KUBE_SCHEDULER_TOKEN="$(secure_random 32)"
   KUBE_CLUSTER_AUTOSCALER_TOKEN="$(secure_random 32)"
+  if [[ "${ENABLE_L7_LOADBALANCING:-}" == "glbc" ]]; then
+    GCE_GLBC_TOKEN="$(secure_random 32)"
+  fi
 
   setup-os-params
   config-ip-firewall

cluster/gce/manifests/glbc.manifest

@@ -34,6 +34,9 @@ spec:
     - mountPath: /var/log/glbc.log
       name: logfile
       readOnly: false
+    - name: srvkube
+      mountPath: /etc/srv/kubernetes/l7-lb-controller
+      readOnly: true
     resources:
       # Request is set to accommodate this pod alongside the other
       # master components on a single core master.
@@ -45,7 +48,7 @@ spec:
     # TODO: split this out into args when we no longer need to pipe stdout to a file #6428
     - sh
     - -c
-    - 'exec /glbc --gce-ratelimit=ga.Operations.Get,qps,10,100 --gce-ratelimit=alpha.Operations.Get,qps,10,100 --gce-ratelimit=ga.BackendServices.Get,qps,1.8,1 --gce-ratelimit=ga.HealthChecks.Get,qps,1.8,1 --gce-ratelimit=alpha.HealthChecks.Get,qps,1.8,1 --gce-ratelimit=beta.NetworkEndpointGroups.Get,qps,1.8,1 --gce-ratelimit=beta.NetworkEndpointGroups.AttachNetworkEndpoints,qps,1.8,1 --gce-ratelimit=beta.NetworkEndpointGroups.DetachNetworkEndpoints,qps,1.8,1 --gce-ratelimit=beta.NetworkEndpointGroups.ListNetworkEndpoints,qps,1.8,1 --verbose --apiserver-host=http://localhost:8080 --default-backend-service=kube-system/default-http-backend --sync-period=600s --running-in-cluster=false --use-real-cloud=true --config-file-path=/etc/gce.conf --healthz-port=8086 1>>/var/log/glbc.log 2>&1'
+    - 'exec /glbc --gce-ratelimit=ga.Operations.Get,qps,10,100 --gce-ratelimit=alpha.Operations.Get,qps,10,100 --gce-ratelimit=ga.BackendServices.Get,qps,1.8,1 --gce-ratelimit=ga.HealthChecks.Get,qps,1.8,1 --gce-ratelimit=alpha.HealthChecks.Get,qps,1.8,1 --gce-ratelimit=beta.NetworkEndpointGroups.Get,qps,1.8,1 --gce-ratelimit=beta.NetworkEndpointGroups.AttachNetworkEndpoints,qps,1.8,1 --gce-ratelimit=beta.NetworkEndpointGroups.DetachNetworkEndpoints,qps,1.8,1 --gce-ratelimit=beta.NetworkEndpointGroups.ListNetworkEndpoints,qps,1.8,1 --verbose --kubeconfig=/etc/srv/kubernetes/l7-lb-controller/kubeconfig --default-backend-service=kube-system/default-http-backend --sync-period=600s --running-in-cluster=false --use-real-cloud=true --config-file-path=/etc/gce.conf --healthz-port=8086 1>>/var/log/glbc.log 2>&1'
   volumes:
   - hostPath:
       path: /etc/gce.conf
@@ -55,3 +58,6 @@ spec:
       path: /var/log/glbc.log
       type: FileOrCreate
     name: logfile
+  - name: srvkube
+    hostPath:
+      path: /etc/srv/kubernetes/l7-lb-controller