Add audit-logging, feature-gates & few admission plugins to kubemark

cluster/gce/config-test.sh

@@ -341,9 +341,6 @@ SOFTLOCKUP_PANIC="${SOFTLOCKUP_PANIC:-true}" # true, false
 
 # Enable a simple "AdvancedAuditing" setup for testing.
 ENABLE_APISERVER_ADVANCED_AUDIT="${ENABLE_APISERVER_ADVANCED_AUDIT:-true}" # true, false
-if [[ "${ENABLE_APISERVER_ADVANCED_AUDIT}" == "true" ]]; then
-  FEATURE_GATES="${FEATURE_GATES},AdvancedAuditing=true"
-fi
 
 ENABLE_BIG_CLUSTER_SUBNETS="${ENABLE_BIG_CLUSTER_SUBNETS:-false}"
 

cluster/gce/gci/configure-helper.sh

@@ -520,7 +520,7 @@ rules:
       - group: "" # core
         resources: ["endpoints", "services", "services/status"]
   - level: None
-    # Ingress controller reads `configmaps/ingress-uid` through the unsecured port.
+    # Ingress controller reads 'configmaps/ingress-uid' through the unsecured port.
     # TODO(#46983): Change this to the ingress controller service account.
     users: ["system:unsecured"]
     namespaces: ["kube-system"]

cluster/kubemark/gce/config-default.sh

@@ -31,29 +31,40 @@ MASTER_DISK_SIZE=${MASTER_DISK_SIZE:-20GB}
 MASTER_ROOT_DISK_SIZE=${KUBEMARK_MASTER_ROOT_DISK_SIZE:-10GB}
 REGISTER_MASTER_KUBELET=${REGISTER_MASTER:-false}
 PREEMPTIBLE_NODE=${PREEMPTIBLE_NODE:-false}
+NODE_ACCELERATORS=${NODE_ACCELERATORS:-""}
 
 MASTER_OS_DISTRIBUTION=${KUBE_MASTER_OS_DISTRIBUTION:-gci}
 NODE_OS_DISTRIBUTION=${KUBE_NODE_OS_DISTRIBUTION:-gci}
 MASTER_IMAGE=${KUBE_GCE_MASTER_IMAGE:-cos-stable-60-9592-90-0}
 MASTER_IMAGE_PROJECT=${KUBE_GCE_MASTER_PROJECT:-cos-cloud}
 
+# GPUs supported in GCE do not have compatible drivers in Debian 7.
+if [[ "${NODE_OS_DISTRIBUTION}" == "debian" ]]; then
+    NODE_ACCELERATORS=""
+fi
+
 NETWORK=${KUBE_GCE_NETWORK:-e2e}
 INSTANCE_PREFIX="${INSTANCE_PREFIX:-"default"}"
 MASTER_NAME="${INSTANCE_PREFIX}-kubemark-master"
 AGGREGATOR_MASTER_NAME="${INSTANCE_PREFIX}-kubemark-aggregator"
 MASTER_TAG="kubemark-master"
+ETCD_QUORUM_READ="${ENABLE_ETCD_QUORUM_READ:-false}"
 EVENT_STORE_NAME="${INSTANCE_PREFIX}-event-store"
 MASTER_IP_RANGE="${MASTER_IP_RANGE:-10.246.0.0/24}"
 CLUSTER_IP_RANGE="${CLUSTER_IP_RANGE:-10.224.0.0/11}"
 RUNTIME_CONFIG="${KUBE_RUNTIME_CONFIG:-}"
 TERMINATED_POD_GC_THRESHOLD=${TERMINATED_POD_GC_THRESHOLD:-100}
+KUBE_APISERVER_REQUEST_TIMEOUT=300
 
 # Set etcd image (e.g. 3.0.17-alpha.1) and version (e.g. 3.0.17) if you need
 # non-default version.
 ETCD_IMAGE="${TEST_ETCD_IMAGE:-}"
 ETCD_VERSION="${TEST_ETCD_VERSION:-}"
-# Storage backend. 'etcd2' supported, 'etcd3' experimental.
+
+# Storage backend. 'etcd2' and 'etcd3' are supported.
 STORAGE_BACKEND=${STORAGE_BACKEND:-}
+# Storage media type: application/json and application/vnd.kubernetes.protobuf are supported.
+STORAGE_MEDIA_TYPE=${STORAGE_MEDIA_TYPE:-}
 
 # Default Log level for all components in test clusters and variables to override it in specific components.
 TEST_CLUSTER_LOG_LEVEL="${TEST_CLUSTER_LOG_LEVEL:---v=2}"
@@ -67,23 +78,16 @@ TEST_CLUSTER_DELETE_COLLECTION_WORKERS="${TEST_CLUSTER_DELETE_COLLECTION_WORKERS
 TEST_CLUSTER_MAX_REQUESTS_INFLIGHT="${TEST_CLUSTER_MAX_REQUESTS_INFLIGHT:-}"
 TEST_CLUSTER_RESYNC_PERIOD="${TEST_CLUSTER_RESYNC_PERIOD:-}"
 
-KUBEMARK_MASTER_COMPONENTS_QPS_LIMITS="${KUBEMARK_MASTER_COMPONENTS_QPS_LIMITS:-}"
-
 # ContentType used by all components to communicate with apiserver.
 TEST_CLUSTER_API_CONTENT_TYPE="${TEST_CLUSTER_API_CONTENT_TYPE:-}"
-# ContentType used to store objects in underlying database.
-TEST_CLUSTER_STORAGE_MEDIA_TYPE=""
-if [ -n "${STORAGE_MEDIA_TYPE:-}" ]; then
-  TEST_CLUSTER_STORAGE_MEDIA_TYPE="--storage-media-type=${STORAGE_MEDIA_TYPE}"
-fi
 
-ENABLE_GARBAGE_COLLECTOR=${ENABLE_GARBAGE_COLLECTOR:-true}
+KUBEMARK_MASTER_COMPONENTS_QPS_LIMITS="${KUBEMARK_MASTER_COMPONENTS_QPS_LIMITS:-}"
 
-CUSTOM_ADMISSION_PLUGINS="${CUSTOM_ADMISSION_PLUGINS:-Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PodPreset,DefaultTolerationSeconds,NodeRestriction,ResourceQuota}"
+CUSTOM_ADMISSION_PLUGINS="${CUSTOM_ADMISSION_PLUGINS:-Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,PodPreset,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,Priority,ResourceQuota}"
 
 # Master components' test arguments.
-APISERVER_TEST_ARGS="${KUBEMARK_APISERVER_TEST_ARGS:-} --runtime-config=extensions/v1beta1 ${API_SERVER_TEST_LOG_LEVEL} ${TEST_CLUSTER_STORAGE_MEDIA_TYPE} ${TEST_CLUSTER_MAX_REQUESTS_INFLIGHT} ${TEST_CLUSTER_DELETE_COLLECTION_WORKERS} --enable-garbage-collector=${ENABLE_GARBAGE_COLLECTOR}"
-CONTROLLER_MANAGER_TEST_ARGS="${KUBEMARK_CONTROLLER_MANAGER_TEST_ARGS:-} ${CONTROLLER_MANAGER_TEST_LOG_LEVEL} ${TEST_CLUSTER_RESYNC_PERIOD} ${TEST_CLUSTER_API_CONTENT_TYPE} ${KUBEMARK_MASTER_COMPONENTS_QPS_LIMITS} --enable-garbage-collector=${ENABLE_GARBAGE_COLLECTOR}"
+APISERVER_TEST_ARGS="${KUBEMARK_APISERVER_TEST_ARGS:-} --runtime-config=extensions/v1beta1 ${API_SERVER_TEST_LOG_LEVEL} ${TEST_CLUSTER_MAX_REQUESTS_INFLIGHT} ${TEST_CLUSTER_DELETE_COLLECTION_WORKERS}"
+CONTROLLER_MANAGER_TEST_ARGS="${KUBEMARK_CONTROLLER_MANAGER_TEST_ARGS:-} ${CONTROLLER_MANAGER_TEST_LOG_LEVEL} ${TEST_CLUSTER_RESYNC_PERIOD} ${TEST_CLUSTER_API_CONTENT_TYPE} ${KUBEMARK_MASTER_COMPONENTS_QPS_LIMITS}"
 SCHEDULER_TEST_ARGS="${KUBEMARK_SCHEDULER_TEST_ARGS:-} ${SCHEDULER_TEST_LOG_LEVEL} ${TEST_CLUSTER_API_CONTENT_TYPE} ${KUBEMARK_MASTER_COMPONENTS_QPS_LIMITS}"
 
 # Hollow-node components' test arguments.
@@ -96,3 +100,19 @@ ALLOCATE_NODE_CIDRS=true
 
 # Optional: Enable cluster autoscaler.
 ENABLE_KUBEMARK_CLUSTER_AUTOSCALER="${ENABLE_KUBEMARK_CLUSTER_AUTOSCALER:-false}"
+
+# Optional: set feature gates
+FEATURE_GATES="${KUBE_FEATURE_GATES:-ExperimentalCriticalPodAnnotation=true}"
+
+if [[ ! -z "${NODE_ACCELERATORS}" ]]; then
+    FEATURE_GATES="${FEATURE_GATES},Accelerators=true"
+fi
+
+# Enable a simple "AdvancedAuditing" setup for testing.
+ENABLE_APISERVER_ADVANCED_AUDIT="${ENABLE_APISERVER_ADVANCED_AUDIT:-false}"
+
+# Optional: enable pod priority
+ENABLE_POD_PRIORITY="${ENABLE_POD_PRIORITY:-}"
+if [[ "${ENABLE_POD_PRIORITY}" == "true" ]]; then
+    FEATURE_GATES="${FEATURE_GATES},PodPriority=true"
+fi

test/kubemark/resources/manifests/kube-apiserver.yaml

@@ -36,6 +36,7 @@ spec:
       hostPort: 8080
       protocol: TCP
     volumeMounts:
+{{audit_policy_config_mount}}
     - name: srvkube
       mountPath: /etc/srv/kubernetes
       readOnly: true
@@ -50,6 +51,7 @@ spec:
     - name: srvsshproxy
       mountPath: /etc/srv/sshproxy
   volumes:
+{{audit_policy_config_volume}}
   - name: srvkube
     hostPath:
       path: /etc/srv/kubernetes

test/kubemark/resources/start-kubemark-master.sh

@@ -313,6 +313,156 @@ function setup-addon-manifests {
   chmod 644 "${dst_dir}"/*
 }
 
+# Write the config for the audit policy.
+# Note: This duplicates the function in cluster/gce/gci/configure-helper.sh.
+# TODO: Get rid of this function when #53321 is fixed.
+function create-master-audit-policy {
+  local -r path="${1}"
+  local -r policy="${2:-}"
+
+  if [[ -n "${policy}" ]]; then
+    echo "${policy}" > "${path}"
+    return
+  fi
+
+  # Known api groups
+  local -r known_apis='
+      - group: "" # core
+      - group: "admissionregistration.k8s.io"
+      - group: "apiextensions.k8s.io"
+      - group: "apiregistration.k8s.io"
+      - group: "apps"
+      - group: "authentication.k8s.io"
+      - group: "authorization.k8s.io"
+      - group: "autoscaling"
+      - group: "batch"
+      - group: "certificates.k8s.io"
+      - group: "extensions"
+      - group: "metrics"
+      - group: "networking.k8s.io"
+      - group: "policy"
+      - group: "rbac.authorization.k8s.io"
+      - group: "settings.k8s.io"
+      - group: "storage.k8s.io"'
+
+  cat <<EOF >"${path}"
+apiVersion: audit.k8s.io/v1beta1
+kind: Policy
+rules:
+  # The following requests were manually identified as high-volume and low-risk,
+  # so drop them.
+  - level: None
+    users: ["system:kube-proxy"]
+    verbs: ["watch"]
+    resources:
+      - group: "" # core
+        resources: ["endpoints", "services", "services/status"]
+  - level: None
+    # Ingress controller reads 'configmaps/ingress-uid' through the unsecured port.
+    # TODO(#46983): Change this to the ingress controller service account.
+    users: ["system:unsecured"]
+    namespaces: ["kube-system"]
+    verbs: ["get"]
+    resources:
+      - group: "" # core
+        resources: ["configmaps"]
+  - level: None
+    users: ["kubelet"] # legacy kubelet identity
+    verbs: ["get"]
+    resources:
+      - group: "" # core
+        resources: ["nodes", "nodes/status"]
+  - level: None
+    userGroups: ["system:nodes"]
+    verbs: ["get"]
+    resources:
+      - group: "" # core
+        resources: ["nodes", "nodes/status"]
+  - level: None
+    users:
+      - system:kube-controller-manager
+      - system:kube-scheduler
+      - system:serviceaccount:kube-system:endpoint-controller
+    verbs: ["get", "update"]
+    namespaces: ["kube-system"]
+    resources:
+      - group: "" # core
+        resources: ["endpoints"]
+  - level: None
+    users: ["system:apiserver"]
+    verbs: ["get"]
+    resources:
+      - group: "" # core
+        resources: ["namespaces", "namespaces/status", "namespaces/finalize"]
+  # Don't log HPA fetching metrics.
+  - level: None
+    users:
+      - system:kube-controller-manager
+    verbs: ["get", "list"]
+    resources:
+      - group: "metrics"
+  # Don't log these read-only URLs.
+  - level: None
+    nonResourceURLs:
+      - /healthz*
+      - /version
+      - /swagger*
+  # Don't log events requests.
+  - level: None
+    resources:
+      - group: "" # core
+        resources: ["events"]
+  # node and pod status calls from nodes are high-volume and can be large, don't log responses for expected updates from nodes
+  - level: Request
+    users: ["kubelet", "system:node-problem-detector", "system:serviceaccount:kube-system:node-problem-detector"]
+    verbs: ["update","patch"]
+    resources:
+      - group: "" # core
+        resources: ["nodes/status", "pods/status"]
+    omitStages:
+      - "RequestReceived"
+  - level: Request
+    userGroups: ["system:nodes"]
+    verbs: ["update","patch"]
+    resources:
+      - group: "" # core
+        resources: ["nodes/status", "pods/status"]
+    omitStages:
+      - "RequestReceived"
+  # deletecollection calls can be large, don't log responses for expected namespace deletions
+  - level: Request
+    users: ["system:serviceaccount:kube-system:namespace-controller"]
+    verbs: ["deletecollection"]
+    omitStages:
+      - "RequestReceived"
+  # Secrets, ConfigMaps, and TokenReviews can contain sensitive & binary data,
+  # so only log at the Metadata level.
+  - level: Metadata
+    resources:
+      - group: "" # core
+        resources: ["secrets", "configmaps"]
+      - group: authentication.k8s.io
+        resources: ["tokenreviews"]
+    omitStages:
+      - "RequestReceived"
+  # Get repsonses can be large; skip them.
+  - level: Request
+    verbs: ["get", "list", "watch"]
+    resources: ${known_apis}
+    omitStages:
+      - "RequestReceived"
+  # Default level for known APIs
+  - level: RequestResponse
+    resources: ${known_apis}
+    omitStages:
+      - "RequestReceived"
+  # Default level for all other requests.
+  - level: Metadata
+    omitStages:
+      - "RequestReceived"
+EOF
+}
+
 # Computes command line arguments to be passed to etcd.
 function compute-etcd-params {
 	local params="${ETCD_TEST_ARGS:-}"
@@ -360,10 +510,40 @@ function compute-kube-apiserver-params {
 	params+=" --secure-port=443"
 	params+=" --basic-auth-file=/etc/srv/kubernetes/basic_auth.csv"
 	params+=" --target-ram-mb=$((${NUM_NODES} * 60))"
-	params+=" --storage-backend=${STORAGE_BACKEND}"
 	params+=" --service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE}"
 	params+=" --admission-control=${CUSTOM_ADMISSION_PLUGINS}"
 	params+=" --authorization-mode=Node,RBAC"
+	params+=" --allow-privileged=true"
+	if [[ -n "${STORAGE_BACKEND:-}" ]]; then
+		params+=" --storage-backend=${STORAGE_BACKEND}"
+	fi
+	if [[ -n "${STORAGE_MEDIA_TYPE:-}" ]]; then
+		params+=" --storage-media-type=${STORAGE_MEDIA_TYPE}"
+	fi
+	if [[ -n "${ETCD_QUORUM_READ:-}" ]]; then
+		params+=" --etcd-quorum-read=${ETCD_QUORUM_READ}"
+	fi
+	if [[ -n "${KUBE_APISERVER_REQUEST_TIMEOUT:-}" ]]; then
+		params+=" --min-request-timeout=${KUBE_APISERVER_REQUEST_TIMEOUT}"
+	fi
+	if [[ -n "${RUNTIME_CONFIG:-}" ]]; then
+		params+=" --runtime-config=${RUNTIME_CONFIG}"
+	fi
+	if [[ -n "${FEATURE_GATES:-}" ]]; then
+		params+=" --feature-gates=${FEATURE_GATES}"
+	fi
+	if [[ "${ENABLE_APISERVER_ADVANCED_AUDIT:-}" == "true" ]]; then
+		# Create the audit policy file, and mount it into the apiserver pod.
+		create-master-audit-policy "${audit_policy_file}" "${ADVANCED_AUDIT_POLICY:-}"
+
+		# The config below matches the one in cluster/gce/gci/configure-helper.sh.
+		# TODO: Currently supporting just log backend. Support webhook if needed.
+		params+=" --audit-policy-file=${audit_policy_file}"
+		params+=" --audit-log-path=/var/log/kube-apiserver-audit.log"
+		params+=" --audit-log-maxage=0"
+		params+=" --audit-log-maxbackup=0"
+		params+=" --audit-log-maxsize=2000000000"
+	fi
 	echo "${params}"
 }
 
@@ -424,6 +604,25 @@ function start-kubemaster-component() {
 	else
 		local -r component_docker_tag=$(cat ${KUBE_BINDIR}/${component}.docker_tag)
 		sed -i -e "s@{{${component}_docker_tag}}@${component_docker_tag}@g" "${src_file}"
+		if [ "${component}" == "kube-apiserver" ]; then
+			local audit_policy_config_mount=""
+			local audit_policy_config_volume=""
+			if [[ "${ENABLE_APISERVER_ADVANCED_AUDIT:-}" == "true" ]]; then
+				read -d '' audit_policy_config_mount << EOF
+- name: auditpolicyconfigmount
+  mountPath: ${audit_policy_file}
+  readOnly: true
+EOF
+				read -d '' audit_policy_config_volume << EOF
+- name: auditpolicyconfigmount
+  hostPath:
+    path: ${audit_policy_file}
+    type: FileOrCreate
+EOF
+			fi
+			sed -i -e "s@{{audit_policy_config_mount}}@${audit_policy_config_mount}@g" "${src_file}"
+			sed -i -e "s@{{audit_policy_config_volume}}@${audit_policy_config_volume}@g" "${src_file}"
+		fi
 	fi
 	cp "${src_file}" /etc/kubernetes/manifests
 }
@@ -497,6 +696,8 @@ assemble-docker-flags
 DOCKER_REGISTRY="gcr.io/google_containers"
 load-docker-images
 
+readonly audit_policy_file="/etc/audit_policy.config"
+
 # Start kubelet as a supervisord process and master components as pods.
 start-kubelet
 start-kubemaster-component "etcd"

test/kubemark/start-kubemark.sh

@@ -68,9 +68,15 @@ SCHEDULER_TEST_ARGS="${SCHEDULER_TEST_ARGS:-}"
 
 # Apiserver related variables.
 APISERVER_TEST_ARGS="${APISERVER_TEST_ARGS:-}"
-STORAGE_BACKEND="${STORAGE_BACKEND:-}"
+STORAGE_MEDIA_TYPE="${STORAGE_MEDIA_TYPE:-}"
+STORAGE_BACKEND="${STORAGE_BACKEND:-etcd3}"
+ETCD_QUORUM_READ="${ETCD_QUORUM_READ:-}"
+RUNTIME_CONFIG="${RUNTIME_CONFIG:-}"
 NUM_NODES="${NUM_NODES:-}"
 CUSTOM_ADMISSION_PLUGINS="${CUSTOM_ADMISSION_PLUGINS:-}"
+FEATURE_GATES="${FEATURE_GATES:-}"
+KUBE_APISERVER_REQUEST_TIMEOUT="${KUBE_APISERVER_REQUEST_TIMEOUT:-}"
+ENABLE_APISERVER_ADVANCED_AUDIT="${ENABLE_APISERVER_ADVANCED_AUDIT:-}"
 EOF
   echo "Created the environment file for master."
 }