Skip to content

K
k3s
Commit 1eb221e9 authored by k8s-merge-robot's avatar k8s-merge-robot
Browse files
Options

Merge pull request #25561 from andyzheng0831/webhook

Automatic merge from submit-queue Configuration for GCP webhook authentication and authorization This PR adds configuration for GCP webhook authentication and authorization in ContainerVM and GCI. The change of configure-vm.sh and kube-apiserver.manifest is directly copied from @cjcullen's PR #25380 and #25296. The change in GCI script configure-helper.sh includes the support for webhook authentication and authorization, and also some code refactor to improve readability. @cjcullen @roberthbailey @zmerlynn please review it. The original PRs are P1, please mark this as P1. cc/ @fabioy @kubernetes/goog-image FYI. I verified it by running e2e tests on GCI cluster. Without the GCI side change, cluster creation fails as being capture by GKE Jenkins tests. I don't test when the two env GCP_AUTHN_URL and GCP_AUTHZ_URL are set, because they are only set in GKE. After this PR is merged, @cjcullen will test in GKE.
parents 529135dd 9811009d
Hide whitespace changes
Inline Side-by-side
Showing with 153 additions and 12 deletions
cluster/gce/configure-vm.sh
View file @ 1eb221e9
......@@ -798,14 +798,60 @@ EOF
CLOUD_CONFIG=/etc/gce.conf
fi
if [[ -n ${CLOUD_CONFIG:-} ]]; then
cat <<EOF >>/etc/salt/minion.d/grains.conf
if [[ -n "${CLOUD_CONFIG:-}" ]]; then
cat <<EOF >>/etc/salt/minion.d/grains.conf
cloud_config: ${CLOUD_CONFIG}
EOF
else
rm -f /etc/gce.conf
fi
if [[ -n "${GCP_AUTHN_URL:-}" ]]; then
cat <<EOF >>/etc/salt/minion.d/grains.conf
webhook_authentication_config: /etc/gcp_authn.config
EOF
cat <<EOF >/etc/gcp_authn.config
clusters:
- name: gcp-authentication-server
cluster:
server: ${GCP_AUTHN_URL}
users:
- name: kube-apiserver
user:
auth-provider:
name: gcp
current-context: webhook
contexts:
- context:
cluster: gcp-authentication-server
user: kube-apiserver
name: webhook
EOF
fi
if [[ -n "${GCP_AUTHZ_URL:-}" ]]; then
cat <<EOF >>/etc/salt/minion.d/grains.conf
webhook_authorization_config: /etc/gcp_authz.config
EOF
cat <<EOF >/etc/gcp_authz.config
clusters:
- name: gcp-authorization-server
cluster:
server: ${GCP_AUTHZ_URL}
users:
- name: kube-apiserver
user:
auth-provider:
name: gcp
current-context: webhook
contexts:
- context:
cluster: gcp-authorization-server
user: kube-apiserver
name: webhook
EOF
fi
# If the kubelet on the master is enabled, give it the same CIDR range
# as a generic node.
if [[ ! -z "${KUBELET_APISERVER:-}" ]] && [[ ! -z "${KUBELET_CERT:-}" ]] && [[ ! -z "${KUBELET_KEY:-}" ]]; then
......
cluster/gce/trusty/configure-helper.sh
View file @ 1eb221e9
......@@ -316,6 +316,46 @@ EOF
multizone = ${MULTIZONE}
EOF
fi
if [ -n "${GCP_AUTHN_URL:-}" ]; then
cat <<EOF >/etc/gcp_authn.config
clusters:
- name: gcp-authentication-server
cluster:
server: ${GCP_AUTHN_URL}
users:
- name: kube-apiserver
user:
auth-provider:
name: gcp
current-context: webhook
contexts:
- context:
cluster: gcp-authentication-server
user: kube-apiserver
name: webhook
EOF
fi
if [ -n "${GCP_AUTHZ_URL:-}" ]; then
cat <<EOF >/etc/gcp_authz.config
clusters:
- name: gcp-authorization-server
cluster:
server: ${GCP_AUTHZ_URL}
users:
- name: kube-apiserver
user:
auth-provider:
name: gcp
current-context: webhook
contexts:
- context:
cluster: gcp-authorization-server
user: kube-apiserver
name: webhook
EOF
fi
}
# Uses KUBELET_CA_CERT (falling back to CA_CERT), KUBELET_CERT, and KUBELET_KEY
......@@ -413,7 +453,20 @@ start_kube_apiserver() {
timeout 30 docker load -i /home/kubernetes/kube-docker-files/kube-apiserver.tar
# Calculate variables and assemble the command line.
params="--cloud-provider=gce --address=127.0.0.1 --etcd-servers=http://127.0.0.1:4001 --tls-cert-file=/etc/srv/kubernetes/server.cert --tls-private-key-file=/etc/srv/kubernetes/server.key --secure-port=443 --client-ca-file=/etc/srv/kubernetes/ca.crt --token-auth-file=/etc/srv/kubernetes/known_tokens.csv --basic-auth-file=/etc/srv/kubernetes/basic_auth.csv --allow-privileged=true --authorization-mode=ABAC --authorization-policy-file=/etc/srv/kubernetes/abac-authz-policy.jsonl --etcd-servers-overrides=/events#http://127.0.0.1:4002 ${APISERVER_TEST_ARGS:-}"
params="${APISERVER_TEST_ARGS:-} ${API_SERVER_TEST_LOG_LEVEL:-"--v=2"}"
params="${params} --cloud-provider=gce"
params="${params} --address=127.0.0.1"
params="${params} --etcd-servers=http://127.0.0.1:4001"
params="${params} --tls-cert-file=/etc/srv/kubernetes/server.cert"
params="${params} --tls-private-key-file=/etc/srv/kubernetes/server.key"
params="${params} --secure-port=443"
params="${params} --client-ca-file=/etc/srv/kubernetes/ca.crt"
params="${params} --token-auth-file=/etc/srv/kubernetes/known_tokens.csv"
params="${params} --basic-auth-file=/etc/srv/kubernetes/basic_auth.csv"
params="${params} --allow-privileged=true"
params="${params} --authorization-policy-file=/etc/srv/kubernetes/abac-authz-policy.jsonl"
params="${params} --etcd-servers-overrides=/events#http://127.0.0.1:4002"
if [ -n "${SERVICE_CLUSTER_IP_RANGE:-}" ]; then
params="${params} --service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE}"
fi
......@@ -426,18 +479,32 @@ start_kube_apiserver() {
if [ -n "${RUNTIME_CONFIG:-}" ]; then
params="${params} --runtime-config=${RUNTIME_CONFIG}"
fi
log_level="--v=2"
if [ -n "${API_SERVER_TEST_LOG_LEVEL:-}" ]; then
log_level="${API_SERVER_TEST_LOG_LEVEL}"
fi
params="${params} ${log_level}"
if [ -n "${PROJECT_ID:-}" ] && [ -n "${TOKEN_URL:-}" ] && [ -n "${TOKEN_BODY:-}" ] && [ -n "${NODE_NETWORK:-}" ]; then
readonly vm_external_ip=$(curl --fail --silent -H 'Metadata-Flavor: Google' "http://metadata/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip")
params="${params} --cloud-config=/etc/gce.conf --advertise-address=${vm_external_ip} --ssh-user=${PROXY_SSH_USER} --ssh-keyfile=/etc/srv/sshproxy/.sshkeyfile"
readonly vm_external_ip=$(curl --retry 5 --retry-delay 3 --fail --silent -H 'Metadata-Flavor: Google' "http://metadata/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip")
params="${params} --cloud-config=/etc/gce.conf"
params="${params} --advertise-address=${vm_external_ip}"
params="${params} --ssh-user=${PROXY_SSH_USER}"
params="${params} --ssh-keyfile=/etc/srv/sshproxy/.sshkeyfile"
fi
readonly kube_apiserver_docker_tag=$(cat /home/kubernetes/kube-docker-files/kube-apiserver.docker_tag)
webhook_authn_config_mount=""
webhook_authn_config_volume=""
if [ -n "${GCP_AUTHN_URL:-}" ]; then
params="${params} --authentication-token-webhook-config-file=/etc/gcp_authn.config"
webhook_authn_config_mount="{\"name\": \"webhookauthnconfigmount\",\"mountPath\": \"/etc/gcp_authn.config\", \"readOnly\": false},"
webhook_authn_config_volume="{\"name\": \"webhookauthnconfigmount\",\"hostPath\": {\"path\": \"/etc/gcp_authn.config\"}},"
fi
params="${params} --authorization-mode=ABAC"
webhook_config_mount=""
webhook_config_volume=""
if [ -n "${GCP_AUTHZ_URL:-}" ]; then
params="${params},Webhook --authorization-webhook-config-file=/etc/gcp_authz.config"
webhook_config_mount="{\"name\": \"webhookconfigmount\",\"mountPath\": \"/etc/gcp_authz.config\", \"readOnly\": false},"
webhook_config_volume="{\"name\": \"webhookconfigmount\",\"hostPath\": {\"path\": \"/etc/gcp_authz.config\"}},"
fi
src_dir="/home/kubernetes/kube-manifests/kubernetes/gci-trusty"
cp "${src_dir}/abac-authz-policy.jsonl" /etc/srv/kubernetes/
src_file="${src_dir}/kube-apiserver.manifest"
......@@ -455,6 +522,11 @@ start_kube_apiserver() {
sed -i -e "s@{{secure_port}}@8080@g" "${src_file}"
sed -i -e "s@{{additional_cloud_config_mount}}@@g" "${src_file}"
sed -i -e "s@{{additional_cloud_config_volume}}@@g" "${src_file}"
sed -i -e "s@{{webhook_authn_config_mount}}@${webhook_authn_config_mount}@g" "${src_file}"
sed -i -e "s@{{webhook_authn_config_volume}}@${webhook_authn_config_volume}@g" "${src_file}"
sed -i -e "s@{{webhook_config_mount}}@${webhook_config_mount}@g" "${src_file}"
sed -i -e "s@{{webhook_config_volume}}@${webhook_config_volume}@g" "${src_file}"
cp "${src_file}" /etc/kubernetes/manifests
}
......
cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest
View file @ 1eb221e9
......@@ -83,6 +83,25 @@
{% set abac_policy_file = " --authorization-policy-file=/srv/kubernetes/abac-authz-policy.jsonl" -%}
{% endif -%}
{% set webhook_authentication_config = "" -%}
{% set webhook_authn_config_mount = "" -%}
{% set webhook_authn_config_volume = "" -%}
{% if grains.webhook_authentication_config is defined -%}
{% set webhook_authentication_config = " --authentication-token-webhook-config-file=" + grains.webhook_authentication_config -%}
{% set webhook_authn_config_mount = "{\"name\": \"webhookauthnconfigmount\",\"mountPath\": \"" + grains.webhook_authentication_config + "\", \"readOnly\": false}," -%}
{% set webhook_authn_config_volume = "{\"name\": \"webhookauthnconfigmount\",\"hostPath\": {\"path\": \"" + grains.webhook_authentication_config + "\"}}," -%}
{% endif -%}
{% set webhook_authorization_config = "" -%}
{% set webhook_config_mount = "" -%}
{% set webhook_config_volume = "" -%}
{% if grains.webhook_authorization_config is defined -%}
{% set webhook_authorization_config = " --authorization-webhook-config-file=" + grains.webhook_authorization_config -%}
{% set webhook_config_mount = "{\"name\": \"webhookconfigmount\",\"mountPath\": \"" + grains.webhook_authorization_config + "\", \"readOnly\": false}," -%}
{% set webhook_config_volume = "{\"name\": \"webhookconfigmount\",\"hostPath\": {\"path\": \"" + grains.webhook_authorization_config + "\"}}," -%}
{% set authz_mode = authz_mode + ",Webhook" -%}
{% endif -%}
{% set admission_control = "" -%}
{% if pillar['admission_control'] is defined -%}
{% set admission_control = "--admission-control=" + pillar['admission_control'] -%}
......@@ -99,7 +118,7 @@
{% endif -%}
{% set params = address + " " + etcd_servers + " " + etcd_servers_overrides + " " + cloud_provider + " " + cloud_config + " " + runtime_config + " " + admission_control + " " + service_cluster_ip_range + " " + client_ca_file + basic_auth_file + " " + min_request_timeout -%}
{% set params = params + " " + cert_file + " " + key_file + " --secure-port=" + secure_port + token_auth_file + " " + bind_address + " " + log_level + " " + advertise_address + " " + proxy_ssh_options + authz_mode + abac_policy_file -%}
{% set params = params + " " + cert_file + " " + key_file + " --secure-port=" + secure_port + token_auth_file + " " + bind_address + " " + log_level + " " + advertise_address + " " + proxy_ssh_options + authz_mode + abac_policy_file + webhook_authentication_config + webhook_authorization_config -%}
# test_args has to be kept at the end, so they'll overwrite any prior configuration
{% if pillar['apiserver_test_args'] is defined -%}
......@@ -153,6 +172,8 @@
"volumeMounts": [
{{cloud_config_mount}}
{{additional_cloud_config_mount}}
{{webhook_config_mount}}
{{webhook_authn_config_mount}}
{ "name": "srvkube",
"mountPath": "{{srv_kube_path}}",
"readOnly": true},
......@@ -180,6 +201,8 @@
"volumes":[
{{cloud_config_volume}}
{{additional_cloud_config_volume}}
{{webhook_config_volume}}
{{webhook_authn_config_volume}}
{ "name": "srvkube",
"hostPath": {
"path": "{{srv_kube_path}}"}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment