Skip to content

K
k3s
Commit da7d17c8 authored by Kubernetes Submit Queue's avatar Kubernetes Submit Queue Committed by GitHub
Browse files
Options

Merge pull request #39951 from shyamjvs/fix-kubemark-npd

Automatic merge from submit-queue (batch tested with PRs 40081, 39951) Passing correct master address to kubemark NPD & authenticating+authorizing it with apiserver Fixes #39245 Fixes https://github.com/kubernetes/node-problem-detector/issues/50 Added RBAC for npd and fixed issue with the npd falling back to inClusterConfig. cc @kubernetes/sig-scalability-misc @wojtek-t @gmarek
parents c40a668a cc78a3f4
Hide whitespace changes
Inline Side-by-side
Showing with 119 additions and 83 deletions
test/kubemark/resources/hollow-node_template.json
View file @ da7d17c8
...@@ -27,9 +27,9 @@ ...@@ -27,9 +27,9 @@
} }
}, },
{ {
"name": "npdconfig-volume", "name": "kernelmonitorconfig-volume",
"configMap": { "configMap": {
"name": "node-problem-detector-config" "name": "node-configmap"
} }
}, },
{ {
...@@ -43,6 +43,10 @@ ...@@ -43,6 +43,10 @@
"hostPath": { "hostPath": {
"path": "/var/log" "path": "/var/log"
} }
},
{
"name": "no-serviceaccount-access-to-real-master",
"emptyDir": {}
} }
], ],
"containers": [ "containers": [
...@@ -65,7 +69,7 @@ ...@@ -65,7 +69,7 @@
} }
}, },
{ {
"name": "MY_POD_NAME", "name": "NODE_NAME",
"valueFrom": { "valueFrom": {
"fieldRef": { "fieldRef": {
"fieldPath": "metadata.name" "fieldPath": "metadata.name"
...@@ -76,12 +80,13 @@ ...@@ -76,12 +80,13 @@
"command": [ "command": [
"/bin/sh", "/bin/sh",
"-c", "-c",
"./kubemark.sh --morph=kubelet --kubeconfig=/kubeconfig/kubelet.kubeconfig $(CONTENT_TYPE) --v=2 1>>/var/logs/kubelet_$(MY_POD_NAME).log 2>&1" "./kubemark.sh --morph=kubelet --name=$(NODE_NAME) --kubeconfig=/kubeconfig/kubelet.kubeconfig $(CONTENT_TYPE) --v=2 1>>/var/logs/kubelet_$(MY_POD_NAME).log 2>&1"
], ],
"volumeMounts": [ "volumeMounts": [
{ {
"name": "kubeconfig-volume", "name": "kubeconfig-volume",
"mountPath": "/kubeconfig" "mountPath": "/kubeconfig",
"readOnly": true
}, },
{ {
"name": "logs-volume", "name": "logs-volume",
...@@ -113,7 +118,7 @@ ...@@ -113,7 +118,7 @@
} }
}, },
{ {
"name": "MY_POD_NAME", "name": "NODE_NAME",
"valueFrom": { "valueFrom": {
"fieldRef": { "fieldRef": {
"fieldPath": "metadata.name" "fieldPath": "metadata.name"
...@@ -124,12 +129,13 @@ ...@@ -124,12 +129,13 @@
"command": [ "command": [
"/bin/sh", "/bin/sh",
"-c", "-c",
"./kubemark.sh --morph=proxy --kubeconfig=/kubeconfig/kubeproxy.kubeconfig $(CONTENT_TYPE) --v=2 1>>/var/logs/kube_proxy_$(MY_POD_NAME).log 2>&1" "./kubemark.sh --morph=proxy --name=$(NODE_NAME) --kubeconfig=/kubeconfig/kubeproxy.kubeconfig $(CONTENT_TYPE) --v=2 1>>/var/logs/kube_proxy_$(MY_POD_NAME).log 2>&1"
], ],
"volumeMounts": [ "volumeMounts": [
{ {
"name": "kubeconfig-volume", "name": "kubeconfig-volume",
"mountPath": "/kubeconfig" "mountPath": "/kubeconfig",
"readOnly": true
}, },
{ {
"name": "logs-volume", "name": "logs-volume",
...@@ -145,20 +151,32 @@ ...@@ -145,20 +151,32 @@
}, },
{ {
"name": "hollow-node-problem-detector", "name": "hollow-node-problem-detector",
"image": "gcr.io/google_containers/node-problem-detector:v0.2", "image": "gcr.io/google_containers/node-problem-detector:v0.3",
"env": [ "env": [
{ {
"name": "NODE_NAME", "name": "NODE_NAME",
"valueFrom": { "valueFrom": {
"fieldRef": { "fieldRef": {
"fieldPath": "spec.nodeName" "fieldPath": "metadata.name"
} }
} }
} }
], ],
"command": [
"/node-problem-detector",
"--kernel-monitor=/config/kernel.monitor",
"--apiserver-override=https://{{master_ip}}:443?inClusterConfig=false&auth=/kubeconfig/npd.kubeconfig",
"--alsologtostderr",
"1>>/var/logs/npd_$(NODE_NAME).log 2>&1"
],
"volumeMounts": [ "volumeMounts": [
{ {
"name": "npdconfig-volume", "name": "kubeconfig-volume",
"mountPath": "/kubeconfig",
"readOnly": true
},
{
"name": "kernelmonitorconfig-volume",
"mountPath": "/config", "mountPath": "/config",
"readOnly": true "readOnly": true
}, },
...@@ -166,6 +184,15 @@ ...@@ -166,6 +184,15 @@
"name": "kernellog-volume", "name": "kernellog-volume",
"mountPath": "/log", "mountPath": "/log",
"readOnly": true "readOnly": true
},
{
"name": "no-serviceaccount-access-to-real-master",
"mountPath": "/var/run/secrets/kubernetes.io/serviceaccount",
"readOnly": true
},
{
"name": "logs-volume",
"mountPath": "/var/logs"
} }
], ],
"resources": { "resources": {
......
test/kubemark/resources/manifests/addons/kubemark-rbac-bindings/npd-binding.yaml 0 → 100644
View file @ da7d17c8
# This is the role binding for the node-problem-detector.
apiVersion: rbac.authorization.k8s.io/v1alpha1
kind: ClusterRoleBinding
metadata:
name: node-problem-detector-binding
labels:
kubernetes.io/cluster-service: "true"
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:node-problem-detector
subjects:
- apiVersion: rbac/v1alpha1
kind: User
name: system:node-problem-detector
test/kubemark/start-kubemark.sh
View file @ da7d17c8
...@@ -51,35 +51,6 @@ EOF ...@@ -51,35 +51,6 @@ EOF
writeEnvironmentFile writeEnvironmentFile
MAKE_DIR="${KUBE_ROOT}/cluster/images/kubemark"
KUBEMARK_BIN="$(kube::util::find-binary-for-platform kubemark linux/amd64)"
if [[ -z "${KUBEMARK_BIN}" ]]; then
echo 'Cannot find cmd/kubemark binary'
exit 1
fi
echo "Copying kubemark to ${MAKE_DIR}"
cp "${KUBEMARK_BIN}" "${MAKE_DIR}"
CURR_DIR=`pwd`
cd "${MAKE_DIR}"
RETRIES=3
for attempt in $(seq 1 ${RETRIES}); do
if ! make; then
if [[ $((attempt)) -eq "${RETRIES}" ]]; then
echo "${color_red}Make failed. Exiting.${color_norm}"
exit 1
fi
echo -e "${color_yellow}Make attempt $(($attempt)) failed. Retrying.${color_norm}" >& 2
sleep $(($attempt * 5))
else
break
fi
done
rm kubemark
cd $CURR_DIR
GCLOUD_COMMON_ARGS="--project ${PROJECT} --zone ${ZONE}" GCLOUD_COMMON_ARGS="--project ${PROJECT} --zone ${ZONE}"
run-gcloud-compute-with-retries disks create "${MASTER_NAME}-pd" \ run-gcloud-compute-with-retries disks create "${MASTER_NAME}-pd" \
...@@ -134,10 +105,7 @@ create-certs ${MASTER_IP} ...@@ -134,10 +105,7 @@ create-certs ${MASTER_IP}
KUBELET_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) KUBELET_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
KUBE_PROXY_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) KUBE_PROXY_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
HEAPSTER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) HEAPSTER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
NODE_PROBLEM_DETECTOR_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
echo "${CA_CERT_BASE64}" | base64 --decode > "${RESOURCE_DIRECTORY}/ca.crt"
echo "${KUBECFG_CERT_BASE64}" | base64 --decode > "${RESOURCE_DIRECTORY}/kubecfg.crt"
echo "${KUBECFG_KEY_BASE64}" | base64 --decode > "${RESOURCE_DIRECTORY}/kubecfg.key"
until gcloud compute ssh --zone="${ZONE}" --project="${PROJECT}" "${MASTER_NAME}" --command="ls" &> /dev/null; do until gcloud compute ssh --zone="${ZONE}" --project="${PROJECT}" "${MASTER_NAME}" --command="ls" &> /dev/null; do
sleep 1 sleep 1
...@@ -156,6 +124,7 @@ run-gcloud-compute-with-retries ssh --zone="${ZONE}" --project="${PROJECT}" "${M ...@@ -156,6 +124,7 @@ run-gcloud-compute-with-retries ssh --zone="${ZONE}" --project="${PROJECT}" "${M
sudo bash -c \"echo \"${KUBELET_TOKEN},system:node:node-name,uid:kubelet,system:nodes\" >> /etc/srv/kubernetes/known_tokens.csv\" && \ sudo bash -c \"echo \"${KUBELET_TOKEN},system:node:node-name,uid:kubelet,system:nodes\" >> /etc/srv/kubernetes/known_tokens.csv\" && \
sudo bash -c \"echo \"${KUBE_PROXY_TOKEN},system:kube-proxy,uid:kube_proxy\" >> /etc/srv/kubernetes/known_tokens.csv\" && \ sudo bash -c \"echo \"${KUBE_PROXY_TOKEN},system:kube-proxy,uid:kube_proxy\" >> /etc/srv/kubernetes/known_tokens.csv\" && \
sudo bash -c \"echo \"${HEAPSTER_TOKEN},system:heapster,uid:heapster\" >> /etc/srv/kubernetes/known_tokens.csv\" && \ sudo bash -c \"echo \"${HEAPSTER_TOKEN},system:heapster,uid:heapster\" >> /etc/srv/kubernetes/known_tokens.csv\" && \
sudo bash -c \"echo \"${NODE_PROBLEM_DETECTOR_TOKEN},system:node-problem-detector,uid:system:node-problem-detector\" >> /etc/srv/kubernetes/known_tokens.csv\" && \
sudo bash -c \"echo ${password},admin,admin > /etc/srv/kubernetes/basic_auth.csv\"" sudo bash -c \"echo ${password},admin,admin > /etc/srv/kubernetes/basic_auth.csv\""
run-gcloud-compute-with-retries copy-files --zone="${ZONE}" --project="${PROJECT}" \ run-gcloud-compute-with-retries copy-files --zone="${ZONE}" --project="${PROJECT}" \
...@@ -177,6 +146,34 @@ gcloud compute ssh "${MASTER_NAME}" --zone="${ZONE}" --project="${PROJECT}" \ ...@@ -177,6 +146,34 @@ gcloud compute ssh "${MASTER_NAME}" --zone="${ZONE}" --project="${PROJECT}" \
sudo chmod a+x /home/kubernetes/start-kubemark-master.sh && \ sudo chmod a+x /home/kubernetes/start-kubemark-master.sh && \
sudo bash /home/kubernetes/start-kubemark-master.sh" sudo bash /home/kubernetes/start-kubemark-master.sh"
# Setup the docker image for kubemark hollow-node.
MAKE_DIR="${KUBE_ROOT}/cluster/images/kubemark"
KUBEMARK_BIN="$(kube::util::find-binary-for-platform kubemark linux/amd64)"
if [[ -z "${KUBEMARK_BIN}" ]]; then
echo 'Cannot find cmd/kubemark binary'
exit 1
fi
echo "Copying kubemark to ${MAKE_DIR}"
cp "${KUBEMARK_BIN}" "${MAKE_DIR}"
CURR_DIR=`pwd`
cd "${MAKE_DIR}"
RETRIES=3
for attempt in $(seq 1 ${RETRIES}); do
if ! make; then
if [[ $((attempt)) -eq "${RETRIES}" ]]; then
echo "${color_red}Make failed. Exiting.${color_norm}"
exit 1
fi
echo -e "${color_yellow}Make attempt $(($attempt)) failed. Retrying.${color_norm}" >& 2
sleep $(($attempt * 5))
else
break
fi
done
rm kubemark
cd $CURR_DIR
# Create kubeconfig for Kubelet. # Create kubeconfig for Kubelet.
KUBELET_KUBECONFIG_CONTENTS=$(echo "apiVersion: v1 KUBELET_KUBECONFIG_CONTENTS=$(echo "apiVersion: v1
kind: Config kind: Config
...@@ -195,7 +192,7 @@ contexts: ...@@ -195,7 +192,7 @@ contexts:
cluster: kubemark cluster: kubemark
user: kubelet user: kubelet
name: kubemark-context name: kubemark-context
current-context: kubemark-context" | base64 | tr -d "\n\r") current-context: kubemark-context")
# Create kubeconfig for Kubeproxy. # Create kubeconfig for Kubeproxy.
KUBEPROXY_KUBECONFIG_CONTENTS=$(echo "apiVersion: v1 KUBEPROXY_KUBECONFIG_CONTENTS=$(echo "apiVersion: v1
...@@ -214,7 +211,7 @@ contexts: ...@@ -214,7 +211,7 @@ contexts:
cluster: kubemark cluster: kubemark
user: kube-proxy user: kube-proxy
name: kubemark-context name: kubemark-context
current-context: kubemark-context" | base64 | tr -d "\n\r") current-context: kubemark-context")
# Create kubeconfig for Heapster. # Create kubeconfig for Heapster.
HEAPSTER_KUBECONFIG_CONTENTS=$(echo "apiVersion: v1 HEAPSTER_KUBECONFIG_CONTENTS=$(echo "apiVersion: v1
...@@ -233,39 +230,28 @@ contexts: ...@@ -233,39 +230,28 @@ contexts:
cluster: kubemark cluster: kubemark
user: heapster user: heapster
name: kubemark-context name: kubemark-context
current-context: kubemark-context" | base64 | tr -d "\n\r") current-context: kubemark-context")
KUBECONFIG_SECRET="${RESOURCE_DIRECTORY}/kubeconfig_secret.json"
cat > "${KUBECONFIG_SECRET}" << EOF
{
"apiVersion": "v1",
"kind": "Secret",
"metadata": {
"name": "kubeconfig"
},
"type": "Opaque",
"data": {
"kubelet.kubeconfig": "${KUBELET_KUBECONFIG_CONTENTS}",
"kubeproxy.kubeconfig": "${KUBEPROXY_KUBECONFIG_CONTENTS}",
"heapster.kubeconfig": "${HEAPSTER_KUBECONFIG_CONTENTS}"
}
}
EOF
NODE_CONFIGMAP="${RESOURCE_DIRECTORY}/node_config_map.json" # Create kubeconfig for NodeProblemDetector.
cat > "${NODE_CONFIGMAP}" << EOF NPD_KUBECONFIG_CONTENTS=$(echo "apiVersion: v1
{ kind: Config
"apiVersion": "v1", users:
"kind": "ConfigMap", - name: node-problem-detector
"metadata": { user:
"name": "node-configmap" token: ${NODE_PROBLEM_DETECTOR_TOKEN}
}, clusters:
"data": { - name: kubemark
"content.type": "${TEST_CLUSTER_API_CONTENT_TYPE}" cluster:
} insecure-skip-tls-verify: true
} server: https://${MASTER_IP}
EOF contexts:
- context:
cluster: kubemark
user: node-problem-detector
name: kubemark-npd-context
current-context: kubemark-npd-context")
# Create kubeconfig for local kubectl.
LOCAL_KUBECONFIG="${RESOURCE_DIRECTORY}/kubeconfig.kubemark" LOCAL_KUBECONFIG="${RESOURCE_DIRECTORY}/kubeconfig.kubemark"
cat > "${LOCAL_KUBECONFIG}" << EOF cat > "${LOCAL_KUBECONFIG}" << EOF
apiVersion: v1 apiVersion: v1
...@@ -292,6 +278,7 @@ EOF ...@@ -292,6 +278,7 @@ EOF
sed "s/{{numreplicas}}/${NUM_NODES:-10}/g" "${RESOURCE_DIRECTORY}/hollow-node_template.json" > "${RESOURCE_DIRECTORY}/hollow-node.json" sed "s/{{numreplicas}}/${NUM_NODES:-10}/g" "${RESOURCE_DIRECTORY}/hollow-node_template.json" > "${RESOURCE_DIRECTORY}/hollow-node.json"
sed -i'' -e "s/{{project}}/${PROJECT}/g" "${RESOURCE_DIRECTORY}/hollow-node.json" sed -i'' -e "s/{{project}}/${PROJECT}/g" "${RESOURCE_DIRECTORY}/hollow-node.json"
sed -i'' -e "s/{{master_ip}}/${MASTER_IP}/g" "${RESOURCE_DIRECTORY}/hollow-node.json"
mkdir "${RESOURCE_DIRECTORY}/addons" || true mkdir "${RESOURCE_DIRECTORY}/addons" || true
...@@ -303,16 +290,23 @@ eventer_mem_per_node=500 ...@@ -303,16 +290,23 @@ eventer_mem_per_node=500
eventer_mem=$((200 * 1024 + ${eventer_mem_per_node}*${NUM_NODES:-10})) eventer_mem=$((200 * 1024 + ${eventer_mem_per_node}*${NUM_NODES:-10}))
sed -i'' -e "s/{{EVENTER_MEM}}/${eventer_mem}/g" "${RESOURCE_DIRECTORY}/addons/heapster.json" sed -i'' -e "s/{{EVENTER_MEM}}/${eventer_mem}/g" "${RESOURCE_DIRECTORY}/addons/heapster.json"
# Create kubemark namespace.
"${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/kubemark-ns.json" "${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/kubemark-ns.json"
"${KUBECTL}" create -f "${KUBECONFIG_SECRET}" --namespace="kubemark" # Create configmap for configuring hollow- kubelet, proxy and npd.
"${KUBECTL}" create -f "${NODE_CONFIGMAP}" --namespace="kubemark" "${KUBECTL}" create configmap "node-configmap" --namespace="kubemark" \
--from-literal=content.type="${TEST_CLUSTER_API_CONTENT_TYPE}" \
--from-file=kernel.monitor="${RESOURCE_DIRECTORY}/kernel-monitor.json"
# Create secret for passing kubeconfigs to kubelet, kubeproxy and npd.
"${KUBECTL}" create secret generic "kubeconfig" --type=Opaque --namespace="kubemark" \
--from-literal=kubelet.kubeconfig="${KUBELET_KUBECONFIG_CONTENTS}" \
--from-literal=kubeproxy.kubeconfig="${KUBEPROXY_KUBECONFIG_CONTENTS}" \
--from-literal=heapster.kubeconfig="${HEAPSTER_KUBECONFIG_CONTENTS}" \
--from-literal=npd.kubeconfig="${NPD_KUBECONFIG_CONTENTS}"
# Create addon pods.
"${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/addons" --namespace="kubemark" "${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/addons" --namespace="kubemark"
"${KUBECTL}" create configmap node-problem-detector-config --from-file="${RESOURCE_DIRECTORY}/kernel-monitor.json" --namespace="kubemark" # Create the replication controller for hollow-nodes.
"${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/hollow-node.json" --namespace="kubemark" "${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/hollow-node.json" --namespace="kubemark"
rm "${KUBECONFIG_SECRET}"
rm "${NODE_CONFIGMAP}"
echo "Waiting for all HollowNodes to become Running..." echo "Waiting for all HollowNodes to become Running..."
start=$(date +%s) start=$(date +%s)
nodes=$("${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" get node 2> /dev/null) || true nodes=$("${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" get node 2> /dev/null) || true
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment