Skip to content

K
k3s
Commit 32d59cbb authored by Kubernetes Submit Queue's avatar Kubernetes Submit Queue Committed by GitHub
Browse files
Options

Merge pull request #42201 from shyamjvs/inotify-limit

Automatic merge from submit-queue (batch tested with PRs 42316, 41618, 42201, 42113, 42191) [Kubemark] Add init container in hollow node for setting inotify limit of node to 200 Fixes #41713 Along with adding the init container, I also changed the manifest to a yaml as otherwise the entire init container annotation would have to be in a single line (with escaped characters), as json doesn't allow multi-line strings. cc @kubernetes/sig-scalability-misc @wojtek-t @gmarek @Random-Liu
parents 44a7be98 75e602ca
Hide whitespace changes
Inline Side-by-side
Showing with 133 additions and 205 deletions
test/kubemark/resources/hollow-node_template.json deleted 100644 → 0
View file @ 44a7be98
{
"kind": "ReplicationController",
"apiVersion": "v1",
"metadata": {
"name": "hollow-node",
"labels": {
"name": "hollow-node"
}
},
"spec": {
"replicas": {{numreplicas}},
"selector": {
"name": "hollow-node"
},
"template": {
"metadata": {
"labels": {
"name": "hollow-node"
}
},
"spec": {
"volumes": [
{
"name": "kubeconfig-volume",
"secret": {
"secretName": "kubeconfig"
}
},
{
"name": "kernelmonitorconfig-volume",
"configMap": {
"name": "node-configmap"
}
},
{
"name": "logs-volume",
"hostPath": {
"path": "/var/log"
}
},
{
"name": "no-serviceaccount-access-to-real-master",
"emptyDir": {}
}
],
"containers": [
{
"name": "hollow-kubelet",
"image": "{{registry}}/{{project}}/kubemark:latest",
"ports": [
{"containerPort": 4194},
{"containerPort": 10250},
{"containerPort": 10255}
],
"env": [
{
"name": "CONTENT_TYPE",
"valueFrom": {
"configMapKeyRef": {
"name": "node-configmap",
"key": "content.type"
}
}
},
{
"name": "NODE_NAME",
"valueFrom": {
"fieldRef": {
"fieldPath": "metadata.name"
}
}
}
],
"command": [
"/bin/sh",
"-c",
"./kubemark.sh --morph=kubelet --name=$(NODE_NAME) --kubeconfig=/kubeconfig/kubelet.kubeconfig $(CONTENT_TYPE) --v=2 1>>/var/log/kubelet-$(NODE_NAME).log 2>&1"
],
"volumeMounts": [
{
"name": "kubeconfig-volume",
"mountPath": "/kubeconfig",
"readOnly": true
},
{
"name": "logs-volume",
"mountPath": "/var/log"
}
],
"resources": {
"requests": {
"cpu": "50m",
"memory": "100M"
}
},
"securityContext": {
"privileged": true
}
},
{
"name": "hollow-proxy",
"image": "{{registry}}/{{project}}/kubemark:latest",
"env": [
{
"name": "CONTENT_TYPE",
"valueFrom": {
"configMapKeyRef": {
"name": "node-configmap",
"key": "content.type"
}
}
},
{
"name": "NODE_NAME",
"valueFrom": {
"fieldRef": {
"fieldPath": "metadata.name"
}
}
}
],
"command": [
"/bin/sh",
"-c",
"./kubemark.sh --morph=proxy --name=$(NODE_NAME) --kubeconfig=/kubeconfig/kubeproxy.kubeconfig $(CONTENT_TYPE) --v=2 1>>/var/log/kubeproxy-$(NODE_NAME).log 2>&1"
],
"volumeMounts": [
{
"name": "kubeconfig-volume",
"mountPath": "/kubeconfig",
"readOnly": true
},
{
"name": "logs-volume",
"mountPath": "/var/log"
}
],
"resources": {
"requests": {
"cpu": "{{HOLLOW_PROXY_CPU}}m",
"memory": "{{HOLLOW_PROXY_MEM}}Ki"
}
}
},
{
"name": "hollow-node-problem-detector",
"image": "gcr.io/google_containers/node-problem-detector:v0.3.0-alpha.0",
"env": [
{
"name": "NODE_NAME",
"valueFrom": {
"fieldRef": {
"fieldPath": "metadata.name"
}
}
}
],
"command": [
"/bin/sh",
"-c",
"/node-problem-detector --kernel-monitor=/config/kernel.monitor --apiserver-override=\"https://{{master_ip}}:443?inClusterConfig=false&auth=/kubeconfig/npd.kubeconfig\" --alsologtostderr 1>>/var/log/npd-$(NODE_NAME).log 2>&1"
],
"volumeMounts": [
{
"name": "kubeconfig-volume",
"mountPath": "/kubeconfig",
"readOnly": true
},
{
"name": "kernelmonitorconfig-volume",
"mountPath": "/config",
"readOnly": true
},
{
"name": "no-serviceaccount-access-to-real-master",
"mountPath": "/var/run/secrets/kubernetes.io/serviceaccount",
"readOnly": true
},
{
"name": "logs-volume",
"mountPath": "/var/log"
}
],
"resources": {
"requests": {
"cpu": "20m",
"memory": "20Mi"
}
},
"securityContext": {
"privileged": true
}
}]
}
}
}
}
test/kubemark/resources/hollow-node_template.yaml 0 → 100644
View file @ 32d59cbb
apiVersion: v1
kind: ReplicationController
metadata:
name: hollow-node
labels:
name: hollow-node
spec:
replicas: {{numreplicas}}
selector:
name: hollow-node
template:
metadata:
labels:
name: hollow-node
annotations:
pod.beta.kubernetes.io/init-containers: '[
{
"name": "init-inotify-limit",
"image": "busybox",
"command": ["sysctl", "-w", "fs.inotify.max_user_instances=200"],
"securityContext": {
"privileged": true
}
}
]'
spec:
volumes:
- name: kubeconfig-volume
secret:
secretName: kubeconfig
- name: kernelmonitorconfig-volume
configMap:
name: node-configmap
- name: logs-volume
hostPath:
path: /var/log
- name: no-serviceaccount-access-to-real-master
emptyDir: {}
containers:
- name: hollow-kubelet
image: {{registry}}/{{project}}/kubemark:latest
ports:
- containerPort: 4194
- containerPort: 10250
- containerPort: 10255
env:
- name: CONTENT_TYPE
valueFrom:
configMapKeyRef:
name: node-configmap
key: content.type
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
command:
- /bin/sh
- -c
- ./kubemark.sh --morph=kubelet --name=$(NODE_NAME) --kubeconfig=/kubeconfig/kubelet.kubeconfig $(CONTENT_TYPE) --v=2 1>>/var/log/kubelet-$(NODE_NAME).log 2>&1
volumeMounts:
- name: kubeconfig-volume
mountPath: /kubeconfig
readOnly: true
- name: logs-volume
mountPath: /var/log
resources:
requests:
cpu: 50m
memory: 100M
securityContext:
privileged: true
- name: hollow-proxy
image: {{registry}}/{{project}}/kubemark:latest
env:
- name: CONTENT_TYPE
valueFrom:
configMapKeyRef:
name: node-configmap
key: content.type
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
command:
- /bin/sh
- -c
- ./kubemark.sh --morph=proxy --name=$(NODE_NAME) --kubeconfig=/kubeconfig/kubeproxy.kubeconfig $(CONTENT_TYPE) --v=2 1>>/var/log/kubeproxy-$(NODE_NAME).log 2>&1
volumeMounts:
- name: kubeconfig-volume
mountPath: /kubeconfig
readOnly: true
- name: logs-volume
mountPath: /var/log
resources:
requests:
cpu: {{HOLLOW_PROXY_CPU}}m
memory: {{HOLLOW_PROXY_MEM}}Ki
- name: hollow-node-problem-detector
image: gcr.io/google_containers/node-problem-detector:v0.3.0-alpha.0
env:
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
command:
- /bin/sh
- -c
- /node-problem-detector --kernel-monitor=/config/kernel.monitor --apiserver-override="https://{{master_ip}}:443?inClusterConfig=false&auth=/kubeconfig/npd.kubeconfig" --alsologtostderr 1>>/var/log/npd-$(NODE_NAME).log 2>&1
volumeMounts:
- name: kubeconfig-volume
mountPath: /kubeconfig
readOnly: true
- name: kernelmonitorconfig-volume
mountPath: /config
readOnly: true
- name: no-serviceaccount-access-to-real-master
mountPath: /var/run/secrets/kubernetes.io/serviceaccount
readOnly: true
- name: logs-volume
mountPath: /var/log
resources:
requests:
cpu: 20m
memory: 20Mi
securityContext:
privileged: true
test/kubemark/start-kubemark.sh
View file @ 32d59cbb
...@@ -308,19 +308,19 @@ current-context: kubemark-context") ...@@ -308,19 +308,19 @@ current-context: kubemark-context")
"${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/addons" --namespace="kubemark" "${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/addons" --namespace="kubemark"
# Create the replication controller for hollow-nodes. # Create the replication controller for hollow-nodes.
sed "s/{{numreplicas}}/${NUM_NODES:-10}/g" "${RESOURCE_DIRECTORY}/hollow-node_template.json" > "${RESOURCE_DIRECTORY}/hollow-node.json" sed "s/{{numreplicas}}/${NUM_NODES:-10}/g" "${RESOURCE_DIRECTORY}/hollow-node_template.yaml" > "${RESOURCE_DIRECTORY}/hollow-node.yaml"
proxy_cpu=20 proxy_cpu=20
if [ "${NUM_NODES:-10}" -gt 1000 ]; then if [ "${NUM_NODES:-10}" -gt 1000 ]; then
proxy_cpu=50 proxy_cpu=50
fi fi
proxy_mem_per_node=100 proxy_mem_per_node=100
proxy_mem=$((100 * 1024 + ${proxy_mem_per_node}*${NUM_NODES:-10})) proxy_mem=$((100 * 1024 + ${proxy_mem_per_node}*${NUM_NODES:-10}))
sed -i'' -e "s/{{HOLLOW_PROXY_CPU}}/${proxy_cpu}/g" "${RESOURCE_DIRECTORY}/hollow-node.json" sed -i'' -e "s/{{HOLLOW_PROXY_CPU}}/${proxy_cpu}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
sed -i'' -e "s/{{HOLLOW_PROXY_MEM}}/${proxy_mem}/g" "${RESOURCE_DIRECTORY}/hollow-node.json" sed -i'' -e "s/{{HOLLOW_PROXY_MEM}}/${proxy_mem}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
sed -i'' -e "s/{{registry}}/${CONTAINER_REGISTRY}/g" "${RESOURCE_DIRECTORY}/hollow-node.json" sed -i'' -e "s/{{registry}}/${CONTAINER_REGISTRY}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
sed -i'' -e "s/{{project}}/${PROJECT}/g" "${RESOURCE_DIRECTORY}/hollow-node.json" sed -i'' -e "s/{{project}}/${PROJECT}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
sed -i'' -e "s/{{master_ip}}/${MASTER_IP}/g" "${RESOURCE_DIRECTORY}/hollow-node.json" sed -i'' -e "s/{{master_ip}}/${MASTER_IP}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
"${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/hollow-node.json" --namespace="kubemark" "${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/hollow-node.yaml" --namespace="kubemark"
echo "Created secrets, configMaps, replication-controllers required for hollow-nodes." echo "Created secrets, configMaps, replication-controllers required for hollow-nodes."
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment