Merge pull request #27874 from ingvagabund/kubelet-kernel-tunning-behaviour-new-flags

cmd/kubelet/app/options/options.go

@@ -183,4 +183,5 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) {
 	fs.Int32Var(&s.EvictionMaxPodGracePeriod, "eviction-max-pod-grace-period", s.EvictionMaxPodGracePeriod, "Maximum allowed grace period (in seconds) to use when terminating pods in response to a soft eviction threshold being met.  If negative, defer to pod specified value.")
 	fs.StringVar(&s.EvictionMinimumReclaim, "eviction-minimum-reclaim", s.EvictionMinimumReclaim, "A set of minimum reclaims (e.g. imagefs.available=2Gi) that describes the minimum amount of resource the kubelet will reclaim when performing a pod eviction if that resource is under pressure.")
 	fs.Int32Var(&s.PodsPerCore, "pods-per-core", s.PodsPerCore, "Number of Pods per core that can run on this Kubelet. The total number of Pods on this Kubelet cannot exceed max-pods, so max-pods will be used if this calculation results in a larger number of Pods allowed on the Kubelet. A value of 0 disables this limit.")
+	fs.BoolVar(&s.ProtectKernelDefaults, "protect-kernel-defaults", s.ProtectKernelDefaults, "Default kubelet behaviour for kernel tuning. If set, kubelet errors if any of kernel tunables is different than kubelet defaults.")
 }

cmd/kubelet/app/server.go

@@ -285,6 +285,7 @@ func UnsecuredKubeletConfig(s *options.KubeletServer) (*KubeletConfig, error) {
 		NodeIP:                net.ParseIP(s.NodeIP),
 		EvictionConfig:        evictionConfig,
 		PodsPerCore:           int(s.PodsPerCore),
+		ProtectKernelDefaults: s.ProtectKernelDefaults,
 	}, nil
 }
 
@@ -388,6 +389,7 @@ func run(s *options.KubeletServer, kcfg *KubeletConfig) (err error) {
 			ContainerRuntime:      kcfg.ContainerRuntime,
 			CgroupsPerQOS:         kcfg.CgroupsPerQOS,
 			CgroupRoot:            kcfg.CgroupRoot,
+			ProtectKernelDefaults: kcfg.ProtectKernelDefaults,
 		})
 		if err != nil {
 			return err
@@ -636,6 +638,7 @@ func SimpleKubelet(client *clientset.Clientset,
 		OutOfDiskTransitionFrequency: outOfDiskTransitionFrequency,
 		EvictionConfig:               evictionConfig,
 		PodsPerCore:                  podsPerCore,
+		ProtectKernelDefaults:        false,
 	}
 	return &kcfg
 }
@@ -888,6 +891,8 @@ type KubeletConfig struct {
 	HairpinMode                string
 	BabysitDaemons             bool
 	Options                    []kubelet.Option
+
+	ProtectKernelDefaults bool
 }
 
 func CreateAndInitKubelet(kc *KubeletConfig) (k KubeletBootstrap, pc *config.PodConfig, err error) {

hack/verify-flags/known-flags.txt

@@ -374,6 +374,7 @@ portal-net
 prepull-images
 private-mountns
 prom-push-gateway
+protect-kernel-defaults
 proto-import
 proxy-bindall
 proxy-kubeconfig

pkg/apis/componentconfig/types.go

@@ -397,6 +397,8 @@ type KubeletConfiguration struct {
 	// Currently only cpu and memory are supported. [default=none]
 	// See http://releases.k8s.io/HEAD/docs/user-guide/compute-resources.md for more detail.
 	KubeReserved utilconfig.ConfigurationMap `json:"kubeReserved"`
+	// Default behaviour for kernel tuning
+	ProtectKernelDefaults bool `json:"protectKernelDefaults"`
 }
 
 type KubeSchedulerConfiguration struct {

pkg/apis/componentconfig/v1alpha1/types.go

@@ -452,4 +452,6 @@ type KubeletConfiguration struct {
 	// Currently only cpu and memory are supported. [default=none]
 	// See http://releases.k8s.io/HEAD/docs/user-guide/compute-resources.md for more detail.
 	KubeReserved map[string]string `json:"kubeReserved"`
+	// Default behaviour for kernel tuning
+	ProtectKernelDefaults bool `json:"protectKernelDefaults"`
 }

pkg/apis/componentconfig/v1alpha1/zz_generated.conversion.go

@@ -319,6 +319,7 @@ func autoConvert_v1alpha1_KubeletConfiguration_To_componentconfig_KubeletConfigu
 	} else {
 		out.KubeReserved = nil
 	}
+	out.ProtectKernelDefaults = in.ProtectKernelDefaults
 	return nil
 }
 
@@ -483,6 +484,7 @@ func autoConvert_componentconfig_KubeletConfiguration_To_v1alpha1_KubeletConfigu
 	} else {
 		out.KubeReserved = nil
 	}
+	out.ProtectKernelDefaults = in.ProtectKernelDefaults
 	return nil
 }
 

pkg/apis/componentconfig/v1alpha1/zz_generated.deepcopy.go

@@ -377,6 +377,7 @@ func DeepCopy_v1alpha1_KubeletConfiguration(in interface{}, out interface{}, c *
 		} else {
 			out.KubeReserved = nil
 		}
+		out.ProtectKernelDefaults = in.ProtectKernelDefaults
 		return nil
 	}
 }

pkg/apis/componentconfig/zz_generated.deepcopy.go

@@ -325,6 +325,7 @@ func DeepCopy_componentconfig_KubeletConfiguration(in interface{}, out interface
 		} else {
 			out.KubeReserved = nil
 		}
+		out.ProtectKernelDefaults = in.ProtectKernelDefaults
 		return nil
 	}
 }

pkg/kubelet/cm/container_manager.go

@@ -45,6 +45,7 @@ type NodeConfig struct {
 	ContainerRuntime      string
 	CgroupsPerQOS         bool
 	CgroupRoot            string
+	ProtectKernelDefaults bool
 }
 
 type Status struct {

pkg/kubelet/cm/container_manager_linux.go

@@ -199,7 +199,6 @@ func createManager(containerName string) *fs.Manager {
 	}
 }
 
-// TODO: plumb this up as a flag to Kubelet in a future PR
 type KernelTunableBehavior string
 
 const (
@@ -288,8 +287,11 @@ func (cm *containerManagerImpl) setupNode() error {
 	if !f.cpuHardcapping {
 		cm.status.SoftRequirements = fmt.Errorf("CPU hardcapping unsupported")
 	}
-	// TODO: plumb kernel tunable options into container manager, right now, we modify by default
-	if err := setupKernelTunables(KernelTunableModify); err != nil {
+	b := KernelTunableModify
+	if cm.GetNodeConfig().ProtectKernelDefaults {
+		b = KernelTunableError
+	}
+	if err := setupKernelTunables(b); err != nil {
 		return err
 	}