Commit 953d8838 authored by Kubernetes Submit Queue's avatar Kubernetes Submit Queue Committed by GitHub

Merge pull request #40117 from mtaufen/flags-struct

Automatic merge from submit-queue Add separate KubeletFlags struct and remove HostnameOverride and NodeIP from config type Add a separate flags struct for Kubelet flags Kubelet flags are not necessarily appropriate for the KubeletConfiguration object. For example, this PR also removes HostnameOverride and NodeIP from KubeletConfiguration.This is a preleminary step to enabling Nodes to share configurations, as part of the dynamic Kubelet configuration feature (#29459). Fields that must be unique for each node inhibit sharing, because their values, by definition, cannot be shared. /cc @ncdc @kubernetes/sig-node-misc @kubernetes/sig-cluster-lifecycle-misc
parents e2d011e4 f5eed7e9
......@@ -126,7 +126,7 @@ HTTP server: The kubelet can also listen for HTTP and respond to a simple API
func UnsecuredKubeletDeps(s *options.KubeletServer) (*kubelet.KubeletDeps, error) {
// Initialize the TLS Options
tlsOptions, err := InitializeTLS(&s.KubeletConfiguration)
tlsOptions, err := InitializeTLS(&s.KubeletFlags, &s.KubeletConfiguration)
if err != nil {
return nil, err
}
......@@ -574,7 +574,7 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.KubeletDeps) (err error) {
glog.Warning(err)
}
if err := RunKubelet(&s.KubeletConfiguration, kubeDeps, s.RunOnce, standaloneMode); err != nil {
if err := RunKubelet(&s.KubeletFlags, &s.KubeletConfiguration, kubeDeps, s.RunOnce, standaloneMode); err != nil {
return err
}
......@@ -620,7 +620,7 @@ func getNodeName(cloud cloudprovider.Interface, hostname string) (types.NodeName
// InitializeTLS checks for a configured TLSCertFile and TLSPrivateKeyFile: if unspecified a new self-signed
// certificate and key file are generated. Returns a configured server.TLSOptions object.
func InitializeTLS(kc *componentconfig.KubeletConfiguration) (*server.TLSOptions, error) {
func InitializeTLS(kf *options.KubeletFlags, kc *componentconfig.KubeletConfiguration) (*server.TLSOptions, error) {
if kc.TLSCertFile == "" && kc.TLSPrivateKeyFile == "" {
kc.TLSCertFile = path.Join(kc.CertDirectory, "kubelet.crt")
kc.TLSPrivateKeyFile = path.Join(kc.CertDirectory, "kubelet.key")
......@@ -630,7 +630,7 @@ func InitializeTLS(kc *componentconfig.KubeletConfiguration) (*server.TLSOptions
return nil, err
}
if !canReadCertAndKey {
cert, key, err := certutil.GenerateSelfSignedCertKey(nodeutil.GetHostname(kc.HostnameOverride), nil, nil)
cert, key, err := certutil.GenerateSelfSignedCertKey(nodeutil.GetHostname(kf.HostnameOverride), nil, nil)
if err != nil {
return nil, fmt.Errorf("unable to generate self signed cert: %v", err)
}
......@@ -761,8 +761,8 @@ func addChaosToClientConfig(s *options.KubeletServer, config *restclient.Config)
// 2 Kubelet binary
// 3 Standalone 'kubernetes' binary
// Eventually, #2 will be replaced with instances of #3
func RunKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *kubelet.KubeletDeps, runOnce bool, standaloneMode bool) error {
hostname := nodeutil.GetHostname(kubeCfg.HostnameOverride)
func RunKubelet(kubeFlags *options.KubeletFlags, kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *kubelet.KubeletDeps, runOnce bool, standaloneMode bool) error {
hostname := nodeutil.GetHostname(kubeFlags.HostnameOverride)
// Query the cloud provider for our node name, default to hostname if kcfg.Cloud == nil
nodeName, err := getNodeName(kubeDeps.Cloud, hostname)
if err != nil {
......@@ -808,7 +808,7 @@ func RunKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *kubelet
if kubeDeps.OSInterface == nil {
kubeDeps.OSInterface = kubecontainer.RealOS{}
}
k, err := builder(kubeCfg, kubeDeps, standaloneMode)
k, err := builder(kubeCfg, kubeDeps, standaloneMode, kubeFlags.HostnameOverride, kubeFlags.NodeIP)
if err != nil {
return fmt.Errorf("failed to create kubelet: %v", err)
}
......@@ -888,11 +888,11 @@ func startKubelet(k kubelet.KubeletBootstrap, podCfg *config.PodConfig, kubeCfg
}
}
func CreateAndInitKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *kubelet.KubeletDeps, standaloneMode bool) (k kubelet.KubeletBootstrap, err error) {
func CreateAndInitKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *kubelet.KubeletDeps, standaloneMode bool, hostnameOverride string, nodeIP string) (k kubelet.KubeletBootstrap, err error) {
// TODO: block until all sources have delivered at least one update to the channel, or break the sync loop
// up into "per source" synchronizations
k, err = kubelet.NewMainKubelet(kubeCfg, kubeDeps, standaloneMode)
k, err = kubelet.NewMainKubelet(kubeCfg, kubeDeps, standaloneMode, hostnameOverride, nodeIP)
if err != nil {
return nil, err
}
......
......@@ -120,6 +120,12 @@ const (
HairpinNone = "none"
)
// A configuration field should go in KubeletFlags instead of KubeletConfiguration if any of these are true:
// - its value will never, or cannot safely be changed during the lifetime of a node
// - its value cannot be safely shared between nodes at the same time (e.g. a hostname)
// KubeletConfiguration is intended to be shared between nodes
// In general, please try to avoid adding flags or configuration fields,
// we already have a confusingly large amount of them.
// TODO: curate the ordering and structure of this config object
type KubeletConfiguration struct {
metav1.TypeMeta
......@@ -167,9 +173,6 @@ type KubeletConfiguration struct {
Authentication KubeletAuthentication
// authorization specifies how requests to the Kubelet's server are authorized
Authorization KubeletAuthorization
// hostnameOverride is the hostname used to identify the kubelet instead
// of the actual hostname.
HostnameOverride string
// podInfraContainerImage is the image whose network/ipc namespaces
// containers in each pod will use.
PodInfraContainerImage string
......@@ -405,10 +408,6 @@ type KubeletConfiguration struct {
// wait before transitioning out of out-of-disk node condition status.
// +optional
OutOfDiskTransitionFrequency metav1.Duration
// nodeIP is IP address of the node. If set, kubelet will use this IP
// address for the node.
// +optional
NodeIP string
// nodeLabels to add when registering the node in the cluster.
NodeLabels map[string]string
// nonMasqueradeCIDR configures masquerading: traffic to IPs outside this range will use IP masquerade.
......
......@@ -175,6 +175,12 @@ type LeaderElectionConfiguration struct {
RetryPeriod metav1.Duration `json:"retryPeriod"`
}
// A configuration field should go in KubeletFlags instead of KubeletConfiguration if any of these are true:
// - its value will never, or cannot safely be changed during the lifetime of a node
// - its value cannot be safely shared between nodes at the same time (e.g. a hostname)
// KubeletConfiguration is intended to be shared between nodes
// In general, please try to avoid adding flags or configuration fields,
// we already have a confusingly large amount of them.
type KubeletConfiguration struct {
metav1.TypeMeta `json:",inline"`
......@@ -221,9 +227,6 @@ type KubeletConfiguration struct {
Authentication KubeletAuthentication `json:"authentication"`
// authorization specifies how requests to the Kubelet's server are authorized
Authorization KubeletAuthorization `json:"authorization"`
// hostnameOverride is the hostname used to identify the kubelet instead
// of the actual hostname.
HostnameOverride string `json:"hostnameOverride"`
// podInfraContainerImage is the image whose network/ipc namespaces
// containers in each pod will use.
PodInfraContainerImage string `json:"podInfraContainerImage"`
......@@ -449,9 +452,6 @@ type KubeletConfiguration struct {
// outOfDiskTransitionFrequency is duration for which the kubelet has to
// wait before transitioning out of out-of-disk node condition status.
OutOfDiskTransitionFrequency metav1.Duration `json:"outOfDiskTransitionFrequency"`
// nodeIP is IP address of the node. If set, kubelet will use this IP
// address for the node.
NodeIP string `json:"nodeIP"`
// nodeLabels to add when registering the node in the cluster.
NodeLabels map[string]string `json:"nodeLabels"`
// nonMasqueradeCIDR configures masquerading: traffic to IPs outside this range will use IP masquerade.
......
......@@ -271,7 +271,6 @@ func autoConvert_v1alpha1_KubeletConfiguration_To_componentconfig_KubeletConfigu
if err := Convert_v1alpha1_KubeletAuthorization_To_componentconfig_KubeletAuthorization(&in.Authorization, &out.Authorization, s); err != nil {
return err
}
out.HostnameOverride = in.HostnameOverride
out.PodInfraContainerImage = in.PodInfraContainerImage
out.DockerEndpoint = in.DockerEndpoint
out.RootDirectory = in.RootDirectory
......@@ -377,7 +376,6 @@ func autoConvert_v1alpha1_KubeletConfiguration_To_componentconfig_KubeletConfigu
return err
}
out.OutOfDiskTransitionFrequency = in.OutOfDiskTransitionFrequency
out.NodeIP = in.NodeIP
out.NodeLabels = *(*map[string]string)(unsafe.Pointer(&in.NodeLabels))
out.NonMasqueradeCIDR = in.NonMasqueradeCIDR
out.EnableCustomMetrics = in.EnableCustomMetrics
......@@ -450,7 +448,6 @@ func autoConvert_componentconfig_KubeletConfiguration_To_v1alpha1_KubeletConfigu
if err := Convert_componentconfig_KubeletAuthorization_To_v1alpha1_KubeletAuthorization(&in.Authorization, &out.Authorization, s); err != nil {
return err
}
out.HostnameOverride = in.HostnameOverride
out.PodInfraContainerImage = in.PodInfraContainerImage
out.DockerEndpoint = in.DockerEndpoint
out.RootDirectory = in.RootDirectory
......@@ -576,7 +573,6 @@ func autoConvert_componentconfig_KubeletConfiguration_To_v1alpha1_KubeletConfigu
return err
}
out.OutOfDiskTransitionFrequency = in.OutOfDiskTransitionFrequency
out.NodeIP = in.NodeIP
out.NodeLabels = *(*map[string]string)(unsafe.Pointer(&in.NodeLabels))
out.NonMasqueradeCIDR = in.NonMasqueradeCIDR
out.EnableCustomMetrics = in.EnableCustomMetrics
......
......@@ -12958,6 +12958,7 @@ func GetOpenAPIDefinitions(ref openapi.ReferenceCallback) map[string]openapi.Ope
"k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1.KubeletConfiguration": {
Schema: spec.Schema{
SchemaProps: spec.SchemaProps{
Description: "A configuration field should go in KubeletFlags instead of KubeletConfiguration if any of these are true: - its value will never, or cannot safely be changed during the lifetime of a node - its value cannot be safely shared between nodes at the same time (e.g. a hostname)\n KubeletConfiguration is intended to be shared between nodes\nIn general, please try to avoid adding flags or configuration fields, we already have a confusingly large amount of them.",
Properties: map[string]spec.Schema{
"kind": {
SchemaProps: spec.SchemaProps{
......@@ -13073,13 +13074,6 @@ func GetOpenAPIDefinitions(ref openapi.ReferenceCallback) map[string]openapi.Ope
Ref: ref("k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1.KubeletAuthorization"),
},
},
"hostnameOverride": {
SchemaProps: spec.SchemaProps{
Description: "hostnameOverride is the hostname used to identify the kubelet instead of the actual hostname.",
Type: []string{"string"},
Format: "",
},
},
"podInfraContainerImage": {
SchemaProps: spec.SchemaProps{
Description: "podInfraContainerImage is the image whose network/ipc namespaces containers in each pod will use.",
......@@ -13617,13 +13611,6 @@ func GetOpenAPIDefinitions(ref openapi.ReferenceCallback) map[string]openapi.Ope
Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Duration"),
},
},
"nodeIP": {
SchemaProps: spec.SchemaProps{
Description: "nodeIP is IP address of the node. If set, kubelet will use this IP address for the node.",
Type: []string{"string"},
Format: "",
},
},
"nodeLabels": {
SchemaProps: spec.SchemaProps{
Description: "nodeLabels to add when registering the node in the cluster.",
......@@ -13869,7 +13856,7 @@ func GetOpenAPIDefinitions(ref openapi.ReferenceCallback) map[string]openapi.Ope
},
},
},
Required: []string{"podManifestPath", "syncFrequency", "fileCheckFrequency", "httpCheckFrequency", "manifestURL", "manifestURLHeader", "enableServer", "address", "port", "readOnlyPort", "tlsCertFile", "tlsPrivateKeyFile", "certDirectory", "authentication", "authorization", "hostnameOverride", "podInfraContainerImage", "dockerEndpoint", "rootDirectory", "seccompProfileRoot", "allowPrivileged", "hostNetworkSources", "hostPIDSources", "hostIPCSources", "registryPullQPS", "registryBurst", "eventRecordQPS", "eventBurst", "enableDebuggingHandlers", "enableContentionProfiling", "minimumGCAge", "maxPerPodContainerCount", "maxContainerCount", "cAdvisorPort", "healthzPort", "healthzBindAddress", "oomScoreAdj", "registerNode", "clusterDomain", "masterServiceNamespace", "clusterDNS", "streamingConnectionIdleTimeout", "nodeStatusUpdateFrequency", "imageMinimumGCAge", "imageGCHighThresholdPercent", "imageGCLowThresholdPercent", "lowDiskSpaceThresholdMB", "volumeStatsAggPeriod", "networkPluginName", "networkPluginDir", "cniConfDir", "cniBinDir", "networkPluginMTU", "volumePluginDir", "cloudProvider", "cloudConfigFile", "kubeletCgroups", "runtimeCgroups", "systemCgroups", "cgroupRoot", "containerRuntime", "remoteRuntimeEndpoint", "remoteImageEndpoint", "runtimeRequestTimeout", "rktPath", "rktAPIEndpoint", "rktStage1Image", "lockFilePath", "exitOnLockContention", "hairpinMode", "babysitDaemons", "maxPods", "dockerExecHandlerName", "podCIDR", "resolvConf", "cpuCFSQuota", "containerized", "maxOpenFiles", "registerSchedulable", "registerWithTaints", "contentType", "kubeAPIQPS", "kubeAPIBurst", "serializeImagePulls", "outOfDiskTransitionFrequency", "nodeIP", "nodeLabels", "nonMasqueradeCIDR", "enableCustomMetrics", "evictionHard", "evictionSoft", "evictionSoftGracePeriod", "evictionPressureTransitionPeriod", "evictionMaxPodGracePeriod", "evictionMinimumReclaim", "experimentalKernelMemcgNotification", "podsPerCore", "enableControllerAttachDetach", "experimentalQOSReserved", "protectKernelDefaults", "makeIPTablesUtilChains", "iptablesMasqueradeBit", "iptablesDropBit", "systemReserved", "kubeReserved"},
Required: []string{"podManifestPath", "syncFrequency", "fileCheckFrequency", "httpCheckFrequency", "manifestURL", "manifestURLHeader", "enableServer", "address", "port", "readOnlyPort", "tlsCertFile", "tlsPrivateKeyFile", "certDirectory", "authentication", "authorization", "podInfraContainerImage", "dockerEndpoint", "rootDirectory", "seccompProfileRoot", "allowPrivileged", "hostNetworkSources", "hostPIDSources", "hostIPCSources", "registryPullQPS", "registryBurst", "eventRecordQPS", "eventBurst", "enableDebuggingHandlers", "enableContentionProfiling", "minimumGCAge", "maxPerPodContainerCount", "maxContainerCount", "cAdvisorPort", "healthzPort", "healthzBindAddress", "oomScoreAdj", "registerNode", "clusterDomain", "masterServiceNamespace", "clusterDNS", "streamingConnectionIdleTimeout", "nodeStatusUpdateFrequency", "imageMinimumGCAge", "imageGCHighThresholdPercent", "imageGCLowThresholdPercent", "lowDiskSpaceThresholdMB", "volumeStatsAggPeriod", "networkPluginName", "networkPluginDir", "cniConfDir", "cniBinDir", "networkPluginMTU", "volumePluginDir", "cloudProvider", "cloudConfigFile", "kubeletCgroups", "runtimeCgroups", "systemCgroups", "cgroupRoot", "containerRuntime", "remoteRuntimeEndpoint", "remoteImageEndpoint", "runtimeRequestTimeout", "rktPath", "rktAPIEndpoint", "rktStage1Image", "lockFilePath", "exitOnLockContention", "hairpinMode", "babysitDaemons", "maxPods", "dockerExecHandlerName", "podCIDR", "resolvConf", "cpuCFSQuota", "containerized", "maxOpenFiles", "registerSchedulable", "registerWithTaints", "contentType", "kubeAPIQPS", "kubeAPIBurst", "serializeImagePulls", "outOfDiskTransitionFrequency", "nodeLabels", "nonMasqueradeCIDR", "enableCustomMetrics", "evictionHard", "evictionSoft", "evictionSoftGracePeriod", "evictionPressureTransitionPeriod", "evictionMaxPodGracePeriod", "evictionMinimumReclaim", "experimentalKernelMemcgNotification", "podsPerCore", "enableControllerAttachDetach", "experimentalQOSReserved", "protectKernelDefaults", "makeIPTablesUtilChains", "iptablesMasqueradeBit", "iptablesDropBit", "systemReserved", "kubeReserved"},
},
},
Dependencies: []string{
......
......@@ -187,7 +187,7 @@ type KubeletBootstrap interface {
}
// create and initialize a Kubelet instance
type KubeletBuilder func(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *KubeletDeps, standaloneMode bool) (KubeletBootstrap, error)
type KubeletBuilder func(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *KubeletDeps, standaloneMode bool, hostnameOverride string, nodeIP string) (KubeletBootstrap, error)
// KubeletDeps is a bin for things we might consider "injected dependencies" -- objects constructed
// at runtime that are necessary for running the Kubelet. This is a temporary solution for grouping
......@@ -282,7 +282,7 @@ func getRuntimeAndImageServices(config *componentconfig.KubeletConfiguration) (i
// NewMainKubelet instantiates a new Kubelet object along with all the required internal modules.
// No initialization of Kubelet and its modules should happen here.
func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *KubeletDeps, standaloneMode bool) (*Kubelet, error) {
func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *KubeletDeps, standaloneMode bool, hostnameOverride string, nodeIP string) (*Kubelet, error) {
if kubeCfg.RootDirectory == "" {
return nil, fmt.Errorf("invalid root directory %q", kubeCfg.RootDirectory)
}
......@@ -302,7 +302,7 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Kub
}
}
hostname := nodeutil.GetHostname(kubeCfg.HostnameOverride)
hostname := nodeutil.GetHostname(hostnameOverride)
// Query the cloud provider for our node name, default to hostname
nodeName := types.NodeName(hostname)
if kubeDeps.Cloud != nil {
......@@ -462,7 +462,7 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Kub
cpuCFSQuota: kubeCfg.CPUCFSQuota,
daemonEndpoints: daemonEndpoints,
containerManager: kubeDeps.ContainerManager,
nodeIP: net.ParseIP(kubeCfg.NodeIP),
nodeIP: net.ParseIP(nodeIP),
clock: clock.RealClock{},
outOfDiskTransitionFrequency: kubeCfg.OutOfDiskTransitionFrequency.Duration,
enableCustomMetrics: kubeCfg.EnableCustomMetrics,
......
......@@ -18,6 +18,7 @@ go_library(
"//cmd/kube-proxy/app:go_default_library",
"//cmd/kube-proxy/app/options:go_default_library",
"//cmd/kubelet/app:go_default_library",
"//cmd/kubelet/app/options:go_default_library",
"//pkg/api:go_default_library",
"//pkg/apis/componentconfig:go_default_library",
"//pkg/apis/componentconfig/v1alpha1:go_default_library",
......
......@@ -21,6 +21,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kubeletapp "k8s.io/kubernetes/cmd/kubelet/app"
"k8s.io/kubernetes/cmd/kubelet/app/options"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/apis/componentconfig"
"k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1"
......@@ -42,6 +43,7 @@ import (
)
type HollowKubelet struct {
KubeletFlags *options.KubeletFlags
KubeletConfiguration *componentconfig.KubeletConfiguration
KubeletDeps *kubelet.KubeletDeps
}
......@@ -58,7 +60,7 @@ func NewHollowKubelet(
// -----------------
// Static config
// -----------------
c := GetHollowKubeletConfig(nodeName, kubeletPort, kubeletReadOnlyPort, maxPods, podsPerCore)
f, c := GetHollowKubeletConfig(nodeName, kubeletPort, kubeletReadOnlyPort, maxPods, podsPerCore)
// -----------------
// Injected objects
......@@ -80,6 +82,7 @@ func NewHollowKubelet(
}
return &HollowKubelet{
KubeletFlags: f,
KubeletConfiguration: c,
KubeletDeps: d,
}
......@@ -87,7 +90,7 @@ func NewHollowKubelet(
// Starts this HollowKubelet and blocks.
func (hk *HollowKubelet) Run() {
kubeletapp.RunKubelet(hk.KubeletConfiguration, hk.KubeletDeps, false, false)
kubeletapp.RunKubelet(hk.KubeletFlags, hk.KubeletConfiguration, hk.KubeletDeps, false, false)
select {}
}
......@@ -98,12 +101,18 @@ func GetHollowKubeletConfig(
kubeletPort int,
kubeletReadOnlyPort int,
maxPods int,
podsPerCore int) *componentconfig.KubeletConfiguration {
podsPerCore int) (*options.KubeletFlags, *componentconfig.KubeletConfiguration) {
testRootDir := utils.MakeTempDirOrDie("hollow-kubelet.", "")
manifestFilePath := utils.MakeTempDirOrDie("manifest", testRootDir)
glog.Infof("Using %s as root dir for hollow-kubelet", testRootDir)
// Flags struct
f := &options.KubeletFlags{
HostnameOverride: nodeName,
}
// Config struct
// Do the external -> internal conversion to make sure that defaults
// are set for fields not overridden in NewHollowKubelet.
tmp := &v1alpha1.KubeletConfiguration{}
......@@ -111,7 +120,6 @@ func GetHollowKubeletConfig(
c := &componentconfig.KubeletConfiguration{}
api.Scheme.Convert(tmp, c, nil)
c.HostnameOverride = nodeName
c.RootDirectory = testRootDir
c.ManifestURL = ""
c.Address = "0.0.0.0" /* bind address */
......@@ -166,6 +174,6 @@ func GetHollowKubeletConfig(
// have actually wanted the default).
// The default will be present in the KubeletConfiguration contstructed above.
return c
return f, c
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment