Kubelet Refactoring

cmd/kubelet/app/options/options.go

@@ -49,12 +49,17 @@ type KubeletServer struct {
 	AuthPath          util.StringFlag // Deprecated -- use KubeConfig instead
 	APIServerList     []string        // Deprecated -- use KubeConfig instead
 
-	RunOnce bool
-
 	// Insert a probability of random errors during calls to the master.
 	ChaosChance float64
 	// Crash immediately, rather than eating panics.
 	ReallyCrashForTesting bool
+
+	// TODO(mtaufen): It is increasingly looking like nobody actually uses the
+	//                Kubelet's runonce mode anymore, so it may be a candidate
+	//                for deprecation and removal.
+	// If runOnce is true, the Kubelet will check the API server once for pods,
+	// run those in addition to the pods specified by the local manifest, and exit.
+	RunOnce bool
 }
 
 // NewKubeletServer will create a new KubeletServer with default values.

cmd/kubelet/app/server_test.go

@@ -19,6 +19,7 @@ package app
 import (
 	"testing"
 
+	"k8s.io/kubernetes/pkg/kubelet"
 	"k8s.io/kubernetes/pkg/util/config"
 )
 
@@ -56,7 +57,7 @@ func TestValueOfAllocatableResources(t *testing.T) {
 		kubeReservedCM.Set(test.kubeReserved)
 		systemReservedCM.Set(test.systemReserved)
 
-		_, err := parseReservation(kubeReservedCM, systemReservedCM)
+		_, err := kubelet.ParseReservation(kubeReservedCM, systemReservedCM)
 		if err != nil {
 			t.Logf("%s: error returned: %v", test.name, err)
 		}

contrib/mesos/pkg/executor/service/service.go

@@ -35,6 +35,7 @@ import (
 	"k8s.io/kubernetes/contrib/mesos/pkg/podutil"
 	"k8s.io/kubernetes/contrib/mesos/pkg/scheduler/meta"
 	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/apis/componentconfig"
 	"k8s.io/kubernetes/pkg/client/cache"
 	clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
 	"k8s.io/kubernetes/pkg/fields"
@@ -158,16 +159,16 @@ func (s *KubeletExecutorServer) runKubelet(
 		}
 	}()
 
-	kcfg, err := kubeletapp.UnsecuredKubeletConfig(s.KubeletServer)
+	kubeDeps, err := kubeletapp.UnsecuredKubeletDeps(s.KubeletServer)
 	if err != nil {
 		return err
 	}
 
 	// apply Mesos specific settings
-	kcfg.Builder = func(kc *kubeletapp.KubeletConfig) (kubeletapp.KubeletBootstrap, *kconfig.PodConfig, error) {
-		k, pc, err := kubeletapp.CreateAndInitKubelet(kc)
+	kubeDeps.Builder = func(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *kubelet.KubeletDeps, standaloneMode bool) (kubelet.KubeletBootstrap, error) {
+		k, err := kubeletapp.CreateAndInitKubelet(kubeCfg, kubeDeps, standaloneMode)
 		if err != nil {
-			return k, pc, err
+			return k, err
 		}
 
 		// decorate kubelet such that it shuts down when the executor is
@@ -177,11 +178,10 @@ func (s *KubeletExecutorServer) runKubelet(
 			executorDone: executorDone,
 		}
 
-		return decorated, pc, nil
+		return decorated, nil
 	}
-	kcfg.RuntimeCgroups = "" // don't move the docker daemon into a cgroup
-	kcfg.Hostname = kcfg.HostnameOverride
-	kcfg.KubeClient = apiclient
+	s.RuntimeCgroups = "" // don't move the docker daemon into a cgroup
+	kubeDeps.KubeClient = apiclient
 
 	// taken from KubeletServer#Run(*KubeletConfig)
 	eventClientConfig, err := kubeletapp.CreateAPIServerClientConfig(s.KubeletServer)
@@ -192,16 +192,16 @@ func (s *KubeletExecutorServer) runKubelet(
 	// make a separate client for events
 	eventClientConfig.QPS = float32(s.EventRecordQPS)
 	eventClientConfig.Burst = int(s.EventBurst)
-	kcfg.EventClient, err = clientset.NewForConfig(eventClientConfig)
+	kubeDeps.EventClient, err = clientset.NewForConfig(eventClientConfig)
 	if err != nil {
 		return err
 	}
 
-	kcfg.NodeName = kcfg.HostnameOverride
-	kcfg.PodConfig = kconfig.NewPodConfig(kconfig.PodConfigNotificationIncremental, kcfg.Recorder) // override the default pod source
-	kcfg.StandaloneMode = false
-	kcfg.SystemCgroups = "" // don't take control over other system processes.
-	if kcfg.Cloud != nil {
+	kubeDeps.PodConfig = kconfig.NewPodConfig(kconfig.PodConfigNotificationIncremental, kubeDeps.Recorder) // override the default pod source
+
+	s.SystemCgroups = "" // don't take control over other system processes.
+
+	if kubeDeps.Cloud != nil {
 		// fail early and hard because having the cloud provider loaded would go unnoticed,
 		// but break bigger cluster because accessing the state.json from every slave kills the master.
 		panic("cloud provider must not be set")
@@ -209,17 +209,17 @@ func (s *KubeletExecutorServer) runKubelet(
 
 	// create custom cAdvisor interface which return the resource values that Mesos reports
 	ni := <-nodeInfos
-	cAdvisorInterface, err := NewMesosCadvisor(ni.Cores, ni.Mem, uint(s.CAdvisorPort), kcfg.ContainerRuntime)
+	cAdvisorInterface, err := NewMesosCadvisor(ni.Cores, ni.Mem, uint(s.CAdvisorPort), s.ContainerRuntime)
 	if err != nil {
 		return err
 	}
 
-	kcfg.CAdvisorInterface = cAdvisorInterface
-	kcfg.ContainerManager, err = cm.NewContainerManager(kcfg.Mounter, cAdvisorInterface, cm.NodeConfig{
-		RuntimeCgroupsName: kcfg.RuntimeCgroups,
-		SystemCgroupsName:  kcfg.SystemCgroups,
-		KubeletCgroupsName: kcfg.KubeletCgroups,
-		ContainerRuntime:   kcfg.ContainerRuntime,
+	kubeDeps.CAdvisorInterface = cAdvisorInterface
+	kubeDeps.ContainerManager, err = cm.NewContainerManager(kubeDeps.Mounter, cAdvisorInterface, cm.NodeConfig{
+		RuntimeCgroupsName: s.RuntimeCgroups,
+		SystemCgroupsName:  s.SystemCgroups,
+		KubeletCgroupsName: s.KubeletCgroups,
+		ContainerRuntime:   s.ContainerRuntime,
 	})
 	if err != nil {
 		return err
@@ -239,26 +239,26 @@ func (s *KubeletExecutorServer) runKubelet(
 		containerOptions = append(containerOptions, podsource.ContainerEnvOverlay([]api.EnvVar{
 			{Name: envContainerID, Value: s.containerID},
 		}))
-		kcfg.ContainerRuntimeOptions = append(kcfg.ContainerRuntimeOptions,
+		kubeDeps.ContainerRuntimeOptions = append(kubeDeps.ContainerRuntimeOptions,
 			dockertools.PodInfraContainerEnv(map[string]string{
 				envContainerID: s.containerID,
 			}))
 	}
 
-	podsource.Mesos(executorDone, kcfg.PodConfig.Channel(podsource.MesosSource), podLW, registry, containerOptions...)
+	podsource.Mesos(executorDone, kubeDeps.PodConfig.Channel(podsource.MesosSource), podLW, registry, containerOptions...)
 
 	// create static-pods directory file source
 	log.V(2).Infof("initializing static pods source factory, configured at path %q", staticPodsConfigPath)
-	fileSourceUpdates := kcfg.PodConfig.Channel(kubetypes.FileSource)
-	kconfig.NewSourceFile(staticPodsConfigPath, kcfg.HostnameOverride, kcfg.FileCheckFrequency, fileSourceUpdates)
+	fileSourceUpdates := kubeDeps.PodConfig.Channel(kubetypes.FileSource)
+	kconfig.NewSourceFile(staticPodsConfigPath, s.HostnameOverride, s.FileCheckFrequency.Duration, fileSourceUpdates)
 
 	// run the kubelet
-	// NOTE: because kcfg != nil holds, the upstream Run function will not
+	// NOTE: because kubeDeps != nil holds, the upstream Run function will not
 	//       initialize the cloud provider. We explicitly wouldn't want
 	//       that because then every kubelet instance would query the master
 	//       state.json which does not scale.
 	s.KubeletServer.LockFilePath = "" // disable lock file
-	err = kubeletapp.Run(s.KubeletServer, kcfg)
+	err = kubeletapp.Run(s.KubeletServer, kubeDeps)
 	return
 }
 

pkg/kubemark/hollow_kubelet.go

@@ -21,11 +21,18 @@ import (
 
 	kubeletapp "k8s.io/kubernetes/cmd/kubelet/app"
 	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/apis/componentconfig"
+	"k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1"
 	clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
+	"k8s.io/kubernetes/pkg/kubelet"
 	"k8s.io/kubernetes/pkg/kubelet/cadvisor"
 	"k8s.io/kubernetes/pkg/kubelet/cm"
 	containertest "k8s.io/kubernetes/pkg/kubelet/container/testing"
 	"k8s.io/kubernetes/pkg/kubelet/dockertools"
+	kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
+	kubeio "k8s.io/kubernetes/pkg/util/io"
+	"k8s.io/kubernetes/pkg/util/mount"
+	"k8s.io/kubernetes/pkg/util/oom"
 	"k8s.io/kubernetes/pkg/volume/empty_dir"
 	"k8s.io/kubernetes/test/utils"
 
@@ -33,7 +40,8 @@ import (
 )
 
 type HollowKubelet struct {
-	KubeletConfig *kubeletapp.KubeletConfig
+	KubeletConfiguration *componentconfig.KubeletConfiguration
+	KubeletDeps          *kubelet.KubeletDeps
 }
 
 func NewHollowKubelet(
@@ -45,44 +53,116 @@ func NewHollowKubelet(
 	containerManager cm.ContainerManager,
 	maxPods int, podsPerCore int,
 ) *HollowKubelet {
-	testRootDir := utils.MakeTempDirOrDie("hollow-kubelet.", "")
-	manifestFilePath := utils.MakeTempDirOrDie("manifest", testRootDir)
-	glog.Infof("Using %s as root dir for hollow-kubelet", testRootDir)
+	// -----------------
+	// Static config
+	// -----------------
+	c := GetHollowKubeletConfig(nodeName, kubeletPort, kubeletReadOnlyPort, maxPods, podsPerCore)
+
+	// -----------------
+	// Injected objects
+	// -----------------
+	d := &kubelet.KubeletDeps{
+		KubeClient:        client,
+		DockerClient:      dockerClient,
+		CAdvisorInterface: cadvisorInterface,
+		Cloud:             nil,
+		OSInterface:       &containertest.FakeOS{},
+		ContainerManager:  containerManager,
+		VolumePlugins:     empty_dir.ProbeVolumePlugins(),
+		TLSOptions:        nil,
+		OOMAdjuster:       oom.NewFakeOOMAdjuster(),
+		Writer:            &kubeio.StdWriter{},
+		Mounter:           mount.New(),
+	}
 
 	return &HollowKubelet{
-		KubeletConfig: kubeletapp.SimpleKubelet(
-			client,
-			dockerClient,
-			nodeName,
-			testRootDir,
-			"",        /* manifest-url */
-			"0.0.0.0", /* bind address */
-			uint(kubeletPort),
-			uint(kubeletReadOnlyPort),
-			api.NamespaceDefault,
-			empty_dir.ProbeVolumePlugins(),
-			nil, /* tls-options */
-			cadvisorInterface,
-			manifestFilePath,
-			nil, /* cloud-provider */
-			&containertest.FakeOS{}, /* os-interface */
-			20*time.Second,          /* FileCheckFrequency */
-			20*time.Second,          /* HTTPCheckFrequency */
-			1*time.Minute,           /* MinimumGCAge */
-			10*time.Second,          /* NodeStatusUpdateFrequency */
-			10*time.Second,          /* SyncFrequency */
-			5*time.Minute,           /* OutOfDiskTransitionFrequency */
-			5*time.Minute,           /* EvictionPressureTransitionPeriod */
-			maxPods,
-			podsPerCore,
-			containerManager,
-			nil,
-		),
+		KubeletConfiguration: c,
+		KubeletDeps:          d,
 	}
 }
 
 // Starts this HollowKubelet and blocks.
 func (hk *HollowKubelet) Run() {
-	kubeletapp.RunKubelet(hk.KubeletConfig)
+	kubeletapp.RunKubelet(hk.KubeletConfiguration, hk.KubeletDeps, false, false)
 	select {}
 }
+
+// Builds a KubeletConfiguration for the HollowKubelet, ensuring that the
+// usual defaults are applied for fields we do not override.
+func GetHollowKubeletConfig(
+	nodeName string,
+	kubeletPort int,
+	kubeletReadOnlyPort int,
+	maxPods int,
+	podsPerCore int) *componentconfig.KubeletConfiguration {
+
+	testRootDir := utils.MakeTempDirOrDie("hollow-kubelet.", "")
+	manifestFilePath := utils.MakeTempDirOrDie("manifest", testRootDir)
+	glog.Infof("Using %s as root dir for hollow-kubelet", testRootDir)
+
+	// Do the external -> internal conversion to make sure that defaults
+	// are set for fields not overridden in NewHollowKubelet.
+	tmp := &v1alpha1.KubeletConfiguration{}
+	c := &componentconfig.KubeletConfiguration{}
+	api.Scheme.Convert(tmp, c, nil)
+
+	c.HostnameOverride = nodeName
+	c.RootDirectory = testRootDir
+	c.ManifestURL = ""
+	c.Address = "0.0.0.0" /* bind address */
+	c.Port = int32(kubeletPort)
+	c.ReadOnlyPort = int32(kubeletReadOnlyPort)
+	c.MasterServiceNamespace = api.NamespaceDefault
+	c.PodManifestPath = manifestFilePath
+	c.FileCheckFrequency.Duration = 20 * time.Second
+	c.HTTPCheckFrequency.Duration = 20 * time.Second
+	c.MinimumGCAge.Duration = 1 * time.Minute
+	c.NodeStatusUpdateFrequency.Duration = 10 * time.Second
+	c.SyncFrequency.Duration = 10 * time.Second
+	c.OutOfDiskTransitionFrequency.Duration = 5 * time.Minute
+	c.EvictionPressureTransitionPeriod.Duration = 5 * time.Minute
+	c.MaxPods = int32(maxPods)
+	c.PodsPerCore = int32(podsPerCore)
+	c.ClusterDNS = ""
+	c.DockerExecHandlerName = "native"
+	c.ImageGCHighThresholdPercent = 90
+	c.ImageGCLowThresholdPercent = 80
+	c.LowDiskSpaceThresholdMB = 256
+	c.VolumeStatsAggPeriod.Duration = time.Minute
+	c.CgroupRoot = ""
+	c.ContainerRuntime = "docker"
+	c.CPUCFSQuota = true
+	c.RuntimeCgroups = ""
+	c.EnableControllerAttachDetach = false
+	c.EnableCustomMetrics = false
+	c.EnableDebuggingHandlers = true
+	c.EnableServer = true
+	c.CgroupsPerQOS = false
+	// Since this kubelet runs with --configure-cbr0=false, it needs to use
+	// hairpin-veth to allow hairpin packets. Note that this deviates from
+	// what the "real" kubelet currently does, because there's no way to
+	// set promiscuous mode on docker0.
+	c.HairpinMode = componentconfig.HairpinVeth
+	c.MaxContainerCount = 100
+	c.MaxOpenFiles = 1024
+	c.MaxPerPodContainerCount = 2
+	c.NvidiaGPUs = 0
+	c.RegisterNode = true
+	c.RegisterSchedulable = true
+	c.RegistryBurst = 10
+	c.RegistryPullQPS = 5.0
+	c.ResolverConfig = kubetypes.ResolvConfDefault
+	c.KubeletCgroups = "/kubelet"
+	c.SerializeImagePulls = true
+	c.SystemCgroups = ""
+	c.ProtectKernelDefaults = false
+
+	// TODO(mtaufen): Note that PodInfraContainerImage was being set to the empty value before,
+	//                but this may not have been intentional. (previous code (SimpleKubelet)
+	//                was peeling it off of a componentconfig.KubeletConfiguration{}, but may
+	//                have actually wanted the default).
+	//                The default will be present in the KubeletConfiguration contstructed above.
+
+	return c
+
+}