collect metrics on the /kubepods cgroup on-demand

pkg/kubelet/apis/stats/v1alpha1/types.go

@@ -87,6 +87,8 @@ const (
 	SystemContainerRuntime = "runtime"
 	// SystemContainerMisc is the container name for the system container tracking non-kubernetes processes.
 	SystemContainerMisc = "misc"
+	// SystemContainerPods is the container name for the system container tracking user pods.
+	SystemContainerPods = "pods"
 )
 
 // PodStats holds pod-level unprocessed sample stats.

pkg/kubelet/cm/container_manager.go

@@ -91,6 +91,9 @@ type ContainerManager interface {
 	UpdatePluginResources(*schedulercache.NodeInfo, *lifecycle.PodAdmitAttributes) error
 
 	InternalContainerLifecycle() InternalContainerLifecycle
+
+	// GetPodCgroupRoot returns the cgroup which contains all pods.
+	GetPodCgroupRoot() string
 }
 
 type NodeConfig struct {

pkg/kubelet/cm/container_manager_linux.go

@@ -500,6 +500,11 @@ func (cm *containerManagerImpl) GetNodeConfig() NodeConfig {
 	return cm.NodeConfig
 }
 
+// GetPodCgroupRoot returns the literal cgroupfs value for the cgroup containing all pods.
+func (cm *containerManagerImpl) GetPodCgroupRoot() string {
+	return cm.cgroupManager.Name(CgroupName(cm.cgroupRoot))
+}
+
 func (cm *containerManagerImpl) GetMountedSubsystems() *CgroupSubsystems {
 	return cm.subsystems
 }

pkg/kubelet/cm/container_manager_stub.go

@@ -90,6 +90,10 @@ func (cm *containerManagerStub) InternalContainerLifecycle() InternalContainerLi
 	return &internalContainerLifecycleImpl{cpumanager.NewFakeManager()}
 }
 
+func (cm *containerManagerStub) GetPodCgroupRoot() string {
+	return ""
+}
+
 func NewStubContainerManager() ContainerManager {
 	return &containerManagerStub{}
 }

pkg/kubelet/eviction/eviction_manager.go

@@ -149,11 +149,11 @@ func (m *managerImpl) Admit(attrs *lifecycle.PodAdmitAttributes) lifecycle.PodAd
 }
 
 // Start starts the control loop to observe and response to low compute resources.
-func (m *managerImpl) Start(diskInfoProvider DiskInfoProvider, podFunc ActivePodsFunc, podCleanedUpFunc PodCleanedUpFunc, capacityProvider CapacityProvider, monitoringInterval time.Duration) {
+func (m *managerImpl) Start(diskInfoProvider DiskInfoProvider, podFunc ActivePodsFunc, podCleanedUpFunc PodCleanedUpFunc, monitoringInterval time.Duration) {
 	// start the eviction manager monitoring
 	go func() {
 		for {
-			if evictedPods := m.synchronize(diskInfoProvider, podFunc, capacityProvider); evictedPods != nil {
+			if evictedPods := m.synchronize(diskInfoProvider, podFunc); evictedPods != nil {
 				glog.Infof("eviction manager: pods %s evicted, waiting for pod to be cleaned up", format.Pods(evictedPods))
 				m.waitForPodsCleanup(podCleanedUpFunc, evictedPods)
 			} else {
@@ -219,7 +219,7 @@ func startMemoryThresholdNotifier(thresholds []evictionapi.Threshold, observatio
 
 // synchronize is the main control loop that enforces eviction thresholds.
 // Returns the pod that was killed, or nil if no pod was killed.
-func (m *managerImpl) synchronize(diskInfoProvider DiskInfoProvider, podFunc ActivePodsFunc, capacityProvider CapacityProvider) []*v1.Pod {
+func (m *managerImpl) synchronize(diskInfoProvider DiskInfoProvider, podFunc ActivePodsFunc) []*v1.Pod {
 	// if we have nothing to do, just return
 	thresholds := m.config.Thresholds
 	if len(thresholds) == 0 {
@@ -248,7 +248,7 @@ func (m *managerImpl) synchronize(diskInfoProvider DiskInfoProvider, podFunc Act
 	}
 
 	// make observations and get a function to derive pod usage stats relative to those observations.
-	observations, statsFunc := makeSignalObservations(summary, capacityProvider, activePods)
+	observations, statsFunc := makeSignalObservations(summary)
 	debugLogObservations("observations", observations)
 
 	// attempt to create a threshold notifier to improve eviction response time
@@ -259,7 +259,7 @@ func (m *managerImpl) synchronize(diskInfoProvider DiskInfoProvider, podFunc Act
 		err = startMemoryThresholdNotifier(m.config.Thresholds, observations, false, func(desc string) {
 			glog.Infof("soft memory eviction threshold crossed at %s", desc)
 			// TODO wait grace period for soft memory limit
-			m.synchronize(diskInfoProvider, podFunc, capacityProvider)
+			m.synchronize(diskInfoProvider, podFunc)
 		})
 		if err != nil {
 			glog.Warningf("eviction manager: failed to create soft memory threshold notifier: %v", err)
@@ -267,7 +267,7 @@ func (m *managerImpl) synchronize(diskInfoProvider DiskInfoProvider, podFunc Act
 		// start hard memory notification
 		err = startMemoryThresholdNotifier(m.config.Thresholds, observations, true, func(desc string) {
 			glog.Infof("hard memory eviction threshold crossed at %s", desc)
-			m.synchronize(diskInfoProvider, podFunc, capacityProvider)
+			m.synchronize(diskInfoProvider, podFunc)
 		})
 		if err != nil {
 			glog.Warningf("eviction manager: failed to create hard memory threshold notifier: %v", err)
@@ -349,7 +349,7 @@ func (m *managerImpl) synchronize(diskInfoProvider DiskInfoProvider, podFunc Act
 	m.recorder.Eventf(m.nodeRef, v1.EventTypeWarning, "EvictionThresholdMet", "Attempting to reclaim %s", resourceToReclaim)
 
 	// check if there are node-level resources we can reclaim to reduce pressure before evicting end-user pods.
-	if m.reclaimNodeLevelResources(resourceToReclaim, capacityProvider, activePods) {
+	if m.reclaimNodeLevelResources(resourceToReclaim) {
 		glog.Infof("eviction manager: able to reduce %v pressure without evicting pods.", resourceToReclaim)
 		return nil
 	}
@@ -437,7 +437,7 @@ func (m *managerImpl) waitForPodsCleanup(podCleanedUpFunc PodCleanedUpFunc, pods
 }
 
 // reclaimNodeLevelResources attempts to reclaim node level resources.  returns true if thresholds were satisfied and no pod eviction is required.
-func (m *managerImpl) reclaimNodeLevelResources(resourceToReclaim v1.ResourceName, capacityProvider CapacityProvider, pods []*v1.Pod) bool {
+func (m *managerImpl) reclaimNodeLevelResources(resourceToReclaim v1.ResourceName) bool {
 	nodeReclaimFuncs := m.resourceToNodeReclaimFuncs[resourceToReclaim]
 	for _, nodeReclaimFunc := range nodeReclaimFuncs {
 		// attempt to reclaim the pressured resource.
@@ -454,7 +454,7 @@ func (m *managerImpl) reclaimNodeLevelResources(resourceToReclaim v1.ResourceNam
 		}
 
 		// make observations and get a function to derive pod usage stats relative to those observations.
-		observations, _ := makeSignalObservations(summary, capacityProvider, pods)
+		observations, _ := makeSignalObservations(summary)
 		debugLogObservations("observations after resource reclaim", observations)
 
 		// determine the set of thresholds met independent of grace period

pkg/kubelet/eviction/helpers.go

@@ -100,9 +100,6 @@ func validSignal(signal evictionapi.Signal) bool {
 // ParseThresholdConfig parses the flags for thresholds.
 func ParseThresholdConfig(allocatableConfig []string, evictionHard, evictionSoft, evictionSoftGracePeriod, evictionMinimumReclaim map[string]string) ([]evictionapi.Threshold, error) {
 	results := []evictionapi.Threshold{}
-	allocatableThresholds := getAllocatableThreshold(allocatableConfig)
-	results = append(results, allocatableThresholds...)
-
 	hardThresholds, err := parseThresholdStatements(evictionHard)
 	if err != nil {
 		return nil, err
@@ -137,9 +134,31 @@ func ParseThresholdConfig(allocatableConfig []string, evictionHard, evictionSoft
 			}
 		}
 	}
+	for _, key := range allocatableConfig {
+		if key == kubetypes.NodeAllocatableEnforcementKey {
+			results = addAllocatableThresholds(results)
+			break
+		}
+	}
 	return results, nil
 }
 
+func addAllocatableThresholds(thresholds []evictionapi.Threshold) []evictionapi.Threshold {
+	additionalThresholds := []evictionapi.Threshold{}
+	for _, threshold := range thresholds {
+		if threshold.Signal == evictionapi.SignalMemoryAvailable && isHardEvictionThreshold(threshold) {
+			// Copy the SignalMemoryAvailable to SignalAllocatableMemoryAvailable
+			additionalThresholds = append(additionalThresholds, evictionapi.Threshold{
+				Signal:     evictionapi.SignalAllocatableMemoryAvailable,
+				Operator:   threshold.Operator,
+				Value:      threshold.Value,
+				MinReclaim: threshold.MinReclaim,
+			})
+		}
+	}
+	return append(thresholds, additionalThresholds...)
+}
+
 // parseThresholdStatements parses the input statements into a list of Threshold objects.
 func parseThresholdStatements(statements map[string]string) ([]evictionapi.Threshold, error) {
 	if len(statements) == 0 {
@@ -204,27 +223,6 @@ func parseThresholdStatement(signal evictionapi.Signal, val string) (*evictionap
 	}, nil
 }
 
-// getAllocatableThreshold returns the thresholds applicable for the allocatable configuration
-func getAllocatableThreshold(allocatableConfig []string) []evictionapi.Threshold {
-	for _, key := range allocatableConfig {
-		if key == kubetypes.NodeAllocatableEnforcementKey {
-			return []evictionapi.Threshold{
-				{
-					Signal:   evictionapi.SignalAllocatableMemoryAvailable,
-					Operator: evictionapi.OpLessThan,
-					Value: evictionapi.ThresholdValue{
-						Quantity: resource.NewQuantity(int64(0), resource.BinarySI),
-					},
-					MinReclaim: &evictionapi.ThresholdValue{
-						Quantity: resource.NewQuantity(int64(0), resource.BinarySI),
-					},
-				},
-			}
-		}
-	}
-	return []evictionapi.Threshold{}
-}
-
 // parsePercentage parses a string representing a percentage value
 func parsePercentage(input string) (float32, error) {
 	value, err := strconv.ParseFloat(strings.TrimRight(input, "%"), 32)
@@ -724,7 +722,7 @@ func (a byEvictionPriority) Less(i, j int) bool {
 }
 
 // makeSignalObservations derives observations using the specified summary provider.
-func makeSignalObservations(summary *statsapi.Summary, capacityProvider CapacityProvider, pods []*v1.Pod) (signalObservations, statsFunc) {
+func makeSignalObservations(summary *statsapi.Summary) (signalObservations, statsFunc) {
 	// build the function to work against for pod stats
 	statsFunc := cachedStatsFunc(summary.Pods)
 	// build an evaluation context for current eviction signals
@@ -737,6 +735,17 @@ func makeSignalObservations(summary *statsapi.Summary, capacityProvider Capacity
 			time:      memory.Time,
 		}
 	}
+	if allocatableContainer, err := getSysContainer(summary.Node.SystemContainers, statsapi.SystemContainerPods); err != nil {
+		glog.Errorf("eviction manager: failed to construct signal: %q error: %v", evictionapi.SignalAllocatableMemoryAvailable, err)
+	} else {
+		if memory := allocatableContainer.Memory; memory != nil && memory.AvailableBytes != nil && memory.WorkingSetBytes != nil {
+			result[evictionapi.SignalAllocatableMemoryAvailable] = signalObservation{
+				available: resource.NewQuantity(int64(*memory.AvailableBytes), resource.BinarySI),
+				capacity:  resource.NewQuantity(int64(*memory.AvailableBytes+*memory.WorkingSetBytes), resource.BinarySI),
+				time:      memory.Time,
+			}
+		}
+	}
 	if nodeFs := summary.Node.Fs; nodeFs != nil {
 		if nodeFs.AvailableBytes != nil && nodeFs.CapacityBytes != nil {
 			result[evictionapi.SignalNodeFsAvailable] = signalObservation{
@@ -771,7 +780,6 @@ func makeSignalObservations(summary *statsapi.Summary, capacityProvider Capacity
 			}
 		}
 	}
-
 	if rlimit := summary.Node.Rlimit; rlimit != nil {
 		if rlimit.NumOfRunningProcesses != nil && rlimit.MaxPID != nil {
 			available := int64(*rlimit.MaxPID) - int64(*rlimit.NumOfRunningProcesses)
@@ -782,27 +790,16 @@ func makeSignalObservations(summary *statsapi.Summary, capacityProvider Capacity
 			}
 		}
 	}
+	return result, statsFunc
+}
 
-	if memoryAllocatableCapacity, ok := capacityProvider.GetCapacity()[v1.ResourceMemory]; ok {
-		memoryAllocatableAvailable := memoryAllocatableCapacity.Copy()
-		if reserved, exists := capacityProvider.GetNodeAllocatableReservation()[v1.ResourceMemory]; exists {
-			memoryAllocatableAvailable.Sub(reserved)
-		}
-		for _, pod := range summary.Pods {
-			mu, err := podMemoryUsage(pod)
-			if err == nil {
-				memoryAllocatableAvailable.Sub(mu[v1.ResourceMemory])
-			}
-		}
-		result[evictionapi.SignalAllocatableMemoryAvailable] = signalObservation{
-			available: memoryAllocatableAvailable,
-			capacity:  &memoryAllocatableCapacity,
+func getSysContainer(sysContainers []statsapi.ContainerStats, name string) (*statsapi.ContainerStats, error) {
+	for _, cont := range sysContainers {
+		if cont.Name == name {
+			return &cont, nil
 		}
-	} else {
-		glog.Errorf("Could not find capacity information for resource %v", v1.ResourceMemory)
 	}
-
-	return result, statsFunc
+	return nil, fmt.Errorf("system container %q not found in metrics", name)
 }
 
 // thresholdsMet returns the set of thresholds that were met independent of grace period

pkg/kubelet/eviction/helpers_test.go

@@ -72,7 +72,7 @@ func TestParseThresholdConfig(t *testing.T) {
 					Signal:   evictionapi.SignalAllocatableMemoryAvailable,
 					Operator: evictionapi.OpLessThan,
 					Value: evictionapi.ThresholdValue{
-						Quantity: quantityMustParse("0"),
+						Quantity: quantityMustParse("150Mi"),
 					},
 					MinReclaim: &evictionapi.ThresholdValue{
 						Quantity: quantityMustParse("0"),
@@ -1004,6 +1004,15 @@ func TestMakeSignalObservations(t *testing.T) {
 				InodesFree:     &nodeFsInodesFree,
 				Inodes:         &nodeFsInodes,
 			},
+			SystemContainers: []statsapi.ContainerStats{
+				{
+					Name: statsapi.SystemContainerPods,
+					Memory: &statsapi.MemoryStats{
+						AvailableBytes:  &nodeAvailableBytes,
+						WorkingSetBytes: &nodeWorkingSetBytes,
+					},
+				},
+			},
 		},
 		Pods: []statsapi.PodStats{},
 	}
@@ -1017,20 +1026,19 @@ func TestMakeSignalObservations(t *testing.T) {
 		fakeStats.Pods = append(fakeStats.Pods, newPodStats(pod, containerWorkingSetBytes))
 	}
 	res := quantityMustParse("5Gi")
-	capacityProvider := newMockCapacityProvider(v1.ResourceList{v1.ResourceMemory: *quantityMustParse("5Gi")}, v1.ResourceList{v1.ResourceMemory: *quantityMustParse("0Gi")})
 	// Allocatable thresholds are always 100%.  Verify that Threshold == Capacity.
 	if res.CmpInt64(int64(allocatableMemoryCapacity)) != 0 {
 		t.Errorf("Expected Threshold %v to be equal to value %v", res.Value(), allocatableMemoryCapacity)
 	}
-	actualObservations, statsFunc := makeSignalObservations(fakeStats, capacityProvider, pods)
+	actualObservations, statsFunc := makeSignalObservations(fakeStats)
 	allocatableMemQuantity, found := actualObservations[evictionapi.SignalAllocatableMemoryAvailable]
 	if !found {
 		t.Errorf("Expected allocatable memory observation, but didnt find one")
 	}
-	if allocatableMemQuantity.available.Value() != 2*containerWorkingSetBytes {
-		t.Errorf("Expected %v, actual: %v", containerWorkingSetBytes, allocatableMemQuantity.available.Value())
+	if expectedBytes := int64(nodeAvailableBytes); allocatableMemQuantity.available.Value() != expectedBytes {
+		t.Errorf("Expected %v, actual: %v", expectedBytes, allocatableMemQuantity.available.Value())
 	}
-	if expectedBytes := int64(allocatableMemoryCapacity); allocatableMemQuantity.capacity.Value() != expectedBytes {
+	if expectedBytes := int64(nodeWorkingSetBytes + nodeAvailableBytes); allocatableMemQuantity.capacity.Value() != expectedBytes {
 		t.Errorf("Expected %v, actual: %v", expectedBytes, allocatableMemQuantity.capacity.Value())
 	}
 	memQuantity, found := actualObservations[evictionapi.SignalMemoryAvailable]

pkg/kubelet/eviction/types.go

@@ -53,7 +53,7 @@ type Config struct {
 // Manager evaluates when an eviction threshold for node stability has been met on the node.
 type Manager interface {
 	// Start starts the control loop to monitor eviction thresholds at specified interval.
-	Start(diskInfoProvider DiskInfoProvider, podFunc ActivePodsFunc, podCleanedUpFunc PodCleanedUpFunc, capacityProvider CapacityProvider, monitoringInterval time.Duration)
+	Start(diskInfoProvider DiskInfoProvider, podFunc ActivePodsFunc, podCleanedUpFunc PodCleanedUpFunc, monitoringInterval time.Duration)
 
 	// IsUnderMemoryPressure returns true if the node is under memory pressure.
 	IsUnderMemoryPressure() bool
@@ -71,14 +71,6 @@ type DiskInfoProvider interface {
 	HasDedicatedImageFs() (bool, error)
 }
 
-// CapacityProvider is responsible for providing the resource capacity and reservation information
-type CapacityProvider interface {
-	// GetCapacity returns the amount of compute resources tracked by container manager available on the node.
-	GetCapacity() v1.ResourceList
-	// GetNodeAllocatableReservation returns the amount of compute resources that have to be reserved from scheduling.
-	GetNodeAllocatableReservation() v1.ResourceList
-}
-
 // ImageGC is responsible for performing garbage collection of unused images.
 type ImageGC interface {
 	// DeleteUnusedImages deletes unused images.

pkg/kubelet/kubelet.go

@@ -1318,7 +1318,7 @@ func (kl *Kubelet) initializeRuntimeDependentModules() {
 		glog.Fatalf("Failed to start cAdvisor %v", err)
 	}
 	// eviction manager must start after cadvisor because it needs to know if the container runtime has a dedicated imagefs
-	kl.evictionManager.Start(kl.StatsProvider, kl.GetActivePods, kl.podResourcesAreReclaimed, kl.containerManager, evictionMonitoringPeriod)
+	kl.evictionManager.Start(kl.StatsProvider, kl.GetActivePods, kl.podResourcesAreReclaimed, evictionMonitoringPeriod)
 
 	// trigger on-demand stats collection once so that we have capacity information for ephemeral storage.
 	// ignore any errors, since if stats collection is not successful, the container manager will fail to start below.

pkg/kubelet/kubelet_getters.go

@@ -211,6 +211,11 @@ func (kl *Kubelet) GetNodeConfig() cm.NodeConfig {
 	return kl.containerManager.GetNodeConfig()
 }
 
+// GetPodCgroupRoot returns the listeral cgroupfs value for the cgroup containing all pods
+func (kl *Kubelet) GetPodCgroupRoot() string {
+	return kl.containerManager.GetPodCgroupRoot()
+}
+
 // GetHostIP returns host IP or nil in case of error.
 func (kl *Kubelet) GetHostIP() (net.IP, error) {
 	node, err := kl.GetNode()

pkg/kubelet/server/server_test.go

@@ -168,6 +168,7 @@ func (fk *fakeKubelet) StreamingConnectionIdleTimeout() time.Duration {
 // Unused functions
 func (_ *fakeKubelet) GetNode() (*v1.Node, error)   { return nil, nil }
 func (_ *fakeKubelet) GetNodeConfig() cm.NodeConfig { return cm.NodeConfig{} }
+func (_ *fakeKubelet) GetPodCgroupRoot() string     { return "" }
 
 func (fk *fakeKubelet) ListVolumesForPod(podUID types.UID) (map[string]volume.Volume, bool) {
 	return map[string]volume.Volume{}, true

pkg/kubelet/server/stats/handler.go

@@ -81,6 +81,9 @@ type StatsProvider interface {
 
 	// RlimitStats returns the rlimit stats of system.
 	RlimitStats() (*statsapi.RlimitStats, error)
+
+	// GetPodCgroupRoot returns the literal cgroupfs value for the cgroup containing all pods
+	GetPodCgroupRoot() string
 }
 
 type handler struct {

pkg/kubelet/server/stats/summary.go

@@ -83,19 +83,23 @@ func (sp *summaryProviderImpl) Get(updateStats bool) (*statsapi.Summary, error)
 		Rlimit:    rlimit,
 	}
 
-	systemContainers := map[string]string{
-		statsapi.SystemContainerKubelet: nodeConfig.KubeletCgroupsName,
-		statsapi.SystemContainerRuntime: nodeConfig.RuntimeCgroupsName,
-		statsapi.SystemContainerMisc:    nodeConfig.SystemCgroupsName,
+	systemContainers := map[string]struct {
+		name             string
+		forceStatsUpdate bool
+	}{
+		statsapi.SystemContainerKubelet: {nodeConfig.KubeletCgroupsName, false},
+		statsapi.SystemContainerRuntime: {nodeConfig.RuntimeCgroupsName, false},
+		statsapi.SystemContainerMisc:    {nodeConfig.SystemCgroupsName, false},
+		statsapi.SystemContainerPods:    {sp.provider.GetPodCgroupRoot(), updateStats},
 	}
-	for sys, name := range systemContainers {
+	for sys, cont := range systemContainers {
 		// skip if cgroup name is undefined (not all system containers are required)
-		if name == "" {
+		if cont.name == "" {
 			continue
 		}
-		s, _, err := sp.provider.GetCgroupStats(name, false)
+		s, _, err := sp.provider.GetCgroupStats(cont.name, cont.forceStatsUpdate)
 		if err != nil {
-			glog.Errorf("Failed to get system container stats for %q: %v", name, err)
+			glog.Errorf("Failed to get system container stats for %q: %v", cont.name, err)
 			continue
 		}
 		// System containers don't have a filesystem associated with them.

pkg/kubelet/server/stats/summary_test.go

@@ -49,6 +49,7 @@ func TestSummaryProvider(t *testing.T) {
 			SystemCgroupsName:  "/misc",
 			KubeletCgroupsName: "/kubelet",
 		}
+		cgroupRoot     = "/kubepods"
 		cgroupStatsMap = map[string]struct {
 			cs *statsapi.ContainerStats
 			ns *statsapi.NetworkStats
@@ -57,6 +58,7 @@ func TestSummaryProvider(t *testing.T) {
 			"/runtime": {cs: getContainerStats(), ns: getNetworkStats()},
 			"/misc":    {cs: getContainerStats(), ns: getNetworkStats()},
 			"/kubelet": {cs: getContainerStats(), ns: getNetworkStats()},
+			"/pods":    {cs: getContainerStats(), ns: getNetworkStats()},
 		}
 		rlimitStats = getRlimitStats()
 	)
@@ -67,6 +69,7 @@ func TestSummaryProvider(t *testing.T) {
 	mockStatsProvider.
 		On("GetNode").Return(node, nil).
 		On("GetNodeConfig").Return(nodeConfig).
+		On("GetPodCgroupRoot").Return(cgroupRoot).
 		On("ListPodStats").Return(podStats, nil).
 		On("ImageFsStats").Return(imageFsStats, nil).
 		On("RootFsStats").Return(rootFsStats, nil).
@@ -74,7 +77,8 @@ func TestSummaryProvider(t *testing.T) {
 		On("GetCgroupStats", "/", true).Return(cgroupStatsMap["/"].cs, cgroupStatsMap["/"].ns, nil).
 		On("GetCgroupStats", "/runtime", false).Return(cgroupStatsMap["/runtime"].cs, cgroupStatsMap["/runtime"].ns, nil).
 		On("GetCgroupStats", "/misc", false).Return(cgroupStatsMap["/misc"].cs, cgroupStatsMap["/misc"].ns, nil).
-		On("GetCgroupStats", "/kubelet", false).Return(cgroupStatsMap["/kubelet"].cs, cgroupStatsMap["/kubelet"].ns, nil)
+		On("GetCgroupStats", "/kubelet", false).Return(cgroupStatsMap["/kubelet"].cs, cgroupStatsMap["/kubelet"].ns, nil).
+		On("GetCgroupStats", "/kubepods", true).Return(cgroupStatsMap["/pods"].cs, cgroupStatsMap["/pods"].ns, nil)
 
 	provider := NewSummaryProvider(mockStatsProvider)
 	summary, err := provider.Get(true)
@@ -88,7 +92,7 @@ func TestSummaryProvider(t *testing.T) {
 	assert.Equal(summary.Node.Fs, rootFsStats)
 	assert.Equal(summary.Node.Runtime, &statsapi.RuntimeStats{ImageFs: imageFsStats})
 
-	assert.Equal(len(summary.Node.SystemContainers), 3)
+	assert.Equal(len(summary.Node.SystemContainers), 4)
 	assert.Contains(summary.Node.SystemContainers, statsapi.ContainerStats{
 		Name:               "kubelet",
 		StartTime:          cgroupStatsMap["/kubelet"].cs.StartTime,
@@ -113,6 +117,14 @@ func TestSummaryProvider(t *testing.T) {
 		Accelerators:       cgroupStatsMap["/runtime"].cs.Accelerators,
 		UserDefinedMetrics: cgroupStatsMap["/runtime"].cs.UserDefinedMetrics,
 	})
+	assert.Contains(summary.Node.SystemContainers, statsapi.ContainerStats{
+		Name:               "pods",
+		StartTime:          cgroupStatsMap["/pods"].cs.StartTime,
+		CPU:                cgroupStatsMap["/pods"].cs.CPU,
+		Memory:             cgroupStatsMap["/pods"].cs.Memory,
+		Accelerators:       cgroupStatsMap["/pods"].cs.Accelerators,
+		UserDefinedMetrics: cgroupStatsMap["/pods"].cs.UserDefinedMetrics,
+	})
 	assert.Equal(summary.Pods, podStats)
 }
 

pkg/kubelet/server/stats/testing/mock_stats_provider.go

@@ -125,6 +125,20 @@ func (_m *StatsProvider) GetNodeConfig() cm.NodeConfig {
 	return r0
 }
 
+// GetPodCgroupRoot provides a mock function with given fields:
+func (_m *StatsProvider) GetPodCgroupRoot() string {
+	ret := _m.Called()
+
+	var r0 string
+	if rf, ok := ret.Get(0).(func() string); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Get(0).(string)
+	}
+
+	return r0
+}
+
 // GetPodByName provides a mock function with given fields: namespace, name
 func (_m *StatsProvider) GetPodByName(namespace string, name string) (*corev1.Pod, bool) {
 	ret := _m.Called(namespace, name)

test/e2e_node/summary_test.go

@@ -107,9 +107,21 @@ var _ = framework.KubeDescribe("Summary API", func() {
 					"UserDefinedMetrics": BeEmpty(),
 				})
 			}
+			podsContExpectations := sysContExpectations().(*gstruct.FieldsMatcher)
+			podsContExpectations.Fields["Memory"] = ptrMatchAllFields(gstruct.Fields{
+				"Time": recent(maxStatsAge),
+				// Pods are limited by Node Allocatable
+				"AvailableBytes":  bounded(1*framework.Kb, memoryLimit),
+				"UsageBytes":      bounded(10*framework.Kb, 20*framework.Mb),
+				"WorkingSetBytes": bounded(10*framework.Kb, 20*framework.Mb),
+				"RSSBytes":        bounded(1*framework.Kb, 20*framework.Mb),
+				"PageFaults":      bounded(0, 1000000),
+				"MajorPageFaults": bounded(0, 10),
+			})
 			systemContainers := gstruct.Elements{
 				"kubelet": sysContExpectations(),
 				"runtime": sysContExpectations(),
+				"pods":    podsContExpectations,
 			}
 			// The Kubelet only manages the 'misc' system container if the host is not running systemd.
 			if !systemdutil.IsRunningSystemd() {