Merge pull request #68841 from krzysztof-jastrzebski/cpuandmemeory2

pkg/kubelet/server/server_test.go

@@ -258,11 +258,15 @@ func (fk *fakeKubelet) ListVolumesForPod(podUID types.UID) (map[string]volume.Vo
 
 func (_ *fakeKubelet) RootFsStats() (*statsapi.FsStats, error)                { return nil, nil }
 func (_ *fakeKubelet) ListPodStats() ([]statsapi.PodStats, error)             { return nil, nil }
+func (_ *fakeKubelet) ListPodCPUAndMemoryStats() ([]statsapi.PodStats, error) { return nil, nil }
 func (_ *fakeKubelet) ImageFsStats() (*statsapi.FsStats, error)               { return nil, nil }
 func (_ *fakeKubelet) RlimitStats() (*statsapi.RlimitStats, error)            { return nil, nil }
 func (_ *fakeKubelet) GetCgroupStats(cgroupName string, updateStats bool) (*statsapi.ContainerStats, *statsapi.NetworkStats, error) {
 	return nil, nil, nil
 }
+func (_ *fakeKubelet) GetCgroupCPUAndMemoryStats(cgroupName string, updateStats bool) (*statsapi.ContainerStats, error) {
+	return nil, nil
+}
 
 type fakeAuth struct {
 	authenticateFunc func(*http.Request) (user.Info, bool, error)

pkg/kubelet/server/stats/handler.go

@@ -43,6 +43,8 @@ type StatsProvider interface {
 	//
 	// ListPodStats returns the stats of all the containers managed by pods.
 	ListPodStats() ([]statsapi.PodStats, error)
+	// ListPodCPUAndMemoryStats returns the CPU and memory stats of all the containers managed by pods.
+	ListPodCPUAndMemoryStats() ([]statsapi.PodStats, error)
 	// ImageFsStats returns the stats of the image filesystem.
 	ImageFsStats() (*statsapi.FsStats, error)
 
@@ -51,6 +53,9 @@ type StatsProvider interface {
 	// GetCgroupStats returns the stats and the networking usage of the cgroup
 	// with the specified cgroupName.
 	GetCgroupStats(cgroupName string, updateStats bool) (*statsapi.ContainerStats, *statsapi.NetworkStats, error)
+	// GetCgroupCPUAndMemoryStats returns the CPU and memory stats of the cgroup with the specified cgroupName.
+	GetCgroupCPUAndMemoryStats(cgroupName string, updateStats bool) (*statsapi.ContainerStats, error)
+
 	// RootFsStats returns the stats of the node root filesystem.
 	RootFsStats() (*statsapi.FsStats, error)
 

pkg/kubelet/server/stats/summary.go

@@ -91,30 +91,33 @@ func (sp *summaryProviderImpl) Get(updateStats bool) (*statsapi.Summary, error)
 }
 
 func (sp *summaryProviderImpl) GetCPUAndMemoryStats() (*statsapi.Summary, error) {
-	summary, err := sp.Get(false)
+	// TODO(timstclair): Consider returning a best-effort response if any of
+	// the following errors occur.
+	node, err := sp.provider.GetNode()
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to get node info: %v", err)
 	}
-	summary.Node.Network = nil
-	summary.Node.Fs = nil
-	summary.Node.Runtime = nil
-	summary.Node.Rlimit = nil
-	for i := 0; i < len(summary.Node.SystemContainers); i++ {
-		summary.Node.SystemContainers[i].Accelerators = nil
-		summary.Node.SystemContainers[i].Rootfs = nil
-		summary.Node.SystemContainers[i].Logs = nil
-		summary.Node.SystemContainers[i].UserDefinedMetrics = nil
+	nodeConfig := sp.provider.GetNodeConfig()
+	rootStats, err := sp.provider.GetCgroupCPUAndMemoryStats("/", false)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get root cgroup stats: %v", err)
 	}
-	for i := 0; i < len(summary.Pods); i++ {
-		summary.Pods[i].Network = nil
-		summary.Pods[i].VolumeStats = nil
-		summary.Pods[i].EphemeralStorage = nil
-		for j := 0; j < len(summary.Pods[i].Containers); j++ {
-			summary.Pods[i].Containers[j].Accelerators = nil
-			summary.Pods[i].Containers[j].Rootfs = nil
-			summary.Pods[i].Containers[j].Logs = nil
-			summary.Pods[i].Containers[j].UserDefinedMetrics = nil
+
+	podStats, err := sp.provider.ListPodCPUAndMemoryStats()
+	if err != nil {
+		return nil, fmt.Errorf("failed to list pod stats: %v", err)
 	}
+
+	nodeStats := statsapi.NodeStats{
+		NodeName:         node.Name,
+		CPU:              rootStats.CPU,
+		Memory:           rootStats.Memory,
+		StartTime:        rootStats.StartTime,
+		SystemContainers: sp.GetSystemContainersCPUAndMemoryStats(nodeConfig, podStats, false),
 	}
-	return summary, nil
+	summary := statsapi.Summary{
+		Node: nodeStats,
+		Pods: podStats,
+	}
+	return &summary, nil
 }

pkg/kubelet/server/stats/summary_sys_containers.go

@@ -53,3 +53,30 @@ func (sp *summaryProviderImpl) GetSystemContainersStats(nodeConfig cm.NodeConfig
 
 	return stats
 }
+
+func (sp *summaryProviderImpl) GetSystemContainersCPUAndMemoryStats(nodeConfig cm.NodeConfig, podStats []statsapi.PodStats, updateStats bool) (stats []statsapi.ContainerStats) {
+	systemContainers := map[string]struct {
+		name             string
+		forceStatsUpdate bool
+	}{
+		statsapi.SystemContainerKubelet: {nodeConfig.KubeletCgroupsName, false},
+		statsapi.SystemContainerRuntime: {nodeConfig.RuntimeCgroupsName, false},
+		statsapi.SystemContainerMisc:    {nodeConfig.SystemCgroupsName, false},
+		statsapi.SystemContainerPods:    {sp.provider.GetPodCgroupRoot(), updateStats},
+	}
+	for sys, cont := range systemContainers {
+		// skip if cgroup name is undefined (not all system containers are required)
+		if cont.name == "" {
+			continue
+		}
+		s, err := sp.provider.GetCgroupCPUAndMemoryStats(cont.name, cont.forceStatsUpdate)
+		if err != nil {
+			glog.Errorf("Failed to get system container stats for %q: %v", cont.name, err)
+			continue
+		}
+		s.Name = sys
+		stats = append(stats, *s)
+	}
+
+	return stats
+}

pkg/kubelet/server/stats/summary_sys_containers_windows.go

@@ -27,11 +27,16 @@ import (
 )
 
 func (sp *summaryProviderImpl) GetSystemContainersStats(nodeConfig cm.NodeConfig, podStats []statsapi.PodStats, updateStats bool) (stats []statsapi.ContainerStats) {
-	stats = append(stats, sp.getSystemPodsStats(nodeConfig, podStats, updateStats))
+	stats = append(stats, sp.getSystemPodsCPUAndMemoryStats(nodeConfig, podStats, updateStats))
 	return stats
 }
 
-func (sp *summaryProviderImpl) getSystemPodsStats(nodeConfig cm.NodeConfig, podStats []statsapi.PodStats, updateStats bool) statsapi.ContainerStats {
+func (sp *summaryProviderImpl) GetSystemContainersCPUAndMemoryStats(nodeConfig cm.NodeConfig, podStats []statsapi.PodStats, updateStats bool) (stats []statsapi.ContainerStats) {
+	stats = append(stats, sp.getSystemPodsCPUAndMemoryStats(nodeConfig, podStats, updateStats))
+	return stats
+}
+
+func (sp *summaryProviderImpl) getSystemPodsCPUAndMemoryStats(nodeConfig cm.NodeConfig, podStats []statsapi.PodStats, updateStats bool) statsapi.ContainerStats {
 	now := metav1.NewTime(time.Now())
 	podsSummary := statsapi.ContainerStats{
 		StartTime: now,

pkg/kubelet/server/stats/summary_test.go

@@ -33,15 +33,6 @@ import (
 )
 
 var (
-	podStats = []statsapi.PodStats{
-		{
-			PodRef:      statsapi.PodReference{Name: "test-pod", Namespace: "test-namespace", UID: "UID_test-pod"},
-			StartTime:   metav1.NewTime(time.Now()),
-			Containers:  []statsapi.ContainerStats{*getContainerStats()},
-			Network:     getNetworkStats(),
-			VolumeStats: []statsapi.VolumeStats{*getVolumeStats()},
-		},
-	}
 	imageFsStats = getFsStats()
 	rootFsStats  = getFsStats()
 	node         = &v1.Node{ObjectMeta: metav1.ObjectMeta{Name: "test-node"}}
@@ -51,7 +42,22 @@ var (
 		KubeletCgroupsName: "/kubelet",
 	}
 	cgroupRoot  = "/kubepods"
-	cgroupStatsMap = map[string]struct {
+	rlimitStats = getRlimitStats()
+)
+
+func TestSummaryProviderGetStats(t *testing.T) {
+	assert := assert.New(t)
+
+	podStats := []statsapi.PodStats{
+		{
+			PodRef:      statsapi.PodReference{Name: "test-pod", Namespace: "test-namespace", UID: "UID_test-pod"},
+			StartTime:   metav1.NewTime(time.Now()),
+			Containers:  []statsapi.ContainerStats{*getContainerStats()},
+			Network:     getNetworkStats(),
+			VolumeStats: []statsapi.VolumeStats{*getVolumeStats()},
+		},
+	}
+	cgroupStatsMap := map[string]struct {
 		cs *statsapi.ContainerStats
 		ns *statsapi.NetworkStats
 	}{
@@ -61,11 +67,6 @@ var (
 		"/kubelet": {cs: getContainerStats(), ns: getNetworkStats()},
 		"/pods":    {cs: getContainerStats(), ns: getNetworkStats()},
 	}
-	rlimitStats = getRlimitStats()
-)
-
-func TestSummaryProviderGetStats(t *testing.T) {
-	assert := assert.New(t)
 
 	mockStatsProvider := new(statstest.StatsProvider)
 	mockStatsProvider.
@@ -133,20 +134,34 @@ func TestSummaryProviderGetStats(t *testing.T) {
 func TestSummaryProviderGetCPUAndMemoryStats(t *testing.T) {
 	assert := assert.New(t)
 
+	podStats := []statsapi.PodStats{
+		{
+			PodRef:     statsapi.PodReference{Name: "test-pod", Namespace: "test-namespace", UID: "UID_test-pod"},
+			StartTime:  metav1.NewTime(time.Now()),
+			Containers: []statsapi.ContainerStats{*getContainerStats()},
+		},
+	}
+	cgroupStatsMap := map[string]struct {
+		cs *statsapi.ContainerStats
+	}{
+		"/":        {cs: getVolumeCPUAndMemoryStats()},
+		"/runtime": {cs: getVolumeCPUAndMemoryStats()},
+		"/misc":    {cs: getVolumeCPUAndMemoryStats()},
+		"/kubelet": {cs: getVolumeCPUAndMemoryStats()},
+		"/pods":    {cs: getVolumeCPUAndMemoryStats()},
+	}
+
 	mockStatsProvider := new(statstest.StatsProvider)
 	mockStatsProvider.
 		On("GetNode").Return(node, nil).
 		On("GetNodeConfig").Return(nodeConfig).
 		On("GetPodCgroupRoot").Return(cgroupRoot).
-		On("ListPodStats").Return(podStats, nil).
-		On("ImageFsStats").Return(imageFsStats, nil).
-		On("RootFsStats").Return(rootFsStats, nil).
-		On("RlimitStats").Return(rlimitStats, nil).
-		On("GetCgroupStats", "/", false).Return(cgroupStatsMap["/"].cs, cgroupStatsMap["/"].ns, nil).
-		On("GetCgroupStats", "/runtime", false).Return(cgroupStatsMap["/runtime"].cs, cgroupStatsMap["/runtime"].ns, nil).
-		On("GetCgroupStats", "/misc", false).Return(cgroupStatsMap["/misc"].cs, cgroupStatsMap["/misc"].ns, nil).
-		On("GetCgroupStats", "/kubelet", false).Return(cgroupStatsMap["/kubelet"].cs, cgroupStatsMap["/kubelet"].ns, nil).
-		On("GetCgroupStats", "/kubepods", false).Return(cgroupStatsMap["/pods"].cs, cgroupStatsMap["/pods"].ns, nil)
+		On("ListPodCPUAndMemoryStats").Return(podStats, nil).
+		On("GetCgroupCPUAndMemoryStats", "/", false).Return(cgroupStatsMap["/"].cs, nil).
+		On("GetCgroupCPUAndMemoryStats", "/runtime", false).Return(cgroupStatsMap["/runtime"].cs, nil).
+		On("GetCgroupCPUAndMemoryStats", "/misc", false).Return(cgroupStatsMap["/misc"].cs, nil).
+		On("GetCgroupCPUAndMemoryStats", "/kubelet", false).Return(cgroupStatsMap["/kubelet"].cs, nil).
+		On("GetCgroupCPUAndMemoryStats", "/kubepods", false).Return(cgroupStatsMap["/pods"].cs, nil)
 
 	provider := NewSummaryProvider(mockStatsProvider)
 	summary, err := provider.GetCPUAndMemoryStats()
@@ -201,6 +216,15 @@ func getContainerStats() *statsapi.ContainerStats {
 	f.Fuzz(v)
 	return v
 }
+func getVolumeCPUAndMemoryStats() *statsapi.ContainerStats {
+	f := fuzz.New().NilChance(0)
+	v := &statsapi.ContainerStats{}
+	f.Fuzz(&v.Name)
+	f.Fuzz(&v.StartTime)
+	f.Fuzz(v.CPU)
+	f.Fuzz(v.Memory)
+	return v
+}
 
 func getVolumeStats() *statsapi.VolumeStats {
 	f := fuzz.New().NilChance(0)

pkg/kubelet/server/stats/testing/mock_stats_provider.go

@@ -64,6 +64,29 @@ func (_m *StatsProvider) GetCgroupStats(cgroupName string, updateStats bool) (*v
 	return r0, r1, r2
 }
 
+// GetCgroupCPUAndMemoryStats provides a mock function with given fields: cgroupName, updateStats
+func (_m *StatsProvider) GetCgroupCPUAndMemoryStats(cgroupName string, updateStats bool) (*v1alpha1.ContainerStats, error) {
+	ret := _m.Called(cgroupName, updateStats)
+
+	var r0 *v1alpha1.ContainerStats
+	if rf, ok := ret.Get(0).(func(string, bool) *v1alpha1.ContainerStats); ok {
+		r0 = rf(cgroupName, updateStats)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*v1alpha1.ContainerStats)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(string, bool) error); ok {
+		r1 = rf(cgroupName, updateStats)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
 // GetPodByCgroupfs provides the pod that maps to the specified cgroup, as well
 // as whether the pod was found.
 func (_m *StatsProvider) GetPodByCgroupfs(cgroupfs string) (*corev1.Pod, bool) {
@@ -252,6 +275,29 @@ func (_m *StatsProvider) ListPodStats() ([]v1alpha1.PodStats, error) {
 	return r0, r1
 }
 
+// ListPodCPUAndMemoryStats provides a mock function with given fields:
+func (_m *StatsProvider) ListPodCPUAndMemoryStats() ([]v1alpha1.PodStats, error) {
+	ret := _m.Called()
+
+	var r0 []v1alpha1.PodStats
+	if rf, ok := ret.Get(0).(func() []v1alpha1.PodStats); ok {
+		r0 = rf()
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).([]v1alpha1.PodStats)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func() error); ok {
+		r1 = rf()
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
 // ListVolumesForPod provides a mock function with given fields: podUID
 func (_m *StatsProvider) ListVolumesForPod(podUID types.UID) (map[string]volume.Volume, bool) {
 	ret := _m.Called(podUID)

pkg/kubelet/stats/cadvisor_stats_provider.go

@@ -145,6 +145,68 @@ func (p *cadvisorStatsProvider) ListPodStats() ([]statsapi.PodStats, error) {
 	return result, nil
 }
 
+// ListPodCPUAndMemoryStats returns the cpu and memory stats of all the pod-managed containers.
+func (p *cadvisorStatsProvider) ListPodCPUAndMemoryStats() ([]statsapi.PodStats, error) {
+	infos, err := getCadvisorContainerInfo(p.cadvisor)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get container info from cadvisor: %v", err)
+	}
+	// removeTerminatedContainerInfo will also remove pod level cgroups, so save the infos into allInfos first
+	allInfos := infos
+	infos = removeTerminatedContainerInfo(infos)
+	// Map each container to a pod and update the PodStats with container data.
+	podToStats := map[statsapi.PodReference]*statsapi.PodStats{}
+	for key, cinfo := range infos {
+		// On systemd using devicemapper each mount into the container has an
+		// associated cgroup. We ignore them to ensure we do not get duplicate
+		// entries in our summary. For details on .mount units:
+		// http://man7.org/linux/man-pages/man5/systemd.mount.5.html
+		if strings.HasSuffix(key, ".mount") {
+			continue
+		}
+		// Build the Pod key if this container is managed by a Pod
+		if !isPodManagedContainer(&cinfo) {
+			continue
+		}
+		ref := buildPodRef(cinfo.Spec.Labels)
+
+		// Lookup the PodStats for the pod using the PodRef. If none exists,
+		// initialize a new entry.
+		podStats, found := podToStats[ref]
+		if !found {
+			podStats = &statsapi.PodStats{PodRef: ref}
+			podToStats[ref] = podStats
+		}
+
+		// Update the PodStats entry with the stats from the container by
+		// adding it to podStats.Containers.
+		containerName := kubetypes.GetContainerName(cinfo.Spec.Labels)
+		if containerName == leaky.PodInfraContainerName {
+			// Special case for infrastructure container which is hidden from
+			// the user and has network stats.
+			podStats.StartTime = metav1.NewTime(cinfo.Spec.CreationTime)
+		} else {
+			podStats.Containers = append(podStats.Containers, *cadvisorInfoToContainerCPUAndMemoryStats(containerName, &cinfo))
+		}
+	}
+
+	// Add each PodStats to the result.
+	result := make([]statsapi.PodStats, 0, len(podToStats))
+	for _, podStats := range podToStats {
+		podUID := types.UID(podStats.PodRef.UID)
+		// Lookup the pod-level cgroup's CPU and memory stats
+		podInfo := getCadvisorPodInfoFromPodUID(podUID, allInfos)
+		if podInfo != nil {
+			cpu, memory := cadvisorInfoToCPUandMemoryStats(podInfo)
+			podStats.CPU = cpu
+			podStats.Memory = memory
+		}
+		result = append(result, *podStats)
+	}
+
+	return result, nil
+}
+
 func calcEphemeralStorage(containers []statsapi.ContainerStats, volumes []statsapi.VolumeStats, rootFsInfo *cadvisorapiv2.FsInfo) *statsapi.FsStats {
 	result := &statsapi.FsStats{
 		Time:           metav1.NewTime(rootFsInfo.Timestamp),

pkg/kubelet/stats/cadvisor_stats_provider_test.go

@@ -258,6 +258,169 @@ func TestCadvisorListPodStats(t *testing.T) {
 	checkNetworkStats(t, "Pod2", seedPod2Infra, ps.Network)
 }
 
+func TestCadvisorListPodCPUAndMemoryStats(t *testing.T) {
+	const (
+		namespace0 = "test0"
+		namespace2 = "test2"
+	)
+	const (
+		seedRoot              = 0
+		seedRuntime           = 100
+		seedKubelet           = 200
+		seedMisc              = 300
+		seedPod0Infra         = 1000
+		seedPod0Container0    = 2000
+		seedPod0Container1    = 2001
+		seedPod1Infra         = 3000
+		seedPod1Container     = 4000
+		seedPod2Infra         = 5000
+		seedPod2Container     = 6000
+		seedEphemeralVolume1  = 10000
+		seedEphemeralVolume2  = 10001
+		seedPersistentVolume1 = 20000
+		seedPersistentVolume2 = 20001
+	)
+	const (
+		pName0 = "pod0"
+		pName1 = "pod1"
+		pName2 = "pod0" // ensure pName2 conflicts with pName0, but is in a different namespace
+	)
+	const (
+		cName00 = "c0"
+		cName01 = "c1"
+		cName10 = "c0" // ensure cName10 conflicts with cName02, but is in a different pod
+		cName20 = "c1" // ensure cName20 conflicts with cName01, but is in a different pod + namespace
+	)
+
+	prf0 := statsapi.PodReference{Name: pName0, Namespace: namespace0, UID: "UID" + pName0}
+	prf1 := statsapi.PodReference{Name: pName1, Namespace: namespace0, UID: "UID" + pName1}
+	prf2 := statsapi.PodReference{Name: pName2, Namespace: namespace2, UID: "UID" + pName2}
+	infos := map[string]cadvisorapiv2.ContainerInfo{
+		"/":              getTestContainerInfo(seedRoot, "", "", ""),
+		"/docker-daemon": getTestContainerInfo(seedRuntime, "", "", ""),
+		"/kubelet":       getTestContainerInfo(seedKubelet, "", "", ""),
+		"/system":        getTestContainerInfo(seedMisc, "", "", ""),
+		// Pod0 - Namespace0
+		"/pod0-i":  getTestContainerInfo(seedPod0Infra, pName0, namespace0, leaky.PodInfraContainerName),
+		"/pod0-c0": getTestContainerInfo(seedPod0Container0, pName0, namespace0, cName00),
+		"/pod0-c1": getTestContainerInfo(seedPod0Container1, pName0, namespace0, cName01),
+		// Pod1 - Namespace0
+		"/pod1-i":  getTestContainerInfo(seedPod1Infra, pName1, namespace0, leaky.PodInfraContainerName),
+		"/pod1-c0": getTestContainerInfo(seedPod1Container, pName1, namespace0, cName10),
+		// Pod2 - Namespace2
+		"/pod2-i":                        getTestContainerInfo(seedPod2Infra, pName2, namespace2, leaky.PodInfraContainerName),
+		"/pod2-c0":                       getTestContainerInfo(seedPod2Container, pName2, namespace2, cName20),
+		"/kubepods/burstable/podUIDpod0": getTestContainerInfo(seedPod0Infra, pName0, namespace0, leaky.PodInfraContainerName),
+		"/kubepods/podUIDpod1":           getTestContainerInfo(seedPod1Infra, pName1, namespace0, leaky.PodInfraContainerName),
+	}
+
+	// memory limit overrides for each container (used to test available bytes if a memory limit is known)
+	memoryLimitOverrides := map[string]uint64{
+		"/":        uint64(1 << 30),
+		"/pod2-c0": uint64(1 << 15),
+	}
+	for name, memoryLimitOverride := range memoryLimitOverrides {
+		info, found := infos[name]
+		if !found {
+			t.Errorf("No container defined with name %v", name)
+		}
+		info.Spec.Memory.Limit = memoryLimitOverride
+		infos[name] = info
+	}
+
+	options := cadvisorapiv2.RequestOptions{
+		IdType:    cadvisorapiv2.TypeName,
+		Count:     2,
+		Recursive: true,
+	}
+
+	mockCadvisor := new(cadvisortest.Mock)
+	mockCadvisor.
+		On("ContainerInfoV2", "/", options).Return(infos, nil)
+
+	ephemeralVolumes := []statsapi.VolumeStats{getPodVolumeStats(seedEphemeralVolume1, "ephemeralVolume1"),
+		getPodVolumeStats(seedEphemeralVolume2, "ephemeralVolume2")}
+	persistentVolumes := []statsapi.VolumeStats{getPodVolumeStats(seedPersistentVolume1, "persistentVolume1"),
+		getPodVolumeStats(seedPersistentVolume2, "persistentVolume2")}
+	volumeStats := serverstats.PodVolumeStats{
+		EphemeralVolumes:  ephemeralVolumes,
+		PersistentVolumes: persistentVolumes,
+	}
+
+	resourceAnalyzer := &fakeResourceAnalyzer{podVolumeStats: volumeStats}
+
+	p := NewCadvisorStatsProvider(mockCadvisor, resourceAnalyzer, nil, nil, nil)
+	pods, err := p.ListPodCPUAndMemoryStats()
+	assert.NoError(t, err)
+
+	assert.Equal(t, 3, len(pods))
+	indexPods := make(map[statsapi.PodReference]statsapi.PodStats, len(pods))
+	for _, pod := range pods {
+		indexPods[pod.PodRef] = pod
+	}
+
+	// Validate Pod0 Results
+	ps, found := indexPods[prf0]
+	assert.True(t, found)
+	assert.Len(t, ps.Containers, 2)
+	indexCon := make(map[string]statsapi.ContainerStats, len(ps.Containers))
+	for _, con := range ps.Containers {
+		indexCon[con.Name] = con
+	}
+	con := indexCon[cName00]
+	assert.EqualValues(t, testTime(creationTime, seedPod0Container0).Unix(), con.StartTime.Time.Unix())
+	checkCPUStats(t, "Pod0Container0", seedPod0Container0, con.CPU)
+	checkMemoryStats(t, "Pod0Conainer0", seedPod0Container0, infos["/pod0-c0"], con.Memory)
+	assert.Nil(t, con.Rootfs)
+	assert.Nil(t, con.Logs)
+	assert.Nil(t, con.Accelerators)
+	assert.Nil(t, con.UserDefinedMetrics)
+
+	con = indexCon[cName01]
+	assert.EqualValues(t, testTime(creationTime, seedPod0Container1).Unix(), con.StartTime.Time.Unix())
+	checkCPUStats(t, "Pod0Container1", seedPod0Container1, con.CPU)
+	checkMemoryStats(t, "Pod0Container1", seedPod0Container1, infos["/pod0-c1"], con.Memory)
+	assert.Nil(t, con.Rootfs)
+	assert.Nil(t, con.Logs)
+	assert.Nil(t, con.Accelerators)
+	assert.Nil(t, con.UserDefinedMetrics)
+
+	assert.EqualValues(t, testTime(creationTime, seedPod0Infra).Unix(), ps.StartTime.Time.Unix())
+	assert.Nil(t, ps.EphemeralStorage)
+	assert.Nil(t, ps.VolumeStats)
+	assert.Nil(t, ps.Network)
+	if ps.CPU != nil {
+		checkCPUStats(t, "Pod0", seedPod0Infra, ps.CPU)
+	}
+	if ps.Memory != nil {
+		checkMemoryStats(t, "Pod0", seedPod0Infra, infos["/pod0-i"], ps.Memory)
+	}
+
+	// Validate Pod1 Results
+	ps, found = indexPods[prf1]
+	assert.True(t, found)
+	assert.Len(t, ps.Containers, 1)
+	con = ps.Containers[0]
+	assert.Equal(t, cName10, con.Name)
+	checkCPUStats(t, "Pod1Container0", seedPod1Container, con.CPU)
+	checkMemoryStats(t, "Pod1Container0", seedPod1Container, infos["/pod1-c0"], con.Memory)
+	assert.Nil(t, ps.EphemeralStorage)
+	assert.Nil(t, ps.VolumeStats)
+	assert.Nil(t, ps.Network)
+
+	// Validate Pod2 Results
+	ps, found = indexPods[prf2]
+	assert.True(t, found)
+	assert.Len(t, ps.Containers, 1)
+	con = ps.Containers[0]
+	assert.Equal(t, cName20, con.Name)
+	checkCPUStats(t, "Pod2Container0", seedPod2Container, con.CPU)
+	checkMemoryStats(t, "Pod2Container0", seedPod2Container, infos["/pod2-c0"], con.Memory)
+	assert.Nil(t, ps.EphemeralStorage)
+	assert.Nil(t, ps.VolumeStats)
+	assert.Nil(t, ps.Network)
+}
+
 func TestCadvisorImagesFsStats(t *testing.T) {
 	var (
 		assert       = assert.New(t)

pkg/kubelet/stats/cri_stats_provider.go

@@ -169,6 +169,87 @@ func (p *criStatsProvider) ListPodStats() ([]statsapi.PodStats, error) {
 	return result, nil
 }
 
+// ListPodCPUAndMemoryStats returns the CPU and Memory stats of all the pod-managed containers.
+func (p *criStatsProvider) ListPodCPUAndMemoryStats() ([]statsapi.PodStats, error) {
+	containers, err := p.runtimeService.ListContainers(&runtimeapi.ContainerFilter{})
+	if err != nil {
+		return nil, fmt.Errorf("failed to list all containers: %v", err)
+	}
+
+	// Creates pod sandbox map.
+	podSandboxMap := make(map[string]*runtimeapi.PodSandbox)
+	podSandboxes, err := p.runtimeService.ListPodSandbox(&runtimeapi.PodSandboxFilter{})
+	if err != nil {
+		return nil, fmt.Errorf("failed to list all pod sandboxes: %v", err)
+	}
+	for _, s := range podSandboxes {
+		podSandboxMap[s.Id] = s
+	}
+
+	// sandboxIDToPodStats is a temporary map from sandbox ID to its pod stats.
+	sandboxIDToPodStats := make(map[string]*statsapi.PodStats)
+
+	resp, err := p.runtimeService.ListContainerStats(&runtimeapi.ContainerStatsFilter{})
+	if err != nil {
+		return nil, fmt.Errorf("failed to list all container stats: %v", err)
+	}
+
+	containers = removeTerminatedContainer(containers)
+	// Creates container map.
+	containerMap := make(map[string]*runtimeapi.Container)
+	for _, c := range containers {
+		containerMap[c.Id] = c
+	}
+
+	allInfos, err := getCadvisorContainerInfo(p.cadvisor)
+	if err != nil {
+		return nil, fmt.Errorf("failed to fetch cadvisor stats: %v", err)
+	}
+	caInfos := getCRICadvisorStats(allInfos)
+
+	for _, stats := range resp {
+		containerID := stats.Attributes.Id
+		container, found := containerMap[containerID]
+		if !found {
+			continue
+		}
+
+		podSandboxID := container.PodSandboxId
+		podSandbox, found := podSandboxMap[podSandboxID]
+		if !found {
+			continue
+		}
+
+		// Creates the stats of the pod (if not created yet) which the
+		// container belongs to.
+		ps, found := sandboxIDToPodStats[podSandboxID]
+		if !found {
+			ps = buildPodStats(podSandbox)
+			sandboxIDToPodStats[podSandboxID] = ps
+		}
+
+		// Fill available CPU and memory stats for full set of required pod stats
+		cs := p.makeContainerCPUAndMemoryStats(stats, container)
+		p.addPodCPUMemoryStats(ps, types.UID(podSandbox.Metadata.Uid), allInfos, cs)
+
+		// If cadvisor stats is available for the container, use it to populate
+		// container stats
+		caStats, caFound := caInfos[containerID]
+		if !caFound {
+			glog.V(4).Infof("Unable to find cadvisor stats for %q", containerID)
+		} else {
+			p.addCadvisorContainerStats(cs, &caStats)
+		}
+		ps.Containers = append(ps.Containers, *cs)
+	}
+
+	result := make([]statsapi.PodStats, 0, len(sandboxIDToPodStats))
+	for _, s := range sandboxIDToPodStats {
+		result = append(result, *s)
+	}
+	return result, nil
+}
+
 // ImageFsStats returns the stats of the image filesystem.
 func (p *criStatsProvider) ImageFsStats() (*statsapi.FsStats, error) {
 	resp, err := p.imageService.ImageFsInfo()
@@ -393,6 +474,33 @@ func (p *criStatsProvider) makeContainerStats(
 	return result
 }
 
+func (p *criStatsProvider) makeContainerCPUAndMemoryStats(
+	stats *runtimeapi.ContainerStats,
+	container *runtimeapi.Container,
+) *statsapi.ContainerStats {
+	result := &statsapi.ContainerStats{
+		Name: stats.Attributes.Metadata.Name,
+		// The StartTime in the summary API is the container creation time.
+		StartTime: metav1.NewTime(time.Unix(0, container.CreatedAt)),
+		CPU:       &statsapi.CPUStats{},
+		Memory:    &statsapi.MemoryStats{},
+		// UserDefinedMetrics is not supported by CRI.
+	}
+	if stats.Cpu != nil {
+		result.CPU.Time = metav1.NewTime(time.Unix(0, stats.Cpu.Timestamp))
+		if stats.Cpu.UsageCoreNanoSeconds != nil {
+			result.CPU.UsageCoreNanoSeconds = &stats.Cpu.UsageCoreNanoSeconds.Value
+		}
+	}
+	if stats.Memory != nil {
+		result.Memory.Time = metav1.NewTime(time.Unix(0, stats.Memory.Timestamp))
+		if stats.Memory.WorkingSetBytes != nil {
+			result.Memory.WorkingSetBytes = &stats.Memory.WorkingSetBytes.Value
+		}
+	}
+	return result
+}
+
 // removeTerminatedContainer returns the specified container but with
 // the stats of the terminated containers removed.
 func removeTerminatedContainer(containers []*runtimeapi.Container) []*runtimeapi.Container {

pkg/kubelet/stats/cri_stats_provider_test.go

@@ -46,8 +46,7 @@ const (
 	offsetUsage
 )
 
-func TestCRIListPodStats(t *testing.T) {
-	const (
+const (
 	seedRoot       = 0
 	seedKubelet    = 200
 	seedMisc       = 300
@@ -58,21 +57,22 @@ func TestCRIListPodStats(t *testing.T) {
 	seedContainer2 = 5000
 	seedSandbox2   = 6000
 	seedContainer3 = 7000
-	)
+)
 
-	const (
+const (
 	pName0 = "pod0"
 	pName1 = "pod1"
 	pName2 = "pod2"
-	)
+)
 
-	const (
+const (
 	cName0 = "container0-name"
 	cName1 = "container1-name"
 	cName2 = "container2-name"
 	cName3 = "container3-name"
-	)
+)
 
+func TestCRIListPodStats(t *testing.T) {
 	var (
 		imageFsMountpoint = "/test/mount/point"
 		unknownMountpoint = "/unknown/mount/point"
@@ -242,6 +242,166 @@ func TestCRIListPodStats(t *testing.T) {
 	mockCadvisor.AssertExpectations(t)
 }
 
+func TestCRIListPodCPUAndMemoryStats(t *testing.T) {
+
+	var (
+		imageFsMountpoint = "/test/mount/point"
+		unknownMountpoint = "/unknown/mount/point"
+
+		sandbox0        = makeFakePodSandbox("sandbox0-name", "sandbox0-uid", "sandbox0-ns")
+		sandbox0Cgroup  = "/" + cm.GetPodCgroupNameSuffix(types.UID(sandbox0.PodSandboxStatus.Metadata.Uid))
+		container0      = makeFakeContainer(sandbox0, cName0, 0, false)
+		containerStats0 = makeFakeContainerStats(container0, imageFsMountpoint)
+		container1      = makeFakeContainer(sandbox0, cName1, 0, false)
+		containerStats1 = makeFakeContainerStats(container1, unknownMountpoint)
+
+		sandbox1        = makeFakePodSandbox("sandbox1-name", "sandbox1-uid", "sandbox1-ns")
+		sandbox1Cgroup  = "/" + cm.GetPodCgroupNameSuffix(types.UID(sandbox1.PodSandboxStatus.Metadata.Uid))
+		container2      = makeFakeContainer(sandbox1, cName2, 0, false)
+		containerStats2 = makeFakeContainerStats(container2, imageFsMountpoint)
+
+		sandbox2        = makeFakePodSandbox("sandbox2-name", "sandbox2-uid", "sandbox2-ns")
+		sandbox2Cgroup  = "/" + cm.GetPodCgroupNameSuffix(types.UID(sandbox2.PodSandboxStatus.Metadata.Uid))
+		container3      = makeFakeContainer(sandbox2, cName3, 0, true)
+		containerStats3 = makeFakeContainerStats(container3, imageFsMountpoint)
+		container4      = makeFakeContainer(sandbox2, cName3, 1, false)
+		containerStats4 = makeFakeContainerStats(container4, imageFsMountpoint)
+	)
+
+	var (
+		mockCadvisor       = new(cadvisortest.Mock)
+		mockRuntimeCache   = new(kubecontainertest.MockRuntimeCache)
+		mockPodManager     = new(kubepodtest.MockManager)
+		resourceAnalyzer   = new(fakeResourceAnalyzer)
+		fakeRuntimeService = critest.NewFakeRuntimeService()
+	)
+
+	infos := map[string]cadvisorapiv2.ContainerInfo{
+		"/":                           getTestContainerInfo(seedRoot, "", "", ""),
+		"/kubelet":                    getTestContainerInfo(seedKubelet, "", "", ""),
+		"/system":                     getTestContainerInfo(seedMisc, "", "", ""),
+		sandbox0.PodSandboxStatus.Id:  getTestContainerInfo(seedSandbox0, pName0, sandbox0.PodSandboxStatus.Metadata.Namespace, leaky.PodInfraContainerName),
+		sandbox0Cgroup:                getTestContainerInfo(seedSandbox0, "", "", ""),
+		container0.ContainerStatus.Id: getTestContainerInfo(seedContainer0, pName0, sandbox0.PodSandboxStatus.Metadata.Namespace, cName0),
+		container1.ContainerStatus.Id: getTestContainerInfo(seedContainer1, pName0, sandbox0.PodSandboxStatus.Metadata.Namespace, cName1),
+		sandbox1.PodSandboxStatus.Id:  getTestContainerInfo(seedSandbox1, pName1, sandbox1.PodSandboxStatus.Metadata.Namespace, leaky.PodInfraContainerName),
+		sandbox1Cgroup:                getTestContainerInfo(seedSandbox1, "", "", ""),
+		container2.ContainerStatus.Id: getTestContainerInfo(seedContainer2, pName1, sandbox1.PodSandboxStatus.Metadata.Namespace, cName2),
+		sandbox2.PodSandboxStatus.Id:  getTestContainerInfo(seedSandbox2, pName2, sandbox2.PodSandboxStatus.Metadata.Namespace, leaky.PodInfraContainerName),
+		sandbox2Cgroup:                getTestContainerInfo(seedSandbox2, "", "", ""),
+		container4.ContainerStatus.Id: getTestContainerInfo(seedContainer3, pName2, sandbox2.PodSandboxStatus.Metadata.Namespace, cName3),
+	}
+
+	options := cadvisorapiv2.RequestOptions{
+		IdType:    cadvisorapiv2.TypeName,
+		Count:     2,
+		Recursive: true,
+	}
+
+	mockCadvisor.
+		On("ContainerInfoV2", "/", options).Return(infos, nil)
+	fakeRuntimeService.SetFakeSandboxes([]*critest.FakePodSandbox{
+		sandbox0, sandbox1, sandbox2,
+	})
+	fakeRuntimeService.SetFakeContainers([]*critest.FakeContainer{
+		container0, container1, container2, container3, container4,
+	})
+	fakeRuntimeService.SetFakeContainerStats([]*runtimeapi.ContainerStats{
+		containerStats0, containerStats1, containerStats2, containerStats3, containerStats4,
+	})
+
+	ephemeralVolumes := makeFakeVolumeStats([]string{"ephVolume1, ephVolumes2"})
+	persistentVolumes := makeFakeVolumeStats([]string{"persisVolume1, persisVolumes2"})
+	resourceAnalyzer.podVolumeStats = serverstats.PodVolumeStats{
+		EphemeralVolumes:  ephemeralVolumes,
+		PersistentVolumes: persistentVolumes,
+	}
+
+	provider := NewCRIStatsProvider(
+		mockCadvisor,
+		resourceAnalyzer,
+		mockPodManager,
+		mockRuntimeCache,
+		fakeRuntimeService,
+		nil,
+		nil,
+	)
+
+	stats, err := provider.ListPodCPUAndMemoryStats()
+	assert := assert.New(t)
+	assert.NoError(err)
+	assert.Equal(3, len(stats))
+
+	podStatsMap := make(map[statsapi.PodReference]statsapi.PodStats)
+	for _, s := range stats {
+		podStatsMap[s.PodRef] = s
+	}
+
+	p0 := podStatsMap[statsapi.PodReference{Name: "sandbox0-name", UID: "sandbox0-uid", Namespace: "sandbox0-ns"}]
+	assert.Equal(sandbox0.CreatedAt, p0.StartTime.UnixNano())
+	assert.Equal(2, len(p0.Containers))
+	assert.Nil(p0.EphemeralStorage)
+	assert.Nil(p0.VolumeStats)
+	assert.Nil(p0.Network)
+	checkCRIPodCPUAndMemoryStats(assert, p0, infos[sandbox0Cgroup].Stats[0])
+
+	containerStatsMap := make(map[string]statsapi.ContainerStats)
+	for _, s := range p0.Containers {
+		containerStatsMap[s.Name] = s
+	}
+
+	c0 := containerStatsMap[cName0]
+	assert.Equal(container0.CreatedAt, c0.StartTime.UnixNano())
+	checkCRICPUAndMemoryStats(assert, c0, infos[container0.ContainerStatus.Id].Stats[0])
+	assert.Nil(c0.Rootfs)
+	assert.Nil(c0.Logs)
+	assert.Nil(c0.Accelerators)
+	assert.Nil(c0.UserDefinedMetrics)
+	c1 := containerStatsMap[cName1]
+	assert.Equal(container1.CreatedAt, c1.StartTime.UnixNano())
+	checkCRICPUAndMemoryStats(assert, c1, infos[container1.ContainerStatus.Id].Stats[0])
+	assert.Nil(c1.Rootfs)
+	assert.Nil(c1.Logs)
+	assert.Nil(c1.Accelerators)
+	assert.Nil(c1.UserDefinedMetrics)
+
+	p1 := podStatsMap[statsapi.PodReference{Name: "sandbox1-name", UID: "sandbox1-uid", Namespace: "sandbox1-ns"}]
+	assert.Equal(sandbox1.CreatedAt, p1.StartTime.UnixNano())
+	assert.Equal(1, len(p1.Containers))
+	assert.Nil(p1.EphemeralStorage)
+	assert.Nil(p1.VolumeStats)
+	assert.Nil(p1.Network)
+	checkCRIPodCPUAndMemoryStats(assert, p1, infos[sandbox1Cgroup].Stats[0])
+
+	c2 := p1.Containers[0]
+	assert.Equal(cName2, c2.Name)
+	assert.Equal(container2.CreatedAt, c2.StartTime.UnixNano())
+	checkCRICPUAndMemoryStats(assert, c2, infos[container2.ContainerStatus.Id].Stats[0])
+	assert.Nil(c2.Rootfs)
+	assert.Nil(c2.Logs)
+	assert.Nil(c2.Accelerators)
+	assert.Nil(c2.UserDefinedMetrics)
+
+	p2 := podStatsMap[statsapi.PodReference{Name: "sandbox2-name", UID: "sandbox2-uid", Namespace: "sandbox2-ns"}]
+	assert.Equal(sandbox2.CreatedAt, p2.StartTime.UnixNano())
+	assert.Equal(1, len(p2.Containers))
+	assert.Nil(p2.EphemeralStorage)
+	assert.Nil(p2.VolumeStats)
+	assert.Nil(p2.Network)
+	checkCRIPodCPUAndMemoryStats(assert, p2, infos[sandbox2Cgroup].Stats[0])
+
+	c3 := p2.Containers[0]
+	assert.Equal(cName3, c3.Name)
+	assert.Equal(container4.CreatedAt, c3.StartTime.UnixNano())
+	checkCRICPUAndMemoryStats(assert, c3, infos[container4.ContainerStatus.Id].Stats[0])
+	assert.Nil(c2.Rootfs)
+	assert.Nil(c2.Logs)
+	assert.Nil(c2.Accelerators)
+	assert.Nil(c2.UserDefinedMetrics)
+
+	mockCadvisor.AssertExpectations(t)
+}
+
 func TestCRIImagesFsStats(t *testing.T) {
 	var (
 		imageFsMountpoint = "/test/mount/point"

pkg/kubelet/stats/helper.go

@@ -132,6 +132,21 @@ func cadvisorInfoToContainerStats(name string, info *cadvisorapiv2.ContainerInfo
 	return result
 }
 
+// cadvisorInfoToContainerCPUAndMemoryStats returns the statsapi.ContainerStats converted
+// from the container and filesystem info.
+func cadvisorInfoToContainerCPUAndMemoryStats(name string, info *cadvisorapiv2.ContainerInfo) *statsapi.ContainerStats {
+	result := &statsapi.ContainerStats{
+		StartTime: metav1.NewTime(info.Spec.CreationTime),
+		Name:      name,
+	}
+
+	cpu, memory := cadvisorInfoToCPUandMemoryStats(info)
+	result.CPU = cpu
+	result.Memory = memory
+
+	return result
+}
+
 // cadvisorInfoToNetworkStats returns the statsapi.NetworkStats converted from
 // the container info from cadvisor.
 func cadvisorInfoToNetworkStats(name string, info *cadvisorapiv2.ContainerInfo) *statsapi.NetworkStats {

pkg/kubelet/stats/stats_provider.go

@@ -85,6 +85,7 @@ type StatsProvider struct {
 // containers managed by pods.
 type containerStatsProvider interface {
 	ListPodStats() ([]statsapi.PodStats, error)
+	ListPodCPUAndMemoryStats() ([]statsapi.PodStats, error)
 	ImageFsStats() (*statsapi.FsStats, error)
 	ImageFsDevice() (string, error)
 }
@@ -106,6 +107,18 @@ func (p *StatsProvider) GetCgroupStats(cgroupName string, updateStats bool) (*st
 	return s, n, nil
 }
 
+// GetCgroupCPUAndMemoryStats returns the CPU and memory stats of the cgroup with the cgroupName. Note that
+// this function doesn't generate filesystem stats.
+func (p *StatsProvider) GetCgroupCPUAndMemoryStats(cgroupName string, updateStats bool) (*statsapi.ContainerStats, error) {
+	info, err := getCgroupInfo(p.cadvisor, cgroupName, updateStats)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get cgroup stats for %q: %v", cgroupName, err)
+	}
+	// Rootfs and imagefs doesn't make sense for raw cgroup.
+	s := cadvisorInfoToContainerCPUAndMemoryStats(cgroupName, info)
+	return s, nil
+}
+
 // RootFsStats returns the stats of the node root filesystem.
 func (p *StatsProvider) RootFsStats() (*statsapi.FsStats, error) {
 	rootFsInfo, err := p.cadvisor.RootFsInfo()

pkg/kubelet/stats/stats_provider_test.go

@@ -100,6 +100,39 @@ func TestGetCgroupStats(t *testing.T) {
 	mockCadvisor.AssertExpectations(t)
 }
 
+func TestGetCgroupCPUAndMemoryStats(t *testing.T) {
+	const (
+		cgroupName        = "test-cgroup-name"
+		containerInfoSeed = 1000
+		updateStats       = false
+	)
+	var (
+		mockCadvisor     = new(cadvisortest.Mock)
+		mockPodManager   = new(kubepodtest.MockManager)
+		mockRuntimeCache = new(kubecontainertest.MockRuntimeCache)
+
+		assert  = assert.New(t)
+		options = cadvisorapiv2.RequestOptions{IdType: cadvisorapiv2.TypeName, Count: 2, Recursive: false}
+
+		containerInfo    = getTestContainerInfo(containerInfoSeed, "test-pod", "test-ns", "test-container")
+		containerInfoMap = map[string]cadvisorapiv2.ContainerInfo{cgroupName: containerInfo}
+	)
+
+	mockCadvisor.On("ContainerInfoV2", cgroupName, options).Return(containerInfoMap, nil)
+
+	provider := newStatsProvider(mockCadvisor, mockPodManager, mockRuntimeCache, fakeContainerStatsProvider{})
+	cs, err := provider.GetCgroupCPUAndMemoryStats(cgroupName, updateStats)
+	assert.NoError(err)
+
+	checkCPUStats(t, "", containerInfoSeed, cs.CPU)
+	checkMemoryStats(t, "", containerInfoSeed, containerInfo, cs.Memory)
+
+	assert.Equal(cgroupName, cs.Name)
+	assert.Equal(metav1.NewTime(containerInfo.Spec.CreationTime), cs.StartTime)
+
+	mockCadvisor.AssertExpectations(t)
+}
+
 func TestRootFsStats(t *testing.T) {
 	const (
 		rootFsInfoSeed    = 1000
@@ -648,6 +681,11 @@ type fakeContainerStatsProvider struct {
 func (p fakeContainerStatsProvider) ListPodStats() ([]statsapi.PodStats, error) {
 	return nil, fmt.Errorf("not implemented")
 }
+
+func (p fakeContainerStatsProvider) ListPodCPUAndMemoryStats() ([]statsapi.PodStats, error) {
+	return nil, fmt.Errorf("not implemented")
+}
+
 func (p fakeContainerStatsProvider) ImageFsStats() (*statsapi.FsStats, error) {
 	return nil, fmt.Errorf("not implemented")
 }