Commit 68d2722b authored by Kubernetes Submit Queue's avatar Kubernetes Submit Queue Committed by GitHub

Merge pull request #53107 from Random-Liu/fix-cri-stats

Automatic merge from submit-queue (batch tested with PRs 53234, 53252, 53267, 53276, 53107). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>. Fix imagefs stats Without this CRI stats based summary api won't work: ```console $ curl localhost:10255/stats/summary Internal Error: failed to get root cgroup stats: failed to get imageFs info: no imagefs label for configured runtime ``` With this PR, we could get summary api from cri-containerd now: ```console $ curl localhost:10255/stats/summary { "node": { "nodeName": "127.0.0.1", "startTime": "2017-09-23T06:26:49Z", "cpu": { "time": "2017-09-27T05:12:08Z", "usageNanoCores": 275510572, "usageCoreNanoSeconds": 11924595625329 }, "memory": { "time": "2017-09-27T05:12:08Z", "availableBytes": 27737075712, "usageBytes": 6028234752, "workingSetBytes": 3884470272, "rssBytes": 652304384, "pageFaults": 98472, "majorPageFaults": 87 }, "fs": { "time": "2017-09-27T05:12:08Z", "availableBytes": 75281231872, "capacityBytes": 104022159360, "usedBytes": 28724150272, "inodesFree": 12003204, "inodes": 12800000, "inodesUsed": 796796 }, "runtime": { "imageFs": { "time": "2017-09-27T05:12:00Z", "availableBytes": 75281231872, "capacityBytes": 104022159360, "usedBytes": 247732356, "inodesFree": 12003204, "inodes": 12800000, "inodesUsed": 6103 } } }, "pods": [ { "podRef": { "name": "kube-dns-7797cb8758-qxkrz", "namespace": "kube-system", "uid": "4425b069-a342-11e7-ac90-42010af00002" }, "startTime": "2017-09-27T05:11:23Z", "containers": [ { "name": "kubedns", "startTime": "2017-09-27T05:11:24Z", "cpu": { "time": "1970-01-01T00:00:01Z", "usageCoreNanoSeconds": 154194917 }, "memory": { "time": "1970-01-01T00:00:01Z", "workingSetBytes": 7643136 }, "rootfs": { "time": "2017-09-27T05:12:00Z", "availableBytes": 75281231872, "capacityBytes": 104022159360, "usedBytes": 9, "inodesFree": 12003204, "inodes": 12800000, "inodesUsed": 32768 }, "logs": { "time": "2017-09-27T05:12:08Z", "availableBytes": 75281231872, "capacityBytes": 104022159360, "inodesFree": 12003204, "inodes": 12800000 }, "userDefinedMetrics": null }, { "name": "dnsmasq", "startTime": "2017-09-27T05:11:24Z", "cpu": { "time": "1970-01-01T00:00:01Z", "usageCoreNanoSeconds": 114482989 }, "memory": { "time": "1970-01-01T00:00:01Z", "workingSetBytes": 7966720 }, "rootfs": { "time": "2017-09-27T05:12:00Z", "availableBytes": 75281231872, "capacityBytes": 104022159360, "usedBytes": 9, "inodesFree": 12003204, "inodes": 12800000, "inodesUsed": 28675 }, "logs": { "time": "2017-09-27T05:12:08Z", "availableBytes": 75281231872, "capacityBytes": 104022159360, "inodesFree": 12003204, "inodes": 12800000 }, "userDefinedMetrics": null }, { "name": "sidecar", "startTime": "2017-09-27T05:11:24Z", "cpu": { "time": "1970-01-01T00:00:01Z", "usageCoreNanoSeconds": 140797580 }, "memory": { "time": "1970-01-01T00:00:01Z", "workingSetBytes": 7430144 }, "rootfs": { "time": "2017-09-27T05:12:00Z", "availableBytes": 75281231872, "capacityBytes": 104022159360, "usedBytes": 8, "inodesFree": 12003204, "inodes": 12800000, "inodesUsed": 28672 }, "logs": { "time": "2017-09-27T05:12:08Z", "availableBytes": 75281231872, "capacityBytes": 104022159360, "inodesFree": 12003204, "inodes": 12800000 }, "userDefinedMetrics": null } ], "volume": [ { "time": "2017-09-27T05:12:03Z", "availableBytes": 15810760704, "capacityBytes": 15810772992, "usedBytes": 12288, "inodesFree": 3860043, "inodes": 3860052, "inodesUsed": 9, "name": "kube-dns-token-l2blr" } ] } ] } ``` Signed-off-by: 's avatarLantao Liu <lantaol@google.com> ```release-note Fix the bug that query Kubelet's stats summary with CRI stats enabled results in error. ```
parents 0ca0d76d f6be1388
...@@ -74,6 +74,7 @@ func cadvisorInfoToContainerStats(name string, info *cadvisorapiv2.ContainerInfo ...@@ -74,6 +74,7 @@ func cadvisorInfoToContainerStats(name string, info *cadvisorapiv2.ContainerInfo
} }
} }
if rootFs != nil {
// The container logs live on the node rootfs device // The container logs live on the node rootfs device
result.Logs = &statsapi.FsStats{ result.Logs = &statsapi.FsStats{
Time: metav1.NewTime(cstat.Timestamp), Time: metav1.NewTime(cstat.Timestamp),
...@@ -87,7 +88,9 @@ func cadvisorInfoToContainerStats(name string, info *cadvisorapiv2.ContainerInfo ...@@ -87,7 +88,9 @@ func cadvisorInfoToContainerStats(name string, info *cadvisorapiv2.ContainerInfo
logsInodesUsed := *rootFs.Inodes - *rootFs.InodesFree logsInodesUsed := *rootFs.Inodes - *rootFs.InodesFree
result.Logs.InodesUsed = &logsInodesUsed result.Logs.InodesUsed = &logsInodesUsed
} }
}
if imageFs != nil {
// The container rootFs lives on the imageFs devices (which may not be the node root fs) // The container rootFs lives on the imageFs devices (which may not be the node root fs)
result.Rootfs = &statsapi.FsStats{ result.Rootfs = &statsapi.FsStats{
Time: metav1.NewTime(cstat.Timestamp), Time: metav1.NewTime(cstat.Timestamp),
...@@ -96,24 +99,28 @@ func cadvisorInfoToContainerStats(name string, info *cadvisorapiv2.ContainerInfo ...@@ -96,24 +99,28 @@ func cadvisorInfoToContainerStats(name string, info *cadvisorapiv2.ContainerInfo
InodesFree: imageFs.InodesFree, InodesFree: imageFs.InodesFree,
Inodes: imageFs.Inodes, Inodes: imageFs.Inodes,
} }
}
cfs := cstat.Filesystem cfs := cstat.Filesystem
if cfs != nil { if cfs != nil {
if cfs.BaseUsageBytes != nil { if cfs.BaseUsageBytes != nil {
if result.Rootfs != nil {
rootfsUsage := *cfs.BaseUsageBytes rootfsUsage := *cfs.BaseUsageBytes
result.Rootfs.UsedBytes = &rootfsUsage result.Rootfs.UsedBytes = &rootfsUsage
if cfs.TotalUsageBytes != nil { }
if cfs.TotalUsageBytes != nil && result.Logs != nil {
logsUsage := *cfs.TotalUsageBytes - *cfs.BaseUsageBytes logsUsage := *cfs.TotalUsageBytes - *cfs.BaseUsageBytes
result.Logs.UsedBytes = &logsUsage result.Logs.UsedBytes = &logsUsage
} }
} }
if cfs.InodeUsage != nil { if cfs.InodeUsage != nil && result.Rootfs != nil {
rootInodes := *cfs.InodeUsage rootInodes := *cfs.InodeUsage
result.Rootfs.InodesUsed = &rootInodes result.Rootfs.InodesUsed = &rootInodes
} }
} }
result.UserDefinedMetrics = cadvisorInfoToUserDefinedMetrics(info) result.UserDefinedMetrics = cadvisorInfoToUserDefinedMetrics(info)
return result return result
} }
......
...@@ -86,21 +86,15 @@ type containerStatsProvider interface { ...@@ -86,21 +86,15 @@ type containerStatsProvider interface {
ImageFsStats() (*statsapi.FsStats, error) ImageFsStats() (*statsapi.FsStats, error)
} }
// GetCgroupStats returns the stats of the cgroup with the cgroupName. // GetCgroupStats returns the stats of the cgroup with the cgroupName. Note that
// this function doesn't generate filesystem stats.
func (p *StatsProvider) GetCgroupStats(cgroupName string) (*statsapi.ContainerStats, *statsapi.NetworkStats, error) { func (p *StatsProvider) GetCgroupStats(cgroupName string) (*statsapi.ContainerStats, *statsapi.NetworkStats, error) {
info, err := getCgroupInfo(p.cadvisor, cgroupName) info, err := getCgroupInfo(p.cadvisor, cgroupName)
if err != nil { if err != nil {
return nil, nil, fmt.Errorf("failed to get cgroup stats for %q: %v", cgroupName, err) return nil, nil, fmt.Errorf("failed to get cgroup stats for %q: %v", cgroupName, err)
} }
rootFsInfo, err := p.cadvisor.RootFsInfo() // Rootfs and imagefs doesn't make sense for raw cgroup.
if err != nil { s := cadvisorInfoToContainerStats(cgroupName, info, nil, nil)
return nil, nil, fmt.Errorf("failed to get rootFs info: %v", err)
}
imageFsInfo, err := p.cadvisor.ImagesFsInfo()
if err != nil {
return nil, nil, fmt.Errorf("failed to get imageFs info: %v", err)
}
s := cadvisorInfoToContainerStats(cgroupName, info, &rootFsInfo, &imageFsInfo)
n := cadvisorInfoToNetworkStats(cgroupName, info) n := cadvisorInfoToNetworkStats(cgroupName, info)
return s, n, nil return s, n, nil
} }
......
...@@ -69,9 +69,7 @@ var ( ...@@ -69,9 +69,7 @@ var (
func TestGetCgroupStats(t *testing.T) { func TestGetCgroupStats(t *testing.T) {
const ( const (
cgroupName = "test-cgroup-name" cgroupName = "test-cgroup-name"
rootFsInfoSeed = 1000 containerInfoSeed = 1000
imageFsInfoSeed = 2000
containerInfoSeed = 3000
) )
var ( var (
mockCadvisor = new(cadvisortest.Mock) mockCadvisor = new(cadvisortest.Mock)
...@@ -81,16 +79,11 @@ func TestGetCgroupStats(t *testing.T) { ...@@ -81,16 +79,11 @@ func TestGetCgroupStats(t *testing.T) {
assert = assert.New(t) assert = assert.New(t)
options = cadvisorapiv2.RequestOptions{IdType: cadvisorapiv2.TypeName, Count: 2, Recursive: false} options = cadvisorapiv2.RequestOptions{IdType: cadvisorapiv2.TypeName, Count: 2, Recursive: false}
rootFsInfo = getTestFsInfo(rootFsInfoSeed)
imageFsInfo = getTestFsInfo(imageFsInfoSeed)
containerInfo = getTestContainerInfo(containerInfoSeed, "test-pod", "test-ns", "test-container") containerInfo = getTestContainerInfo(containerInfoSeed, "test-pod", "test-ns", "test-container")
containerInfoMap = map[string]cadvisorapiv2.ContainerInfo{cgroupName: containerInfo} containerInfoMap = map[string]cadvisorapiv2.ContainerInfo{cgroupName: containerInfo}
) )
mockCadvisor. mockCadvisor.On("ContainerInfoV2", cgroupName, options).Return(containerInfoMap, nil)
On("RootFsInfo").Return(rootFsInfo, nil).
On("ImagesFsInfo").Return(imageFsInfo, nil).
On("ContainerInfoV2", cgroupName, options).Return(containerInfoMap, nil)
provider := newStatsProvider(mockCadvisor, mockPodManager, mockRuntimeCache, fakeContainerStatsProvider{}) provider := newStatsProvider(mockCadvisor, mockPodManager, mockRuntimeCache, fakeContainerStatsProvider{})
cs, ns, err := provider.GetCgroupStats(cgroupName) cs, ns, err := provider.GetCgroupStats(cgroupName)
...@@ -98,21 +91,11 @@ func TestGetCgroupStats(t *testing.T) { ...@@ -98,21 +91,11 @@ func TestGetCgroupStats(t *testing.T) {
checkCPUStats(t, "", containerInfoSeed, cs.CPU) checkCPUStats(t, "", containerInfoSeed, cs.CPU)
checkMemoryStats(t, "", containerInfoSeed, containerInfo, cs.Memory) checkMemoryStats(t, "", containerInfoSeed, containerInfo, cs.Memory)
checkFsStats(t, "", imageFsInfoSeed, cs.Rootfs)
checkFsStats(t, "", rootFsInfoSeed, cs.Logs)
checkNetworkStats(t, "", containerInfoSeed, ns) checkNetworkStats(t, "", containerInfoSeed, ns)
assert.Equal(cgroupName, cs.Name) assert.Equal(cgroupName, cs.Name)
assert.Equal(metav1.NewTime(containerInfo.Spec.CreationTime), cs.StartTime) assert.Equal(metav1.NewTime(containerInfo.Spec.CreationTime), cs.StartTime)
assert.Equal(metav1.NewTime(containerInfo.Stats[0].Timestamp), cs.Rootfs.Time)
assert.Equal(*containerInfo.Stats[0].Filesystem.BaseUsageBytes, *cs.Rootfs.UsedBytes)
assert.Equal(*containerInfo.Stats[0].Filesystem.InodeUsage, *cs.Rootfs.InodesUsed)
assert.Equal(metav1.NewTime(containerInfo.Stats[0].Timestamp), cs.Logs.Time)
assert.Equal(*containerInfo.Stats[0].Filesystem.TotalUsageBytes-*containerInfo.Stats[0].Filesystem.BaseUsageBytes, *cs.Logs.UsedBytes)
assert.Equal(*rootFsInfo.Inodes-*rootFsInfo.InodesFree, *cs.Logs.InodesUsed)
mockCadvisor.AssertExpectations(t) mockCadvisor.AssertExpectations(t)
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment