Skip to content

K
k3s
Commit 2244bfed authored by Kubernetes Submit Queue's avatar Kubernetes Submit Queue Committed by GitHub
Browse files
Options

Merge pull request #35137 from dashpole/per_container_inode_eviction

Automatic merge from submit-queue Eviction manager evicts based on inode consumption Fixes: #32526 Integrate Cadvisor per-container inode stats into the summary api. Make the eviction manager act based on inode consumption to evict pods using the most inodes. This PR is pending on a cadvisor godeps update which will be included in PR #35136
parents 1fa83690 4ca7f9f9
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 105 additions and 36 deletions
pkg/kubelet/api/v1alpha1/stats/types.go
View file @ 2244bfed
...@@ -208,6 +208,10 @@ type FsStats struct { ...@@ -208,6 +208,10 @@ type FsStats struct {
// Inodes represents the total inodes in the filesystem. // Inodes represents the total inodes in the filesystem.
// +optional // +optional
Inodes *uint64 `json:"inodes,omitempty"` Inodes *uint64 `json:"inodes,omitempty"`
// InodesUsed represents the inodes used by the filesystem
// This may not equal Inodes - InodesFree because this filesystem may share inodes with other "filesystems"
// e.g. For ContainerStats.Rootfs, this is the inodes used only by that container, and does not count inodes used by other containers.
InodesUsed *uint64 `json:"inodesUsed,omitempty"`
} }
// UserDefinedMetricType defines how the metric should be interpreted by the user. // UserDefinedMetricType defines how the metric should be interpreted by the user.
......
pkg/kubelet/eviction/helpers.go
View file @ 2244bfed
...@@ -319,14 +319,11 @@ func diskUsage(fsStats *statsapi.FsStats) *resource.Quantity { ...@@ -319,14 +319,11 @@ func diskUsage(fsStats *statsapi.FsStats) *resource.Quantity {
// inodeUsage converts inodes consumed into a resource quantity. // inodeUsage converts inodes consumed into a resource quantity.
func inodeUsage(fsStats *statsapi.FsStats) *resource.Quantity { func inodeUsage(fsStats *statsapi.FsStats) *resource.Quantity {
// TODO: cadvisor needs to support inodes used per container if fsStats == nil || fsStats.InodesUsed == nil {
// right now, cadvisor reports total inodes and inodes free per filesystem. return &resource.Quantity{Format: resource.BinarySI}
// this is insufficient to know how many inodes are consumed by the container. }
// for example, with the overlay driver, the rootfs and each container filesystem usage := int64(*fsStats.InodesUsed)
// will report the same total inode and inode free values but no way of knowing return resource.NewQuantity(usage, resource.BinarySI)
// how many inodes consumed in that filesystem are charged to this container.
// for now, we report 0 as inode usage pending support in cadvisor.
return resource.NewQuantity(int64(0), resource.BinarySI)
} }
// memoryUsage converts working set into a resource quantity. // memoryUsage converts working set into a resource quantity.
......
pkg/kubelet/eviction/helpers_test.go
View file @ 2244bfed
...@@ -418,8 +418,17 @@ func TestOrderedByQoS(t *testing.T) { ...@@ -418,8 +418,17 @@ func TestOrderedByQoS(t *testing.T) {
} }
} }
// TestOrderedByDisk ensures we order pods by greediest disk consumer func TestOrderedbyDisk(t *testing.T) {
func TestOrderedByDisk(t *testing.T) { testOrderedByResource(t, resourceDisk, newPodDiskStats)
}
func TestOrderedbyInodes(t *testing.T) {
testOrderedByResource(t, resourceInodes, newPodInodeStats)
}
// testOrderedByDisk ensures we order pods by greediest resource consumer
func testOrderedByResource(t *testing.T, orderedByResource api.ResourceName,
newPodStatsFunc func(pod *api.Pod, rootFsUsed, logsUsed, perLocalVolumeUsed resource.Quantity) statsapi.PodStats) {
pod1 := newPod("best-effort-high", []api.Container{ pod1 := newPod("best-effort-high", []api.Container{
newContainer("best-effort-high", newResourceList("", ""), newResourceList("", "")), newContainer("best-effort-high", newResourceList("", ""), newResourceList("", "")),
}, []api.Volume{ }, []api.Volume{
...@@ -463,19 +472,19 @@ func TestOrderedByDisk(t *testing.T) { ...@@ -463,19 +472,19 @@ func TestOrderedByDisk(t *testing.T) {
}), }),
}) })
stats := map[*api.Pod]statsapi.PodStats{ stats := map[*api.Pod]statsapi.PodStats{
pod1: newPodDiskStats(pod1, resource.MustParse("50Mi"), resource.MustParse("100Mi"), resource.MustParse("50Mi")), // 200Mi pod1: newPodStatsFunc(pod1, resource.MustParse("50Mi"), resource.MustParse("100Mi"), resource.MustParse("50Mi")), // 200Mi
pod2: newPodDiskStats(pod2, resource.MustParse("100Mi"), resource.MustParse("150Mi"), resource.MustParse("50Mi")), // 300Mi pod2: newPodStatsFunc(pod2, resource.MustParse("100Mi"), resource.MustParse("150Mi"), resource.MustParse("50Mi")), // 300Mi
pod3: newPodDiskStats(pod3, resource.MustParse("200Mi"), resource.MustParse("150Mi"), resource.MustParse("50Mi")), // 400Mi pod3: newPodStatsFunc(pod3, resource.MustParse("200Mi"), resource.MustParse("150Mi"), resource.MustParse("50Mi")), // 400Mi
pod4: newPodDiskStats(pod4, resource.MustParse("300Mi"), resource.MustParse("100Mi"), resource.MustParse("50Mi")), // 450Mi pod4: newPodStatsFunc(pod4, resource.MustParse("300Mi"), resource.MustParse("100Mi"), resource.MustParse("50Mi")), // 450Mi
pod5: newPodDiskStats(pod5, resource.MustParse("400Mi"), resource.MustParse("100Mi"), resource.MustParse("50Mi")), // 550Mi pod5: newPodStatsFunc(pod5, resource.MustParse("400Mi"), resource.MustParse("100Mi"), resource.MustParse("50Mi")), // 550Mi
pod6: newPodDiskStats(pod6, resource.MustParse("500Mi"), resource.MustParse("100Mi"), resource.MustParse("50Mi")), // 650Mi pod6: newPodStatsFunc(pod6, resource.MustParse("500Mi"), resource.MustParse("100Mi"), resource.MustParse("50Mi")), // 650Mi
} }
statsFn := func(pod *api.Pod) (statsapi.PodStats, bool) { statsFn := func(pod *api.Pod) (statsapi.PodStats, bool) {
result, found := stats[pod] result, found := stats[pod]
return result, found return result, found
} }
pods := []*api.Pod{pod1, pod2, pod3, pod4, pod5, pod6} pods := []*api.Pod{pod1, pod2, pod3, pod4, pod5, pod6}
orderedBy(disk(statsFn, []fsStatsType{fsStatsRoot, fsStatsLogs, fsStatsLocalVolumeSource}, resourceDisk)).Sort(pods) orderedBy(disk(statsFn, []fsStatsType{fsStatsRoot, fsStatsLogs, fsStatsLocalVolumeSource}, orderedByResource)).Sort(pods)
expected := []*api.Pod{pod6, pod5, pod4, pod3, pod2, pod1} expected := []*api.Pod{pod6, pod5, pod4, pod3, pod2, pod1}
for i := range expected { for i := range expected {
if pods[i] != expected[i] { if pods[i] != expected[i] {
...@@ -484,8 +493,17 @@ func TestOrderedByDisk(t *testing.T) { ...@@ -484,8 +493,17 @@ func TestOrderedByDisk(t *testing.T) {
} }
} }
// TestOrderedByQoSDisk ensures we order pods by qos and then greediest disk consumer func TestOrderedbyQoSDisk(t *testing.T) {
func TestOrderedByQoSDisk(t *testing.T) { testOrderedByQoSResource(t, resourceDisk, newPodDiskStats)
}
func TestOrderedbyQoSInodes(t *testing.T) {
testOrderedByQoSResource(t, resourceInodes, newPodInodeStats)
}
// testOrderedByQoSDisk ensures we order pods by qos and then greediest resource consumer
func testOrderedByQoSResource(t *testing.T, orderedByResource api.ResourceName,
newPodStatsFunc func(pod *api.Pod, rootFsUsed, logsUsed, perLocalVolumeUsed resource.Quantity) statsapi.PodStats) {
pod1 := newPod("best-effort-high", []api.Container{ pod1 := newPod("best-effort-high", []api.Container{
newContainer("best-effort-high", newResourceList("", ""), newResourceList("", "")), newContainer("best-effort-high", newResourceList("", ""), newResourceList("", "")),
}, []api.Volume{ }, []api.Volume{
...@@ -529,19 +547,19 @@ func TestOrderedByQoSDisk(t *testing.T) { ...@@ -529,19 +547,19 @@ func TestOrderedByQoSDisk(t *testing.T) {
}), }),
}) })
stats := map[*api.Pod]statsapi.PodStats{ stats := map[*api.Pod]statsapi.PodStats{
pod1: newPodDiskStats(pod1, resource.MustParse("50Mi"), resource.MustParse("100Mi"), resource.MustParse("50Mi")), // 200Mi pod1: newPodStatsFunc(pod1, resource.MustParse("50Mi"), resource.MustParse("100Mi"), resource.MustParse("50Mi")), // 200Mi
pod2: newPodDiskStats(pod2, resource.MustParse("100Mi"), resource.MustParse("150Mi"), resource.MustParse("50Mi")), // 300Mi pod2: newPodStatsFunc(pod2, resource.MustParse("100Mi"), resource.MustParse("150Mi"), resource.MustParse("50Mi")), // 300Mi
pod3: newPodDiskStats(pod3, resource.MustParse("200Mi"), resource.MustParse("150Mi"), resource.MustParse("50Mi")), // 400Mi pod3: newPodStatsFunc(pod3, resource.MustParse("200Mi"), resource.MustParse("150Mi"), resource.MustParse("50Mi")), // 400Mi
pod4: newPodDiskStats(pod4, resource.MustParse("300Mi"), resource.MustParse("100Mi"), resource.MustParse("50Mi")), // 450Mi pod4: newPodStatsFunc(pod4, resource.MustParse("300Mi"), resource.MustParse("100Mi"), resource.MustParse("50Mi")), // 450Mi
pod5: newPodDiskStats(pod5, resource.MustParse("400Mi"), resource.MustParse("100Mi"), resource.MustParse("50Mi")), // 550Mi pod5: newPodStatsFunc(pod5, resource.MustParse("400Mi"), resource.MustParse("100Mi"), resource.MustParse("50Mi")), // 550Mi
pod6: newPodDiskStats(pod6, resource.MustParse("500Mi"), resource.MustParse("100Mi"), resource.MustParse("50Mi")), // 650Mi pod6: newPodStatsFunc(pod6, resource.MustParse("500Mi"), resource.MustParse("100Mi"), resource.MustParse("50Mi")), // 650Mi
} }
statsFn := func(pod *api.Pod) (statsapi.PodStats, bool) { statsFn := func(pod *api.Pod) (statsapi.PodStats, bool) {
result, found := stats[pod] result, found := stats[pod]
return result, found return result, found
} }
pods := []*api.Pod{pod1, pod2, pod3, pod4, pod5, pod6} pods := []*api.Pod{pod1, pod2, pod3, pod4, pod5, pod6}
orderedBy(qosComparator, disk(statsFn, []fsStatsType{fsStatsRoot, fsStatsLogs, fsStatsLocalVolumeSource}, resourceDisk)).Sort(pods) orderedBy(qosComparator, disk(statsFn, []fsStatsType{fsStatsRoot, fsStatsLogs, fsStatsLocalVolumeSource}, orderedByResource)).Sort(pods)
expected := []*api.Pod{pod2, pod1, pod4, pod3, pod6, pod5} expected := []*api.Pod{pod2, pod1, pod4, pod3, pod6, pod5}
for i := range expected { for i := range expected {
if pods[i] != expected[i] { if pods[i] != expected[i] {
...@@ -1430,16 +1448,32 @@ func testCompareThresholdValue(t *testing.T) { ...@@ -1430,16 +1448,32 @@ func testCompareThresholdValue(t *testing.T) {
} }
// newPodInodeStats returns stats with specified usage amounts. // newPodInodeStats returns stats with specified usage amounts.
// TODO: in future, this should take a value for inodesUsed per container. func newPodInodeStats(pod *api.Pod, rootFsInodesUsed, logsInodesUsed, perLocalVolumeInodesUsed resource.Quantity) statsapi.PodStats {
func newPodInodeStats(pod *api.Pod) statsapi.PodStats {
result := statsapi.PodStats{ result := statsapi.PodStats{
PodRef: statsapi.PodReference{ PodRef: statsapi.PodReference{
Name: pod.Name, Namespace: pod.Namespace, UID: string(pod.UID), Name: pod.Name, Namespace: pod.Namespace, UID: string(pod.UID),
}, },
} }
rootFsUsed := uint64(rootFsInodesUsed.Value())
logsUsed := uint64(logsInodesUsed.Value())
for range pod.Spec.Containers { for range pod.Spec.Containers {
result.Containers = append(result.Containers, statsapi.ContainerStats{ result.Containers = append(result.Containers, statsapi.ContainerStats{
Rootfs: &statsapi.FsStats{}, Rootfs: &statsapi.FsStats{
InodesUsed: &rootFsUsed,
},
Logs: &statsapi.FsStats{
InodesUsed: &logsUsed,
},
})
}
perLocalVolumeUsed := uint64(perLocalVolumeInodesUsed.Value())
for _, volumeName := range localVolumeNames(pod) {
result.VolumeStats = append(result.VolumeStats, statsapi.VolumeStats{
Name: volumeName,
FsStats: statsapi.FsStats{
InodesUsed: &perLocalVolumeUsed,
},
}) })
} }
return result return result
...@@ -1528,10 +1562,12 @@ func newVolume(name string, volumeSource api.VolumeSource) api.Volume { ...@@ -1528,10 +1562,12 @@ func newVolume(name string, volumeSource api.VolumeSource) api.Volume {
} }
} }
// newPod uses the name as the uid. Make names unique for testing.
func newPod(name string, containers []api.Container, volumes []api.Volume) *api.Pod { func newPod(name string, containers []api.Container, volumes []api.Volume) *api.Pod {
return &api.Pod{ return &api.Pod{
ObjectMeta: api.ObjectMeta{ ObjectMeta: api.ObjectMeta{
Name: name, Name: name,
UID: types.UID(name),
}, },
Spec: api.PodSpec{ Spec: api.PodSpec{
Containers: containers, Containers: containers,
......
pkg/kubelet/server/stats/summary.go
View file @ 2244bfed
...@@ -115,6 +115,18 @@ func (sb *summaryBuilder) build() (*stats.Summary, error) { ...@@ -115,6 +115,18 @@ func (sb *summaryBuilder) build() (*stats.Summary, error) {
return nil, fmt.Errorf("Missing stats for root container") return nil, fmt.Errorf("Missing stats for root container")
} }
var nodeFsInodesUsed *uint64
if sb.rootFsInfo.Inodes != nil && sb.rootFsInfo.InodesFree != nil {
nodeFsIU := *sb.rootFsInfo.Inodes - *sb.rootFsInfo.InodesFree
nodeFsInodesUsed = &nodeFsIU
}
var imageFsInodesUsed *uint64
if sb.imageFsInfo.Inodes != nil && sb.imageFsInfo.InodesFree != nil {
imageFsIU := *sb.imageFsInfo.Inodes - *sb.imageFsInfo.InodesFree
imageFsInodesUsed = &imageFsIU
}
rootStats := sb.containerInfoV2ToStats("", &rootInfo) rootStats := sb.containerInfoV2ToStats("", &rootInfo)
nodeStats := stats.NodeStats{ nodeStats := stats.NodeStats{
NodeName: sb.node.Name, NodeName: sb.node.Name,
...@@ -126,7 +138,9 @@ func (sb *summaryBuilder) build() (*stats.Summary, error) { ...@@ -126,7 +138,9 @@ func (sb *summaryBuilder) build() (*stats.Summary, error) {
CapacityBytes: &sb.rootFsInfo.Capacity, CapacityBytes: &sb.rootFsInfo.Capacity,
UsedBytes: &sb.rootFsInfo.Usage, UsedBytes: &sb.rootFsInfo.Usage,
InodesFree: sb.rootFsInfo.InodesFree, InodesFree: sb.rootFsInfo.InodesFree,
Inodes: sb.rootFsInfo.Inodes}, Inodes: sb.rootFsInfo.Inodes,
InodesUsed: nodeFsInodesUsed,
},
StartTime: rootStats.StartTime, StartTime: rootStats.StartTime,
Runtime: &stats.RuntimeStats{ Runtime: &stats.RuntimeStats{
ImageFs: &stats.FsStats{ ImageFs: &stats.FsStats{
...@@ -135,6 +149,7 @@ func (sb *summaryBuilder) build() (*stats.Summary, error) { ...@@ -135,6 +149,7 @@ func (sb *summaryBuilder) build() (*stats.Summary, error) {
UsedBytes: &sb.imageStats.TotalStorageBytes, UsedBytes: &sb.imageStats.TotalStorageBytes,
InodesFree: sb.imageFsInfo.InodesFree, InodesFree: sb.imageFsInfo.InodesFree,
Inodes: sb.imageFsInfo.Inodes, Inodes: sb.imageFsInfo.Inodes,
InodesUsed: imageFsInodesUsed,
}, },
}, },
} }
...@@ -174,6 +189,11 @@ func (sb *summaryBuilder) containerInfoV2FsStats( ...@@ -174,6 +189,11 @@ func (sb *summaryBuilder) containerInfoV2FsStats(
Inodes: sb.rootFsInfo.Inodes, Inodes: sb.rootFsInfo.Inodes,
} }
if sb.rootFsInfo.Inodes != nil && sb.rootFsInfo.InodesFree != nil {
logsInodesUsed := *sb.rootFsInfo.Inodes - *sb.rootFsInfo.InodesFree
cs.Logs.InodesUsed = &logsInodesUsed
}
// The container rootFs lives on the imageFs devices (which may not be the node root fs) // The container rootFs lives on the imageFs devices (which may not be the node root fs)
cs.Rootfs = &stats.FsStats{ cs.Rootfs = &stats.FsStats{
AvailableBytes: &sb.imageFsInfo.Available, AvailableBytes: &sb.imageFsInfo.Available,
...@@ -186,12 +206,19 @@ func (sb *summaryBuilder) containerInfoV2FsStats( ...@@ -186,12 +206,19 @@ func (sb *summaryBuilder) containerInfoV2FsStats(
return return
} }
cfs := lcs.Filesystem cfs := lcs.Filesystem
if cfs != nil && cfs.BaseUsageBytes != nil {
rootfsUsage := *cfs.BaseUsageBytes if cfs != nil {
cs.Rootfs.UsedBytes = &rootfsUsage if cfs.BaseUsageBytes != nil {
if cfs.TotalUsageBytes != nil { rootfsUsage := *cfs.BaseUsageBytes
logsUsage := *cfs.TotalUsageBytes - *cfs.BaseUsageBytes cs.Rootfs.UsedBytes = &rootfsUsage
cs.Logs.UsedBytes = &logsUsage if cfs.TotalUsageBytes != nil {
logsUsage := *cfs.TotalUsageBytes - *cfs.BaseUsageBytes
cs.Logs.UsedBytes = &logsUsage
}
}
if cfs.InodeUsage != nil {
rootInodes := *cfs.InodeUsage
cs.Rootfs.InodesUsed = &rootInodes
} }
} }
} }
......
test/e2e_node/summary_test.go
View file @ 2244bfed
...@@ -111,6 +111,7 @@ var _ = framework.KubeDescribe("Summary API", func() { ...@@ -111,6 +111,7 @@ var _ = framework.KubeDescribe("Summary API", func() {
"UsedBytes": bounded(kb, 10*mb), "UsedBytes": bounded(kb, 10*mb),
"InodesFree": bounded(1E4, 1E8), "InodesFree": bounded(1E4, 1E8),
"Inodes": bounded(1E4, 1E8), "Inodes": bounded(1E4, 1E8),
"InodesUsed": bounded(0, 1E8),
}), }),
"Logs": ptrMatchAllFields(gstruct.Fields{ "Logs": ptrMatchAllFields(gstruct.Fields{
"AvailableBytes": fsCapacityBounds, "AvailableBytes": fsCapacityBounds,
...@@ -118,6 +119,7 @@ var _ = framework.KubeDescribe("Summary API", func() { ...@@ -118,6 +119,7 @@ var _ = framework.KubeDescribe("Summary API", func() {
"UsedBytes": bounded(kb, 10*mb), "UsedBytes": bounded(kb, 10*mb),
"InodesFree": bounded(1E4, 1E8), "InodesFree": bounded(1E4, 1E8),
"Inodes": bounded(1E4, 1E8), "Inodes": bounded(1E4, 1E8),
"InodesUsed": bounded(0, 1E8),
}), }),
"UserDefinedMetrics": BeEmpty(), "UserDefinedMetrics": BeEmpty(),
}), }),
...@@ -139,6 +141,7 @@ var _ = framework.KubeDescribe("Summary API", func() { ...@@ -139,6 +141,7 @@ var _ = framework.KubeDescribe("Summary API", func() {
// Inodes are not reported for Volumes. // Inodes are not reported for Volumes.
"InodesFree": BeNil(), "InodesFree": BeNil(),
"Inodes": BeNil(), "Inodes": BeNil(),
"InodesUsed": BeNil(),
}), }),
}), }),
}), }),
...@@ -179,6 +182,7 @@ var _ = framework.KubeDescribe("Summary API", func() { ...@@ -179,6 +182,7 @@ var _ = framework.KubeDescribe("Summary API", func() {
"UsedBytes": bounded(kb, 10*gb), "UsedBytes": bounded(kb, 10*gb),
"InodesFree": bounded(1E4, 1E8), "InodesFree": bounded(1E4, 1E8),
"Inodes": bounded(1E4, 1E8), "Inodes": bounded(1E4, 1E8),
"InodesUsed": bounded(0, 1E8),
}), }),
"Runtime": ptrMatchAllFields(gstruct.Fields{ "Runtime": ptrMatchAllFields(gstruct.Fields{
"ImageFs": ptrMatchAllFields(gstruct.Fields{ "ImageFs": ptrMatchAllFields(gstruct.Fields{
...@@ -187,6 +191,7 @@ var _ = framework.KubeDescribe("Summary API", func() { ...@@ -187,6 +191,7 @@ var _ = framework.KubeDescribe("Summary API", func() {
"UsedBytes": bounded(kb, 10*gb), "UsedBytes": bounded(kb, 10*gb),
"InodesFree": bounded(1E4, 1E8), "InodesFree": bounded(1E4, 1E8),
"Inodes": bounded(1E4, 1E8), "Inodes": bounded(1E4, 1E8),
"InodesUsed": bounded(0, 1E8),
}), }),
}), }),
}), }),
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment