Skip to content

K
k3s
Commit 2244bfed authored by Kubernetes Submit Queue's avatar Kubernetes Submit Queue Committed by GitHub
Browse files
Options

Merge pull request #35137 from dashpole/per_container_inode_eviction

Automatic merge from submit-queue Eviction manager evicts based on inode consumption Fixes: #32526 Integrate Cadvisor per-container inode stats into the summary api. Make the eviction manager act based on inode consumption to evict pods using the most inodes. This PR is pending on a cadvisor godeps update which will be included in PR #35136
parents 1fa83690 4ca7f9f9
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 105 additions and 36 deletions
pkg/kubelet/api/v1alpha1/stats/types.go
View file @ 2244bfed
......@@ -208,6 +208,10 @@ type FsStats struct {
// Inodes represents the total inodes in the filesystem.
// +optional
Inodes *uint64 `json:"inodes,omitempty"`
// InodesUsed represents the inodes used by the filesystem
// This may not equal Inodes - InodesFree because this filesystem may share inodes with other "filesystems"
// e.g. For ContainerStats.Rootfs, this is the inodes used only by that container, and does not count inodes used by other containers.
InodesUsed *uint64 `json:"inodesUsed,omitempty"`
}
// UserDefinedMetricType defines how the metric should be interpreted by the user.
......
pkg/kubelet/eviction/helpers.go
View file @ 2244bfed
......@@ -319,14 +319,11 @@ func diskUsage(fsStats *statsapi.FsStats) *resource.Quantity {
// inodeUsage converts inodes consumed into a resource quantity.
func inodeUsage(fsStats *statsapi.FsStats) *resource.Quantity {
// TODO: cadvisor needs to support inodes used per container
// right now, cadvisor reports total inodes and inodes free per filesystem.
// this is insufficient to know how many inodes are consumed by the container.
// for example, with the overlay driver, the rootfs and each container filesystem
// will report the same total inode and inode free values but no way of knowing
// how many inodes consumed in that filesystem are charged to this container.
// for now, we report 0 as inode usage pending support in cadvisor.
return resource.NewQuantity(int64(0), resource.BinarySI)
if fsStats == nil || fsStats.InodesUsed == nil {
return &resource.Quantity{Format: resource.BinarySI}
}
usage := int64(*fsStats.InodesUsed)
return resource.NewQuantity(usage, resource.BinarySI)
}
// memoryUsage converts working set into a resource quantity.
......
pkg/kubelet/eviction/helpers_test.go
View file @ 2244bfed
......@@ -418,8 +418,17 @@ func TestOrderedByQoS(t *testing.T) {
}
}
// TestOrderedByDisk ensures we order pods by greediest disk consumer
func TestOrderedByDisk(t *testing.T) {
func TestOrderedbyDisk(t *testing.T) {
testOrderedByResource(t, resourceDisk, newPodDiskStats)
}
func TestOrderedbyInodes(t *testing.T) {
testOrderedByResource(t, resourceInodes, newPodInodeStats)
}
// testOrderedByDisk ensures we order pods by greediest resource consumer
func testOrderedByResource(t *testing.T, orderedByResource api.ResourceName,
newPodStatsFunc func(pod *api.Pod, rootFsUsed, logsUsed, perLocalVolumeUsed resource.Quantity) statsapi.PodStats) {
pod1 := newPod("best-effort-high", []api.Container{
newContainer("best-effort-high", newResourceList("", ""), newResourceList("", "")),
}, []api.Volume{
......@@ -463,19 +472,19 @@ func TestOrderedByDisk(t *testing.T) {
}),
})
stats := map[*api.Pod]statsapi.PodStats{
pod1: newPodDiskStats(pod1, resource.MustParse("50Mi"), resource.MustParse("100Mi"), resource.MustParse("50Mi")), // 200Mi
pod2: newPodDiskStats(pod2, resource.MustParse("100Mi"), resource.MustParse("150Mi"), resource.MustParse("50Mi")), // 300Mi
pod3: newPodDiskStats(pod3, resource.MustParse("200Mi"), resource.MustParse("150Mi"), resource.MustParse("50Mi")), // 400Mi
pod4: newPodDiskStats(pod4, resource.MustParse("300Mi"), resource.MustParse("100Mi"), resource.MustParse("50Mi")), // 450Mi
pod5: newPodDiskStats(pod5, resource.MustParse("400Mi"), resource.MustParse("100Mi"), resource.MustParse("50Mi")), // 550Mi
pod6: newPodDiskStats(pod6, resource.MustParse("500Mi"), resource.MustParse("100Mi"), resource.MustParse("50Mi")), // 650Mi
pod1: newPodStatsFunc(pod1, resource.MustParse("50Mi"), resource.MustParse("100Mi"), resource.MustParse("50Mi")), // 200Mi
pod2: newPodStatsFunc(pod2, resource.MustParse("100Mi"), resource.MustParse("150Mi"), resource.MustParse("50Mi")), // 300Mi
pod3: newPodStatsFunc(pod3, resource.MustParse("200Mi"), resource.MustParse("150Mi"), resource.MustParse("50Mi")), // 400Mi
pod4: newPodStatsFunc(pod4, resource.MustParse("300Mi"), resource.MustParse("100Mi"), resource.MustParse("50Mi")), // 450Mi
pod5: newPodStatsFunc(pod5, resource.MustParse("400Mi"), resource.MustParse("100Mi"), resource.MustParse("50Mi")), // 550Mi
pod6: newPodStatsFunc(pod6, resource.MustParse("500Mi"), resource.MustParse("100Mi"), resource.MustParse("50Mi")), // 650Mi
}
statsFn := func(pod *api.Pod) (statsapi.PodStats, bool) {
result, found := stats[pod]
return result, found
}
pods := []*api.Pod{pod1, pod2, pod3, pod4, pod5, pod6}
orderedBy(disk(statsFn, []fsStatsType{fsStatsRoot, fsStatsLogs, fsStatsLocalVolumeSource}, resourceDisk)).Sort(pods)
orderedBy(disk(statsFn, []fsStatsType{fsStatsRoot, fsStatsLogs, fsStatsLocalVolumeSource}, orderedByResource)).Sort(pods)
expected := []*api.Pod{pod6, pod5, pod4, pod3, pod2, pod1}
for i := range expected {
if pods[i] != expected[i] {
......@@ -484,8 +493,17 @@ func TestOrderedByDisk(t *testing.T) {
}
}
// TestOrderedByQoSDisk ensures we order pods by qos and then greediest disk consumer
func TestOrderedByQoSDisk(t *testing.T) {
func TestOrderedbyQoSDisk(t *testing.T) {
testOrderedByQoSResource(t, resourceDisk, newPodDiskStats)
}
func TestOrderedbyQoSInodes(t *testing.T) {
testOrderedByQoSResource(t, resourceInodes, newPodInodeStats)
}
// testOrderedByQoSDisk ensures we order pods by qos and then greediest resource consumer
func testOrderedByQoSResource(t *testing.T, orderedByResource api.ResourceName,
newPodStatsFunc func(pod *api.Pod, rootFsUsed, logsUsed, perLocalVolumeUsed resource.Quantity) statsapi.PodStats) {
pod1 := newPod("best-effort-high", []api.Container{
newContainer("best-effort-high", newResourceList("", ""), newResourceList("", "")),
}, []api.Volume{
......@@ -529,19 +547,19 @@ func TestOrderedByQoSDisk(t *testing.T) {
}),
})
stats := map[*api.Pod]statsapi.PodStats{
pod1: newPodDiskStats(pod1, resource.MustParse("50Mi"), resource.MustParse("100Mi"), resource.MustParse("50Mi")), // 200Mi
pod2: newPodDiskStats(pod2, resource.MustParse("100Mi"), resource.MustParse("150Mi"), resource.MustParse("50Mi")), // 300Mi
pod3: newPodDiskStats(pod3, resource.MustParse("200Mi"), resource.MustParse("150Mi"), resource.MustParse("50Mi")), // 400Mi
pod4: newPodDiskStats(pod4, resource.MustParse("300Mi"), resource.MustParse("100Mi"), resource.MustParse("50Mi")), // 450Mi
pod5: newPodDiskStats(pod5, resource.MustParse("400Mi"), resource.MustParse("100Mi"), resource.MustParse("50Mi")), // 550Mi
pod6: newPodDiskStats(pod6, resource.MustParse("500Mi"), resource.MustParse("100Mi"), resource.MustParse("50Mi")), // 650Mi
pod1: newPodStatsFunc(pod1, resource.MustParse("50Mi"), resource.MustParse("100Mi"), resource.MustParse("50Mi")), // 200Mi
pod2: newPodStatsFunc(pod2, resource.MustParse("100Mi"), resource.MustParse("150Mi"), resource.MustParse("50Mi")), // 300Mi
pod3: newPodStatsFunc(pod3, resource.MustParse("200Mi"), resource.MustParse("150Mi"), resource.MustParse("50Mi")), // 400Mi
pod4: newPodStatsFunc(pod4, resource.MustParse("300Mi"), resource.MustParse("100Mi"), resource.MustParse("50Mi")), // 450Mi
pod5: newPodStatsFunc(pod5, resource.MustParse("400Mi"), resource.MustParse("100Mi"), resource.MustParse("50Mi")), // 550Mi
pod6: newPodStatsFunc(pod6, resource.MustParse("500Mi"), resource.MustParse("100Mi"), resource.MustParse("50Mi")), // 650Mi
}
statsFn := func(pod *api.Pod) (statsapi.PodStats, bool) {
result, found := stats[pod]
return result, found
}
pods := []*api.Pod{pod1, pod2, pod3, pod4, pod5, pod6}
orderedBy(qosComparator, disk(statsFn, []fsStatsType{fsStatsRoot, fsStatsLogs, fsStatsLocalVolumeSource}, resourceDisk)).Sort(pods)
orderedBy(qosComparator, disk(statsFn, []fsStatsType{fsStatsRoot, fsStatsLogs, fsStatsLocalVolumeSource}, orderedByResource)).Sort(pods)
expected := []*api.Pod{pod2, pod1, pod4, pod3, pod6, pod5}
for i := range expected {
if pods[i] != expected[i] {
......@@ -1430,16 +1448,32 @@ func testCompareThresholdValue(t *testing.T) {
}
// newPodInodeStats returns stats with specified usage amounts.
// TODO: in future, this should take a value for inodesUsed per container.
func newPodInodeStats(pod *api.Pod) statsapi.PodStats {
func newPodInodeStats(pod *api.Pod, rootFsInodesUsed, logsInodesUsed, perLocalVolumeInodesUsed resource.Quantity) statsapi.PodStats {
result := statsapi.PodStats{
PodRef: statsapi.PodReference{
Name: pod.Name, Namespace: pod.Namespace, UID: string(pod.UID),
},
}
rootFsUsed := uint64(rootFsInodesUsed.Value())
logsUsed := uint64(logsInodesUsed.Value())
for range pod.Spec.Containers {
result.Containers = append(result.Containers, statsapi.ContainerStats{
Rootfs: &statsapi.FsStats{},
Rootfs: &statsapi.FsStats{
InodesUsed: &rootFsUsed,
},
Logs: &statsapi.FsStats{
InodesUsed: &logsUsed,
},
})
}
perLocalVolumeUsed := uint64(perLocalVolumeInodesUsed.Value())
for _, volumeName := range localVolumeNames(pod) {
result.VolumeStats = append(result.VolumeStats, statsapi.VolumeStats{
Name: volumeName,
FsStats: statsapi.FsStats{
InodesUsed: &perLocalVolumeUsed,
},
})
}
return result
......@@ -1528,10 +1562,12 @@ func newVolume(name string, volumeSource api.VolumeSource) api.Volume {
}
}
// newPod uses the name as the uid. Make names unique for testing.
func newPod(name string, containers []api.Container, volumes []api.Volume) *api.Pod {
return &api.Pod{
ObjectMeta: api.ObjectMeta{
Name: name,
UID: types.UID(name),
},
Spec: api.PodSpec{
Containers: containers,
......
pkg/kubelet/server/stats/summary.go
View file @ 2244bfed
......@@ -115,6 +115,18 @@ func (sb *summaryBuilder) build() (*stats.Summary, error) {
return nil, fmt.Errorf("Missing stats for root container")
}
var nodeFsInodesUsed *uint64
if sb.rootFsInfo.Inodes != nil && sb.rootFsInfo.InodesFree != nil {
nodeFsIU := *sb.rootFsInfo.Inodes - *sb.rootFsInfo.InodesFree
nodeFsInodesUsed = &nodeFsIU
}
var imageFsInodesUsed *uint64
if sb.imageFsInfo.Inodes != nil && sb.imageFsInfo.InodesFree != nil {
imageFsIU := *sb.imageFsInfo.Inodes - *sb.imageFsInfo.InodesFree
imageFsInodesUsed = &imageFsIU
}
rootStats := sb.containerInfoV2ToStats("", &rootInfo)
nodeStats := stats.NodeStats{
NodeName: sb.node.Name,
......@@ -126,7 +138,9 @@ func (sb *summaryBuilder) build() (*stats.Summary, error) {
CapacityBytes: &sb.rootFsInfo.Capacity,
UsedBytes: &sb.rootFsInfo.Usage,
InodesFree: sb.rootFsInfo.InodesFree,
Inodes: sb.rootFsInfo.Inodes},
Inodes: sb.rootFsInfo.Inodes,
InodesUsed: nodeFsInodesUsed,
},
StartTime: rootStats.StartTime,
Runtime: &stats.RuntimeStats{
ImageFs: &stats.FsStats{
......@@ -135,6 +149,7 @@ func (sb *summaryBuilder) build() (*stats.Summary, error) {
UsedBytes: &sb.imageStats.TotalStorageBytes,
InodesFree: sb.imageFsInfo.InodesFree,
Inodes: sb.imageFsInfo.Inodes,
InodesUsed: imageFsInodesUsed,
},
},
}
......@@ -174,6 +189,11 @@ func (sb *summaryBuilder) containerInfoV2FsStats(
Inodes: sb.rootFsInfo.Inodes,
}
if sb.rootFsInfo.Inodes != nil && sb.rootFsInfo.InodesFree != nil {
logsInodesUsed := *sb.rootFsInfo.Inodes - *sb.rootFsInfo.InodesFree
cs.Logs.InodesUsed = &logsInodesUsed
}
// The container rootFs lives on the imageFs devices (which may not be the node root fs)
cs.Rootfs = &stats.FsStats{
AvailableBytes: &sb.imageFsInfo.Available,
......@@ -186,12 +206,19 @@ func (sb *summaryBuilder) containerInfoV2FsStats(
return
}
cfs := lcs.Filesystem
if cfs != nil && cfs.BaseUsageBytes != nil {
rootfsUsage := *cfs.BaseUsageBytes
cs.Rootfs.UsedBytes = &rootfsUsage
if cfs.TotalUsageBytes != nil {
logsUsage := *cfs.TotalUsageBytes - *cfs.BaseUsageBytes
cs.Logs.UsedBytes = &logsUsage
if cfs != nil {
if cfs.BaseUsageBytes != nil {
rootfsUsage := *cfs.BaseUsageBytes
cs.Rootfs.UsedBytes = &rootfsUsage
if cfs.TotalUsageBytes != nil {
logsUsage := *cfs.TotalUsageBytes - *cfs.BaseUsageBytes
cs.Logs.UsedBytes = &logsUsage
}
}
if cfs.InodeUsage != nil {
rootInodes := *cfs.InodeUsage
cs.Rootfs.InodesUsed = &rootInodes
}
}
}
......
test/e2e_node/summary_test.go
View file @ 2244bfed
......@@ -111,6 +111,7 @@ var _ = framework.KubeDescribe("Summary API", func() {
"UsedBytes": bounded(kb, 10*mb),
"InodesFree": bounded(1E4, 1E8),
"Inodes": bounded(1E4, 1E8),
"InodesUsed": bounded(0, 1E8),
}),
"Logs": ptrMatchAllFields(gstruct.Fields{
"AvailableBytes": fsCapacityBounds,
......@@ -118,6 +119,7 @@ var _ = framework.KubeDescribe("Summary API", func() {
"UsedBytes": bounded(kb, 10*mb),
"InodesFree": bounded(1E4, 1E8),
"Inodes": bounded(1E4, 1E8),
"InodesUsed": bounded(0, 1E8),
}),
"UserDefinedMetrics": BeEmpty(),
}),
......@@ -139,6 +141,7 @@ var _ = framework.KubeDescribe("Summary API", func() {
// Inodes are not reported for Volumes.
"InodesFree": BeNil(),
"Inodes": BeNil(),
"InodesUsed": BeNil(),
}),
}),
}),
......@@ -179,6 +182,7 @@ var _ = framework.KubeDescribe("Summary API", func() {
"UsedBytes": bounded(kb, 10*gb),
"InodesFree": bounded(1E4, 1E8),
"Inodes": bounded(1E4, 1E8),
"InodesUsed": bounded(0, 1E8),
}),
"Runtime": ptrMatchAllFields(gstruct.Fields{
"ImageFs": ptrMatchAllFields(gstruct.Fields{
......@@ -187,6 +191,7 @@ var _ = framework.KubeDescribe("Summary API", func() {
"UsedBytes": bounded(kb, 10*gb),
"InodesFree": bounded(1E4, 1E8),
"Inodes": bounded(1E4, 1E8),
"InodesUsed": bounded(0, 1E8),
}),
}),
}),
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment