Merge pull request #63415 from dashpole/eviction_event

pkg/kubelet/eviction/BUILD

@@ -20,7 +20,6 @@ go_test(
         "//pkg/kubelet/eviction/api:go_default_library",
         "//pkg/kubelet/lifecycle:go_default_library",
         "//pkg/kubelet/types:go_default_library",
-        "//pkg/quota:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",

pkg/kubelet/eviction/api/types.go

@@ -38,8 +38,6 @@ const (
 	SignalImageFsInodesFree Signal = "imagefs.inodesFree"
 	// SignalAllocatableMemoryAvailable is amount of memory available for pod allocation (i.e. allocatable - workingSet (of pods), in bytes.
 	SignalAllocatableMemoryAvailable Signal = "allocatableMemory.available"
-	// SignalAllocatableNodeFsAvailable is amount of local storage available for pod allocation
-	SignalAllocatableNodeFsAvailable Signal = "allocatableNodeFs.available"
 	// SignalPIDAvailable is amount of PID available for pod allocation
 	SignalPIDAvailable Signal = "pid.available"
 )
@@ -60,13 +58,11 @@ const (
 // from either above or below, never both). There is thus no reason to expose the
 // operator in the Kubelet's public API. Instead, we internally map signal types to operators.
 var OpForSignal = map[Signal]ThresholdOperator{
-	SignalMemoryAvailable:            OpLessThan,
-	SignalNodeFsAvailable:            OpLessThan,
-	SignalNodeFsInodesFree:           OpLessThan,
-	SignalImageFsAvailable:           OpLessThan,
-	SignalImageFsInodesFree:          OpLessThan,
-	SignalAllocatableMemoryAvailable: OpLessThan,
-	SignalAllocatableNodeFsAvailable: OpLessThan,
+	SignalMemoryAvailable:   OpLessThan,
+	SignalNodeFsAvailable:   OpLessThan,
+	SignalNodeFsInodesFree:  OpLessThan,
+	SignalImageFsAvailable:  OpLessThan,
+	SignalImageFsInodesFree: OpLessThan,
 }
 
 // ThresholdValue is a value holder that abstracts literal versus percentage based quantity

pkg/kubelet/eviction/eviction_manager.go

@@ -76,10 +76,10 @@ type managerImpl struct {
 	thresholdsFirstObservedAt thresholdsObservedAt
 	// records the set of thresholds that have been met (including graceperiod) but not yet resolved
 	thresholdsMet []evictionapi.Threshold
-	// resourceToRankFunc maps a resource to ranking function for that resource.
-	resourceToRankFunc map[v1.ResourceName]rankFunc
-	// resourceToNodeReclaimFuncs maps a resource to an ordered list of functions that know how to reclaim that resource.
-	resourceToNodeReclaimFuncs map[v1.ResourceName]nodeReclaimFuncs
+	// signalToRankFunc maps a resource to ranking function for that resource.
+	signalToRankFunc map[evictionapi.Signal]rankFunc
+	// signalToNodeReclaimFuncs maps a resource to an ordered list of functions that know how to reclaim that resource.
+	signalToNodeReclaimFuncs map[evictionapi.Signal]nodeReclaimFuncs
 	// last observations from synchronize
 	lastObservations signalObservations
 	// notifierStopCh is a channel used to stop all thresholdNotifiers
@@ -239,8 +239,8 @@ func (m *managerImpl) synchronize(diskInfoProvider DiskInfoProvider, podFunc Act
 			return nil
 		}
 		m.dedicatedImageFs = &hasImageFs
-		m.resourceToRankFunc = buildResourceToRankFunc(hasImageFs)
-		m.resourceToNodeReclaimFuncs = buildResourceToNodeReclaimFuncs(m.imageGC, m.containerGC, hasImageFs)
+		m.signalToRankFunc = buildSignalToRankFunc(hasImageFs)
+		m.signalToNodeReclaimFuncs = buildSignalToNodeReclaimFuncs(m.imageGC, m.containerGC, hasImageFs)
 	}
 
 	activePods := podFunc()
@@ -333,26 +333,26 @@ func (m *managerImpl) synchronize(diskInfoProvider DiskInfoProvider, podFunc Act
 		}
 	}
 
-	// determine the set of resources under starvation
-	starvedResources := getStarvedResources(thresholds)
-	if len(starvedResources) == 0 {
+	if len(thresholds) == 0 {
 		glog.V(3).Infof("eviction manager: no resources are starved")
 		return nil
 	}
 
-	// rank the resources to reclaim by eviction priority
-	sort.Sort(byEvictionPriority(starvedResources))
-	resourceToReclaim := starvedResources[0]
+	// rank the thresholds by eviction priority
+	sort.Sort(byEvictionPriority(thresholds))
+	thresholdToReclaim := thresholds[0]
+	resourceToReclaim, found := signalToResource[thresholdToReclaim.Signal]
+	if !found {
+		glog.V(3).Infof("eviction manager: threshold %s was crossed, but reclaim is not implemented for this threshold.", thresholdToReclaim.Signal)
+		return nil
+	}
 	glog.Warningf("eviction manager: attempting to reclaim %v", resourceToReclaim)
 
-	// determine if this is a soft or hard eviction associated with the resource
-	softEviction := isSoftEvictionThresholds(thresholds, resourceToReclaim)
-
 	// record an event about the resources we are now attempting to reclaim via eviction
 	m.recorder.Eventf(m.nodeRef, v1.EventTypeWarning, "EvictionThresholdMet", "Attempting to reclaim %s", resourceToReclaim)
 
 	// check if there are node-level resources we can reclaim to reduce pressure before evicting end-user pods.
-	if m.reclaimNodeLevelResources(resourceToReclaim) {
+	if m.reclaimNodeLevelResources(thresholdToReclaim.Signal, resourceToReclaim) {
 		glog.Infof("eviction manager: able to reduce %v pressure without evicting pods.", resourceToReclaim)
 		return nil
 	}
@@ -360,9 +360,9 @@ func (m *managerImpl) synchronize(diskInfoProvider DiskInfoProvider, podFunc Act
 	glog.Infof("eviction manager: must evict pod(s) to reclaim %v", resourceToReclaim)
 
 	// rank the pods for eviction
-	rank, ok := m.resourceToRankFunc[resourceToReclaim]
+	rank, ok := m.signalToRankFunc[thresholdToReclaim.Signal]
 	if !ok {
-		glog.Errorf("eviction manager: no ranking function for resource %s", resourceToReclaim)
+		glog.Errorf("eviction manager: no ranking function for signal %s", thresholdToReclaim.Signal)
 		return nil
 	}
 
@@ -388,30 +388,13 @@ func (m *managerImpl) synchronize(diskInfoProvider DiskInfoProvider, podFunc Act
 	// we kill at most a single pod during each eviction interval
 	for i := range activePods {
 		pod := activePods[i]
-		// If the pod is marked as critical and static, and support for critical pod annotations is enabled,
-		// do not evict such pods. Static pods are not re-admitted after evictions.
-		// https://github.com/kubernetes/kubernetes/issues/40573 has more details.
-		if utilfeature.DefaultFeatureGate.Enabled(features.ExperimentalCriticalPodAnnotation) &&
-			kubelettypes.IsCriticalPod(pod) && kubepod.IsStaticPod(pod) {
-			continue
-		}
-		status := v1.PodStatus{
-			Phase:   v1.PodFailed,
-			Message: fmt.Sprintf(message, resourceToReclaim),
-			Reason:  reason,
-		}
-		// record that we are evicting the pod
-		m.recorder.Eventf(pod, v1.EventTypeWarning, reason, fmt.Sprintf(message, resourceToReclaim))
 		gracePeriodOverride := int64(0)
-		if softEviction {
+		if !isHardEvictionThreshold(thresholdToReclaim) {
 			gracePeriodOverride = m.config.MaxPodGracePeriodSeconds
 		}
-		// this is a blocking call and should only return when the pod and its containers are killed.
-		err := m.killPodFunc(pod, status, &gracePeriodOverride)
-		if err != nil {
-			glog.Warningf("eviction manager: error while evicting pod %s: %v", format.Pod(pod), err)
+		if m.evictPod(pod, gracePeriodOverride, evictionMessage(resourceToReclaim, pod, statsFunc)) {
+			return []*v1.Pod{pod}
 		}
-		return []*v1.Pod{pod}
 	}
 	glog.Infof("eviction manager: unable to evict any pods from the node")
 	return nil
@@ -440,8 +423,8 @@ func (m *managerImpl) waitForPodsCleanup(podCleanedUpFunc PodCleanedUpFunc, pods
 }
 
 // reclaimNodeLevelResources attempts to reclaim node level resources.  returns true if thresholds were satisfied and no pod eviction is required.
-func (m *managerImpl) reclaimNodeLevelResources(resourceToReclaim v1.ResourceName) bool {
-	nodeReclaimFuncs := m.resourceToNodeReclaimFuncs[resourceToReclaim]
+func (m *managerImpl) reclaimNodeLevelResources(signalToReclaim evictionapi.Signal, resourceToReclaim v1.ResourceName) bool {
+	nodeReclaimFuncs := m.signalToNodeReclaimFuncs[signalToReclaim]
 	for _, nodeReclaimFunc := range nodeReclaimFuncs {
 		// attempt to reclaim the pressured resource.
 		if err := nodeReclaimFunc(); err != nil {
@@ -512,7 +495,7 @@ func (m *managerImpl) emptyDirLimitEviction(podStats statsapi.PodStats, pod *v1.
 			used := podVolumeUsed[pod.Spec.Volumes[i].Name]
 			if used != nil && size != nil && size.Sign() == 1 && used.Cmp(*size) > 0 {
 				// the emptyDir usage exceeds the size limit, evict the pod
-				return m.evictPod(pod, v1.ResourceName("EmptyDir"), fmt.Sprintf("emptyDir usage exceeds the limit %q", size.String()))
+				return m.evictPod(pod, 0, fmt.Sprintf(emptyDirMessage, pod.Spec.Volumes[i].Name, size.String()))
 			}
 		}
 	}
@@ -540,10 +523,11 @@ func (m *managerImpl) podEphemeralStorageLimitEviction(podStats statsapi.PodStat
 		return false
 	}
 
-	podEphemeralStorageTotalUsage.Add(podEphemeralUsage[resourceDisk])
-	if podEphemeralStorageTotalUsage.Cmp(podLimits[v1.ResourceEphemeralStorage]) > 0 {
+	podEphemeralStorageTotalUsage.Add(podEphemeralUsage[v1.ResourceEphemeralStorage])
+	podEphemeralStorageLimit := podLimits[v1.ResourceEphemeralStorage]
+	if podEphemeralStorageTotalUsage.Cmp(podEphemeralStorageLimit) > 0 {
 		// the total usage of pod exceeds the total size limit of containers, evict the pod
-		return m.evictPod(pod, v1.ResourceEphemeralStorage, fmt.Sprintf("pod ephemeral local storage usage exceeds the total limit of containers %v", podLimits[v1.ResourceEphemeralStorage]))
+		return m.evictPod(pod, 0, fmt.Sprintf(podEphemeralStorageMessage, podEphemeralStorageLimit.String()))
 	}
 	return false
 }
@@ -565,7 +549,7 @@ func (m *managerImpl) containerEphemeralStorageLimitEviction(podStats statsapi.P
 
 		if ephemeralStorageThreshold, ok := thresholdsMap[containerStat.Name]; ok {
 			if ephemeralStorageThreshold.Cmp(*containerUsed) < 0 {
-				return m.evictPod(pod, v1.ResourceEphemeralStorage, fmt.Sprintf("container's ephemeral local storage usage exceeds the limit %q", ephemeralStorageThreshold.String()))
+				return m.evictPod(pod, 0, fmt.Sprintf(containerEphemeralStorageMessage, containerStat.Name, ephemeralStorageThreshold.String()))
 
 			}
 		}
@@ -573,21 +557,24 @@ func (m *managerImpl) containerEphemeralStorageLimitEviction(podStats statsapi.P
 	return false
 }
 
-func (m *managerImpl) evictPod(pod *v1.Pod, resourceName v1.ResourceName, evictMsg string) bool {
+func (m *managerImpl) evictPod(pod *v1.Pod, gracePeriodOverride int64, evictMsg string) bool {
+	// If the pod is marked as critical and static, and support for critical pod annotations is enabled,
+	// do not evict such pods. Static pods are not re-admitted after evictions.
+	// https://github.com/kubernetes/kubernetes/issues/40573 has more details.
 	if utilfeature.DefaultFeatureGate.Enabled(features.ExperimentalCriticalPodAnnotation) &&
 		kubelettypes.IsCriticalPod(pod) && kubepod.IsStaticPod(pod) {
-		glog.Errorf("eviction manager: cannot evict a critical pod %s", format.Pod(pod))
+		glog.Errorf("eviction manager: cannot evict a critical static pod %s", format.Pod(pod))
 		return false
 	}
 	status := v1.PodStatus{
 		Phase:   v1.PodFailed,
-		Message: fmt.Sprintf(message, resourceName),
+		Message: evictMsg,
 		Reason:  reason,
 	}
 	// record that we are evicting the pod
 	m.recorder.Eventf(pod, v1.EventTypeWarning, reason, evictMsg)
-	gracePeriod := int64(0)
-	err := m.killPodFunc(pod, status, &gracePeriod)
+	// this is a blocking call and should only return when the pod and its containers are killed.
+	err := m.killPodFunc(pod, status, &gracePeriodOverride)
 	if err != nil {
 		glog.Errorf("eviction manager: pod %s failed to evict %v", format.Pod(pod), err)
 	} else {

pkg/kubelet/eviction/helpers_test.go

@@ -19,6 +19,7 @@ package eviction
 import (
 	"fmt"
 	"reflect"
+	"sort"
 	"sync"
 	"testing"
 	"time"
@@ -29,12 +30,10 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/clock"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
-	api "k8s.io/kubernetes/pkg/apis/core"
 	"k8s.io/kubernetes/pkg/features"
 	statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
 	evictionapi "k8s.io/kubernetes/pkg/kubelet/eviction/api"
 	kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
-	"k8s.io/kubernetes/pkg/quota"
 )
 
 func quantityMustParse(value string) *resource.Quantity {
@@ -470,7 +469,7 @@ func TestOrderedByExceedsRequestDisk(t *testing.T) {
 		return result, found
 	}
 	pods := []*v1.Pod{below, exceeds}
-	orderedBy(exceedDiskRequests(statsFn, []fsStatsType{fsStatsRoot, fsStatsLogs, fsStatsLocalVolumeSource}, resourceDisk)).Sort(pods)
+	orderedBy(exceedDiskRequests(statsFn, []fsStatsType{fsStatsRoot, fsStatsLogs, fsStatsLocalVolumeSource}, v1.ResourceEphemeralStorage)).Sort(pods)
 
 	expected := []*v1.Pod{exceeds, below}
 	for i := range expected {
@@ -584,7 +583,7 @@ func TestOrderedbyDisk(t *testing.T) {
 		return result, found
 	}
 	pods := []*v1.Pod{pod1, pod2, pod3, pod4, pod5, pod6}
-	orderedBy(disk(statsFn, []fsStatsType{fsStatsRoot, fsStatsLogs, fsStatsLocalVolumeSource}, resourceDisk)).Sort(pods)
+	orderedBy(disk(statsFn, []fsStatsType{fsStatsRoot, fsStatsLogs, fsStatsLocalVolumeSource}, v1.ResourceEphemeralStorage)).Sort(pods)
 	expected := []*v1.Pod{pod1, pod3, pod2, pod4, pod5, pod6}
 	for i := range expected {
 		if pods[i] != expected[i] {
@@ -651,7 +650,7 @@ func TestOrderedbyDiskDisableLocalStorage(t *testing.T) {
 		return result, found
 	}
 	pods := []*v1.Pod{pod1, pod3, pod2, pod4, pod5, pod6}
-	orderedBy(disk(statsFn, []fsStatsType{fsStatsRoot, fsStatsLogs, fsStatsLocalVolumeSource}, resourceDisk)).Sort(pods)
+	orderedBy(disk(statsFn, []fsStatsType{fsStatsRoot, fsStatsLogs, fsStatsLocalVolumeSource}, v1.ResourceEphemeralStorage)).Sort(pods)
 	expected := []*v1.Pod{pod5, pod3, pod1, pod6, pod4, pod2}
 	for i := range expected {
 		if pods[i] != expected[i] {
@@ -780,7 +779,7 @@ func TestOrderedByPriorityDisk(t *testing.T) {
 	pods := []*v1.Pod{pod8, pod7, pod6, pod5, pod4, pod3, pod2, pod1}
 	expected := []*v1.Pod{pod1, pod2, pod3, pod4, pod5, pod6, pod7, pod8}
 	fsStatsToMeasure := []fsStatsType{fsStatsRoot, fsStatsLogs, fsStatsLocalVolumeSource}
-	orderedBy(exceedDiskRequests(statsFn, fsStatsToMeasure, resourceDisk), priority, disk(statsFn, fsStatsToMeasure, resourceDisk)).Sort(pods)
+	orderedBy(exceedDiskRequests(statsFn, fsStatsToMeasure, v1.ResourceEphemeralStorage), priority, disk(statsFn, fsStatsToMeasure, v1.ResourceEphemeralStorage)).Sort(pods)
 	for i := range expected {
 		if pods[i] != expected[i] {
 			t.Errorf("Expected pod[%d]: %s, but got: %s", i, expected[i].Name, pods[i].Name)
@@ -932,6 +931,80 @@ func TestOrderedByPriorityMemory(t *testing.T) {
 	}
 }
 
+func TestSortByEvictionPriority(t *testing.T) {
+	for _, tc := range []struct {
+		name       string
+		thresholds []evictionapi.Threshold
+		expected   []evictionapi.Threshold
+	}{
+		{
+			name:       "empty threshold list",
+			thresholds: []evictionapi.Threshold{},
+			expected:   []evictionapi.Threshold{},
+		},
+		{
+			name: "memory first, PID last",
+			thresholds: []evictionapi.Threshold{
+				{
+					Signal: evictionapi.SignalPIDAvailable,
+				},
+				{
+					Signal: evictionapi.SignalNodeFsAvailable,
+				},
+				{
+					Signal: evictionapi.SignalMemoryAvailable,
+				},
+			},
+			expected: []evictionapi.Threshold{
+				{
+					Signal: evictionapi.SignalMemoryAvailable,
+				},
+				{
+					Signal: evictionapi.SignalNodeFsAvailable,
+				},
+				{
+					Signal: evictionapi.SignalPIDAvailable,
+				},
+			},
+		},
+		{
+			name: "allocatable memory first, PID last",
+			thresholds: []evictionapi.Threshold{
+				{
+					Signal: evictionapi.SignalPIDAvailable,
+				},
+				{
+					Signal: evictionapi.SignalNodeFsAvailable,
+				},
+				{
+					Signal: evictionapi.SignalAllocatableMemoryAvailable,
+				},
+			},
+			expected: []evictionapi.Threshold{
+				{
+					Signal: evictionapi.SignalAllocatableMemoryAvailable,
+				},
+				{
+					Signal: evictionapi.SignalNodeFsAvailable,
+				},
+				{
+					Signal: evictionapi.SignalPIDAvailable,
+				},
+			},
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			sort.Sort(byEvictionPriority(tc.thresholds))
+			for i := range tc.expected {
+				if tc.thresholds[i].Signal != tc.expected[i].Signal {
+					t.Errorf("At index %d, expected threshold with signal %s, but got %s", i, tc.expected[i].Signal, tc.thresholds[i].Signal)
+				}
+			}
+
+		})
+	}
+}
+
 type fakeSummaryProvider struct {
 	result *statsapi.Summary
 }
@@ -1622,47 +1695,6 @@ func TestHasNodeConditions(t *testing.T) {
 	}
 }
 
-func TestGetStarvedResources(t *testing.T) {
-	testCases := map[string]struct {
-		inputs []evictionapi.Threshold
-		result []v1.ResourceName
-	}{
-		"memory.available": {
-			inputs: []evictionapi.Threshold{
-				{Signal: evictionapi.SignalMemoryAvailable},
-			},
-			result: []v1.ResourceName{v1.ResourceMemory},
-		},
-		"imagefs.available": {
-			inputs: []evictionapi.Threshold{
-				{Signal: evictionapi.SignalImageFsAvailable},
-			},
-			result: []v1.ResourceName{resourceImageFs},
-		},
-		"nodefs.available": {
-			inputs: []evictionapi.Threshold{
-				{Signal: evictionapi.SignalNodeFsAvailable},
-			},
-			result: []v1.ResourceName{resourceNodeFs},
-		},
-	}
-	var internalResourceNames = func(in []v1.ResourceName) []api.ResourceName {
-		var out []api.ResourceName
-		for _, name := range in {
-			out = append(out, api.ResourceName(name))
-		}
-		return out
-	}
-	for testName, testCase := range testCases {
-		actual := getStarvedResources(testCase.inputs)
-		actualSet := quota.ToSet(internalResourceNames(actual))
-		expectedSet := quota.ToSet(internalResourceNames(testCase.result))
-		if !actualSet.Equal(expectedSet) {
-			t.Errorf("Test case: %s, expected: %v, actual: %v", testName, expectedSet, actualSet)
-		}
-	}
-}
-
 func TestParsePercentage(t *testing.T) {
 	testCases := map[string]struct {
 		hasError bool