Skip to content

K
k3s
Commit 407bef47 authored by Kubernetes Submit Queue's avatar Kubernetes Submit Queue Committed by GitHub
Browse files
Options

Merge pull request #52373 from dashpole/eviction_cleanup

Automatic merge from submit-queue (batch tested with PRs 52960, 52373). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.. Refactor eviction tests fixes: #52203 We have a bunch of eviction tests, which each break independently, and take a large amount of time to fix. This refactors these tests to share the core eviction testing logic. Each tests needs only to set kubelet flags, and specify which pods to run. I decided to omit the memory eviction tests because they work. Best not to disturb them. A large portion of the code changes are the renaming of inode_eviction_test.go -> eviction_test.go This should probably wait until after https://github.com/kubernetes/kubernetes/pull/50392 /assign @mtaufen @Random-Liu
parents 4714cddc 828c2d96
Expand all Show whitespace changes
Inline Side-by-side
Showing with 6 additions and 369 deletions
test/e2e_node/BUILD
View file @ 407bef47
...@@ -76,23 +76,19 @@ go_library( ...@@ -76,23 +76,19 @@ go_library(
go_test( go_test(
name = "go_default_test", name = "go_default_test",
srcs = [ srcs = [
"allocatable_eviction_test.go",
"apparmor_test.go", "apparmor_test.go",
"cpu_manager_test.go", "cpu_manager_test.go",
"critical_pod_test.go", "critical_pod_test.go",
"disk_eviction_test.go",
"docker_test.go", "docker_test.go",
"dockershim_checkpoint_test.go", "dockershim_checkpoint_test.go",
"dynamic_kubelet_config_test.go", "dynamic_kubelet_config_test.go",
"e2e_node_suite_test.go", "e2e_node_suite_test.go",
"eviction_test.go",
"garbage_collector_test.go", "garbage_collector_test.go",
"gke_environment_test.go", "gke_environment_test.go",
"image_id_test.go", "image_id_test.go",
"inode_eviction_test.go",
"kubelet_test.go", "kubelet_test.go",
"lifecycle_hook_test.go", "lifecycle_hook_test.go",
"local_storage_allocatable_eviction_test.go",
"local_storage_isolation_eviction_test.go",
"log_path_test.go", "log_path_test.go",
"memory_eviction_test.go", "memory_eviction_test.go",
"mirror_pod_test.go", "mirror_pod_test.go",
...@@ -157,7 +153,6 @@ go_test( ...@@ -157,7 +153,6 @@ go_test(
"//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/yaml:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/yaml:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/watch:go_default_library", "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library",
"//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library", "//vendor/k8s.io/client-go/kubernetes:go_default_library",
"//vendor/k8s.io/client-go/kubernetes/scheme:go_default_library", "//vendor/k8s.io/client-go/kubernetes/scheme:go_default_library",
] + select({ ] + select({
......
test/e2e_node/allocatable_eviction_test.go deleted 100644 → 0
View file @ 4714cddc
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package e2e_node
import (
"fmt"
"time"
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
nodeutil "k8s.io/kubernetes/pkg/api/v1/node"
"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
"k8s.io/kubernetes/pkg/kubelet/cm"
"k8s.io/kubernetes/test/e2e/framework"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
// Eviction Policy is described here:
// https://github.com/kubernetes/kubernetes/blob/master/docs/proposals/kubelet-eviction.md
var _ = framework.KubeDescribe("MemoryAllocatableEviction [Slow] [Serial] [Disruptive] [Flaky]", func() {
f := framework.NewDefaultFramework("memory-allocatable-eviction-test")
podTestSpecs := []podTestSpec{
{
evictionPriority: 1, // This pod should be evicted before the innocent pod
pod: getMemhogPod("memory-hog-pod", "memory-hog", v1.ResourceRequirements{}),
},
{
evictionPriority: 0, // This pod should never be evicted
pod: getInnocentPod(),
},
}
evictionTestTimeout := 10 * time.Minute
testCondition := "Memory Pressure"
Context(fmt.Sprintf("when we run containers that should cause %s", testCondition), func() {
tempSetCurrentKubeletConfig(f, func(initialConfig *kubeletconfig.KubeletConfiguration) {
// Set large system and kube reserved values to trigger allocatable thresholds far before hard eviction thresholds.
kubeReserved := getNodeCPUAndMemoryCapacity(f)[v1.ResourceMemory]
// The default hard eviction threshold is 250Mb, so Allocatable = Capacity - Reserved - 250Mb
// We want Allocatable = 50Mb, so set Reserved = Capacity - Allocatable - 250Mb = Capacity - 300Mb
kubeReserved.Sub(resource.MustParse("300Mi"))
initialConfig.KubeReserved = kubeletconfig.ConfigurationMap(map[string]string{string(v1.ResourceMemory): kubeReserved.String()})
initialConfig.EnforceNodeAllocatable = []string{cm.NodeAllocatableEnforcementKey}
initialConfig.ExperimentalNodeAllocatableIgnoreEvictionThreshold = false
initialConfig.CgroupsPerQOS = true
})
// Place the remainder of the test within a context so that the kubelet config is set before and after the test.
Context("With kubeconfig updated", func() {
runEvictionTest(f, testCondition, podTestSpecs, evictionTestTimeout, hasMemoryPressure)
})
})
})
// Returns TRUE if the node has Memory Pressure, FALSE otherwise
func hasMemoryPressure(f *framework.Framework, testCondition string) (bool, error) {
localNodeStatus := getLocalNode(f).Status
_, pressure := nodeutil.GetNodeCondition(&localNodeStatus, v1.NodeMemoryPressure)
Expect(pressure).NotTo(BeNil())
hasPressure := pressure.Status == v1.ConditionTrue
By(fmt.Sprintf("checking if pod has %s: %v", testCondition, hasPressure))
// Additional Logging relating to Memory
summary, err := getNodeSummary()
if err != nil {
return false, err
}
if summary.Node.Memory != nil && summary.Node.Memory.WorkingSetBytes != nil && summary.Node.Memory.AvailableBytes != nil {
framework.Logf("Node.Memory.WorkingSetBytes: %d, summary.Node.Memory.AvailableBytes: %d", *summary.Node.Memory.WorkingSetBytes, *summary.Node.Memory.AvailableBytes)
}
for _, pod := range summary.Pods {
framework.Logf("Pod: %s", pod.PodRef.Name)
for _, container := range pod.Containers {
if container.Memory != nil && container.Memory.WorkingSetBytes != nil {
framework.Logf("--- summary Container: %s WorkingSetBytes: %d", container.Name, *container.Memory.WorkingSetBytes)
}
}
}
return hasPressure, nil
}
test/e2e_node/disk_eviction_test.go deleted 100644 → 0
View file @ 4714cddc
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package e2e_node
import (
"fmt"
"strings"
"time"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/uuid"
"k8s.io/kubernetes/test/e2e/framework"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
clientset "k8s.io/client-go/kubernetes"
)
const (
// podCheckInterval is the interval seconds between pod status checks.
podCheckInterval = time.Second * 2
// containerGCPeriod is the period of container garbage collect loop. It should be the same
// with ContainerGCPeriod in kubelet.go. However we don't want to include kubelet package
// directly which will introduce a lot more dependencies.
containerGCPeriod = time.Minute * 1
dummyFile = "dummy."
)
// TODO: Leverage dynamic Kubelet settings when it's implemented to only modify the kubelet eviction option in this test.
var _ = framework.KubeDescribe("Kubelet Eviction Manager [Serial] [Disruptive]", func() {
f := framework.NewDefaultFramework("kubelet-eviction-manager")
var podClient *framework.PodClient
var c clientset.Interface
BeforeEach(func() {
podClient = f.PodClient()
c = f.ClientSet
})
Describe("hard eviction test", func() {
Context("pod using the most disk space gets evicted when the node disk usage is above the eviction hard threshold", func() {
var busyPodName, idlePodName, verifyPodName string
BeforeEach(func() {
if !isImageSupported() {
framework.Skipf("test skipped because the image is not supported by the test")
}
if !evictionOptionIsSet() {
framework.Skipf("test skipped because eviction option is not set")
}
busyPodName = "to-evict" + string(uuid.NewUUID())
idlePodName = "idle" + string(uuid.NewUUID())
verifyPodName = "verify" + string(uuid.NewUUID())
createIdlePod(idlePodName, podClient)
podClient.Create(&v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: busyPodName,
},
Spec: v1.PodSpec{
RestartPolicy: v1.RestartPolicyNever,
Containers: []v1.Container{
{
Image: busyboxImage,
Name: busyPodName,
// Filling the disk
Command: []string{"sh", "-c",
fmt.Sprintf("for NUM in `seq 1 1 100000`; do dd if=/dev/urandom of=%s.$NUM bs=50000000 count=10; sleep 0.5; done",
dummyFile)},
},
},
},
})
})
AfterEach(func() {
if !isImageSupported() || !evictionOptionIsSet() { // Skip the after each
return
}
podClient.DeleteSync(busyPodName, &metav1.DeleteOptions{}, framework.DefaultPodDeletionTimeout)
podClient.DeleteSync(idlePodName, &metav1.DeleteOptions{}, framework.DefaultPodDeletionTimeout)
podClient.DeleteSync(verifyPodName, &metav1.DeleteOptions{}, framework.DefaultPodDeletionTimeout)
// Wait for 2 container gc loop to ensure that the containers are deleted. The containers
// created in this test consume a lot of disk, we don't want them to trigger disk eviction
// again after the test.
time.Sleep(containerGCPeriod * 2)
if framework.TestContext.PrepullImages {
// The disk eviction test may cause the prepulled images to be evicted,
// prepull those images again to ensure this test not affect following tests.
PrePullAllImages()
}
})
It("should evict the pod using the most disk space [Slow]", func() {
evictionOccurred := false
nodeDiskPressureCondition := false
podRescheduleable := false
Eventually(func() error {
// Avoid the test using up all the disk space
err := checkDiskUsage(0.05)
if err != nil {
return err
}
// The pod should be evicted.
if !evictionOccurred {
podData, err := podClient.Get(busyPodName, metav1.GetOptions{})
if err != nil {
return err
}
err = verifyPodEviction(podData)
if err != nil {
return err
}
podData, err = podClient.Get(idlePodName, metav1.GetOptions{})
if err != nil {
return err
}
if podData.Status.Phase != v1.PodRunning {
err = verifyPodEviction(podData)
if err != nil {
return err
}
}
evictionOccurred = true
return fmt.Errorf("waiting for node disk pressure condition to be set")
}
// The node should have disk pressure condition after the pods are evicted.
if !nodeDiskPressureCondition {
if !nodeHasDiskPressure(f.ClientSet) {
return fmt.Errorf("expected disk pressure condition is not set")
}
nodeDiskPressureCondition = true
return fmt.Errorf("waiting for node disk pressure condition to be cleared")
}
// After eviction happens the pod is evicted so eventually the node disk pressure should be relieved.
if !podRescheduleable {
if nodeHasDiskPressure(f.ClientSet) {
return fmt.Errorf("expected disk pressure condition relief has not happened")
}
createIdlePod(verifyPodName, podClient)
podRescheduleable = true
return fmt.Errorf("waiting for the node to accept a new pod")
}
// The new pod should be able to be scheduled and run after the disk pressure is relieved.
podData, err := podClient.Get(verifyPodName, metav1.GetOptions{})
if err != nil {
return err
}
if podData.Status.Phase != v1.PodRunning {
return fmt.Errorf("waiting for the new pod to be running")
}
return nil
}, time.Minute*15 /* based on n1-standard-1 machine type */, podCheckInterval).Should(BeNil())
})
})
})
})
func createIdlePod(podName string, podClient *framework.PodClient) {
podClient.Create(&v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: podName,
},
Spec: v1.PodSpec{
RestartPolicy: v1.RestartPolicyNever,
Containers: []v1.Container{
{
Image: framework.GetPauseImageNameForHostArch(),
Name: podName,
},
},
},
})
}
func verifyPodEviction(podData *v1.Pod) error {
if podData.Status.Phase != v1.PodFailed {
return fmt.Errorf("expected phase to be failed. got %+v", podData.Status.Phase)
}
if podData.Status.Reason != "Evicted" {
return fmt.Errorf("expected failed reason to be evicted. got %+v", podData.Status.Reason)
}
return nil
}
func nodeHasDiskPressure(cs clientset.Interface) bool {
nodeList := framework.GetReadySchedulableNodesOrDie(cs)
for _, condition := range nodeList.Items[0].Status.Conditions {
if condition.Type == v1.NodeDiskPressure {
return condition.Status == v1.ConditionTrue
}
}
return false
}
func evictionOptionIsSet() bool {
return len(framework.TestContext.KubeletConfig.EvictionHard) > 0
}
// TODO(random-liu): Use OSImage in node status to do the check.
func isImageSupported() bool {
// TODO: Only images with image fs is selected for testing for now. When the kubelet settings can be dynamically updated,
// instead of skipping images the eviction thresholds should be adjusted based on the images.
return strings.Contains(framework.TestContext.NodeName, "-gci-dev-")
}
// checkDiskUsage verifies that the available bytes on disk are above the limit.
func checkDiskUsage(limit float64) error {
summary, err := getNodeSummary()
if err != nil {
return err
}
if nodeFs := summary.Node.Fs; nodeFs != nil {
if nodeFs.AvailableBytes != nil && nodeFs.CapacityBytes != nil {
if float64(*nodeFs.CapacityBytes)*limit > float64(*nodeFs.AvailableBytes) {
return fmt.Errorf("available nodefs byte is less than %v%%", limit*float64(100))
}
}
}
if summary.Node.Runtime != nil {
if imageFs := summary.Node.Runtime.ImageFs; imageFs != nil {
if float64(*imageFs.CapacityBytes)*limit > float64(*imageFs.AvailableBytes) {
return fmt.Errorf("available imagefs byte is less than %v%%", limit*float64(100))
}
}
}
return nil
}
test/e2e_node/jenkins/jenkins-flaky.properties
View file @ 407bef47
...@@ -4,8 +4,8 @@ GCE_ZONE=us-central1-f ...@@ -4,8 +4,8 @@ GCE_ZONE=us-central1-f
GCE_PROJECT=k8s-jkns-ci-node-e2e GCE_PROJECT=k8s-jkns-ci-node-e2e
CLEANUP=true CLEANUP=true
GINKGO_FLAGS='--focus="\[Flaky\]"' GINKGO_FLAGS='--focus="\[Flaky\]"'
TEST_ARGS='--feature-gates=DynamicKubeletConfig=true' TEST_ARGS='--feature-gates=DynamicKubeletConfig=true,LocalStorageCapacityIsolation=true'
KUBELET_ARGS='--cgroups-per-qos=true --cgroup-root=/' KUBELET_ARGS='--cgroups-per-qos=true --cgroup-root=/'
PARALLELISM=1 PARALLELISM=1
TIMEOUT=2h TIMEOUT=3h
test/e2e_node/util.go
View file @ 407bef47
...@@ -100,17 +100,14 @@ func tempSetCurrentKubeletConfig(f *framework.Framework, updateFunction func(ini ...@@ -100,17 +100,14 @@ func tempSetCurrentKubeletConfig(f *framework.Framework, updateFunction func(ini
BeforeEach(func() { BeforeEach(func() {
configEnabled, err := isKubeletConfigEnabled(f) configEnabled, err := isKubeletConfigEnabled(f)
framework.ExpectNoError(err) framework.ExpectNoError(err)
if configEnabled { Expect(configEnabled).To(BeTrue(), "The Dynamic Kubelet Configuration feature is not enabled.\n"+
"Pass --feature-gates=DynamicKubeletConfig=true to the Kubelet to enable this feature.\n"+
"For `make test-e2e-node`, you can set `TEST_ARGS='--feature-gates=DynamicKubeletConfig=true'`.")
oldCfg, err = getCurrentKubeletConfig() oldCfg, err = getCurrentKubeletConfig()
framework.ExpectNoError(err) framework.ExpectNoError(err)
newCfg := oldCfg.DeepCopy() newCfg := oldCfg.DeepCopy()
updateFunction(newCfg) updateFunction(newCfg)
framework.ExpectNoError(setKubeletConfiguration(f, newCfg)) framework.ExpectNoError(setKubeletConfiguration(f, newCfg))
} else {
framework.Logf("The Dynamic Kubelet Configuration feature is not enabled.\n" +
"Pass --feature-gates=DynamicKubeletConfig=true to the Kubelet to enable this feature.\n" +
"For `make test-e2e-node`, you can set `TEST_ARGS='--feature-gates=DynamicKubeletConfig=true'`.")
}
}) })
AfterEach(func() { AfterEach(func() {
if oldCfg != nil { if oldCfg != nil {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment