Skip to content

K
k3s
Commit 826069e6 authored by Kubernetes Submit Queue's avatar Kubernetes Submit Queue Committed by GitHub
Browse files
Options

Merge pull request #43453 from MaciekPytel/ca_more_drain_e2e

Automatic merge from submit-queue (batch tested with PRs 43453, 42992) Cluster-autoscaler multistep node drain e2e **What this PR does / why we need it**: This is another Cluster-Autoscaler node drain test. It tests draining a node, when there is pdb allowing for moving some pods, but not all the required pods in one go. Basically it's a more complex scenario of other CA drain e2e, testing the retry logic in CA scale down. **Which issue this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close that issue when PR gets merged)*: fixes # **Special notes for your reviewer**: **Release note**: ```release-note ```
parents e9a91b8c 2630cefc
Show whitespace changes
Inline Side-by-side
Showing with 39 additions and 13 deletions
test/e2e/autoscaling/cluster_size_autoscaling.go
View file @ 826069e6
...@@ -330,7 +330,7 @@ var _ = framework.KubeDescribe("Cluster size autoscaling [Slow]", func() { ...@@ -330,7 +330,7 @@ var _ = framework.KubeDescribe("Cluster size autoscaling [Slow]", func() {
}) })
It("should be able to scale down when rescheduling a pod is required and pdb allows for it[Feature:ClusterSizeAutoscalingScaleDown]", func() { It("should be able to scale down when rescheduling a pod is required and pdb allows for it[Feature:ClusterSizeAutoscalingScaleDown]", func() {
runDrainTest(f, originalSizes, 1, func(increasedSize int) { runDrainTest(f, originalSizes, 1, 1, func(increasedSize int) {
By("Some node should be removed") By("Some node should be removed")
framework.ExpectNoError(WaitForClusterSizeFunc(f.ClientSet, framework.ExpectNoError(WaitForClusterSizeFunc(f.ClientSet,
func(size int) bool { return size < increasedSize }, scaleDownTimeout)) func(size int) bool { return size < increasedSize }, scaleDownTimeout))
...@@ -338,7 +338,7 @@ var _ = framework.KubeDescribe("Cluster size autoscaling [Slow]", func() { ...@@ -338,7 +338,7 @@ var _ = framework.KubeDescribe("Cluster size autoscaling [Slow]", func() {
}) })
It("shouldn't be able to scale down when rescheduling a pod is required, but pdb doesn't allow drain[Feature:ClusterSizeAutoscalingScaleDown]", func() { It("shouldn't be able to scale down when rescheduling a pod is required, but pdb doesn't allow drain[Feature:ClusterSizeAutoscalingScaleDown]", func() {
runDrainTest(f, originalSizes, 0, func(increasedSize int) { runDrainTest(f, originalSizes, 1, 0, func(increasedSize int) {
By("No nodes should be removed") By("No nodes should be removed")
time.Sleep(scaleDownTimeout) time.Sleep(scaleDownTimeout)
nodes := framework.GetReadySchedulableNodesOrDie(f.ClientSet) nodes := framework.GetReadySchedulableNodesOrDie(f.ClientSet)
...@@ -346,9 +346,17 @@ var _ = framework.KubeDescribe("Cluster size autoscaling [Slow]", func() { ...@@ -346,9 +346,17 @@ var _ = framework.KubeDescribe("Cluster size autoscaling [Slow]", func() {
}) })
}) })
It("should be able to scale down by draining multiple pods one by one as dictated by pdb[Feature:ClusterSizeAutoscalingScaleDown]", func() {
runDrainTest(f, originalSizes, 2, 1, func(increasedSize int) {
By("Some node should be removed")
framework.ExpectNoError(WaitForClusterSizeFunc(f.ClientSet,
func(size int) bool { return size < increasedSize }, scaleDownTimeout))
})
})
}) })
func runDrainTest(f *framework.Framework, migSizes map[string]int, pdbSize int, verifyFunction func(int)) { func runDrainTest(f *framework.Framework, migSizes map[string]int, podsPerNode, pdbSize int, verifyFunction func(int)) {
increasedSize := manuallyIncreaseClusterSize(f, migSizes) increasedSize := manuallyIncreaseClusterSize(f, migSizes)
nodes, err := f.ClientSet.Core().Nodes().List(metav1.ListOptions{FieldSelector: fields.Set{ nodes, err := f.ClientSet.Core().Nodes().List(metav1.ListOptions{FieldSelector: fields.Set{
...@@ -356,10 +364,10 @@ func runDrainTest(f *framework.Framework, migSizes map[string]int, pdbSize int, ...@@ -356,10 +364,10 @@ func runDrainTest(f *framework.Framework, migSizes map[string]int, pdbSize int,
}.AsSelector().String()}) }.AsSelector().String()})
framework.ExpectNoError(err) framework.ExpectNoError(err)
namespace := f.Namespace.Name namespace := f.Namespace.Name
numPods := len(nodes.Items) numPods := len(nodes.Items) * podsPerNode
testId := string(uuid.NewUUID()) // So that we can label and find pods testId := string(uuid.NewUUID()) // So that we can label and find pods
labelMap := map[string]string{"test_id": testId} labelMap := map[string]string{"test_id": testId}
framework.ExpectNoError(runReplicatedPodOnEachNode(f, nodes.Items, "reschedulable-pods", labelMap)) framework.ExpectNoError(runReplicatedPodOnEachNode(f, nodes.Items, podsPerNode, "reschedulable-pods", labelMap))
defer framework.DeleteRCAndPods(f.ClientSet, f.InternalClientset, f.Namespace.Name, "reschedulable-pods") defer framework.DeleteRCAndPods(f.ClientSet, f.InternalClientset, f.Namespace.Name, "reschedulable-pods")
...@@ -666,6 +674,7 @@ func setMigSizes(sizes map[string]int) { ...@@ -666,6 +674,7 @@ func setMigSizes(sizes map[string]int) {
func makeNodeUnschedulable(c clientset.Interface, node *v1.Node) error { func makeNodeUnschedulable(c clientset.Interface, node *v1.Node) error {
By(fmt.Sprintf("Taint node %s", node.Name)) By(fmt.Sprintf("Taint node %s", node.Name))
for j := 0; j < 3; j++ {
freshNode, err := c.Core().Nodes().Get(node.Name, metav1.GetOptions{}) freshNode, err := c.Core().Nodes().Get(node.Name, metav1.GetOptions{})
if err != nil { if err != nil {
return err return err
...@@ -681,11 +690,20 @@ func makeNodeUnschedulable(c clientset.Interface, node *v1.Node) error { ...@@ -681,11 +690,20 @@ func makeNodeUnschedulable(c clientset.Interface, node *v1.Node) error {
Effect: v1.TaintEffectNoSchedule, Effect: v1.TaintEffectNoSchedule,
}) })
_, err = c.Core().Nodes().Update(freshNode) _, err = c.Core().Nodes().Update(freshNode)
if err == nil {
return nil
}
if !errors.IsConflict(err) {
return err return err
}
glog.Warningf("Got 409 conflict when trying to taint node, retries left: %v", 3-j)
}
return fmt.Errorf("Failed to taint node in allowed number of retries")
} }
func makeNodeSchedulable(c clientset.Interface, node *v1.Node) error { func makeNodeSchedulable(c clientset.Interface, node *v1.Node) error {
By(fmt.Sprintf("Remove taint from node %s", node.Name)) By(fmt.Sprintf("Remove taint from node %s", node.Name))
for j := 0; j < 3; j++ {
freshNode, err := c.Core().Nodes().Get(node.Name, metav1.GetOptions{}) freshNode, err := c.Core().Nodes().Get(node.Name, metav1.GetOptions{})
if err != nil { if err != nil {
return err return err
...@@ -697,24 +715,32 @@ func makeNodeSchedulable(c clientset.Interface, node *v1.Node) error { ...@@ -697,24 +715,32 @@ func makeNodeSchedulable(c clientset.Interface, node *v1.Node) error {
} }
} }
if len(newTaints) != len(freshNode.Spec.Taints) { if len(newTaints) == len(freshNode.Spec.Taints) {
return nil
}
freshNode.Spec.Taints = newTaints freshNode.Spec.Taints = newTaints
_, err = c.Core().Nodes().Update(freshNode) _, err = c.Core().Nodes().Update(freshNode)
if err == nil {
return nil
}
if !errors.IsConflict(err) {
return err return err
} }
return nil glog.Warningf("Got 409 conflict when trying to taint node, retries left: %v", 3-j)
}
return fmt.Errorf("Failed to remove taint from node in allowed number of retries")
} }
// Creat an RC running a single pod on each node without adding any constraint forcing such // Creat an RC running a given number of pods on each node without adding any constraint forcing
// pod distribution. This is meant to create a bunch of underutilized (but not unused) nodes // such pod distribution. This is meant to create a bunch of underutilized (but not unused) nodes
// with pods that can be rescheduled on different nodes. // with pods that can be rescheduled on different nodes.
// This is achieved using the following method: // This is achieved using the following method:
// 1. disable scheduling on each node // 1. disable scheduling on each node
// 2. create an empty RC // 2. create an empty RC
// 3. for each node: // 3. for each node:
// 3a. enable scheduling on that node // 3a. enable scheduling on that node
// 3b. increase number of replicas in RC by 1 // 3b. increase number of replicas in RC by podsPerNode
func runReplicatedPodOnEachNode(f *framework.Framework, nodes []v1.Node, id string, labels map[string]string) error { func runReplicatedPodOnEachNode(f *framework.Framework, nodes []v1.Node, podsPerNode int, id string, labels map[string]string) error {
By("Run a pod on each node") By("Run a pod on each node")
for _, node := range nodes { for _, node := range nodes {
err := makeNodeUnschedulable(f.ClientSet, &node) err := makeNodeUnschedulable(f.ClientSet, &node)
...@@ -754,7 +780,7 @@ func runReplicatedPodOnEachNode(f *framework.Framework, nodes []v1.Node, id stri ...@@ -754,7 +780,7 @@ func runReplicatedPodOnEachNode(f *framework.Framework, nodes []v1.Node, id stri
// Update replicas count, to create new pods that will be allocated on node // Update replicas count, to create new pods that will be allocated on node
// (we retry 409 errors in case rc reference got out of sync) // (we retry 409 errors in case rc reference got out of sync)
for j := 0; j < 3; j++ { for j := 0; j < 3; j++ {
*rc.Spec.Replicas = int32(i + 1) *rc.Spec.Replicas = int32((i + 1) * podsPerNode)
rc, err = f.ClientSet.Core().ReplicationControllers(f.Namespace.Name).Update(rc) rc, err = f.ClientSet.Core().ReplicationControllers(f.Namespace.Name).Update(rc)
if err == nil { if err == nil {
break break
...@@ -771,7 +797,7 @@ func runReplicatedPodOnEachNode(f *framework.Framework, nodes []v1.Node, id stri ...@@ -771,7 +797,7 @@ func runReplicatedPodOnEachNode(f *framework.Framework, nodes []v1.Node, id stri
err = wait.PollImmediate(5*time.Second, podTimeout, func() (bool, error) { err = wait.PollImmediate(5*time.Second, podTimeout, func() (bool, error) {
rc, err = f.ClientSet.Core().ReplicationControllers(f.Namespace.Name).Get(id, metav1.GetOptions{}) rc, err = f.ClientSet.Core().ReplicationControllers(f.Namespace.Name).Get(id, metav1.GetOptions{})
if err != nil || rc.Status.ReadyReplicas < int32(i+1) { if err != nil || rc.Status.ReadyReplicas < int32((i+1)*podsPerNode) {
return false, nil return false, nil
} }
return true, nil return true, nil
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment