Merge pull request #61941 from bskiba/retry-ng-deletion

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>. Retry node pool deletion in autoscaling tests. **What this PR does / why we need it**: Currently, when there is already an operation running on the cluster, the node pool deletion will fail. This will in turn make other tests that try to add extra node pool fail. With this PR we will retry the node pool deletion operation 3 times over the course of ~15 minutes which will hopefully be enough to finish any operations running on the cluster. This will deflake autoscaling e2e tests on gke regional clusters. **Release note**: ``` NONE ``` @aleksandra-malinowska

Merge pull request #61941 from bskiba/retry-ng-deletion
cd570c09 · Kubernetes Submit Queue · GitHub · aa61d4f0 · b0f44e3b · cd570c09
Unverified Commit cd570c09 authored Apr 04, 2018 by Kubernetes Submit Queue Committed by GitHub Apr 04, 2018
Hide whitespace changes
Inline Side-by-side

Showing with 12 additions and 5 deletions

cluster_size_autoscaling.go test/e2e/autoscaling/cluster_size_autoscaling.go +12 -5

No files found.
--- a/test/e2e/autoscaling/cluster_size_autoscaling.go
+++ b/test/e2e/autoscaling/cluster_size_autoscaling.go
@@ -1304,11 +1304,18 @@ func deleteNodePool(name string) {
 	glog.Infof("Deleting node pool %s", name)
 	args := []string{"container", "node-pools", "delete", name, "--quiet",
 		"--cluster=" + framework.TestContext.CloudConfig.Cluster}
-	output, err := execCmd(getGcloudCommand(args)...).CombinedOutput()
-	if err != nil {
-		glog.Infof("Error: %v", err)
-	}
-	glog.Infof("Node-pool deletion output: %s", output)
+	err := wait.ExponentialBackoff(
+		wait.Backoff{Duration: 1 * time.Minute, Factor: float64(3), Steps: 3},
+		func() (bool, error) {
+			output, err := execCmd(getGcloudCommand(args)...).CombinedOutput()
+			if err != nil {
+				glog.Warningf("Error deleting nodegroup - error:%v, output: %s", err, output)
+				return false, nil
+			}
+			glog.Infof("Node-pool deletion output: %s", output)
+			return true, nil
+		})
+	framework.ExpectNoError(err)
 }

 func getPoolNodes(f *framework.Framework, poolName string) []*v1.Node {