Merge pull request #52724 from shyamjvs/fix-density-test

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.. Retry if possible while creating latency pods in density test Saw the [last run](https://k8s-gubernator.appspot.com/build/kubernetes-jenkins/logs/ci-kubernetes-e2e-gce-scale-performance/37) of density test on 5k-node fail due to it: ``` Expected error: <*errors.StatusError | 0xc44f2fd7a0>: { ErrStatus: { TypeMeta: {Kind: "", APIVersion: ""}, ListMeta: {SelfLink: "", ResourceVersion: "", Continue: ""}, Status: "Failure", Message: "timeout", Reason: "", Details: nil, Code: 500, }, } timeout not to have occurred ``` cc @kubernetes/sig-scalability-misc

Merge pull request #52724 from shyamjvs/fix-density-test
9505c01f · Kubernetes Submit Queue · GitHub · f7dd62f1 · 419bbd26 · 9505c01f
Commit 9505c01f authored Sep 20, 2017 by Kubernetes Submit Queue Committed by GitHub Sep 20, 2017
Show whitespace changes
Inline Side-by-side

Showing with 9 additions and 1 deletion

BUILD test/e2e/scalability/BUILD +1 -0

density.go test/e2e/scalability/density.go +8 -1

No files found.
--- a/test/e2e/scalability/BUILD
+++ b/test/e2e/scalability/BUILD
@@ -23,6 +23,7 @@ go_library(
        "//vendor/github.com/onsi/ginkgo:go_default_library",
        "//vendor/github.com/onsi/gomega:go_default_library",
        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
        "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
        "//vendor/k8s.io/apimachinery/pkg/fields:go_default_library",

--- a/test/e2e/scalability/density.go
+++ b/test/e2e/scalability/density.go
@@ -26,6 +26,7 @@ import (
 	"time"
 	"k8s.io/api/core/v1"
+	apierrs "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
@@ -53,6 +54,7 @@ const (
 	MinSaturationThreshold     = 2 * time.Minute
 	MinPodsPerSecondThroughput = 8
 	DensityPollInterval        = 10 * time.Second
+	MaxLatencyPodCreationTries = 5
 )
 // Maximum container failures this test tolerates before failing.
@@ -819,8 +821,13 @@ func createRunningPodFromRC(wg *sync.WaitGroup, c clientset.Interface, name, ns,
 			},
 		},
 	}
+	for attempt := 1; attempt <= MaxLatencyPodCreationTries; attempt++ {
 		_, err := c.Core().ReplicationControllers(ns).Create(rc)
-	framework.ExpectNoError(err)
+		if err == nil || apierrs.IsAlreadyExists(err) {
+			break
+		}
+		Expect(attempt < MaxLatencyPodCreationTries && framework.IsRetryableAPIError(err)).To(Equal(true))
+	}
 	framework.ExpectNoError(framework.WaitForControlledPodsRunning(c, ns, name, api.Kind("ReplicationController")))
 	framework.Logf("Found pod '%s' running", name)
 }