Merge pull request #43220 from MaciekPytel/montoring_e2e_timeout

Automatic merge from submit-queue Add retry to monitoring e2e **What this PR does / why we need it**: Add retry to monitoring e2e to prevent it from failing because heapster have not yet been started after cluster creation. @piosz @jszczepkowski **Which issue this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close that issue when PR gets merged)*: fixes #43024 **Special notes for your reviewer**: **Release note**: ```release-note ```

Merge pull request #43220 from MaciekPytel/montoring_e2e_timeout
44146189 · Kubernetes Submit Queue · GitHub · 223c721d · 5b5a9149 · 44146189
Commit 44146189 authored Mar 17, 2017 by Kubernetes Submit Queue Committed by GitHub Mar 17, 2017
Hide whitespace changes
Inline Side-by-side

Showing with 29 additions and 10 deletions

monitoring.go test/e2e/monitoring.go +29 -10

No files found.
--- a/test/e2e/monitoring.go
+++ b/test/e2e/monitoring.go
@@ -26,6 +26,7 @@ import (
 	influxdb "github.com/influxdata/influxdb/client"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
 	"k8s.io/kubernetes/test/e2e/framework"
@@ -45,12 +46,13 @@ var _ = framework.KubeDescribe("Monitoring", func() {
 })
 const (
-	influxdbService      = "monitoring-influxdb"
+	influxdbService       = "monitoring-influxdb"
-	influxdbDatabaseName = "k8s"
+	influxdbDatabaseName  = "k8s"
-	podlistQuery         = "show tag values from \"cpu/usage\" with key = pod_id"
+	podlistQuery          = "show tag values from \"cpu/usage\" with key = pod_id"
-	nodelistQuery        = "show tag values from \"cpu/usage\" with key = hostname"
+	nodelistQuery         = "show tag values from \"cpu/usage\" with key = hostname"
-	sleepBetweenAttempts = 5 * time.Second
+	sleepBetweenAttempts  = 5 * time.Second
-	testTimeout          = 5 * time.Minute
+	testTimeout           = 5 * time.Minute
+	initializationTimeout = 5 * time.Minute
 )
 var (
@@ -286,10 +288,27 @@ func validatePodsAndNodes(c clientset.Interface, expectedPods, expectedNodes []s
 func testMonitoringUsingHeapsterInfluxdb(c clientset.Interface) {
 	// Check if heapster pods and services are up.
-	expectedPods, err := verifyExpectedRcsExistAndGetExpectedPods(c)
+	var expectedPods []string
-	framework.ExpectNoError(err)
+	rcErr := fmt.Errorf("failed to verify expected RCs within timeout")
-	framework.ExpectNoError(expectedServicesExist(c))
+	serviceErr := fmt.Errorf("failed to verify expected services within timeout")
-	// TODO: Wait for all pods and services to be running.
+	err := wait.PollImmediate(sleepBetweenAttempts, initializationTimeout, func() (bool, error) {
+		expectedPods, rcErr = verifyExpectedRcsExistAndGetExpectedPods(c)
+		if rcErr != nil {
+			framework.Logf("Waiting for expected RCs (got error: %v)", rcErr)
+			return false, nil
+		}
+		serviceErr = expectedServicesExist(c)
+		if serviceErr != nil {
+			framework.Logf("Waiting for expected services (got error: %v)", serviceErr)
+			return false, nil
+		}
+		return true, nil
+	})
+	if err != nil {
+		framework.ExpectNoError(rcErr)
+		framework.ExpectNoError(serviceErr)
+		framework.Failf("Failed to verify RCs and services within timeout: %v", err)
+	}
 	expectedNodes, err := getAllNodesInCluster(c)
 	framework.ExpectNoError(err)