Merge pull request #45094 from aleksandra-malinowska/stackdriver-monitoring-test-debug

test/e2e/stackdriver_monitoring.go

@@ -48,7 +48,7 @@ var (
 	}
 
 	pollFrequency = time.Second * 5
-	pollTimeout   = time.Minute * 7
+	pollTimeout   = time.Minute * 20
 
 	rcName            = "resource-consumer"
 	replicas          = 1
@@ -79,22 +79,34 @@ var _ = framework.KubeDescribe("Stackdriver Monitoring", func() {
 
 		rc.WaitForReplicas(replicas)
 
-		pollingFunction := checkForMetrics(projectId, gcmService, time.Now())
-		framework.ExpectNoError(wait.Poll(pollFrequency, pollTimeout, pollingFunction))
+		metricsMap := map[string]bool{}
+		pollingFunction := checkForMetrics(projectId, gcmService, time.Now(), metricsMap)
+		err = wait.Poll(pollFrequency, pollTimeout, pollingFunction)
+		if err != nil {
+			framework.Logf("Missing metrics: %+v\n", metricsMap)
+		}
+		framework.ExpectNoError(err)
 	})
 })
 
-func checkForMetrics(projectId string, gcmService *gcm.Service, start time.Time) func() (bool, error) {
+func checkForMetrics(projectId string, gcmService *gcm.Service, start time.Time, metricsMap map[string]bool) func() (bool, error) {
 	return func() (bool, error) {
 		// TODO: list which metrics are missing in case of failure
 		counter := 0
 		correctUtilization := false
 		for _, metric := range stackdriverMetrics {
+			metricsMap[metric] = false
+		}
+		for _, metric := range stackdriverMetrics {
 			// TODO: check only for metrics from this cluster
 			ts, err := fetchTimeSeries(projectId, gcmService, metric, start, time.Now())
 			framework.ExpectNoError(err)
 			if len(ts) > 0 {
 				counter = counter + 1
+				metricsMap[metric] = true
+				framework.Logf("Received %v timeseries for metric %v\n", len(ts), metric)
+			} else {
+				framework.Logf("No timeseries for metric %v\n", metric)
 			}
 
 			var sum float64 = 0
@@ -111,7 +123,10 @@ func checkForMetrics(projectId string, gcmService *gcm.Service, start time.Time)
 						}
 					}
 					sum = sum + *max.Value.DoubleValue
+					framework.Logf("Received %v points for metric %v\n",
+						len(t.Points), metric)
 				}
+				framework.Logf("Most recent cpu/utilization sum: %v\n", sum)
 				if math.Abs(sum*float64(cpuLimit)-float64(cpuUsed)) > tolerance*float64(cpuUsed) {
 					return false, nil
 				} else {