Increase threshold for LIST apicall latencies to 2s

test/e2e/framework/metrics_util.go

@@ -50,6 +50,10 @@ const (
 	// portion of CPU and basically stop all the real work.
 	// Increasing threshold to 1s is within our SLO and should solve this problem.
 	apiCallLatencyThreshold time.Duration = 1 * time.Second
+
+	// We set a higher threshold for list apicalls as they can take more time when
+	// the list is really big. For eg. list nodes in a 5000-node cluster.
+	apiListCallLatencyThreshold time.Duration = 2 * time.Second
 )
 
 type MetricsForE2E metrics.MetricsCollection
@@ -341,7 +345,9 @@ func HighLatencyRequests(c clientset.Interface) (int, *APIResponsiveness, error)
 	top := 5
 	for i := range metrics.APICalls {
 		isBad := false
-		if metrics.APICalls[i].Latency.Perc99 > apiCallLatencyThreshold {
+		verb := metrics.APICalls[i].Verb
+		if verb != "LIST" && metrics.APICalls[i].Latency.Perc99 > apiCallLatencyThreshold ||
+			verb == "LIST" && metrics.APICalls[i].Latency.Perc99 > apiListCallLatencyThreshold {
 			badMetrics++
 			isBad = true
 		}