Merge pull request #52237 from smarterclayton/watch_metric

staging/src/k8s.io/apiserver/pkg/endpoints/handlers/proxy.go

@@ -96,6 +96,9 @@ func (r *ProxyHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 	if namespace != "" {
 		scope = "namespace"
 	}
+	if requestInfo.Name != "" {
+		scope = "resource"
+	}
 
 	ctx = request.WithNamespace(ctx, namespace)
 	if len(parts) < 2 {

staging/src/k8s.io/apiserver/pkg/endpoints/metrics/metrics.go

@@ -41,7 +41,7 @@ var (
 			Name: "apiserver_request_count",
 			Help: "Counter of apiserver requests broken out for each verb, API resource, client, and HTTP response contentType and code.",
 		},
-		[]string{"verb", "resource", "subresource", "client", "contentType", "code"},
+		[]string{"verb", "resource", "subresource", "scope", "client", "contentType", "code"},
 	)
 	requestLatencies = prometheus.NewHistogramVec(
 		prometheus.HistogramOpts{
@@ -50,7 +50,7 @@ var (
 			// Use buckets ranging from 125 ms to 8 seconds.
 			Buckets: prometheus.ExponentialBuckets(125000, 2.0, 7),
 		},
-		[]string{"verb", "resource", "subresource"},
+		[]string{"verb", "resource", "subresource", "scope"},
 	)
 	requestLatenciesSummary = prometheus.NewSummaryVec(
 		prometheus.SummaryOpts{
@@ -59,7 +59,7 @@ var (
 			// Make the sliding window of 1h.
 			MaxAge: time.Hour,
 		},
-		[]string{"verb", "resource", "subresource"},
+		[]string{"verb", "resource", "subresource", "scope"},
 	)
 	responseSizes = prometheus.NewHistogramVec(
 		prometheus.HistogramOpts{
@@ -85,9 +85,9 @@ func Register() {
 // uppercase to be backwards compatible with existing monitoring tooling.
 func Monitor(verb, resource, subresource, scope, client, contentType string, httpCode, respSize int, reqStart time.Time) {
 	elapsed := float64((time.Since(reqStart)) / time.Microsecond)
-	requestCounter.WithLabelValues(verb, resource, subresource, client, contentType, codeToString(httpCode)).Inc()
-	requestLatencies.WithLabelValues(verb, resource, subresource).Observe(elapsed)
-	requestLatenciesSummary.WithLabelValues(verb, resource, subresource).Observe(elapsed)
+	requestCounter.WithLabelValues(verb, resource, subresource, scope, client, contentType, codeToString(httpCode)).Inc()
+	requestLatencies.WithLabelValues(verb, resource, subresource, scope).Observe(elapsed)
+	requestLatenciesSummary.WithLabelValues(verb, resource, subresource, scope).Observe(elapsed)
 	// We are only interested in response sizes of read requests.
 	if verb == "GET" || verb == "LIST" {
 		responseSizes.WithLabelValues(verb, resource, subresource, scope).Observe(float64(respSize))
@@ -106,6 +106,10 @@ func MonitorRequest(request *http.Request, verb, resource, subresource, scope, c
 			}
 		}
 	}
+	// normalize the legacy WATCHLIST to WATCH to ensure users aren't surprised by metrics
+	if verb == "WATCHLIST" {
+		reportedVerb = "WATCH"
+	}
 
 	client := cleanUserAgent(utilnet.GetHTTPClient(request))
 	Monitor(reportedVerb, resource, subresource, scope, client, contentType, httpCode, respSize, reqStart)

staging/src/k8s.io/apiserver/pkg/server/filters/maxinflight.go

@@ -112,6 +112,9 @@ func WithMaxInFlightLimit(
 				if requestInfo.Namespace != "" {
 					scope = "namespace"
 				}
+				if requestInfo.Name != "" {
+					scope = "resource"
+				}
 				if requestInfo.IsResourceRequest {
 					metrics.MonitorRequest(r, strings.ToUpper(requestInfo.Verb), requestInfo.Resource, requestInfo.Subresource, "", scope, http.StatusTooManyRequests, 0, time.Now())
 				} else {

staging/src/k8s.io/apiserver/pkg/server/filters/timeout.go

@@ -61,6 +61,9 @@ func WithTimeoutForNonLongRunningRequests(handler http.Handler, requestContextMa
 			if requestInfo.Namespace != "" {
 				scope = "namespace"
 			}
+			if requestInfo.Name != "" {
+				scope = "resource"
+			}
 			if requestInfo.IsResourceRequest {
 				metrics.MonitorRequest(req, strings.ToUpper(requestInfo.Verb), requestInfo.Resource, requestInfo.Subresource, "", scope, http.StatusGatewayTimeout, 0, now)
 			} else {

test/e2e/framework/metrics_util.go

@@ -230,6 +230,7 @@ type APICall struct {
 	Resource    string        `json:"resource"`
 	Subresource string        `json:"subresource"`
 	Verb        string        `json:"verb"`
+	Scope       string        `json:"scope"`
 	Latency     LatencyMetric `json:"latency"`
 	Count       int           `json:"count"`
 }
@@ -261,14 +262,14 @@ func (a *APIResponsiveness) Less(i, j int) bool {
 // Set request latency for a particular quantile in the APICall metric entry (creating one if necessary).
 // 0 <= quantile <=1 (e.g. 0.95 is 95%tile, 0.5 is median)
 // Only 0.5, 0.9 and 0.99 quantiles are supported.
-func (a *APIResponsiveness) addMetricRequestLatency(resource, subresource, verb string, quantile float64, latency time.Duration) {
+func (a *APIResponsiveness) addMetricRequestLatency(resource, subresource, verb, scope string, quantile float64, latency time.Duration) {
 	for i, apicall := range a.APICalls {
-		if apicall.Resource == resource && apicall.Subresource == subresource && apicall.Verb == verb {
+		if apicall.Resource == resource && apicall.Subresource == subresource && apicall.Verb == verb && apicall.Scope == scope {
 			a.APICalls[i] = setQuantileAPICall(apicall, quantile, latency)
 			return
 		}
 	}
-	apicall := setQuantileAPICall(APICall{Resource: resource, Subresource: subresource, Verb: verb}, quantile, latency)
+	apicall := setQuantileAPICall(APICall{Resource: resource, Subresource: subresource, Verb: verb, Scope: scope}, quantile, latency)
 	a.APICalls = append(a.APICalls, apicall)
 }
 
@@ -292,14 +293,14 @@ func setQuantile(metric *LatencyMetric, quantile float64, latency time.Duration)
 }
 
 // Add request count to the APICall metric entry (creating one if necessary).
-func (a *APIResponsiveness) addMetricRequestCount(resource, subresource, verb string, count int) {
+func (a *APIResponsiveness) addMetricRequestCount(resource, subresource, verb, scope string, count int) {
 	for i, apicall := range a.APICalls {
-		if apicall.Resource == resource && apicall.Subresource == subresource && apicall.Verb == verb {
+		if apicall.Resource == resource && apicall.Subresource == subresource && apicall.Verb == verb && apicall.Scope == scope {
 			a.APICalls[i].Count += count
 			return
 		}
 	}
-	apicall := APICall{Resource: resource, Subresource: subresource, Verb: verb, Count: count}
+	apicall := APICall{Resource: resource, Subresource: subresource, Verb: verb, Count: count, Scope: scope}
 	a.APICalls = append(a.APICalls, apicall)
 }
 
@@ -332,6 +333,7 @@ func readLatencyMetrics(c clientset.Interface) (*APIResponsiveness, error) {
 		resource := string(sample.Metric["resource"])
 		subresource := string(sample.Metric["subresource"])
 		verb := string(sample.Metric["verb"])
+		scope := string(sample.Metric["scope"])
 		if ignoredResources.Has(resource) || ignoredVerbs.Has(verb) {
 			continue
 		}
@@ -343,10 +345,10 @@ func readLatencyMetrics(c clientset.Interface) (*APIResponsiveness, error) {
 			if err != nil {
 				return nil, err
 			}
-			a.addMetricRequestLatency(resource, subresource, verb, quantile, time.Duration(int64(latency))*time.Microsecond)
+			a.addMetricRequestLatency(resource, subresource, verb, scope, quantile, time.Duration(int64(latency))*time.Microsecond)
 		case "apiserver_request_count":
 			count := sample.Value
-			a.addMetricRequestCount(resource, subresource, verb, int(count))
+			a.addMetricRequestCount(resource, subresource, verb, scope, int(count))
 
 		}
 	}

test/e2e/framework/perf_util.go

@@ -44,6 +44,7 @@ func ApiCallToPerfData(apicalls *APIResponsiveness) *perftype.PerfData {
 				"Verb":        apicall.Verb,
 				"Resource":    apicall.Resource,
 				"Subresource": apicall.Subresource,
+				"Scope":       apicall.Scope,
 				"Count":       fmt.Sprintf("%v", apicall.Count),
 			},
 		}