Merge pull request #20632 from wojtek-t/fix_metrics

pkg/apiserver/apiserver.go

@@ -168,13 +168,10 @@ func (g *APIGroupVersion) newInstaller() *APIInstaller {
 
 // TODO: document all handlers
 // InstallSupport registers the APIServer support functions
-func InstallSupport(mux Mux, ws *restful.WebService, enableResettingMetrics bool, checks ...healthz.HealthzChecker) {
+func InstallSupport(mux Mux, ws *restful.WebService, checks ...healthz.HealthzChecker) {
 	// TODO: convert healthz and metrics to restful and remove container arg
 	healthz.InstallHandler(mux, checks...)
 	mux.Handle("/metrics", prometheus.Handler())
-	if enableResettingMetrics {
-		mux.HandleFunc("/resetMetrics", metrics.Reset)
-	}
 
 	// Set up a service to return the git code version.
 	ws.Path("/version")

pkg/apiserver/apiserver_test.go

@@ -297,7 +297,7 @@ func handleInternal(storage map[string]rest.Storage, admissionControl admission.
 	}
 
 	ws := new(restful.WebService)
-	InstallSupport(mux, ws, false)
+	InstallSupport(mux, ws)
 	container.Add(ws)
 	return &defaultAPIServer{mux, container}
 }

pkg/apiserver/metrics/metrics.go

@@ -17,8 +17,6 @@ limitations under the License.
 package metrics
 
 import (
-	"io"
-	"net/http"
 	"strconv"
 	"time"
 
@@ -68,9 +66,8 @@ func Monitor(verb, resource *string, client string, httpCode *int, reqStart time
 	requestLatenciesSummary.WithLabelValues(*verb, *resource).Observe(float64((time.Since(reqStart)) / time.Microsecond))
 }
 
-func Reset(w http.ResponseWriter, req *http.Request) {
+func Reset() {
 	requestCounter.Reset()
 	requestLatencies.Reset()
 	requestLatenciesSummary.Reset()
-	io.WriteString(w, "metrics reset\n")
 }

pkg/master/master.go

@@ -18,6 +18,7 @@ package master
 
 import (
 	"fmt"
+	"io"
 	"net"
 	"net/http"
 	"net/url"
@@ -32,6 +33,7 @@ import (
 	"k8s.io/kubernetes/pkg/apimachinery/registered"
 	"k8s.io/kubernetes/pkg/apis/extensions"
 	"k8s.io/kubernetes/pkg/apiserver"
+	apiservermetrics "k8s.io/kubernetes/pkg/apiserver/metrics"
 	"k8s.io/kubernetes/pkg/genericapiserver"
 	"k8s.io/kubernetes/pkg/healthz"
 	kubeletclient "k8s.io/kubernetes/pkg/kubelet/client"
@@ -68,6 +70,7 @@ import (
 	thirdpartyresourcedataetcd "k8s.io/kubernetes/pkg/registry/thirdpartyresourcedata/etcd"
 	"k8s.io/kubernetes/pkg/runtime"
 	"k8s.io/kubernetes/pkg/storage"
+	etcdmetrics "k8s.io/kubernetes/pkg/storage/etcd/metrics"
 	etcdutil "k8s.io/kubernetes/pkg/storage/etcd/util"
 	"k8s.io/kubernetes/pkg/util"
 	"k8s.io/kubernetes/pkg/util/sets"
@@ -158,6 +161,12 @@ func New(c *Config) *Master {
 	return m
 }
 
+func resetMetrics(w http.ResponseWriter, req *http.Request) {
+	apiservermetrics.Reset()
+	etcdmetrics.Reset()
+	io.WriteString(w, "metrics reset\n")
+}
+
 func (m *Master) InstallAPIs(c *Config) {
 	apiGroupsInfo := []genericapiserver.APIGroupInfo{}
 
@@ -190,7 +199,10 @@ func (m *Master) InstallAPIs(c *Config) {
 	}
 
 	// TODO(nikhiljindal): Refactor generic parts of support services (like /versions) to genericapiserver.
-	apiserver.InstallSupport(m.MuxHelper, m.RootWebService, c.EnableProfiling, healthzChecks...)
+	apiserver.InstallSupport(m.MuxHelper, m.RootWebService, healthzChecks...)
+	if c.EnableProfiling {
+		m.MuxHelper.HandleFunc("/resetMetrics", resetMetrics)
+	}
 
 	// Install root web services
 	m.HandlerContainer.Add(m.RootWebService)

pkg/storage/etcd/metrics/metrics.go

@@ -102,3 +102,12 @@ func ObserveCacheMiss() {
 func ObserveNewEntry() {
 	cacheEntryCounter.Inc()
 }
+
+func Reset() {
+	cacheHitCounter.Set(0)
+	cacheMissCounter.Set(0)
+	cacheEntryCounter.Set(0)
+	// TODO: Reset cacheAddLatency.
+	// TODO: Reset cacheGetLatency.
+	etcdRequestLatenciesSummary.Reset()
+}

test/e2e/metrics_util.go

@@ -92,25 +92,16 @@ func (m *MetricsForE2E) PrintJSON() string {
 
 var InterestingApiServerMetrics = []string{
 	"apiserver_request_count",
-	"apiserver_request_latencies_bucket",
+	"apiserver_request_latencies_summary",
 	"etcd_helper_cache_entry_count",
 	"etcd_helper_cache_hit_count",
 	"etcd_helper_cache_miss_count",
 	"etcd_request_cache_add_latencies_summary",
 	"etcd_request_cache_get_latencies_summary",
 	"etcd_request_latencies_summary",
-	"go_gc_duration_seconds",
-	"go_goroutines",
-	"process_cpu_seconds_total",
-	"process_open_fds",
-	"process_resident_memory_bytes",
-	"process_start_time_seconds",
-	"process_virtual_memory_bytes",
 }
 
 var InterestingKubeletMetrics = []string{
-	"go_gc_duration_seconds",
-	"go_goroutines",
 	"kubelet_container_manager_latency_microseconds",
 	"kubelet_docker_errors",
 	"kubelet_docker_operations_latency_microseconds",
@@ -119,11 +110,6 @@ var InterestingKubeletMetrics = []string{
 	"kubelet_pod_worker_latency_microseconds",
 	"kubelet_pod_worker_start_latency_microseconds",
 	"kubelet_sync_pods_latency_microseconds",
-	"process_cpu_seconds_total",
-	"process_open_fds",
-	"process_resident_memory_bytes",
-	"process_start_time_seconds",
-	"process_virtual_memory_bytes",
 }
 
 // Dashboard metrics