Merge pull request #19263 from hongchaodeng/metrics

plugin/pkg/scheduler/metrics/metrics.go

@@ -28,28 +28,28 @@ const schedulerSubsystem = "scheduler"
 var BindingSaturationReportInterval = 1 * time.Second
 
 var (
-	E2eSchedulingLatency = prometheus.NewSummary(
-		prometheus.SummaryOpts{
+	E2eSchedulingLatency = prometheus.NewHistogram(
+		prometheus.HistogramOpts{
 			Subsystem: schedulerSubsystem,
 			Name:      "e2e_scheduling_latency_microseconds",
 			Help:      "E2e scheduling latency (scheduling algorithm + binding)",
-			MaxAge:    time.Hour,
+			Buckets:   prometheus.ExponentialBuckets(1000, 2, 15),
 		},
 	)
-	SchedulingAlgorithmLatency = prometheus.NewSummary(
-		prometheus.SummaryOpts{
+	SchedulingAlgorithmLatency = prometheus.NewHistogram(
+		prometheus.HistogramOpts{
 			Subsystem: schedulerSubsystem,
 			Name:      "scheduling_algorithm_latency_microseconds",
 			Help:      "Scheduling algorithm latency",
-			MaxAge:    time.Hour,
+			Buckets:   prometheus.ExponentialBuckets(1000, 2, 15),
 		},
 	)
-	BindingLatency = prometheus.NewSummary(
-		prometheus.SummaryOpts{
+	BindingLatency = prometheus.NewHistogram(
+		prometheus.HistogramOpts{
 			Subsystem: schedulerSubsystem,
 			Name:      "binding_latency_microseconds",
 			Help:      "Binding latency",
-			MaxAge:    time.Hour,
+			Buckets:   prometheus.ExponentialBuckets(1000, 2, 15),
 		},
 	)
 	BindingRateLimiterSaturation = prometheus.NewGauge(

plugin/pkg/scheduler/scheduler.go

@@ -125,17 +125,15 @@ func (s *Scheduler) scheduleOne() {
 
 	glog.V(3).Infof("Attempting to schedule: %+v", pod)
 	start := time.Now()
-	defer func() {
-		metrics.E2eSchedulingLatency.Observe(metrics.SinceInMicroseconds(start))
-	}()
 	dest, err := s.config.Algorithm.Schedule(pod, s.config.NodeLister)
-	metrics.SchedulingAlgorithmLatency.Observe(metrics.SinceInMicroseconds(start))
 	if err != nil {
 		glog.V(1).Infof("Failed to schedule: %+v", pod)
 		s.config.Recorder.Eventf(pod, api.EventTypeWarning, "FailedScheduling", "%v", err)
 		s.config.Error(pod, err)
 		return
 	}
+	metrics.SchedulingAlgorithmLatency.Observe(metrics.SinceInMicroseconds(start))
+
 	b := &api.Binding{
 		ObjectMeta: api.ObjectMeta{Namespace: pod.Namespace, Name: pod.Name},
 		Target: api.ObjectReference{
@@ -149,17 +147,19 @@ func (s *Scheduler) scheduleOne() {
 	s.config.Modeler.LockedAction(func() {
 		bindingStart := time.Now()
 		err := s.config.Binder.Bind(b)
-		metrics.BindingLatency.Observe(metrics.SinceInMicroseconds(bindingStart))
 		if err != nil {
 			glog.V(1).Infof("Failed to bind pod: %+v", err)
 			s.config.Recorder.Eventf(pod, api.EventTypeNormal, "FailedScheduling", "Binding rejected: %v", err)
 			s.config.Error(pod, err)
 			return
 		}
+		metrics.BindingLatency.Observe(metrics.SinceInMicroseconds(bindingStart))
 		s.config.Recorder.Eventf(pod, api.EventTypeNormal, "Scheduled", "Successfully assigned %v to %v", pod.Name, dest)
 		// tell the model to assume that this binding took effect.
 		assumed := *pod
 		assumed.Spec.NodeName = dest
 		s.config.Modeler.AssumePod(&assumed)
 	})
+
+	metrics.E2eSchedulingLatency.Observe(metrics.SinceInMicroseconds(start))
 }