Merge pull request #71300 from danielqsj/71165

pkg/util/workqueue/prometheus/prometheus.go

@@ -25,6 +25,18 @@ import (
 // Package prometheus sets the workqueue DefaultMetricsFactory to produce
 // prometheus metrics. To use this package, you just have to import it.
 
+// Metrics subsystem and keys used by the workqueue.
+const (
+	WorkQueueSubsystem         = "workqueue"
+	DepthKey                   = "depth"
+	AddsKey                    = "adds_total"
+	QueueLatencyKey            = "queue_latency_seconds"
+	WorkDurationKey            = "work_duration_seconds"
+	UnfinishedWorkKey          = "unfinished_work_seconds"
+	LongestRunningProcessorKey = "longest_running_processor_seconds"
+	RetriesKey                 = "retries_total"
+)
+
 func init() {
 	workqueue.SetProvider(prometheusMetricsProvider{})
 }
@@ -33,6 +45,90 @@ type prometheusMetricsProvider struct{}
 
 func (prometheusMetricsProvider) NewDepthMetric(name string) workqueue.GaugeMetric {
 	depth := prometheus.NewGauge(prometheus.GaugeOpts{
+		Subsystem:   WorkQueueSubsystem,
+		Name:        DepthKey,
+		Help:        "Current depth of workqueue",
+		ConstLabels: prometheus.Labels{"name": name},
+	})
+	prometheus.Register(depth)
+	return depth
+}
+
+func (prometheusMetricsProvider) NewAddsMetric(name string) workqueue.CounterMetric {
+	adds := prometheus.NewCounter(prometheus.CounterOpts{
+		Subsystem:   WorkQueueSubsystem,
+		Name:        AddsKey,
+		Help:        "Total number of adds handled by workqueue",
+		ConstLabels: prometheus.Labels{"name": name},
+	})
+	prometheus.Register(adds)
+	return adds
+}
+
+func (prometheusMetricsProvider) NewLatencyMetric(name string) workqueue.HistogramMetric {
+	latency := prometheus.NewHistogram(prometheus.HistogramOpts{
+		Subsystem:   WorkQueueSubsystem,
+		Name:        QueueLatencyKey,
+		Help:        "How long in seconds an item stays in workqueue before being requested.",
+		ConstLabels: prometheus.Labels{"name": name},
+		Buckets:     prometheus.ExponentialBuckets(10e-9, 10, 10),
+	})
+	prometheus.Register(latency)
+	return latency
+}
+
+func (prometheusMetricsProvider) NewWorkDurationMetric(name string) workqueue.HistogramMetric {
+	workDuration := prometheus.NewHistogram(prometheus.HistogramOpts{
+		Subsystem:   WorkQueueSubsystem,
+		Name:        WorkDurationKey,
+		Help:        "How long in seconds processing an item from workqueue takes.",
+		ConstLabels: prometheus.Labels{"name": name},
+		Buckets:     prometheus.ExponentialBuckets(10e-9, 10, 10),
+	})
+	prometheus.Register(workDuration)
+	return workDuration
+}
+
+func (prometheusMetricsProvider) NewUnfinishedWorkSecondsMetric(name string) workqueue.SettableGaugeMetric {
+	unfinished := prometheus.NewGauge(prometheus.GaugeOpts{
+		Subsystem: WorkQueueSubsystem,
+		Name:      UnfinishedWorkKey,
+		Help: "How many seconds of work has done that " +
+			"is in progress and hasn't been observed by work_duration. Large " +
+			"values indicate stuck threads. One can deduce the number of stuck " +
+			"threads by observing the rate at which this increases.",
+		ConstLabels: prometheus.Labels{"name": name},
+	})
+	prometheus.Register(unfinished)
+	return unfinished
+}
+
+func (prometheusMetricsProvider) NewLongestRunningProcessorSecondsMetric(name string) workqueue.SettableGaugeMetric {
+	unfinished := prometheus.NewGauge(prometheus.GaugeOpts{
+		Subsystem: WorkQueueSubsystem,
+		Name:      LongestRunningProcessorKey,
+		Help: "How many seconds has the longest running " +
+			"processor for workqueue been running.",
+		ConstLabels: prometheus.Labels{"name": name},
+	})
+	prometheus.Register(unfinished)
+	return unfinished
+}
+
+func (prometheusMetricsProvider) NewRetriesMetric(name string) workqueue.CounterMetric {
+	retries := prometheus.NewCounter(prometheus.CounterOpts{
+		Subsystem:   WorkQueueSubsystem,
+		Name:        RetriesKey,
+		Help:        "Total number of retries handled by workqueue",
+		ConstLabels: prometheus.Labels{"name": name},
+	})
+	prometheus.Register(retries)
+	return retries
+}
+
+// TODO(danielqsj): Remove the following metrics, they are deprecated
+func (prometheusMetricsProvider) NewDeprecatedDepthMetric(name string) workqueue.GaugeMetric {
+	depth := prometheus.NewGauge(prometheus.GaugeOpts{
 		Subsystem: name,
 		Name:      "depth",
 		Help:      "Current depth of workqueue: " + name,
@@ -41,7 +137,7 @@ func (prometheusMetricsProvider) NewDepthMetric(name string) workqueue.GaugeMetr
 	return depth
 }
 
-func (prometheusMetricsProvider) NewAddsMetric(name string) workqueue.CounterMetric {
+func (prometheusMetricsProvider) NewDeprecatedAddsMetric(name string) workqueue.CounterMetric {
 	adds := prometheus.NewCounter(prometheus.CounterOpts{
 		Subsystem: name,
 		Name:      "adds",
@@ -51,7 +147,7 @@ func (prometheusMetricsProvider) NewAddsMetric(name string) workqueue.CounterMet
 	return adds
 }
 
-func (prometheusMetricsProvider) NewLatencyMetric(name string) workqueue.SummaryMetric {
+func (prometheusMetricsProvider) NewDeprecatedLatencyMetric(name string) workqueue.SummaryMetric {
 	latency := prometheus.NewSummary(prometheus.SummaryOpts{
 		Subsystem: name,
 		Name:      "queue_latency",
@@ -61,7 +157,7 @@ func (prometheusMetricsProvider) NewLatencyMetric(name string) workqueue.Summary
 	return latency
 }
 
-func (prometheusMetricsProvider) NewWorkDurationMetric(name string) workqueue.SummaryMetric {
+func (prometheusMetricsProvider) NewDeprecatedWorkDurationMetric(name string) workqueue.SummaryMetric {
 	workDuration := prometheus.NewSummary(prometheus.SummaryOpts{
 		Subsystem: name,
 		Name:      "work_duration",
@@ -71,7 +167,7 @@ func (prometheusMetricsProvider) NewWorkDurationMetric(name string) workqueue.Su
 	return workDuration
 }
 
-func (prometheusMetricsProvider) NewUnfinishedWorkSecondsMetric(name string) workqueue.SettableGaugeMetric {
+func (prometheusMetricsProvider) NewDeprecatedUnfinishedWorkSecondsMetric(name string) workqueue.SettableGaugeMetric {
 	unfinished := prometheus.NewGauge(prometheus.GaugeOpts{
 		Subsystem: name,
 		Name:      "unfinished_work_seconds",
@@ -84,7 +180,7 @@ func (prometheusMetricsProvider) NewUnfinishedWorkSecondsMetric(name string) wor
 	return unfinished
 }
 
-func (prometheusMetricsProvider) NewLongestRunningProcessorMicrosecondsMetric(name string) workqueue.SettableGaugeMetric {
+func (prometheusMetricsProvider) NewDeprecatedLongestRunningProcessorMicrosecondsMetric(name string) workqueue.SettableGaugeMetric {
 	unfinished := prometheus.NewGauge(prometheus.GaugeOpts{
 		Subsystem: name,
 		Name:      "longest_running_processor_microseconds",
@@ -95,7 +191,7 @@ func (prometheusMetricsProvider) NewLongestRunningProcessorMicrosecondsMetric(na
 	return unfinished
 }
 
-func (prometheusMetricsProvider) NewRetriesMetric(name string) workqueue.CounterMetric {
+func (prometheusMetricsProvider) NewDeprecatedRetriesMetric(name string) workqueue.CounterMetric {
 	retries := prometheus.NewCounter(prometheus.CounterOpts{
 		Subsystem: name,
 		Name:      "retries",

staging/src/k8s.io/client-go/util/workqueue/delaying_queue.go

@@ -43,12 +43,13 @@ func NewNamedDelayingQueue(name string) DelayingInterface {
 
 func newDelayingQueue(clock clock.Clock, name string) DelayingInterface {
 	ret := &delayingType{
-		Interface:       NewNamed(name),
-		clock:           clock,
-		heartbeat:       clock.NewTicker(maxWait),
-		stopCh:          make(chan struct{}),
-		waitingForAddCh: make(chan *waitFor, 1000),
-		metrics:         newRetryMetrics(name),
+		Interface:         NewNamed(name),
+		clock:             clock,
+		heartbeat:         clock.NewTicker(maxWait),
+		stopCh:            make(chan struct{}),
+		waitingForAddCh:   make(chan *waitFor, 1000),
+		metrics:           newRetryMetrics(name),
+		deprecatedMetrics: newDeprecatedRetryMetrics(name),
 	}
 
 	go ret.waitingLoop()
@@ -73,7 +74,8 @@ type delayingType struct {
 	waitingForAddCh chan *waitFor
 
 	// metrics counts the number of retries
-	metrics retryMetrics
+	metrics           retryMetrics
+	deprecatedMetrics retryMetrics
 }
 
 // waitFor holds the data to add and the time it should be added
@@ -146,6 +148,7 @@ func (q *delayingType) AddAfter(item interface{}, duration time.Duration) {
 	}
 
 	q.metrics.retry()
+	q.deprecatedMetrics.retry()
 
 	// immediately add things with no delay
 	if duration <= 0 {

staging/src/k8s.io/client-go/util/workqueue/metrics.go

@@ -57,6 +57,11 @@ type SummaryMetric interface {
 	Observe(float64)
 }
 
+// HistogramMetric counts individual observations.
+type HistogramMetric interface {
+	Observe(float64)
+}
+
 type noopMetric struct{}
 
 func (noopMetric) Inc()            {}
@@ -73,15 +78,23 @@ type defaultQueueMetrics struct {
 	// total number of adds handled by a workqueue
 	adds CounterMetric
 	// how long an item stays in a workqueue
-	latency SummaryMetric
+	latency HistogramMetric
 	// how long processing an item from a workqueue takes
-	workDuration         SummaryMetric
+	workDuration         HistogramMetric
 	addTimes             map[t]time.Time
 	processingStartTimes map[t]time.Time
 
 	// how long have current threads been working?
 	unfinishedWorkSeconds   SettableGaugeMetric
 	longestRunningProcessor SettableGaugeMetric
+
+	// TODO(danielqsj): Remove the following metrics, they are deprecated
+	deprecatedDepth                   GaugeMetric
+	deprecatedAdds                    CounterMetric
+	deprecatedLatency                 SummaryMetric
+	deprecatedWorkDuration            SummaryMetric
+	deprecatedUnfinishedWorkSeconds   SettableGaugeMetric
+	deprecatedLongestRunningProcessor SettableGaugeMetric
 }
 
 func (m *defaultQueueMetrics) add(item t) {
@@ -90,7 +103,9 @@ func (m *defaultQueueMetrics) add(item t) {
 	}
 
 	m.adds.Inc()
+	m.deprecatedAdds.Inc()
 	m.depth.Inc()
+	m.deprecatedDepth.Inc()
 	if _, exists := m.addTimes[item]; !exists {
 		m.addTimes[item] = m.clock.Now()
 	}
@@ -102,9 +117,11 @@ func (m *defaultQueueMetrics) get(item t) {
 	}
 
 	m.depth.Dec()
+	m.deprecatedDepth.Dec()
 	m.processingStartTimes[item] = m.clock.Now()
 	if startTime, exists := m.addTimes[item]; exists {
-		m.latency.Observe(m.sinceInMicroseconds(startTime))
+		m.latency.Observe(m.sinceInSeconds(startTime))
+		m.deprecatedLatency.Observe(m.sinceInMicroseconds(startTime))
 		delete(m.addTimes, item)
 	}
 }
@@ -115,7 +132,8 @@ func (m *defaultQueueMetrics) done(item t) {
 	}
 
 	if startTime, exists := m.processingStartTimes[item]; exists {
-		m.workDuration.Observe(m.sinceInMicroseconds(startTime))
+		m.workDuration.Observe(m.sinceInSeconds(startTime))
+		m.deprecatedWorkDuration.Observe(m.sinceInMicroseconds(startTime))
 		delete(m.processingStartTimes, item)
 	}
 }
@@ -135,7 +153,9 @@ func (m *defaultQueueMetrics) updateUnfinishedWork() {
 	// Convert to seconds; microseconds is unhelpfully granular for this.
 	total /= 1000000
 	m.unfinishedWorkSeconds.Set(total)
-	m.longestRunningProcessor.Set(oldest) // in microseconds.
+	m.deprecatedUnfinishedWorkSeconds.Set(total)
+	m.longestRunningProcessor.Set(oldest / 1000000)
+	m.deprecatedLongestRunningProcessor.Set(oldest) // in microseconds.
 }
 
 type noMetrics struct{}
@@ -150,6 +170,11 @@ func (m *defaultQueueMetrics) sinceInMicroseconds(start time.Time) float64 {
 	return float64(m.clock.Since(start).Nanoseconds() / time.Microsecond.Nanoseconds())
 }
 
+// Gets the time since the specified start in seconds.
+func (m *defaultQueueMetrics) sinceInSeconds(start time.Time) float64 {
+	return m.clock.Since(start).Seconds()
+}
+
 type retryMetrics interface {
 	retry()
 }
@@ -170,11 +195,18 @@ func (m *defaultRetryMetrics) retry() {
 type MetricsProvider interface {
 	NewDepthMetric(name string) GaugeMetric
 	NewAddsMetric(name string) CounterMetric
-	NewLatencyMetric(name string) SummaryMetric
-	NewWorkDurationMetric(name string) SummaryMetric
+	NewLatencyMetric(name string) HistogramMetric
+	NewWorkDurationMetric(name string) HistogramMetric
 	NewUnfinishedWorkSecondsMetric(name string) SettableGaugeMetric
-	NewLongestRunningProcessorMicrosecondsMetric(name string) SettableGaugeMetric
+	NewLongestRunningProcessorSecondsMetric(name string) SettableGaugeMetric
 	NewRetriesMetric(name string) CounterMetric
+	NewDeprecatedDepthMetric(name string) GaugeMetric
+	NewDeprecatedAddsMetric(name string) CounterMetric
+	NewDeprecatedLatencyMetric(name string) SummaryMetric
+	NewDeprecatedWorkDurationMetric(name string) SummaryMetric
+	NewDeprecatedUnfinishedWorkSecondsMetric(name string) SettableGaugeMetric
+	NewDeprecatedLongestRunningProcessorMicrosecondsMetric(name string) SettableGaugeMetric
+	NewDeprecatedRetriesMetric(name string) CounterMetric
 }
 
 type noopMetricsProvider struct{}
@@ -187,11 +219,11 @@ func (_ noopMetricsProvider) NewAddsMetric(name string) CounterMetric {
 	return noopMetric{}
 }
 
-func (_ noopMetricsProvider) NewLatencyMetric(name string) SummaryMetric {
+func (_ noopMetricsProvider) NewLatencyMetric(name string) HistogramMetric {
 	return noopMetric{}
 }
 
-func (_ noopMetricsProvider) NewWorkDurationMetric(name string) SummaryMetric {
+func (_ noopMetricsProvider) NewWorkDurationMetric(name string) HistogramMetric {
 	return noopMetric{}
 }
 
@@ -199,7 +231,7 @@ func (_ noopMetricsProvider) NewUnfinishedWorkSecondsMetric(name string) Settabl
 	return noopMetric{}
 }
 
-func (_ noopMetricsProvider) NewLongestRunningProcessorMicrosecondsMetric(name string) SettableGaugeMetric {
+func (_ noopMetricsProvider) NewLongestRunningProcessorSecondsMetric(name string) SettableGaugeMetric {
 	return noopMetric{}
 }
 
@@ -207,6 +239,34 @@ func (_ noopMetricsProvider) NewRetriesMetric(name string) CounterMetric {
 	return noopMetric{}
 }
 
+func (_ noopMetricsProvider) NewDeprecatedDepthMetric(name string) GaugeMetric {
+	return noopMetric{}
+}
+
+func (_ noopMetricsProvider) NewDeprecatedAddsMetric(name string) CounterMetric {
+	return noopMetric{}
+}
+
+func (_ noopMetricsProvider) NewDeprecatedLatencyMetric(name string) SummaryMetric {
+	return noopMetric{}
+}
+
+func (_ noopMetricsProvider) NewDeprecatedWorkDurationMetric(name string) SummaryMetric {
+	return noopMetric{}
+}
+
+func (_ noopMetricsProvider) NewDeprecatedUnfinishedWorkSecondsMetric(name string) SettableGaugeMetric {
+	return noopMetric{}
+}
+
+func (_ noopMetricsProvider) NewDeprecatedLongestRunningProcessorMicrosecondsMetric(name string) SettableGaugeMetric {
+	return noopMetric{}
+}
+
+func (_ noopMetricsProvider) NewDeprecatedRetriesMetric(name string) CounterMetric {
+	return noopMetric{}
+}
+
 var globalMetricsFactory = queueMetricsFactory{
 	metricsProvider: noopMetricsProvider{},
 }
@@ -229,15 +289,21 @@ func (f *queueMetricsFactory) newQueueMetrics(name string, clock clock.Clock) qu
 		return noMetrics{}
 	}
 	return &defaultQueueMetrics{
-		clock:                   clock,
-		depth:                   mp.NewDepthMetric(name),
-		adds:                    mp.NewAddsMetric(name),
-		latency:                 mp.NewLatencyMetric(name),
-		workDuration:            mp.NewWorkDurationMetric(name),
-		unfinishedWorkSeconds:   mp.NewUnfinishedWorkSecondsMetric(name),
-		longestRunningProcessor: mp.NewLongestRunningProcessorMicrosecondsMetric(name),
-		addTimes:                map[t]time.Time{},
-		processingStartTimes:    map[t]time.Time{},
+		clock:                             clock,
+		depth:                             mp.NewDepthMetric(name),
+		adds:                              mp.NewAddsMetric(name),
+		latency:                           mp.NewLatencyMetric(name),
+		workDuration:                      mp.NewWorkDurationMetric(name),
+		unfinishedWorkSeconds:             mp.NewUnfinishedWorkSecondsMetric(name),
+		longestRunningProcessor:           mp.NewLongestRunningProcessorSecondsMetric(name),
+		deprecatedDepth:                   mp.NewDeprecatedDepthMetric(name),
+		deprecatedAdds:                    mp.NewDeprecatedAddsMetric(name),
+		deprecatedLatency:                 mp.NewDeprecatedLatencyMetric(name),
+		deprecatedWorkDuration:            mp.NewDeprecatedWorkDurationMetric(name),
+		deprecatedUnfinishedWorkSeconds:   mp.NewDeprecatedUnfinishedWorkSecondsMetric(name),
+		deprecatedLongestRunningProcessor: mp.NewDeprecatedLongestRunningProcessorMicrosecondsMetric(name),
+		addTimes:                          map[t]time.Time{},
+		processingStartTimes:              map[t]time.Time{},
 	}
 }
 
@@ -251,6 +317,16 @@ func newRetryMetrics(name string) retryMetrics {
 	}
 }
 
+func newDeprecatedRetryMetrics(name string) retryMetrics {
+	var ret *defaultRetryMetrics
+	if len(name) == 0 {
+		return ret
+	}
+	return &defaultRetryMetrics{
+		retries: globalMetricsFactory.metricsProvider.NewDeprecatedRetriesMetric(name),
+	}
+}
+
 // SetProvider sets the metrics provider for all subsequently created work
 // queues. Only the first call has an effect.
 func SetProvider(metricsProvider MetricsProvider) {

staging/src/k8s.io/client-go/util/workqueue/metrics_test.go

@@ -137,6 +137,14 @@ type testMetricsProvider struct {
 	unfinished testMetric
 	longest    testMetric
 	retries    testMetric
+	// deprecated metrics
+	deprecatedDepth      testMetric
+	deprecatedAdds       testMetric
+	deprecatedLatency    testMetric
+	deprecatedDuration   testMetric
+	deprecatedUnfinished testMetric
+	deprecatedLongest    testMetric
+	deprecatedRetries    testMetric
 }
 
 func (m *testMetricsProvider) NewDepthMetric(name string) GaugeMetric {
@@ -147,11 +155,11 @@ func (m *testMetricsProvider) NewAddsMetric(name string) CounterMetric {
 	return &m.adds
 }
 
-func (m *testMetricsProvider) NewLatencyMetric(name string) SummaryMetric {
+func (m *testMetricsProvider) NewLatencyMetric(name string) HistogramMetric {
 	return &m.latency
 }
 
-func (m *testMetricsProvider) NewWorkDurationMetric(name string) SummaryMetric {
+func (m *testMetricsProvider) NewWorkDurationMetric(name string) HistogramMetric {
 	return &m.duration
 }
 
@@ -159,7 +167,7 @@ func (m *testMetricsProvider) NewUnfinishedWorkSecondsMetric(name string) Settab
 	return &m.unfinished
 }
 
-func (m *testMetricsProvider) NewLongestRunningProcessorMicrosecondsMetric(name string) SettableGaugeMetric {
+func (m *testMetricsProvider) NewLongestRunningProcessorSecondsMetric(name string) SettableGaugeMetric {
 	return &m.longest
 }
 
@@ -167,6 +175,34 @@ func (m *testMetricsProvider) NewRetriesMetric(name string) CounterMetric {
 	return &m.retries
 }
 
+func (m *testMetricsProvider) NewDeprecatedDepthMetric(name string) GaugeMetric {
+	return &m.deprecatedDepth
+}
+
+func (m *testMetricsProvider) NewDeprecatedAddsMetric(name string) CounterMetric {
+	return &m.deprecatedAdds
+}
+
+func (m *testMetricsProvider) NewDeprecatedLatencyMetric(name string) SummaryMetric {
+	return &m.deprecatedLatency
+}
+
+func (m *testMetricsProvider) NewDeprecatedWorkDurationMetric(name string) SummaryMetric {
+	return &m.deprecatedDuration
+}
+
+func (m *testMetricsProvider) NewDeprecatedUnfinishedWorkSecondsMetric(name string) SettableGaugeMetric {
+	return &m.deprecatedUnfinished
+}
+
+func (m *testMetricsProvider) NewDeprecatedLongestRunningProcessorMicrosecondsMetric(name string) SettableGaugeMetric {
+	return &m.deprecatedLongest
+}
+
+func (m *testMetricsProvider) NewDeprecatedRetriesMetric(name string) CounterMetric {
+	return &m.deprecatedRetries
+}
+
 func TestSinceInMicroseconds(t *testing.T) {
 	mp := testMetricsProvider{}
 	c := clock.NewFakeClock(time.Now())
@@ -201,10 +237,18 @@ func TestMetrics(t *testing.T) {
 		t.Errorf("expected %v, got %v", e, a)
 	}
 
+	if e, a := 1.0, mp.deprecatedAdds.gaugeValue(); e != a {
+		t.Errorf("expected %v, got %v", e, a)
+	}
+
 	if e, a := 1.0, mp.depth.gaugeValue(); e != a {
 		t.Errorf("expected %v, got %v", e, a)
 	}
 
+	if e, a := 1.0, mp.deprecatedDepth.gaugeValue(); e != a {
+		t.Errorf("expected %v, got %v", e, a)
+	}
+
 	c.Step(50 * time.Microsecond)
 
 	// Start processing
@@ -213,15 +257,24 @@ func TestMetrics(t *testing.T) {
 		t.Errorf("Expected %v, got %v", "foo", i)
 	}
 
-	if e, a := 50.0, mp.latency.observationValue(); e != a {
+	if e, a := 5e-05, mp.latency.observationValue(); e != a {
 		t.Errorf("expected %v, got %v", e, a)
 	}
 	if e, a := 1, mp.latency.observationCount(); e != a {
 		t.Errorf("expected %v, got %v", e, a)
 	}
+	if e, a := 50.0, mp.deprecatedLatency.observationValue(); e != a {
+		t.Errorf("expected %v, got %v", e, a)
+	}
+	if e, a := 1, mp.deprecatedLatency.observationCount(); e != a {
+		t.Errorf("expected %v, got %v", e, a)
+	}
 	if e, a := 0.0, mp.depth.gaugeValue(); e != a {
 		t.Errorf("expected %v, got %v", e, a)
 	}
+	if e, a := 0.0, mp.deprecatedDepth.gaugeValue(); e != a {
+		t.Errorf("expected %v, got %v", e, a)
+	}
 
 	// Add it back while processing; multiple adds of the same item are
 	// de-duped.
@@ -233,27 +286,42 @@ func TestMetrics(t *testing.T) {
 	if e, a := 2.0, mp.adds.gaugeValue(); e != a {
 		t.Errorf("expected %v, got %v", e, a)
 	}
+	if e, a := 2.0, mp.deprecatedAdds.gaugeValue(); e != a {
+		t.Errorf("expected %v, got %v", e, a)
+	}
 	// One thing remains in the queue
 	if e, a := 1.0, mp.depth.gaugeValue(); e != a {
 		t.Errorf("expected %v, got %v", e, a)
 	}
+	if e, a := 1.0, mp.deprecatedDepth.gaugeValue(); e != a {
+		t.Errorf("expected %v, got %v", e, a)
+	}
 
 	c.Step(25 * time.Microsecond)
 
 	// Finish it up
 	q.Done(i)
 
-	if e, a := 25.0, mp.duration.observationValue(); e != a {
+	if e, a := 2.5e-05, mp.duration.observationValue(); e != a {
 		t.Errorf("expected %v, got %v", e, a)
 	}
 	if e, a := 1, mp.duration.observationCount(); e != a {
 		t.Errorf("expected %v, got %v", e, a)
 	}
+	if e, a := 25.0, mp.deprecatedDuration.observationValue(); e != a {
+		t.Errorf("expected %v, got %v", e, a)
+	}
+	if e, a := 1, mp.deprecatedDuration.observationCount(); e != a {
+		t.Errorf("expected %v, got %v", e, a)
+	}
 
 	// One thing remains in the queue
 	if e, a := 1.0, mp.depth.gaugeValue(); e != a {
 		t.Errorf("expected %v, got %v", e, a)
 	}
+	if e, a := 1.0, mp.deprecatedDepth.gaugeValue(); e != a {
+		t.Errorf("expected %v, got %v", e, a)
+	}
 
 	// It should be back on the queue
 	i, _ = q.Get()
@@ -261,33 +329,54 @@ func TestMetrics(t *testing.T) {
 		t.Errorf("Expected %v, got %v", "foo", i)
 	}
 
-	if e, a := 25.0, mp.latency.observationValue(); e != a {
+	if e, a := 2.5e-05, mp.latency.observationValue(); e != a {
 		t.Errorf("expected %v, got %v", e, a)
 	}
 	if e, a := 2, mp.latency.observationCount(); e != a {
 		t.Errorf("expected %v, got %v", e, a)
 	}
+	if e, a := 25.0, mp.deprecatedLatency.observationValue(); e != a {
+		t.Errorf("expected %v, got %v", e, a)
+	}
+	if e, a := 2, mp.deprecatedLatency.observationCount(); e != a {
+		t.Errorf("expected %v, got %v", e, a)
+	}
 
 	// use a channel to ensure we don't look at the metric before it's
 	// been set.
 	ch := make(chan struct{}, 1)
 	mp.unfinished.notifyCh = ch
+	mp.deprecatedUnfinished.notifyCh = ch
 	c.Step(time.Millisecond)
 	<-ch
+	<-ch
 	mp.unfinished.notifyCh = nil
+	mp.deprecatedUnfinished.notifyCh = nil
 	if e, a := .001, mp.unfinished.gaugeValue(); e != a {
 		t.Errorf("expected %v, got %v", e, a)
 	}
-	if e, a := 1000.0, mp.longest.gaugeValue(); e != a {
+	if e, a := .001, mp.deprecatedUnfinished.gaugeValue(); e != a {
+		t.Errorf("expected %v, got %v", e, a)
+	}
+	if e, a := .001, mp.longest.gaugeValue(); e != a {
+		t.Errorf("expected %v, got %v", e, a)
+	}
+	if e, a := 1000.0, mp.deprecatedLongest.gaugeValue(); e != a {
 		t.Errorf("expected %v, got %v", e, a)
 	}
 
 	// Finish that one up
 	q.Done(i)
-	if e, a := 1000.0, mp.duration.observationValue(); e != a {
+	if e, a := .001, mp.duration.observationValue(); e != a {
 		t.Errorf("expected %v, got %v", e, a)
 	}
 	if e, a := 2, mp.duration.observationCount(); e != a {
 		t.Errorf("expected %v, got %v", e, a)
 	}
+	if e, a := 1000.0, mp.deprecatedDuration.observationValue(); e != a {
+		t.Errorf("expected %v, got %v", e, a)
+	}
+	if e, a := 2, mp.deprecatedDuration.observationCount(); e != a {
+		t.Errorf("expected %v, got %v", e, a)
+	}
 }

staging/src/k8s.io/client-go/util/workqueue/rate_limiting_queue_test.go

@@ -28,12 +28,13 @@ func TestRateLimitingQueue(t *testing.T) {
 	queue := NewRateLimitingQueue(limiter).(*rateLimitingType)
 	fakeClock := clock.NewFakeClock(time.Now())
 	delayingQueue := &delayingType{
-		Interface:       New(),
-		clock:           fakeClock,
-		heartbeat:       fakeClock.NewTicker(maxWait),
-		stopCh:          make(chan struct{}),
-		waitingForAddCh: make(chan *waitFor, 1000),
-		metrics:         newRetryMetrics(""),
+		Interface:         New(),
+		clock:             fakeClock,
+		heartbeat:         fakeClock.NewTicker(maxWait),
+		stopCh:            make(chan struct{}),
+		waitingForAddCh:   make(chan *waitFor, 1000),
+		metrics:           newRetryMetrics(""),
+		deprecatedMetrics: newDeprecatedRetryMetrics(""),
 	}
 	queue.DelayingInterface = delayingQueue