Add extra metrics for PV Controller

pkg/controller/volume/persistentvolume/metrics/metrics.go

@@ -56,6 +56,8 @@ type PVCLister interface {
 func Register(pvLister PVLister, pvcLister PVCLister) {
 	registerMetrics.Do(func() {
 		prometheus.MustRegister(newPVAndPVCCountCollector(pvLister, pvcLister))
+		prometheus.MustRegister(volumeOperationMetric)
+		prometheus.MustRegister(volumeOperationErrorsMetric)
 	})
 }
 
@@ -89,6 +91,19 @@ var (
 		prometheus.BuildFQName("", pvControllerSubsystem, unboundPVCKey),
 		"Gauge measuring number of persistent volume claim currently unbound",
 		[]string{namespaceLabel}, nil)
+
+	volumeOperationMetric = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name: "volume_operation_total_seconds",
+			Help: "Total volume operation time",
+		},
+		[]string{"plugin_name", "operation_name"})
+	volumeOperationErrorsMetric = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "volume_operation_total_errors",
+			Help: "Total volume operation erros",
+		},
+		[]string{"plugin_name", "operation_name"})
 )
 
 func (collector *pvAndPVCCountCollector) Describe(ch chan<- *prometheus.Desc) {
@@ -182,3 +197,15 @@ func (collector *pvAndPVCCountCollector) pvcCollect(ch chan<- prometheus.Metric)
 		ch <- metric
 	}
 }
+
+// RecordVolumeOperationMetric records the latency and errors of volume operations.
+func RecordVolumeOperationMetric(pluginName, opName string, timeTaken float64, err error) {
+	if pluginName == "" {
+		pluginName = "N/A"
+	}
+	if err != nil {
+		volumeOperationErrorsMetric.WithLabelValues(pluginName, opName).Inc()
+		return
+	}
+	volumeOperationMetric.WithLabelValues(pluginName, opName).Observe(timeTaken)
+}

test/e2e/storage/volume_metrics.go

@@ -167,6 +167,37 @@ var _ = utils.SIGDescribe("[Serial] Volume metrics", func() {
 		framework.ExpectNoError(framework.DeletePodWithWait(f, c, pod))
 	})
 
+	It("should create metrics for total time taken in volume operations in P/V Controller", func() {
+		var err error
+		pvc, err = c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(pvc)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(pvc).ToNot(Equal(nil))
+
+		claims := []*v1.PersistentVolumeClaim{pvc}
+		pod := framework.MakePod(ns, nil, claims, false, "")
+		pod, err = c.CoreV1().Pods(ns).Create(pod)
+		Expect(err).NotTo(HaveOccurred())
+
+		err = framework.WaitForPodRunningInNamespace(c, pod)
+		framework.ExpectNoError(framework.WaitForPodRunningInNamespace(c, pod), "Error starting pod ", pod.Name)
+
+		pod, err = c.CoreV1().Pods(ns).Get(pod.Name, metav1.GetOptions{})
+		Expect(err).NotTo(HaveOccurred())
+
+		controllerMetrics, err := metricsGrabber.GrabFromControllerManager()
+		if err != nil {
+			framework.Skipf("Could not get controller-manager metrics - skipping")
+		}
+
+		metricKey := "volume_operation_total_seconds_count"
+		dimensions := []string{"operation_name", "plugin_name"}
+		valid := hasValidMetrics(metrics.Metrics(controllerMetrics), metricKey, dimensions...)
+		Expect(valid).To(BeTrue(), "Invalid metric in P/V Controller metrics: %q", metricKey)
+
+		framework.Logf("Deleting pod %q/%q", pod.Namespace, pod.Name)
+		framework.ExpectNoError(framework.DeletePodWithWait(f, c, pod))
+	})
+
 	// Test for pv controller metrics, concretely: bound/unbound pv/pvc count.
 	Describe("PVController", func() {
 		const (
@@ -454,3 +485,23 @@ func calculateRelativeValues(originValues, updatedValues map[string]int64) map[s
 	}
 	return relativeValues
 }
+
+func hasValidMetrics(metrics metrics.Metrics, metricKey string, dimensions ...string) bool {
+	var errCount int
+	framework.Logf("Looking for sample in metric %q", metricKey)
+	samples, ok := metrics[metricKey]
+	if !ok {
+		framework.Logf("Key %q was not found in metrics", metricKey)
+		return false
+	}
+	for _, sample := range samples {
+		framework.Logf("Found sample %q", sample.String())
+		for _, d := range dimensions {
+			if _, ok := sample.Metric[model.LabelName(d)]; !ok {
+				framework.Logf("Error getting dimension %q for metric %q, sample %q", d, metricKey, sample.String())
+				errCount++
+			}
+		}
+	}
+	return errCount == 0
+}