Merge pull request #64527 from gnufied/add-pvc-in-use-metrics

pkg/controller/volume/attachdetach/BUILD

@@ -14,6 +14,7 @@ go_library(
         "//pkg/cloudprovider:go_default_library",
         "//pkg/controller:go_default_library",
         "//pkg/controller/volume/attachdetach/cache:go_default_library",
+        "//pkg/controller/volume/attachdetach/metrics:go_default_library",
         "//pkg/controller/volume/attachdetach/populator:go_default_library",
         "//pkg/controller/volume/attachdetach/reconciler:go_default_library",
         "//pkg/controller/volume/attachdetach/statusupdater:go_default_library",
@@ -69,6 +70,7 @@ filegroup(
     srcs = [
         ":package-srcs",
         "//pkg/controller/volume/attachdetach/cache:all-srcs",
+        "//pkg/controller/volume/attachdetach/metrics:all-srcs",
         "//pkg/controller/volume/attachdetach/populator:all-srcs",
         "//pkg/controller/volume/attachdetach/reconciler:all-srcs",
         "//pkg/controller/volume/attachdetach/statusupdater:all-srcs",

pkg/controller/volume/attachdetach/attach_detach_controller.go

@@ -39,6 +39,7 @@ import (
 	"k8s.io/kubernetes/pkg/cloudprovider"
 	"k8s.io/kubernetes/pkg/controller"
 	"k8s.io/kubernetes/pkg/controller/volume/attachdetach/cache"
+	"k8s.io/kubernetes/pkg/controller/volume/attachdetach/metrics"
 	"k8s.io/kubernetes/pkg/controller/volume/attachdetach/populator"
 	"k8s.io/kubernetes/pkg/controller/volume/attachdetach/reconciler"
 	"k8s.io/kubernetes/pkg/controller/volume/attachdetach/statusupdater"
@@ -273,6 +274,7 @@ func (adc *attachDetachController) Run(stopCh <-chan struct{}) {
 	}
 	go adc.reconciler.Run(stopCh)
 	go adc.desiredStateOfWorldPopulator.Run(stopCh)
+	metrics.Register(adc.pvcLister, adc.pvLister, adc.podLister, &adc.volumePluginMgr)
 
 	<-stopCh
 }

pkg/controller/volume/attachdetach/metrics/BUILD

+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["metrics.go"],
+    importpath = "k8s.io/kubernetes/pkg/controller/volume/attachdetach/metrics",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/controller/volume/attachdetach/util:go_default_library",
+        "//pkg/volume:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//staging/src/k8s.io/client-go/listers/core/v1:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["metrics_test.go"],
+    embed = [":go_default_library"],
+    deps = [
+        "//pkg/controller:go_default_library",
+        "//pkg/volume/testing:go_default_library",
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/client-go/informers:go_default_library",
+        "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

pkg/controller/volume/attachdetach/metrics/metrics.go

+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metrics
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/golang/glog"
+	"github.com/prometheus/client_golang/prometheus"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/types"
+	corelisters "k8s.io/client-go/listers/core/v1"
+	"k8s.io/kubernetes/pkg/controller/volume/attachdetach/util"
+	"k8s.io/kubernetes/pkg/volume"
+)
+
+var (
+	inUseVolumeMetricDesc = prometheus.NewDesc(
+		prometheus.BuildFQName("", "storage_count", "attachable_volumes_in_use"),
+		"Measure number of volumes in use",
+		[]string{"node", "volume_plugin"}, nil)
+)
+var registerMetrics sync.Once
+
+type volumeInUseCollector struct {
+	pvcLister       corelisters.PersistentVolumeClaimLister
+	podLister       corelisters.PodLister
+	pvLister        corelisters.PersistentVolumeLister
+	volumePluginMgr *volume.VolumePluginMgr
+}
+
+// nodeVolumeCount contains map of {"nodeName": {"pluginName": volume_count }}
+// For example :
+//     node 172.168.1.100.ec2.internal has 10 EBS and 3 glusterfs PVC in use
+//     {"172.168.1.100.ec2.internal": {"aws-ebs": 10, "glusterfs": 3}}
+type nodeVolumeCount map[types.NodeName]map[string]int
+
+// Register registers pvc's in-use metrics
+func Register(pvcLister corelisters.PersistentVolumeClaimLister,
+	pvLister corelisters.PersistentVolumeLister,
+	podLister corelisters.PodLister,
+	pluginMgr *volume.VolumePluginMgr) {
+	registerMetrics.Do(func() {
+		prometheus.MustRegister(newVolumeInUseCollector(pvcLister, podLister, pvLister, pluginMgr))
+	})
+
+}
+
+func (volumeInUse nodeVolumeCount) add(nodeName types.NodeName, pluginName string) {
+	nodeCount, ok := volumeInUse[nodeName]
+	if !ok {
+		nodeCount = map[string]int{}
+	}
+	nodeCount[pluginName]++
+	volumeInUse[nodeName] = nodeCount
+}
+
+func newVolumeInUseCollector(
+	pvcLister corelisters.PersistentVolumeClaimLister,
+	podLister corelisters.PodLister,
+	pvLister corelisters.PersistentVolumeLister,
+	pluginMgr *volume.VolumePluginMgr) *volumeInUseCollector {
+	fmt.Println("Doing this crap again")
+	return &volumeInUseCollector{pvcLister, podLister, pvLister, pluginMgr}
+}
+
+// Check if our collector implements necessary collector interface
+var _ prometheus.Collector = &volumeInUseCollector{}
+
+func (collector *volumeInUseCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- inUseVolumeMetricDesc
+}
+
+func (collector *volumeInUseCollector) Collect(ch chan<- prometheus.Metric) {
+	nodeVolumeMap := collector.getVolumeInUseCount()
+	for nodeName, pluginCount := range nodeVolumeMap {
+		for pluginName, count := range pluginCount {
+			metric, err := prometheus.NewConstMetric(inUseVolumeMetricDesc,
+				prometheus.GaugeValue,
+				float64(count),
+				string(nodeName),
+				pluginName)
+			if err != nil {
+				glog.Warningf("Failed to create metric : %v", err)
+			}
+			ch <- metric
+		}
+	}
+}
+
+func (collector *volumeInUseCollector) getVolumeInUseCount() nodeVolumeCount {
+	pods, err := collector.podLister.List(labels.Everything())
+	if err != nil {
+		glog.Errorf("Error getting pod list")
+		return nil
+	}
+
+	nodeVolumeMap := make(nodeVolumeCount)
+	for _, pod := range pods {
+		if len(pod.Spec.Volumes) <= 0 {
+			continue
+		}
+
+		nodeName := types.NodeName(pod.Spec.NodeName)
+		if nodeName == "" {
+			continue
+		}
+		for _, podVolume := range pod.Spec.Volumes {
+			volumeSpec, err := util.CreateVolumeSpec(podVolume, pod.Namespace, collector.pvcLister, collector.pvLister)
+			if err != nil {
+				continue
+			}
+			volumePlugin, err := collector.volumePluginMgr.FindPluginBySpec(volumeSpec)
+			if err != nil {
+				continue
+			}
+			nodeVolumeMap.add(nodeName, volumePlugin.GetPluginName())
+		}
+	}
+	return nodeVolumeMap
+}

pkg/controller/volume/attachdetach/metrics/metrics_test.go

+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metrics
+
+import (
+	"testing"
+
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/informers"
+	"k8s.io/client-go/kubernetes/fake"
+	"k8s.io/kubernetes/pkg/controller"
+	volumetesting "k8s.io/kubernetes/pkg/volume/testing"
+)
+
+func TestMetricCollection(t *testing.T) {
+	fakeVolumePluginMgr, _ := volumetesting.GetTestVolumePluginMgr(t)
+	fakeClient := &fake.Clientset{}
+
+	fakeInformerFactory := informers.NewSharedInformerFactory(fakeClient, controller.NoResyncPeriodFunc())
+	fakePodInformer := fakeInformerFactory.Core().V1().Pods()
+	pod := &v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "metric-test-pod",
+			UID:       "metric-test-pod-uid",
+			Namespace: "metric-test",
+		},
+		Spec: v1.PodSpec{
+			NodeName: "metric-test-host",
+			Volumes: []v1.Volume{
+				{
+					Name: "metric-test-volume-name",
+					VolumeSource: v1.VolumeSource{
+						PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
+							ClaimName: "metric-test-pvc",
+						},
+					},
+				},
+			},
+		},
+		Status: v1.PodStatus{
+			Phase: v1.PodPhase("Running"),
+		},
+	}
+
+	fakePodInformer.Informer().GetStore().Add(pod)
+	pvcInformer := fakeInformerFactory.Core().V1().PersistentVolumeClaims()
+	pvInformer := fakeInformerFactory.Core().V1().PersistentVolumes()
+
+	pvc := &v1.PersistentVolumeClaim{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "metric-test-pvc",
+			Namespace: "metric-test",
+			UID:       "metric-test-pvc-1",
+		},
+		Spec: v1.PersistentVolumeClaimSpec{
+			AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadOnlyMany, v1.ReadWriteOnce},
+			Resources: v1.ResourceRequirements{
+				Requests: v1.ResourceList{
+					v1.ResourceName(v1.ResourceStorage): resource.MustParse("2G"),
+				},
+			},
+			VolumeName: "test-metric-pv-1",
+		},
+		Status: v1.PersistentVolumeClaimStatus{
+			Phase: v1.ClaimBound,
+		},
+	}
+	pv := &v1.PersistentVolume{
+		ObjectMeta: metav1.ObjectMeta{
+			UID:  "test-metric-pv-1",
+			Name: "test-metric-pv-1",
+		},
+		Spec: v1.PersistentVolumeSpec{
+			Capacity: v1.ResourceList{
+				v1.ResourceName(v1.ResourceStorage): resource.MustParse("5G"),
+			},
+			PersistentVolumeSource: v1.PersistentVolumeSource{
+				GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{},
+			},
+			AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce, v1.ReadOnlyMany},
+			// this one we're pretending is already bound
+			ClaimRef: &v1.ObjectReference{UID: "metric-test-pvc-1", Namespace: "metric-test"},
+		},
+	}
+	pvcInformer.Informer().GetStore().Add(pvc)
+	pvInformer.Informer().GetStore().Add(pv)
+	pvcLister := pvcInformer.Lister()
+	pvLister := pvInformer.Lister()
+
+	metricCollector := newVolumeInUseCollector(pvcLister, fakePodInformer.Lister(), pvLister, fakeVolumePluginMgr)
+	nodeUseMap := metricCollector.getVolumeInUseCount()
+	if len(nodeUseMap) < 1 {
+		t.Errorf("Expected one volume in use got %d", len(nodeUseMap))
+	}
+	testNodeMetric := nodeUseMap["metric-test-host"]
+	pluginUseCount, ok := testNodeMetric["fake-plugin"]
+	if !ok {
+		t.Errorf("Expected fake plugin pvc got nothing")
+	}
+
+	if pluginUseCount < 1 {
+		t.Errorf("Expected at least in-use volume metric got %d", pluginUseCount)
+	}
+
+}