Skip to content

K
k3s
Commit 3ed34183 authored by Matthew Wong's avatar Matthew Wong
Browse files
Options

Add volume operation metrics to operation executor and PV controller

parent bc1a58ae
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 146 additions and 59 deletions
pkg/controller/volume/persistentvolume/BUILD
View file @ 3ed34183
......@@ -24,6 +24,7 @@ go_library(
"//pkg/util/io:go_default_library",
"//pkg/util/mount:go_default_library",
"//pkg/volume:go_default_library",
"//pkg/volume/util:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/api/storage/v1:go_default_library",
......
pkg/controller/volume/persistentvolume/pv_controller.go
View file @ 3ed34183
......@@ -39,6 +39,7 @@ import (
"k8s.io/kubernetes/pkg/util/goroutinemap"
"k8s.io/kubernetes/pkg/util/goroutinemap/exponentialbackoff"
vol "k8s.io/kubernetes/pkg/volume"
"k8s.io/kubernetes/pkg/volume/util"
"github.com/golang/glog"
)
......@@ -1216,7 +1217,10 @@ func (ctrl *PersistentVolumeController) doDeleteVolume(volume *v1.PersistentVolu
return false, fmt.Errorf("Failed to create deleter for volume %q: %v", volume.Name, err)
}
if err = deleter.Delete(); err != nil {
opComplete := util.OperationCompleteHook(plugin.GetPluginName(), "volume_delete")
err = deleter.Delete()
opComplete(err)
if err != nil {
// Deleter failed
return false, err
}
......@@ -1326,7 +1330,9 @@ func (ctrl *PersistentVolumeController) provisionClaimOperation(claimObj interfa
return
}
opComplete := util.OperationCompleteHook(plugin.GetPluginName(), "volume_provision")
volume, err = provisioner.Provision()
opComplete(err)
if err != nil {
strerr := fmt.Sprintf("Failed to provision volume with StorageClass %q: %v", storageClass.Name, err)
glog.V(2).Infof("failed to provision volume for claim %q with StorageClass %q: %v", claimToClaimKey(claim), storageClass.Name, err)
......
pkg/volume/util/BUILD
View file @ 3ed34183
......@@ -15,6 +15,7 @@ go_library(
"doc.go",
"fs_unsupported.go",
"io_util.go",
"metrics.go",
"util.go",
] + select({
"@io_bazel_rules_go//go/platform:darwin_amd64": [
......@@ -31,6 +32,7 @@ go_library(
"//pkg/api/v1/helper:go_default_library",
"//pkg/util/mount:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/api/storage/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
......
pkg/volume/util/metrics.go 0 → 100644
View file @ 3ed34183
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package util
import (
"time"
"github.com/prometheus/client_golang/prometheus"
)
var storageOperationMetric = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "storage_operation_duration_seconds",
Help: "Storage operation duration",
},
[]string{"volume_plugin", "operation_name"},
)
var storageOperationErrorMetric = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "storage_operation_errors_total",
Help: "Storage operation errors",
},
[]string{"volume_plugin", "operation_name"},
)
func init() {
registerMetrics()
}
func registerMetrics() {
prometheus.MustRegister(storageOperationMetric)
prometheus.MustRegister(storageOperationErrorMetric)
}
// OperationCompleteHook returns a hook to call when an operation is completed
func OperationCompleteHook(plugin, operationName string) func(error) {
requestTime := time.Now()
opComplete := func(err error) {
timeTaken := time.Since(requestTime).Seconds()
// Create metric with operation name and plugin name
if err != nil {
storageOperationErrorMetric.WithLabelValues(plugin, operationName).Inc()
} else {
storageOperationMetric.WithLabelValues(plugin, operationName).Observe(timeTaken)
}
}
return opComplete
}
pkg/volume/util/nestedpendingoperations/nestedpendingoperations.go
View file @ 3ed34183
......@@ -55,7 +55,7 @@ type NestedPendingOperations interface {
// concatenation of volumeName and podName is removed from the list of
// executing operations allowing a new operation to be started with the
// volumeName without error.
Run(volumeName v1.UniqueVolumeName, podName types.UniquePodName, operationFunc func() error) error
Run(volumeName v1.UniqueVolumeName, podName types.UniquePodName, operationFunc func() error, operationCompleteFunc func(error)) error
// Wait blocks until all operations are completed. This is typically
// necessary during tests - the test should wait until all operations finish
......@@ -94,7 +94,8 @@ type operation struct {
func (grm *nestedPendingOperations) Run(
volumeName v1.UniqueVolumeName,
podName types.UniquePodName,
operationFunc func() error) error {
operationFunc func() error,
operationCompleteFunc func(error)) error {
grm.lock.Lock()
defer grm.lock.Unlock()
opExists, previousOpIndex := grm.isOperationExists(volumeName, podName)
......@@ -132,6 +133,7 @@ func (grm *nestedPendingOperations) Run(
defer k8sRuntime.HandleCrash()
// Handle completion of and error, if any, from operationFunc()
defer grm.operationComplete(volumeName, podName, &err)
defer operationCompleteFunc(err)
// Handle panic, if any, from operationFunc()
defer k8sRuntime.RecoverFromPanic(&err)
return operationFunc()
......
pkg/volume/util/operationexecutor/operation_executor.go
View file @ 3ed34183
......@@ -31,6 +31,7 @@ import (
"k8s.io/apimachinery/pkg/types"
"k8s.io/kubernetes/pkg/util/mount"
"k8s.io/kubernetes/pkg/volume"
"k8s.io/kubernetes/pkg/volume/util"
"k8s.io/kubernetes/pkg/volume/util/nestedpendingoperations"
volumetypes "k8s.io/kubernetes/pkg/volume/util/types"
"k8s.io/kubernetes/pkg/volume/util/volumehelper"
......@@ -535,29 +536,32 @@ func (oe *operationExecutor) IsOperationPending(volumeName v1.UniqueVolumeName,
func (oe *operationExecutor) AttachVolume(
volumeToAttach VolumeToAttach,
actualStateOfWorld ActualStateOfWorldAttacherUpdater) error {
attachFunc, err :=
attachFunc, plugin, err :=
oe.operationGenerator.GenerateAttachVolumeFunc(volumeToAttach, actualStateOfWorld)
if err != nil {
return err
}
opCompleteFunc := util.OperationCompleteHook(plugin, "volume_attach")
return oe.pendingOperations.Run(
volumeToAttach.VolumeName, "" /* podName */, attachFunc)
volumeToAttach.VolumeName, "" /* podName */, attachFunc, opCompleteFunc)
}
func (oe *operationExecutor) DetachVolume(
volumeToDetach AttachedVolume,
verifySafeToDetach bool,
actualStateOfWorld ActualStateOfWorldAttacherUpdater) error {
detachFunc, err :=
detachFunc, plugin, err :=
oe.operationGenerator.GenerateDetachVolumeFunc(volumeToDetach, verifySafeToDetach, actualStateOfWorld)
if err != nil {
return err
}
opCompleteFunc := util.OperationCompleteHook(plugin, "volume_detach")
return oe.pendingOperations.Run(
volumeToDetach.VolumeName, "" /* podName */, detachFunc)
volumeToDetach.VolumeName, "" /* podName */, detachFunc, opCompleteFunc)
}
func (oe *operationExecutor) VerifyVolumesAreAttached(
attachedVolumes map[types.NodeName][]AttachedVolume,
actualStateOfWorld ActualStateOfWorldAttacherUpdater) {
......@@ -630,9 +634,11 @@ func (oe *operationExecutor) VerifyVolumesAreAttached(
if err != nil {
glog.Errorf("BulkVerifyVolumes.GenerateBulkVolumeVerifyFunc error bulk verifying volumes for plugin %q with %v", pluginName, err)
}
opCompleteFunc := util.OperationCompleteHook(pluginName, "verify_volumes_are_attached")
// Ugly hack to ensure - we don't do parallel bulk polling of same volume plugin
uniquePluginName := v1.UniqueVolumeName(pluginName)
err = oe.pendingOperations.Run(uniquePluginName, "" /* Pod Name */, bulkVerifyVolumeFunc)
err = oe.pendingOperations.Run(uniquePluginName, "" /* Pod Name */, bulkVerifyVolumeFunc, opCompleteFunc)
if err != nil {
glog.Errorf("BulkVerifyVolumes.Run Error bulk volume verification for plugin %q with %v", pluginName, err)
}
......@@ -648,8 +654,10 @@ func (oe *operationExecutor) VerifyVolumesAreAttachedPerNode(
if err != nil {
return err
}
opCompleteFunc := util.OperationCompleteHook("<n/a>", "verify_volumes_are_attached_per_node")
// Give an empty UniqueVolumeName so that this operation could be executed concurrently.
return oe.pendingOperations.Run("" /* volumeName */, "" /* podName */, volumesAreAttachedFunc)
return oe.pendingOperations.Run("" /* volumeName */, "" /* podName */, volumesAreAttachedFunc, opCompleteFunc)
}
func (oe *operationExecutor) MountVolume(
......@@ -657,7 +665,7 @@ func (oe *operationExecutor) MountVolume(
volumeToMount VolumeToMount,
actualStateOfWorld ActualStateOfWorldMounterUpdater,
isRemount bool) error {
mountFunc, err := oe.operationGenerator.GenerateMountVolumeFunc(
mountFunc, plugin, err := oe.operationGenerator.GenerateMountVolumeFunc(
waitForAttachTimeout, volumeToMount, actualStateOfWorld, isRemount)
if err != nil {
return err
......@@ -671,15 +679,17 @@ func (oe *operationExecutor) MountVolume(
podName = volumehelper.GetUniquePodName(volumeToMount.Pod)
}
// TODO mount_device
opCompleteFunc := util.OperationCompleteHook(plugin, "volume_mount")
return oe.pendingOperations.Run(
volumeToMount.VolumeName, podName, mountFunc)
volumeToMount.VolumeName, podName, mountFunc, opCompleteFunc)
}
func (oe *operationExecutor) UnmountVolume(
volumeToUnmount MountedVolume,
actualStateOfWorld ActualStateOfWorldMounterUpdater) error {
unmountFunc, err :=
unmountFunc, plugin, err :=
oe.operationGenerator.GenerateUnmountVolumeFunc(volumeToUnmount, actualStateOfWorld)
if err != nil {
return err
......@@ -689,36 +699,39 @@ func (oe *operationExecutor) UnmountVolume(
// same volume in parallel
podName := volumetypes.UniquePodName(volumeToUnmount.PodUID)
opCompleteFunc := util.OperationCompleteHook(plugin, "volume_unmount")
return oe.pendingOperations.Run(
volumeToUnmount.VolumeName, podName, unmountFunc)
volumeToUnmount.VolumeName, podName, unmountFunc, opCompleteFunc)
}
func (oe *operationExecutor) UnmountDevice(
deviceToDetach AttachedVolume,
actualStateOfWorld ActualStateOfWorldMounterUpdater,
mounter mount.Interface) error {
unmountDeviceFunc, err :=
unmountDeviceFunc, plugin, err :=
oe.operationGenerator.GenerateUnmountDeviceFunc(deviceToDetach, actualStateOfWorld, mounter)
if err != nil {
return err
}
opCompleteFunc := util.OperationCompleteHook(plugin, "unmount_device")
return oe.pendingOperations.Run(
deviceToDetach.VolumeName, "" /* podName */, unmountDeviceFunc)
deviceToDetach.VolumeName, "" /* podName */, unmountDeviceFunc, opCompleteFunc)
}
func (oe *operationExecutor) VerifyControllerAttachedVolume(
volumeToMount VolumeToMount,
nodeName types.NodeName,
actualStateOfWorld ActualStateOfWorldAttacherUpdater) error {
verifyControllerAttachedVolumeFunc, err :=
verifyControllerAttachedVolumeFunc, plugin, err :=
oe.operationGenerator.GenerateVerifyControllerAttachedVolumeFunc(volumeToMount, nodeName, actualStateOfWorld)
if err != nil {
return err
}
opCompleteFunc := util.OperationCompleteHook(plugin, "verify_controller_attached_volume")
return oe.pendingOperations.Run(
volumeToMount.VolumeName, "" /* podName */, verifyControllerAttachedVolumeFunc)
volumeToMount.VolumeName, "" /* podName */, verifyControllerAttachedVolumeFunc, opCompleteFunc)
}
// TODO: this is a workaround for the unmount device issue caused by gci mounter.
......
pkg/volume/util/operationexecutor/operation_executor_test.go
View file @ 3ed34183
......@@ -239,29 +239,29 @@ func newFakeOperationGenerator(ch chan interface{}, quit chan interface{}) Opera
}
}
func (fopg *fakeOperationGenerator) GenerateMountVolumeFunc(waitForAttachTimeout time.Duration, volumeToMount VolumeToMount, actualStateOfWorldMounterUpdater ActualStateOfWorldMounterUpdater, isRemount bool) (func() error, error) {
func (fopg *fakeOperationGenerator) GenerateMountVolumeFunc(waitForAttachTimeout time.Duration, volumeToMount VolumeToMount, actualStateOfWorldMounterUpdater ActualStateOfWorldMounterUpdater, isRemount bool) (func() error, string, error) {
return func() error {
startOperationAndBlock(fopg.ch, fopg.quit)
return nil
}, nil
}, "", nil
}
func (fopg *fakeOperationGenerator) GenerateUnmountVolumeFunc(volumeToUnmount MountedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater) (func() error, error) {
func (fopg *fakeOperationGenerator) GenerateUnmountVolumeFunc(volumeToUnmount MountedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater) (func() error, string, error) {
return func() error {
startOperationAndBlock(fopg.ch, fopg.quit)
return nil
}, nil
}, "", nil
}
func (fopg *fakeOperationGenerator) GenerateAttachVolumeFunc(volumeToAttach VolumeToAttach, actualStateOfWorld ActualStateOfWorldAttacherUpdater) (func() error, error) {
func (fopg *fakeOperationGenerator) GenerateAttachVolumeFunc(volumeToAttach VolumeToAttach, actualStateOfWorld ActualStateOfWorldAttacherUpdater) (func() error, string, error) {
return func() error {
startOperationAndBlock(fopg.ch, fopg.quit)
return nil
}, nil
}, "", nil
}
func (fopg *fakeOperationGenerator) GenerateDetachVolumeFunc(volumeToDetach AttachedVolume, verifySafeToDetach bool, actualStateOfWorld ActualStateOfWorldAttacherUpdater) (func() error, error) {
func (fopg *fakeOperationGenerator) GenerateDetachVolumeFunc(volumeToDetach AttachedVolume, verifySafeToDetach bool, actualStateOfWorld ActualStateOfWorldAttacherUpdater) (func() error, string, error) {
return func() error {
startOperationAndBlock(fopg.ch, fopg.quit)
return nil
}, nil
}, "", nil
}
func (fopg *fakeOperationGenerator) GenerateVolumesAreAttachedFunc(attachedVolumes []AttachedVolume, nodeName types.NodeName, actualStateOfWorld ActualStateOfWorldAttacherUpdater) (func() error, error) {
return func() error {
......@@ -269,17 +269,17 @@ func (fopg *fakeOperationGenerator) GenerateVolumesAreAttachedFunc(attachedVolum
return nil
}, nil
}
func (fopg *fakeOperationGenerator) GenerateUnmountDeviceFunc(deviceToDetach AttachedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater, mounter mount.Interface) (func() error, error) {
func (fopg *fakeOperationGenerator) GenerateUnmountDeviceFunc(deviceToDetach AttachedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater, mounter mount.Interface) (func() error, string, error) {
return func() error {
startOperationAndBlock(fopg.ch, fopg.quit)
return nil
}, nil
}, "", nil
}
func (fopg *fakeOperationGenerator) GenerateVerifyControllerAttachedVolumeFunc(volumeToMount VolumeToMount, nodeName types.NodeName, actualStateOfWorld ActualStateOfWorldAttacherUpdater) (func() error, error) {
func (fopg *fakeOperationGenerator) GenerateVerifyControllerAttachedVolumeFunc(volumeToMount VolumeToMount, nodeName types.NodeName, actualStateOfWorld ActualStateOfWorldAttacherUpdater) (func() error, string, error) {
return func() error {
startOperationAndBlock(fopg.ch, fopg.quit)
return nil
}, nil
}, "", nil
}
func (fopg *fakeOperationGenerator) GenerateBulkVolumeVerifyFunc(
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment