Add block volume support to Cinder volume plugin

pkg/volume/cinder/BUILD

@@ -11,6 +11,7 @@ go_library(
     srcs = [
         "attacher.go",
         "cinder.go",
+        "cinder_block.go",
         "cinder_util.go",
         "doc.go",
     ],
@@ -18,12 +19,14 @@ go_library(
     deps = [
         "//pkg/cloudprovider:go_default_library",
         "//pkg/cloudprovider/providers/openstack:go_default_library",
+        "//pkg/features:go_default_library",
         "//pkg/kubelet/apis:go_default_library",
         "//pkg/util/keymutex:go_default_library",
         "//pkg/util/mount:go_default_library",
         "//pkg/util/strings:go_default_library",
         "//pkg/volume:go_default_library",
         "//pkg/volume/util:go_default_library",
+        "//pkg/volume/util/volumepathhandler:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
@@ -31,6 +34,7 @@ go_library(
         "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes:go_default_library",
         "//vendor/k8s.io/utils/exec:go_default_library",
     ],
@@ -40,6 +44,7 @@ go_test(
     name = "go_default_test",
     srcs = [
         "attacher_test.go",
+        "cinder_block_test.go",
         "cinder_test.go",
     ],
     embed = [":go_default_library"],
@@ -51,6 +56,7 @@ go_test(
         "//vendor/github.com/golang/glog:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
         "//vendor/k8s.io/client-go/util/testing:go_default_library",
     ],

pkg/volume/cinder/cinder.go

@@ -27,8 +27,10 @@ import (
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/kubernetes/pkg/cloudprovider"
 	"k8s.io/kubernetes/pkg/cloudprovider/providers/openstack"
+	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/util/keymutex"
 	"k8s.io/kubernetes/pkg/util/mount"
 	kstrings "k8s.io/kubernetes/pkg/util/strings"
@@ -505,10 +507,6 @@ func (c *cinderVolumeProvisioner) Provision(selectedNode *v1.Node, allowedTopolo
 		return nil, fmt.Errorf("invalid AccessModes %v: only AccessModes %v are supported", c.options.PVC.Spec.AccessModes, c.plugin.GetAccessModes())
 	}
 
-	if util.CheckPersistentVolumeClaimModeBlock(c.options.PVC) {
-		return nil, fmt.Errorf("%s does not support block volume provisioning", c.plugin.GetPluginName())
-	}
-
 	volumeID, sizeGB, labels, fstype, err := c.manager.CreateVolume(c)
 	if err != nil {
 		return nil, err
@@ -542,6 +540,10 @@ func (c *cinderVolumeProvisioner) Provision(selectedNode *v1.Node, allowedTopolo
 		pv.Spec.AccessModes = c.plugin.GetAccessModes()
 	}
 
+	if utilfeature.DefaultFeatureGate.Enabled(features.BlockVolume) {
+		pv.Spec.VolumeMode = c.options.PVC.Spec.VolumeMode
+	}
+
 	return pv, nil
 }
 

pkg/volume/cinder/cinder_block.go

+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cinder
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/golang/glog"
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/kubernetes/pkg/util/mount"
+	kstrings "k8s.io/kubernetes/pkg/util/strings"
+	"k8s.io/kubernetes/pkg/volume"
+	"k8s.io/kubernetes/pkg/volume/util"
+	"k8s.io/kubernetes/pkg/volume/util/volumepathhandler"
+)
+
+var _ volume.VolumePlugin = &cinderPlugin{}
+var _ volume.PersistentVolumePlugin = &cinderPlugin{}
+var _ volume.BlockVolumePlugin = &cinderPlugin{}
+var _ volume.DeletableVolumePlugin = &cinderPlugin{}
+var _ volume.ProvisionableVolumePlugin = &cinderPlugin{}
+var _ volume.ExpandableVolumePlugin = &cinderPlugin{}
+
+func (plugin *cinderPlugin) ConstructBlockVolumeSpec(podUID types.UID, volumeName, mapPath string) (*volume.Spec, error) {
+	pluginDir := plugin.host.GetVolumeDevicePluginDir(cinderVolumePluginName)
+	blkutil := volumepathhandler.NewBlockVolumePathHandler()
+	globalMapPathUUID, err := blkutil.FindGlobalMapPathUUIDFromPod(pluginDir, mapPath, podUID)
+	if err != nil {
+		return nil, err
+	}
+	glog.V(5).Infof("globalMapPathUUID: %v, err: %v", globalMapPathUUID, err)
+
+	globalMapPath := filepath.Dir(globalMapPathUUID)
+	if len(globalMapPath) <= 1 {
+		return nil, fmt.Errorf("failed to get volume plugin information from globalMapPathUUID: %v", globalMapPathUUID)
+	}
+
+	return getVolumeSpecFromGlobalMapPath(globalMapPath)
+}
+
+func getVolumeSpecFromGlobalMapPath(globalMapPath string) (*volume.Spec, error) {
+	// Get volume spec information from globalMapPath
+	// globalMapPath example:
+	//   plugins/kubernetes.io/{PluginName}/{DefaultKubeletVolumeDevicesDirName}/{volumeID}
+	//   plugins/kubernetes.io/cinder/volumeDevices/vol-XXXXXX
+	vID := filepath.Base(globalMapPath)
+	if len(vID) <= 1 {
+		return nil, fmt.Errorf("failed to get volumeID from global path=%s", globalMapPath)
+	}
+	block := v1.PersistentVolumeBlock
+	cinderVolume := &v1.PersistentVolume{
+		Spec: v1.PersistentVolumeSpec{
+			PersistentVolumeSource: v1.PersistentVolumeSource{
+				Cinder: &v1.CinderPersistentVolumeSource{
+					VolumeID: vID,
+				},
+			},
+			VolumeMode: &block,
+		},
+	}
+	return volume.NewSpecFromPersistentVolume(cinderVolume, true), nil
+}
+
+// NewBlockVolumeMapper creates a new volume.BlockVolumeMapper from an API specification.
+func (plugin *cinderPlugin) NewBlockVolumeMapper(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.BlockVolumeMapper, error) {
+	// If this is called via GenerateUnmapDeviceFunc(), pod is nil.
+	// Pass empty string as dummy uid since uid isn't used in the case.
+	var uid types.UID
+	if pod != nil {
+		uid = pod.UID
+	}
+
+	return plugin.newBlockVolumeMapperInternal(spec, uid, &DiskUtil{}, plugin.host.GetMounter(plugin.GetPluginName()))
+}
+
+func (plugin *cinderPlugin) newBlockVolumeMapperInternal(spec *volume.Spec, podUID types.UID, manager cdManager, mounter mount.Interface) (volume.BlockVolumeMapper, error) {
+	pdName, fsType, readOnly, err := getVolumeInfo(spec)
+	if err != nil {
+		return nil, err
+	}
+
+	return &cinderVolumeMapper{
+		cinderVolume: &cinderVolume{
+			podUID:  podUID,
+			volName: spec.Name(),
+			pdName:  pdName,
+			fsType:  fsType,
+			manager: manager,
+			mounter: mounter,
+			plugin:  plugin,
+		},
+		readOnly: readOnly}, nil
+}
+
+func (plugin *cinderPlugin) NewBlockVolumeUnmapper(volName string, podUID types.UID) (volume.BlockVolumeUnmapper, error) {
+	return plugin.newUnmapperInternal(volName, podUID, &DiskUtil{}, plugin.host.GetMounter(plugin.GetPluginName()))
+}
+
+func (plugin *cinderPlugin) newUnmapperInternal(volName string, podUID types.UID, manager cdManager, mounter mount.Interface) (volume.BlockVolumeUnmapper, error) {
+	return &cinderPluginUnmapper{
+		cinderVolume: &cinderVolume{
+			podUID:  podUID,
+			volName: volName,
+			manager: manager,
+			mounter: mounter,
+			plugin:  plugin,
+		}}, nil
+}
+
+func (c *cinderPluginUnmapper) TearDownDevice(mapPath, devicePath string) error {
+	return nil
+}
+
+type cinderPluginUnmapper struct {
+	*cinderVolume
+}
+
+var _ volume.BlockVolumeUnmapper = &cinderPluginUnmapper{}
+
+type cinderVolumeMapper struct {
+	*cinderVolume
+	readOnly bool
+}
+
+var _ volume.BlockVolumeMapper = &cinderVolumeMapper{}
+
+func (b *cinderVolumeMapper) SetUpDevice() (string, error) {
+	return "", nil
+}
+
+func (b *cinderVolumeMapper) MapDevice(devicePath, globalMapPath, volumeMapPath, volumeMapName string, podUID types.UID) error {
+	return util.MapBlockVolume(devicePath, globalMapPath, volumeMapPath, volumeMapName, podUID)
+}
+
+// GetGlobalMapPath returns global map path and error
+// path: plugins/kubernetes.io/{PluginName}/volumeDevices/volumeID
+//       plugins/kubernetes.io/cinder/volumeDevices/vol-XXXXXX
+func (cd *cinderVolume) GetGlobalMapPath(spec *volume.Spec) (string, error) {
+	pdName, _, _, err := getVolumeInfo(spec)
+	if err != nil {
+		return "", err
+	}
+	return filepath.Join(cd.plugin.host.GetVolumeDevicePluginDir(cinderVolumePluginName), pdName), nil
+}
+
+// GetPodDeviceMapPath returns pod device map path and volume name
+// path: pods/{podUid}/volumeDevices/kubernetes.io~cinder
+func (cd *cinderVolume) GetPodDeviceMapPath() (string, string) {
+	name := cinderVolumePluginName
+	return cd.plugin.host.GetPodVolumeDeviceDir(cd.podUID, kstrings.EscapeQualifiedNameForDisk(name)), cd.volName
+}

pkg/volume/cinder/cinder_block_test.go

+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cinder
+
+import (
+	"os"
+	"path"
+	"testing"
+
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	utiltesting "k8s.io/client-go/util/testing"
+	"k8s.io/kubernetes/pkg/volume"
+	volumetest "k8s.io/kubernetes/pkg/volume/testing"
+)
+
+const (
+	testVolName    = "vol-1234"
+	testPVName     = "pv1"
+	testGlobalPath = "plugins/kubernetes.io/cinder/volumeDevices/vol-1234"
+	testPodPath    = "pods/poduid/volumeDevices/kubernetes.io~cinder"
+)
+
+func TestGetVolumeSpecFromGlobalMapPath(t *testing.T) {
+	// make our test path for fake GlobalMapPath
+	// /tmp symbolized our pluginDir
+	// /tmp/testGlobalPathXXXXX/plugins/kubernetes.io/cinder/volumeDevices/pdVol1
+	tmpVDir, err := utiltesting.MkTmpdir("cinderBlockTest")
+	if err != nil {
+		t.Fatalf("can't make a temp dir: %v", err)
+	}
+	//deferred clean up
+	defer os.RemoveAll(tmpVDir)
+
+	expectedGlobalPath := path.Join(tmpVDir, testGlobalPath)
+
+	//Bad Path
+	badspec, err := getVolumeSpecFromGlobalMapPath("")
+	if badspec != nil || err == nil {
+		t.Errorf("Expected not to get spec from GlobalMapPath but did")
+	}
+
+	// Good Path
+	spec, err := getVolumeSpecFromGlobalMapPath(expectedGlobalPath)
+	if spec == nil || err != nil {
+		t.Fatalf("Failed to get spec from GlobalMapPath: %v", err)
+	}
+	if spec.PersistentVolume.Spec.Cinder.VolumeID != testVolName {
+		t.Errorf("Invalid volumeID from GlobalMapPath spec: %s", spec.PersistentVolume.Spec.Cinder.VolumeID)
+	}
+	block := v1.PersistentVolumeBlock
+	specMode := spec.PersistentVolume.Spec.VolumeMode
+	if &specMode == nil {
+		t.Errorf("Invalid volumeMode from GlobalMapPath spec: %v expected: %v", &specMode, block)
+	}
+	if *specMode != block {
+		t.Errorf("Invalid volumeMode from GlobalMapPath spec: %v expected: %v", *specMode, block)
+	}
+}
+
+func getTestVolume(readOnly bool, isBlock bool) *volume.Spec {
+	pv := &v1.PersistentVolume{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: testPVName,
+		},
+		Spec: v1.PersistentVolumeSpec{
+			PersistentVolumeSource: v1.PersistentVolumeSource{
+				Cinder: &v1.CinderPersistentVolumeSource{
+					VolumeID: testVolName,
+				},
+			},
+		},
+	}
+
+	if isBlock {
+		blockMode := v1.PersistentVolumeBlock
+		pv.Spec.VolumeMode = &blockMode
+	}
+	return volume.NewSpecFromPersistentVolume(pv, readOnly)
+}
+
+func TestGetPodAndPluginMapPaths(t *testing.T) {
+	tmpVDir, err := utiltesting.MkTmpdir("cinderBlockTest")
+	if err != nil {
+		t.Fatalf("can't make a temp dir: %v", err)
+	}
+	//deferred clean up
+	defer os.RemoveAll(tmpVDir)
+
+	expectedGlobalPath := path.Join(tmpVDir, testGlobalPath)
+	expectedPodPath := path.Join(tmpVDir, testPodPath)
+
+	spec := getTestVolume(false, true /*isBlock*/)
+	plugMgr := volume.VolumePluginMgr{}
+	plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpVDir, nil, nil))
+	plug, err := plugMgr.FindMapperPluginByName(cinderVolumePluginName)
+	if err != nil {
+		os.RemoveAll(tmpVDir)
+		t.Fatalf("Can't find the plugin by name: %q", cinderVolumePluginName)
+	}
+	if plug.GetPluginName() != cinderVolumePluginName {
+		t.Fatalf("Wrong name: %s", plug.GetPluginName())
+	}
+	pod := &v1.Pod{ObjectMeta: metav1.ObjectMeta{UID: types.UID("poduid")}}
+	mapper, err := plug.NewBlockVolumeMapper(spec, pod, volume.VolumeOptions{})
+	if err != nil {
+		t.Fatalf("Failed to make a new Mounter: %v", err)
+	}
+	if mapper == nil {
+		t.Fatalf("Got a nil Mounter")
+	}
+
+	//GetGlobalMapPath
+	gMapPath, err := mapper.GetGlobalMapPath(spec)
+	if err != nil || len(gMapPath) == 0 {
+		t.Fatalf("Invalid GlobalMapPath from spec: %s", spec.PersistentVolume.Spec.Cinder.VolumeID)
+	}
+	if gMapPath != expectedGlobalPath {
+		t.Errorf("Failed to get GlobalMapPath: %s %s", gMapPath, expectedGlobalPath)
+	}
+
+	//GetPodDeviceMapPath
+	gDevicePath, gVolName := mapper.GetPodDeviceMapPath()
+	if gDevicePath != expectedPodPath {
+		t.Errorf("Got unexpected pod path: %s, expected %s", gDevicePath, expectedPodPath)
+	}
+	if gVolName != testPVName {
+		t.Errorf("Got unexpected volNamne: %s, expected %s", gVolName, testPVName)
+	}
+}