Inital Quobyte dynamic provision

cmd/kube-controller-manager/app/plugins.go

@@ -45,6 +45,7 @@ import (
 	"k8s.io/kubernetes/pkg/volume/glusterfs"
 	"k8s.io/kubernetes/pkg/volume/host_path"
 	"k8s.io/kubernetes/pkg/volume/nfs"
+	"k8s.io/kubernetes/pkg/volume/quobyte"
 	"k8s.io/kubernetes/pkg/volume/rbd"
 	"k8s.io/kubernetes/pkg/volume/vsphere_volume"
 )
@@ -105,6 +106,8 @@ func ProbeControllerVolumePlugins(cloud cloudprovider.Interface, config componen
 	allPlugins = append(allPlugins, glusterfs.ProbeVolumePlugins()...)
 	// add rbd provisioner
 	allPlugins = append(allPlugins, rbd.ProbeVolumePlugins()...)
+	allPlugins = append(allPlugins, quobyte.ProbeVolumePlugins()...)
+
 	if cloud != nil {
 		switch {
 		case aws.ProviderName == cloud.ProviderName():

examples/experimental/persistent-volume-provisioning/README.md

@@ -147,6 +147,94 @@ parameters:
 * `userId`: Ceph client ID that is used to map the RBD image. Default is the same as `adminId`.
 * `userSecretName`: The name of Ceph Secret for `userId` to map RBD image. It must exist in the same namespace as PVCs. It is required.
 
+#### Quobyte
+
+<!-- BEGIN MUNGE: EXAMPLE quobyte/quobyte-storage-class.yaml -->
+
+```yaml
+apiVersion: storage.k8s.io/v1beta1
+kind: StorageClass
+metadata:
+   name: slow
+provisioner: kubernetes.io/quobyte
+parameters:
+    quobyteApiServer: "http://138.68.74.142:7860"
+    registry: "138.68.74.142:7861"
+    adminSecretName: "quobyte-admin-secret"
+    adminSecretNamespace: "kube-system"
+    user: "root"
+    group: "root"
+    quobyteConfig: "BASE"
+    quobyteTenant: "DEFAULT"
+```
+
+[Download example](quobyte/quobyte-storage-class.yaml?raw=true)
+<!-- END MUNGE: EXAMPLE quobyte/quobyte-storage-class.yaml -->
+
+* **quobyteApiServer** API Server of Quobyte in the format http(s)://api-server:7860
+* **registry** Quobyte registry to use to mount the volume. You can specifiy the registry as <host>:<port> pair or if you want to specify multiple registries you just have to put a comma between them e.q. <host1>:<port>,<host2>:<port>,<host3>:<port>. The host can be an IP address or if you have a working DNS you can also provide the DNS names.
+* **adminSecretName** secret that holds information about the Quobyte user and the password to authenticate agains the API server.
+* **adminSecretNamespace** The namespace for **adminSecretName**. Default is `default`.
+* **user** maps all access to this user. Default is `root`.
+* **group** maps all access to this group. Default is `nfsnobody`.
+* **quobyteConfig** use the specified configuration to create the volume. You can create a new configuration or modify an existing one with the Web console or the quobyte CLI. Default is `BASE`
+* **quobyteTenant** use the specified tenant to create/delete the volume. This Quobyte tenant has to be already present in Quobyte. Default is `DEFAULT`
+
+First create Quobyte admin's Secret in the system namespace. Here the Secret is created in `kube-system`:
+
+```
+$ kubectl create -f examples/experimental/persistent-volume-provisioning/quobyte/quobyte-admin-secret.yaml --namespace=kube-system
+```
+
+Then create the Quobyte storage class:
+
+```
+$ kubectl create -f examples/experimental/persistent-volume-provisioning/quobyte/quobyte-storage-class.yaml
+```
+
+Now create a PVC
+
+```
+$ kubectl create -f examples/experimental/persistent-volume-provisioning/claim1.json
+```
+
+Check the created PVC:
+
+```
+$ kubectl describe pvc
+Name:       claim1
+Namespace:      default
+Status:     Bound
+Volume:     pvc-bdb82652-694a-11e6-b811-080027242396
+Labels:     <none>
+Capacity:       3Gi
+Access Modes:   RWO
+No events.
+
+$ kubectl describe pv
+Name:  		pvc-bdb82652-694a-11e6-b811-080027242396
+Labels:		<none>
+Status:		Bound
+Claim: 		default/claim1
+Reclaim Policy:	Delete
+Access Modes:  	RWO
+Capacity:      	3Gi
+Message:
+Source:
+    Type:      	Quobyte (a Quobyte mount on the host that shares a pod's lifetime)
+    Registry:  	138.68.79.14:7861
+    Volume:    	kubernetes-dynamic-pvc-bdb97c58-694a-11e6-91b6-080027242396
+    ReadOnly:  	false
+No events.
+```
+
+Create a Pod to use the PVC:
+
+```
+$ kubectl create -f examples/experimental/persistent-volume-provisioning/quobyte/example-pod.yaml
+```
+
+
 ### User provisioning requests
 
 Users request dynamically provisioned storage by including a storage class in their `PersistentVolumeClaim`.

examples/experimental/persistent-volume-provisioning/quobyte/example-pod.yaml

+apiVersion: v1
+kind: ReplicationController
+metadata:
+  name: server
+spec:
+  replicas: 1
+  selector:
+    role: server
+  template:
+    metadata:
+      labels:
+        role: server
+    spec:
+      containers:
+      - name: server
+        image: nginx 
+        volumeMounts:
+          - mountPath: /var/lib/www/html
+            name: quobytepvc
+      volumes:
+        - name: quobytepvc
+          persistentVolumeClaim:
+            claimName: claim1

examples/experimental/persistent-volume-provisioning/quobyte/quobyte-admin-secret.yaml

+apiVersion: v1
+kind: Secret
+metadata:
+  name: quobyte-admin-secret
+data:
+  password: cXVvYnl0ZQ==
+  user: YWRtaW4=

examples/experimental/persistent-volume-provisioning/quobyte/quobyte-storage-class.yaml

+apiVersion: storage.k8s.io/v1beta1
+kind: StorageClass
+metadata:
+   name: slow
+provisioner: kubernetes.io/quobyte
+parameters:
+    quobyteApiServer: "http://138.68.74.142:7860"
+    registry: "138.68.74.142:7861"
+    adminSecretName: "quobyte-admin-secret"
+    adminSecretNamespace: "kube-system"
+    user: "root"
+    group: "root"
+    quobyteConfig: "BASE"
+    quobyteTenant: "DEFAULT"

examples/volumes/quobyte/Readme.md

@@ -94,11 +94,11 @@ spec:
 <!-- END MUNGE: EXAMPLE ./quobyte-pod.yaml -->
 
 Parameters:
-* **registry** Quobyte registry to use to mount the volume. You can specifiy the registry as <host>:<port> pair or if you want to specify multiple registries you just have to put a semicolon between them e.q. <host1>:<port>,<host2>:<port>,<host3>:<port>. The host can be an IP address or if you have a working DNS you can also provide the DNS names.
+* **registry** Quobyte registry to use to mount the volume. You can specifiy the registry as <host>:<port> pair or if you want to specify multiple registries you just have to put a comma between them e.q. <host1>:<port>,<host2>:<port>,<host3>:<port>. The host can be an IP address or if you have a working DNS you can also provide the DNS names.
 * **volume** volume represents a Quobyte volume which must be created before usage.
 * **readOnly** is the boolean that sets the mountpoint readOnly or readWrite.
-* **user** maps all access to this user. Default is root.
-* **group** maps all access to this group. Default is empty.
+* **user** maps all access to this user. Default is `root`.
+* **group** maps all access to this group. Default is `nfsnobody`.
 
 Creating the pod:
 

pkg/volume/quobyte/quobyte.go

@@ -20,14 +20,18 @@ import (
 	"fmt"
 	"os"
 	"path"
+	goStrings "strings"
 
 	"github.com/golang/glog"
+	"github.com/pborman/uuid"
 	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/api/resource"
 	"k8s.io/kubernetes/pkg/types"
 	"k8s.io/kubernetes/pkg/util/exec"
 	"k8s.io/kubernetes/pkg/util/mount"
 	"k8s.io/kubernetes/pkg/util/strings"
 	"k8s.io/kubernetes/pkg/volume"
+	"k8s.io/kubernetes/pkg/volume/util"
 )
 
 // ProbeVolumePlugins is the primary entrypoint for volume plugins.
@@ -39,11 +43,27 @@ type quobytePlugin struct {
 	host volume.VolumeHost
 }
 
+// This user is used to authenticate against the
+// Quobyte API server and holds all information
+type quobyteAPIConfig struct {
+	quobyteUser      string
+	quobytePassword  string
+	quobyteApiServer string
+}
+
 var _ volume.VolumePlugin = &quobytePlugin{}
 var _ volume.PersistentVolumePlugin = &quobytePlugin{}
+var _ volume.DeletableVolumePlugin = &quobytePlugin{}
+var _ volume.ProvisionableVolumePlugin = &quobytePlugin{}
+var _ volume.Provisioner = &quobyteVolumeProvisioner{}
+var _ volume.Deleter = &quobyteVolumeDeleter{}
 
 const (
 	quobytePluginName = "kubernetes.io/quobyte"
+
+	annotationQuobyteAPIServer          = "quobyte.kubernetes.io/api"
+	annotationQuobyteAPISecret          = "quobyte.kubernetes.io/apiuser"
+	annotationQuobyteAPISecretNamespace = "quobyte.kubernetes.io/apipassword"
 )
 
 func (plugin *quobytePlugin) Init(host volume.VolumeHost) error {
@@ -149,7 +169,8 @@ func (plugin *quobytePlugin) newMounterInternal(spec *volume.Spec, pod *api.Pod,
 			plugin:  plugin,
 		},
 		registry: source.Registry,
-		readOnly: readOnly}, nil
+		readOnly: readOnly,
+	}, nil
 }
 
 func (plugin *quobytePlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) {
@@ -157,21 +178,26 @@ func (plugin *quobytePlugin) NewUnmounter(volName string, podUID types.UID) (vol
 }
 
 func (plugin *quobytePlugin) newUnmounterInternal(volName string, podUID types.UID, mounter mount.Interface) (volume.Unmounter, error) {
-	return &quobyteUnmounter{&quobyte{
-		volName: volName,
-		mounter: mounter,
-		pod:     &api.Pod{ObjectMeta: api.ObjectMeta{UID: podUID}},
-		plugin:  plugin,
-	}}, nil
+	return &quobyteUnmounter{
+		&quobyte{
+			volName: volName,
+			mounter: mounter,
+			pod:     &api.Pod{ObjectMeta: api.ObjectMeta{UID: podUID}},
+			plugin:  plugin,
+		},
+	}, nil
 }
 
 // Quobyte volumes represent a bare host directory mount of an quobyte export.
+//TODO add configuration + tenant
 type quobyte struct {
 	volName string
 	pod     *api.Pod
 	user    string
 	group   string
 	volume  string
+	tenant  string
+	config  string
 	mounter mount.Interface
 	plugin  *quobytePlugin
 	volume.MetricsNil
@@ -226,22 +252,22 @@ func (mounter *quobyteMounter) SetUpAt(dir string, fsGroup *int64) error {
 }
 
 // GetPath returns the path to the user specific mount of a Quobyte volume
-// Returns a path in the format ../user@volume e.g. ../root@MyVolume
-// or if a group is set ../user#group@volume
+// Returns a path in the format ../user#group@volume
 func (quobyteVolume *quobyte) GetPath() string {
 	user := quobyteVolume.user
 	if len(user) == 0 {
 		user = "root"
 	}
 
+	group := quobyteVolume.group
+	if len(group) == 0 {
+		group = "nfsnobody"
+	}
+
 	// Quobyte has only one mount in the PluginDir where all Volumes are mounted
 	// The Quobyte client does a fixed-user mapping
 	pluginDir := quobyteVolume.plugin.host.GetPluginDir(strings.EscapeQualifiedNameForDisk(quobytePluginName))
-	if len(quobyteVolume.group) > 0 {
-		return path.Join(pluginDir, fmt.Sprintf("%s#%s@%s", user, quobyteVolume.group, quobyteVolume.volume))
-	}
-
-	return path.Join(pluginDir, fmt.Sprintf("%s@%s", user, quobyteVolume.volume))
+	return path.Join(pluginDir, fmt.Sprintf("%s#%s@%s", user, group, quobyteVolume.volume))
 }
 
 type quobyteUnmounter struct {
@@ -258,3 +284,192 @@ func (unmounter *quobyteUnmounter) TearDown() error {
 func (unmounter *quobyteUnmounter) TearDownAt(dir string) error {
 	return nil
 }
+
+type quobyteVolumeDeleter struct {
+	*quobyteMounter
+	pv *api.PersistentVolume
+}
+
+func (plugin *quobytePlugin) NewDeleter(spec *volume.Spec) (volume.Deleter, error) {
+	if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.Quobyte == nil {
+		return nil, fmt.Errorf("spec.PersistentVolumeSource.Spec.Quobyte is nil")
+	}
+
+	return plugin.newDeleterInternal(spec)
+}
+
+func (plugin *quobytePlugin) newDeleterInternal(spec *volume.Spec) (volume.Deleter, error) {
+	source, readOnly, err := getVolumeSource(spec)
+	if err != nil {
+		return nil, err
+	}
+
+	return &quobyteVolumeDeleter{
+		quobyteMounter: &quobyteMounter{
+			quobyte: &quobyte{
+				volName: spec.Name(),
+				user:    source.User,
+				group:   source.Group,
+				volume:  source.Volume,
+				plugin:  plugin,
+			},
+			registry: source.Registry,
+			readOnly: readOnly,
+		},
+		pv: spec.PersistentVolume,
+	}, nil
+}
+
+func (plugin *quobytePlugin) NewProvisioner(options volume.VolumeOptions) (volume.Provisioner, error) {
+	if len(options.AccessModes) == 0 {
+		options.AccessModes = plugin.GetAccessModes()
+	}
+
+	return plugin.newProvisionerInternal(options)
+}
+
+func (plugin *quobytePlugin) newProvisionerInternal(options volume.VolumeOptions) (volume.Provisioner, error) {
+	return &quobyteVolumeProvisioner{
+		quobyteMounter: &quobyteMounter{
+			quobyte: &quobyte{
+				plugin: plugin,
+			},
+		},
+		options: options,
+	}, nil
+}
+
+type quobyteVolumeProvisioner struct {
+	*quobyteMounter
+	options volume.VolumeOptions
+}
+
+func (provisioner *quobyteVolumeProvisioner) Provision() (*api.PersistentVolume, error) {
+	if provisioner.options.Selector != nil {
+		return nil, fmt.Errorf("claim Selector is not supported")
+	}
+	var apiServer, adminSecretName, quobyteUser, quobytePassword string
+	adminSecretNamespace := "default"
+	provisioner.config = "BASE"
+	provisioner.tenant = "DEFAULT"
+
+	for k, v := range provisioner.options.Parameters {
+		switch goStrings.ToLower(k) {
+		case "registry":
+			provisioner.registry = v
+		case "adminsecretname":
+			adminSecretName = v
+		case "adminsecretnamespace":
+			adminSecretNamespace = v
+		case "quobyteapiserver":
+			apiServer = v
+		case "user":
+			provisioner.user = v
+		case "group":
+			provisioner.group = v
+		case "quobytetenant":
+			provisioner.tenant = v
+		case "quobyteconfig":
+			provisioner.config = v
+		default:
+			return nil, fmt.Errorf("invalid option %q for volume plugin %s", k, provisioner.plugin.GetPluginName())
+		}
+	}
+
+	secretMap, err := util.GetSecret(adminSecretNamespace, adminSecretName, provisioner.plugin.host.GetKubeClient())
+	if err != nil {
+		return nil, err
+	}
+
+	var ok bool
+	if quobyteUser, ok = secretMap["user"]; !ok {
+		return nil, fmt.Errorf("Missing \"user\" in secret")
+	}
+
+	if quobytePassword, ok = secretMap["password"]; !ok {
+		return nil, fmt.Errorf("Missing \"password\" in secret")
+	}
+
+	if !validateRegistry(provisioner.registry) {
+		return nil, fmt.Errorf("Quoybte registry missing or malformed: must be a host:port pair or multiple pairs seperated by commas")
+	}
+
+	if len(apiServer) == 0 {
+		return nil, fmt.Errorf("Quoybte API server missing or malformed: must be a http(s)://host:port pair or multiple pairs seperated by commas")
+	}
+
+	// create random image name
+	provisioner.volume = fmt.Sprintf("kubernetes-dynamic-pvc-%s", uuid.NewUUID())
+
+	cfg := &quobyteAPIConfig{
+		quobyteApiServer: apiServer,
+		quobyteUser:      quobyteUser,
+		quobytePassword:  quobytePassword,
+	}
+	manager := &quobyteVolumeManager{
+		config: cfg,
+	}
+
+	vol, sizeGB, err := manager.createVolume(provisioner)
+	if err != nil {
+		return nil, err
+	}
+	pv := new(api.PersistentVolume)
+	pv.Spec.PersistentVolumeSource.Quobyte = vol
+	pv.Spec.PersistentVolumeReclaimPolicy = provisioner.options.PersistentVolumeReclaimPolicy
+	pv.Spec.AccessModes = provisioner.options.AccessModes
+	pv.Spec.Capacity = api.ResourceList{
+		api.ResourceName(api.ResourceStorage): resource.MustParse(fmt.Sprintf("%dGi", sizeGB)),
+	}
+
+	util.AddVolumeAnnotations(pv, map[string]string{
+		annotationQuobyteAPIServer:          apiServer,
+		annotationQuobyteAPISecret:          adminSecretName,
+		annotationQuobyteAPISecretNamespace: adminSecretNamespace,
+	})
+
+	return pv, nil
+}
+
+func (deleter *quobyteVolumeDeleter) GetPath() string {
+	return deleter.quobyte.GetPath()
+}
+
+func (deleter *quobyteVolumeDeleter) Delete() error {
+	var quobyteUser, quobytePassword string
+	annotations, err := util.ParseVolumeAnnotations(deleter.pv, []string{
+		annotationQuobyteAPISecret,
+		annotationQuobyteAPISecretNamespace,
+		annotationQuobyteAPIServer})
+
+	if err != nil {
+		return err
+	}
+
+	secretMap, err := util.GetSecret(
+		annotations[annotationQuobyteAPISecretNamespace],
+		annotations[annotationQuobyteAPISecret],
+		deleter.plugin.host.GetKubeClient())
+
+	if err != nil {
+		return err
+	}
+
+	var ok bool
+	if quobyteUser, ok = secretMap["user"]; !ok {
+		return fmt.Errorf("Missing \"user\" in secret")
+	}
+
+	if quobytePassword, ok = secretMap["password"]; !ok {
+		return fmt.Errorf("Missing \"password\" in secret")
+	}
+
+	manager := &quobyteVolumeManager{
+		config: &quobyteAPIConfig{
+			quobyteUser:      quobyteUser,
+			quobytePassword:  quobytePassword,
+			quobyteApiServer: annotations[annotationQuobyteAPIServer],
+		},
+	}
+	return manager.deleteVolume(deleter)
+}

pkg/volume/quobyte/quobyte_util.go

@@ -17,12 +17,58 @@ limitations under the License.
 package quobyte
 
 import (
+	"net"
 	"path"
 	"strings"
 
+	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/volume"
+
 	"github.com/golang/glog"
+	quobyte_api "github.com/quobyte/api"
 )
 
+type quobyteVolumeManager struct {
+	config *quobyteAPIConfig
+}
+
+func (manager *quobyteVolumeManager) createVolume(provisioner *quobyteVolumeProvisioner) (quobyte *api.QuobyteVolumeSource, size int, err error) {
+	volumeSize := int(volume.RoundUpSize(provisioner.options.Capacity.Value(), 1024*1024*1024))
+	// Quobyte has the concept of Volumes which doen't have a specific size (they can grow unlimited)
+	// to simulate a size constraint we could set here a Quota
+	volumeRequest := &quobyte_api.CreateVolumeRequest{
+		Name:              provisioner.volume,
+		RootUserID:        provisioner.user,
+		RootGroupID:       provisioner.group,
+		TenantID:          provisioner.tenant,
+		ConfigurationName: provisioner.config,
+	}
+
+	if _, err := manager.createQuobyteClient().CreateVolume(volumeRequest); err != nil {
+		return &api.QuobyteVolumeSource{}, volumeSize, err
+	}
+
+	glog.V(4).Infof("Created Quobyte volume %s", provisioner.volume)
+	return &api.QuobyteVolumeSource{
+		Registry: provisioner.registry,
+		Volume:   provisioner.volume,
+		User:     provisioner.user,
+		Group:    provisioner.group,
+	}, volumeSize, nil
+}
+
+func (manager *quobyteVolumeManager) deleteVolume(deleter *quobyteVolumeDeleter) error {
+	return manager.createQuobyteClient().DeleteVolumeByName(deleter.volume, deleter.tenant)
+}
+
+func (manager *quobyteVolumeManager) createQuobyteClient() *quobyte_api.QuobyteClient {
+	return quobyte_api.NewQuobyteClient(
+		manager.config.quobyteApiServer,
+		manager.config.quobyteUser,
+		manager.config.quobytePassword,
+	)
+}
+
 func (mounter *quobyteMounter) pluginDirIsMounted(pluginDir string) (bool, error) {
 	mounts, err := mounter.mounter.List()
 	if err != nil {
@@ -46,3 +92,17 @@ func (mounter *quobyteMounter) pluginDirIsMounted(pluginDir string) (bool, error
 func (mounter *quobyteMounter) correctTraillingSlash(regStr string) string {
 	return path.Clean(regStr) + "/"
 }
+
+func validateRegistry(registry string) bool {
+	if len(registry) == 0 {
+		return false
+	}
+
+	for _, hostPortPair := range strings.Split(registry, ",") {
+		if _, _, err := net.SplitHostPort(hostPortPair); err != nil {
+			return false
+		}
+	}
+
+	return true
+}

pkg/volume/util/util.go

@@ -22,6 +22,7 @@ import (
 	"path"
 
 	"github.com/golang/glog"
+	"k8s.io/kubernetes/pkg/api"
 	clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
 	"k8s.io/kubernetes/pkg/util/mount"
 )
@@ -126,3 +127,33 @@ func GetSecret(namespace, secretName string, kubeClient clientset.Interface) (ma
 	}
 	return secret, nil
 }
+
+// AddVolumeAnnotations adds a golang Map as annotation to a PersistentVolume
+func AddVolumeAnnotations(pv *api.PersistentVolume, annotations map[string]string) {
+	if pv.Annotations == nil {
+		pv.Annotations = map[string]string{}
+	}
+
+	for k, v := range annotations {
+		pv.Annotations[k] = v
+	}
+}
+
+// ParseVolumeAnnotations reads the defined annoations from a PersistentVolume
+func ParseVolumeAnnotations(pv *api.PersistentVolume, parseAnnotations []string) (map[string]string, error) {
+	result := map[string]string{}
+
+	if pv.Annotations == nil {
+		return result, fmt.Errorf("cannot parse volume annotations: no annotations found")
+	}
+
+	for _, annotation := range parseAnnotations {
+		if val, ok := pv.Annotations[annotation]; ok {
+			result[annotation] = val
+		} else {
+			return result, fmt.Errorf("cannot parse volume annotations: annotation %s not found", annotation)
+		}
+	}
+
+	return result, nil
+}