e2e: deploy from manifest files + enhance CSI volume output

test/e2e/storage/testsuites/provisioning.go

@@ -41,6 +41,7 @@ type StorageClassTest struct {
 	Name               string
 	CloudProviders     []string
 	Provisioner        string
+	StorageClassName   string
 	Parameters         map[string]string
 	DelayBinding       bool
 	ClaimSize          string

test/e2e/storage/utils/deployment.go

+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package utils
+
+import (
+	"path"
+	"strings"
+
+	appsv1 "k8s.io/api/apps/v1"
+	"k8s.io/api/core/v1"
+	storagev1 "k8s.io/api/storage/v1"
+	"k8s.io/kubernetes/test/e2e/framework"
+)
+
+// PatchCSIDeployment modifies the CSI driver deployment:
+// - replaces the provisioner name
+// - forces pods onto a specific host
+//
+// All of that is optional, see PatchCSIOptions. Just beware
+// that not renaming the CSI driver deployment can be problematic:
+// - when multiple tests deploy the driver, they need
+//   to run sequentially
+// - might conflict with manual deployments
+//
+// This function is written so that it works for CSI driver deployments
+// that follow these conventions:
+// - driver and provisioner names are identical
+// - the driver binary accepts a --drivername parameter
+// - the provisioner binary accepts a --provisioner parameter
+// - the paths inside the container are either fixed
+//   and don't need to be patch (for example, --csi-address=/csi/csi.sock is
+//   okay) or are specified directly in a parameter (for example,
+//   --kubelet-registration-path=/var/lib/kubelet/plugins/csi-hostpath/csi.sock)
+//
+// Driver deployments that are different will have to do the patching
+// without this function, or skip patching entirely.
+//
+// TODO (?): the storage.csi.image.version and storage.csi.image.registry
+// settings are ignored. We could patch the image definitions or deprecate
+// those options.
+func PatchCSIDeployment(f *framework.Framework, o PatchCSIOptions, object interface{}) error {
+	rename := o.OldDriverName != "" && o.NewDriverName != "" &&
+		o.OldDriverName != o.NewDriverName
+
+	patchVolumes := func(volumes []v1.Volume) {
+		if !rename {
+			return
+		}
+		for i := range volumes {
+			volume := &volumes[i]
+			if volume.HostPath != nil {
+				// Update paths like /var/lib/kubelet/plugins/<provisioner>.
+				p := &volume.HostPath.Path
+				dir, file := path.Split(*p)
+				if file == o.OldDriverName {
+					*p = path.Join(dir, o.NewDriverName)
+				}
+			}
+		}
+	}
+
+	patchContainers := func(containers []v1.Container) {
+		for i := range containers {
+			container := &containers[i]
+			if rename {
+				for e := range container.Args {
+					// Inject test-specific provider name into paths like this one:
+					// --kubelet-registration-path=/var/lib/kubelet/plugins/csi-hostpath/csi.sock
+					container.Args[e] = strings.Replace(container.Args[e], "/"+o.OldDriverName+"/", "/"+o.NewDriverName+"/", 1)
+				}
+			}
+			// Overwrite driver name resp. provider name
+			// by appending a parameter with the right
+			// value.
+			switch container.Name {
+			case o.DriverContainerName:
+				container.Args = append(container.Args, "--drivername="+o.NewDriverName)
+			case o.ProvisionerContainerName:
+				// Driver name is expected to be the same
+				// as the provisioner here.
+				container.Args = append(container.Args, "--provisioner="+o.NewDriverName)
+			}
+		}
+	}
+
+	patchPodSpec := func(spec *v1.PodSpec) {
+		patchContainers(spec.Containers)
+		patchVolumes(spec.Volumes)
+		if o.NodeName != "" {
+			spec.NodeName = o.NodeName
+		}
+	}
+
+	switch object := object.(type) {
+	case *appsv1.ReplicaSet:
+		patchPodSpec(&object.Spec.Template.Spec)
+	case *appsv1.DaemonSet:
+		patchPodSpec(&object.Spec.Template.Spec)
+	case *appsv1.StatefulSet:
+		patchPodSpec(&object.Spec.Template.Spec)
+	case *appsv1.Deployment:
+		patchPodSpec(&object.Spec.Template.Spec)
+	case *storagev1.StorageClass:
+		if o.NewDriverName != "" {
+			// Driver name is expected to be the same
+			// as the provisioner name here.
+			object.Provisioner = o.NewDriverName
+		}
+	}
+
+	return nil
+}
+
+// PatchCSIOptions controls how PatchCSIDeployment patches the objects.
+type PatchCSIOptions struct {
+	// The original driver name.
+	OldDriverName string
+	// The driver name that replaces the original name.
+	// Can be empty (not used at all) or equal to OldDriverName
+	// (then it will be added were appropriate without renaming
+	// in existing fields).
+	NewDriverName string
+	// The name of the container which has the CSI driver binary.
+	// If non-empty, --drivername with the new name will be
+	// appended to the argument list.
+	DriverContainerName string
+	// The name of the container which has the provisioner binary.
+	// If non-empty, --provisioner with new name will be appended
+	// to the argument list.
+	ProvisionerContainerName string
+	// If non-empty, all pods are forced to run on this node.
+	NodeName string
+}

test/e2e/testing-manifests/storage-csi/controller-role.yaml

-# Role for external CSI provisioner and attacher.
-# They need to modify Endpoints and ConfigMap for leader election.
-
-kind: Role
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: csi-controller
-rules:
-- apiGroups: [""]
-  resources: ["configmaps", "endpoints"]
-  verbs: ["get", "watch", "list", "delete", "update", "create"]
+# Replaced by individual roles for external-attacher and external-provisioner:
+# - https://github.com/kubernetes-csi/external-attacher/blob/master/deploy/kubernetes/rbac.yaml
+# - https://github.com/kubernetes-csi/external-provisioner/blob/master/deploy/kubernetes/rbac.yaml

test/e2e/testing-manifests/storage-csi/driver-registrar/README.md

+The original file is (or will be) https://github.com/kubernetes-csi/driver-registrar/blob/master/deploy/kubernetes/rbac.yaml

test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml

+# This YAML file contains all RBAC objects that are necessary to run external
+# CSI provisioner.
+#
+# In production, each CSI driver deployment has to be customized:
+# - to avoid conflicts, use non-default namespace and different names
+#   for non-namespaced entities like the ClusterRole
+# - decide whether the deployment replicates the external CSI
+#   provisioner, in which case leadership election must be enabled;
+#   this influences the RBAC setup, see below
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: csi-driver-registrar
+  # replace with non-default namespace name
+  namespace: default
+
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: driver-registrar-runner
+rules:
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["get", "list", "watch", "create", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "update", "patch"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-driver-registrar-role
+subjects:
+  - kind: ServiceAccount
+    name: csi-driver-registrar
+    # replace with non-default namespace name
+    namespace: default
+roleRef:
+  kind: ClusterRole
+  name: driver-registrar-runner
+  apiGroup: rbac.authorization.k8s.io

test/e2e/testing-manifests/storage-csi/external-attacher/README.md

+The original file is (or will be) https://github.com/kubernetes-csi/external-attacher/blob/master/deploy/kubernetes/rbac.yaml

test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml

+# This YAML file contains all RBAC objects that are necessary to run external
+# CSI attacher.
+#
+# In production, each CSI driver deployment has to be customized:
+# - to avoid conflicts, use non-default namespace and different names
+#   for non-namespaced entities like the ClusterRole
+# - decide whether the deployment replicates the external CSI
+#   attacher, in which case leadership election must be enabled;
+#   this influences the RBAC setup, see below
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: csi-attacher
+  # replace with non-default namespace name
+  namespace: default
+
+---
+# Attacher must be able to work with PVs, nodes and VolumeAttachments
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: external-attacher-runner
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["csi.storage.k8s.io"]
+    resources: ["csinodeinfos"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattachments"]
+    verbs: ["get", "list", "watch", "update"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-attacher-role
+subjects:
+  - kind: ServiceAccount
+    name: csi-attacher
+    # replace with non-default namespace name
+    namespace: default
+roleRef:
+  kind: ClusterRole
+  name: external-attacher-runner
+  apiGroup: rbac.authorization.k8s.io
+
+---
+# Attacher must be able to work with config map in current namespace
+# if (and only if) leadership election is enabled
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  # replace with non-default namespace name
+  namespace: default
+  name: external-attacher-cfg
+rules:
+- apiGroups: [""]
+  resources: ["configmaps"]
+  verbs: ["get", "watch", "list", "delete", "update", "create"]
+
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-attacher-role-cfg
+  # replace with non-default namespace name
+  namespace: default
+subjects:
+  - kind: ServiceAccount
+    name: csi-attacher
+    # replace with non-default namespace name
+    namespace: default
+roleRef:
+  kind: Role
+  name: external-attacher-cfg

test/e2e/testing-manifests/storage-csi/external-provisioner/README.md

+The original file is (or will be) https://github.com/kubernetes-csi/external-provisioner/blob/master/deploy/kubernetes/rbac.yaml

test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml

+# This YAML file contains all RBAC objects that are necessary to run external
+# CSI provisioner.
+#
+# In production, each CSI driver deployment has to be customized:
+# - to avoid conflicts, use non-default namespace and different names
+#   for non-namespaced entities like the ClusterRole
+# - decide whether the deployment replicates the external CSI
+#   provisioner, in which case leadership election must be enabled;
+#   this influences the RBAC setup, see below
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: csi-provisioner
+  # replace with non-default namespace name
+  namespace: default
+
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: external-provisioner-runner
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "list"]
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "create", "delete"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["list", "watch", "create", "update", "patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots"]
+    verbs: ["get", "list"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents"]
+    verbs: ["get", "list"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-provisioner-role
+subjects:
+  - kind: ServiceAccount
+    name: csi-provisioner
+    # replace with non-default namespace name
+    namespace: default
+roleRef:
+  kind: ClusterRole
+  name: external-provisioner-runner
+  apiGroup: rbac.authorization.k8s.io
+
+---
+# Provisioner must be able to work with endpoints in current namespace
+# if (and only if) leadership election is enabled
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  # replace with non-default namespace name
+  namespace: default
+  name: external-provisioner-cfg
+rules:
+- apiGroups: [""]
+  resources: ["endpoints"]
+  verbs: ["get", "watch", "list", "delete", "update", "create"]
+
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-provisioner-role-cfg
+  # replace with non-default namespace name
+  namespace: default
+subjects:
+  - kind: ServiceAccount
+    name: csi-provisioner
+    # replace with non-default namespace name
+    namespace: default
+roleRef:
+  kind: Role
+  name: external-provisioner-cfg

test/e2e/testing-manifests/storage-csi/gce-pd/controller_ss.yaml

@@ -13,6 +13,7 @@ spec:
       labels:
         app: csi-gce-pd-driver
     spec:
+      serviceAccountName: csi-controller
       containers:
         - name: csi-external-provisioner
           imagePullPolicy: Always

test/e2e/testing-manifests/storage-csi/gce-pd/csi-controller-rbac.yaml

+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: csi-controller
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-controller-attacher-role
+subjects:
+  - kind: ServiceAccount
+    name: csi-controller
+    namespace: default
+roleRef:
+  kind: ClusterRole
+  name: external-attacher-runner
+  apiGroup: rbac.authorization.k8s.io
+
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-controller-attacher-role-cfg
+  namespace: default
+subjects:
+  - kind: ServiceAccount
+    name: csi-controller
+    namespace: default
+roleRef:
+  kind: Role
+  name: external-attacher-cfg
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-controller-provisioner-role
+subjects:
+  - kind: ServiceAccount
+    name: csi-controller
+    namespace: default
+roleRef:
+  kind: ClusterRole
+  name: external-provisioner-runner
+  apiGroup: rbac.authorization.k8s.io
+
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-controller-provisioner-role-cfg
+  namespace: default
+subjects:
+  - kind: ServiceAccount
+    name: csi-controller
+    namespace: default
+roleRef:
+  kind: Role
+  name: external-provisioner-cfg
+
+---
+# priviledged Pod Security Policy, previously defined via PrivilegedTestPSPClusterRoleBinding()
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: psp-csi-controller-driver-registrar-role
+subjects:
+  - kind: ServiceAccount
+    name: csi-controller
+    namespace: default
+  - kind: ServiceAccount
+    name: csi-driver-registrar
+    namespace: default
+roleRef:
+  kind: ClusterRole
+  name: e2e-test-privileged-psp
+  apiGroup: rbac.authorization.k8s.io

test/e2e/testing-manifests/storage-csi/gce-pd/node_ds.yaml

@@ -12,6 +12,7 @@ spec:
       labels:
         app: csi-gce-driver
     spec:
+      serviceAccountName: csi-driver-registrar
       containers:
         - name: csi-driver-registrar
           imagePullPolicy: Always

test/e2e/testing-manifests/storage-csi/hostpath/README.md

+A partial copy of https://github.com/kubernetes-csi/docs/tree/master/book/src/example,
+with some modifications:
+- serviceAccountName is used instead of the deprecated serviceAccount
+- the RBAC roles from driver-registrar, external-attacher and external-provisioner
+  are used

test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-attacher.yaml

+kind: Service
+apiVersion: v1
+metadata:
+  name: csi-hostpath-attacher
+  labels:
+    app: csi-hostpath-attacher
+spec:
+  selector:
+    app: csi-hostpath-attacher
+  ports:
+    - name: dummy
+      port: 12345
+
+---
+kind: StatefulSet
+apiVersion: apps/v1
+metadata:
+  name: csi-hostpath-attacher
+spec:
+  serviceName: "csi-hostpath-attacher"
+  replicas: 1
+  selector:
+    matchLabels:
+      app: csi-hostpath-attacher
+  template:
+    metadata:
+      labels:
+        app: csi-hostpath-attacher
+    spec:
+      serviceAccountName: csi-attacher
+      containers:
+        - name: csi-attacher
+          image: quay.io/k8scsi/csi-attacher:v0.4.0
+          args:
+            - --v=5
+            - --csi-address=$(ADDRESS)
+          env:
+            - name: ADDRESS
+              value: /csi/csi.sock
+          imagePullPolicy: Always
+          volumeMounts:
+          - mountPath: /csi
+            name: socket-dir
+      volumes:
+        - hostPath:
+            path: /var/lib/kubelet/plugins/csi-hostpath
+            type: DirectoryOrCreate
+          name: socket-dir

test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-provisioner.yaml

+kind: Service
+apiVersion: v1
+metadata:
+  name: csi-hostpath-provisioner 
+  labels:
+    app: csi-hostpath-provisioner 
+spec:
+  selector:
+    app: csi-hostpath-provisioner 
+  ports:
+    - name: dummy
+      port: 12345
+
+---
+kind: StatefulSet
+apiVersion: apps/v1
+metadata:
+  name: csi-hostpath-provisioner
+spec:
+  serviceName: "csi-hostpath-provisioner"
+  replicas: 1
+  selector:
+    matchLabels:
+      app: csi-hostpath-provisioner
+  template:
+    metadata:
+      labels:
+        app: csi-hostpath-provisioner
+    spec:
+      serviceAccountName: csi-provisioner
+      containers:
+        - name: csi-provisioner
+          image: quay.io/k8scsi/csi-provisioner:v0.4.0
+          args:
+            - "--provisioner=csi-hostpath"
+            - "--csi-address=$(ADDRESS)"
+            - "--connection-timeout=15s"
+          env:
+            - name: ADDRESS
+              value: /csi/csi.sock
+          imagePullPolicy: Always
+          volumeMounts:
+            - mountPath: /csi
+              name: socket-dir
+      volumes:
+        - hostPath:
+            path: /var/lib/kubelet/plugins/csi-hostpath
+            type: DirectoryOrCreate
+          name: socket-dir

test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpathplugin.yaml

+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: csi-hostpathplugin
+spec:
+  selector:
+    matchLabels:
+      app: csi-hostpathplugin
+  template:
+    metadata:
+      labels:
+        app: csi-hostpathplugin
+    spec:
+      serviceAccountName: csi-driver-registrar
+      hostNetwork: true
+      containers:
+        - name: driver-registrar
+          image: quay.io/k8scsi/driver-registrar:v0.4.0
+          args:
+            - --v=5
+            - --csi-address=/csi/csi.sock
+            - --kubelet-registration-path=/var/lib/kubelet/plugins/csi-hostpath/csi.sock
+          env:
+            - name: KUBE_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: spec.nodeName
+          imagePullPolicy: Always
+          volumeMounts:
+          - mountPath: /csi
+            name: socket-dir
+          - mountPath: /registration
+            name: registration-dir
+        - name: hostpath
+          image: quay.io/k8scsi/hostpathplugin:v0.4.0
+          args:
+            - "--v=5"
+            - "--endpoint=$(CSI_ENDPOINT)"
+            - "--nodeid=$(KUBE_NODE_NAME)"
+          env:
+            - name: CSI_ENDPOINT
+              value: unix:///csi/csi.sock
+            - name: KUBE_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: spec.nodeName
+          imagePullPolicy: Always
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - mountPath: /csi
+              name: socket-dir
+            - mountPath: /var/lib/kubelet/pods
+              mountPropagation: Bidirectional
+              name: mountpoint-dir
+      volumes:
+        - hostPath:
+            path: /var/lib/kubelet/plugins/csi-hostpath
+            type: DirectoryOrCreate
+          name: socket-dir
+        - hostPath:
+            path: /var/lib/kubelet/pods
+            type: DirectoryOrCreate
+          name: mountpoint-dir
+        - hostPath:
+            path: /var/lib/kubelet/plugins
+            type: Directory
+          name: registration-dir

test/e2e/testing-manifests/storage-csi/hostpath/hostpath/e2e-test-rbac.yaml

+# priviledged Pod Security Policy, previously defined just for gcePD via PrivilegedTestPSPClusterRoleBinding()
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: psp-csi-hostpath-role
+subjects:
+  - kind: ServiceAccount
+    name: csi-attacher
+    namespace: default
+  - kind: ServiceAccount
+    name: csi-driver-registrar
+    namespace: default
+  - kind: ServiceAccount
+    name: csi-provisioner
+    namespace: default
+roleRef:
+  kind: ClusterRole
+  name: e2e-test-privileged-psp
+  apiGroup: rbac.authorization.k8s.io

test/e2e/testing-manifests/storage-csi/hostpath/usage/csi-storageclass.yaml

+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: csi-hostpath-sc
+provisioner: csi-hostpath
+reclaimPolicy: Delete
+volumeBindingMode: Immediate