Make csi drivers and in-tree drivers share e2e tests

test/e2e/storage/BUILD

@@ -3,7 +3,6 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library")
 go_library(
     name = "go_default_library",
     srcs = [
-        "csi_objects.go",
         "csi_volumes.go",
         "empty_dir_wrapper.go",
         "ephemeral_volume.go",
@@ -72,6 +71,7 @@ go_library(
         "//test/e2e/framework/providers/gce:go_default_library",
         "//test/e2e/framework/testfiles:go_default_library",
         "//test/e2e/storage/drivers:go_default_library",
+        "//test/e2e/storage/testpatterns:go_default_library",
         "//test/e2e/storage/testsuites:go_default_library",
         "//test/e2e/storage/utils:go_default_library",
         "//test/utils/image:go_default_library",

test/e2e/storage/drivers/BUILD

@@ -4,6 +4,8 @@ go_library(
     name = "go_default_library",
     srcs = [
         "base.go",
+        "csi.go",
+        "csi_objects.go",
         "in_tree.go",
     ],
     importpath = "k8s.io/kubernetes/test/e2e/storage/drivers",
@@ -17,7 +19,9 @@ go_library(
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/authentication/serviceaccount:go_default_library",
+        "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
         "//test/e2e/framework:go_default_library",
         "//test/e2e/storage/testpatterns:go_default_library",
         "//test/e2e/storage/utils:go_default_library",

test/e2e/storage/drivers/csi.go

+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/*
+ * This file defines various csi volume test drivers for TestSuites.
+ *
+ * There are two ways, how to prepare test drivers:
+ * 1) With containerized server (NFS, Ceph, Gluster, iSCSI, ...)
+ * It creates a server pod which defines one volume for the tests.
+ * These tests work only when privileged containers are allowed, exporting
+ * various filesystems (NFS, GlusterFS, ...) usually needs some mounting or
+ * other privileged magic in the server pod.
+ *
+ * Note that the server containers are for testing purposes only and should not
+ * be used in production.
+ *
+ * 2) With server or cloud provider outside of Kubernetes (Cinder, GCE, AWS, Azure, ...)
+ * Appropriate server or cloud provider must exist somewhere outside
+ * the tested Kubernetes cluster. CreateVolume will create a new volume to be
+ * used in the TestSuites for inlineVolume or DynamicPV tests.
+ */
+
+package drivers
+
+import (
+	"fmt"
+	"math/rand"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	storagev1 "k8s.io/api/storage/v1"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/storage/testpatterns"
+	"k8s.io/kubernetes/test/e2e/storage/utils"
+)
+
+// hostpathCSI
+type hostpathCSIDriver struct {
+	cleanup    func()
+	driverInfo DriverInfo
+}
+
+var _ TestDriver = &hostpathCSIDriver{}
+var _ DynamicPVTestDriver = &hostpathCSIDriver{}
+
+// InitHostPathCSIDriver returns hostpathCSIDriver that implements TestDriver interface
+func InitHostPathCSIDriver() TestDriver {
+	return &hostpathCSIDriver{
+		driverInfo: DriverInfo{
+			Name:        "csi-hostpath",
+			FeatureTag:  "",
+			MaxFileSize: testpatterns.FileSizeMedium,
+			SupportedFsType: sets.NewString(
+				"", // Default fsType
+			),
+			IsPersistent:       true,
+			IsFsGroupSupported: false,
+			IsBlockSupported:   false,
+		},
+	}
+}
+
+func (h *hostpathCSIDriver) GetDriverInfo() *DriverInfo {
+	return &h.driverInfo
+}
+
+func (h *hostpathCSIDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) {
+}
+
+func (h *hostpathCSIDriver) GetDynamicProvisionStorageClass(fsType string) *storagev1.StorageClass {
+	provisioner := h.driverInfo.Name + h.driverInfo.Framework.UniqueName
+	parameters := map[string]string{}
+	ns := h.driverInfo.Framework.Namespace.Name
+	suffix := fmt.Sprintf("%s-sc", provisioner)
+
+	return getStorageClass(provisioner, parameters, nil, ns, suffix)
+}
+
+func (h *hostpathCSIDriver) CreateDriver() {
+	By("deploying csi hostpath driver")
+	f := h.driverInfo.Framework
+	cs := f.ClientSet
+
+	// pods should be scheduled on the node
+	nodes := framework.GetReadySchedulableNodesOrDie(cs)
+	node := nodes.Items[rand.Intn(len(nodes.Items))]
+	h.driverInfo.Config.ClientNodeName = node.Name
+	h.driverInfo.Config.ServerNodeName = node.Name
+
+	// TODO (?): the storage.csi.image.version and storage.csi.image.registry
+	// settings are ignored for this test. We could patch the image definitions.
+	o := utils.PatchCSIOptions{
+		OldDriverName:            h.driverInfo.Name,
+		NewDriverName:            h.driverInfo.Name + h.driverInfo.Framework.UniqueName,
+		DriverContainerName:      "hostpath",
+		ProvisionerContainerName: "csi-provisioner",
+		NodeName:                 h.driverInfo.Config.ServerNodeName,
+	}
+	cleanup, err := h.driverInfo.Framework.CreateFromManifests(func(item interface{}) error {
+		return utils.PatchCSIDeployment(h.driverInfo.Framework, o, item)
+	},
+		"test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml",
+		"test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml",
+		"test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml",
+		"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-attacher.yaml",
+		"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-provisioner.yaml",
+		"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpathplugin.yaml",
+		"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/e2e-test-rbac.yaml",
+	)
+	h.cleanup = cleanup
+	if err != nil {
+		framework.Failf("deploying csi hostpath driver: %v", err)
+	}
+}
+
+func (h *hostpathCSIDriver) CleanupDriver() {
+	if h.cleanup != nil {
+		By("uninstalling csi hostpath driver")
+		h.cleanup()
+	}
+}
+
+// gce-pd
+type gcePDCSIDriver struct {
+	cleanup    func()
+	driverInfo DriverInfo
+}
+
+var _ TestDriver = &gcePDCSIDriver{}
+var _ DynamicPVTestDriver = &gcePDCSIDriver{}
+
+// InitGcePDCSIDriver returns gcePDCSIDriver that implements TestDriver interface
+func InitGcePDCSIDriver() TestDriver {
+	return &gcePDCSIDriver{
+		driverInfo: DriverInfo{
+			Name:        "com.google.csi.gcepd",
+			FeatureTag:  "[Serial]",
+			MaxFileSize: testpatterns.FileSizeMedium,
+			SupportedFsType: sets.NewString(
+				"", // Default fsType
+				"ext2",
+				"ext3",
+				"ext4",
+				"xfs",
+			),
+			IsPersistent:       true,
+			IsFsGroupSupported: true,
+			IsBlockSupported:   false,
+		},
+	}
+}
+
+func (g *gcePDCSIDriver) GetDriverInfo() *DriverInfo {
+	return &g.driverInfo
+}
+
+func (g *gcePDCSIDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) {
+	f := g.driverInfo.Framework
+	cs := f.ClientSet
+	config := g.driverInfo.Config
+	framework.SkipUnlessProviderIs("gce", "gke")
+	framework.SkipIfMultizone(cs)
+
+	// TODO(#62561): Use credentials through external pod identity when that goes GA instead of downloading keys.
+	createGCESecrets(cs, config)
+	framework.SkipUnlessSecretExistsAfterWait(cs, "cloud-sa", config.Namespace, 3*time.Minute)
+}
+
+func (g *gcePDCSIDriver) GetDynamicProvisionStorageClass(fsType string) *storagev1.StorageClass {
+	ns := g.driverInfo.Framework.Namespace.Name
+	provisioner := g.driverInfo.Name
+	suffix := fmt.Sprintf("%s-sc", g.driverInfo.Name)
+
+	parameters := map[string]string{"type": "pd-standard"}
+
+	return getStorageClass(provisioner, parameters, nil, ns, suffix)
+}
+
+func (g *gcePDCSIDriver) CreateDriver() {
+	By("deploying csi gce-pd driver")
+	// It would be safer to rename the gcePD driver, but that
+	// hasn't been done before either and attempts to do so now led to
+	// errors during driver registration, therefore it is disabled
+	// by passing a nil function below.
+	//
+	// These are the options which would have to be used:
+	// o := utils.PatchCSIOptions{
+	// 	OldDriverName:            "com.google.csi.gcepd",
+	// 	NewDriverName:            "com.google.csi.gcepd-" + g.f.UniqueName,
+	// 	DriverContainerName:      "gce-driver",
+	// 	ProvisionerContainerName: "csi-external-provisioner",
+	// }
+	cleanup, err := g.driverInfo.Framework.CreateFromManifests(nil,
+		"test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml",
+		"test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml",
+		"test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml",
+		"test/e2e/testing-manifests/storage-csi/gce-pd/csi-controller-rbac.yaml",
+		"test/e2e/testing-manifests/storage-csi/gce-pd/node_ds.yaml",
+		"test/e2e/testing-manifests/storage-csi/gce-pd/controller_ss.yaml",
+	)
+	g.cleanup = cleanup
+	if err != nil {
+		framework.Failf("deploying csi gce-pd driver: %v", err)
+	}
+}
+
+func (g *gcePDCSIDriver) CleanupDriver() {
+	By("uninstalling gce-pd driver")
+	if g.cleanup != nil {
+		g.cleanup()
+	}
+}
+
+// gcePd-external
+type gcePDExternalCSIDriver struct {
+	driverInfo DriverInfo
+}
+
+var _ TestDriver = &gcePDExternalCSIDriver{}
+var _ DynamicPVTestDriver = &gcePDExternalCSIDriver{}
+
+// InitGcePDExternalCSIDriver returns gcePDExternalCSIDriver that implements TestDriver interface
+func InitGcePDExternalCSIDriver() TestDriver {
+	return &gcePDExternalCSIDriver{
+		driverInfo: DriverInfo{
+			Name: "com.google.csi.gcepd",
+			// TODO(#70258): this is temporary until we can figure out how to make e2e tests a library
+			FeatureTag:  "[Serial][Feature: gcePD-external]",
+			MaxFileSize: testpatterns.FileSizeMedium,
+			SupportedFsType: sets.NewString(
+				"", // Default fsType
+				"ext2",
+				"ext3",
+				"ext4",
+				"xfs",
+			),
+			IsPersistent:       true,
+			IsFsGroupSupported: true,
+			IsBlockSupported:   false,
+		},
+	}
+}
+
+func (g *gcePDExternalCSIDriver) GetDriverInfo() *DriverInfo {
+	return &g.driverInfo
+}
+
+func (g *gcePDExternalCSIDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) {
+	framework.SkipUnlessProviderIs("gce", "gke")
+	framework.SkipIfMultizone(g.driverInfo.Framework.ClientSet)
+}
+
+func (g *gcePDExternalCSIDriver) GetDynamicProvisionStorageClass(fsType string) *storagev1.StorageClass {
+	ns := g.driverInfo.Framework.Namespace.Name
+	provisioner := g.driverInfo.Name
+	suffix := fmt.Sprintf("%s-sc", g.driverInfo.Name)
+
+	parameters := map[string]string{"type": "pd-standard"}
+
+	return getStorageClass(provisioner, parameters, nil, ns, suffix)
+}
+
+func (g *gcePDExternalCSIDriver) CreateDriver() {
+}
+
+func (g *gcePDExternalCSIDriver) CleanupDriver() {
+}

test/e2e/storage/csi_objects.go → test/e2e/storage/drivers/csi_objects.go

@@ -17,7 +17,7 @@ limitations under the License.
 // This file is used to deploy the CSI hostPath plugin
 // More Information: https://github.com/kubernetes-csi/drivers/tree/master/pkg/hostpath
 
-package storage
+package drivers
 
 import (
 	"flag"

test/e2e/storage/in_tree_volumes.go

@@ -23,6 +23,7 @@ import (
 	"k8s.io/api/core/v1"
 	"k8s.io/kubernetes/test/e2e/framework"
 	"k8s.io/kubernetes/test/e2e/storage/drivers"
+	"k8s.io/kubernetes/test/e2e/storage/testpatterns"
 	"k8s.io/kubernetes/test/e2e/storage/testsuites"
 	"k8s.io/kubernetes/test/e2e/storage/utils"
 )
@@ -53,6 +54,10 @@ var testSuites = []func() testsuites.TestSuite{
 	testsuites.InitProvisioningTestSuite,
 }
 
+func intreeTunePattern(patterns []testpatterns.TestPattern) []testpatterns.TestPattern {
+	return patterns
+}
+
 // This executes testSuites for in-tree volumes.
 var _ = utils.SIGDescribe("In-tree Volumes", func() {
 	f := framework.NewDefaultFramework("volumes")
@@ -86,7 +91,7 @@ var _ = utils.SIGDescribe("In-tree Volumes", func() {
 				driver.CleanupDriver()
 			})
 
-			testsuites.RunTestSuite(f, config, driver, testSuites)
+			testsuites.RunTestSuite(f, config, driver, testSuites, intreeTunePattern)
 		})
 	}
 })

test/e2e/storage/testsuites/base.go

@@ -66,12 +66,12 @@ func getTestNameStr(suite TestSuite, pattern testpatterns.TestPattern) string {
 }
 
 // RunTestSuite runs all testpatterns of all testSuites for a driver
-func RunTestSuite(f *framework.Framework, config framework.VolumeTestConfig, driver drivers.TestDriver, tsInits []func() TestSuite) {
+func RunTestSuite(f *framework.Framework, config framework.VolumeTestConfig, driver drivers.TestDriver, tsInits []func() TestSuite, tunePatternFunc func([]testpatterns.TestPattern) []testpatterns.TestPattern) {
 	for _, testSuiteInit := range tsInits {
 		suite := testSuiteInit()
-		tsInfo := suite.getTestSuiteInfo()
+		patterns := tunePatternFunc(suite.getTestSuiteInfo().testPatterns)
 
-		for _, pattern := range tsInfo.testPatterns {
+		for _, pattern := range patterns {
 			suite.execTest(driver, pattern)
 		}
 	}
@@ -164,7 +164,7 @@ func (r *genericVolumeTestResource) setupResource(driver drivers.TestDriver, pat
 	case testpatterns.DynamicPV:
 		framework.Logf("Creating resource for dynamic PV")
 		if dDriver, ok := driver.(drivers.DynamicPVTestDriver); ok {
-			claimSize := "2Gi"
+			claimSize := "5Gi"
 			r.sc = dDriver.GetDynamicProvisionStorageClass(fsType)
 
 			By("creating a StorageClass " + r.sc.Name)

test/e2e/storage/testsuites/provisioning.go

@@ -151,7 +151,7 @@ func (p *provisioningTestResource) setupResource(driver drivers.TestDriver, patt
 				framework.Skipf("Driver %q does not define Dynamic Provision StorageClass - skipping", driver.GetDriverInfo().Name)
 			}
 			p.driver = driver
-			p.claimSize = "2Gi"
+			p.claimSize = "5Gi"
 			p.pvc = getClaim(p.claimSize, driver.GetDriverInfo().Framework.Namespace.Name)
 			p.pvc.Spec.StorageClassName = &p.sc.Name
 			framework.Logf("In creating storage class object and pvc object for driver - sc: %v, pvc: %v", p.sc, p.pvc)

test/e2e/storage/testsuites/subpath.go

@@ -19,6 +19,7 @@ package testsuites
 import (
 	"fmt"
 	"path/filepath"
+	"regexp"
 	"strings"
 
 	"k8s.io/api/core/v1"
@@ -315,7 +316,8 @@ func testSubPath(input *subPathTestInput) {
 	})
 
 	It("should unmount if pod is force deleted while kubelet is down [Disruptive][Slow]", func() {
-		if input.volType == "hostPath" || input.volType == "hostPathSymlink" {
+		if strings.HasPrefix(input.volType, "hostPath") || strings.HasPrefix(input.volType, "csi-hostpath") {
+			// TODO: This skip should be removed once #61446 is fixed
 			framework.Skipf("%s volume type does not support reconstruction, skipping", input.volType)
 		}
 		testSubpathReconstruction(input.f, input.pod, true)
@@ -380,10 +382,23 @@ func TestBasicSubpathFile(f *framework.Framework, contents string, pod *v1.Pod,
 	Expect(err).NotTo(HaveOccurred(), "while deleting pod")
 }
 
+func generateSuffixForPodName(s string) string {
+	// Pod name must:
+	//   1. consist of lower case alphanumeric characters or '-',
+	//   2. start and end with an alphanumeric character.
+	// (e.g. 'my-name',  or '123-abc', regex used for validation is '[a-z0-9]([-a-z0-9]*[a-z0-9])?')
+	// Therefore, suffix is generated by following steps:
+	//   1. all strings other than [A-Za-z0-9] is replaced with "-",
+	//   2. add lower case alphanumeric characters at the end ('-[a-z0-9]{4}' is added),
+	//   3. convert the entire strings to lower case.
+	re := regexp.MustCompile("[^A-Za-z0-9]")
+	return strings.ToLower(fmt.Sprintf("%s-%s", re.ReplaceAllString(s, "-"), rand.String(4)))
+}
+
 // SubpathTestPod returns a pod spec for subpath tests
 func SubpathTestPod(f *framework.Framework, subpath, volumeType string, source *v1.VolumeSource, privilegedSecurityContext bool) *v1.Pod {
 	var (
-		suffix          = strings.ToLower(fmt.Sprintf("%s-%s", volumeType, rand.String(4)))
+		suffix          = generateSuffixForPodName(volumeType)
 		gracePeriod     = int64(1)
 		probeVolumeName = "liveness-probe-volume"
 	)

test/e2e/storage/testsuites/volumemode.go

@@ -197,7 +197,7 @@ func (s *volumeModeTestResource) setupResource(driver drivers.TestDriver, patter
 			}
 			s.sc.VolumeBindingMode = &volBindMode
 
-			claimSize := "2Gi"
+			claimSize := "5Gi"
 			s.pvc = getClaim(claimSize, ns.Name)
 			s.pvc.Spec.StorageClassName = &s.sc.Name
 			s.pvc.Spec.VolumeMode = &volMode