Merge pull request #74693 from mkimuram/issue/74545

test/e2e/framework/pv_util.go

@@ -93,6 +93,14 @@ type PersistentVolumeClaimConfig struct {
 	VolumeMode       *v1.PersistentVolumeMode
 }
 
+// NodeSelection specifies where to run a pod, using a combination of fixed node name,
+// node selector and/or affinity.
+type NodeSelection struct {
+	Name     string
+	Selector map[string]string
+	Affinity *v1.Affinity
+}
+
 // Clean up a pv and pvc in a single pv/pvc test case.
 // Note: delete errors are appended to []error so that we can attempt to delete both the pvc and pv.
 func PVPVCCleanup(c clientset.Interface, ns string, pv *v1.PersistentVolume, pvc *v1.PersistentVolumeClaim) []error {
@@ -873,14 +881,16 @@ func CreateNginxPod(client clientset.Interface, namespace string, nodeSelector m
 
 // create security pod with given claims
 func CreateSecPod(client clientset.Interface, namespace string, pvclaims []*v1.PersistentVolumeClaim, isPrivileged bool, command string, hostIPC bool, hostPID bool, seLinuxLabel *v1.SELinuxOptions, fsGroup *int64, timeout time.Duration) (*v1.Pod, error) {
-	return CreateSecPodWithNodeName(client, namespace, pvclaims, isPrivileged, command, hostIPC, hostPID, seLinuxLabel, fsGroup, "", timeout)
+	return CreateSecPodWithNodeSelection(client, namespace, pvclaims, isPrivileged, command, hostIPC, hostPID, seLinuxLabel, fsGroup, NodeSelection{}, timeout)
 }
 
 // create security pod with given claims
-func CreateSecPodWithNodeName(client clientset.Interface, namespace string, pvclaims []*v1.PersistentVolumeClaim, isPrivileged bool, command string, hostIPC bool, hostPID bool, seLinuxLabel *v1.SELinuxOptions, fsGroup *int64, nodeName string, timeout time.Duration) (*v1.Pod, error) {
+func CreateSecPodWithNodeSelection(client clientset.Interface, namespace string, pvclaims []*v1.PersistentVolumeClaim, isPrivileged bool, command string, hostIPC bool, hostPID bool, seLinuxLabel *v1.SELinuxOptions, fsGroup *int64, node NodeSelection, timeout time.Duration) (*v1.Pod, error) {
 	pod := MakeSecPod(namespace, pvclaims, isPrivileged, command, hostIPC, hostPID, seLinuxLabel, fsGroup)
-	// Setting nodeName
-	pod.Spec.NodeName = nodeName
+	// Setting node
+	pod.Spec.NodeName = node.Name
+	pod.Spec.NodeSelector = node.Selector
+	pod.Spec.Affinity = node.Affinity
 
 	pod, err := client.CoreV1().Pods(namespace).Create(pod)
 	if err != nil {
@@ -900,6 +910,44 @@ func CreateSecPodWithNodeName(client clientset.Interface, namespace string, pvcl
 	return pod, nil
 }
 
+// SetNodeAffinityRequirement sets affinity with specified operator to nodeName to nodeSelection
+func SetNodeAffinityRequirement(nodeSelection *NodeSelection, operator v1.NodeSelectorOperator, nodeName string) {
+	// Add node-anti-affinity.
+	if nodeSelection.Affinity == nil {
+		nodeSelection.Affinity = &v1.Affinity{}
+	}
+	if nodeSelection.Affinity.NodeAffinity == nil {
+		nodeSelection.Affinity.NodeAffinity = &v1.NodeAffinity{}
+	}
+	if nodeSelection.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil {
+		nodeSelection.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution = &v1.NodeSelector{}
+	}
+	nodeSelection.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = append(nodeSelection.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms,
+		v1.NodeSelectorTerm{
+			// https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity warns
+			// that "the value of kubernetes.io/hostname may be the same as the Node name in some environments and a different value in other environments".
+			// So this might be cleaner:
+			// MatchFields: []v1.NodeSelectorRequirement{
+			// 	{Key: "name", Operator: v1.NodeSelectorOpNotIn, Values: []string{nodeName}},
+			// },
+			// However, "name", "Name", "ObjectMeta.Name" all got rejected with "not a valid field selector key".
+
+			MatchExpressions: []v1.NodeSelectorRequirement{
+				{Key: "kubernetes.io/hostname", Operator: operator, Values: []string{nodeName}},
+			},
+		})
+}
+
+// SetAffinity sets affinity to nodeName to nodeSelection
+func SetAffinity(nodeSelection *NodeSelection, nodeName string) {
+	SetNodeAffinityRequirement(nodeSelection, v1.NodeSelectorOpIn, nodeName)
+}
+
+// SetAntiAffinity sets anti-affinity to nodeName to nodeSelection
+func SetAntiAffinity(nodeSelection *NodeSelection, nodeName string) {
+	SetNodeAffinityRequirement(nodeSelection, v1.NodeSelectorOpNotIn, nodeName)
+}
+
 // Define and create a pod with a mounted PV.  Pod runs infinite loop until killed.
 func CreateClientPod(c clientset.Interface, ns string, pvc *v1.PersistentVolumeClaim) (*v1.Pod, error) {
 	return CreatePod(c, ns, nil, []*v1.PersistentVolumeClaim{pvc}, true, "")

test/e2e/storage/csi_mock_volume.go

@@ -153,12 +153,12 @@ var _ = utils.SIGDescribe("CSI mock volume", func() {
 			scTest.AllowVolumeExpansion = true
 		}
 
-		nodeSelection := testsuites.NodeSelection{
+		nodeSelection := framework.NodeSelection{
 			// The mock driver only works when everything runs on a single node.
 			Name: nodeName,
 		}
 		if len(m.nodeLabel) > 0 {
-			nodeSelection = testsuites.NodeSelection{
+			nodeSelection = framework.NodeSelection{
 				Selector: m.nodeLabel,
 			}
 		}
@@ -177,11 +177,11 @@ var _ = utils.SIGDescribe("CSI mock volume", func() {
 
 	createPodWithPVC := func(pvc *v1.PersistentVolumeClaim) (*v1.Pod, error) {
 		nodeName := m.config.ClientNodeName
-		nodeSelection := testsuites.NodeSelection{
+		nodeSelection := framework.NodeSelection{
 			Name: nodeName,
 		}
 		if len(m.nodeLabel) > 0 {
-			nodeSelection = testsuites.NodeSelection{
+			nodeSelection = framework.NodeSelection{
 				Selector: m.nodeLabel,
 			}
 		}
@@ -597,7 +597,7 @@ func checkNodeForLimits(nodeName string, attachKey v1.ResourceName, cs clientset
 	return int(attachLimit), waitErr
 }
 
-func startPausePod(cs clientset.Interface, t testsuites.StorageClassTest, node testsuites.NodeSelection, ns string) (*storagev1.StorageClass, *v1.PersistentVolumeClaim, *v1.Pod) {
+func startPausePod(cs clientset.Interface, t testsuites.StorageClassTest, node framework.NodeSelection, ns string) (*storagev1.StorageClass, *v1.PersistentVolumeClaim, *v1.Pod) {
 	class := newStorageClass(t, ns, "")
 	var err error
 	_, err = cs.StorageV1().StorageClasses().Get(class.Name, metav1.GetOptions{})
@@ -659,7 +659,7 @@ func startPausePod(cs clientset.Interface, t testsuites.StorageClassTest, node t
 	return class, claim, pod
 }
 
-func startPausePodWithClaim(cs clientset.Interface, pvc *v1.PersistentVolumeClaim, node testsuites.NodeSelection, ns string) (*v1.Pod, error) {
+func startPausePodWithClaim(cs clientset.Interface, pvc *v1.PersistentVolumeClaim, node framework.NodeSelection, ns string) (*v1.Pod, error) {
 	pod := &v1.Pod{
 		ObjectMeta: metav1.ObjectMeta{
 			GenerateName: "pvc-volume-tester-",

test/e2e/storage/csi_volumes.go

@@ -47,6 +47,7 @@ var csiTestSuites = []func() testsuites.TestSuite{
 	testsuites.InitSubPathTestSuite,
 	testsuites.InitProvisioningTestSuite,
 	testsuites.InitSnapshottableTestSuite,
+	testsuites.InitMultiVolumeTestSuite,
 }
 
 // This executes testSuites for csi volumes.
@@ -155,7 +156,7 @@ func testTopologyNegative(cs clientset.Interface, suffix, namespace string, dela
 		test.PvCheck = func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
 			// Ensure that a pod cannot be scheduled in an unsuitable zone.
 			pod := testsuites.StartInPodWithVolume(cs, namespace, claim.Name, "pvc-tester-unschedulable", "sleep 100000",
-				testsuites.NodeSelection{Selector: nodeSelector})
+				framework.NodeSelection{Selector: nodeSelector})
 			defer testsuites.StopPod(cs, pod)
 			framework.ExpectNoError(framework.WaitForPodNameUnschedulableInNamespace(cs, pod.Name, pod.Namespace), "pod should be unschedulable")
 		}

test/e2e/storage/drivers/in_tree.go

@@ -95,6 +95,7 @@ func InitNFSDriver() testsuites.TestDriver {
 			Capabilities: map[testsuites.Capability]bool{
 				testsuites.CapPersistence: true,
 				testsuites.CapExec:        true,
+				testsuites.CapRWX:         true,
 			},
 		},
 	}
@@ -232,6 +233,7 @@ func InitGlusterFSDriver() testsuites.TestDriver {
 			Capabilities: map[testsuites.Capability]bool{
 				testsuites.CapPersistence: true,
 				testsuites.CapExec:        true,
+				testsuites.CapRWX:         true,
 			},
 		},
 	}
@@ -586,6 +588,7 @@ func InitCephFSDriver() testsuites.TestDriver {
 			Capabilities: map[testsuites.Capability]bool{
 				testsuites.CapPersistence: true,
 				testsuites.CapExec:        true,
+				testsuites.CapRWX:         true,
 			},
 		},
 	}

test/e2e/storage/in_tree_volumes.go

@@ -55,6 +55,7 @@ var testSuites = []func() testsuites.TestSuite{
 	testsuites.InitVolumeModeTestSuite,
 	testsuites.InitSubPathTestSuite,
 	testsuites.InitProvisioningTestSuite,
+	testsuites.InitMultiVolumeTestSuite,
 }
 
 // This executes testSuites for in-tree volumes.

test/e2e/storage/persistent_volumes.go

@@ -30,6 +30,7 @@ import (
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/storage/testsuites"
 	"k8s.io/kubernetes/test/e2e/storage/utils"
 	imageutils "k8s.io/kubernetes/test/utils/image"
 )
@@ -308,68 +309,22 @@ var _ = utils.SIGDescribe("PersistentVolumes", func() {
 	Describe("Default StorageClass", func() {
 		Context("pods that use multiple volumes", func() {
 
-			AfterEach(func() {
-				framework.DeleteAllStatefulSets(c, ns)
-			})
-
 			It("should be reschedulable [Slow]", func() {
 				// Only run on providers with default storageclass
 				framework.SkipUnlessProviderIs("openstack", "gce", "gke", "vsphere", "azure")
 
 				numVols := 4
-				ssTester := framework.NewStatefulSetTester(c)
-
-				By("Creating a StatefulSet pod to initialize data")
-				writeCmd := "true"
-				for i := 0; i < numVols; i++ {
-					writeCmd += fmt.Sprintf("&& touch %v", getVolumeFile(i))
-				}
-				writeCmd += "&& sleep 10000"
-
-				probe := &v1.Probe{
-					Handler: v1.Handler{
-						Exec: &v1.ExecAction{
-							// Check that the last file got created
-							Command: []string{"test", "-f", getVolumeFile(numVols - 1)},
-						},
-					},
-					InitialDelaySeconds: 1,
-					PeriodSeconds:       1,
-				}
 
-				mounts := []v1.VolumeMount{}
-				claims := []v1.PersistentVolumeClaim{}
+				By("Creating pvcs")
+				claims := []*v1.PersistentVolumeClaim{}
 				for i := 0; i < numVols; i++ {
 					pvc := framework.MakePersistentVolumeClaim(framework.PersistentVolumeClaimConfig{}, ns)
-					pvc.Name = getVolName(i)
-					mounts = append(mounts, v1.VolumeMount{Name: pvc.Name, MountPath: getMountPath(i)})
-					claims = append(claims, *pvc)
+					claims = append(claims, pvc)
 				}
 
-				spec := makeStatefulSetWithPVCs(ns, writeCmd, mounts, claims, probe)
-				ss, err := c.AppsV1().StatefulSets(ns).Create(spec)
-				Expect(err).NotTo(HaveOccurred())
-				ssTester.WaitForRunningAndReady(1, ss)
-
-				By("Deleting the StatefulSet but not the volumes")
-				// Scale down to 0 first so that the Delete is quick
-				ss, err = ssTester.Scale(ss, 0)
-				Expect(err).NotTo(HaveOccurred())
-				ssTester.WaitForStatusReplicas(ss, 0)
-				err = c.AppsV1().StatefulSets(ns).Delete(ss.Name, &metav1.DeleteOptions{})
-				Expect(err).NotTo(HaveOccurred())
-
-				By("Creating a new Statefulset and validating the data")
-				validateCmd := "true"
-				for i := 0; i < numVols; i++ {
-					validateCmd += fmt.Sprintf("&& test -f %v", getVolumeFile(i))
-				}
-				validateCmd += "&& sleep 10000"
-
-				spec = makeStatefulSetWithPVCs(ns, validateCmd, mounts, claims, probe)
-				ss, err = c.AppsV1().StatefulSets(ns).Create(spec)
-				Expect(err).NotTo(HaveOccurred())
-				ssTester.WaitForRunningAndReady(1, ss)
+				By("Testing access to pvcs before and after pod recreation on differetn node")
+				testsuites.TestAccessMultipleVolumesAcrossPodRecreation(f, c, ns,
+					framework.NodeSelection{}, claims, false /* sameNode */)
 			})
 		})
 	})

test/e2e/storage/regional_pd.go

@@ -115,7 +115,7 @@ func testVolumeProvisioning(c clientset.Interface, ns string) {
 				err = verifyZonesInPV(volume, sets.NewString(cloudZones...), true /* match */)
 				Expect(err).NotTo(HaveOccurred(), "verifyZonesInPV")
 
-				testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, testsuites.NodeSelection{})
+				testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, framework.NodeSelection{})
 			},
 		},
 		{
@@ -137,7 +137,7 @@ func testVolumeProvisioning(c clientset.Interface, ns string) {
 				err = verifyZonesInPV(volume, zones, false /* match */)
 				Expect(err).NotTo(HaveOccurred(), "verifyZonesInPV")
 
-				testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, testsuites.NodeSelection{})
+				testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, framework.NodeSelection{})
 			},
 		},
 	}

test/e2e/storage/testsuites/BUILD

@@ -5,6 +5,7 @@ go_library(
     srcs = [
         "base.go",
         "driveroperations.go",
+        "multivolume.go",
         "provisioning.go",
         "snapshottable.go",
         "subpath.go",
@@ -28,6 +29,7 @@ go_library(
         "//staging/src/k8s.io/apimachinery/pkg/util/rand:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/storage/names:go_default_library",
         "//staging/src/k8s.io/client-go/dynamic:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
         "//test/e2e/framework:go_default_library",

test/e2e/storage/testsuites/base.go

@@ -188,7 +188,7 @@ func createGenericVolumeTestResource(driver TestDriver, config *PerTestConfig, p
 		if pDriver, ok := driver.(PreprovisionedPVTestDriver); ok {
 			pvSource, volumeNodeAffinity := pDriver.GetPersistentVolumeSource(false, fsType, r.volume)
 			if pvSource != nil {
-				r.volSource, r.pv, r.pvc = createVolumeSourceWithPVCPV(f, dInfo.Name, pvSource, volumeNodeAffinity, false)
+				r.volSource, r.pv, r.pvc = createVolumeSourceWithPVCPV(f, dInfo.Name, pvSource, volumeNodeAffinity, false, pattern.VolMode)
 			}
 			r.volType = fmt.Sprintf("%s-preprovisionedPV", dInfo.Name)
 		}
@@ -205,7 +205,7 @@ func createGenericVolumeTestResource(driver TestDriver, config *PerTestConfig, p
 
 			if r.sc != nil {
 				r.volSource, r.pv, r.pvc = createVolumeSourceWithPVCPVFromDynamicProvisionSC(
-					f, dInfo.Name, claimSize, r.sc, false, nil)
+					f, dInfo.Name, claimSize, r.sc, false, pattern.VolMode)
 			}
 			r.volType = fmt.Sprintf("%s-dynamicPV", dInfo.Name)
 		}
@@ -269,6 +269,7 @@ func createVolumeSourceWithPVCPV(
 	pvSource *v1.PersistentVolumeSource,
 	volumeNodeAffinity *v1.VolumeNodeAffinity,
 	readOnly bool,
+	volMode v1.PersistentVolumeMode,
 ) (*v1.VolumeSource, *v1.PersistentVolume, *v1.PersistentVolumeClaim) {
 	pvConfig := framework.PersistentVolumeConfig{
 		NamePrefix:       fmt.Sprintf("%s-", name),
@@ -276,10 +277,16 @@ func createVolumeSourceWithPVCPV(
 		PVSource:         *pvSource,
 		NodeAffinity:     volumeNodeAffinity,
 	}
+
 	pvcConfig := framework.PersistentVolumeClaimConfig{
 		StorageClassName: &f.Namespace.Name,
 	}
 
+	if volMode != "" {
+		pvConfig.VolumeMode = &volMode
+		pvcConfig.VolumeMode = &volMode
+	}
+
 	framework.Logf("Creating PVC and PV")
 	pv, pvc, err := framework.CreatePVCPV(f.ClientSet, pvConfig, pvcConfig, f.Namespace.Name, false)
 	Expect(err).NotTo(HaveOccurred(), "PVC, PV creation failed")
@@ -302,7 +309,7 @@ func createVolumeSourceWithPVCPVFromDynamicProvisionSC(
 	claimSize string,
 	sc *storagev1.StorageClass,
 	readOnly bool,
-	volMode *v1.PersistentVolumeMode,
+	volMode v1.PersistentVolumeMode,
 ) (*v1.VolumeSource, *v1.PersistentVolume, *v1.PersistentVolumeClaim) {
 	cs := f.ClientSet
 	ns := f.Namespace.Name
@@ -310,8 +317,8 @@ func createVolumeSourceWithPVCPVFromDynamicProvisionSC(
 	By("creating a claim")
 	pvc := getClaim(claimSize, ns)
 	pvc.Spec.StorageClassName = &sc.Name
-	if volMode != nil {
-		pvc.Spec.VolumeMode = volMode
+	if volMode != "" {
+		pvc.Spec.VolumeMode = &volMode
 	}
 
 	var err error

test/e2e/storage/testsuites/driveroperations.go

@@ -22,6 +22,7 @@ import (
 	storagev1 "k8s.io/api/storage/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apiserver/pkg/storage/names"
 	"k8s.io/kubernetes/test/e2e/framework"
 	"k8s.io/kubernetes/test/e2e/storage/testpatterns"
 )
@@ -71,8 +72,8 @@ func GetStorageClass(
 			Kind: "StorageClass",
 		},
 		ObjectMeta: metav1.ObjectMeta{
-			// Name must be unique, so let's base it on namespace name
-			Name: ns + "-" + suffix,
+			// Name must be unique, so let's base it on namespace name and use GenerateName
+			Name: names.SimpleNameGenerator.GenerateName(ns + "-" + suffix),
 		},
 		Provisioner:       provisioner,
 		Parameters:        parameters,

test/e2e/storage/testsuites/provisioning.go

@@ -18,7 +18,6 @@ package testsuites
 
 import (
 	"fmt"
-	"sync"
 	"time"
 
 	. "github.com/onsi/ginkgo"
@@ -180,7 +179,7 @@ func (p *provisioningTestSuite) defineTests(driver TestDriver, pattern testpatte
 			framework.Skipf("need more than one node - skipping")
 		}
 		l.testCase.PvCheck = func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
-			PVMultiNodeCheck(l.cs, claim, volume, NodeSelection{Name: l.config.ClientNodeName})
+			PVMultiNodeCheck(l.cs, claim, volume, framework.NodeSelection{Name: l.config.ClientNodeName})
 		}
 		l.testCase.TestDynamicProvisioning()
 	})
@@ -214,62 +213,14 @@ func (p *provisioningTestSuite) defineTests(driver TestDriver, pattern testpatte
 
 		dc := l.config.Framework.DynamicClient
 		vsc := sDriver.GetSnapshotClass(l.config)
-		dataSource, cleanupFunc := prepareDataSourceForProvisioning(NodeSelection{Name: l.config.ClientNodeName}, l.cs, dc, l.pvc, l.sc, vsc)
+		dataSource, cleanupFunc := prepareDataSourceForProvisioning(framework.NodeSelection{Name: l.config.ClientNodeName}, l.cs, dc, l.pvc, l.sc, vsc)
 		defer cleanupFunc()
 
 		l.pvc.Spec.DataSource = dataSource
 		l.testCase.PvCheck = func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
 			By("checking whether the created volume has the pre-populated data")
 			command := fmt.Sprintf("grep '%s' /mnt/test/initialData", claim.Namespace)
-			RunInPodWithVolume(l.cs, claim.Namespace, claim.Name, "pvc-snapshot-tester", command, NodeSelection{Name: l.config.ClientNodeName})
-		}
-		l.testCase.TestDynamicProvisioning()
-	})
-
-	It("should allow concurrent writes on the same node", func() {
-		if !dInfo.Capabilities[CapMultiPODs] {
-			framework.Skipf("Driver %q does not support multiple concurrent pods - skipping", dInfo.Name)
-		}
-
-		init()
-		defer cleanup()
-
-		l.testCase.PvCheck = func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
-			// We start two pods concurrently on the same node,
-			// using the same PVC. Both wait for other to create a
-			// file before returning. The pods are forced onto the
-			// same node via pod affinity.
-			wg := sync.WaitGroup{}
-			wg.Add(2)
-			firstPodName := "pvc-tester-first"
-			secondPodName := "pvc-tester-second"
-			run := func(podName, command string) {
-				defer GinkgoRecover()
-				defer wg.Done()
-				node := NodeSelection{
-					Name: l.config.ClientNodeName,
-				}
-				if podName == secondPodName {
-					node.Affinity = &v1.Affinity{
-						PodAffinity: &v1.PodAffinity{
-							RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{
-								{LabelSelector: &metav1.LabelSelector{
-									MatchLabels: map[string]string{
-										// Set by RunInPodWithVolume.
-										"app": firstPodName,
-									},
-								},
-									TopologyKey: "kubernetes.io/hostname",
-								},
-							},
-						},
-					}
-				}
-				RunInPodWithVolume(l.cs, claim.Namespace, claim.Name, podName, command, node)
-			}
-			go run(firstPodName, "touch /mnt/test/first && while ! [ -f /mnt/test/second ]; do sleep 1; done")
-			go run(secondPodName, "touch /mnt/test/second && while ! [ -f /mnt/test/first ]; do sleep 1; done")
-			wg.Wait()
+			RunInPodWithVolume(l.cs, claim.Namespace, claim.Name, "pvc-snapshot-tester", command, framework.NodeSelection{Name: l.config.ClientNodeName})
 		}
 		l.testCase.TestDynamicProvisioning()
 	})
@@ -384,7 +335,7 @@ func (t StorageClassTest) TestDynamicProvisioning() *v1.PersistentVolume {
 // persistent across pods.
 //
 // This is a common test that can be called from a StorageClassTest.PvCheck.
-func PVWriteReadSingleNodeCheck(client clientset.Interface, claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume, node NodeSelection) {
+func PVWriteReadSingleNodeCheck(client clientset.Interface, claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume, node framework.NodeSelection) {
 	By(fmt.Sprintf("checking the created volume is writable and has the PV's mount options on node %+v", node))
 	command := "echo 'hello world' > /mnt/test/data"
 	// We give the first pod the secondary responsibility of checking the volume has
@@ -408,7 +359,7 @@ func PVWriteReadSingleNodeCheck(client clientset.Interface, claim *v1.Persistent
 
 	By(fmt.Sprintf("checking the created volume is readable and retains data on the same node %q", actualNodeName))
 	command = "grep 'hello world' /mnt/test/data"
-	RunInPodWithVolume(client, claim.Namespace, claim.Name, "pvc-volume-tester-reader", command, NodeSelection{Name: actualNodeName})
+	RunInPodWithVolume(client, claim.Namespace, claim.Name, "pvc-volume-tester-reader", command, framework.NodeSelection{Name: actualNodeName})
 }
 
 // PVMultiNodeCheck checks that a PV retains data when moved between nodes.
@@ -425,7 +376,7 @@ func PVWriteReadSingleNodeCheck(client clientset.Interface, claim *v1.Persistent
 // persistent across pods and across nodes.
 //
 // This is a common test that can be called from a StorageClassTest.PvCheck.
-func PVMultiNodeCheck(client clientset.Interface, claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume, node NodeSelection) {
+func PVMultiNodeCheck(client clientset.Interface, claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume, node framework.NodeSelection) {
 	Expect(node.Name).To(Equal(""), "this test only works when not locked onto a single node")
 
 	var pod *v1.Pod
@@ -446,30 +397,7 @@ func PVMultiNodeCheck(client clientset.Interface, claim *v1.PersistentVolumeClai
 
 	// Add node-anti-affinity.
 	secondNode := node
-	if secondNode.Affinity == nil {
-		secondNode.Affinity = &v1.Affinity{}
-	}
-	if secondNode.Affinity.NodeAffinity == nil {
-		secondNode.Affinity.NodeAffinity = &v1.NodeAffinity{}
-	}
-	if secondNode.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil {
-		secondNode.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution = &v1.NodeSelector{}
-	}
-	secondNode.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = append(secondNode.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms,
-		v1.NodeSelectorTerm{
-			// https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity warns
-			// that "the value of kubernetes.io/hostname may be the same as the Node name in some environments and a different value in other environments".
-			// So this might be cleaner:
-			// MatchFields: []v1.NodeSelectorRequirement{
-			// 	{Key: "name", Operator: v1.NodeSelectorOpNotIn, Values: []string{actualNodeName}},
-			// },
-			// However, "name", "Name", "ObjectMeta.Name" all got rejected with "not a valid field selector key".
-
-			MatchExpressions: []v1.NodeSelectorRequirement{
-				{Key: "kubernetes.io/hostname", Operator: v1.NodeSelectorOpNotIn, Values: []string{actualNodeName}},
-			},
-		})
-
+	framework.SetAntiAffinity(&secondNode, actualNodeName)
 	By(fmt.Sprintf("checking the created volume is readable and retains data on another node %+v", secondNode))
 	command = "grep 'hello world' /mnt/test/data"
 	if framework.NodeOSDistroIs("windows") {
@@ -573,17 +501,9 @@ func (t StorageClassTest) TestBindingWaitForFirstConsumerMultiPVC(claims []*v1.P
 	return pvs, node
 }
 
-// NodeSelection specifies where to run a pod, using a combination of fixed node name,
-// node selector and/or affinity.
-type NodeSelection struct {
-	Name     string
-	Selector map[string]string
-	Affinity *v1.Affinity
-}
-
 // RunInPodWithVolume runs a command in a pod with given claim mounted to /mnt directory.
 // It starts, checks, collects output and stops it.
-func RunInPodWithVolume(c clientset.Interface, ns, claimName, podName, command string, node NodeSelection) {
+func RunInPodWithVolume(c clientset.Interface, ns, claimName, podName, command string, node framework.NodeSelection) {
 	pod := StartInPodWithVolume(c, ns, claimName, podName, command, node)
 	defer StopPod(c, pod)
 	framework.ExpectNoError(framework.WaitForPodSuccessInNamespaceSlow(c, pod.Name, pod.Namespace))
@@ -591,7 +511,7 @@ func RunInPodWithVolume(c clientset.Interface, ns, claimName, podName, command s
 
 // StartInPodWithVolume starts a command in a pod with given claim mounted to /mnt directory
 // The caller is responsible for checking the pod and deleting it.
-func StartInPodWithVolume(c clientset.Interface, ns, claimName, podName, command string, node NodeSelection) *v1.Pod {
+func StartInPodWithVolume(c clientset.Interface, ns, claimName, podName, command string, node framework.NodeSelection) *v1.Pod {
 	pod := &v1.Pod{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "Pod",
@@ -664,7 +584,7 @@ func verifyPVCsPending(client clientset.Interface, pvcs []*v1.PersistentVolumeCl
 }
 
 func prepareDataSourceForProvisioning(
-	node NodeSelection,
+	node framework.NodeSelection,
 	client clientset.Interface,
 	dynamicClient dynamic.Interface,
 	initClaim *v1.PersistentVolumeClaim,

test/e2e/storage/testsuites/testdriver.go

@@ -121,6 +121,8 @@ const (
 	// - https://github.com/container-storage-interface/spec/issues/178
 	// - NodeStageVolume in the spec
 	CapMultiPODs Capability = "multipods"
+
+	CapRWX Capability = "RWX" // support ReadWriteMany access modes
 )
 
 // DriverInfo represents static information about a TestDriver.

test/e2e/storage/testsuites/volumemode.go

@@ -182,9 +182,9 @@ func (t *volumeModeTestSuite) defineTests(driver TestDriver, pattern testpattern
 				framework.ExpectNoError(framework.WaitOnPVandPVC(l.cs, l.ns.Name, l.pv, l.pvc))
 
 				By("Creating pod")
-				pod, err := framework.CreateSecPodWithNodeName(l.cs, l.ns.Name, []*v1.PersistentVolumeClaim{l.pvc},
+				pod, err := framework.CreateSecPodWithNodeSelection(l.cs, l.ns.Name, []*v1.PersistentVolumeClaim{l.pvc},
 					false, "", false, false, framework.SELinuxLabel,
-					nil, l.config.ClientNodeName, framework.PodStartTimeout)
+					nil, framework.NodeSelection{Name: l.config.ClientNodeName}, framework.PodStartTimeout)
 				defer func() {
 					framework.ExpectNoError(framework.DeletePodWithWait(f, l.cs, pod))
 				}()
@@ -213,9 +213,9 @@ func (t *volumeModeTestSuite) defineTests(driver TestDriver, pattern testpattern
 				framework.ExpectNoError(framework.WaitOnPVandPVC(l.cs, l.ns.Name, l.pv, l.pvc))
 
 				By("Creating pod")
-				pod, err := framework.CreateSecPodWithNodeName(l.cs, l.ns.Name, []*v1.PersistentVolumeClaim{l.pvc},
+				pod, err := framework.CreateSecPodWithNodeSelection(l.cs, l.ns.Name, []*v1.PersistentVolumeClaim{l.pvc},
 					false, "", false, false, framework.SELinuxLabel,
-					nil, l.config.ClientNodeName, framework.PodStartTimeout)
+					nil, framework.NodeSelection{Name: l.config.ClientNodeName}, framework.PodStartTimeout)
 				defer func() {
 					framework.ExpectNoError(framework.DeletePodWithWait(f, l.cs, pod))
 				}()
@@ -273,9 +273,9 @@ func (t *volumeModeTestSuite) defineTests(driver TestDriver, pattern testpattern
 				Expect(err).NotTo(HaveOccurred())
 
 				By("Creating pod")
-				pod, err := framework.CreateSecPodWithNodeName(l.cs, l.ns.Name, []*v1.PersistentVolumeClaim{l.pvc},
+				pod, err := framework.CreateSecPodWithNodeSelection(l.cs, l.ns.Name, []*v1.PersistentVolumeClaim{l.pvc},
 					false, "", false, false, framework.SELinuxLabel,
-					nil, l.config.ClientNodeName, framework.PodStartTimeout)
+					nil, framework.NodeSelection{Name: l.config.ClientNodeName}, framework.PodStartTimeout)
 				defer func() {
 					framework.ExpectNoError(framework.DeletePodWithWait(f, l.cs, pod))
 				}()

test/e2e/storage/utils/utils.go

@@ -17,7 +17,11 @@ limitations under the License.
 package utils
 
 import (
+	"crypto/sha256"
+	"encoding/base64"
 	"fmt"
+	"math/rand"
+	"path/filepath"
 	"strings"
 	"time"
 
@@ -483,3 +487,43 @@ func CheckReadWriteToPath(pod *v1.Pod, volMode v1.PersistentVolumeMode, path str
 		VerifyExecInPodFail(pod, fmt.Sprintf("dd if=/dev/urandom of=%s bs=64 count=1", path), 1)
 	}
 }
+
+func genBinDataFromSeed(len int, seed int64) []byte {
+	binData := make([]byte, len)
+	rand.Seed(seed)
+
+	len, err := rand.Read(binData)
+	if err != nil {
+		fmt.Printf("Error: %v", err)
+	}
+
+	return binData
+}
+
+func CheckReadFromPath(pod *v1.Pod, volMode v1.PersistentVolumeMode, path string, len int, seed int64) {
+	var pathForVolMode string
+	if volMode == v1.PersistentVolumeBlock {
+		pathForVolMode = path
+	} else {
+		pathForVolMode = filepath.Join(path, "file1.txt")
+	}
+
+	sum := sha256.Sum256(genBinDataFromSeed(len, seed))
+
+	VerifyExecInPodSucceed(pod, fmt.Sprintf("dd if=%s bs=%d count=1 | sha256sum", pathForVolMode, len))
+	VerifyExecInPodSucceed(pod, fmt.Sprintf("dd if=%s bs=%d count=1 | sha256sum | grep -Fq %x", pathForVolMode, len, sum))
+}
+
+func CheckWriteToPath(pod *v1.Pod, volMode v1.PersistentVolumeMode, path string, len int, seed int64) {
+	var pathForVolMode string
+	if volMode == v1.PersistentVolumeBlock {
+		pathForVolMode = path
+	} else {
+		pathForVolMode = filepath.Join(path, "file1.txt")
+	}
+
+	encoded := base64.StdEncoding.EncodeToString(genBinDataFromSeed(len, seed))
+
+	VerifyExecInPodSucceed(pod, fmt.Sprintf("echo %s | base64 -d | sha256sum", encoded))
+	VerifyExecInPodSucceed(pod, fmt.Sprintf("echo %s | base64 -d | dd of=%s bs=%d count=1", encoded, pathForVolMode, len))
+}

test/e2e/storage/volume_provisioning.go

@@ -276,7 +276,7 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
 					PvCheck: func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
 						err := checkGCEPD(volume, "pd-ssd")
 						Expect(err).NotTo(HaveOccurred(), "checkGCEPD pd-ssd")
-						testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, testsuites.NodeSelection{})
+						testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, framework.NodeSelection{})
 					},
 				},
 				{
@@ -291,7 +291,7 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
 					PvCheck: func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
 						err := checkGCEPD(volume, "pd-standard")
 						Expect(err).NotTo(HaveOccurred(), "checkGCEPD pd-standard")
-						testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, testsuites.NodeSelection{})
+						testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, framework.NodeSelection{})
 					},
 				},
 				// AWS
@@ -308,7 +308,7 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
 					PvCheck: func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
 						err := checkAWSEBS(volume, "gp2", false)
 						Expect(err).NotTo(HaveOccurred(), "checkAWSEBS gp2")
-						testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, testsuites.NodeSelection{})
+						testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, framework.NodeSelection{})
 					},
 				},
 				{
@@ -324,7 +324,7 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
 					PvCheck: func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
 						err := checkAWSEBS(volume, "io1", false)
 						Expect(err).NotTo(HaveOccurred(), "checkAWSEBS io1")
-						testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, testsuites.NodeSelection{})
+						testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, framework.NodeSelection{})
 					},
 				},
 				{
@@ -339,7 +339,7 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
 					PvCheck: func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
 						err := checkAWSEBS(volume, "sc1", false)
 						Expect(err).NotTo(HaveOccurred(), "checkAWSEBS sc1")
-						testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, testsuites.NodeSelection{})
+						testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, framework.NodeSelection{})
 					},
 				},
 				{
@@ -354,7 +354,7 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
 					PvCheck: func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
 						err := checkAWSEBS(volume, "st1", false)
 						Expect(err).NotTo(HaveOccurred(), "checkAWSEBS st1")
-						testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, testsuites.NodeSelection{})
+						testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, framework.NodeSelection{})
 					},
 				},
 				{
@@ -369,7 +369,7 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
 					PvCheck: func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
 						err := checkAWSEBS(volume, "gp2", true)
 						Expect(err).NotTo(HaveOccurred(), "checkAWSEBS gp2 encrypted")
-						testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, testsuites.NodeSelection{})
+						testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, framework.NodeSelection{})
 					},
 				},
 				// OpenStack generic tests (works on all OpenStack deployments)
@@ -381,7 +381,7 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
 					ClaimSize:      "1.5Gi",
 					ExpectedSize:   "2Gi",
 					PvCheck: func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
-						testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, testsuites.NodeSelection{})
+						testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, framework.NodeSelection{})
 					},
 				},
 				{
@@ -395,7 +395,7 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
 					ClaimSize:    "1.5Gi",
 					ExpectedSize: "2Gi",
 					PvCheck: func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
-						testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, testsuites.NodeSelection{})
+						testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, framework.NodeSelection{})
 					},
 				},
 				// vSphere generic test
@@ -407,7 +407,7 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
 					ClaimSize:      "1.5Gi",
 					ExpectedSize:   "1.5Gi",
 					PvCheck: func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
-						testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, testsuites.NodeSelection{})
+						testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, framework.NodeSelection{})
 					},
 				},
 				// Azure
@@ -419,7 +419,7 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
 					ClaimSize:      "1Gi",
 					ExpectedSize:   "1Gi",
 					PvCheck: func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
-						testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, testsuites.NodeSelection{})
+						testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, framework.NodeSelection{})
 					},
 				},
 			}
@@ -480,7 +480,7 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
 				PvCheck: func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
 					err := checkGCEPD(volume, "pd-standard")
 					Expect(err).NotTo(HaveOccurred(), "checkGCEPD")
-					testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, testsuites.NodeSelection{})
+					testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, framework.NodeSelection{})
 				},
 			}
 			test.Class = newStorageClass(test, ns, "reclaimpolicy")