Skip to content

K
k3s
Commit b4c88ace authored by Masaki Kimura's avatar Masaki Kimura
Browse files
Options

Add e2e tests for multiAttach

parent 8bde75e6
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 164 additions and 180 deletions
test/e2e/framework/pv_util.go
View file @ b4c88ace
......@@ -93,6 +93,14 @@ type PersistentVolumeClaimConfig struct {
VolumeMode *v1.PersistentVolumeMode
}
// NodeSelection specifies where to run a pod, using a combination of fixed node name,
// node selector and/or affinity.
type NodeSelection struct {
Name string
Selector map[string]string
Affinity *v1.Affinity
}
// Clean up a pv and pvc in a single pv/pvc test case.
// Note: delete errors are appended to []error so that we can attempt to delete both the pvc and pv.
func PVPVCCleanup(c clientset.Interface, ns string, pv *v1.PersistentVolume, pvc *v1.PersistentVolumeClaim) []error {
......@@ -874,14 +882,16 @@ func CreateNginxPod(client clientset.Interface, namespace string, nodeSelector m
// create security pod with given claims
func CreateSecPod(client clientset.Interface, namespace string, pvclaims []*v1.PersistentVolumeClaim, isPrivileged bool, command string, hostIPC bool, hostPID bool, seLinuxLabel *v1.SELinuxOptions, fsGroup *int64, timeout time.Duration) (*v1.Pod, error) {
return CreateSecPodWithNodeName(client, namespace, pvclaims, isPrivileged, command, hostIPC, hostPID, seLinuxLabel, fsGroup, "", timeout)
return CreateSecPodWithNodeSelection(client, namespace, pvclaims, isPrivileged, command, hostIPC, hostPID, seLinuxLabel, fsGroup, NodeSelection{}, timeout)
}
// create security pod with given claims
func CreateSecPodWithNodeName(client clientset.Interface, namespace string, pvclaims []*v1.PersistentVolumeClaim, isPrivileged bool, command string, hostIPC bool, hostPID bool, seLinuxLabel *v1.SELinuxOptions, fsGroup *int64, nodeName string, timeout time.Duration) (*v1.Pod, error) {
func CreateSecPodWithNodeSelection(client clientset.Interface, namespace string, pvclaims []*v1.PersistentVolumeClaim, isPrivileged bool, command string, hostIPC bool, hostPID bool, seLinuxLabel *v1.SELinuxOptions, fsGroup *int64, node NodeSelection, timeout time.Duration) (*v1.Pod, error) {
pod := MakeSecPod(namespace, pvclaims, isPrivileged, command, hostIPC, hostPID, seLinuxLabel, fsGroup)
// Setting nodeName
pod.Spec.NodeName = nodeName
// Setting node
pod.Spec.NodeName = node.Name
pod.Spec.NodeSelector = node.Selector
pod.Spec.Affinity = node.Affinity
pod, err := client.CoreV1().Pods(namespace).Create(pod)
if err != nil {
......@@ -901,6 +911,44 @@ func CreateSecPodWithNodeName(client clientset.Interface, namespace string, pvcl
return pod, nil
}
// SetNodeAffinityRequirement sets affinity with specified operator to nodeName to nodeSelection
func SetNodeAffinityRequirement(nodeSelection *NodeSelection, operator v1.NodeSelectorOperator, nodeName string) {
// Add node-anti-affinity.
if nodeSelection.Affinity == nil {
nodeSelection.Affinity = &v1.Affinity{}
}
if nodeSelection.Affinity.NodeAffinity == nil {
nodeSelection.Affinity.NodeAffinity = &v1.NodeAffinity{}
}
if nodeSelection.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil {
nodeSelection.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution = &v1.NodeSelector{}
}
nodeSelection.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = append(nodeSelection.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms,
v1.NodeSelectorTerm{
// https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity warns
// that "the value of kubernetes.io/hostname may be the same as the Node name in some environments and a different value in other environments".
// So this might be cleaner:
// MatchFields: []v1.NodeSelectorRequirement{
// {Key: "name", Operator: v1.NodeSelectorOpNotIn, Values: []string{nodeName}},
// },
// However, "name", "Name", "ObjectMeta.Name" all got rejected with "not a valid field selector key".
MatchExpressions: []v1.NodeSelectorRequirement{
{Key: "kubernetes.io/hostname", Operator: operator, Values: []string{nodeName}},
},
})
}
// SetAffinity sets affinity to nodeName to nodeSelection
func SetAffinity(nodeSelection *NodeSelection, nodeName string) {
SetNodeAffinityRequirement(nodeSelection, v1.NodeSelectorOpIn, nodeName)
}
// SetAntiAffinity sets anti-affinity to nodeName to nodeSelection
func SetAntiAffinity(nodeSelection *NodeSelection, nodeName string) {
SetNodeAffinityRequirement(nodeSelection, v1.NodeSelectorOpNotIn, nodeName)
}
// Define and create a pod with a mounted PV. Pod runs infinite loop until killed.
func CreateClientPod(c clientset.Interface, ns string, pvc *v1.PersistentVolumeClaim) (*v1.Pod, error) {
return CreatePod(c, ns, nil, []*v1.PersistentVolumeClaim{pvc}, true, "")
......
test/e2e/storage/csi_mock_volume.go
View file @ b4c88ace
......@@ -153,12 +153,12 @@ var _ = utils.SIGDescribe("CSI mock volume", func() {
scTest.AllowVolumeExpansion = true
}
nodeSelection := testsuites.NodeSelection{
nodeSelection := framework.NodeSelection{
// The mock driver only works when everything runs on a single node.
Name: nodeName,
}
if len(m.nodeLabel) > 0 {
nodeSelection = testsuites.NodeSelection{
nodeSelection = framework.NodeSelection{
Selector: m.nodeLabel,
}
}
......@@ -177,11 +177,11 @@ var _ = utils.SIGDescribe("CSI mock volume", func() {
createPodWithPVC := func(pvc *v1.PersistentVolumeClaim) (*v1.Pod, error) {
nodeName := m.config.ClientNodeName
nodeSelection := testsuites.NodeSelection{
nodeSelection := framework.NodeSelection{
Name: nodeName,
}
if len(m.nodeLabel) > 0 {
nodeSelection = testsuites.NodeSelection{
nodeSelection = framework.NodeSelection{
Selector: m.nodeLabel,
}
}
......@@ -597,7 +597,7 @@ func checkNodeForLimits(nodeName string, attachKey v1.ResourceName, cs clientset
return int(attachLimit), waitErr
}
func startPausePod(cs clientset.Interface, t testsuites.StorageClassTest, node testsuites.NodeSelection, ns string) (*storagev1.StorageClass, *v1.PersistentVolumeClaim, *v1.Pod) {
func startPausePod(cs clientset.Interface, t testsuites.StorageClassTest, node framework.NodeSelection, ns string) (*storagev1.StorageClass, *v1.PersistentVolumeClaim, *v1.Pod) {
class := newStorageClass(t, ns, "")
var err error
_, err = cs.StorageV1().StorageClasses().Get(class.Name, metav1.GetOptions{})
......@@ -659,7 +659,7 @@ func startPausePod(cs clientset.Interface, t testsuites.StorageClassTest, node t
return class, claim, pod
}
func startPausePodWithClaim(cs clientset.Interface, pvc *v1.PersistentVolumeClaim, node testsuites.NodeSelection, ns string) (*v1.Pod, error) {
func startPausePodWithClaim(cs clientset.Interface, pvc *v1.PersistentVolumeClaim, node framework.NodeSelection, ns string) (*v1.Pod, error) {
pod := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "pvc-volume-tester-",
......
test/e2e/storage/csi_volumes.go
View file @ b4c88ace
......@@ -47,6 +47,7 @@ var csiTestSuites = []func() testsuites.TestSuite{
testsuites.InitSubPathTestSuite,
testsuites.InitProvisioningTestSuite,
testsuites.InitSnapshottableTestSuite,
testsuites.InitMultiVolumeTestSuite,
}
// This executes testSuites for csi volumes.
......@@ -155,7 +156,7 @@ func testTopologyNegative(cs clientset.Interface, suffix, namespace string, dela
test.PvCheck = func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
// Ensure that a pod cannot be scheduled in an unsuitable zone.
pod := testsuites.StartInPodWithVolume(cs, namespace, claim.Name, "pvc-tester-unschedulable", "sleep 100000",
testsuites.NodeSelection{Selector: nodeSelector})
framework.NodeSelection{Selector: nodeSelector})
defer testsuites.StopPod(cs, pod)
framework.ExpectNoError(framework.WaitForPodNameUnschedulableInNamespace(cs, pod.Name, pod.Namespace), "pod should be unschedulable")
}
......
test/e2e/storage/drivers/in_tree.go
View file @ b4c88ace
......@@ -95,6 +95,7 @@ func InitNFSDriver() testsuites.TestDriver {
Capabilities: map[testsuites.Capability]bool{
testsuites.CapPersistence: true,
testsuites.CapExec: true,
testsuites.CapRWX: true,
},
},
}
......@@ -232,6 +233,7 @@ func InitGlusterFSDriver() testsuites.TestDriver {
Capabilities: map[testsuites.Capability]bool{
testsuites.CapPersistence: true,
testsuites.CapExec: true,
testsuites.CapRWX: true,
},
},
}
......@@ -586,6 +588,7 @@ func InitCephFSDriver() testsuites.TestDriver {
Capabilities: map[testsuites.Capability]bool{
testsuites.CapPersistence: true,
testsuites.CapExec: true,
testsuites.CapRWX: true,
},
},
}
......
test/e2e/storage/in_tree_volumes.go
View file @ b4c88ace
......@@ -55,6 +55,7 @@ var testSuites = []func() testsuites.TestSuite{
testsuites.InitVolumeModeTestSuite,
testsuites.InitSubPathTestSuite,
testsuites.InitProvisioningTestSuite,
testsuites.InitMultiVolumeTestSuite,
}
// This executes testSuites for in-tree volumes.
......
test/e2e/storage/persistent_volumes.go
View file @ b4c88ace
......@@ -30,6 +30,7 @@ import (
utilerrors "k8s.io/apimachinery/pkg/util/errors"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/test/e2e/framework"
"k8s.io/kubernetes/test/e2e/storage/testsuites"
"k8s.io/kubernetes/test/e2e/storage/utils"
imageutils "k8s.io/kubernetes/test/utils/image"
)
......@@ -308,68 +309,22 @@ var _ = utils.SIGDescribe("PersistentVolumes", func() {
Describe("Default StorageClass", func() {
Context("pods that use multiple volumes", func() {
AfterEach(func() {
framework.DeleteAllStatefulSets(c, ns)
})
It("should be reschedulable [Slow]", func() {
// Only run on providers with default storageclass
framework.SkipUnlessProviderIs("openstack", "gce", "gke", "vsphere", "azure")
numVols := 4
ssTester := framework.NewStatefulSetTester(c)
By("Creating a StatefulSet pod to initialize data")
writeCmd := "true"
for i := 0; i < numVols; i++ {
writeCmd += fmt.Sprintf("&& touch %v", getVolumeFile(i))
}
writeCmd += "&& sleep 10000"
probe := &v1.Probe{
Handler: v1.Handler{
Exec: &v1.ExecAction{
// Check that the last file got created
Command: []string{"test", "-f", getVolumeFile(numVols - 1)},
},
},
InitialDelaySeconds: 1,
PeriodSeconds: 1,
}
mounts := []v1.VolumeMount{}
claims := []v1.PersistentVolumeClaim{}
By("Creating pvcs")
claims := []*v1.PersistentVolumeClaim{}
for i := 0; i < numVols; i++ {
pvc := framework.MakePersistentVolumeClaim(framework.PersistentVolumeClaimConfig{}, ns)
pvc.Name = getVolName(i)
mounts = append(mounts, v1.VolumeMount{Name: pvc.Name, MountPath: getMountPath(i)})
claims = append(claims, *pvc)
claims = append(claims, pvc)
}
spec := makeStatefulSetWithPVCs(ns, writeCmd, mounts, claims, probe)
ss, err := c.AppsV1().StatefulSets(ns).Create(spec)
Expect(err).NotTo(HaveOccurred())
ssTester.WaitForRunningAndReady(1, ss)
By("Deleting the StatefulSet but not the volumes")
// Scale down to 0 first so that the Delete is quick
ss, err = ssTester.Scale(ss, 0)
Expect(err).NotTo(HaveOccurred())
ssTester.WaitForStatusReplicas(ss, 0)
err = c.AppsV1().StatefulSets(ns).Delete(ss.Name, &metav1.DeleteOptions{})
Expect(err).NotTo(HaveOccurred())
By("Creating a new Statefulset and validating the data")
validateCmd := "true"
for i := 0; i < numVols; i++ {
validateCmd += fmt.Sprintf("&& test -f %v", getVolumeFile(i))
}
validateCmd += "&& sleep 10000"
spec = makeStatefulSetWithPVCs(ns, validateCmd, mounts, claims, probe)
ss, err = c.AppsV1().StatefulSets(ns).Create(spec)
Expect(err).NotTo(HaveOccurred())
ssTester.WaitForRunningAndReady(1, ss)
By("Testing access to pvcs before and after pod recreation on differetn node")
testsuites.TestAccessMultipleVolumesAcrossPodRecreation(f, c, ns,
framework.NodeSelection{}, claims, false /* sameNode */)
})
})
})
......
test/e2e/storage/regional_pd.go
View file @ b4c88ace
......@@ -116,7 +116,7 @@ func testVolumeProvisioning(c clientset.Interface, ns string) {
err = verifyZonesInPV(volume, sets.NewString(cloudZones...), true /* match */)
Expect(err).NotTo(HaveOccurred(), "verifyZonesInPV")
testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, testsuites.NodeSelection{})
testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, framework.NodeSelection{})
},
},
{
......@@ -138,7 +138,7 @@ func testVolumeProvisioning(c clientset.Interface, ns string) {
err = verifyZonesInPV(volume, zones, false /* match */)
Expect(err).NotTo(HaveOccurred(), "verifyZonesInPV")
testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, testsuites.NodeSelection{})
testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, framework.NodeSelection{})
},
},
}
......
test/e2e/storage/testsuites/BUILD
View file @ b4c88ace
......@@ -5,6 +5,7 @@ go_library(
srcs = [
"base.go",
"driveroperations.go",
"multivolume.go",
"provisioning.go",
"snapshottable.go",
"subpath.go",
......@@ -28,6 +29,7 @@ go_library(
"//staging/src/k8s.io/apimachinery/pkg/util/rand:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/storage/names:go_default_library",
"//staging/src/k8s.io/client-go/dynamic:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//test/e2e/framework:go_default_library",
......
test/e2e/storage/testsuites/base.go
View file @ b4c88ace
......@@ -188,7 +188,7 @@ func createGenericVolumeTestResource(driver TestDriver, config *PerTestConfig, p
if pDriver, ok := driver.(PreprovisionedPVTestDriver); ok {
pvSource, volumeNodeAffinity := pDriver.GetPersistentVolumeSource(false, fsType, r.volume)
if pvSource != nil {
r.volSource, r.pv, r.pvc = createVolumeSourceWithPVCPV(f, dInfo.Name, pvSource, volumeNodeAffinity, false)
r.volSource, r.pv, r.pvc = createVolumeSourceWithPVCPV(f, dInfo.Name, pvSource, volumeNodeAffinity, false, pattern.VolMode)
}
r.volType = fmt.Sprintf("%s-preprovisionedPV", dInfo.Name)
}
......@@ -205,7 +205,7 @@ func createGenericVolumeTestResource(driver TestDriver, config *PerTestConfig, p
if r.sc != nil {
r.volSource, r.pv, r.pvc = createVolumeSourceWithPVCPVFromDynamicProvisionSC(
f, dInfo.Name, claimSize, r.sc, false, nil)
f, dInfo.Name, claimSize, r.sc, false, pattern.VolMode)
}
r.volType = fmt.Sprintf("%s-dynamicPV", dInfo.Name)
}
......@@ -269,6 +269,7 @@ func createVolumeSourceWithPVCPV(
pvSource *v1.PersistentVolumeSource,
volumeNodeAffinity *v1.VolumeNodeAffinity,
readOnly bool,
volMode v1.PersistentVolumeMode,
) (*v1.VolumeSource, *v1.PersistentVolume, *v1.PersistentVolumeClaim) {
pvConfig := framework.PersistentVolumeConfig{
NamePrefix: fmt.Sprintf("%s-", name),
......@@ -276,10 +277,16 @@ func createVolumeSourceWithPVCPV(
PVSource: *pvSource,
NodeAffinity: volumeNodeAffinity,
}
pvcConfig := framework.PersistentVolumeClaimConfig{
StorageClassName: &f.Namespace.Name,
}
if volMode != "" {
pvConfig.VolumeMode = &volMode
pvcConfig.VolumeMode = &volMode
}
framework.Logf("Creating PVC and PV")
pv, pvc, err := framework.CreatePVCPV(f.ClientSet, pvConfig, pvcConfig, f.Namespace.Name, false)
Expect(err).NotTo(HaveOccurred(), "PVC, PV creation failed")
......@@ -302,7 +309,7 @@ func createVolumeSourceWithPVCPVFromDynamicProvisionSC(
claimSize string,
sc *storagev1.StorageClass,
readOnly bool,
volMode *v1.PersistentVolumeMode,
volMode v1.PersistentVolumeMode,
) (*v1.VolumeSource, *v1.PersistentVolume, *v1.PersistentVolumeClaim) {
cs := f.ClientSet
ns := f.Namespace.Name
......@@ -310,8 +317,8 @@ func createVolumeSourceWithPVCPVFromDynamicProvisionSC(
By("creating a claim")
pvc := getClaim(claimSize, ns)
pvc.Spec.StorageClassName = &sc.Name
if volMode != nil {
pvc.Spec.VolumeMode = volMode
if volMode != "" {
pvc.Spec.VolumeMode = &volMode
}
var err error
......
test/e2e/storage/testsuites/driveroperations.go
View file @ b4c88ace
......@@ -22,6 +22,7 @@ import (
storagev1 "k8s.io/api/storage/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apiserver/pkg/storage/names"
"k8s.io/kubernetes/test/e2e/framework"
"k8s.io/kubernetes/test/e2e/storage/testpatterns"
)
......@@ -71,8 +72,8 @@ func GetStorageClass(
Kind: "StorageClass",
},
ObjectMeta: metav1.ObjectMeta{
// Name must be unique, so let's base it on namespace name
Name: ns + "-" + suffix,
// Name must be unique, so let's base it on namespace name and use GenerateName
Name: names.SimpleNameGenerator.GenerateName(ns + "-" + suffix),
},
Provisioner: provisioner,
Parameters: parameters,
......
test/e2e/storage/testsuites/provisioning.go
View file @ b4c88ace
......@@ -18,7 +18,6 @@ package testsuites
import (
"fmt"
"sync"
"time"
. "github.com/onsi/ginkgo"
......@@ -180,7 +179,7 @@ func (p *provisioningTestSuite) defineTests(driver TestDriver, pattern testpatte
framework.Skipf("need more than one node - skipping")
}
l.testCase.PvCheck = func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
PVMultiNodeCheck(l.cs, claim, volume, NodeSelection{Name: l.config.ClientNodeName})
PVMultiNodeCheck(l.cs, claim, volume, framework.NodeSelection{Name: l.config.ClientNodeName})
}
l.testCase.TestDynamicProvisioning()
})
......@@ -214,62 +213,14 @@ func (p *provisioningTestSuite) defineTests(driver TestDriver, pattern testpatte
dc := l.config.Framework.DynamicClient
vsc := sDriver.GetSnapshotClass(l.config)
dataSource, cleanupFunc := prepareDataSourceForProvisioning(NodeSelection{Name: l.config.ClientNodeName}, l.cs, dc, l.pvc, l.sc, vsc)
dataSource, cleanupFunc := prepareDataSourceForProvisioning(framework.NodeSelection{Name: l.config.ClientNodeName}, l.cs, dc, l.pvc, l.sc, vsc)
defer cleanupFunc()
l.pvc.Spec.DataSource = dataSource
l.testCase.PvCheck = func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
By("checking whether the created volume has the pre-populated data")
command := fmt.Sprintf("grep '%s' /mnt/test/initialData", claim.Namespace)
RunInPodWithVolume(l.cs, claim.Namespace, claim.Name, "pvc-snapshot-tester", command, NodeSelection{Name: l.config.ClientNodeName})
}
l.testCase.TestDynamicProvisioning()
})
It("should allow concurrent writes on the same node", func() {
if !dInfo.Capabilities[CapMultiPODs] {
framework.Skipf("Driver %q does not support multiple concurrent pods - skipping", dInfo.Name)
}
init()
defer cleanup()
l.testCase.PvCheck = func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
// We start two pods concurrently on the same node,
// using the same PVC. Both wait for other to create a
// file before returning. The pods are forced onto the
// same node via pod affinity.
wg := sync.WaitGroup{}
wg.Add(2)
firstPodName := "pvc-tester-first"
secondPodName := "pvc-tester-second"
run := func(podName, command string) {
defer GinkgoRecover()
defer wg.Done()
node := NodeSelection{
Name: l.config.ClientNodeName,
}
if podName == secondPodName {
node.Affinity = &v1.Affinity{
PodAffinity: &v1.PodAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{
{LabelSelector: &metav1.LabelSelector{
MatchLabels: map[string]string{
// Set by RunInPodWithVolume.
"app": firstPodName,
},
},
TopologyKey: "kubernetes.io/hostname",
},
},
},
}
}
RunInPodWithVolume(l.cs, claim.Namespace, claim.Name, podName, command, node)
}
go run(firstPodName, "touch /mnt/test/first && while ! [ -f /mnt/test/second ]; do sleep 1; done")
go run(secondPodName, "touch /mnt/test/second && while ! [ -f /mnt/test/first ]; do sleep 1; done")
wg.Wait()
RunInPodWithVolume(l.cs, claim.Namespace, claim.Name, "pvc-snapshot-tester", command, framework.NodeSelection{Name: l.config.ClientNodeName})
}
l.testCase.TestDynamicProvisioning()
})
......@@ -384,7 +335,7 @@ func (t StorageClassTest) TestDynamicProvisioning() *v1.PersistentVolume {
// persistent across pods.
//
// This is a common test that can be called from a StorageClassTest.PvCheck.
func PVWriteReadSingleNodeCheck(client clientset.Interface, claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume, node NodeSelection) {
func PVWriteReadSingleNodeCheck(client clientset.Interface, claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume, node framework.NodeSelection) {
By(fmt.Sprintf("checking the created volume is writable and has the PV's mount options on node %+v", node))
command := "echo 'hello world' > /mnt/test/data"
// We give the first pod the secondary responsibility of checking the volume has
......@@ -408,7 +359,7 @@ func PVWriteReadSingleNodeCheck(client clientset.Interface, claim *v1.Persistent
By(fmt.Sprintf("checking the created volume is readable and retains data on the same node %q", actualNodeName))
command = "grep 'hello world' /mnt/test/data"
RunInPodWithVolume(client, claim.Namespace, claim.Name, "pvc-volume-tester-reader", command, NodeSelection{Name: actualNodeName})
RunInPodWithVolume(client, claim.Namespace, claim.Name, "pvc-volume-tester-reader", command, framework.NodeSelection{Name: actualNodeName})
}
// PVMultiNodeCheck checks that a PV retains data when moved between nodes.
......@@ -425,7 +376,7 @@ func PVWriteReadSingleNodeCheck(client clientset.Interface, claim *v1.Persistent
// persistent across pods and across nodes.
//
// This is a common test that can be called from a StorageClassTest.PvCheck.
func PVMultiNodeCheck(client clientset.Interface, claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume, node NodeSelection) {
func PVMultiNodeCheck(client clientset.Interface, claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume, node framework.NodeSelection) {
Expect(node.Name).To(Equal(""), "this test only works when not locked onto a single node")
var pod *v1.Pod
......@@ -446,30 +397,7 @@ func PVMultiNodeCheck(client clientset.Interface, claim *v1.PersistentVolumeClai
// Add node-anti-affinity.
secondNode := node
if secondNode.Affinity == nil {
secondNode.Affinity = &v1.Affinity{}
}
if secondNode.Affinity.NodeAffinity == nil {
secondNode.Affinity.NodeAffinity = &v1.NodeAffinity{}
}
if secondNode.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil {
secondNode.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution = &v1.NodeSelector{}
}
secondNode.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = append(secondNode.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms,
v1.NodeSelectorTerm{
// https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity warns
// that "the value of kubernetes.io/hostname may be the same as the Node name in some environments and a different value in other environments".
// So this might be cleaner:
// MatchFields: []v1.NodeSelectorRequirement{
// {Key: "name", Operator: v1.NodeSelectorOpNotIn, Values: []string{actualNodeName}},
// },
// However, "name", "Name", "ObjectMeta.Name" all got rejected with "not a valid field selector key".
MatchExpressions: []v1.NodeSelectorRequirement{
{Key: "kubernetes.io/hostname", Operator: v1.NodeSelectorOpNotIn, Values: []string{actualNodeName}},
},
})
framework.SetAntiAffinity(&secondNode, actualNodeName)
By(fmt.Sprintf("checking the created volume is readable and retains data on another node %+v", secondNode))
command = "grep 'hello world' /mnt/test/data"
if framework.NodeOSDistroIs("windows") {
......@@ -573,17 +501,9 @@ func (t StorageClassTest) TestBindingWaitForFirstConsumerMultiPVC(claims []*v1.P
return pvs, node
}
// NodeSelection specifies where to run a pod, using a combination of fixed node name,
// node selector and/or affinity.
type NodeSelection struct {
Name string
Selector map[string]string
Affinity *v1.Affinity
}
// RunInPodWithVolume runs a command in a pod with given claim mounted to /mnt directory.
// It starts, checks, collects output and stops it.
func RunInPodWithVolume(c clientset.Interface, ns, claimName, podName, command string, node NodeSelection) {
func RunInPodWithVolume(c clientset.Interface, ns, claimName, podName, command string, node framework.NodeSelection) {
pod := StartInPodWithVolume(c, ns, claimName, podName, command, node)
defer StopPod(c, pod)
framework.ExpectNoError(framework.WaitForPodSuccessInNamespaceSlow(c, pod.Name, pod.Namespace))
......@@ -591,7 +511,7 @@ func RunInPodWithVolume(c clientset.Interface, ns, claimName, podName, command s
// StartInPodWithVolume starts a command in a pod with given claim mounted to /mnt directory
// The caller is responsible for checking the pod and deleting it.
func StartInPodWithVolume(c clientset.Interface, ns, claimName, podName, command string, node NodeSelection) *v1.Pod {
func StartInPodWithVolume(c clientset.Interface, ns, claimName, podName, command string, node framework.NodeSelection) *v1.Pod {
pod := &v1.Pod{
TypeMeta: metav1.TypeMeta{
Kind: "Pod",
......@@ -664,7 +584,7 @@ func verifyPVCsPending(client clientset.Interface, pvcs []*v1.PersistentVolumeCl
}
func prepareDataSourceForProvisioning(
node NodeSelection,
node framework.NodeSelection,
client clientset.Interface,
dynamicClient dynamic.Interface,
initClaim *v1.PersistentVolumeClaim,
......
test/e2e/storage/testsuites/testdriver.go
View file @ b4c88ace
......@@ -121,6 +121,8 @@ const (
// - https://github.com/container-storage-interface/spec/issues/178
// - NodeStageVolume in the spec
CapMultiPODs Capability = "multipods"
CapRWX Capability = "RWX" // support ReadWriteMany access modes
)
// DriverInfo represents static information about a TestDriver.
......
test/e2e/storage/testsuites/volumemode.go
View file @ b4c88ace
......@@ -182,9 +182,9 @@ func (t *volumeModeTestSuite) defineTests(driver TestDriver, pattern testpattern
framework.ExpectNoError(framework.WaitOnPVandPVC(l.cs, l.ns.Name, l.pv, l.pvc))
By("Creating pod")
pod, err := framework.CreateSecPodWithNodeName(l.cs, l.ns.Name, []*v1.PersistentVolumeClaim{l.pvc},
pod, err := framework.CreateSecPodWithNodeSelection(l.cs, l.ns.Name, []*v1.PersistentVolumeClaim{l.pvc},
false, "", false, false, framework.SELinuxLabel,
nil, l.config.ClientNodeName, framework.PodStartTimeout)
nil, framework.NodeSelection{Name: l.config.ClientNodeName}, framework.PodStartTimeout)
defer func() {
framework.ExpectNoError(framework.DeletePodWithWait(f, l.cs, pod))
}()
......@@ -213,9 +213,9 @@ func (t *volumeModeTestSuite) defineTests(driver TestDriver, pattern testpattern
framework.ExpectNoError(framework.WaitOnPVandPVC(l.cs, l.ns.Name, l.pv, l.pvc))
By("Creating pod")
pod, err := framework.CreateSecPodWithNodeName(l.cs, l.ns.Name, []*v1.PersistentVolumeClaim{l.pvc},
pod, err := framework.CreateSecPodWithNodeSelection(l.cs, l.ns.Name, []*v1.PersistentVolumeClaim{l.pvc},
false, "", false, false, framework.SELinuxLabel,
nil, l.config.ClientNodeName, framework.PodStartTimeout)
nil, framework.NodeSelection{Name: l.config.ClientNodeName}, framework.PodStartTimeout)
defer func() {
framework.ExpectNoError(framework.DeletePodWithWait(f, l.cs, pod))
}()
......@@ -273,9 +273,9 @@ func (t *volumeModeTestSuite) defineTests(driver TestDriver, pattern testpattern
Expect(err).NotTo(HaveOccurred())
By("Creating pod")
pod, err := framework.CreateSecPodWithNodeName(l.cs, l.ns.Name, []*v1.PersistentVolumeClaim{l.pvc},
pod, err := framework.CreateSecPodWithNodeSelection(l.cs, l.ns.Name, []*v1.PersistentVolumeClaim{l.pvc},
false, "", false, false, framework.SELinuxLabel,
nil, l.config.ClientNodeName, framework.PodStartTimeout)
nil, framework.NodeSelection{Name: l.config.ClientNodeName}, framework.PodStartTimeout)
defer func() {
framework.ExpectNoError(framework.DeletePodWithWait(f, l.cs, pod))
}()
......
test/e2e/storage/utils/utils.go
View file @ b4c88ace
......@@ -17,7 +17,11 @@ limitations under the License.
package utils
import (
"crypto/sha256"
"encoding/base64"
"fmt"
"math/rand"
"path/filepath"
"strings"
"time"
......@@ -483,3 +487,43 @@ func CheckReadWriteToPath(pod *v1.Pod, volMode v1.PersistentVolumeMode, path str
VerifyExecInPodFail(pod, fmt.Sprintf("dd if=/dev/urandom of=%s bs=64 count=1", path), 1)
}
}
func genBinDataFromSeed(len int, seed int64) []byte {
binData := make([]byte, len)
rand.Seed(seed)
len, err := rand.Read(binData)
if err != nil {
fmt.Printf("Error: %v", err)
}
return binData
}
func CheckReadFromPath(pod *v1.Pod, volMode v1.PersistentVolumeMode, path string, len int, seed int64) {
var pathForVolMode string
if volMode == v1.PersistentVolumeBlock {
pathForVolMode = path
} else {
pathForVolMode = filepath.Join(path, "file1.txt")
}
sum := sha256.Sum256(genBinDataFromSeed(len, seed))
VerifyExecInPodSucceed(pod, fmt.Sprintf("dd if=%s bs=%d count=1 | sha256sum", pathForVolMode, len))
VerifyExecInPodSucceed(pod, fmt.Sprintf("dd if=%s bs=%d count=1 | sha256sum | grep -Fq %x", pathForVolMode, len, sum))
}
func CheckWriteToPath(pod *v1.Pod, volMode v1.PersistentVolumeMode, path string, len int, seed int64) {
var pathForVolMode string
if volMode == v1.PersistentVolumeBlock {
pathForVolMode = path
} else {
pathForVolMode = filepath.Join(path, "file1.txt")
}
encoded := base64.StdEncoding.EncodeToString(genBinDataFromSeed(len, seed))
VerifyExecInPodSucceed(pod, fmt.Sprintf("echo %s | base64 -d | sha256sum", encoded))
VerifyExecInPodSucceed(pod, fmt.Sprintf("echo %s | base64 -d | dd of=%s bs=%d count=1", encoded, pathForVolMode, len))
}
test/e2e/storage/volume_provisioning.go
View file @ b4c88ace
......@@ -276,7 +276,7 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
PvCheck: func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
err := checkGCEPD(volume, "pd-ssd")
Expect(err).NotTo(HaveOccurred(), "checkGCEPD pd-ssd")
testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, testsuites.NodeSelection{})
testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, framework.NodeSelection{})
},
},
{
......@@ -291,7 +291,7 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
PvCheck: func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
err := checkGCEPD(volume, "pd-standard")
Expect(err).NotTo(HaveOccurred(), "checkGCEPD pd-standard")
testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, testsuites.NodeSelection{})
testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, framework.NodeSelection{})
},
},
// AWS
......@@ -308,7 +308,7 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
PvCheck: func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
err := checkAWSEBS(volume, "gp2", false)
Expect(err).NotTo(HaveOccurred(), "checkAWSEBS gp2")
testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, testsuites.NodeSelection{})
testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, framework.NodeSelection{})
},
},
{
......@@ -324,7 +324,7 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
PvCheck: func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
err := checkAWSEBS(volume, "io1", false)
Expect(err).NotTo(HaveOccurred(), "checkAWSEBS io1")
testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, testsuites.NodeSelection{})
testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, framework.NodeSelection{})
},
},
{
......@@ -339,7 +339,7 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
PvCheck: func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
err := checkAWSEBS(volume, "sc1", false)
Expect(err).NotTo(HaveOccurred(), "checkAWSEBS sc1")
testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, testsuites.NodeSelection{})
testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, framework.NodeSelection{})
},
},
{
......@@ -354,7 +354,7 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
PvCheck: func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
err := checkAWSEBS(volume, "st1", false)
Expect(err).NotTo(HaveOccurred(), "checkAWSEBS st1")
testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, testsuites.NodeSelection{})
testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, framework.NodeSelection{})
},
},
{
......@@ -369,7 +369,7 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
PvCheck: func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
err := checkAWSEBS(volume, "gp2", true)
Expect(err).NotTo(HaveOccurred(), "checkAWSEBS gp2 encrypted")
testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, testsuites.NodeSelection{})
testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, framework.NodeSelection{})
},
},
// OpenStack generic tests (works on all OpenStack deployments)
......@@ -381,7 +381,7 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
ClaimSize: "1.5Gi",
ExpectedSize: "2Gi",
PvCheck: func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, testsuites.NodeSelection{})
testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, framework.NodeSelection{})
},
},
{
......@@ -395,7 +395,7 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
ClaimSize: "1.5Gi",
ExpectedSize: "2Gi",
PvCheck: func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, testsuites.NodeSelection{})
testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, framework.NodeSelection{})
},
},
// vSphere generic test
......@@ -407,7 +407,7 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
ClaimSize: "1.5Gi",
ExpectedSize: "1.5Gi",
PvCheck: func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, testsuites.NodeSelection{})
testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, framework.NodeSelection{})
},
},
// Azure
......@@ -419,7 +419,7 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
ClaimSize: "1Gi",
ExpectedSize: "1Gi",
PvCheck: func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, testsuites.NodeSelection{})
testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, framework.NodeSelection{})
},
},
}
......@@ -480,7 +480,7 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
PvCheck: func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
err := checkGCEPD(volume, "pd-standard")
Expect(err).NotTo(HaveOccurred(), "checkGCEPD")
testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, testsuites.NodeSelection{})
testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, framework.NodeSelection{})
},
}
test.Class = newStorageClass(test, ns, "reclaimpolicy")
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment