Adds e2e test for the VMware vpxd restart scenario

test/e2e/storage/vsphere/BUILD

@@ -33,6 +33,7 @@ go_library(
         "vsphere_volume_ops_storm.go",
         "vsphere_volume_perf.go",
         "vsphere_volume_placement.go",
+        "vsphere_volume_vpxd_restart.go",
         "vsphere_volume_vsan_policy.go",
     ],
     importpath = "k8s.io/kubernetes/test/e2e/storage/vsphere",

test/e2e/storage/vsphere/vsphere_utils.go

@@ -23,6 +23,7 @@ import (
 	"time"
 
 	"github.com/golang/glog"
+	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	"github.com/vmware/govmomi/object"
 
@@ -374,7 +375,7 @@ func getVSpherePodSpecWithVolumePaths(volumePaths []string, keyValuelabel map[st
 	return pod
 }
 
-func verifyFilesExistOnVSphereVolume(namespace string, podName string, filePaths []string) {
+func verifyFilesExistOnVSphereVolume(namespace string, podName string, filePaths ...string) {
 	for _, filePath := range filePaths {
 		_, err := framework.RunKubectl("exec", fmt.Sprintf("--namespace=%s", namespace), podName, "--", "/bin/ls", filePath)
 		Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("failed to verify file: %q on the pod: %q", filePath, podName))
@@ -756,3 +757,72 @@ func GetReadySchedulableRandomNodeInfo() *NodeInfo {
 	Expect(nodesInfo).NotTo(BeEmpty())
 	return nodesInfo[rand.Int()%len(nodesInfo)]
 }
+
+// invokeVCenterServiceControl invokes the given command for the given service
+// via service-control on the given vCenter host over SSH.
+func invokeVCenterServiceControl(command, service, host string) error {
+	sshCmd := fmt.Sprintf("service-control --%s %s", command, service)
+	framework.Logf("Invoking command %v on vCenter host %v", sshCmd, host)
+	result, err := framework.SSH(sshCmd, host, framework.TestContext.Provider)
+	if err != nil || result.Code != 0 {
+		framework.LogSSHResult(result)
+		return fmt.Errorf("couldn't execute command: %s on vCenter host: %v", sshCmd, err)
+	}
+	return nil
+}
+
+// expectVolumeToBeAttached checks if the given Volume is attached to the given
+// Node, else fails.
+func expectVolumeToBeAttached(nodeName, volumePath string) {
+	isAttached, err := diskIsAttached(volumePath, nodeName)
+	Expect(err).NotTo(HaveOccurred())
+	Expect(isAttached).To(BeTrue(), fmt.Sprintf("disk: %s is not attached with the node", volumePath))
+}
+
+// expectVolumesToBeAttached checks if the given Volumes are attached to the
+// corresponding set of Nodes, else fails.
+func expectVolumesToBeAttached(pods []*v1.Pod, volumePaths []string) {
+	for i, pod := range pods {
+		nodeName := pod.Spec.NodeName
+		volumePath := volumePaths[i]
+		By(fmt.Sprintf("Verifying that volume %v is attached to node %v", volumePath, nodeName))
+		expectVolumeToBeAttached(nodeName, volumePath)
+	}
+}
+
+// expectFilesToBeAccessible checks if the given files are accessible on the
+// corresponding set of Nodes, else fails.
+func expectFilesToBeAccessible(namespace string, pods []*v1.Pod, filePaths []string) {
+	for i, pod := range pods {
+		podName := pod.Name
+		filePath := filePaths[i]
+		By(fmt.Sprintf("Verifying that file %v is accessible on pod %v", filePath, podName))
+		verifyFilesExistOnVSphereVolume(namespace, podName, filePath)
+	}
+}
+
+// writeContentToPodFile writes the given content to the specified file.
+func writeContentToPodFile(namespace, podName, filePath, content string) error {
+	_, err := framework.RunKubectl("exec", fmt.Sprintf("--namespace=%s", namespace), podName,
+		"--", "/bin/sh", "-c", fmt.Sprintf("echo '%s' > %s", content, filePath))
+	return err
+}
+
+// expectFileContentToMatch checks if a given file contains the specified
+// content, else fails.
+func expectFileContentToMatch(namespace, podName, filePath, content string) {
+	_, err := framework.RunKubectl("exec", fmt.Sprintf("--namespace=%s", namespace), podName,
+		"--", "/bin/sh", "-c", fmt.Sprintf("grep '%s' %s", content, filePath))
+	Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("failed to match content of file: %q on the pod: %q", filePath, podName))
+}
+
+// expectFileContentsToMatch checks if the given contents match the ones present
+// in corresponding files on respective Pods, else fails.
+func expectFileContentsToMatch(namespace string, pods []*v1.Pod, filePaths []string, contents []string) {
+	for i, pod := range pods {
+		podName := pod.Name
+		filePath := filePaths[i]
+		By(fmt.Sprintf("Matching file content for %v on pod %v", filePath, podName))
+		expectFileContentToMatch(namespace, podName, filePath, contents[i])
+	}
+}

test/e2e/storage/vsphere/vsphere_volume_cluster_ds.go

@@ -17,7 +17,6 @@ limitations under the License.
 package vsphere
 
 import (
-	"fmt"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -98,9 +97,7 @@ var _ = utils.SIGDescribe("Volume Provisioning On Clustered Datastore [Feature:v
 		nodeName := pod.Spec.NodeName
 
 		By("Verifying volume is attached")
-		isAttached, err := diskIsAttached(volumePath, nodeName)
-		Expect(err).NotTo(HaveOccurred())
-		Expect(isAttached).To(BeTrue(), fmt.Sprintf("disk: %s is not attached with the node: %v", volumePath, nodeName))
+		expectVolumeToBeAttached(nodeName, volumePath)
 
 		By("Deleting pod")
 		err = framework.DeletePodWithWait(f, client, pod)

test/e2e/storage/vsphere/vsphere_volume_master_restart.go

@@ -103,10 +103,7 @@ var _ = utils.SIGDescribe("Volume Attach Verify [Feature:vsphere][Serial][Disrup
 
 			nodeName := pod.Spec.NodeName
 			By(fmt.Sprintf("Verify volume %s is attached to the pod %s", volumePath, nodeName))
-			isAttached, err := diskIsAttached(volumePath, nodeName)
-			Expect(err).NotTo(HaveOccurred())
-			Expect(isAttached).To(BeTrue(), fmt.Sprintf("disk: %s is not attached with the node", volumePath))
-
+			expectVolumeToBeAttached(nodeName, volumePath)
 		}
 
 		By("Restarting kubelet on master node")
@@ -121,10 +118,9 @@ var _ = utils.SIGDescribe("Volume Attach Verify [Feature:vsphere][Serial][Disrup
 		for i, pod := range pods {
 			volumePath := volumePaths[i]
 			nodeName := pod.Spec.NodeName
+
 			By(fmt.Sprintf("After master restart, verify volume %v is attached to the pod %v", volumePath, nodeName))
-			isAttached, err := diskIsAttached(volumePaths[i], nodeName)
-			Expect(err).NotTo(HaveOccurred())
-			Expect(isAttached).To(BeTrue(), fmt.Sprintf("disk: %s is not attached with the node", volumePath))
+			expectVolumeToBeAttached(nodeName, volumePath)
 
 			By(fmt.Sprintf("Deleting pod on node %s", nodeName))
 			err = framework.DeletePodWithWait(f, client, pod)

test/e2e/storage/vsphere/vsphere_volume_placement.go

@@ -320,9 +320,9 @@ var _ = utils.SIGDescribe("Volume Placement", func() {
 
 			// Verify newly and previously created files present on the volume mounted on the pod
 			By("Verify newly Created file and previously created files present on volume mounted on pod-A")
-			verifyFilesExistOnVSphereVolume(ns, podA.Name, podAFiles)
+			verifyFilesExistOnVSphereVolume(ns, podA.Name, podAFiles...)
 			By("Verify newly Created file and previously created files present on volume mounted on pod-B")
-			verifyFilesExistOnVSphereVolume(ns, podB.Name, podBFiles)
+			verifyFilesExistOnVSphereVolume(ns, podB.Name, podBFiles...)
 
 			By("Deleting pod-A")
 			framework.ExpectNoError(framework.DeletePodWithWait(f, c, podA), "Failed to delete pod ", podA.Name)
@@ -378,7 +378,7 @@ func createAndVerifyFilesOnVolume(namespace string, podname string, newEmptyfile
 
 	// Verify newly and previously created files present on the volume mounted on the pod
 	By(fmt.Sprintf("Verify newly Created file and previously created files present on volume mounted on: %v", podname))
-	verifyFilesExistOnVSphereVolume(namespace, podname, filesToCheck)
+	verifyFilesExistOnVSphereVolume(namespace, podname, filesToCheck...)
 }
 
 func deletePodAndWaitForVolumeToDetach(f *framework.Framework, c clientset.Interface, pod *v1.Pod, nodeName string, volumePaths []string) {

test/e2e/storage/vsphere/vsphere_volume_vpxd_restart.go

+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package vsphere
+
+import (
+	"fmt"
+	"strconv"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/uuid"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/storage/utils"
+)
+
+/*
+	Test to verify that a volume remains attached through vpxd restart.
+
+	For the number of schedulable nodes:
+	1. Create a Volume with default options.
+	2. Create a Pod with the created Volume.
+	3. Verify that the Volume is attached.
+	4. Create a file with random contents under the Volume's mount point on the Pod.
+	5. Stop the vpxd service on the vCenter host.
+	6. Verify that the file is accessible on the Pod and that it's contents match.
+	7. Start the vpxd service on the vCenter host.
+	8. Verify that the Volume remains attached, the file is accessible on the Pod, and that it's contents match.
+	9. Delete the Pod and wait for the Volume to be detached.
+	10. Delete the Volume.
+*/
+var _ = utils.SIGDescribe("Verify Volume Attach Through vpxd Restart [Feature:vsphere][Serial][Disruptive]", func() {
+	f := framework.NewDefaultFramework("restart-vpxd")
+
+	type node struct {
+		name     string
+		kvLabels map[string]string
+		nodeInfo *NodeInfo
+	}
+
+	const (
+		labelKey        = "vsphere_e2e_label_vpxd_restart"
+		vpxdServiceName = "vmware-vpxd"
+	)
+
+	var (
+		client     clientset.Interface
+		namespace  string
+		vcNodesMap map[string][]node
+	)
+
+	BeforeEach(func() {
+		framework.SkipUnlessProviderIs("vsphere")
+		Bootstrap(f)
+		client = f.ClientSet
+		namespace = f.Namespace.Name
+		framework.ExpectNoError(framework.WaitForAllNodesSchedulable(client, framework.TestContext.NodeSchedulableTimeout))
+
+		nodes := framework.GetReadySchedulableNodesOrDie(client)
+		numNodes := len(nodes.Items)
+		if numNodes < 1 {
+			framework.Skipf("Requires at least %d nodes (not %d)", 1, len(nodes.Items))
+		}
+
+		vcNodesMap = make(map[string][]node)
+		for i := 0; i < numNodes; i++ {
+			nodeInfo := TestContext.NodeMapper.GetNodeInfo(nodes.Items[i].Name)
+			nodeName := nodes.Items[i].Name
+			nodeLabel := "vsphere_e2e_" + string(uuid.NewUUID())
+			framework.AddOrUpdateLabelOnNode(client, nodeName, labelKey, nodeLabel)
+
+			vcHost := nodeInfo.VSphere.Config.Hostname
+			vcNodesMap[vcHost] = append(vcNodesMap[vcHost], node{
+				name:     nodeName,
+				kvLabels: map[string]string{labelKey: nodeLabel},
+				nodeInfo: nodeInfo,
+			})
+		}
+	})
+
+	It("verify volume remains attached through vpxd restart", func() {
+		for vcHost, nodes := range vcNodesMap {
+			var (
+				volumePaths  []string
+				filePaths    []string
+				fileContents []string
+				pods         []*v1.Pod
+			)
+
+			framework.Logf("Testing for nodes on vCenter host: %s", vcHost)
+
+			for i, node := range nodes {
+				By(fmt.Sprintf("Creating test vsphere volume %d", i))
+				volumePath, err := node.nodeInfo.VSphere.CreateVolume(&VolumeOptions{}, node.nodeInfo.DataCenterRef)
+				Expect(err).NotTo(HaveOccurred())
+				volumePaths = append(volumePaths, volumePath)
+
+				By(fmt.Sprintf("Creating pod %d on node %v", i, node.name))
+				podspec := getVSpherePodSpecWithVolumePaths([]string{volumePath}, node.kvLabels, nil)
+				pod, err := client.CoreV1().Pods(namespace).Create(podspec)
+				Expect(err).NotTo(HaveOccurred())
+
+				By(fmt.Sprintf("Waiting for pod %d to be ready", i))
+				Expect(framework.WaitForPodNameRunningInNamespace(client, pod.Name, namespace)).To(Succeed())
+
+				pod, err = client.CoreV1().Pods(namespace).Get(pod.Name, metav1.GetOptions{})
+				Expect(err).NotTo(HaveOccurred())
+				pods = append(pods, pod)
+
+				nodeName := pod.Spec.NodeName
+				By(fmt.Sprintf("Verifying that volume %v is attached to node %v", volumePath, nodeName))
+				expectVolumeToBeAttached(nodeName, volumePath)
+
+				By(fmt.Sprintf("Creating a file with random content on the volume mounted on pod %d", i))
+				filePath := fmt.Sprintf("/mnt/volume1/%v_vpxd_restart_test_%v.txt", namespace, strconv.FormatInt(time.Now().UnixNano(), 10))
+				randomContent := fmt.Sprintf("Random Content -- %v", strconv.FormatInt(time.Now().UnixNano(), 10))
+				err = writeContentToPodFile(namespace, pod.Name, filePath, randomContent)
+				Expect(err).NotTo(HaveOccurred())
+				filePaths = append(filePaths, filePath)
+				fileContents = append(fileContents, randomContent)
+			}
+
+			By("Stopping vpxd on the vCenter host")
+			vcAddress := vcHost + ":22"
+			err := invokeVCenterServiceControl("stop", vpxdServiceName, vcAddress)
+			Expect(err).NotTo(HaveOccurred(), "Unable to stop vpxd on the vCenter host")
+
+			expectFilesToBeAccessible(namespace, pods, filePaths)
+			expectFileContentsToMatch(namespace, pods, filePaths, fileContents)
+
+			By("Starting vpxd on the vCenter host")
+			err = invokeVCenterServiceControl("start", vpxdServiceName, vcAddress)
+			Expect(err).NotTo(HaveOccurred(), "Unable to start vpxd on the vCenter host")
+
+			expectVolumesToBeAttached(pods, volumePaths)
+			expectFilesToBeAccessible(namespace, pods, filePaths)
+			expectFileContentsToMatch(namespace, pods, filePaths, fileContents)
+
+			for i, node := range nodes {
+				pod := pods[i]
+				nodeName := pod.Spec.NodeName
+				volumePath := volumePaths[i]
+
+				By(fmt.Sprintf("Deleting pod on node %s", nodeName))
+				err = framework.DeletePodWithWait(f, client, pod)
+				Expect(err).NotTo(HaveOccurred())
+
+				By(fmt.Sprintf("Waiting for volume %s to be detached from node %s", volumePath, nodeName))
+				err = waitForVSphereDiskToDetach(volumePath, nodeName)
+				Expect(err).NotTo(HaveOccurred())
+
+				By(fmt.Sprintf("Deleting volume %s", volumePath))
+				err = node.nodeInfo.VSphere.DeleteVolume(volumePath, node.nodeInfo.DataCenterRef)
+				Expect(err).NotTo(HaveOccurred())
+			}
+		}
+	})
+})