Merge pull request #16766 from saad-ali/fixPDTestLongTimeout

test/e2e/framework.go

@@ -136,6 +136,12 @@ func (f *Framework) WaitForPodRunning(podName string) error {
 	return waitForPodRunningInNamespace(f.Client, podName, f.Namespace.Name)
 }
 
+// WaitForPodRunningSlow waits for the pod to run in the namespace.
+// It has a longer timeout then WaitForPodRunning (util.slowPodStartTimeout).
+func (f *Framework) WaitForPodRunningSlow(podName string) error {
+	return waitForPodRunningInNamespaceSlow(f.Client, podName, f.Namespace.Name)
+}
+
 // Runs the given pod and verifies that the output of exact container matches the desired output.
 func (f *Framework) TestContainerOutput(scenarioName string, pod *api.Pod, containerIndex int, expectedOutput []string) {
 	testContainerOutput(scenarioName, f.Client, pod, containerIndex, expectedOutput, f.Namespace.Name)

test/e2e/pd.go

@@ -91,7 +91,7 @@ var _ = Describe("Pod Disks", func() {
 		_, err = podClient.Create(host0Pod)
 		expectNoError(err, fmt.Sprintf("Failed to create host0Pod: %v", err))
 
-		expectNoError(framework.WaitForPodRunning(host0Pod.Name))
+		expectNoError(framework.WaitForPodRunningSlow(host0Pod.Name))
 
 		testFile := "/testpd1/tracker"
 		testFileContents := fmt.Sprintf("%v", mathrand.Int())
@@ -106,7 +106,7 @@ var _ = Describe("Pod Disks", func() {
 		_, err = podClient.Create(host1Pod)
 		expectNoError(err, "Failed to create host1Pod")
 
-		expectNoError(framework.WaitForPodRunning(host1Pod.Name))
+		expectNoError(framework.WaitForPodRunningSlow(host1Pod.Name))
 
 		v, err := framework.ReadFileViaContainer(host1Pod.Name, containerName, testFile)
 		expectNoError(err)
@@ -150,7 +150,7 @@ var _ = Describe("Pod Disks", func() {
 		By("submitting rwPod to ensure PD is formatted")
 		_, err = podClient.Create(rwPod)
 		expectNoError(err, "Failed to create rwPod")
-		expectNoError(framework.WaitForPodRunning(rwPod.Name))
+		expectNoError(framework.WaitForPodRunningSlow(rwPod.Name))
 		expectNoError(podClient.Delete(rwPod.Name, api.NewDeleteOptions(0)), "Failed to delete host0Pod")
 		expectNoError(waitForPDDetach(diskName, host0Name))
 
@@ -162,9 +162,9 @@ var _ = Describe("Pod Disks", func() {
 		_, err = podClient.Create(host1ROPod)
 		expectNoError(err, "Failed to create host1ROPod")
 
-		expectNoError(framework.WaitForPodRunning(host0ROPod.Name))
+		expectNoError(framework.WaitForPodRunningSlow(host0ROPod.Name))
 
-		expectNoError(framework.WaitForPodRunning(host1ROPod.Name))
+		expectNoError(framework.WaitForPodRunningSlow(host1ROPod.Name))
 
 		By("deleting host0ROPod")
 		expectNoError(podClient.Delete(host0ROPod.Name, api.NewDeleteOptions(0)), "Failed to delete host0ROPod")
@@ -204,7 +204,7 @@ var _ = Describe("Pod Disks", func() {
 			_, err = podClient.Create(host0Pod)
 			expectNoError(err, fmt.Sprintf("Failed to create host0Pod: %v", err))
 
-			expectNoError(framework.WaitForPodRunning(host0Pod.Name))
+			expectNoError(framework.WaitForPodRunningSlow(host0Pod.Name))
 
 			// randomly select a container and read/verify pd contents from it
 			containerName := fmt.Sprintf("mycontainer%v", mathrand.Intn(numContainers)+1)
@@ -263,7 +263,7 @@ var _ = Describe("Pod Disks", func() {
 			_, err = podClient.Create(host0Pod)
 			expectNoError(err, fmt.Sprintf("Failed to create host0Pod: %v", err))
 
-			expectNoError(framework.WaitForPodRunning(host0Pod.Name))
+			expectNoError(framework.WaitForPodRunningSlow(host0Pod.Name))
 
 			// Read/verify pd contents for both disks from container
 			verifyPDContentsViaContainer(framework, host0Pod.Name, containerName, fileAndContentToVerify)

test/e2e/util.go

@@ -64,6 +64,9 @@ const (
 	// TODO: Make this 30 seconds once #4566 is resolved.
 	podStartTimeout = 5 * time.Minute
 
+	// Some pods can take much longer to get ready due to volume attach/detach latency.
+	slowPodStartTimeout = 15 * time.Minute
+
 	// How long to wait for a service endpoint to be resolvable.
 	serviceStartTimeout = 1 * time.Minute
 
@@ -575,8 +578,20 @@ func deleteNS(c *client.Client, namespace string, timeout time.Duration) error {
 	return nil
 }
 
+// Waits default ammount of time (podStartTimeout) for the specified pod to become running.
+// Returns an error if timeout occurs first, or pod goes in to failed state.
 func waitForPodRunningInNamespace(c *client.Client, podName string, namespace string) error {
-	return waitForPodCondition(c, namespace, podName, "running", podStartTimeout, func(pod *api.Pod) (bool, error) {
+	return waitTimeoutForPodRunningInNamespace(c, podName, namespace, podStartTimeout)
+}
+
+// Waits an extended ammount of time (slowPodStartTimeout) for the specified pod to become running.
+// Returns an error if timeout occurs first, or pod goes in to failed state.
+func waitForPodRunningInNamespaceSlow(c *client.Client, podName string, namespace string) error {
+	return waitTimeoutForPodRunningInNamespace(c, podName, namespace, slowPodStartTimeout)
+}
+
+func waitTimeoutForPodRunningInNamespace(c *client.Client, podName string, namespace string, timeout time.Duration) error {
+	return waitForPodCondition(c, namespace, podName, "running", timeout, func(pod *api.Pod) (bool, error) {
 		if pod.Status.Phase == api.PodRunning {
 			Logf("Found pod '%s' on node '%s'", podName, pod.Spec.NodeName)
 			return true, nil