Merge pull request #53831 from gnufied/fix-multiattach-error-flake

Automatic merge from submit-queue (batch tested with PRs 51840, 53542, 53857, 53831, 53702). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>. Fix possible flake in multiattach unit test It is possible that by the time we check for multiattach error on node, the reconciler loop may not have processed second volume and hence we are going to retry for multiattach error on node before giving up and marking the test as failed. Fixes https://github.com/openshift/origin/issues/16836

Merge pull request #53831 from gnufied/fix-multiattach-error-flake
dcac65ae · Kubernetes Submit Queue · GitHub · e6e23ae1 · 68d417d7 · dcac65ae
Commit dcac65ae authored Oct 13, 2017 by Kubernetes Submit Queue Committed by GitHub Oct 13, 2017
Hide whitespace changes
Inline Side-by-side

Showing with 24 additions and 8 deletions

reconciler_test.go ...troller/volume/attachdetach/reconciler/reconciler_test.go +24 -8

No files found.
--- a/pkg/controller/volume/attachdetach/reconciler/reconciler_test.go
+++ b/pkg/controller/volume/attachdetach/reconciler/reconciler_test.go
@@ -456,15 +456,9 @@ func Test_Run_OneVolumeAttachAndDetachMultipleNodesWithReadWriteOnce(t *testing.
 	nodesForVolume := asw.GetNodesForVolume(generatedVolumeName)
 	// check if multiattach is marked
-	// at least one volume should be marked with multiattach error
+	// at least one volume+node should be marked with multiattach error
 	nodeAttachedTo := nodesForVolume[0]
-	for _, volumeToAttach := range dsw.GetVolumesToAttach() {
+	waitForMultiAttachErrorOnNode(t, nodeAttachedTo, dsw)
-		if volumeToAttach.NodeName != nodeAttachedTo {
-			if !volumeToAttach.MultiAttachErrorReported {
-				t.Fatalf("Expected volume %q on node %q to have multiattach error", volumeToAttach.VolumeName, volumeToAttach.NodeName)
-			}
-		}
-	}
 	// Act
 	podToDelete := ""
@@ -495,6 +489,28 @@ func Test_Run_OneVolumeAttachAndDetachMultipleNodesWithReadWriteOnce(t *testing.
 	waitForTotalAttachCallCount(t, 2 /* expectedAttachCallCount */, fakePlugin)
 }
+func waitForMultiAttachErrorOnNode(
+	t *testing.T,
+	attachedNode k8stypes.NodeName,
+	dsow cache.DesiredStateOfWorld) {
+	multAttachCheckFunc := func() (bool, error) {
+		for _, volumeToAttach := range dsow.GetVolumesToAttach() {
+			if volumeToAttach.NodeName != attachedNode {
+				if volumeToAttach.MultiAttachErrorReported {
+					return true, nil
+				}
+			}
+		}
+		t.Logf("Warning: MultiAttach error not yet set on Node. Will retry.")
+		return false, nil
+	}
+	err := retryWithExponentialBackOff(100*time.Millisecond, multAttachCheckFunc)
+	if err != nil {
+		t.Fatalf("Timed out waiting for MultiAttach Error to be set on non-attached node")
+	}
+}
 func waitForNewAttacherCallCount(
 	t *testing.T,
 	expectedCallCount int,