Merge pull request #51039 from enisoc/deflake-sts-saturate

Automatic merge from submit-queue StatefulSet: Deflake e2e "Saturate" phase. This should reduce one source of flakiness found while investigating #48031. The "Saturate" phase of StatefulSet e2e tests verifies orderly startup by controlling when each Pod is allowed to report Ready. If a Pod unexepectedly goes down during the test, the replacement Pod created by the controller will forget if it was already allowed to report Ready. After this change, the signal that allows each Pod to report Ready is persisted in the Pod's PVC. Thus, the replacement Pod will remember that it was already told to proceed to a Ready state.

Merge pull request #51039 from enisoc/deflake-sts-saturate
a44e538d · Kubernetes Submit Queue · GitHub · 601b6d3e · 3bc76760 · a44e538d
Commit a44e538d authored Aug 22, 2017 by Kubernetes Submit Queue Committed by GitHub Aug 22, 2017
Showing with 8 additions and 8 deletions

statefulset.go test/e2e/apps/statefulset.go +5 -5

statefulset_utils.go test/e2e/framework/statefulset_utils.go +2 -2

statefulset.go test/e2e/upgrades/apps/statefulset.go +1 -1

No files found.
--- a/test/e2e/apps/statefulset.go
+++ b/test/e2e/apps/statefulset.go
@@ -115,7 +115,7 @@ var _ = SIGDescribe("StatefulSet", func() {
 			By("Restarting statefulset " + ss.Name)
 			sst.Restart(ss)
-			sst.Saturate(ss)
+			sst.WaitForRunningAndReady(*ss.Spec.Replicas, ss)
 			By("Verifying statefulset mounted data directory is usable")
 			framework.ExpectNoError(sst.CheckMount(ss, "/data"))
@@ -233,13 +233,13 @@ var _ = SIGDescribe("StatefulSet", func() {
 			sst.DeleteStatefulPodAtIndex(0, ss)
 			By("Confirming stateful pod at index 0 is recreated.")
-			sst.WaitForRunning(2, 0, ss)
+			sst.WaitForRunning(2, 1, ss)
-			By("Deleting unhealthy stateful pod at index 1.")
+			By("Resuming stateful pod at index 1.")
-			sst.DeleteStatefulPodAtIndex(1, ss)
+			sst.ResumeNextPod(ss)
 			By("Confirming all stateful pods in statefulset are created.")
-			sst.Saturate(ss)
+			sst.WaitForRunningAndReady(*ss.Spec.Replicas, ss)
 		})
 		It("should perform rolling updates and roll backs of template modifications", func() {

--- a/test/e2e/framework/statefulset_utils.go
+++ b/test/e2e/framework/statefulset_utils.go
@@ -548,7 +548,7 @@ func (s *StatefulSetTester) RestorePodHttpProbe(ss *apps.StatefulSet, pod *v1.Po
 var pauseProbe = &v1.Probe{
 	Handler: v1.Handler{
-		Exec: &v1.ExecAction{Command: []string{"test", "-f", "/tmp/statefulset-continue"}},
+		Exec: &v1.ExecAction{Command: []string{"test", "-f", "/data/statefulset-continue"}},
 	},
 	PeriodSeconds:    1,
 	SuccessThreshold: 1,
@@ -586,7 +586,7 @@ func (s *StatefulSetTester) ResumeNextPod(ss *apps.StatefulSet) {
 		if resumedPod != "" {
 			Failf("Found multiple paused stateful pods: %v and %v", pod.Name, resumedPod)
 		}
-		_, err := RunHostCmd(pod.Namespace, pod.Name, "touch /tmp/statefulset-continue")
+		_, err := RunHostCmd(pod.Namespace, pod.Name, "touch /data/statefulset-continue")
 		ExpectNoError(err)
 		Logf("Resumed pod %v", pod.Name)
 		resumedPod = pod.Name

--- a/test/e2e/upgrades/apps/statefulset.go
+++ b/test/e2e/upgrades/apps/statefulset.go
@@ -110,5 +110,5 @@ func (t *StatefulSetUpgradeTest) verify() {
 func (t *StatefulSetUpgradeTest) restart() {
 	By("Restarting statefulset " + t.set.Name)
 	t.tester.Restart(t.set)
-	t.tester.Saturate(t.set)
+	t.tester.WaitForRunningAndReady(*t.set.Spec.Replicas, t.set)
 }