Conformance: Add StatefulSet tests.

test/conformance/testdata/conformance.txt

 test/e2e/apimachinery/custom_resource_definition.go: "creating/deleting custom resource definition objects works "
 test/e2e/apps/rc.go: "should serve a basic image on each replica with a public image "
 test/e2e/apps/replica_set.go: "should serve a basic image on each replica with a public image "
+test/e2e/apps/statefulset.go: "should perform rolling updates and roll backs of template modifications"
+test/e2e/apps/statefulset.go: "should perform canary updates and phased rolling updates of template modifications"
+test/e2e/apps/statefulset.go: "Scaling should happen in predictable order and halt if any stateful pod is unhealthy"
+test/e2e/apps/statefulset.go: "Burst scaling should run to completion even with unhealthy pods"
+test/e2e/apps/statefulset.go: "Should recreate evicted statefulset"
 test/e2e/auth/service_accounts.go: "should mount an API token into pods "
 test/e2e/auth/service_accounts.go: "should allow opting out of API token automount "
 test/e2e/common/configmap.go: "should be consumable via environment variable "

test/e2e/apps/statefulset.go

@@ -88,6 +88,8 @@ var _ = SIGDescribe("StatefulSet", func() {
 			framework.DeleteAllStatefulSets(c, ns)
 		})
 
+		// This can't be Conformance yet because it depends on a default
+		// StorageClass and a dynamic provisioner.
 		It("should provide basic identity", func() {
 			By("Creating statefulset " + ssName + " in namespace " + ns)
 			*(ss.Spec.Replicas) = 3
@@ -125,6 +127,8 @@ var _ = SIGDescribe("StatefulSet", func() {
 			framework.ExpectNoError(sst.ExecInStatefulPods(ss, cmd))
 		})
 
+		// This can't be Conformance yet because it depends on a default
+		// StorageClass and a dynamic provisioner.
 		It("should adopt matching orphans and release non-matching pods", func() {
 			By("Creating statefulset " + ssName + " in namespace " + ns)
 			*(ss.Spec.Replicas) = 1
@@ -208,6 +212,8 @@ var _ = SIGDescribe("StatefulSet", func() {
 			)).To(Succeed(), "wait for pod %q to be readopted", pod.Name)
 		})
 
+		// This can't be Conformance yet because it depends on a default
+		// StorageClass and a dynamic provisioner.
 		It("should not deadlock when a pod's predecessor fails", func() {
 			By("Creating statefulset " + ssName + " in namespace " + ns)
 			*(ss.Spec.Replicas) = 2
@@ -242,7 +248,15 @@ var _ = SIGDescribe("StatefulSet", func() {
 			sst.WaitForRunningAndReady(*ss.Spec.Replicas, ss)
 		})
 
-		It("should perform rolling updates and roll backs of template modifications", func() {
+		/*
+			Testname: StatefulSet-RollingUpdate
+			Description: StatefulSet MUST support the RollingUpdate strategy to automatically replace Pods
+				one at a time when the Pod template changes. The StatefulSet's status MUST indicate the
+				CurrentRevision and UpdateRevision. If the template is changed to match a prior revision,
+				StatefulSet MUST detect this as a rollback instead of creating a new revision.
+				This test does not depend on a preexisting default StorageClass or a dynamic provisioner.
+		*/
+		framework.ConformanceIt("should perform rolling updates and roll backs of template modifications", func() {
 			By("Creating a new StatefulSet")
 			ss := framework.NewStatefulSet("ss2", ns, headlessSvcName, 3, nil, nil, labels)
 			sst := framework.NewStatefulSetTester(c)
@@ -291,7 +305,7 @@ var _ = SIGDescribe("StatefulSet", func() {
 			ss, pods = sst.WaitForPodReady(ss, pods.Items[1].Name)
 			ss, pods = sst.WaitForRollingUpdate(ss)
 			Expect(ss.Status.CurrentRevision).To(Equal(updateRevision),
-				fmt.Sprintf("StatefulSet %s/%s current revision %s does not equal updste revision %s on update completion",
+				fmt.Sprintf("StatefulSet %s/%s current revision %s does not equal update revision %s on update completion",
 					ss.Namespace,
 					ss.Name,
 					ss.Status.CurrentRevision,
@@ -357,7 +371,14 @@ var _ = SIGDescribe("StatefulSet", func() {
 			}
 		})
 
-		It("should perform canary updates and phased rolling updates of template modifications", func() {
+		/*
+			Testname: StatefulSet-RollingUpdatePartition
+			Description: StatefulSet's RollingUpdate strategy MUST support the Partition parameter for
+				canaries and phased rollouts. If a Pod is deleted while a rolling update is in progress,
+				StatefulSet MUST restore the Pod without violating the Partition.
+				This test does not depend on a preexisting default StorageClass or a dynamic provisioner.
+		*/
+		framework.ConformanceIt("should perform canary updates and phased rolling updates of template modifications", func() {
 			By("Creating a new StaefulSet")
 			ss := framework.NewStatefulSet("ss2", ns, headlessSvcName, 3, nil, nil, labels)
 			sst := framework.NewStatefulSetTester(c)
@@ -421,7 +442,7 @@ var _ = SIGDescribe("StatefulSet", func() {
 						currentRevision))
 			}
 
-			By("By performing a canary update")
+			By("Performing a canary update")
 			ss.Spec.UpdateStrategy = apps.StatefulSetUpdateStrategy{
 				Type: apps.RollingUpdateStatefulSetStrategyType,
 				RollingUpdate: func() *apps.RollingUpdateStatefulSetStrategy {
@@ -566,6 +587,8 @@ var _ = SIGDescribe("StatefulSet", func() {
 
 		})
 
+		// Do not mark this as Conformance.
+		// The legacy OnDelete strategy only exists for backward compatibility with pre-v1 APIs.
 		It("should implement legacy replacement when the update strategy is OnDelete", func() {
 			By("Creating a new StatefulSet")
 			ss := framework.NewStatefulSet("ss2", ns, headlessSvcName, 3, nil, nil, labels)
@@ -646,7 +669,14 @@ var _ = SIGDescribe("StatefulSet", func() {
 			}
 		})
 
-		It("Scaling should happen in predictable order and halt if any stateful pod is unhealthy", func() {
+		/*
+			Testname: StatefulSet-Scaling
+			Description: StatefulSet MUST create Pods in ascending order by ordinal index when scaling up,
+				and delete Pods in descending order when scaling down. Scaling up or down MUST pause if any
+				Pods belonging to the StatefulSet are unhealthy.
+				This test does not depend on a preexisting default StorageClass or a dynamic provisioner.
+		*/
+		framework.ConformanceIt("Scaling should happen in predictable order and halt if any stateful pod is unhealthy", func() {
 			psLabels := klabels.Set(labels)
 			By("Initializing watcher for selector " + psLabels.String())
 			watcher, err := f.ClientSet.CoreV1().Pods(ns).Watch(metav1.ListOptions{
@@ -722,7 +752,12 @@ var _ = SIGDescribe("StatefulSet", func() {
 			Expect(err).NotTo(HaveOccurred())
 		})
 
-		It("Burst scaling should run to completion even with unhealthy pods", func() {
+		/*
+			Testname: StatefulSet-BurstScaling
+			Description: StatefulSet MUST support the Parallel PodManagementPolicy for burst scaling.
+				This test does not depend on a preexisting default StorageClass or a dynamic provisioner.
+		*/
+		framework.ConformanceIt("Burst scaling should run to completion even with unhealthy pods", func() {
 			psLabels := klabels.Set(labels)
 
 			By("Creating stateful set " + ssName + " in namespace " + ns)
@@ -760,7 +795,13 @@ var _ = SIGDescribe("StatefulSet", func() {
 			sst.WaitForStatusReplicas(ss, 0)
 		})
 
-		It("Should recreate evicted statefulset", func() {
+		/*
+			Testname: StatefulSet-RecreateFailedPod
+			Description: StatefulSet MUST delete and recreate Pods it owns that go into a Failed state,
+				such as when they are rejected or evicted by a Node.
+				This test does not depend on a preexisting default StorageClass or a dynamic provisioner.
+		*/
+		framework.ConformanceIt("Should recreate evicted statefulset", func() {
 			podName := "test-pod"
 			statefulPodName := ssName + "-0"
 			By("Looking for a node to schedule stateful set and pod")
@@ -901,21 +942,29 @@ var _ = SIGDescribe("StatefulSet", func() {
 			framework.DeleteAllStatefulSets(c, ns)
 		})
 
+		// Do not mark this as Conformance.
+		// StatefulSet Conformance should not be dependent on specific applications.
 		It("should creating a working zookeeper cluster", func() {
 			appTester.statefulPod = &zookeeperTester{tester: sst}
 			appTester.run()
 		})
 
+		// Do not mark this as Conformance.
+		// StatefulSet Conformance should not be dependent on specific applications.
 		It("should creating a working redis cluster", func() {
 			appTester.statefulPod = &redisTester{tester: sst}
 			appTester.run()
 		})
 
+		// Do not mark this as Conformance.
+		// StatefulSet Conformance should not be dependent on specific applications.
 		It("should creating a working mysql cluster", func() {
 			appTester.statefulPod = &mysqlGaleraTester{tester: sst}
 			appTester.run()
 		})
 
+		// Do not mark this as Conformance.
+		// StatefulSet Conformance should not be dependent on specific applications.
 		It("should creating a working CockroachDB cluster", func() {
 			appTester.statefulPod = &cockroachDBTester{tester: sst}
 			appTester.run()