Implement automated downgrade testing.

hack/verify-flags/known-flags.txt

@@ -282,6 +282,7 @@ gather-metrics-at-teardown
 gather-resource-usage
 gce-project
 gce-service-account
+gce-upgrade-script
 gce-zone
 ginkgo-flags
 gke-cluster

test/e2e/cluster_upgrade.go

@@ -113,6 +113,39 @@ var _ = framework.KubeDescribe("Upgrade [Feature:Upgrade]", func() {
 	})
 })
 
+var _ = framework.KubeDescribe("Downgrade [Feature:Downgrade]", func() {
+	f := framework.NewDefaultFramework("cluster-downgrade")
+
+	// Create the frameworks here because we can only create them
+	// in a "Describe".
+	testFrameworks := map[string]*framework.Framework{}
+	for _, t := range upgradeTests {
+		testFrameworks[t.Name()] = framework.NewDefaultFramework(t.Name())
+	}
+
+	framework.KubeDescribe("cluster downgrade", func() {
+		It("should maintain a functioning cluster [Feature:ClusterDowngrade]", func() {
+			cm := chaosmonkey.New(func() {
+				// Yes this really is a downgrade. And nodes must downgrade first.
+				v, err := realVersion(framework.TestContext.UpgradeTarget)
+				framework.ExpectNoError(err)
+				framework.ExpectNoError(framework.NodeUpgrade(f, v, framework.TestContext.UpgradeImage))
+				framework.ExpectNoError(checkNodesVersions(f.ClientSet, v))
+				framework.ExpectNoError(framework.MasterUpgrade(v))
+				framework.ExpectNoError(checkMasterVersion(f.ClientSet, v))
+			})
+			for _, t := range upgradeTests {
+				cm.RegisterInterface(&chaosMonkeyAdapter{
+					test:        t,
+					framework:   testFrameworks[t.Name()],
+					upgradeType: upgrades.ClusterUpgrade,
+				})
+			}
+			cm.Do()
+		})
+	})
+})
+
 var _ = framework.KubeDescribe("etcd Upgrade [Feature:EtcdUpgrade]", func() {
 	// Create the frameworks here because we can only create them
 	// in a "Describe".

test/e2e/framework/nodes_util.go

@@ -57,13 +57,13 @@ func etcdUpgradeGCE(target_storage, target_version string) error {
 		"STORAGE_BACKEND="+target_storage,
 		"TEST_ETCD_IMAGE=3.0.17")
 
-	_, _, err := RunCmdEnv(env, path.Join(TestContext.RepoRoot, "cluster/gce/upgrade.sh"), "-l", "-M")
+	_, _, err := RunCmdEnv(env, gceUpgradeScript(), "-l", "-M")
 	return err
 }
 
 func masterUpgradeGCE(rawV string) error {
 	v := "v" + rawV
-	_, _, err := RunCmd(path.Join(TestContext.RepoRoot, "cluster/gce/upgrade.sh"), "-M", v)
+	_, _, err := RunCmd(gceUpgradeScript(), "-M", v)
 	return err
 }
 
@@ -111,10 +111,10 @@ func nodeUpgradeGCE(rawV, img string) error {
 	v := "v" + rawV
 	if img != "" {
 		env := append(os.Environ(), "KUBE_NODE_OS_DISTRIBUTION="+img)
-		_, _, err := RunCmdEnv(env, path.Join(TestContext.RepoRoot, "cluster/gce/upgrade.sh"), "-N", "-o", v)
+		_, _, err := RunCmdEnv(env, gceUpgradeScript(), "-N", "-o", v)
 		return err
 	}
-	_, _, err := RunCmd(path.Join(TestContext.RepoRoot, "cluster/gce/upgrade.sh"), "-N", v)
+	_, _, err := RunCmd(gceUpgradeScript(), "-N", v)
 	return err
 }
 
@@ -254,3 +254,10 @@ func MigTemplate() (string, error) {
 	}
 	return templ, nil
 }
+
+func gceUpgradeScript() string {
+	if len(TestContext.GCEUpgradeScript) == 0 {
+		return path.Join(TestContext.RepoRoot, "cluster/gce/upgrade.sh")
+	}
+	return TestContext.GCEUpgradeScript
+}

test/e2e/framework/test_context.go

@@ -52,6 +52,7 @@ type TestContextType struct {
 	EtcdUpgradeStorage       string
 	EtcdUpgradeVersion       string
 	UpgradeImage             string
+	GCEUpgradeScript         string
 	PrometheusPushGateway    string
 	ContainerRuntime         string
 	MasterOSDistro           string
@@ -200,6 +201,7 @@ func RegisterClusterFlags() {
 	flag.StringVar(&TestContext.EtcdUpgradeStorage, "etcd-upgrade-storage", "", "The storage version to upgrade to (either 'etcdv2' or 'etcdv3') if doing an etcd upgrade test.")
 	flag.StringVar(&TestContext.EtcdUpgradeVersion, "etcd-upgrade-version", "", "The etcd binary version to upgrade to (e.g., '3.0.14', '2.3.7') if doing an etcd upgrade test.")
 	flag.StringVar(&TestContext.UpgradeImage, "upgrade-image", "", "Image to upgrade to (e.g. 'container_vm' or 'gci') if doing an upgrade test.")
+	flag.StringVar(&TestContext.GCEUpgradeScript, "gce-upgrade-script", "", "Script to use to upgrade a GCE cluster.")
 	flag.StringVar(&TestContext.PrometheusPushGateway, "prom-push-gateway", "", "The URL to prometheus gateway, so that metrics can be pushed during e2es and scraped by prometheus. Typically something like 127.0.0.1:9091.")
 	flag.BoolVar(&TestContext.CleanStart, "clean-start", false, "If true, purge all namespaces except default and system before running tests. This serves to Cleanup test namespaces from failed/interrupted e2e runs in a long-lived cluster.")
 	flag.BoolVar(&TestContext.GarbageCollectorEnabled, "garbage-collector-enabled", true, "Set to true if the garbage collector is enabled in the kube-apiserver and kube-controller-manager, then some tests will rely on the garbage collector to delete dependent resources.")