Add configuration for splitting upgrade jobs instead of using Jenkins steps

hack/ginkgo-e2e.sh

@@ -107,4 +107,5 @@ export PATH=$(dirname "${e2e_test}"):"${PATH}"
   ${E2E_CLEAN_START:+"--clean-start=true"} \
   ${E2E_MIN_STARTUP_PODS:+"--minStartupPods=${E2E_MIN_STARTUP_PODS}"} \
   ${E2E_REPORT_DIR:+"--report-dir=${E2E_REPORT_DIR}"} \
+  ${E2E_REPORT_PREFIX:+"--report-prefix=${E2E_REPORT_PREFIX}"} \
   "${@:-}"

hack/jenkins/e2e-runner.sh

@@ -152,6 +152,22 @@ if [[ "${JENKINS_USE_TRUSTY_IMAGES:-}" =~ ^[yY]$ ]]; then
   export KUBE_OS_DISTRIBUTION="trusty"
 fi
 
+function e2e_test() {
+    local -r ginkgo_test_args="${1}"
+    # Check to make sure the cluster is up before running tests, and fail if it's not.
+    go run ./hack/e2e.go ${E2E_OPT:-} -v --isup
+    # Jenkins will look at the junit*.xml files for test failures, so don't exit with a nonzero
+    # error code if it was only tests that failed.
+    go run ./hack/e2e.go ${E2E_OPT:-} -v --test \
+      ${ginkgo_test_args:+--test_args="${ginkgo_test_args}"} \
+      && exitcode=0 || exitcode=$?
+    if [[ "${E2E_PUBLISH_GREEN_VERSION:-}" == "true" && ${exitcode} == 0 ]]; then
+        # Use plaintext version file packaged with kubernetes.tar.gz
+        echo "Publish version to ci/latest-green.txt: $(cat version)"
+        gsutil cp ./version gs://kubernetes-release/ci/latest-green.txt
+    fi
+}
+
 echo "--------------------------------------------------------------------------------"
 echo "Test Environment:"
 printenv | sort
@@ -231,7 +247,7 @@ fi
 ### Pre Set Up ###
 # Install gcloud from a custom path if provided. Used to test GKE with gcloud
 # at HEAD, release candidate.
-if [[ ! -z "${CLOUDSDK_BUCKET:-}" ]]; then
+if [[ -n "${CLOUDSDK_BUCKET:-}" ]]; then
     gsutil -mq cp -r "${CLOUDSDK_BUCKET}" ~
     rm -rf ~/repo ~/cloudsdk
     mv ~/$(basename "${CLOUDSDK_BUCKET}") ~/repo
@@ -267,21 +283,28 @@ if [[ "${E2E_UP,,}" == "true" ]]; then
     fi
 fi
 
-### Run tests ###
-# Jenkins will look at the junit*.xml files for test failures, so don't exit
-# with a nonzero error code if it was only tests that failed.
-if [[ "${E2E_TEST,,}" == "true" ]]; then
-    # Check to make sure the cluster is up before running tests, and fail if it's not.
-    go run ./hack/e2e.go ${E2E_OPT:-} -v --isup
-    go run ./hack/e2e.go ${E2E_OPT:-} -v --test \
-      ${GINKGO_TEST_ARGS:+--test_args="${GINKGO_TEST_ARGS}"} \
-      && exitcode=0 || exitcode=$?
-    if [[ "${E2E_PUBLISH_GREEN_VERSION:-}" == "true" && ${exitcode} == 0 && -n ${build_version:-} ]]; then
-        echo "Publish build_version to ci/latest-green.txt: ${build_version}"
-        gsutil cp ./version gs://kubernetes-release/ci/latest-green.txt
+# Allow download & unpack of alternate version of tests, for cross-version & upgrade testing.
+if [[ -n "${JENKINS_PUBLISHED_TEST_VERSION:-}" ]]; then
+    cd ..
+    mv kubernetes kubernetes_old
+    fetch_published_version_tars "${JENKINS_PUBLISHED_TEST_VERSION}"
+    cd kubernetes
+    # Upgrade the cluster before running other tests
+    if [[ "${E2E_UPGRADE_TEST,,}" == "true" ]]; then
+	# Add a report prefix for the e2e tests so that the tests don't get overwritten when we run
+	# the rest of the e2es.
+        E2E_REPORT_PREFIX='upgrade' e2e_test "${GINKGO_UPGRADE_TEST_ARGS:-}"
+	# If JENKINS_USE_OLD_TESTS is set, back out into the old tests now that we've upgraded.
+        if [[ "${JENKINS_USE_OLD_TESTS:-}" == "true" ]]; then
+            cd ../kubernetes_old
+        fi
     fi
 fi
 
+if [[ "${E2E_TEST,,}" == "true" ]]; then
+    e2e_test "${GINKGO_TEST_ARGS:-}"
+fi
+
 ### Start Kubemark ###
 if [[ "${USE_KUBEMARK:-}" == "true" ]]; then
   export RUN_FROM_DISTRO=true

hack/jenkins/job-configs/kubernetes-jenkins/kubernetes-e2e.yaml

@@ -572,7 +572,6 @@
     jobs:
         - 'kubernetes-e2e-{suffix}'
 
-# ==============================================================================
 # NOTE: From here on all jobs use Trusty as the image for master and/or nodes.
 # Please add templates/groups/projects/jobs that use ContainerVm above/below
 # this section (search "End of Trusty jobs" for the ending separator).
@@ -842,4 +841,53 @@
                 export KUBE_OS_DISTRIBUTION="trusty"
     jobs:
         - 'kubernetes-e2e-{suffix}'
-#============================== End of Trusty jobs =============================
+
+# End of Trusty jobs
+
+- project:
+    name: kubernetes-e2e-gke-upgrades-experimental
+    trigger-job: 'kubernetes-build'
+    test-owner: 'ihmccreery'
+    emails: 'ihmccreery@google.com'
+    provider-env: '{gke-provider-env}'
+    suffix:
+        - 'gke-kubectl-skew-1.1-1.2':
+            description: 'Deploys a cluster at v1.1 and runs the v1.2 Kubectl tests.'
+            timeout: 30
+            job-env: |
+                export PROJECT="kube-jks-gke-upg-experimental"
+                export E2E_NAME="gke-ctl-skew-1-1-1-2"
+                export JENKINS_PUBLISHED_TEST_VERSION="ci/latest-1.2"
+                export GINKGO_TEST_ARGS="--ginkgo.focus=Kubectl"
+                export GINKGO_PARALLEL="y"
+        - 'gke-upgrade-master-1.1-1.2':
+            description: 'Deploys a cluster at v1.1, upgrades its master to v1.2, and runs v1.1 tests against it.'
+            timeout: 30
+            job-env: |
+                export PROJECT="kube-jks-gke-upg-experimental"
+                export E2E_NAME="gke-upg-mas-1-1-1-2"
+                export JENKINS_PUBLISHED_TEST_VERSION="ci/latest-1.2"
+                export E2E_UPGRADE_TEST="true"
+                export GINKGO_UPGRADE_TEST_ARGS="--ginkgo.focus=\[Feature:MasterUpgrade\] --upgrade-target=ci/latest-1.2"
+                export JENKINS_USE_OLD_TESTS="true"
+        - 'gke-upgrade-cluster-1.1-1.2':
+            description: 'Deploys a cluster at v1.1, upgrades the cluster to v1.2, and runs v1.1 tests against it.'
+            timeout: 30
+            job-env: |
+                export PROJECT="kube-jks-gke-upg-experimental"
+                export E2E_NAME="gke-upg-clu-1-1-1-2"
+                export JENKINS_PUBLISHED_TEST_VERSION="ci/latest-1.2"
+                export E2E_UPGRADE_TEST="true"
+                export GINKGO_UPGRADE_TEST_ARGS="--ginkgo.focus=\[Feature:ClusterUpgrade\] --upgrade-target=ci/latest-1.2"
+                export JENKINS_USE_OLD_TESTS="true"
+        - 'gke-upgrade-cluster-new-1.1-1.2':
+            description: 'Deploys a cluster at v1.1, upgrades the cluster to v1.2, and runs v1.2 tests against it.'
+            timeout: 30
+            job-env: |
+                export PROJECT="kube-jks-gke-upg-experimental"
+                export E2E_NAME="gke-upg-clu-new-1-1-1-2"
+                export JENKINS_PUBLISHED_TEST_VERSION="ci/latest-1.2"
+                export E2E_UPGRADE_TEST="true"
+                export GINKGO_UPGRADE_TEST_ARGS="--ginkgo.focus=\[Feature:ClusterUpgrade\] --upgrade-target=ci/latest-1.2"
+    jobs:
+        - 'kubernetes-e2e-{suffix}'

hack/verify-flags/known-flags.txt

@@ -334,6 +334,7 @@ replicaset-lookup-cache-size
 replication-controller-lookup-cache-size
 repo-root
 report-dir
+report-prefix
 required-contexts
 resolv-conf
 resource-container

test/e2e/e2e.go

@@ -71,6 +71,7 @@ func RegisterFlags() {
 	flag.StringVar(&testContext.KubectlPath, "kubectl-path", "kubectl", "The kubectl binary to use. For development, you might use 'cluster/kubectl.sh' here.")
 	flag.StringVar(&testContext.OutputDir, "e2e-output-dir", "/tmp", "Output directory for interesting/useful test data, like performance data, benchmarks, and other metrics.")
 	flag.StringVar(&testContext.ReportDir, "report-dir", "", "Path to the directory where the JUnit XML reports should be saved. Default is empty, which doesn't generate these reports.")
+	flag.StringVar(&testContext.ReportPrefix, "report-prefix", "", "Optional prefix for JUnit XML reports. Default is empty, which doesn't prepend anything to the default name.")
 	flag.StringVar(&testContext.prefix, "prefix", "e2e", "A prefix to be added to cloud resources created during testing.")
 	flag.StringVar(&testContext.OSDistro, "os-distro", "debian", "The OS distribution of cluster VM instances (debian, trusty, or coreos).")
 
@@ -269,7 +270,7 @@ func RunE2ETests(t *testing.T) {
 		if err := os.MkdirAll(testContext.ReportDir, 0755); err != nil {
 			glog.Errorf("Failed creating report directory: %v", err)
 		} else {
-			r = append(r, reporters.NewJUnitReporter(path.Join(testContext.ReportDir, fmt.Sprintf("junit_%02d.xml", config.GinkgoConfig.ParallelNode))))
+			r = append(r, reporters.NewJUnitReporter(path.Join(testContext.ReportDir, fmt.Sprintf("junit_%v%02d.xml", testContext.ReportPrefix, config.GinkgoConfig.ParallelNode))))
 		}
 	}
 	glog.Infof("Starting e2e run %q on Ginkgo node %d", runId, config.GinkgoConfig.ParallelNode)

test/e2e/util.go

@@ -186,6 +186,7 @@ type TestContextType struct {
 	KubectlPath           string
 	OutputDir             string
 	ReportDir             string
+	ReportPrefix          string
 	prefix                string
 	MinStartupPods        int
 	UpgradeTarget         string