Add CI jobs to track the master and 1.2 branch with Trusty images

hack/jenkins/e2e-runner.sh

@@ -94,11 +94,69 @@ function unpack_binaries() {
     tar -xzf kubernetes-test.tar.gz
 }
 
+# GCP Project to fetch Trusty images.
+function get_trusty_image_project() {
+  local project=""
+  # Retry the gsutil command a couple times to mitigate the effect of
+  # transient server errors.
+  for n in $(seq 3); do
+    project="$(gsutil cat "gs://trusty-images/image-project.txt")" && break || sleep 1
+  done
+  if [[ -z "${project}" ]]; then
+    echo "Failed to find the image project for Trusty images."
+    exit 1
+  fi
+  echo "${project}"
+  # Clean up gsutil artifacts otherwise the later test stage will complain.
+  rm -rf .config &> /dev/null
+  rm -rf .gsutil &> /dev/null
+}
+
+# Get the latest Trusty image for a Jenkins job.
+function get_latest_trusty_image() {
+    local image_project="$1"
+    local image_type="$2"
+    local image_index=""
+    if [[ "${image_type}" == head ]]; then
+      image_index="trusty-head"
+    elif [[ "${image_type}" == dev ]]; then
+      image_index="trusty-dev"
+    elif [[ "${image_type}" == beta ]]; then
+      image_index="trusty-beta"
+    elif [[ "${image_type}" == stable ]]; then
+      image_index="trusty-stable"
+    fi
+
+    local image=""
+    # Retry the gsutil command a couple times to mitigate the effect of
+    # transient server errors.
+    for n in $(seq 3); do
+      image="$(gsutil cat "gs://${image_project}/image-indices/latest-test-image-${image_index}")" && break || sleep 1
+    done
+    if [[ -z "${image}" ]]; then
+      echo "Failed to find Trusty image for ${image_type}"
+      exit 1
+    fi
+    echo "${image}"
+    # Clean up gsutil artifacts otherwise the later test stage will complain.
+    rm -rf .config &> /dev/null
+    rm -rf .gsutil &> /dev/null
+}
+
 echo "--------------------------------------------------------------------------------"
 echo "Test Environment:"
 printenv | sort
 echo "--------------------------------------------------------------------------------"
 
+# We get the image project and name for Trusty dynamically.
+if [[ "${JENKINS_USE_TRUSTY_IMAGES:-}" =~ ^[yY]$ ]]; then
+  trusty_image_project="$(get_trusty_image_project)"
+  trusty_image="$(get_latest_trusty_image "${trusty_image_project}" "head")"
+  export KUBE_MASTER_IMAGE_PROJECT="${trusty_image_project}"
+  export KUBE_MASTER_IMAGE="${trusty_image}"
+  export KUBE_OS_DISTRIBUTION="trusty"
+fi
+
 # We get the Kubernetes tarballs unless we are going to use old ones
 if [[ "${JENKINS_USE_EXISTING_BINARIES:-}" =~ ^[yY]$ ]]; then
     echo "Using existing binaries; not cleaning, fetching, or unpacking new ones."

hack/jenkins/job-configs/kubernetes-jenkins/kubernetes-e2e.yaml

@@ -40,25 +40,7 @@
         - timestamps
         - workspace-cleanup
 
-# Template for e2e test jobs that run on GCE with Trusty images.
-- job-template:
-    name: 'kubernetes-e2e-gce-trusty-{suffix}'
-    <<: *e2e_job_defaults
-    triggers:
-        - timed: 'H H/8 * * *'
-    publishers:
-        - e2e-publishers:
-            recipients: '{emails}'
-        - description-setter:
-            regexp: KUBE_GCE_MINION_IMAGE=(.*)
-        - groovy-postbuild:
-            script: |
-                def trustyImageMatcher = manager.getLogMatcher("KUBE_GCE_MINION_IMAGE=(.*)")
-                if(trustyImageMatcher?.matches()) manager.addShortText("<b>Trusty Image: " + trustyImageMatcher.group(1) + "</b>", "grey", "white", "0px", "white")
-                def k8sVersionMatcher = manager.getLogMatcher("Using\\spublished\\sversion\\s(.*)\\s\\(from.*")
-                if(k8sVersionMatcher?.matches()) manager.addShortText("<br><b>Kubernetes version: " + k8sVersionMatcher.group(1) + "</b>", "grey", "white", "0px", "white")
-
-# Template for the rest of e2e test jobs.
+# Template for most e2e test jobs.
 - job-template:
     name: 'kubernetes-e2e-{suffix}'
     <<: *e2e_job_defaults
@@ -285,7 +267,7 @@
                 export TEST_CLUSTER_DELETE_COLLECTION_WORKERS="--delete-collection-workers=16"
     jobs:
         - 'kubernetes-e2e-{suffix}'
- 
+
 - project:
     name: kubernetes-e2e-gke-1-2
     trigger-job: 'kubernetes-build-1.2'
@@ -390,47 +372,6 @@
     jobs:
         - 'kubernetes-e2e-{suffix}'
 
-- project:
-    name: kubernetes-e2e-gke-trusty
-    trigger-job: 'kubernetes-build-1.1'
-    test-owner: 'wonderfly@google.com'
-    branch: 'release-1.1'
-    runner: '{old-runner-1-1}'
-    post-env: ''
-    emails: 'wonderfly@google.com,qzheng@google.com'
-    suffix:
-        - 'gke-trusty-prod':
-            timeout: 180
-            description: |
-                Run e2e tests with Trusty as node image using the following config:<br>
-                - provider: GKE<br>
-                - api proxy: prod<br>
-                - borg job: prod<br>
-                - client (kubectl): release/stable.txt<br>
-                - cluster (k8s): release/stable.txt<br>
-                - tests: release/stable.txt
-        - 'gke-trusty-staging':
-            timeout: 300
-            description: |
-                Run e2e tests with Trusty as node image using the following config:<br>
-                - provider: GKE<br>
-                - api proxy: staging<br>
-                - borg job: staging<br>
-                - client (kubectl): release/stable.txt<br>
-                - cluster (k8s): release/stable.txt<br>
-                - tests: release/stable.txt
-        - 'gke-trusty-test':
-            timeout: 300
-            description: |
-                Run e2e tests with Trusty as node image using the following config:<br>
-                - provider: GKE<br>
-                - api proxy: staging<br>
-                - borg job: test<br>
-                - client (kubectl): release/stable.txt<br>
-                - cluster (k8s): release/stable.txt<br>
-                - tests: release/stable.txt
-    jobs:
-        - 'kubernetes-e2e-{suffix}'
 
 - project:
     name: kubernetes-e2e-gce-1.1
@@ -544,6 +485,154 @@
         - 'kubernetes-e2e-{suffix}'
 
 - project:
+    name: kubernetes-e2e-gce-enormous-cluster
+    test-owner: 'gmarek'
+    emails: 'gmarek@google.com'
+    cron-string: 'H 8 * * *'
+    trigger-job: ''
+    description: 'Starts and deletes empty 1000 node cluster and runs Density 30 test on it. Does allow few Nodes to fail during startup.'
+    timeout: 480
+    branch: 'master'
+    suffix: 'gce-enormous-cluster'
+    provider-env: '{gce-provider-env}'
+    job-env: |
+        # XXX Not a unique project
+        export E2E_NAME="e2e-enormous-cluster"
+        export GINKGO_TEST_ARGS="--ginkgo.focus=\[Feature:Performance\]"
+        export PROJECT="kubernetes-scale"
+        export FAIL_ON_GCP_RESOURCE_LEAK="false"
+        # Override GCE defaults.
+        export NETWORK_PROVIDER="flannel"
+        # Temporarily switch of Heapster, as this will not schedule anywhere.
+        # TODO: Think of a solution to enable it.
+        export KUBE_ENABLE_CLUSTER_MONITORING="none"
+        export KUBE_GCE_ZONE="us-central1-c"
+        export MASTER_SIZE="n1-standard-32"
+        export NODE_SIZE="n1-standard-1"
+        export NODE_DISK_SIZE="50GB"
+        export NUM_NODES="1000"
+        export ALLOWED_NOTREADY_NODES="2"
+        export EXIT_ON_WEAK_ERROR="false"
+        # Reduce logs verbosity
+        export TEST_CLUSTER_LOG_LEVEL="--v=1"
+        export MAX_INSTANCES_PER_MIG="1000"
+        # Increase resync period to simulate production
+        export TEST_CLUSTER_RESYNC_PERIOD="--min-resync-period=12h"
+        # Increase delete collection parallelism
+        export TEST_CLUSTER_DELETE_COLLECTION_WORKERS="--delete-collection-workers=16"
+    jobs:
+        - 'kubernetes-e2e-{suffix}'
+
+# ==============================================================================
+# NOTE: From here on all jobs use Trusty as the image for master and/or nodes.
+# Please add templates/groups/projects/jobs that use ContainerVm above/below
+# this section (search "End of Trusty jobs" for the ending separator).
+
+# Template for e2e test jobs that run on GCE with Trusty images against
+# kubernetes HEAD.
+- job-template:
+    name: 'kubernetes-e2e-gce-trusty-ci-{suffix}'
+    <<: *e2e_job_defaults
+    triggers:
+        - reverse:
+            jobs: '{trigger-job}'
+            result: success
+        - timed: '{cron-string}'
+    publishers:
+        - e2e-publishers:
+            recipients: '{emails}'
+        - description-setter:
+            regexp: KUBE_GCE_MASTER_IMAGE=(.*)
+        - groovy-postbuild:
+            script: |
+                def trustyImageMatcher = manager.getLogMatcher("KUBE_GCE_MASTER_IMAGE=(.*)")
+                if(trustyImageMatcher?.matches()) manager.addShortText("<b>Trusty Image: " + trustyImageMatcher.group(1) + "</b>", "grey", "white", "0px", "white")
+                def k8sVersionMatcher = manager.getLogMatcher("Using\\spublished\\sversion\\s(.*)\\s\\(from.*")
+                if(k8sVersionMatcher?.matches()) manager.addShortText("<br><b>Kubernetes version: " + k8sVersionMatcher.group(1) + "</b>", "grey", "white", "0px", "white")
+
+# CI jobs that run on GCE with both master and nodes running Trusty images. We
+# use these to guard k8s+Trusty compatibility.
+- project:
+    name: kubernetes-e2e-gce-trusty-ci-master
+    trigger-job: 'kubernetes-build'
+    test-owner: 'wonderfly@google.com'
+    emails: 'wonderfly@google.com,qzheng@google.com'
+    provider-env: |
+        {gce-provider-env}
+        export JENKINS_USE_TRUSTY_IMAGES="y"
+    suffix:
+        # TODO(wonderfly): For Trusty, we currently only run CI and slow tests.
+        # More test coverage under way.
+        - 'master':
+            description: 'Runs all non-slow, non-serial, non-flaky, tests on GCE with Trusty images in parallel on the master branch.'
+            timeout: 30
+            job-env: |
+                export GINKGO_TEST_ARGS="--ginkgo.skip=\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]"
+                export GINKGO_PARALLEL="y"
+                export PROJECT="e2e-gce-trusty-ci-master"
+        - 'slow-master':
+            description: 'Runs slow tests on GCE with Trusty images, sequentially on the master branch.'
+            timeout: 60
+            job-env: |
+                export GINKGO_TEST_ARGS="--ginkgo.focus=\[Slow\] \
+                                         --ginkgo.skip=\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]"
+                export GINKGO_PARALLEL="y"
+                export PROJECT="e2e-gce-trusty-ci-master-slow"
+    jobs:
+        - 'kubernetes-e2e-gce-trusty-ci-{suffix}'
+
+- project:
+    name: kubernetes-e2e-gce-trusty-ci-1-2
+    trigger-job: 'kubernetes-build-1.2'
+    test-owner: 'wonderfly@google.com'
+    emails: 'wonderfly@google.com,qzheng@google.com'
+    provider-env: |
+        {gce-provider-env}
+        export JENKINS_PUBLISHED_VERSION="ci/latest-1.2"
+        export JENKINS_USE_TRUSTY_IMAGES="y"
+    suffix:
+        # TODO(wonderfly): For Trusty, we currently only run CI and slow tests.
+        # More test coverage under way.
+        - 'release-1.2':
+            description: 'Runs all non-slow, non-serial, non-flaky, tests on GCE with Trusty images in parallel on the release-1.2 branch.'
+            timeout: 30
+            job-env: |
+                export GINKGO_TEST_ARGS="--ginkgo.skip=\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]"
+                export GINKGO_PARALLEL="y"
+                export PROJECT="e2e-gce-trusty-ci-1-2"
+        - 'slow-release-1.2':
+            description: 'Runs slow tests on GCE with Trusty images, sequentially on the release-1.2 branch.'
+            timeout: 60
+            job-env: |
+                export GINKGO_TEST_ARGS="--ginkgo.focus=\[Slow\] \
+                                         --ginkgo.skip=\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]"
+                export GINKGO_PARALLEL="y"
+                export PROJECT="e2e-gce-trusty-ci-slow-1-2"
+    jobs:
+        - 'kubernetes-e2e-gce-trusty-ci-{suffix}'
+
+# Template for e2e test jobs that run on GCE with Trusty images against last
+# released k8s version. We use these jobs to test Trusty's continuous build.
+- job-template:
+    name: 'kubernetes-e2e-gce-trusty-{suffix}'
+    <<: *e2e_job_defaults
+    triggers:
+        - timed: 'H H/8 * * *'
+    publishers:
+        - e2e-publishers:
+            recipients: '{emails}'
+        - description-setter:
+            regexp: KUBE_GCE_MINION_IMAGE=(.*)
+        - groovy-postbuild:
+            script: |
+                def trustyImageMatcher = manager.getLogMatcher("KUBE_GCE_MINION_IMAGE=(.*)")
+                if(trustyImageMatcher?.matches()) manager.addShortText("<b>Trusty Image: " + trustyImageMatcher.group(1) + "</b>", "grey", "white", "0px", "white")
+                def k8sVersionMatcher = manager.getLogMatcher("Using\\spublished\\sversion\\s(.*)\\s\\(from.*")
+                if(k8sVersionMatcher?.matches()) manager.addShortText("<br><b>Kubernetes version: " + k8sVersionMatcher.group(1) + "</b>", "grey", "white", "0px", "white")
+
+# Jobs that run e2e tests on GCE with Trusty images against the latest k8s
+# release.
+- project:
     name: kubernetes-e2e-gce-trusty
     test-owner: 'wonderfly@google.com'
     branch: 'release-1.1'
@@ -569,6 +658,7 @@
         - 'dev-slow':
             # Constantly failing due to a known issue of the image. Disabled
             # until the issue is resolved.
+            # TODO(wonderfly): Re-enable it once the internal issue is fixed.
             disable_job: true
             description: 'Run slow E2E tests on latest Trusty dev build.'
             timeout: 270
@@ -581,42 +671,46 @@
     jobs:
         - 'kubernetes-e2e-gce-trusty-{suffix}'
 
+# Jobs that run e2e tests on GKE with Trusty as node image on the release-1.1 branch.
 - project:
-    name: kubernetes-e2e-gce-enormous-cluster
-    test-owner: 'gmarek'
-    emails: 'gmarek@google.com'
-    cron-string: 'H 8 * * *'
-    trigger-job: ''
-    description: 'Starts and deletes empty 1000 node cluster and runs Density 30 test on it. Does allow few Nodes to fail during startup.'
-    timeout: 480
-    branch: 'master'
-    suffix: 'gce-enormous-cluster'
-    provider-env: '{gce-provider-env}'
-    job-env: |
-        # XXX Not a unique project
-        export E2E_NAME="e2e-enormous-cluster"
-        export GINKGO_TEST_ARGS="--ginkgo.focus=\[Feature:Performance\]"
-        export PROJECT="kubernetes-scale"
-        export FAIL_ON_GCP_RESOURCE_LEAK="false"
-        # Override GCE defaults.
-        export NETWORK_PROVIDER="flannel"
-        # Temporarily switch of Heapster, as this will not schedule anywhere.
-        # TODO: Think of a solution to enable it.
-        export KUBE_ENABLE_CLUSTER_MONITORING="none"
-        export KUBE_GCE_ZONE="us-central1-c"
-        export MASTER_SIZE="n1-standard-32"
-        export NODE_SIZE="n1-standard-1"
-        export NODE_DISK_SIZE="50GB"
-        export NUM_NODES="1000"
-        export ALLOWED_NOTREADY_NODES="2"
-        export EXIT_ON_WEAK_ERROR="false"
-        # Reduce logs verbosity
-        export TEST_CLUSTER_LOG_LEVEL="--v=1"
-        export MAX_INSTANCES_PER_MIG="1000"
-        # Increase resync period to simulate production
-        export TEST_CLUSTER_RESYNC_PERIOD="--min-resync-period=12h"
-        # Increase delete collection parallelism
-        export TEST_CLUSTER_DELETE_COLLECTION_WORKERS="--delete-collection-workers=16"
+    name: kubernetes-e2e-gke-trusty
+    trigger-job: 'kubernetes-build-1.1'
+    test-owner: 'wonderfly@google.com'
+    branch: 'release-1.1'
+    runner: '{old-runner-1-1}'
+    post-env: ''
+    emails: 'wonderfly@google.com,qzheng@google.com'
+    suffix:
+        - 'gke-trusty-prod':
+            timeout: 180
+            description: |
+                Run e2e tests with Trusty as node image using the following config:<br>
+                - provider: GKE<br>
+                - api proxy: prod<br>
+                - borg job: prod<br>
+                - client (kubectl): release/stable.txt<br>
+                - cluster (k8s): release/stable.txt<br>
+                - tests: release/stable.txt
+        - 'gke-trusty-staging':
+            timeout: 300
+            description: |
+                Run e2e tests with Trusty as node image using the following config:<br>
+                - provider: GKE<br>
+                - api proxy: staging<br>
+                - borg job: staging<br>
+                - client (kubectl): release/stable.txt<br>
+                - cluster (k8s): release/stable.txt<br>
+                - tests: release/stable.txt
+        - 'gke-trusty-test':
+            timeout: 300
+            description: |
+                Run e2e tests with Trusty as node image using the following config:<br>
+                - provider: GKE<br>
+                - api proxy: staging<br>
+                - borg job: test<br>
+                - client (kubectl): release/stable.txt<br>
+                - cluster (k8s): release/stable.txt<br>
+                - tests: release/stable.txt
     jobs:
         - 'kubernetes-e2e-{suffix}'
-
+#============================== End of Trusty jobs =============================