Add config for the main gke-jenkins jobs to e2e.sh

cluster/gke/config-common.sh

@@ -25,7 +25,7 @@ NETWORK="${NETWORK:-default}"
 NETWORK_RANGE="${NETWORK_RANGE:-10.240.0.0/16}"
 FIREWALL_SSH="${FIREWALL_SSH:-${NETWORK}-allow-ssh}"
 GCLOUD="${GCLOUD:-gcloud}"
-CMD_GROUP="${CMD_GROUP:-beta}"
+CMD_GROUP="${CMD_GROUP:-}"
 GCLOUD_CONFIG_DIR="${GCLOUD_CONFIG_DIR:-${HOME}/.config/gcloud/kubernetes}"
 MINION_SCOPES="${MINION_SCOPES:-"compute-rw,storage-ro"}"
 MACHINE_TYPE="${MACHINE_TYPE:-n1-standard-1}"

cluster/gke/util.sh

@@ -99,7 +99,7 @@ function verify-prereqs() {
     sudo_prefix="sudo"
   fi
   ${sudo_prefix} gcloud ${gcloud_prompt:-} components update preview || true
-  ${sudo_prefix} gcloud ${gcloud_prompt:-} components update "${CMD_GROUP}"|| true
+  ${sudo_prefix} gcloud ${gcloud_prompt:-} components update ${CMD_GROUP:-} || true
   ${sudo_prefix} gcloud ${gcloud_prompt:-} components update kubectl|| true
   ${sudo_prefix} gcloud ${gcloud_prompt:-} components update || true
 }
@@ -150,7 +150,7 @@ function kube-up() {
   )
 
   # Bring up the cluster.
-  "${GCLOUD}" "${CMD_GROUP}" container clusters create "${CLUSTER_NAME}" "${create_args[@]}"
+  "${GCLOUD}" ${CMD_GROUP:-} container clusters create "${CLUSTER_NAME}" "${create_args[@]}"
 }
 
 # Execute prior to running tests to initialize required structure. This is
@@ -200,7 +200,7 @@ function test-setup() {
 function detect-master() {
   echo "... in gke:detect-master()" >&2
   detect-project >&2
-  KUBE_MASTER_IP=$("${GCLOUD}" "${CMD_GROUP}" container clusters describe \
+  KUBE_MASTER_IP=$("${GCLOUD}" ${CMD_GROUP:-} container clusters describe \
     --project="${PROJECT}" --zone="${ZONE}" "${CLUSTER_NAME}" \
     | grep endpoint | cut -f 2 -d ' ')
 }
@@ -242,7 +242,7 @@ function detect-minion-names {
 #   NODE_INSTANCE_GROUP
 function detect-node-instance-group {
   echo "... in gke:detect-node-instance-group()" >&2
-  NODE_INSTANCE_GROUP=$("${GCLOUD}" "${CMD_GROUP}" container clusters describe \
+  NODE_INSTANCE_GROUP=$("${GCLOUD}" ${CMD_GROUP:-} container clusters describe \
     --project="${PROJECT}" --zone="${ZONE}" "${CLUSTER_NAME}" \
     | grep instanceGroupManagers | cut -d '/' -f 11)
 }
@@ -318,6 +318,6 @@ function test-teardown() {
 function kube-down() {
   echo "... in gke:kube-down()" >&2
   detect-project >&2
-  "${GCLOUD}" "${CMD_GROUP}" container clusters delete --project="${PROJECT}" \
+  "${GCLOUD}" ${CMD_GROUP:-} container clusters delete --project="${PROJECT}" \
     --zone="${ZONE}" "${CLUSTER_NAME}" --quiet
 }

hack/jenkins/e2e.sh

@@ -70,6 +70,10 @@ if [[ ${JOB_NAME} =~ ^kubernetes-.*-gce ]]; then
   : ${E2E_MIN_STARTUP_PODS:="1"}
   : ${E2E_ZONE:="us-central1-f"}
   : ${NUM_MINIONS_PARALLEL:="6"}  # Number of nodes required to run all of the tests in parallel
+
+elif [[ ${JOB_NAME} =~ ^kubernetes-.*-gke ]]; then
+  KUBERNETES_PROVIDER="gke"
+  : ${E2E_ZONE:="us-central1-f"}
 fi
 
 if [[ "${KUBERNETES_PROVIDER}" == "aws" ]]; then
@@ -84,8 +88,8 @@ if [[ "${KUBERNETES_PROVIDER}" == "aws" ]]; then
   fi
 fi
 
-# Specialized tests which should be skipped by default for projects.
-GCE_DEFAULT_SKIP_TESTS=(
+# Specialized to skip when running reboot tests.
+REBOOT_SKIP_TESTS=(
     "Autoscaling\sSuite"
     "Skipped"
     "Reboot"
@@ -93,6 +97,20 @@ GCE_DEFAULT_SKIP_TESTS=(
     "Example"
     )
 
+# Specialized tests which should be skipped by default for projects.
+GCE_DEFAULT_SKIP_TESTS=(
+    "${REBOOT_SKIP_TESTS[@]}"
+    "Reboot")
+
+# Tests which cannot be run on GKE, e.g. because they require
+# master ssh access.
+GKE_REQUIRED_SKIP_TESTS=(
+    "Nodes"
+    "Etcd\sFailure"
+    "MasterCerts"
+    "Shell"
+    )
+
 # The following tests are known to be flaky, and are thus run only in their own
 # -flaky- build variants.
 GCE_FLAKY_TESTS=(
@@ -345,6 +363,81 @@ case ${JOB_NAME} in
     : ${KUBE_GCE_INSTANCE_PREFIX="e2e-gce"}
     : ${PROJECT:="k8s-jkns-e2e-gce-release"}
     ;;
+
+  kubernetes-e2e-gke-prod)
+    : ${DOGFOOD_GCLOUD:="true"}
+    : ${E2E_CLUSTER_NAME:="jkns-gke-e2e-prod"}
+    : ${E2E_NETWORK:="e2e-gke-prod"}
+    : ${E2E_SET_CLUSTER_API_VERSION:=y}
+    : ${JENKINS_USE_SERVER_VERSION:=y}
+    : ${PROJECT:="k8s-jkns-e2e-gke-prod"}
+    : ${GINKGO_TEST_ARGS:="--ginkgo.skip=$(join_regex_allow_empty \
+          ${GKE_REQUIRED_SKIP_TESTS[@]:+${GKE_REQUIRED_SKIP_TESTS[@]}} \
+          ${GCE_DEFAULT_SKIP_TESTS[@]:+${GCE_DEFAULT_SKIP_TESTS[@]}} \
+          ${GCE_FLAKY_TESTS[@]:+${GCE_FLAKY_TESTS[@]}} \
+          )"}
+    ;;
+
+  kubernetes-e2e-gke-staging)
+    : ${DOGFOOD_GCLOUD:="true"}
+    : ${GKE_API_ENDPOINT:="https://staging-container.sandbox.googleapis.com/"}
+    : ${E2E_CLUSTER_NAME:="jkns-gke-e2e-staging"}
+    : ${E2E_NETWORK:="e2e-gke-staging"}
+    : ${E2E_SET_CLUSTER_API_VERSION:=y}
+    : ${JENKINS_USE_SERVER_VERSION:=y}
+    : ${PROJECT:="k8s-jkns-e2e-gke-staging"}
+    : ${GINKGO_TEST_ARGS:="--ginkgo.skip=$(join_regex_allow_empty \
+          ${GKE_REQUIRED_SKIP_TESTS[@]:+${GKE_REQUIRED_SKIP_TESTS[@]}} \
+          ${GCE_DEFAULT_SKIP_TESTS[@]:+${GCE_DEFAULT_SKIP_TESTS[@]}} \
+          ${GCE_FLAKY_TESTS[@]:+${GCE_FLAKY_TESTS[@]}} \
+          )"}
+    ;;
+
+  kubernetes-e2e-gke-test)
+    : ${DOGFOOD_GCLOUD:="true"}
+    : ${CLOUDSDK_BUCKET:="gs://cloud-sdk-build/testing/rc"}
+    : ${GKE_API_ENDPOINT:="https://test-container.sandbox.googleapis.com/"}
+    : ${E2E_CLUSTER_NAME:="jkns-gke-e2e-test"}
+    : ${E2E_NETWORK:="e2e-gke-test"}
+    : ${JENKINS_USE_RELEASE_TARS:=y}
+    : ${PROJECT:="k8s-jkns-e2e-gke-ci"}
+    : ${GINKGO_TEST_ARGS:="--ginkgo.skip=$(join_regex_allow_empty \
+          ${GKE_REQUIRED_SKIP_TESTS[@]:+${GKE_REQUIRED_SKIP_TESTS[@]}} \
+          ${GCE_DEFAULT_SKIP_TESTS[@]:+${GCE_DEFAULT_SKIP_TESTS[@]}} \
+          ${GCE_FLAKY_TESTS[@]:+${GCE_FLAKY_TESTS[@]}} \
+          )"}
+    ;;
+
+  kubernetes-e2e-gke-ci)
+    : ${DOGFOOD_GCLOUD:="true"}
+    : ${CLOUDSDK_BUCKET:="gs://cloud-sdk-build/testing/staging"}
+    : ${GKE_API_ENDPOINT:="https://test-container.sandbox.googleapis.com/"}
+    : ${E2E_CLUSTER_NAME:="jkns-gke-e2e-ci"}
+    : ${E2E_NETWORK:="e2e-gke-ci"}
+    : ${E2E_SET_CLUSTER_API_VERSION:=y}
+    : ${PROJECT:="k8s-jkns-e2e-gke-ci"}
+    : ${GINKGO_TEST_ARGS:="--ginkgo.skip=$(join_regex_allow_empty \
+          ${GKE_REQUIRED_SKIP_TESTS[@]:+${GKE_REQUIRED_SKIP_TESTS[@]}} \
+          ${GCE_DEFAULT_SKIP_TESTS[@]:+${GCE_DEFAULT_SKIP_TESTS[@]}} \
+          ${GCE_FLAKY_TESTS[@]:+${GCE_FLAKY_TESTS[@]}} \
+          )"}
+    ;;
+
+  kubernetes-e2e-gke-ci-reboot)
+    : ${DOGFOOD_GCLOUD:="true"}
+    : ${CLOUDSDK_BUCKET:="gs://cloud-sdk-build/testing/staging"}
+    : ${GKE_API_ENDPOINT:="https://test-container.sandbox.googleapis.com/"}
+    : ${E2E_CLUSTER_NAME:="jkns-gke-e2e-ci-reboot"}
+    : ${E2E_NETWORK:="e2e-gke-ci"}
+    : ${E2E_SET_CLUSTER_API_VERSION:=y}
+    : ${PROJECT:="k8s-jkns-e2e-gke-ci"}
+    : ${GINKGO_TEST_ARGS:="--ginkgo.skip=$(join_regex_allow_empty \
+          ${GKE_REQUIRED_SKIP_TESTS[@]:+${GKE_REQUIRED_SKIP_TESTS[@]}} \
+          ${REBOOT_SKIP_TESTS[@]:+${REBOOT_SKIP_TESTS[@]}} \
+          ${GCE_FLAKY_TESTS[@]:+${GCE_FLAKY_TESTS[@]}} \
+          ${GCE_PARALLEL_SKIP_TESTS[@]:+${GCE_PARALLEL_SKIP_TESTS[@]}} \
+          )"}
+    ;;
 esac
 
 # AWS variables
@@ -362,6 +455,13 @@ export KUBE_GCS_STAGING_PATH_SUFFIX=${KUBE_GCS_STAGING_PATH_SUFFIX:-}
 export CLUSTER_NAME=${E2E_CLUSTER_NAME}
 export ZONE=${E2E_ZONE}
 export KUBE_GKE_NETWORK=${E2E_NETWORK}
+export E2E_SET_CLUSTER_API_VERSION=${E2E_SET_CLUSTER_API_VERSION:-}
+export DOGFOOD_GCLOUD=${DOGFOOD_GCLOUD:-}
+export CMD_GROUP=${CMD_GROUP:-}
+
+if [[ ! -z "${GKE_API_ENDPOINT:-}" ]]; then
+  export CLOUDSDK_API_ENDPOINT_OVERRIDES_CONTAINER=${GKE_API_ENDPOINT}
+fi
 
 # Shared cluster variables
 export E2E_MIN_STARTUP_PODS=${E2E_MIN_STARTUP_PODS:-}
@@ -371,6 +471,7 @@ export MINION_SIZE=${MINION_SIZE:-}
 export NUM_MINIONS=${NUM_MINIONS:-}
 export PROJECT=${PROJECT:-}
 
+export KUBERNETES_PROVIDER=${KUBERNETES_PROVIDER}
 export PATH=${PATH}:/usr/local/go/bin
 export KUBE_SKIP_CONFIRMATIONS=y
 
@@ -407,10 +508,13 @@ if [[ "${E2E_UP,,}" == "true" || "${JENKINS_FORCE_GET_TARS:-}" =~ ^[yY]$ ]]; the
         # gcloud bug can cause racing component updates to stomp on each
         # other.
         export KUBE_SKIP_UPDATE=y
-        sudo flock -x -n /var/run/lock/gcloud-components.lock -c "gcloud components update -q" || true
-        sudo flock -x -n /var/run/lock/gcloud-components.lock -c "gcloud components update preview -q" || true
-        sudo flock -x -n /var/run/lock/gcloud-components.lock -c "gcloud components update alpha -q" || true
-        sudo flock -x -n /var/run/lock/gcloud-components.lock -c "gcloud components update beta -q" || true
+        {
+          sudo flock -x -n 9
+          gcloud components update -q || true
+          gcloud components update preview -q || true
+          gcloud components update alpha -q || true
+          gcloud components update beta -q || true
+        } 9>/var/run/lock/gcloud-components.lock
 
         if [[ ! -z ${JENKINS_EXPLICIT_VERSION:-} ]]; then
             # Use an explicit pinned version like "ci/v0.10.0-101-g6c814c4" or
@@ -496,6 +600,21 @@ ARTIFACTS=${WORKSPACE}/_artifacts
 mkdir -p ${ARTIFACTS}
 export E2E_REPORT_DIR=${ARTIFACTS}
 
+### Pre Set Up ###
+# Install gcloud from a custom path if provided. Used to test GKE with gcloud
+# at HEAD, release candidate.
+if [[ ! -z "${CLOUDSDK_BUCKET:-}" ]]; then
+    sudo gsutil -m cp -r "${CLOUDSDK_BUCKET}" ~
+    mv ~/$(basename "${CLOUDSDK_BUCKET}") ~/repo
+    mkdir ~/cloudsdk
+    tar zvxf ~/repo/google-cloud-sdk.tar.gz -C ~/cloudsdk
+    export CLOUDSDK_CORE_DISABLE_PROMPTS=1
+    export CLOUDSDK_COMPONENT_MANAGER_SNAPSHOT_URL=file://${HOME}/repo/components-2.json
+    ~/cloudsdk/google-cloud-sdk/install.sh --disable-installation-options --bash-completion=false --path-update=false --usage-reporting=false
+    export PATH=${HOME}/cloudsdk/google-cloud-sdk/bin:${PATH}
+    export CLOUDSDK_CONFIG=/var/lib/jenkins/.config/gcloud
+fi
+
 ### Set up ###
 if [[ "${E2E_UP,,}" == "true" ]]; then
     go run ./hack/e2e.go ${E2E_OPT} -v --down