Add GUBERNATOR flag for remote node e2e tests to produce gubernator link

Makefile

@@ -162,6 +162,8 @@ test-e2e: ginkgo generated_files
 #  INSTANCE_PREFIX: For REMOTE=true only.  Instances created from images will
 #    have the name "${INSTANCE_PREFIX}-${IMAGE_NAME}".  Defaults to "test".
 #  INSTANCE_METADATA: For REMOTE=true and running on GCE only.
+#  GUBERNATOR: For REMOTE=true only. Produce link to Gubernator to view logs.
+#	 Defaults to false.
 #
 # Example:
 #   make test-e2e-node FOCUS=Kubelet SKIP=container

hack/make-rules/test-e2e-node.sh

@@ -32,6 +32,7 @@ remote=${REMOTE:-"false"}
 images=${IMAGES:-""}
 hosts=${HOSTS:-""}
 metadata=${INSTANCE_METADATA:-""}
+gubernator=${GUBERNATOR:-"false"}
 gci_image=$(gcloud compute images list --project google-containers \
     --no-standard-images --regexp="gci-dev.*" --format="table[no-heading](name)")
 if [[ $hosts == "" && $images == "" ]]; then
@@ -129,12 +130,13 @@ if [ $remote = true ] ; then
   echo "Ginkgo Flags: $ginkgoflags"
   echo "Instance Metadata: $metadata"
   # Invoke the runner
-  go run test/e2e_node/runner/run_e2e.go  --logtostderr --vmodule=*=4 --ssh-env="gce" \
-    --zone="$zone" --project="$project"  \
+  go run test/e2e_node/runner/run_e2e.go  --logtostderr --vmodule=*=2 --ssh-env="gce" \
+    --zone="$zone" --project="$project" --gubernator="$gubernator" \
     --hosts="$hosts" --images="$images" --cleanup="$cleanup" \
     --results-dir="$artifacts" --ginkgo-flags="$ginkgoflags" \
     --image-project="$image_project" --instance-name-prefix="$instance_prefix" --setup-node="true" \
-    --delete-instances="$delete_instances" --test_args="$test_args" --instance-metadata="$metadata"
+    --delete-instances="$delete_instances" --test_args="$test_args" --instance-metadata="$metadata" \
+    2>&1 | tee "${artifacts}/build-log.txt"
   exit $?
 
 else

test/e2e_node/gubernator.sh

+#!/bin/bash
+
+# Copyright 2016 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Make bucket and a folder for e2e-node test logs.
+# Populate the folder from the logs stored in /tmp/_artifacts/ in the same way as a
+# jenkins build would, and then print the URL to view the test results on Gubernator
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+source cluster/lib/logging.sh
+
+
+if [[ $# -eq 0 || ! $1 =~ ^[Yy]$ ]]; then
+  read -p "Do you want to run gubernator.sh and upload logs publicly to GCS? [y/n]" yn
+  echo
+  if [[ ! $yn =~ ^[Yy]$ ]]; then
+      exit 1
+  fi
+fi
+
+# Check that user has gsutil
+if [[ $(which gsutil) == "" ]]; then
+  echo "Could not find gsutil when running:\which gsutil"
+  exit 1
+fi
+
+# Check that user has gcloud
+if [[ $(which gcloud) == "" ]]; then
+  echo "Could not find gcloud when running:\which gcloud"
+  exit 1
+fi
+
+# Check that user has Credentialed Active account
+if ! gcloud auth list | grep -q "ACTIVE"; then
+  echo "Could not find active account when running:\gcloud auth list"
+  exit 1
+fi
+
+readonly gcs_acl="public-read"
+bucket_name="${USER}-g8r-logs"
+echo ""
+V=2 kube::log::status "Using bucket ${bucket_name}"
+
+# Check if the bucket exists
+if ! gsutil ls gs:// | grep -q "gs://${bucket_name}/"; then
+  V=2 kube::log::status "Creating bucket ${bucket_name}"
+  gsutil mb gs://${bucket_name}/
+else
+  V=2 kube::log::status "Bucket already exists"
+fi
+# Path for e2e-node test results
+GCS_JOBS_PATH="gs://${bucket_name}/logs/e2e-node"
+
+ARTIFACTS=${ARTIFACTS:-"/tmp/_artifacts"}
+BUILD_LOG_PATH="${ARTIFACTS}/build-log.txt"
+
+if [[ ! -e $BUILD_LOG_PATH ]]; then
+  echo "Could not find build-log.txt at ${BUILD_LOG_PATH}"
+  exit 1
+fi
+
+# Get start and end timestamps based on build-log.txt file contents
+# Line where the actual tests start
+start_line=$(grep -n -m 1 "^=" ${BUILD_LOG_PATH} | sed 's/\([0-9]*\).*/\1/')
+# Create text file starting where the tests start
+after_start=$(tail -n +${start_line} ${BUILD_LOG_PATH})
+echo "${after_start}" >> build-log-cut.txt
+# Match the first timestamp
+start_time_raw=$(grep -m 1 -o '[0-9][0-9][0-9][0-9][[:blank:]][0-9][0-9]:[0-9][0-9]:[0-9][0-9].[0-9]*' build-log-cut.txt)
+rm build-log-cut.txt
+# Make the date readable by date command (ex: 0101 00:00:00.000 -> 01/01 00:00:00.000)
+start_time=$(echo ${start_time_raw} | sed 's/^.\{2\}/&\//')
+V=2 kube::log::status "Started at ${start_time}"
+# Match the last timestamp in the build-log file
+end_time=$(grep -o '[0-9][0-9][0-9][0-9][[:blank:]][0-9][0-9]:[0-9][0-9]:[0-9][0-9].[0-9]*' ${BUILD_LOG_PATH} | tail -1 | sed 's/^.\{2\}/&\//')
+# Convert to epoch time for Gubernator
+start_time_epoch=$(date -d "${start_time}" +%s)
+end_time_epoch=$(date -d "${end_time}" +%s)
+
+# Make folder name for build from timestamp
+BUILD_STAMP=$(echo $start_time | sed 's/\///' | sed 's/ /_/')
+
+GCS_LOGS_PATH="${GCS_JOBS_PATH}/${BUILD_STAMP}"
+
+# Check if folder for same logs already exists
+if gsutil ls "${GCS_JOBS_PATH}" | grep -q "${BUILD_STAMP}"; then
+  V=2 kube::log::status "Log files already uploaded"
+  echo "Gubernator linked below:"
+  echo "k8s-gubernator.appspot.com/build/${GCS_LOGS_PATH}?local=on"
+  exit
+fi
+
+results=$(find ${ARTIFACTS} -type d -name "results")
+if [[ $results != "" && $results != "${ARTIFACTS}/results" && $results != $ARTIFACTS ]]; then
+  mv $results $ARTIFACTS
+fi
+
+# Upload log files
+for upload_attempt in $(seq 3); do
+  if [[ -d "${ARTIFACTS}" && -n $(ls -A "${ARTIFACTS}") ]]; then
+    V=2 kube::log::status "Uploading artifacts"
+    gsutil -m -q -o "GSUtil:use_magicfile=True" cp -a "${gcs_acl}" -r -c \
+      -z log,xml,xml "${ARTIFACTS}" "${GCS_LOGS_PATH}/artifacts" || continue
+  fi
+  break
+done
+for upload_attempt in $(seq 3); do
+  if [[ -e "${BUILD_LOG_PATH}" ]]; then
+    V=2 kube::log::status "Uploading build log"
+    gsutil -q cp -Z -a "${gcs_acl}" "${BUILD_LOG_PATH}" "${GCS_LOGS_PATH}" || continue
+  fi
+  break
+done
+
+
+# Find the k8s version for started.json
+version=""
+if [[ -e "version" ]]; then
+  version=$(cat "version")
+elif [[ -e "hack/lib/version.sh" ]]; then
+  export KUBE_ROOT="."
+  source "hack/lib/version.sh"
+  kube::version::get_version_vars
+  version="${KUBE_GIT_VERSION-}"
+fi
+if [[ -n "${version}" ]]; then
+  V=2 kube::log::status "Found Kubernetes version: ${version}"
+else
+  V=2 kube::log::status "Could not find Kubernetes version"
+fi
+
+#Find build result from build-log.txt
+if grep -Fxq "Test Suite Passed" "${BUILD_LOG_PATH}"
+  then
+    build_result="SUCCESS"
+else
+    build_result="FAILURE"
+fi
+
+V=4 kube::log::status "Build result is ${build_result}"
+
+if [[ -e "${ARTIFACTS}/started.json" ]]; then
+  rm "${ARTIFACTS}/started.json"
+fi
+
+if [[ -e "${ARTIFACTS}/finished.json" ]]; then
+  rm "${ARTIFACTS}/finished.json"
+fi
+
+V=2 kube::log::status "Constructing started.json and finished.json files"
+echo "{" >> "${ARTIFACTS}/started.json"
+echo "    \"version\": \"${version}\"," >> "${ARTIFACTS}/started.json"
+echo "    \"timestamp\": ${start_time_epoch}," >> "${ARTIFACTS}/started.json"
+echo "    \"jenkins-node\": \"${NODE_NAME:-}\"" >> "${ARTIFACTS}/started.json"
+echo "}" >> "${ARTIFACTS}/started.json"
+
+echo "{" >> "${ARTIFACTS}/finished.json"
+echo "    \"result\": \"${build_result}\"," >> "${ARTIFACTS}/finished.json"
+echo "    \"timestamp\": ${end_time_epoch}" >> "${ARTIFACTS}/finished.json"
+echo "}" >> "${ARTIFACTS}/finished.json"
+
+
+# Upload started.json
+V=2 kube::log::status "Uploading started.json and finished.json"
+V=2 kube::log::status "Run started at ${start_time}"
+json_file="${GCS_LOGS_PATH}/started.json"
+
+for upload_attempt in $(seq 3); do
+  V=2 kube::log::status "Uploading started.json to ${json_file} (attempt ${upload_attempt})"
+  gsutil -q -h "Content-Type:application/json" cp -a "${gcs_acl}" "${ARTIFACTS}/started.json" \
+    "${json_file}" || continue
+  break
+done
+
+# Upload finished.json
+for upload_attempt in $(seq 3); do
+  V=2 kube::log::status "Uploading finished.json to ${GCS_LOGS_PATH} (attempt ${upload_attempt})"
+  gsutil -q -h "Content-Type:application/json" cp -a "${gcs_acl}" "${ARTIFACTS}/finished.json" \
+    "${GCS_LOGS_PATH}/finished.json" || continue
+  break
+done
+
+
+echo "Gubernator linked below:"
+echo "k8s-gubernator.appspot.com/build/${bucket_name}/logs/e2e-node/${BUILD_STAMP}"

test/e2e_node/runner/run_e2e.go

@@ -27,6 +27,7 @@ import (
 	"math/rand"
 	"net/http"
 	"os"
+	"os/exec"
 	"regexp"
 	"sort"
 	"strings"
@@ -56,6 +57,7 @@ var deleteInstances = flag.Bool("delete-instances", true, "If true, delete any i
 var buildOnly = flag.Bool("build-only", false, "If true, build e2e_node_test.tar.gz and exit.")
 var setupNode = flag.Bool("setup-node", false, "When true, current user will be added to docker group on the test machine")
 var instanceMetadata = flag.String("instance-metadata", "", "key/value metadata for instances separated by '=' or '<', 'k=v' means the key is 'k' and the value is 'v'; 'k<p' means the key is 'k' and the value is extracted from the local path 'p', e.g. k1=v1,k2<p2")
+var gubernator = flag.Bool("gubernator", false, "If true, output Gubernator link to view logs")
 
 var computeService *compute.Service
 
@@ -247,8 +249,25 @@ func main() {
 	// Set the exit code if there were failures
 	if !exitOk {
 		fmt.Printf("Failure: %d errors encountered.", errCount)
+		callGubernator(*gubernator)
 		os.Exit(1)
 	}
+	callGubernator(*gubernator)
+}
+
+func callGubernator(gubernator bool) {
+	fmt.Println("Running gubernator.sh")
+	if gubernator {
+		output, err := exec.Command("./test/e2e_node/gubernator.sh", "y").Output()
+
+		if err != nil {
+			fmt.Println("gubernator.sh Failed")
+			fmt.Println(err)
+			return
+		}
+		fmt.Printf("%s", output)
+	}
+	return
 }
 
 func (a *Archive) getArchive() (string, error) {