Run shell tests under Ginkgo, changing all reporting to JUnit

hack/e2e.go

@@ -30,10 +30,8 @@ import (
 	"os/signal"
 	"path"
 	"path/filepath"
-	"sort"
 	"strconv"
 	"strings"
-	"time"
 )
 
 var (
@@ -44,12 +42,8 @@ var (
 	push             = flag.Bool("push", false, "If true, push to e2e cluster. Has no effect if -up is true.")
 	pushup           = flag.Bool("pushup", false, "If true, push to e2e cluster if it's up, otherwise start the e2e cluster.")
 	down             = flag.Bool("down", false, "If true, tear down the cluster before exiting.")
-	orderseed        = flag.Int64("orderseed", 0, "If non-zero, seed of random test shuffle order. (Otherwise random.)")
-	test             = flag.Bool("test", false, "Run all tests in hack/e2e-suite.")
-	tests            = flag.String("tests", "", "Run only tests in hack/e2e-suite matching this glob. Ignored if -test is set.")
-	times            = flag.Int("times", 1, "Number of times each test is eligible to be run. Individual order is determined by shuffling --times instances of each test using --orderseed (like a multi-deck shoe of cards).")
+	test             = flag.Bool("test", false, "Run Ginkgo tests.")
 	root             = flag.String("root", absOrDie(filepath.Clean(filepath.Join(path.Base(os.Args[0]), ".."))), "Root directory of kubernetes repository.")
-	tap              = flag.Bool("tap", false, "Enable Test Anything Protocol (TAP) output (disables --verbose, only failure output recorded)")
 	verbose          = flag.Bool("v", false, "If true, print all command output.")
 	trace_bash       = flag.Bool("trace-bash", false, "If true, pass -x to bash to trace all bash commands")
 	checkVersionSkew = flag.Bool("check_version_skew", true, ""+
@@ -95,21 +89,6 @@ func main() {
 	flag.Parse()
 	signal.Notify(signals, os.Interrupt)
 
-	if *tap {
-		fmt.Printf("TAP version 13\n")
-		log.SetPrefix("# ")
-
-		// TODO: this limitation is fixable by moving runBash to
-		// outputing to temp files, which still lets people check on
-		// stuck things interactively. The current stdout/stderr
-		// approach isn't really going to work with TAP, though.
-		*verbose = false
-	}
-
-	if *test {
-		*tests = "*"
-	}
-
 	if *isup {
 		status := 1
 		if IsUp() {
@@ -168,8 +147,8 @@ func main() {
 	switch {
 	case *ctlCmd != "":
 		failure = !runBash("'kubectl "+*ctlCmd+"'", "$KUBECTL "+*ctlCmd)
-	case *tests != "":
-		failure = PrintResults(Test())
+	case *test:
+		failure = Test()
 	}
 
 	if *down {
@@ -278,7 +257,7 @@ func shuffleStrings(strings []string, r *rand.Rand) {
 	}
 }
 
-func Test() (results ResultsByTest) {
+func Test() bool {
 	defer runBashUntil("watchEvents", "while true; do $KUBECTL --watch-only get events; done")()
 
 	if !IsUp() {
@@ -287,128 +266,7 @@ func Test() (results ResultsByTest) {
 
 	ValidateClusterSize()
 
-	// run tests!
-	dir, err := os.Open(filepath.Join(*root, "hack", "e2e-suite"))
-	if err != nil {
-		log.Fatal("Couldn't open e2e-suite dir")
-	}
-	defer dir.Close()
-	names, err := dir.Readdirnames(0)
-	if err != nil {
-		log.Fatal("Couldn't read names in e2e-suite dir")
-	}
-
-	toRun := make([]string, 0, len(names))
-	for i := range names {
-		name := names[i]
-		if name == "." || name == ".." {
-			continue
-		}
-		if match, err := path.Match(*tests, name); !match && err == nil {
-			continue
-		}
-		if err != nil {
-			log.Fatalf("Bad test pattern: %v", *tests)
-		}
-		toRun = append(toRun, name)
-	}
-
-	if *orderseed == 0 {
-		// Use low order bits of NanoTime as the default seed. (Using
-		// all the bits makes for a long, very similar looking seed
-		// between runs.)
-		*orderseed = time.Now().UnixNano() & (1<<32 - 1)
-	}
-	sort.Strings(toRun)
-	if *times != 1 {
-		if *times <= 0 {
-			log.Fatal("Invalid --times (negative or no testing requested)!")
-		}
-		newToRun := make([]string, 0, *times*len(toRun))
-		for i := 0; i < *times; i++ {
-			newToRun = append(newToRun, toRun...)
-		}
-		toRun = newToRun
-	}
-	shuffleStrings(toRun, rand.New(rand.NewSource(*orderseed)))
-	log.Printf("Running tests matching %v shuffled with seed %#x: %v", *tests, *orderseed, toRun)
-	results = ResultsByTest{}
-	if *tap {
-		fmt.Printf("1..%v\n", len(toRun))
-	}
-	for i, name := range toRun {
-		absName := filepath.Join(*root, "hack", "e2e-suite", name)
-		log.Printf("Starting test [%v/%v]: %v", i+1, len(toRun), name)
-		start := time.Now()
-		testResult := results[name]
-		res, stdout, stderr := runBashWithOutputs(name, absName)
-		// The duration_ms output is an undocumented Jenkins TAP
-		// plugin feature for test duration. One might think _ms means
-		// milliseconds, but Jenkins interprets this field in seconds.
-		duration_secs := time.Now().Sub(start).Seconds()
-		if res {
-			fmt.Printf("ok %v - %v\n", i+1, name)
-			if *tap {
-				fmt.Printf("  ---\n  duration_ms: %.3f\n  ...\n", duration_secs)
-			}
-			testResult.Pass++
-		} else {
-			fmt.Printf("not ok %v - %v\n", i+1, name)
-			if *tap {
-				fmt.Printf("  ---\n  duration_ms: %.3f\n", duration_secs)
-			}
-			printBashOutputs("  ", "    ", stdout, stderr, *tap)
-			if *tap {
-				fmt.Printf("  ...\n")
-			}
-			testResult.Fail++
-		}
-		results[name] = testResult
-	}
-
-	return
-}
-
-func PrintResults(results ResultsByTest) bool {
-	failures := 0
-
-	passed := []string{}
-	flaky := []string{}
-	failed := []string{}
-	for test, result := range results {
-		if result.Pass > 0 && result.Fail == 0 {
-			passed = append(passed, test)
-		} else if result.Pass > 0 && result.Fail > 0 {
-			flaky = append(flaky, test)
-			failures += result.Fail
-		} else {
-			failed = append(failed, test)
-			failures += result.Fail
-		}
-	}
-	sort.Strings(passed)
-	sort.Strings(flaky)
-	sort.Strings(failed)
-	printSubreport("Passed", passed, results)
-	printSubreport("Flaky", flaky, results)
-	printSubreport("Failed", failed, results)
-	if failures > 0 {
-		log.Printf("%v test(s) failed.", failures)
-	} else {
-		log.Printf("Success!")
-	}
-
-	return failures > 0
-}
-
-func printSubreport(title string, tests []string, results ResultsByTest) {
-	report := title + " tests:"
-
-	for _, test := range tests {
-		result := results[test]
-		report += fmt.Sprintf(" %v[%v/%v]", test, result.Pass, result.Pass+result.Fail)
-	}
-	log.Printf(report)
+	return runBash("Ginkgo tests", filepath.Join(*root, "hack", "ginkgo-e2e.sh"))
 }
 
 // All nonsense below is temporary until we have go versions of these things.
@@ -451,10 +309,6 @@ func runBashUntil(stepName, bashFragment string) func() {
 		cmd.Process.Signal(os.Interrupt)
 		headerprefix := stepName + " "
 		lineprefix := "  "
-		if *tap {
-			headerprefix = "# " + headerprefix
-			lineprefix = "# " + lineprefix
-		}
 		printBashOutputs(headerprefix, lineprefix, string(stdout.Bytes()), string(stderr.Bytes()), false)
 	}
 }

hack/e2e-suite/goe2e.sh → hack/ginkgo-e2e.sh

@@ -18,7 +18,7 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
-KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
+KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
 
 : ${KUBE_VERSION_ROOT:=${KUBE_ROOT}}
 : ${KUBECTL:="${KUBE_VERSION_ROOT}/cluster/kubectl.sh"}

hack/jenkins/e2e.sh

@@ -77,11 +77,9 @@ if [[ ! -z ${E2E_SET_CLUSTER_API_VERSION:-} ]]; then
     export CLUSTER_API_VERSION=$(echo ${GITHASH} | cut -c 2-)
 fi
 
-# Have cmd/e2e run by goe2e.sh generate JUnit report in ${WORKSPACE}/junit*.xml
-export E2E_REPORT_DIR=${WORKSPACE}
-
-go run ./hack/e2e.go ${E2E_OPT} -v --down
-go run ./hack/e2e.go ${E2E_OPT} -v --up
-go run ./hack/e2e.go -v --ctl="version --match-server-version=false"
-go run ./hack/e2e.go ${E2E_OPT} --test --tap | tee ../e2e.${JOB_NAME}.${BUILD_NUMBER}.${GITHASH}.tap
-go run ./hack/e2e.go ${E2E_OPT} -v --down
+export KUBE_CONFIG_FILE="config-test.sh"
+cluster/kube-down.sh
+cluster/kube-up.sh
+cluster/kubectl.sh version
+hack/ginkgo-e2e.sh --report_dir=${WORKSPACE}
+cluster/kube-down.sh

test/e2e/shell.go

+/*
+Copyright 2015 Google Inc. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+
+	. "github.com/onsi/ginkgo"
+)
+
+var (
+	root = absOrDie(filepath.Clean(filepath.Join(path.Base(os.Args[0]), "..")))
+)
+
+var _ = Describe("Shell", func() {
+	// Slurp up all the tests in hack/e2e-suite
+	bashE2ERoot := filepath.Join(root, "hack/e2e-suite")
+	files, err := ioutil.ReadDir(bashE2ERoot)
+	if err != nil {
+		Fail(err.Error())
+	}
+
+	for _, file := range files {
+		fileName := file.Name() // Make a copy
+		It(fmt.Sprintf("tests that %v passes", fileName), func() {
+			runCmdTest(filepath.Join(bashE2ERoot, fileName))
+		})
+	}
+})
+
+func absOrDie(path string) string {
+	out, err := filepath.Abs(path)
+	if err != nil {
+		panic(err)
+	}
+	return out
+}
+
+// Runs the given cmd test.
+func runCmdTest(path string) {
+	By(fmt.Sprintf("Running %v", path))
+	cmd := exec.Command(path)
+	cmd.Stdout = bytes.NewBuffer(nil)
+	cmd.Stderr = cmd.Stdout
+
+	if err := cmd.Run(); err != nil {
+		Fail(fmt.Sprintf("Error running %v:\nCommand output:\n%v\n", cmd, cmd.Stdout))
+		return
+	}
+	return
+}