Skip to content

K
k3s
Commit f3a992aa authored by Zach Loafman's avatar Zach Loafman
Browse files
Options

Run e2e tests in deterministic random order

Currently, we run the e2e tests in whatever order readdir happens to return, which is random on some filesystems, name sorted on others, create order on others, etc. Eventually, we may want to be automatically hermetic between e2e tests (especially as we introduce more resource destructive tests), but until then, it would be useful if we permute the test order randomly between runs to ensure that developers don't accidentally rely on a particular order. This introduces a form of forced hermeticism, since improper state cleanup from one test may not perturb a given test, but there's probably *a* test in the suite that the order will perturb, so the RNG will find that order eventually. Adds logging of the generated seed, and an --orderseed argument that can be used to re-run in the same order. Also sorts the pass/fail list now for easier human reading.
parent 3910b2d6
Hide whitespace changes
Inline Side-by-side
Showing with 32 additions and 1 deletion
hack/e2e.go
View file @ f3a992aa
...@@ -22,12 +22,15 @@ import ( ...@@ -22,12 +22,15 @@ import (
"flag" "flag"
"fmt" "fmt"
"log" "log"
"math/rand"
"os" "os"
"os/exec" "os/exec"
"os/signal" "os/signal"
"path" "path"
"path/filepath" "path/filepath"
"sort"
"strings" "strings"
"time"
) )
var ( var (
...@@ -36,6 +39,7 @@ var ( ...@@ -36,6 +39,7 @@ var (
up = flag.Bool("up", false, "If true, start the the e2e cluster. If cluster is already up, recreate it.") up = flag.Bool("up", false, "If true, start the the e2e cluster. If cluster is already up, recreate it.")
push = flag.Bool("push", false, "If true, push to e2e cluster. Has no effect if -up is true.") push = flag.Bool("push", false, "If true, push to e2e cluster. Has no effect if -up is true.")
down = flag.Bool("down", false, "If true, tear down the cluster before exiting.") down = flag.Bool("down", false, "If true, tear down the cluster before exiting.")
orderseed = flag.Int64("orderseed", 0, "If non-zero, seed of random test shuffle order. (Otherwise random.)")
test = flag.Bool("test", false, "Run all tests in hack/e2e-suite.") test = flag.Bool("test", false, "Run all tests in hack/e2e-suite.")
tests = flag.String("tests", "", "Run only tests in hack/e2e-suite matching this glob. Ignored if -test is set.") tests = flag.String("tests", "", "Run only tests in hack/e2e-suite matching this glob. Ignored if -test is set.")
root = flag.String("root", absOrDie(filepath.Clean(filepath.Join(path.Base(os.Args[0]), ".."))), "Root directory of kubernetes repository.") root = flag.String("root", absOrDie(filepath.Clean(filepath.Join(path.Base(os.Args[0]), ".."))), "Root directory of kubernetes repository.")
...@@ -134,6 +138,14 @@ func tryUp() bool { ...@@ -134,6 +138,14 @@ func tryUp() bool {
return runBash("up", path.Join(*root, "/cluster/kube-up.sh; test-setup;")) return runBash("up", path.Join(*root, "/cluster/kube-up.sh; test-setup;"))
} }
// Fisher-Yates shuffle using the given RNG r
func shuffleStrings(strings []string, r *rand.Rand) {
for i := len(strings) - 1; i > 0; i-- {
j := r.Intn(i + 1)
strings[i], strings[j] = strings[j], strings[i]
}
}
func Test() (failed, passed []string) { func Test() (failed, passed []string) {
defer runBashUntil("watchEvents", "$KUBECTL --watch-only get events")() defer runBashUntil("watchEvents", "$KUBECTL --watch-only get events")()
// run tests! // run tests!
...@@ -147,6 +159,7 @@ func Test() (failed, passed []string) { ...@@ -147,6 +159,7 @@ func Test() (failed, passed []string) {
log.Fatal("Couldn't read names in e2e-suite dir") log.Fatal("Couldn't read names in e2e-suite dir")
} }
toRun := make([]string, 0, len(names))
for i := range names { for i := range names {
name := names[i] name := names[i]
if name == "." || name == ".." { if name == "." || name == ".." {
...@@ -155,8 +168,24 @@ func Test() (failed, passed []string) { ...@@ -155,8 +168,24 @@ func Test() (failed, passed []string) {
if match, err := path.Match(*tests, name); !match && err == nil { if match, err := path.Match(*tests, name); !match && err == nil {
continue continue
} }
if err != nil {
log.Fatal("Bad test pattern: %v", tests)
}
toRun = append(toRun, name)
}
if *orderseed == 0 {
// Use low order bits of NanoTime as the default seed. (Using
// all the bits makes for a long, very similar looking seed
// between runs.)
*orderseed = time.Now().UnixNano() & (1<<32 - 1)
}
sort.Strings(toRun)
shuffleStrings(toRun, rand.New(rand.NewSource(*orderseed)))
log.Printf("Running tests matching %v shuffled with seed %#x: %v", *tests, *orderseed, toRun)
for _, name := range toRun {
absName := filepath.Join(*root, "hack", "e2e-suite", name) absName := filepath.Join(*root, "hack", "e2e-suite", name)
log.Printf("%v matches %v. Starting test.", name, *tests) log.Printf("Starting test %v.", name)
if runBash(name, absName) { if runBash(name, absName) {
log.Printf("%v passed", name) log.Printf("%v passed", name)
passed = append(passed, name) passed = append(passed, name)
...@@ -166,6 +195,8 @@ func Test() (failed, passed []string) { ...@@ -166,6 +195,8 @@ func Test() (failed, passed []string) {
} }
} }
sort.Strings(passed)
sort.Strings(failed)
return return
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment