K3s Integration test fixes (#4341)

pkg/etcd/etcd_int_test.go

@@ -32,7 +32,7 @@ var _ = Describe("etcd snapshots", func() {
 		It("starts up with no problems", func() {
 			Eventually(func() (string, error) {
 				return testutil.K3sCmd("kubectl", "get", "pods", "-A")
-			}, "90s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
+			}, "180s", "5s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
 		})
 		It("saves an etcd snapshot", func() {
 			Expect(testutil.K3sCmd("etcd-snapshot", "save")).

scripts/build-tests-sonobuoy

@@ -8,16 +8,22 @@ OUTFILE="./dist/artifacts/k3s-int-tests.yaml"
 
 # Compile all integration tests and containerize them
 mkdir -p dist/artifacts
-go test -c -v -ldflags "-X 'github.com/rancher/k3s/tests/util.existingServer=True'" -o dist/artifacts/k3s-integration-1.test ./tests/integration/... -run Integration
 
+# Integration tests under /pkg
 PKG_TO_TEST=$(find ./pkg/ -type f -name "*_int_test.go" | sed -r 's|/[^/]+$||' |sort -u)
-INDEX=1
 for i in $PKG_TO_TEST; do
-    echo $i
-    go test -c -v -ldflags "-X 'github.com/rancher/k3s/tests/util.existingServer=True'" -o dist/artifacts/k3s-integration-$INDEX.test $i -run Integration
-    INDEX=$(expr $INDEX + 1)
+    name=$(echo "${i##*/}")
+    echo $name
+    go test -c -v -ldflags "-X 'github.com/rancher/k3s/tests/util.existingServer=True'" -o dist/artifacts/k3s-integration-$name.test $i -run Integration
+done
+
+# Integration tests under /tests
+PKG_TO_TEST=$(find ./tests/integration -type f -name "*_int_test.go" | sed -r 's|/[^/]+$||' |sort -u)
+for i in $PKG_TO_TEST; do
+    name=$(echo "${i##*/}")
+    echo $name
+    go test -c -v -ldflags "-X 'github.com/rancher/k3s/tests/util.existingServer=True'" -o dist/artifacts/k3s-integration-$name.test $i -run Integration
 done
-go test -c -v -ldflags "-X 'github.com/rancher/k3s/tests/util.existingServer=True'" -o dist/artifacts/k3s-integration-$INDEX.test ./tests/integration/... -run Integration
 docker build -f ./tests/integration/Dockerfile.test -t $REPO .
 docker save $REPO -o ./dist/artifacts/$REPO.tar
 

tests/TESTING.md

@@ -60,25 +60,25 @@ To facilitate K3s CLI testing, see `tests/util/cmd.go` helper functions.
 Integration tests can be placed in two areas:  
 
 1. Next to the go package they intend to test.
-2. In `tests/integration/` for package agnostic testing.  
+2. In `tests/integration/<TESTNAME>` for package agnostic testing.  
 
 Package specific integration tests should use the `<PACKAGE_UNDER_TEST>_test` package.  
 Package agnostic integration tests should use the `integration` package.  
 All integration test files should be named: `<TEST_NAME>_int_test.go`  
 All integration test functions should be named: `Test_Integration<Test_Name>`.  
 See the [etcd snapshot test](https://github.com/k3s-io/k3s/blob/master/pkg/etcd/etcd_int_test.go) as a package specific example.  
-See the [local storage test](https://github.com/k3s-io/k3s/blob/master/tests/integration/localstorage_int_test.go) as a package agnostic example.
+See the [local storage test](https://github.com/k3s-io/k3s/blob/master/tests/integration/localstorage/localstorage_int_test.go) as a package agnostic example.
 
 ### Running
 
 Integration tests can be run with no k3s cluster present, each test will spin up and kill the appropriate k3s server it needs.
 ```bash
-go test ./pkg/... ./tests/... -run Integration
+go test ./pkg/... ./tests/integration/... -run Integration
 ```
 
 Integration tests can be run on an existing single-node cluster via compile time flag, tests will skip if the server is not configured correctly.
 ```bash
-go test -ldflags "-X 'github.com/rancher/k3s/tests/util.existingServer=True'" ./pkg/... ./tests/... -run Integration
+go test -ldflags "-X 'github.com/rancher/k3s/tests/util.existingServer=True'" ./pkg/... ./tests/integration/... -run Integration
 ```
 
 Integration tests can also be run via a [Sonobuoy](https://sonobuoy.io/docs/v0.53.2/) plugin on an existing single-node cluster.
@@ -111,7 +111,7 @@ See the [upgrade cluster test](https://github.com/k3s-io/k3s/blob/master/tests/e
 Generally, E2E tests are run as a nightly Jenkins job for QA. They can still be run locally but additional setup may be required.
 
 ```bash
-go test ./tests/... -run E2E
+go test ./tests/e2e... -run E2E
 ```
 
 ## Contributing New Or Updated Tests

tests/integration/dual_stack_int_test.go → tests/integration/dualstack/dualstack_int_test.go

@@ -10,11 +10,17 @@ import (
 	testutil "github.com/rancher/k3s/tests/util"
 )
 
-var dualStackServerArgs = []string{"--cluster-init", "--cluster-cidr 10.42.0.0/16,2001:cafe:42:0::/56", "--service-cidr 10.43.0.0/16,2001:cafe:42:1::/112"}
+var dualStackServer *testutil.K3sServer
+var dualStackServerArgs = []string{
+	"--cluster-init",
+	"--cluster-cidr 10.42.0.0/16,2001:cafe:42:0::/56",
+	"--service-cidr 10.43.0.0/16,2001:cafe:42:1::/112",
+	"--disable-network-policy",
+}
 var _ = BeforeSuite(func() {
 	if !testutil.IsExistingServer() {
 		var err error
-		server, err = testutil.K3sStartServer(dualStackServerArgs...)
+		dualStackServer, err = testutil.K3sStartServer(dualStackServerArgs...)
 		Expect(err).ToNot(HaveOccurred())
 	}
 })
@@ -29,21 +35,21 @@ var _ = Describe("dual stack", func() {
 		It("starts up with no problems", func() {
 			Eventually(func() (string, error) {
 				return testutil.K3sCmd("kubectl", "get", "pods", "-A")
-			}, "90s", "1s").Should(MatchRegexp("kube-system.+traefik.+1\\/1.+Running"))
+			}, "180s", "5s").Should(MatchRegexp("kube-system.+traefik.+1\\/1.+Running"))
 		})
 		It("creates pods with two IPs", func() {
-			podname, err := testutil.K3sCmd("kubectl", "get", "pods", "-nkube-system", "-ojsonpath={.items[?(@.metadata.labels.app\\.kubernetes\\.io/name==\"traefik\")].metadata.name}")
-			Expect(err).NotTo(HaveOccurred())
-			result, err := testutil.K3sCmd("kubectl", "exec", podname, "-nkube-system", "--", "ip", "a")
-			Expect(result).To(ContainSubstring("2001:cafe:42:"))
+			podname, err := testutil.K3sCmd("kubectl", "get", "pods", "-n", "kube-system", "-o", "jsonpath={.items[?(@.metadata.labels.app\\.kubernetes\\.io/name==\"traefik\")].metadata.name}")
 			Expect(err).NotTo(HaveOccurred())
+			Eventually(func() (string, error) {
+				return testutil.K3sCmd("kubectl", "exec", podname, "-n", "kube-system", "--", "ip", "a")
+			}, "5s", "1s").Should(ContainSubstring("2001:cafe:42:"))
 		})
 	})
 })
 
 var _ = AfterSuite(func() {
 	if !testutil.IsExistingServer() {
-		Expect(testutil.K3sKillServer(server)).To(Succeed())
+		Expect(testutil.K3sKillServer(dualStackServer)).To(Succeed())
 	}
 })
 

tests/integration/localstorage_int_test.go → tests/integration/localstorage/localstorage_int_test.go

@@ -13,20 +13,21 @@ import (
 	testutil "github.com/rancher/k3s/tests/util"
 )
 
-var server *testutil.K3sServer
-var serverArgs = []string{"--cluster-init"}
+var localStorageServer *testutil.K3sServer
+var localStorageServerArgs = []string{"--cluster-init"}
+var testDataDir = "../../testdata/"
 var _ = BeforeSuite(func() {
 	if !testutil.IsExistingServer() {
 		var err error
-		server, err = testutil.K3sStartServer(serverArgs...)
+		localStorageServer, err = testutil.K3sStartServer(localStorageServerArgs...)
 		Expect(err).ToNot(HaveOccurred())
 	}
 })
 
 var _ = Describe("local storage", func() {
 	BeforeEach(func() {
-		if testutil.IsExistingServer() && !testutil.ServerArgsPresent(serverArgs) {
-			Skip("Test needs k3s server with: " + strings.Join(serverArgs, " "))
+		if testutil.IsExistingServer() && !testutil.ServerArgsPresent(localStorageServerArgs) {
+			Skip("Test needs k3s server with: " + strings.Join(localStorageServerArgs, " "))
 		}
 	})
 	When("a new local storage is created", func() {
@@ -36,12 +37,12 @@ var _ = Describe("local storage", func() {
 			}, "90s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
 		})
 		It("creates a new pvc", func() {
-			result, err := testutil.K3sCmd("kubectl", "create", "-f", "../testdata/localstorage_pvc.yaml")
+			result, err := testutil.K3sCmd("kubectl", "create", "-f", testDataDir+"localstorage_pvc.yaml")
 			Expect(result).To(ContainSubstring("persistentvolumeclaim/local-path-pvc created"))
 			Expect(err).NotTo(HaveOccurred())
 		})
 		It("creates a new pod", func() {
-			Expect(testutil.K3sCmd("kubectl", "create", "-f", "../testdata/localstorage_pod.yaml")).
+			Expect(testutil.K3sCmd("kubectl", "create", "-f", testDataDir+"localstorage_pod.yaml")).
 				To(ContainSubstring("pod/volume-test created"))
 		})
 		It("shows storage up in kubectl", func() {
@@ -81,7 +82,7 @@ var _ = Describe("local storage", func() {
 
 var _ = AfterSuite(func() {
 	if !testutil.IsExistingServer() {
-		Expect(testutil.K3sKillServer(server)).To(Succeed())
+		Expect(testutil.K3sKillServer(localStorageServer)).To(Succeed())
 	}
 })
 

tests/util/cmd.go

@@ -15,6 +15,14 @@ import (
 // Compile-time variable
 var existingServer = "False"
 
+const lockFile = "/var/lock/k3s-test.lock"
+
+type K3sServer struct {
+	cmd     *exec.Cmd
+	scanner *bufio.Scanner
+	lock    int
+}
+
 func findK3sExecutable() string {
 	// if running on an existing cluster, it maybe installed via k3s.service
 	// or run manually from dist/artifacts/k3s
@@ -25,7 +33,8 @@ func findK3sExecutable() string {
 		}
 	}
 	k3sBin := "dist/artifacts/k3s"
-	for {
+	i := 0
+	for ; i < 20; i++ {
 		_, err := os.Stat(k3sBin)
 		if err != nil {
 			k3sBin = "../" + k3sBin
@@ -33,6 +42,9 @@ func findK3sExecutable() string {
 		}
 		break
 	}
+	if i == 20 {
+		logrus.Fatal("Unable to find k3s executable")
+	}
 	return k3sBin
 }
 
@@ -112,22 +124,21 @@ func FindStringInCmdAsync(scanner *bufio.Scanner, target string) bool {
 	return false
 }
 
-type K3sServer struct {
-	cmd     *exec.Cmd
-	scanner *bufio.Scanner
-	lock    int
-}
-
 // K3sStartServer acquires an exclusive lock on a temporary file, then launches a k3s cluster
 // with the provided arguments. Subsequent/parallel calls to this function will block until
 // the original lock is cleared using K3sKillServer
-func K3sStartServer(cmdArgs ...string) (*K3sServer, error) {
+func K3sStartServer(inputArgs ...string) (*K3sServer, error) {
 	logrus.Info("waiting to get server lock")
-	k3sLock, err := flock.Acquire("/var/lock/k3s-test.lock")
+	k3sLock, err := flock.Acquire(lockFile)
 	if err != nil {
 		return nil, err
 	}
 
+	var cmdArgs []string
+	for _, arg := range inputArgs {
+		cmdArgs = append(cmdArgs, strings.Fields(arg)...)
+	}
+
 	k3sBin := findK3sExecutable()
 	var cmd *exec.Cmd
 	if IsRoot() {
@@ -157,5 +168,11 @@ func K3sKillServer(server *K3sServer) error {
 			return err
 		}
 	}
-	return flock.Release(server.lock)
+	if err := flock.Release(server.lock); err != nil {
+		return err
+	}
+	if !flock.CheckLock(lockFile) {
+		return os.Remove(lockFile)
+	}
+	return nil
 }