Convert docker conformance test to go test framework (#11643)

scripts/test

@@ -19,32 +19,34 @@ mkdir -p $artifacts
 
 docker ps
 
+export K3S_IMAGE="rancher/k3s:${VERSION_TAG}${SUFFIX}"
+
 # ---
 # Only run PR tests on arm arch, we use GitHub Actions for amd64 and arm64
 # Run all tests on tag events, as we want test failures to block the release
 if [ "$ARCH" == 'arm' ] ||  [ "$DRONE_BUILD_EVENT" = 'tag' ]; then
 
-  export K3S_IMAGE="rancher/k3s:${VERSION_TAG}${SUFFIX}"
   go test ./tests/docker/basics/basics_test.go -k3sImage="$K3S_IMAGE"
   echo "Did go test basics $?"
 
-  . ./tests/docker/test-run-basics
-  echo "Did test-run-basics $?"
+  # Extract v1.XX minor version for skew and upgrade tests
+  minor_version=$(echo $VERSION_K8S | cut -d '.' -f1,2)
 
-  . ./tests/docker/test-run-cacerts
-  echo "Did test-run-cacerts $?"
+  go test ./tests/docker/cacerts/cacerts_test.go -k3sImage="$K3S_IMAGE"
+  echo "Did go test cacerts $?"
 
-  . ./tests/docker/test-run-compat
-  echo "Did test-run-compat $?"
+  go test ./tests/docker/skew/skew_test.go -k3sImage="$K3S_IMAGE" -channel="$minor_version"
+  echo "Did go test skew $?"
 
-  . ./tests/docker/test-run-bootstraptoken
-  echo "Did test-run-bootstraptoken $?"
+  go test ./tests/docker/bootstraptoken/bootstraptoken_test.go -k3sImage="$K3S_IMAGE"
+  echo "Did go test bootstraptoken $?"
 
-  . ./tests/docker/test-run-upgrade
-  echo "Did test-run-upgrade $?"
+  go test ./tests/docker/upgrade/upgrade_test.go -k3sImage="$K3S_IMAGE" -channel="$minor_version"
+  echo "Did go test upgrade $?"
+  
+  go test ./tests/docker/lazypull/lazypull_test.go -k3sImage="$K3S_IMAGE"
+  echo "Did go test lazypull $?"
 
-  . ./tests/docker/test-run-lazypull
-  echo "Did test-run-lazypull $?"
 fi
 
 
@@ -69,10 +71,10 @@ fi
 # ---
 
 if [ "$DRONE_BUILD_EVENT" = 'cron' ]; then
-  E2E_OUTPUT=$artifacts test-run-sonobuoy serial
-  echo "Did test-run-sonobuoy serial $?"
-  test-run-sonobuoy etcd serial
-  echo "Did test-run-sonobuoy-etcd serial $?"
+  run-go-test ./test/docker/conformance/conformance_test.go -k3sImage="$K3S_IMAGE" -db sqlite -serial -ginkgo.v
+  echo "Did go conformance sqlite serial $?"
+  run-go-test ./test/docker/conformance/conformance_test.go -k3sImage="$K3S_IMAGE" -db etcd -serial -ginkgo.v
+  echo "Did go conformance etcd serial $?"
   test-run-sonobuoy mysql serial
   echo "Did test-run-sonobuoy-mysqk serial $?"
   test-run-sonobuoy postgres serial
@@ -89,8 +91,10 @@ if [ "$DRONE_BUILD_EVENT" = 'cron' ]; then
 
   E2E_OUTPUT=$artifacts test-run-sonobuoy parallel
   echo "Did test-run-sonobuoy parallel $?"
-  test-run-sonobuoy etcd parallel
-  echo "Did test-run-sonobuoy-etcd parallel $?"
+  run-go-test ./test/docker/conformance/conformance_test.go -k3sImage="$K3S_IMAGE" -db sqlite -ginkgo.v
+  echo "Did go conformance sqlite parallel $?"
+  run-go-test ./test/docker/conformance/conformance_test.go -k3sImage="$K3S_IMAGE" -db etcd -ginkgo.v
+  echo "Did go test conformance etcd parallel $?"
   test-run-sonobuoy mysql parallel
   echo "Did test-run-sonobuoy-mysql parallel $?"
   test-run-sonobuoy postgres parallel

tests/docker/conformance/conformance_test.go

+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/k3s-io/k3s/tests"
+	tester "github.com/k3s-io/k3s/tests/docker"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var k3sImage = flag.String("k3sImage", "", "The k3s image used to provision containers")
+var db = flag.String("db", "", "The database to use for the tests (sqlite, etcd, mysql, postgres)")
+var serial = flag.Bool("serial", false, "Run the Serial Conformance Tests")
+var config *tester.TestConfig
+
+func Test_DockerConformance(t *testing.T) {
+	flag.Parse()
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Conformance Docker Test Suite")
+}
+
+var _ = Describe("Conformance Tests", Ordered, func() {
+
+	Context("Setup Cluster", func() {
+		It("should provision servers and agents", func() {
+			var err error
+			config, err = tester.NewTestConfig(*k3sImage)
+			Expect(err).NotTo(HaveOccurred())
+			config.DBType = *db
+			Expect(config.ProvisionServers(1)).To(Succeed())
+			Expect(config.ProvisionAgents(1)).To(Succeed())
+			Eventually(func() error {
+				return tests.CheckDefaultDeployments(config.KubeconfigFile)
+			}, "90s", "5s").Should(Succeed())
+			Eventually(func() error {
+				return tests.NodesReady(config.KubeconfigFile, config.GetNodeNames())
+			}, "40s", "5s").Should(Succeed())
+		})
+	})
+	Context("Run Hydrophone Conformance tests", func() {
+		It("should download hydrophone", func() {
+			hydrophoneVersion := "v0.6.0"
+			hydrophoneArch := runtime.GOARCH
+			if hydrophoneArch == "amd64" {
+				hydrophoneArch = "x86_64"
+			}
+			hydrophoneURL := fmt.Sprintf("https://github.com/kubernetes-sigs/hydrophone/releases/download/%s/hydrophone_Linux_%s.tar.gz",
+				hydrophoneVersion, hydrophoneArch)
+			cmd := fmt.Sprintf("curl -L %s | tar -xzf - -C %s", hydrophoneURL, config.TestDir)
+			_, err := tester.RunCommand(cmd)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(os.Chmod(filepath.Join(config.TestDir, "hydrophone"), 0755)).To(Succeed())
+		})
+		// Takes about 15min to run, so expect nothing to happen for a while
+		It("should run parallel conformance tests", func() {
+			if *serial {
+				Skip("Skipping parallel conformance tests")
+			}
+			cmd := fmt.Sprintf("%s --focus=\"Conformance\" --skip=\"Serial|Flaky\" -v 2 -p %d --kubeconfig %s",
+				filepath.Join(config.TestDir, "hydrophone"),
+				runtime.NumCPU()/2,
+				config.KubeconfigFile)
+			By("Hydrophone: " + cmd)
+			hc, err := StartCmd(cmd)
+			Expect(err).NotTo(HaveOccurred())
+			// Periodically check the number of tests that have run, since the hydrophone output does not support a progress status
+			// Taken from https://github.com/kubernetes-sigs/hydrophone/issues/223#issuecomment-2547174722
+			go func() {
+				cmd := fmt.Sprintf("kubectl exec -n=conformance e2e-conformance-test -c output-container --kubeconfig=%s -- cat /tmp/results/e2e.log | grep -o \"•\" | wc -l",
+					config.KubeconfigFile)
+				for i := 1; ; i++ {
+					time.Sleep(120 * time.Second)
+					if hc.ProcessState != nil {
+						break
+					}
+					res, _ := tester.RunCommand(cmd)
+					res = strings.TrimSpace(res)
+					fmt.Printf("Status Report %d: %s tests complete\n", i, res)
+				}
+			}()
+			Expect(hc.Wait()).To(Succeed())
+		})
+		It("should run serial conformance tests", func() {
+			if !*serial {
+				Skip("Skipping serial conformance tests")
+			}
+			cmd := fmt.Sprintf("%s --focus=\"Serial\" --skip=\"Flaky\"  -v 2 --kubeconfig %s",
+				filepath.Join(config.TestDir, "hydrophone"),
+				config.KubeconfigFile)
+			By("Hydrophone: " + cmd)
+			hc, err := StartCmd(cmd)
+			Expect(err).NotTo(HaveOccurred())
+			go func() {
+				cmd := fmt.Sprintf("kubectl exec -n=conformance e2e-conformance-test -c output-container --kubeconfig=%s -- cat /tmp/results/e2e.log | grep -o \"•\" | wc -l",
+					config.KubeconfigFile)
+				for i := 1; ; i++ {
+					time.Sleep(120 * time.Second)
+					if hc.ProcessState != nil {
+						break
+					}
+					res, _ := tester.RunCommand(cmd)
+					res = strings.TrimSpace(res)
+					fmt.Printf("Status Report %d: %s tests complete\n", i, res)
+				}
+			}()
+			Expect(hc.Wait()).To(Succeed())
+		})
+	})
+})
+
+var failed bool
+var _ = AfterEach(func() {
+	failed = failed || CurrentSpecReport().Failed()
+})
+
+var _ = AfterSuite(func() {
+	if config != nil && !failed {
+		config.Cleanup()
+	}
+})
+
+// StartCmd starts a command and pipes its output to
+// the ginkgo Writr, with the expectation to poll the progress of the command
+func StartCmd(cmd string) (*exec.Cmd, error) {
+	c := exec.Command("sh", "-c", cmd)
+	c.Stdout = GinkgoWriter
+	c.Stderr = GinkgoWriter
+	if err := c.Start(); err != nil {
+		return c, err
+	}
+	return c, nil
+}

tests/docker/test-helpers

@@ -525,6 +525,20 @@ run-test() {
 }
 export -f run-test
 
+run-go-test() {
+    local delay=15
+    (
+      set +x
+      while [ $(count-running-tests) -ge ${MAX_CONCURRENT_TESTS:-3} ]; do
+          sleep $delay
+      done
+    )
+
+    go test -timeout=45m -v "$@" &
+    pids+=($!)
+}
+export -f run-go-test
+
 # ---
 
 cleanup-test-env(){

tests/docker/test-helpers.go

@@ -21,6 +21,7 @@ type TestConfig struct {
 	KubeconfigFile string
 	Token          string
 	K3sImage       string
+	DBType         string
 	Servers        []Server
 	Agents         []DockerNode
 	ServerYaml     string
@@ -115,16 +116,28 @@ func (config *TestConfig) ProvisionServers(numOfServers int) error {
 			yamlMount = fmt.Sprintf("--mount type=bind,src=%s,dst=/etc/rancher/k3s/config.yaml", filepath.Join(config.TestDir, fmt.Sprintf("server-%d.yaml", i)))
 		}
 
-		var joinOrStart string
-		if numOfServers > 0 {
-			if i == 0 {
-				joinOrStart = "--cluster-init"
-			} else {
-				if config.Servers[0].URL == "" {
-					return fmt.Errorf("first server URL is empty")
-				}
-				joinOrStart = fmt.Sprintf("--server %s", config.Servers[0].URL)
+		var joinServer string
+		var dbConnect string
+		var err error
+		if config.DBType == "" && numOfServers > 1 {
+			config.DBType = "etcd"
+		} else if config.DBType == "" {
+			config.DBType = "sqlite"
+		}
+		if i == 0 {
+			dbConnect, err = config.setupDatabase(true)
+			if err != nil {
+				return err
 			}
+		} else {
+			dbConnect, err = config.setupDatabase(false)
+			if err != nil {
+				return err
+			}
+			if config.Servers[0].URL == "" {
+				return fmt.Errorf("first server URL is empty")
+			}
+			joinServer = fmt.Sprintf("--server %s", config.Servers[0].URL)
 		}
 		newServer := Server{
 			DockerNode: DockerNode{
@@ -163,7 +176,7 @@ func (config *TestConfig) ProvisionServers(numOfServers int) error {
 			}
 			// The pipe requires that we use sh -c with "" to run the command
 			cmd = fmt.Sprintf("/bin/sh -c \"curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC='%s' INSTALL_K3S_SKIP_DOWNLOAD=true sh -\"",
-				joinOrStart+" "+os.Getenv(fmt.Sprintf("SERVER_%d_ARGS", i)))
+				dbConnect+" "+joinServer+" "+os.Getenv(fmt.Sprintf("SERVER_%d_ARGS", i)))
 			if out, err := newServer.RunCmdOnNode(cmd); err != nil {
 				return fmt.Errorf("failed to start server: %s: %v", out, err)
 			}
@@ -183,7 +196,7 @@ func (config *TestConfig) ProvisionServers(numOfServers int) error {
 				os.Getenv("REGISTRY_CLUSTER_ARGS"),
 				yamlMount,
 				config.K3sImage,
-				"server", joinOrStart, os.Getenv(fmt.Sprintf("SERVER_%d_ARGS", i))}, " ")
+				"server", dbConnect, joinServer, os.Getenv(fmt.Sprintf("SERVER_%d_ARGS", i))}, " ")
 			if out, err := RunCommand(dRun); err != nil {
 				return fmt.Errorf("failed to run server container: %s: %v", out, err)
 			}
@@ -214,6 +227,40 @@ func (config *TestConfig) ProvisionServers(numOfServers int) error {
 	return copyAndModifyKubeconfig(config)
 }
 
+// setupDatabase will start the configured database if startDB is true,
+// and return the correct flag to join the configured database
+func (config *TestConfig) setupDatabase(startDB bool) (string, error) {
+
+	joinFlag := ""
+	startCmd := ""
+	switch config.DBType {
+	case "mysql":
+		startCmd = "docker run -d --name mysql -e MYSQL_ROOT_PASSWORD=docker -p 3306:3306 mysql:8.4"
+		joinFlag = "--datastore-endpoint='mysql://root:docker@tcp(172.17.0.1:3306)/k3s'"
+	case "postgres":
+		startCmd = "docker run -d --name postgres -e POSTGRES_PASSWORD=docker -p 5432:5432 postgres:16-alpine"
+		joinFlag = "--datastore-endpoint='postgres://postgres:docker@tcp(172.17.0.1:5432)/k3s'"
+	case "etcd":
+		if startDB {
+			joinFlag = "--cluster-init"
+		}
+	case "sqlite":
+		break
+	default:
+		return "", fmt.Errorf("unsupported database type: %s", config.DBType)
+	}
+
+	if startDB && startCmd != "" {
+		if out, err := RunCommand(startCmd); err != nil {
+			return "", fmt.Errorf("failed to start %s container: %s: %v", config.DBType, out, err)
+		}
+		// Wait for DB to start
+		time.Sleep(10 * time.Second)
+	}
+	return joinFlag, nil
+
+}
+
 func (config *TestConfig) ProvisionAgents(numOfAgents int) error {
 	if err := checkVersionSkew(config); err != nil {
 		return err
@@ -340,6 +387,18 @@ func (config *TestConfig) Cleanup() error {
 		}
 	}
 
+	// Stop DB if it was started
+	if config.DBType == "mysql" || config.DBType == "postgres" {
+		cmd := fmt.Sprintf("docker stop %s", config.DBType)
+		if _, err := RunCommand(cmd); err != nil {
+			errs = append(errs, fmt.Errorf("failed to stop %s: %v", config.DBType, err))
+		}
+		cmd = fmt.Sprintf("docker rm %s", config.DBType)
+		if _, err := RunCommand(cmd); err != nil {
+			errs = append(errs, fmt.Errorf("failed to remove %s: %v", config.DBType, err))
+		}
+	}
+
 	// Error out if we hit any issues
 	if len(errs) > 0 {
 		return fmt.Errorf("cleanup failed: %v", errs)

tests/docker/upgrade/upgrade_test.go

@@ -126,6 +126,7 @@ var _ = Describe("Upgrade Tests", Ordered, func() {
 				cVersion := strings.Split(*k3sImage, ":")[1]
 				cVersion = strings.Replace(cVersion, "-amd64", "", 1)
 				cVersion = strings.Replace(cVersion, "-arm64", "", 1)
+				cVersion = strings.Replace(cVersion, "-arm", "", 1)
 				cVersion = strings.Replace(cVersion, "-", "+", 1)
 				Expect(out).To(ContainSubstring(cVersion))
 			}

tests/e2e/btrfs/btrfs_test.go

@@ -61,10 +61,12 @@ var _ = Describe("Verify that btrfs based servers work", Ordered, func() {
 		})
 		It("Checks that btrfs snapshots exist", func() {
 			cmd := "btrfs subvolume list /var/lib/rancher/k3s/agent/containerd/io.containerd.snapshotter.v1.btrfs"
-			res, err := tc.Servers[0].RunCmdOnNode(cmd)
-			Expect(err).NotTo(HaveOccurred())
-			Expect(res).To(MatchRegexp("agent/containerd/io.containerd.snapshotter.v1.btrfs/active/\\d+"))
-			Expect(res).To(MatchRegexp("agent/containerd/io.containerd.snapshotter.v1.btrfs/snapshots/\\d+"))
+			Eventually(func(g Gomega) {
+				res, err := tc.Servers[0].RunCmdOnNode(cmd)
+				g.Expect(err).NotTo(HaveOccurred())
+				g.Expect(res).To(MatchRegexp("agent/containerd/io.containerd.snapshotter.v1.btrfs/active/\\d+"))
+				g.Expect(res).To(MatchRegexp("agent/containerd/io.containerd.snapshotter.v1.btrfs/snapshots/\\d+"))
+			}, "30s", "5s").Should(Succeed())
 		})
 	})
 })