Convert test output to JSON format (#6410)

go.mod

@@ -86,6 +86,7 @@ require (
 	github.com/gorilla/mux v1.8.0
 	github.com/gorilla/websocket v1.4.2
 	github.com/gruntwork-io/terratest v0.40.6
+	github.com/json-iterator/go v1.1.12
 	github.com/k3s-io/helm-controller v0.13.0
 	github.com/k3s-io/kine v0.9.6
 	github.com/klauspost/compress v1.15.9
@@ -264,7 +265,6 @@ require (
 	github.com/jonboulle/clockwork v0.2.2 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/josharian/native v0.0.0-20200817173448-b6b71def0850 // indirect
-	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/karrick/godirwalk v1.16.1 // indirect
 	github.com/klauspost/cpuid/v2 v2.1.0 // indirect
 	github.com/libopenstorage/openstorage v1.0.0 // indirect

tests/e2e/clusterreset/clusterreset_test.go

@@ -17,7 +17,7 @@ import (
 // opensuse/Leap-15.3.x86_64, dweomer/microos.amd64
 var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system")
 var serverCount = flag.Int("serverCount", 3, "number of server nodes")
-var agentCount = flag.Int("agentCount", 2, "number of agent nodes")
+var agentCount = flag.Int("agentCount", 1, "number of agent nodes")
 var hardened = flag.Bool("hardened", false, "true or false")
 var ci = flag.Bool("ci", false, "running on CI")
 var local = flag.Bool("local", false, "deploy a locally built K3s binary")
@@ -29,7 +29,8 @@ var local = flag.Bool("local", false, "deploy a locally built K3s binary")
 func Test_E2EClusterReset(t *testing.T) {
 	RegisterFailHandler(Fail)
 	flag.Parse()
-	RunSpecs(t, "ClusterReset Test Suite")
+	suiteConfig, reporterConfig := GinkgoConfiguration()
+	RunSpecs(t, "ClusterReset Test Suite", suiteConfig, reporterConfig)
 }
 
 var (
@@ -38,6 +39,8 @@ var (
 	agentNodeNames  []string
 )
 
+var _ = ReportAfterEach(e2e.GenReport)
+
 var _ = Describe("Verify Create", Ordered, func() {
 	Context("Cluster :", func() {
 		It("Starts up with no issues", func() {
@@ -169,7 +172,7 @@ var _ = Describe("Verify Create", Ordered, func() {
 	})
 })
 
-var failed = false
+var failed bool
 var _ = AfterEach(func() {
 	failed = failed || CurrentSpecReport().Failed()
 })

tests/e2e/docker/docker_test.go

@@ -20,10 +20,11 @@ var agentCount = flag.Int("agentCount", 1, "number of agent nodes")
 // Environment Variables Info:
 // E2E_RELEASE_VERSION=v1.23.1+k3s2 or nil for latest commit from master
 
-func Test_E2EClusterValidation(t *testing.T) {
+func Test_E2EDockerCRIValidation(t *testing.T) {
 	RegisterFailHandler(Fail)
 	flag.Parse()
-	RunSpecs(t, "Docker CRI Test Suite")
+	suiteConfig, reporterConfig := GinkgoConfiguration()
+	RunSpecs(t, "Docker CRI Test Suite", suiteConfig, reporterConfig)
 }
 
 var (
@@ -32,6 +33,8 @@ var (
 	agentNodeNames  []string
 )
 
+var _ = ReportAfterEach(e2e.GenReport)
+
 var _ = Describe("Verify CRI-Dockerd", Ordered, func() {
 	Context("Cluster :", func() {
 		It("Starts up with no issues", func() {
@@ -74,7 +77,7 @@ var _ = Describe("Verify CRI-Dockerd", Ordered, func() {
 	})
 })
 
-var failed = false
+var failed bool
 var _ = AfterEach(func() {
 	failed = failed || CurrentSpecReport().Failed()
 })

tests/e2e/dualstack/dualstack_test.go

-package validatedualstack
+package dualstack
 
 import (
 	"flag"
@@ -14,14 +14,15 @@ import (
 
 // Valid nodeOS: generic/ubuntu2004, opensuse/Leap-15.3.x86_64
 var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system")
-var serverCount = flag.Int("serverCount", 1, "number of server nodes")
+var serverCount = flag.Int("serverCount", 3, "number of server nodes")
 var agentCount = flag.Int("agentCount", 1, "number of agent nodes")
 var hardened = flag.Bool("hardened", false, "true or false")
 
 func Test_E2EDualStack(t *testing.T) {
 	flag.Parse()
 	RegisterFailHandler(Fail)
-	RunSpecs(t, "DualStack Suite")
+	suiteConfig, reporterConfig := GinkgoConfiguration()
+	RunSpecs(t, "DualStack Test Suite", suiteConfig, reporterConfig)
 }
 
 var (
@@ -30,6 +31,8 @@ var (
 	agentNodeNames  []string
 )
 
+var _ = ReportAfterEach(e2e.GenReport)
+
 var _ = Describe("Verify DualStack Configuration", Ordered, func() {
 
 	It("Starts up with no issues", func() {

tests/e2e/externalip/external_test.go → tests/e2e/externalip/externalip_test.go

@@ -48,7 +48,9 @@ func getClientIPs(kubeConfigFile string) ([]e2e.ObjIP, error) {
 func Test_E2EExternalIP(t *testing.T) {
 	flag.Parse()
 	RegisterFailHandler(Fail)
-	RunSpecs(t, "Validate External-IP config Suite")
+	suiteConfig, reporterConfig := GinkgoConfiguration()
+	RunSpecs(t, "External-IP config Suite", suiteConfig, reporterConfig)
+
 }
 
 var (
@@ -57,6 +59,8 @@ var (
 	agentNodeNames  []string
 )
 
+var _ = ReportAfterEach(e2e.GenReport)
+
 var _ = Describe("Verify External-IP config", Ordered, func() {
 
 	It("Starts up with no issues", func() {

tests/e2e/scripts/run_tests.sh

 #!/bin/bash
 servercount=${5:-3}
-agentcount=${6:-2}
+agentcount=${6:-1}
 db=${7:-"etcd"}
 k3s_version=${k3s_version}
 k3s_channel=${k3s_channel:-"commit"}
@@ -17,34 +17,41 @@ ssh -i "$1"  -o "StrictHostKeyChecking no" $2@$3 'cd k3s && /usr/local/go/bin/go
 
 ssh -i "$1"  -o "StrictHostKeyChecking no" $2@$3 "cd k3s/tests/e2e/dualstack && vagrant global-status | awk '/running/'|cut -c1-7| xargs -r -d '\n' -n 1 -- vagrant destroy -f"
 
-echo "RUNNING DUALSTACK VALIDATION TEST"
-ssh -i "$1"  -o "StrictHostKeyChecking no" $2@$3 "cd k3s/tests/e2e && E2E_HARDENED="$hardened" /usr/local/go/bin/go test -v dualstack/dualstack_test.go -nodeOS="$4" -serverCount=$((servercount)) -agentCount=$((agentcount))  -timeout=1h"
+echo 'RUNNING DUALSTACK VALIDATION TEST'
+ssh -i "$1"  -o "StrictHostKeyChecking no" $2@$3 "cd k3s/tests/e2e && E2E_HARDENED="$hardened" /usr/local/go/bin/go test -v dualstack/dualstack_test.go -nodeOS="$4" -serverCount=$((servercount)) -agentCount=$((agentcount))  -timeout=30m -json" | tee -a testreport.log
 ssh -i "$1"  -o "StrictHostKeyChecking no" $2@$3 'cd k3s/tests/e2e/dualstack && vagrant destroy -f'
 
 echo 'RUNNING CLUSTER VALIDATION TEST'
-ssh -i "$1"  -o "StrictHostKeyChecking no" $2@$3 "cd k3s/tests/e2e && E2E_REGISTRY=true E2E_HARDENED="$hardened" /usr/local/go/bin/go test -v validatecluster/validatecluster_test.go -nodeOS="$4" -serverCount=$((servercount)) -agentCount=$((agentcount))  -timeout=1h"
+ssh -i "$1"  -o "StrictHostKeyChecking no" $2@$3 "cd k3s/tests/e2e && E2E_REGISTRY=true E2E_HARDENED="$hardened" /usr/local/go/bin/go test -v validatecluster/validatecluster_test.go -nodeOS="$4" -serverCount=$((servercount)) -agentCount=$((agentcount))  -timeout=30m -json" | tee -a testreport.log
 ssh -i "$1"  -o "StrictHostKeyChecking no" $2@$3 'cd k3s/tests/e2e/validatecluster && vagrant destroy -f'
 
-
 echo 'RUNNING SECRETS ENCRYPTION TEST'
-ssh -i "$1"  -o "StrictHostKeyChecking no" $2@$3 "cd k3s/tests/e2e && /usr/local/go/bin/go test -v secretsencryption/secretsencryption_test.go -nodeOS="$4" -serverCount=$((servercount)) -timeout=1h"
+ssh -i "$1"  -o "StrictHostKeyChecking no" $2@$3 "cd k3s/tests/e2e && /usr/local/go/bin/go test -v secretsencryption/secretsencryption_test.go -nodeOS="$4" -serverCount=$((servercount)) -timeout=30m -json" | tee -a testreport.log
 ssh -i "$1"  -o "StrictHostKeyChecking no" $2@$3 'cd k3s/tests/e2e/secretsencryption && vagrant destroy -f'
 
-echo 'RUN CLUSTER RESET TEST'
-ssh -i "$1"  -o "StrictHostKeyChecking no" $2@$3 "cd k3s/tests/e2e && /usr/local/go/bin/go test -v clusterreset/clusterreset_test.go -nodeOS="$4" -serverCount=3 -agentCount=1 -timeout=1h"
-ssh -i "$1"  -o "StrictHostKeyChecking no" $2@$3 'cd k3s/tests/e2e/clusterreset && vagrant destroy -f'
+echo 'RUNNING SNAPSHOT RESTORE TEST'
+ssh -i "$1"  -o "StrictHostKeyChecking no" $2@$3 "cd k3s/tests/e2e && /usr/local/go/bin/go test -v snapshotrestore/snapshotrestore_test.go -nodeOS="$4" -serverCount=$((servercount)) -agentCount=$((agentcount)) -timeout=30m -json" | tee -a testreport.log
+ssh -i "$1"  -o "StrictHostKeyChecking no" $2@$3 'cd k3s/tests/e2e/secretsencryption && vagrant destroy -f'
 
 echo 'RUNNING SPLIT SERVER VALIDATION TEST'
-ssh -i "$1"  -o "StrictHostKeyChecking no" $2@$3 "cd k3s/tests/e2e && E2E_HARDENED="$hardened" /usr/local/go/bin/go test -v splitserver/splitserver_test.go -nodeOS="$4" -timeout=1h"
+ssh -i "$1"  -o "StrictHostKeyChecking no" $2@$3 "cd k3s/tests/e2e && E2E_HARDENED="$hardened" /usr/local/go/bin/go test -v splitserver/splitserver_test.go -nodeOS="$4" -timeout=30m -json" | tee -a testreport.log
 ssh -i "$1"  -o "StrictHostKeyChecking no" $2@$3 'cd k3s/tests/e2e/splitserver && vagrant destroy -f'
 
 E2E_RELEASE_VERSION=$k3s_version && export E2E_RELEASE_VERSION
 E2E_RELEASE_CHANNEL=$k3s_channel && export E2E_RELEASE_CHANNEL
 
 echo 'RUNNING CLUSTER UPGRADE TEST'
-ssh -i "$1"  -o "StrictHostKeyChecking no" $2@$3 "cd k3s/tests/e2e && E2E_REGISTRY=true /usr/local/go/bin/go test -v upgradecluster/upgradecluster_test.go -nodeOS="$4" -serverCount=$((servercount)) -agentCount=$((agentcount)) -timeout=1h"
+ssh -i "$1"  -o "StrictHostKeyChecking no" $2@$3 "cd k3s/tests/e2e && E2E_REGISTRY=true /usr/local/go/bin/go test -v upgradecluster/upgradecluster_test.go -nodeOS="$4" -serverCount=$((servercount)) -agentCount=$((agentcount)) -timeout=30m -json" | tee -a testreport.log
 ssh -i "$1"  -o "StrictHostKeyChecking no" $2@$3 'cd k3s/tests/e2e/upgradecluster && vagrant destroy -f'
 
+echo 'RUN CLUSTER RESET TEST'
+ssh -i "$1"  -o "StrictHostKeyChecking no" $2@$3 "cd k3s/tests/e2e && /usr/local/go/bin/go test -v clusterreset/clusterreset_test.go -nodeOS="$4" -serverCount=$((servercount)) -agentCount=$((agentcount)) -timeout=30m -json" | tee -a testreport.log
+ssh -i "$1"  -o "StrictHostKeyChecking no" $2@$3 'cd k3s/tests/e2e/clusterreset && vagrant destroy -f'
+
 echo 'RUNNING DOCKER CRI VALIDATION TEST'
-ssh -i "$1"  -o "StrictHostKeyChecking no" $2@$3 "cd k3s/tests/e2e && /usr/local/go/bin/go test -v docker/docker_test.go -nodeOS="$4" -serverCount=1 -agentCount=1 -timeout=1h"
+ssh -i "$1"  -o "StrictHostKeyChecking no" $2@$3 "cd k3s/tests/e2e && /usr/local/go/bin/go test -v docker/docker_test.go -nodeOS="$4" -serverCount=1 -agentCount=1 -timeout=30m -json" | tee -a testreport.log
 ssh -i "$1"  -o "StrictHostKeyChecking no" $2@$3 'cd k3s/tests/e2e/docker && vagrant destroy -f'
+
+echo 'RUNNING EXTERNALIP VALIDATION TEST'
+ssh -i "$1"  -o "StrictHostKeyChecking no" $2@$3 "cd k3s/tests/e2e && E2E_HARDENED="$hardened" /usr/local/go/bin/go test -v externalip/externalip_test.go -nodeOS="$4" -serverCount=1 -agentCount=1  -timeout=30m -json" | tee -a testreport.log
+ssh -i "$1"  -o "StrictHostKeyChecking no" $2@$3 'cd k3s/tests/e2e/dualstack && vagrant destroy -f'

tests/e2e/secretsencryption/secretsencryption_test.go

@@ -23,7 +23,8 @@ var hardened = flag.Bool("hardened", false, "true or false")
 func Test_E2ESecretsEncryption(t *testing.T) {
 	RegisterFailHandler(Fail)
 	flag.Parse()
-	RunSpecs(t, "Secrets Encryption Test Suite")
+	suiteConfig, reporterConfig := GinkgoConfiguration()
+	RunSpecs(t, "Secrets Encryption Test Suite", suiteConfig, reporterConfig)
 }
 
 var (
@@ -31,6 +32,8 @@ var (
 	serverNodeNames []string
 )
 
+var _ = ReportAfterEach(e2e.GenReport)
+
 var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() {
 	Context("Cluster :", func() {
 		It("Starts up with no issues", func() {
@@ -286,7 +289,7 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() {
 
 })
 
-var failed = false
+var failed bool
 var _ = AfterEach(func() {
 	failed = failed || CurrentSpecReport().Failed()
 })

tests/e2e/snapshotrestore/snapshotrestore_test.go

@@ -30,7 +30,8 @@ var local = flag.Bool("local", false, "deploy a locally built K3s binary")
 func Test_E2ESnapshotRestore(t *testing.T) {
 	RegisterFailHandler(Fail)
 	flag.Parse()
-	RunSpecs(t, "SnapshotRestore Test Suite")
+	suiteConfig, reporterConfig := GinkgoConfiguration()
+	RunSpecs(t, "SnapshotRestore Test Suite", suiteConfig, reporterConfig)
 }
 
 var (
@@ -40,6 +41,8 @@ var (
 	snapshotname    string
 )
 
+var _ = ReportAfterEach(e2e.GenReport)
+
 var _ = Describe("Verify Create", Ordered, func() {
 	Context("Cluster :", func() {
 		It("Starts up with no issues", func() {

tests/e2e/splitserver/splitserver_test.go

@@ -60,7 +60,8 @@ func createSplitCluster(nodeOS string, etcdCount, controlPlaneCount, agentCount
 func Test_E2ESplitServer(t *testing.T) {
 	RegisterFailHandler(Fail)
 	flag.Parse()
-	RunSpecs(t, "Split Server Test Suite")
+	suiteConfig, reporterConfig := GinkgoConfiguration()
+	RunSpecs(t, "Split Server Test Suite", suiteConfig, reporterConfig)
 }
 
 var (
@@ -70,6 +71,8 @@ var (
 	agentNodeNames []string
 )
 
+var _ = ReportAfterEach(e2e.GenReport)
+
 var _ = Describe("Verify Create", Ordered, func() {
 	Context("Cluster :", func() {
 		It("Starts up with no issues", func() {
@@ -219,7 +222,7 @@ var _ = Describe("Verify Create", Ordered, func() {
 	})
 })
 
-var failed = false
+var failed bool
 var _ = AfterEach(func() {
 	failed = failed || CurrentSpecReport().Failed()
 })

tests/e2e/testutils.go

@@ -12,6 +12,8 @@ import (
 	"strings"
 	"time"
 
+	json "github.com/json-iterator/go"
+	ginkgo "github.com/onsi/ginkgo/v2"
 	"golang.org/x/sync/errgroup"
 )
 
@@ -280,6 +282,22 @@ func GenKubeConfigFile(serverName string) (string, error) {
 	return kubeConfigFile, nil
 }
 
+func GenReport(specReport ginkgo.SpecReport) {
+	state := struct {
+		State string        `json:"state"`
+		Name  string        `json:"name"`
+		Type  string        `json:"type"`
+		Time  time.Duration `json:"time"`
+	}{
+		State: specReport.State.String(),
+		Name:  specReport.LeafNodeText,
+		Type:  "k3s test",
+		Time:  specReport.RunTime,
+	}
+	status, _ := json.Marshal(state)
+	fmt.Printf("%s", status)
+}
+
 // GetVagrantLog returns the logs of on vagrant commands that initialize the nodes and provision K3s on each node.
 // It also attempts to fetch the systemctl logs of K3s on nodes where the k3s.service failed.
 func GetVagrantLog(cErr error) string {

tests/e2e/upgradecluster/upgradecluster_test.go

@@ -31,7 +31,8 @@ var ci = flag.Bool("ci", false, "running on CI")
 func Test_E2EUpgradeValidation(t *testing.T) {
 	RegisterFailHandler(Fail)
 	flag.Parse()
-	RunSpecs(t, "Upgrade Cluster Test Suite")
+	suiteConfig, reporterConfig := GinkgoConfiguration()
+	RunSpecs(t, "Upgrade Cluster Test Suite", suiteConfig, reporterConfig)
 }
 
 var (
@@ -40,6 +41,8 @@ var (
 	agentNodeNames  []string
 )
 
+var _ = ReportAfterEach(e2e.GenReport)
+
 var _ = Describe("Verify Upgrade", Ordered, func() {
 	Context("Cluster :", func() {
 		It("Starts up with no issues", func() {
@@ -376,7 +379,7 @@ var _ = Describe("Verify Upgrade", Ordered, func() {
 	})
 })
 
-var failed = false
+var failed bool
 var _ = AfterEach(func() {
 	failed = failed || CurrentSpecReport().Failed()
 })

tests/e2e/validatecluster/validatecluster_test.go

@@ -30,7 +30,8 @@ var local = flag.Bool("local", false, "deploy a locally built K3s binary")
 func Test_E2EClusterValidation(t *testing.T) {
 	RegisterFailHandler(Fail)
 	flag.Parse()
-	RunSpecs(t, "Create Cluster Test Suite")
+	suiteConfig, reporterConfig := GinkgoConfiguration()
+	RunSpecs(t, "Create Cluster Test Suite", suiteConfig, reporterConfig)
 }
 
 var (
@@ -39,6 +40,8 @@ var (
 	agentNodeNames  []string
 )
 
+var _ = ReportAfterEach(e2e.GenReport)
+
 var _ = Describe("Verify Create", Ordered, func() {
 	Context("Cluster :", func() {
 		It("Starts up with no issues", func() {
@@ -270,7 +273,7 @@ var _ = Describe("Verify Create", Ordered, func() {
 	})
 })
 
-var failed = false
+var failed bool
 var _ = AfterEach(func() {
 	failed = failed || CurrentSpecReport().Failed()
 })