Increase e2e log verbosity and collect logs on failure

.github/workflows/e2e.yaml

@@ -91,7 +91,7 @@ jobs:
         run: | 
           chmod +x ./dist/artifacts/k3s
           cd tests/e2e/${{ matrix.etest }}
-          go test -v -timeout=45m ./${{ matrix.etest}}_test.go -ci -local
+          go test -timeout=45m ./${{ matrix.etest}}_test.go -test.v -ginkgo.v -ci -local
       - name: On Failure, Upload Journald Logs
         uses: actions/upload-artifact@v4
         if: ${{ failure() }}
@@ -211,9 +211,9 @@ jobs:
         # These tests use rancher/systemd-node and have different flags.
         CI_TESTS="autoimport hardened secretsencryption snapshotrestore token"
         if [ ${{ matrix.dtest }} = "upgrade" ] || [ ${{ matrix.dtest }} = "skew" ]; then
-          ./${{ matrix.dtest }}.test -k3sImage=$K3S_IMAGE -channel=$CHANNEL
+          ./${{ matrix.dtest }}.test -test.timeout=0 -test.v -ginkgo.v -k3sImage=$K3S_IMAGE -channel=$CHANNEL
         elif [[ $CI_TESTS =~ ${{ matrix.dtest }} ]]; then
-          ./${{ matrix.dtest }}.test -ci 
+          ./${{ matrix.dtest }}.test -test.timeout=0 -test.v -ginkgo.v -ci
         else
-          ./${{ matrix.dtest }}.test -k3sImage=$K3S_IMAGE
-        fi
+          ./${{ matrix.dtest }}.test -test.timeout=0 -test.v -ginkgo.v -k3sImage=$K3S_IMAGE
+        fi
\ No newline at end of file

tests/docker/autoimport/autoimport_test.go

@@ -171,6 +171,9 @@ var _ = AfterEach(func() {
 })
 
 var _ = AfterSuite(func() {
+	if failed {
+		AddReportEntry("journald-logs", docker.TailJournalLogs(1000, append(tc.Servers, tc.Agents...)))
+	}
 	if *ci || (tc != nil && !failed) {
 		Expect(tc.Cleanup()).To(Succeed())
 	}

tests/docker/basics/basics_test.go

@@ -8,6 +8,7 @@ import (
 	"testing"
 
 	"github.com/k3s-io/k3s/tests"
+	"github.com/k3s-io/k3s/tests/docker"
 	tester "github.com/k3s-io/k3s/tests/docker"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -74,6 +75,10 @@ var _ = AfterEach(func() {
 })
 
 var _ = AfterSuite(func() {
+	if failed {
+		AddReportEntry("describe", docker.DescribeNodesAndPods(config))
+		AddReportEntry("docker-logs", docker.TailDockerLogs(1000, append(config.Servers, config.Agents...)))
+	}
 	if config != nil && !failed {
 		config.Cleanup()
 	}

tests/docker/bootstraptoken/bootstraptoken_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	"github.com/k3s-io/k3s/tests"
+	"github.com/k3s-io/k3s/tests/docker"
 	tester "github.com/k3s-io/k3s/tests/docker"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -62,6 +63,10 @@ var _ = AfterEach(func() {
 })
 
 var _ = AfterSuite(func() {
+	if failed {
+		AddReportEntry("describe", docker.DescribeNodesAndPods(config))
+		AddReportEntry("docker-logs", docker.TailDockerLogs(1000, append(config.Servers, config.Agents...)))
+	}
 	if config != nil && !failed {
 		config.Cleanup()
 	}

tests/docker/cacerts/cacerts_test.go

@@ -9,6 +9,7 @@ import (
 	"testing"
 
 	"github.com/k3s-io/k3s/tests"
+	"github.com/k3s-io/k3s/tests/docker"
 	tester "github.com/k3s-io/k3s/tests/docker"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -88,6 +89,10 @@ var _ = AfterEach(func() {
 })
 
 var _ = AfterSuite(func() {
+	if failed {
+		AddReportEntry("describe", docker.DescribeNodesAndPods(config))
+		AddReportEntry("docker-logs", docker.TailDockerLogs(1000, append(config.Servers, config.Agents...)))
+	}
 	if config != nil && !failed {
 		config.Cleanup()
 		cmd := fmt.Sprintf("docker stop k3s-pause-%s", testID)

tests/docker/conformance/conformance_test.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"github.com/k3s-io/k3s/tests"
+	"github.com/k3s-io/k3s/tests/docker"
 	tester "github.com/k3s-io/k3s/tests/docker"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -123,6 +124,10 @@ var _ = AfterEach(func() {
 })
 
 var _ = AfterSuite(func() {
+	if failed {
+		AddReportEntry("describe", docker.DescribeNodesAndPods(config))
+		AddReportEntry("docker-logs", docker.TailDockerLogs(1000, append(config.Servers, config.Agents...)))
+	}
 	if config != nil && !failed {
 		config.Cleanup()
 	}

tests/docker/etcd/etcd_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	"github.com/k3s-io/k3s/tests"
+	"github.com/k3s-io/k3s/tests/docker"
 	tester "github.com/k3s-io/k3s/tests/docker"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -72,6 +73,10 @@ var _ = AfterEach(func() {
 })
 
 var _ = AfterSuite(func() {
+	if failed {
+		AddReportEntry("describe", docker.DescribeNodesAndPods(config))
+		AddReportEntry("docker-logs", docker.TailDockerLogs(1000, append(config.Servers, config.Agents...)))
+	}
 	if config != nil && !failed {
 		config.Cleanup()
 	}

tests/docker/hardened/hardened_test.go

@@ -118,6 +118,9 @@ var _ = AfterEach(func() {
 })
 
 var _ = AfterSuite(func() {
+	if failed {
+		AddReportEntry("journald-logs", docker.TailJournalLogs(1000, append(config.Servers, config.Agents...)))
+	}
 	if *ci || (config != nil && !failed) {
 		Expect(config.Cleanup()).To(Succeed())
 	}

tests/docker/lazypull/lazypull_test.go

@@ -7,6 +7,7 @@ import (
 	"testing"
 
 	"github.com/k3s-io/k3s/tests"
+	"github.com/k3s-io/k3s/tests/docker"
 	tester "github.com/k3s-io/k3s/tests/docker"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -71,6 +72,10 @@ var _ = AfterEach(func() {
 })
 
 var _ = AfterSuite(func() {
+	if failed {
+		AddReportEntry("describe", docker.DescribeNodesAndPods(config))
+		AddReportEntry("docker-logs", docker.TailDockerLogs(1000, append(config.Servers, config.Agents...)))
+	}
 	if config != nil && !failed {
 		config.Cleanup()
 	}

tests/docker/secretsencryption/secretsencryption_test.go

@@ -178,6 +178,9 @@ var _ = AfterEach(func() {
 })
 
 var _ = AfterSuite(func() {
+	if failed {
+		AddReportEntry("journald-logs", docker.TailJournalLogs(1000, append(tc.Servers, tc.Agents...)))
+	}
 	if *ci || (tc != nil && !failed) {
 		Expect(tc.Cleanup()).To(Succeed())
 	}

tests/docker/skew/skew_test.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/blang/semver/v4"
 	"github.com/k3s-io/k3s/tests"
+	"github.com/k3s-io/k3s/tests/docker"
 	tester "github.com/k3s-io/k3s/tests/docker"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -143,6 +144,10 @@ var _ = AfterEach(func() {
 })
 
 var _ = AfterSuite(func() {
+	if failed {
+		AddReportEntry("describe", docker.DescribeNodesAndPods(config))
+		AddReportEntry("docker-logs", docker.TailDockerLogs(1000, append(config.Servers, config.Agents...)))
+	}
 	if config != nil && !failed {
 		config.Cleanup()
 	}

tests/docker/snapshotrestore/snapshotrestore_test.go

@@ -7,6 +7,7 @@ import (
 	"testing"
 
 	"github.com/k3s-io/k3s/tests"
+	"github.com/k3s-io/k3s/tests/docker"
 	tester "github.com/k3s-io/k3s/tests/docker"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -168,6 +169,9 @@ var _ = AfterEach(func() {
 })
 
 var _ = AfterSuite(func() {
+	if failed {
+		AddReportEntry("journald-logs", docker.TailJournalLogs(1000, append(config.Servers, config.Agents...)))
+	}
 	if *ci || (config != nil && !failed) {
 		Expect(config.Cleanup()).To(Succeed())
 	}

tests/docker/test-helpers.go

@@ -611,10 +611,47 @@ func (config TestConfig) DeployWorkload(workload string) (string, error) {
 // RestartCluster restarts the k3s service on each node given
 func RestartCluster(nodes []DockerNode) error {
 	for _, node := range nodes {
-		cmd := "systemctl restart k3s* --all"
+		// Wait 60 seconds for the restart to succeed.
+		// If k3s doesn't report started to systemd within 60 seconds something is wrong.
+		cmd := "timeout -v 60 systemctl restart k3s* --all"
 		if _, err := node.RunCmdOnNode(cmd); err != nil {
 			return err
 		}
 	}
 	return nil
 }
+
+func DescribeNodesAndPods(config *TestConfig) string {
+	cmd := "kubectl describe node,pod -A --kubeconfig=" + config.KubeconfigFile
+	out, err := RunCommand(cmd)
+	if err != nil {
+		return fmt.Sprintf("** %v **\n%s", err, out)
+	}
+	return out
+}
+
+func TailDockerLogs(lines int, nodes []DockerNode) string {
+	logs := &strings.Builder{}
+	for _, node := range nodes {
+		cmd := fmt.Sprintf("docker logs %s --tail=%d", node.Name, lines)
+		if l, err := RunCommand(cmd); err != nil {
+			fmt.Fprintf(logs, "** failed to read docker logs for node %s ***\n%v\n", node.Name, err)
+		} else {
+			fmt.Fprintf(logs, "** docker logs for node %s ***\n%s\n", node.Name, l)
+		}
+	}
+	return logs.String()
+}
+
+func TailJournalLogs(lines int, nodes []DockerNode) string {
+	logs := &strings.Builder{}
+	for _, node := range nodes {
+		cmd := fmt.Sprintf("journalctl -u k3s* --no-pager --lines=%d", lines)
+		if l, err := node.RunCmdOnNode(cmd); err != nil {
+			fmt.Fprintf(logs, "** failed to read journald log for node %s ***\n%v\n", node.Name, err)
+		} else {
+			fmt.Fprintf(logs, "** journald log for node %s ***\n%s\n", node.Name, l)
+		}
+	}
+	return logs.String()
+}

tests/docker/token/token_test.go

@@ -155,6 +155,9 @@ var _ = AfterEach(func() {
 })
 
 var _ = AfterSuite(func() {
+	if failed {
+		AddReportEntry("journald-logs", docker.TailJournalLogs(1000, append(tc.Servers, tc.Agents...)))
+	}
 	if *ci || (tc != nil && !failed) {
 		Expect(tc.Cleanup()).To(Succeed())
 	}

tests/docker/upgrade/upgrade_test.go

@@ -10,6 +10,7 @@ import (
 	"testing"
 
 	"github.com/k3s-io/k3s/tests"
+	"github.com/k3s-io/k3s/tests/docker"
 	tester "github.com/k3s-io/k3s/tests/docker"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -146,6 +147,10 @@ var _ = AfterEach(func() {
 })
 
 var _ = AfterSuite(func() {
+	if failed {
+		AddReportEntry("describe", docker.DescribeNodesAndPods(config))
+		AddReportEntry("docker-logs", docker.TailDockerLogs(1000, append(config.Servers, config.Agents...)))
+	}
 	if config != nil && !failed {
 		config.Cleanup()
 	}