Testing directory and documentation rework. (#5256)

.github/workflows/cgroup.yaml

@@ -6,7 +6,7 @@ on:
       - "channel.yaml"
       - "install.sh"
       - "tests/**"
-      - "!tests/vagrant/cgroup/**"
+      - "!tests/cgroup/**"
       - ".github/**"
       - "!.github/workflows/cgroup.yaml"
   pull_request:
@@ -15,7 +15,7 @@ on:
       - "channel.yaml"
       - "install.sh"
       - "tests/**"
-      - "!tests/vagrant/cgroup/**"
+      - "!tests/cgroup/**"
       - ".github/**"
       - "!.github/workflows/cgroup.yaml"
   workflow_dispatch: {}
@@ -47,7 +47,7 @@ jobs:
       max-parallel: 1
     defaults:
       run:
-        working-directory: tests/vagrant/cgroup/${{ matrix.mode }}/${{ matrix.vm }}
+        working-directory: tests/cgroup/${{ matrix.mode }}/${{ matrix.vm }}
     steps:
       - name: "Checkout"
         uses: actions/checkout@v2
@@ -61,7 +61,7 @@ jobs:
           path: |
             ~/.vagrant.d/boxes
             ~/.vagrant.d/gems
-          key: cgroup-${{ hashFiles(format('tests/vagrant/cgroup/{0}/{1}/Vagrantfile', matrix.mode, matrix.vm)) }}
+          key: cgroup-${{ hashFiles(format('tests/cgroup/{0}/{1}/Vagrantfile', matrix.mode, matrix.vm)) }}
         id: vagrant-cache
         continue-on-error: true
       - name: "Vagrant Plugin(s)"

.github/workflows/install.yaml

@@ -5,12 +5,12 @@ on:
     paths:
       - "channel.yaml"
       - "install.sh"
-      - "tests/vagrant/install/**"
+      - "tests/install/**"
   pull_request:
     branches: [main, master]
     paths:
       - "install.sh"
-      - "tests/vagrant/install/**"
+      - "tests/install/**"
   workflow_dispatch: {}
 jobs:
   test:
@@ -32,7 +32,7 @@ jobs:
       max-parallel: 2
     defaults:
       run:
-        working-directory: tests/vagrant/install/${{ matrix.vm }}
+        working-directory: tests/install/${{ matrix.vm }}
     env:
       INSTALL_K3S_CHANNEL: ${{ matrix.channel }}
     steps:
@@ -45,7 +45,7 @@ jobs:
           path: |
             ~/.vagrant.d/boxes
             ~/.vagrant.d/gems
-          key: install-${{ hashFiles(format('tests/vagrant/install/{0}/Vagrantfile', matrix.vm)) }}
+          key: install-${{ hashFiles(format('tests/install/{0}/Vagrantfile', matrix.vm)) }}
         id: vagrant-cache
         continue-on-error: true
       - name: "Vagrant Plugin(s)"

.github/workflows/integration.yaml

@@ -5,7 +5,9 @@ on:
       - "**.md"
       - "channel.yaml"
       - "install.sh"
-      - "tests/vagrant/**"
+      - "tests/snapshotter/**"
+      - "tests/install/**"
+      - "tests/cgroup/**"
       - ".github/**"
       - "!.github/workflows/integration.yaml"
   pull_request:
@@ -13,7 +15,9 @@ on:
       - "**.md"
       - "channel.yaml"
       - "install.sh"
-      - "tests/vagrant/**"
+      - "tests/snapshotter/**"
+      - "tests/install/**"
+      - "tests/cgroup/**"
       - ".github/**"
       - "!.github/workflows/integration.yaml"
   workflow_dispatch: {}

.github/workflows/snapshotter.yaml

@@ -6,7 +6,7 @@ on:
       - "channel.yaml"
       - "install.sh"
       - "tests/**"
-      - "!tests/vagrant/snapshotter/**"
+      - "!tests/snapshotter/**"
       - ".github/**"
       - "!.github/workflows/snapshotter.yaml"
   pull_request:
@@ -15,7 +15,7 @@ on:
       - "channel.yaml"
       - "install.sh"
       - "tests/**"
-      - "!tests/vagrant/snapshotter/**"
+      - "!tests/snapshotter/**"
       - ".github/**"
       - "!.github/workflows/snapshotter.yaml"
   workflow_dispatch: {}
@@ -47,7 +47,7 @@ jobs:
       max-parallel: 1
     defaults:
       run:
-        working-directory: tests/vagrant/snapshotter/${{ matrix.snapshotter }}/${{ matrix.vm }}
+        working-directory: tests/snapshotter/${{ matrix.snapshotter }}/${{ matrix.vm }}
     env:
       VAGRANT_EXPERIMENTAL: disks
     steps:
@@ -63,7 +63,7 @@ jobs:
           path: |
             ~/.vagrant.d/boxes
             ~/.vagrant.d/gems
-          key: snapshotter-${{ hashFiles(format('tests/vagrant/snapshotter/{0}/{1}/Vagrantfile', matrix.snapshotter, matrix.vm)) }}
+          key: snapshotter-${{ hashFiles(format('tests/snapshotter/{0}/{1}/Vagrantfile', matrix.snapshotter, matrix.vm)) }}
         id: vagrant-cache
         continue-on-error: true
       - name: "Vagrant Plugin(s)"

.github/workflows/unitcoverage.yaml

@@ -5,7 +5,9 @@ on:
       - "**.md"
       - "channel.yaml"
       - "install.sh"
-      - "tests/vagrant/**"
+      - "tests/snapshotter/**"
+      - "tests/install/**"
+      - "tests/cgroup/**"
       - ".github/**"
       - "!.github/workflows/unitcoverage.yaml"
   pull_request:
@@ -13,7 +15,9 @@ on:
       - "**.md"
       - "channel.yaml"
       - "install.sh"
-      - "tests/vagrant/**"
+      - "tests/snapshotter/**"
+      - "tests/install/**"
+      - "tests/cgroup/**"
       - ".github/**"
       - "!.github/workflows/unitcoverage.yaml"
   workflow_dispatch: {}

pkg/etcd/etcd_test.go

@@ -11,7 +11,7 @@ import (
 
 	"github.com/k3s-io/k3s/pkg/clientaccess"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
-	testutil "github.com/k3s-io/k3s/tests/util"
+	testutil "github.com/k3s-io/k3s/tests"
 	"github.com/robfig/cron/v3"
 	clientv3 "go.etcd.io/etcd/client/v3"
 	"go.etcd.io/etcd/server/v3/etcdserver"

pkg/flock/flock_int_test.go

-//go:build linux || darwin || freebsd || openbsd || netbsd || dragonfly
-// +build linux darwin freebsd openbsd netbsd dragonfly
-
-package flock_test
-
-import (
-	"testing"
-
-	"github.com/k3s-io/k3s/pkg/flock"
-	. "github.com/onsi/ginkgo/v2"
-	. "github.com/onsi/gomega"
-)
-
-const lockfile = "/tmp/testlock.test"
-
-var lock int
-var _ = Describe("file locks", func() {
-	When("a new exclusive lock is created", func() {
-		It("starts up with no problems", func() {
-			var err error
-			lock, err = flock.Acquire(lockfile)
-			Expect(err).ToNot(HaveOccurred())
-		})
-		It("has a write lock on the file", func() {
-			Expect(flock.CheckLock(lockfile)).To(BeTrue())
-		})
-		It("release the lock correctly", func() {
-			Expect(flock.Release(lock)).To(Succeed())
-			Expect(flock.CheckLock(lockfile)).To(BeFalse())
-		})
-	})
-})
-
-func TestFlock(t *testing.T) {
-	RegisterFailHandler(Fail)
-	RunSpecs(t, "Flock Suite")
-}

pkg/flock/flock_unix_test.go

+//go:build linux || darwin || freebsd || openbsd || netbsd || dragonfly
+// +build linux darwin freebsd openbsd netbsd dragonfly
+
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package flock
+
+import (
+	"testing"
+)
+
+func Test_UnitFlock(t *testing.T) {
+	tests := []struct {
+		name      string
+		path      string
+		wantCheck bool
+		wantErr   bool
+	}{
+		{
+			name: "Basic Flock Test",
+			path: "/tmp/testlock.test",
+
+			wantCheck: true,
+			wantErr:   false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			lock, err := Acquire(tt.path)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Acquire() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+
+			if got := CheckLock(tt.path); got != tt.wantCheck {
+				t.Errorf("CheckLock() = %+v\nWant = %+v", got, tt.wantCheck)
+			}
+
+			if err := Release(lock); (err != nil) != tt.wantErr {
+				t.Errorf("Release() error = %v, wantErr %v", err, tt.wantErr)
+			}
+
+			if got := CheckLock(tt.path); got == tt.wantCheck {
+				t.Errorf("CheckLock() = %+v\nWant = %+v", got, !tt.wantCheck)
+			}
+		})
+	}
+}

scripts/build-tests-sonobuoy

@@ -14,7 +14,7 @@ PKG_TO_TEST=$(find ./pkg/ -type f -name "*_int_test.go" | sed -r 's|/[^/]+$||' |
 for i in $PKG_TO_TEST; do
     name=$(echo "${i##*/}")
     echo $name
-    go test -c -v -ldflags "-X 'github.com/k3s-io/k3s/tests/util.existingServer=True'" -o dist/artifacts/k3s-integration-$name.test $i -run Integration
+    go test -c -v -ldflags "-X 'github.com/k3s-io/k3s/tests/integration.existingServer=True'" -o dist/artifacts/k3s-integration-$name.test $i -run Integration
 done
 
 # Integration tests under /tests
@@ -22,7 +22,7 @@ PKG_TO_TEST=$(find ./tests/integration -type f -name "*_int_test.go" | sed -r 's
 for i in $PKG_TO_TEST; do
     name=$(echo "${i##*/}")
     echo $name
-    go test -c -v -ldflags "-X 'github.com/k3s-io/k3s/tests/util.existingServer=True'" -o dist/artifacts/k3s-integration-$name.test $i -run Integration
+    go test -c -v -ldflags "-X 'github.com/k3s-io/k3s/tests/integration.existingServer=True'" -o dist/artifacts/k3s-integration-$name.test $i -run Integration
 done
 docker build -f ./tests/integration/Dockerfile.test -t $REPO .
 docker save $REPO -o ./dist/artifacts/$REPO.tar

tests/TESTING.md

 # Testing Standards in K3s
 
-Testing in K3s comes in 4 forms: 
+Testing in K3s comes in 5 forms: 
 - [Unit](#unit-tests)
 - [Integration](#integration-tests)
 - [Smoke](#smoke-tests)
+- [Performance](#performance)
 - [End-to-End (E2E)](#end-to-end-e2e-tests)
 
 This document will explain *when* each test should be written and *how* each test should be
@@ -53,71 +54,27 @@ ___
 Integration tests should be used to test a specific functionality of k3s that exists across multiple Go packages, either via exported function calls, or more often, CLI comands.
 Integration tests should be used for "black box" testing. 
 
-### Framework
-
-All integration tests in K3s follow a [Behavior Diven Development (BDD)](https://en.wikipedia.org/wiki/Behavior-driven_development) style. Specifically, K3s uses [Ginkgo](https://onsi.github.io/ginkgo/) and [Gomega](https://onsi.github.io/gomega/) to drive the tests.  
-To generate an initial test, the command `ginkgo bootstrap` can be used.
-
-To facilitate K3s CLI testing, see `tests/util/cmd.go` helper functions.
-
-### Format
-
-All integration tests should be placed under `tests/integration/<TEST_NAME>`.  
-All integration test files should be named: `<TEST_NAME>_int_test.go`.  
-All integration test functions should be named: `Test_Integration<TEST_NAME>`.  
-See the [local storage test](../tests/integration/localstorage/localstorage_int_test.go) as an example.
-
-### Running
-
-Integration tests can be run with no k3s cluster present, each test will spin up and kill the appropriate k3s server it needs.  
-Note: Integration tests must be run as root, prefix the commands below with `sudo -E env "PATH=$PATH"` if a sudo user.
-```bash
-go test ./tests/integration/... -run Integration
-```
-
-Additionally, to generate JUnit reporting for the tests, the Ginkgo CLI is used
-```
-ginkgo --junit-report=result.xml ./tests/integration/...
-```
-
-Integration tests can be run on an existing single-node cluster via compile time flag, tests will skip if the server is not configured correctly.
-```bash
-go test -ldflags "-X 'github.com/k3s-io/k3s/tests/util.existingServer=True'" ./tests/integration/... -run Integration
-```
-
-Integration tests can also be run via a [Sonobuoy](https://sonobuoy.io/docs/v0.53.2/) plugin on an existing single-node cluster.
-```bash
-./scripts/build-tests-sonobuoy
-sudo KUBECONFIG=/etc/rancher/k3s/k3s.yaml sonobuoy run --plugin ./dist/artifacts/k3s-int-tests.yaml
-```
-Check the sonobuoy status and retrieve results
-``` 
-sudo KUBECONFIG=/etc/rancher/k3s/k3s.yaml sonobuoy status
-sudo KUBECONFIG=/etc/rancher/k3s/k3s.yaml sonobuoy retrieve
-sudo KUBECONFIG=/etc/rancher/k3s/k3s.yaml sonobuoy results <TAR_FILE_FROM_RETRIEVE>
-```
+See [integration/README.md](./integration/README.md) for more info.
 
 ___
 
 ## Smoke Tests
 
-Smoke tests are defined under the [tests/vagrant](../tests/vagrant) path at the root of this repository.
-The sub-directories therein contain fixtures for running simple clusters to assert correct behavior for "happy path"
-scenarios. These fixtures are mostly self-contained Vagrantfiles describing single-node installations that are
-easily spun up with Vagrant for the `libvirt` and `virtualbox` providers:
-
-- [Install Script](../tests/vagrant/install) :arrow_right: on proposed changes to [install.sh](../install.sh) 
-  - [CentOS 7](../tests/vagrant/install/centos-7) (stand-in for RHEL 7)
-  - [CentOS 8](../tests/vagrant/install/centos-8) (stand-in for RHEL 8)
-  - [Leap 15.3](../tests/vagrant/install/opensuse-microos) (stand-in for SLES)
-  - [MicroOS](../tests/vagrant/install/opensuse-microos) (stand-in for SLE-Micro)
-  - [Ubuntu 20.04](../tests/vagrant/install/ubuntu-focal) (Focal Fossa)
-- [Control Groups](../tests/vagrant/cgroup) :arrow_right: on any code change
-  - [mode=unified](../tests/vagrant/cgroup/unified) (cgroups v2)
-    - [Fedora 34](../tests/vagrant/cgroup/unified/fedora-34) (rootfull + rootless)
-- [Snapshotter](../tests/vagrant/snapshotter/btrfs/opensuse-leap) :arrow_right: on any code change
-  - [BTRFS](../tests/vagrant/snapshotter/btrfs) ([containerd built-in](https://github.com/containerd/containerd/tree/main/snapshots/btrfs))
-    - [Leap 15.3](../tests/vagrant/snapshotter/btrfs/opensuse-leap)
+Smoke tests are a collection of tests defined under the [tests](../tests) path at the root of this repository.
+The sub-directories therein contain fixtures for running simple clusters to assert correct behavior for "happy path" scenarios. These fixtures are mostly self-contained Vagrantfiles describing single-node installations that are easily spun up with Vagrant for the `libvirt` and `virtualbox` providers:
+
+- [Install Script](../tests/install) :arrow_right: on proposed changes to [install.sh](../install.sh) 
+  - [CentOS 7](../tests/install/centos-7) (stand-in for RHEL 7)
+  - [CentOS 8](../tests/install/centos-8) (stand-in for RHEL 8)
+  - [Leap 15.3](../tests/install/opensuse-microos) (stand-in for SLES)
+  - [MicroOS](../tests/install/opensuse-microos) (stand-in for SLE-Micro)
+  - [Ubuntu 20.04](../tests/install/ubuntu-focal) (Focal Fossa)
+- [Control Groups](../tests/cgroup) :arrow_right: on any code change
+  - [mode=unified](../tests/cgroup/unified) (cgroups v2)
+    - [Fedora 34](../tests/cgroup/unified/fedora-34) (rootfull + rootless)
+- [Snapshotter](../tests/snapshotter/btrfs/opensuse-leap) :arrow_right: on any code change
+  - [BTRFS](../tests/snapshotter/btrfs) ([containerd built-in](https://github.com/containerd/containerd/tree/main/snapshots/btrfs))
+    - [Leap 15.3](../tests/snapshotter/btrfs/opensuse-leap)
 
 When adding new installer test(s) please copy the prevalent style for the `Vagrantfile`.
 Ideally, the boxes used for additional assertions will support the default `virtualbox` provider which
@@ -153,7 +110,7 @@ These can be set on the CLI or exported before invoking Vagrant:
 
 The **Install Script** tests can be run by changing to the fixture directory and invoking `vagrant up`, e.g.:
 ```shell
-cd tests/vagrant/install/centos-8
+cd tests/install/centos-8
 vagrant up
 # the following provisioners are optional. the do not run by default but are invoked
 # explicitly by github actions workflow to avoid certain timeout issues on slow runners
@@ -173,54 +130,22 @@ an idea of how they can be invoked.
 
 ___
 
-## End-to-End (E2E) Tests
-
-E2E tests cover multi-node K3s configuration and administration: bringup, update, teardown etc. across a wide range of operating systems. E2E tests are run nightly as part of K3s quality assurance (QA).
+## Performance Tests
 
-### Framework 
-End-to-end tests utilize [Ginkgo](https://onsi.github.io/ginkgo/) and [Gomega](https://onsi.github.io/gomega/) like the integration tests, but rely on [Vagrant](https://www.vagrantup.com/) to provide the underlying cluster configuration. 
-
-Currently tested operating systems are:
-- [Ubuntu 20.04](https://app.vagrantup.com/generic/boxes/ubuntu2004)
-- [Leap 15.3](https://app.vagrantup.com/opensuse/boxes/Leap-15.3.x86_64) (stand-in for SLE-Server)
-- [MicroOS](https://app.vagrantup.com/dweomer/boxes/microos.amd64) (stand-in for SLE-Micro)
-
-### Format
+Performance tests use Terraform to test large scale deployments of K3s clusters.
 
-All E2E tests should be placed under `tests/e2e/<TEST_NAME>`.  
-All E2E test functions should be named: `Test_E2E<TEST_NAME>`.  
-A E2E test consists of two parts:
-1. `Vagrantfile`: a vagrant file which describes and configures the VMs upon which the cluster and test will run
-2. `<TEST_NAME>.go`: A go test file which calls `vagrant up` and controls the actual testing of the cluster
-
-See the [validate cluster test](../tests/e2e/validatecluster/validatecluster_test.go) as an example.
-
-### Running
-
-Generally, E2E tests are run as a nightly Jenkins job for QA. They can still be run locally but additional setup may be required. By default, all E2E tests are designed with `libvirt` as the underlying VM provider. Instructions for installing libvirt and its associated vagrant plugin, `vagrant-libvirt` can be found [here.](https://github.com/vagrant-libvirt/vagrant-libvirt#installation) `VirtualBox` is also supported as a backup VM provider.
+See [perf/README.md](./perf/README.md) for more info.
+___
 
-Once setup is complete, all E2E tests can be run with:
-```bash
-go test -timeout=15m ./tests/e2e/... -run E2E
-```
-Tests can be run individually with:
-```bash
-go test -timeout=15m ./tests/e2e/validatecluster/... -run E2E
-#or
-go test -timeout=15m ./tests/e2e/... -run E2EClusterValidation
-```
+## End-to-End (E2E) Tests
 
-Additionally, to generate junit reporting for the tests, the Ginkgo CLI is used. Installation instructions can be found [here.](https://onsi.github.io/ginkgo/#getting-started)  
+E2E tests cover multi-node K3s configuration and administration: bringup, update, teardown etc. across a wide range of operating systems. E2E tests are run nightly as part of K3s quality assurance (QA).
 
-To run the all E2E tests and generate JUnit testing reports:
-```
-ginkgo --junit-report=result.xml ./tests/e2e/...
-```
+See [e2e/README.md](./e2e/README.md) for more info.
 
-Note: The `go test` default timeout is 10 minutes, thus the `-timeout` flag should be used. The `ginkgo` default timeout is 1 hour, no timeout flag is needed.
+___
 
 ## Contributing New Or Updated Tests
 
-___
 We gladly accept new and updated tests of all types. If you wish to create
 a new test or update an existing test, please submit a PR with a title that includes the words `<NAME_OF_TEST> (Created/Updated)`.

tests/vagrant/cgroup/unified/fedora-34/Vagrantfile → tests/cgroup/unified/fedora-34/Vagrantfile

@@ -3,8 +3,8 @@
 #
 # Vagrant box for testing k3s with cgroup v2.
 
-ENV['TEST_UNITFILE_ROOTFULL'] ||= '../../../../../k3s.service'
-ENV['TEST_UNITFILE_ROOTLESS'] ||= '../../../../../k3s-rootless.service'
+ENV['TEST_UNITFILE_ROOTFULL'] ||= '../../../../k3s.service'
+ENV['TEST_UNITFILE_ROOTLESS'] ||= '../../../../k3s-rootless.service'
 
 Vagrant.configure("2") do |config|
   config.vagrant.plugins = {
@@ -12,7 +12,7 @@ Vagrant.configure("2") do |config|
   }
   config.vm.box = "fedora/34-cloud-base"
   config.vm.boot_timeout = ENV['TEST_VM_BOOT_TIMEOUT'] || 600 # seconds
-  config.vm.synced_folder '../../../../../dist/artifacts', '/vagrant', type: 'rsync', disabled: false,
+  config.vm.synced_folder '../../../../dist/artifacts', '/vagrant', type: 'rsync', disabled: false,
     rsync__exclude: ENV['RSYNC_EXCLUDE'] || '*.tar.*'
 
   config.vm.define 'cgroup-unified', primary: true do |test|

tests/e2e/README.md

+# End-to-End (E2E) Tests
+
+E2E tests cover multi-node K3s configuration and administration: bringup, update, teardown etc. across a wide range of operating systems. E2E tests are run nightly as part of K3s quality assurance (QA).
+
+## Framework 
+End-to-end tests utilize [Ginkgo](https://onsi.github.io/ginkgo/) and [Gomega](https://onsi.github.io/gomega/) like the integration tests, but rely on [Vagrant](https://www.vagrantup.com/) to provide the underlying cluster configuration. 
+
+Currently tested operating systems are:
+- [Ubuntu 20.04](https://app.vagrantup.com/generic/boxes/ubuntu2004)
+- [Leap 15.3](https://app.vagrantup.com/opensuse/boxes/Leap-15.3.x86_64) (stand-in for SLE-Server)
+- [MicroOS](https://app.vagrantup.com/dweomer/boxes/microos.amd64) (stand-in for SLE-Micro)
+
+## Format
+
+All E2E tests should be placed under `tests/e2e/<TEST_NAME>`.  
+All E2E test functions should be named: `Test_E2E<TEST_NAME>`.  
+A E2E test consists of two parts:
+1. `Vagrantfile`: a vagrant file which describes and configures the VMs upon which the cluster and test will run
+2. `<TEST_NAME>.go`: A go test file which calls `vagrant up` and controls the actual testing of the cluster
+
+See the [validate cluster test](../tests/e2e/validatecluster/validatecluster_test.go) as an example.
+
+## Running
+
+Generally, E2E tests are run as a nightly Jenkins job for QA. They can still be run locally but additional setup may be required. By default, all E2E tests are designed with `libvirt` as the underlying VM provider. Instructions for installing libvirt and its associated vagrant plugin, `vagrant-libvirt` can be found [here.](https://github.com/vagrant-libvirt/vagrant-libvirt#installation) `VirtualBox` is also supported as a backup VM provider.
+
+Once setup is complete, all E2E tests can be run with:
+```bash
+go test -timeout=15m ./tests/e2e/... -run E2E
+```
+Tests can be run individually with:
+```bash
+go test -timeout=15m ./tests/e2e/validatecluster/... -run E2E
+#or
+go test -timeout=15m ./tests/e2e/... -run E2EClusterValidation
+```
+
+Additionally, to generate junit reporting for the tests, the Ginkgo CLI is used. Installation instructions can be found [here.](https://onsi.github.io/ginkgo/#getting-started)  
+
+To run the all E2E tests and generate JUnit testing reports:
+```
+ginkgo --junit-report=result.xml ./tests/e2e/...
+```
+
+Note: The `go test` default timeout is 10 minutes, thus the `-timeout` flag should be used. The `ginkgo` default timeout is 1 hour, no timeout flag is needed.
\ No newline at end of file

tests/e2e/go.mod

-module github.com/k3s-io/k3s/tests/e2e
-
-go 1.15
-
-require (
-	github.com/kr/pretty v0.2.0 // indirect
-	github.com/onsi/ginkgo/v2 v2.1.1
-	github.com/onsi/gomega v1.17.0
-	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
-)

tests/e2e/secretsencryption/secretsencryption_test.go

@@ -17,8 +17,8 @@ import (
 var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system")
 var serverCount = flag.Int("serverCount", 3, "number of server nodes")
 
-//valid format: RELEASE_VERSION=v1.23.1+k3s2 or nil for latest commit from master
-var installType = flag.String("installType", "", "version or nil to use latest commit")
+// Environment Variables Info:
+// E2E_RELEASE_VERSION=v1.23.1+k3s2 or nil for latest commit from master
 
 func Test_E2EClusterValidation(t *testing.T) {
 	RegisterFailHandler(Fail)
@@ -35,7 +35,7 @@ var _ = Describe("Verify Secrets Encryption Rotation", func() {
 	Context("Cluster :", func() {
 		It("Starts up with no issues", func() {
 			var err error
-			serverNodeNames, _, err = e2e.CreateCluster(*nodeOS, *serverCount, 0, *installType)
+			serverNodeNames, _, err = e2e.CreateCluster(*nodeOS, *serverCount, 0)
 			Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog)
 			fmt.Println("CLUSTER CONFIG")
 			fmt.Println("OS:", *nodeOS)

tests/e2e/testutils.go

@@ -70,18 +70,18 @@ func CreateCluster(nodeOS string, serverCount, agentCount int) ([]string, []stri
 }
 
 func DeployWorkload(workload, kubeconfig string, arch bool) (string, error) {
-	resource_dir := "../amd64_resource_files"
+	resourceDir := "../amd64_resource_files"
 	if arch {
-		resource_dir = "../arm64_resource_files"
+		resourceDir = "../arm64_resource_files"
 	}
-	files, err := ioutil.ReadDir(resource_dir)
+	files, err := ioutil.ReadDir(resourceDir)
 	if err != nil {
 		err = fmt.Errorf("%s : Unable to read resource manifest file for %s", err, workload)
 		return "", err
 	}
 	fmt.Println("\nDeploying", workload)
 	for _, f := range files {
-		filename := filepath.Join(resource_dir, f.Name())
+		filename := filepath.Join(resourceDir, f.Name())
 		if strings.TrimSpace(f.Name()) == workload {
 			cmd := "kubectl apply -f " + filename + " --kubeconfig=" + kubeconfig
 			return RunCommand(cmd)

tests/e2e/upgradecluster/upgradecluster_test.go

@@ -18,7 +18,9 @@ var serverCount = flag.Int("serverCount", 3, "number of server nodes")
 var agentCount = flag.Int("agentCount", 2, "number of agent nodes")
 
 // Environment Variables Info:
-// E2E_RELEASE_VERSION=v1.23.1+k3s2 or nil for latest commit from master
+// E2E_RELEASE_VERSION=v1.23.3+k3s1
+// OR
+// E2E_RELEASE_CHANNEL=(commit|latest|stable), commit pulls latest commit from master
 
 func Test_E2EUpgradeValidation(t *testing.T) {
 	RegisterFailHandler(Fail)
@@ -96,7 +98,7 @@ var _ = Describe("Verify Upgrade", func() {
 			Expect(err).NotTo(HaveOccurred(), "NodePort manifest not deployed")
 
 			for _, nodeName := range serverNodeNames {
-				node_external_ip, _ := e2e.FetchNodeExternalIP(nodeName)
+				nodeExternalIP, _ := e2e.FetchNodeExternalIP(nodeName)
 				cmd := "kubectl get service nginx-nodeport-svc --kubeconfig=" + kubeConfigFile + " --output jsonpath=\"{.spec.ports[0].nodePort}\""
 				nodeport, err := e2e.RunCommand(cmd)
 				Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd)
@@ -106,7 +108,7 @@ var _ = Describe("Verify Upgrade", func() {
 					return e2e.RunCommand(cmd)
 				}, "240s", "5s").Should(ContainSubstring("test-nodeport"), "nodeport pod was not created")
 
-				cmd = "curl -L --insecure http://" + node_external_ip + ":" + nodeport + "/name.html"
+				cmd = "curl -L --insecure http://" + nodeExternalIP + ":" + nodeport + "/name.html"
 				fmt.Println(cmd)
 				Eventually(func() (string, error) {
 					return e2e.RunCommand(cmd)
@@ -291,7 +293,7 @@ var _ = Describe("Verify Upgrade", func() {
 		It("After upgrade verifies NodePort Service", func() {
 
 			for _, nodeName := range serverNodeNames {
-				node_external_ip, _ := e2e.FetchNodeExternalIP(nodeName)
+				nodeExternalIP, _ := e2e.FetchNodeExternalIP(nodeName)
 				cmd := "kubectl get service nginx-nodeport-svc --kubeconfig=" + kubeConfigFile + " --output jsonpath=\"{.spec.ports[0].nodePort}\""
 				nodeport, err := e2e.RunCommand(cmd)
 				Expect(err).NotTo(HaveOccurred())
@@ -301,7 +303,7 @@ var _ = Describe("Verify Upgrade", func() {
 					return e2e.RunCommand(cmd)
 				}, "240s", "5s").Should(ContainSubstring("test-nodeport"), "nodeport pod was not created")
 
-				cmd = "curl -L --insecure http://" + node_external_ip + ":" + nodeport + "/name.html"
+				cmd = "curl -L --insecure http://" + nodeExternalIP + ":" + nodeport + "/name.html"
 				fmt.Println(cmd)
 				Eventually(func() (string, error) {
 					return e2e.RunCommand(cmd)

tests/e2e/validatecluster/validatecluster_test.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/k3s-io/k3s/tests/e2e"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 )
@@ -101,7 +102,7 @@ var _ = Describe("Verify Create", func() {
 			Expect(err).NotTo(HaveOccurred(), "NodePort manifest not deployed")
 
 			for _, nodeName := range serverNodeNames {
-				node_external_ip, _ := e2e.FetchNodeExternalIP(nodeName)
+				nodeExternalIP, _ := e2e.FetchNodeExternalIP(nodeName)
 				cmd := "kubectl get service nginx-nodeport-svc --kubeconfig=" + kubeConfigFile + " --output jsonpath=\"{.spec.ports[0].nodePort}\""
 				nodeport, err := e2e.RunCommand(cmd)
 				Expect(err).NotTo(HaveOccurred())
@@ -113,7 +114,7 @@ var _ = Describe("Verify Create", func() {
 					g.Expect(res).Should(ContainSubstring("test-nodeport"), "nodeport pod was not created")
 				}, "240s", "5s").Should(Succeed())
 
-				cmd = "curl -L --insecure http://" + node_external_ip + ":" + nodeport + "/name.html"
+				cmd = "curl -L --insecure http://" + nodeExternalIP + ":" + nodeport + "/name.html"
 				fmt.Println(cmd)
 				Eventually(func(g Gomega) {
 					res, err := e2e.RunCommand(cmd)

tests/vagrant/install/centos-7/Vagrantfile → tests/install/centos-7/Vagrantfile

@@ -2,7 +2,7 @@
 # vi: set ft=ruby :
 #
 
-ENV['TEST_INSTALL_SH'] ||= '../../../../install.sh'
+ENV['TEST_INSTALL_SH'] ||= '../../../install.sh'
 
 Vagrant.configure("2") do |config|
   config.vagrant.plugins = {

tests/vagrant/install/centos-8/Vagrantfile → tests/install/centos-8/Vagrantfile

@@ -2,7 +2,7 @@
 # vi: set ft=ruby :
 #
 
-ENV['TEST_INSTALL_SH'] ||= '../../../../install.sh'
+ENV['TEST_INSTALL_SH'] ||= '../../../install.sh'
 
 Vagrant.configure("2") do |config|
   config.vagrant.plugins = {

tests/vagrant/install/fedora-coreos/Vagrantfile → tests/install/fedora-coreos/Vagrantfile

@@ -2,7 +2,7 @@
 # vi: set ft=ruby :
 #
 
-ENV['TEST_INSTALL_SH'] ||= '../../../../install.sh'
+ENV['TEST_INSTALL_SH'] ||= '../../../install.sh'
 
 Vagrant.configure("2") do |config|
   config.vagrant.plugins = {

tests/vagrant/install/opensuse-leap/Vagrantfile → tests/install/opensuse-leap/Vagrantfile

@@ -2,7 +2,7 @@
 # vi: set ft=ruby :
 #
 
-ENV['TEST_INSTALL_SH'] ||= '../../../../install.sh'
+ENV['TEST_INSTALL_SH'] ||= '../../../install.sh'
 
 Vagrant.configure("2") do |config|
   config.vagrant.plugins = {

tests/vagrant/install/opensuse-microos/Vagrantfile → tests/install/opensuse-microos/Vagrantfile

@@ -2,7 +2,7 @@
 # vi: set ft=ruby :
 #
 
-ENV['TEST_INSTALL_SH'] ||= '../../../../install.sh'
+ENV['TEST_INSTALL_SH'] ||= '../../../install.sh'
 
 Vagrant.configure("2") do |config|
   config.vagrant.plugins = {

tests/vagrant/install/ubuntu-focal/Vagrantfile → tests/install/ubuntu-focal/Vagrantfile

@@ -2,7 +2,7 @@
 # vi: set ft=ruby :
 #
 
-ENV['TEST_INSTALL_SH'] ||= '../../../../install.sh'
+ENV['TEST_INSTALL_SH'] ||= '../../../install.sh'
 
 Vagrant.configure("2") do |config|
   config.vagrant.plugins = {

tests/integration/README.md

+# Integration Tests
+
+Integration tests should be used to test a specific functionality of k3s that exists across multiple Go packages, either via exported function calls, or more often, CLI comands.
+Integration tests should be used for "black box" testing. 
+
+## Framework
+
+All integration tests in K3s follow a [Behavior Diven Development (BDD)](https://en.wikipedia.org/wiki/Behavior-driven_development) style. Specifically, K3s uses [Ginkgo](https://onsi.github.io/ginkgo/) and [Gomega](https://onsi.github.io/gomega/) to drive the tests.  
+To generate an initial test, the command `ginkgo bootstrap` can be used.
+
+To facilitate K3s CLI testing, see `tests/util/cmd.go` helper functions.
+
+## Format
+
+All integration tests should be placed under `tests/integration/<TEST_NAME>`.  
+All integration test files should be named: `<TEST_NAME>_int_test.go`.  
+All integration test functions should be named: `Test_Integration<TEST_NAME>`.  
+See the [local storage test](../tests/integration/localstorage/localstorage_int_test.go) as an example.
+
+## Running
+
+Integration tests can be run with no k3s cluster present, each test will spin up and kill the appropriate k3s server it needs.  
+Note: Integration tests must be run as root, prefix the commands below with `sudo -E env "PATH=$PATH"` if a sudo user.
+```bash
+go test ./tests/integration/... -run Integration
+```
+
+Additionally, to generate JUnit reporting for the tests, the Ginkgo CLI is used
+```
+ginkgo --junit-report=result.xml ./tests/integration/...
+```
+
+Integration tests can be run on an existing single-node cluster via compile time flag, tests will skip if the server is not configured correctly.
+```bash
+go test -ldflags "-X 'github.com/k3s-io/k3s/tests/integration.existingServer=True'" ./tests/integration/... -run Integration
+```
+
+Integration tests can also be run via a [Sonobuoy](https://sonobuoy.io/docs/v0.53.2/) plugin on an existing single-node cluster.
+```bash
+./scripts/build-tests-sonobuoy
+sudo KUBECONFIG=/etc/rancher/k3s/k3s.yaml sonobuoy run --plugin ./dist/artifacts/k3s-int-tests.yaml
+```
+Check the sonobuoy status and retrieve results
+``` 
+sudo KUBECONFIG=/etc/rancher/k3s/k3s.yaml sonobuoy status
+sudo KUBECONFIG=/etc/rancher/k3s/k3s.yaml sonobuoy retrieve
+sudo KUBECONFIG=/etc/rancher/k3s/k3s.yaml sonobuoy results <TAR_FILE_FROM_RETRIEVE>
+```

tests/integration/custometcdargs/custometcdargs_int_test.go

@@ -5,7 +5,7 @@ import (
 	"strings"
 	"testing"
 
-	testutil "github.com/k3s-io/k3s/tests/util"
+	testutil "github.com/k3s-io/k3s/tests/integration"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 )

tests/integration/dualstack/dualstack_int_test.go

@@ -5,7 +5,7 @@ import (
 	"strings"
 	"testing"
 
-	testutil "github.com/k3s-io/k3s/tests/util"
+	testutil "github.com/k3s-io/k3s/tests/integration"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 )

tests/integration/etcdrestore/etcd_restore_int_test.go

@@ -4,7 +4,7 @@ import (
 	"strings"
 	"testing"
 
-	testutil "github.com/k3s-io/k3s/tests/util"
+	testutil "github.com/k3s-io/k3s/tests/integration"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 )

tests/integration/etcdsnapshot/etcdsnapshot_int_test.go

@@ -6,7 +6,7 @@ import (
 	"testing"
 	"time"
 
-	testutil "github.com/k3s-io/k3s/tests/util"
+	testutil "github.com/k3s-io/k3s/tests/integration"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 )

tests/util/integration.go → tests/integration/integration.go

-package util
+package integration
 
 import (
 	"bufio"

tests/integration/localstorage/localstorage_int_test.go

@@ -7,7 +7,7 @@ import (
 	"strings"
 	"testing"
 
-	testutil "github.com/k3s-io/k3s/tests/util"
+	testutil "github.com/k3s-io/k3s/tests/integration"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 )

tests/integration/secretsencryption/secretsencryption_int_test.go

@@ -6,7 +6,7 @@ import (
 	"testing"
 	"time"
 
-	testutil "github.com/k3s-io/k3s/tests/util"
+	testutil "github.com/k3s-io/k3s/tests/integration"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 )

tests/vagrant/snapshotter/btrfs/opensuse-leap/Vagrantfile → tests/snapshotter/btrfs/opensuse-leap/Vagrantfile

@@ -10,7 +10,7 @@ Vagrant.configure("2") do |config|
   }
   config.vm.box = "opensuse/Leap-15.3.x86_64"
   config.vm.boot_timeout = ENV['TEST_VM_BOOT_TIMEOUT'] || 600 # seconds
-  config.vm.synced_folder '../../../../../dist/artifacts', '/vagrant', type: 'rsync', disabled: false,
+  config.vm.synced_folder '../../../../dist/artifacts', '/vagrant', type: 'rsync', disabled: false,
     rsync__exclude: ENV['RSYNC_EXCLUDE'] || '*.tar.*'
 
   config.vm.define 'snapshotter-btrfs', primary: true do |test|

tests/util/unit.go → tests/unit.go

-package util
+package tests
 
 import (
 	"net"

tests/util/wait-for-coredns.sh

-#!/bin/bash
-# Wait for CoreDNS pods to be ready.
-
-set -x
-echo "Waiting for CoreDNS pods to be ready..."
-counter=0
-# `kubectl wait` fails when the pods with the specified label are not created yet
-until kubectl wait --for=condition=ready pods --namespace=kube-system -l k8s-app=kube-dns; do
-  ((counter++))
-  if [[ $counter -eq 20 ]]; then
-    echo "CoreDNS not running?"
-    kubectl get pods -A
-    kubectl get nodes -o wide
-    exit 1
-  fi
-  sleep 10
-done

tests/vagrant/.gitignore

-.vagrant/