Merge pull request #6086 from dereknola/test_bulk_123

.github/workflows/cgroup.yaml

@@ -42,7 +42,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        vm: [fedora-34]
+        vm: [fedora-35]
         mode: [unified]
       max-parallel: 1
     defaults:

.github/workflows/install.yaml

@@ -13,28 +13,37 @@ on:
       - "tests/install/**"
   workflow_dispatch: {}
 jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-20.04
+    timeout-minutes: 20
+    steps:
+    - name: "Checkout"
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 1
+    - name: "Make"
+      run: DOCKER_BUILDKIT=1 SKIP_VALIDATE=1 make
+    - name: "Upload k3s binary"
+      uses: actions/upload-artifact@v2
+      with:
+        name: k3s
+        path: ./dist/artifacts/k3s
   test:
     name: "Smoke Test"
+    needs: build
     runs-on: macos-12
     timeout-minutes: 40
     strategy:
       fail-fast: false
       matrix:
-        channel: [stable]
         vm: [centos-7, rocky-8, fedora-coreos, opensuse-leap, opensuse-microos, ubuntu-focal]
-        include:
-          - {channel: latest, vm: rocky-8}
-          - {channel: latest, vm: ubuntu-focal}
-          - {channel: latest, vm: opensuse-leap}
-          - {channel: testing, vm: rocky-8}
-          - {channel: testing, vm: opensuse-leap}
-          - {channel: testing, vm: ubuntu-focal}
       max-parallel: 2
     defaults:
       run:
         working-directory: tests/install/${{ matrix.vm }}
     env:
-      INSTALL_K3S_CHANNEL: ${{ matrix.channel }}
+      INSTALL_K3S_SKIP_DOWNLOAD: true
     steps:
       - name: "Checkout"
         uses: actions/checkout@v2
@@ -49,9 +58,25 @@ jobs:
         id: vagrant-cache
         continue-on-error: true
       - name: "Vagrant Plugin(s)"
-        run: vagrant plugin install vagrant-k3s vagrant-reload
-      - name: "Vagrant Up ⏩ Install K3s"
-        run: vagrant up
+        run: vagrant plugin install vagrant-k3s vagrant-reload vagrant-scp
+      - name: "Download k3s binary"
+        uses: actions/download-artifact@v2
+        with:
+          name: k3s
+          path: tests/install/${{ matrix.vm }}
+      - name: "Vagrant Up"
+        run: vagrant up --no-provision
+      - name: "Upload k3s binary"
+        run: |
+          chmod +x k3s
+          vagrant scp k3s /tmp/k3s
+          vagrant ssh -c "sudo mv /tmp/k3s /usr/local/bin/k3s"
+          vagrant provision --provision-with=k3s-upload
+      - name: "⏩ Install K3s"
+        run: |
+          vagrant provision --provision-with=k3s-prepare
+          vagrant provision --provision-with=k3s-install
+          if [ ${{ matrix.vm }} = 'opensuse-microos' ]; then vagrant reload --no-provision; fi
       - name: "⏳ Node"
         run: vagrant provision --provision-with=k3s-wait-for-node
       - name: "⏳ CoreDNS"

.github/workflows/integration.yaml

@@ -60,7 +60,7 @@ jobs:
     - name: Run Integration Tests
       run: | 
         chmod +x ./dist/artifacts/k3s
-        sudo -E env "PATH=$PATH" go test ./tests/integration/... -run Integration
+        sudo -E env "PATH=$PATH" go test -v ./tests/integration/... -run Integration
     - name: On Failure, Launch Debug Session
       if: ${{ failure() }}
       uses: mxschmitt/action-tmate@v3

.github/workflows/nightly-install.yaml

+name: Nightly Install
+on:
+  schedule:
+    - cron: "0 0 * * 1-5"
+  workflow_dispatch: {}
+jobs:
+  test:
+    name: "Smoke Test"
+    runs-on: macos-12
+    timeout-minutes: 40
+    strategy:
+      fail-fast: false
+      matrix:
+        channel: [v1.23]
+        vm: [centos-7, rocky-8, fedora-coreos, opensuse-leap, opensuse-microos, ubuntu-focal]
+      max-parallel: 2
+    defaults:
+      run:
+        working-directory: tests/install/${{ matrix.vm }}
+    env:
+      INSTALL_K3S_CHANNEL: ${{ matrix.channel }}
+    steps:
+      - name: "Checkout"
+        uses: actions/checkout@v2
+        with: {fetch-depth: 1}
+      - name: "Vagrant Cache"
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.vagrant.d/boxes
+            ~/.vagrant.d/gems
+          key: install-${{ hashFiles(format('tests/install/{0}/Vagrantfile', matrix.vm)) }}
+        id: vagrant-cache
+        continue-on-error: true
+      - name: "Vagrant Plugin(s)"
+        run: vagrant plugin install vagrant-k3s vagrant-reload
+      - name: "Vagrant Up ⏩ Install K3s"
+        run: vagrant up
+      - name: "⏳ Node"
+        run: vagrant provision --provision-with=k3s-wait-for-node
+      - name: "⏳ CoreDNS"
+        run: vagrant provision --provision-with=k3s-wait-for-coredns
+      - name: "⏳ Local Storage"
+        run: vagrant provision --provision-with=k3s-wait-for-local-storage
+        continue-on-error: true
+      - name: "⏳ Metrics Server"
+        run: vagrant provision --provision-with=k3s-wait-for-metrics-server
+        continue-on-error: true
+      - name: "⏳ Traefik"
+        run: vagrant provision --provision-with=k3s-wait-for-traefik
+        continue-on-error: true
+      - name: "k3s-status"
+        run: vagrant provision --provision-with=k3s-status
+      - name: "k3s-procps"
+        run: vagrant provision --provision-with=k3s-procps

install.sh

@@ -483,7 +483,7 @@ setup_selinux() {
     ${package_installer} install -y https://${rpm_site}/k3s/${rpm_channel}/common/${rpm_site_infix}/noarch/k3s-selinux-0.4-1.${rpm_target}.noarch.rpm
 "
 
-    if [ "$INSTALL_K3S_SKIP_SELINUX_RPM" = true ] || can_skip_download || [ ! -d /usr/share/selinux ]; then
+    if [ "$INSTALL_K3S_SKIP_SELINUX_RPM" = true ] || [ ! -d /usr/share/selinux ]; then
         info "Skipping installation of SELinux RPM"
     elif  [ "${ID_LIKE:-}" != coreos ] && [ "${VARIANT_ID:-}" != coreos ]; then
         install_selinux_rpm ${rpm_site} ${rpm_channel} ${rpm_target} ${rpm_site_infix}

tests/TESTING.md

@@ -71,7 +71,7 @@ The sub-directories therein contain fixtures for running simple clusters to asse
   - [Ubuntu 20.04](../tests/install/ubuntu-focal) (Focal Fossa)
 - [Control Groups](../tests/cgroup) :arrow_right: on any code change
   - [mode=unified](../tests/cgroup/unified) (cgroups v2)
-    - [Fedora 34](../tests/cgroup/unified/fedora-34) (rootfull + rootless)
+    - [Fedora 35](../tests/cgroup/unified/fedora-35) (rootfull + rootless)
 - [Snapshotter](../tests/snapshotter/btrfs/opensuse-leap) :arrow_right: on any code change
   - [BTRFS](../tests/snapshotter/btrfs) ([containerd built-in](https://github.com/containerd/containerd/tree/main/snapshots/btrfs))
     - [Leap 15.3](../tests/snapshotter/btrfs/opensuse-leap)

tests/cgroup/unified/fedora-34/Vagrantfile → tests/cgroup/unified/fedora-35/Vagrantfile

@@ -10,7 +10,7 @@ Vagrant.configure("2") do |config|
   config.vagrant.plugins = {
     'vagrant-k3s' => {:version => '~> 0.1.3'},
   }
-  config.vm.box = "fedora/34-cloud-base"
+  config.vm.box = "fedora/35-cloud-base"
   config.vm.boot_timeout = ENV['TEST_VM_BOOT_TIMEOUT'] || 600 # seconds
   config.vm.synced_folder '../../../../dist/artifacts', '/vagrant', type: 'rsync', disabled: false,
     rsync__exclude: ENV['RSYNC_EXCLUDE'] || '*.tar.*'
@@ -61,12 +61,16 @@ EOF
     test.vm.provision 'k3s-install', type: 'k3s', run: ENV['CI'] == 'true' ? 'never' : 'once' do |k3s|
       k3s.args = %w[server]
       k3s.env = %w[INSTALL_K3S_NAME=server INSTALL_K3S_SKIP_DOWNLOAD=true K3S_TOKEN=vagrant INSTALL_K3S_SKIP_ENABLE=true]
-      k3s.config = {
-        'disable' => %w[local-storage metrics-server servicelb traefik],
-        'disable-helm-controller' => true,
-        'disable-network-policy' => true,
-        'write-kubeconfig-mode' => '0644',
-      }
+      k3s.config = <<~YAML
+        disable:
+        - local-storage 
+        - metrics-server
+        - servicelb
+        - traefik
+        disable-helm-controller: true
+        disable-network-policy: true
+        write-kubeconfig-mode: '0644'
+      YAML
       k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321
     end
     test.vm.provision "k3s-start", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh|

tests/e2e/Dockerfile.build

-FROM golang:alpine
-
-ARG TERRAFORM_VERSION=0.12.10
-ENV TERRAFORM_VERSION=$TERRAFORM_VERSION
-
-
-RUN apk update && \
-    apk upgrade --update-cache --available && \
-    apk add curl git jq bash openssh unzip gcc g++ make ca-certificates && \
-    curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
-
-RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && \
-    chmod +x ./kubectl && \
-    mv ./kubectl /usr/local/bin
-RUN mkdir tmp && \
-    curl "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip" -o tmp/terraform.zip && \
-    unzip tmp/terraform.zip -d /usr/local/bin && \
-    chmod +x /usr/local/bin/terraform && \
-    rm -rf tmp
-
-WORKDIR $GOPATH/src/github.com/k3s-io/k3s-io/k3s/tests/e2e
-
-COPY . .
-

tests/e2e/README.md

@@ -20,6 +20,32 @@ A E2E test consists of two parts:
 
 See the [validate cluster test](../tests/e2e/validatecluster/validatecluster_test.go) as an example.
 
+
+## Setup
+
+To run the E2E tests, you must first install the following:
+- Vagrant
+- Libvirt
+- Vagrant plugins
+
+### Vagrant 
+
+Download the latest version (currently 2.2.19) of Vagrant [*from the website*](https://www.vagrantup.com/downloads). Do not use built-in packages, they often old or do not include the required ruby library extensions necessary to get certain plugins working.
+
+### Libvirt
+Follow the OS specific guides to install libvirt/qemu on your host:  
+    - [openSUSE](https://documentation.suse.com/sles/15-SP1/html/SLES-all/cha-vt-installation.html)
+    - [ubuntu](https://ubuntu.com/server/docs/virtualization-libvirt)
+    - [debian](https://wiki.debian.org/KVM#Installation)
+    - [fedora](https://developer.fedoraproject.org/tools/virtualization/installing-libvirt-and-virt-install-on-fedora-linux.html)
+
+### Vagrant plugins
+Install the necessary vagrant plugins with the following command:
+
+```bash
+vagrant plugin install vagrant-libvirt vagrant-scp vagrant-k3s vagrant-reload
+```
+
 ## Running
 
 Generally, E2E tests are run as a nightly Jenkins job for QA. They can still be run locally but additional setup may be required. By default, all E2E tests are designed with `libvirt` as the underlying VM provider. Instructions for installing libvirt and its associated vagrant plugin, `vagrant-libvirt` can be found [here.](https://github.com/vagrant-libvirt/vagrant-libvirt#installation) `VirtualBox` is also supported as a backup VM provider.

tests/e2e/amd64_resource_files/clusterip.yaml

@@ -24,7 +24,7 @@ metadata:
   labels:
     k8s-app: nginx-app-clusterip
   name: nginx-clusterip-svc
-  namespace: default 
+  namespace: default
 spec:
   type: ClusterIP
   ports:

tests/e2e/arm_resource_files/clusterip.yaml → tests/e2e/cis_amd64_resource_files/clusterip.yaml

@@ -14,9 +14,9 @@ spec:
     spec:
       containers:
       - name: nginx
-        image: shylajarancher19/shylajaarm64:v1.0
+        image: ranchertest/mytestcontainer:unprivileged
         ports:
-        - containerPort: 80
+        - containerPort: 8080
 ---
 apiVersion: v1
 kind: Service
@@ -24,10 +24,10 @@ metadata:
   labels:
     k8s-app: nginx-app-clusterip
   name: nginx-clusterip-svc
-  namespace: default 
+  namespace: default
 spec:
   type: ClusterIP
   ports:
-    - port: 80
+    - port: 8080
   selector:
     k8s-app: nginx-app-clusterip

tests/e2e/cis_amd64_resource_files/dnsutils.yaml

+apiVersion: v1
+kind: Pod
+metadata:
+  name: dnsutils
+  namespace: default
+spec:
+  containers:
+  - name: dnsutils
+    image: gcr.io/kubernetes-e2e-test-images/dnsutils:1.3
+    command:
+      - sleep
+      - "3600"
+    imagePullPolicy: IfNotPresent
+  restartPolicy: Always

tests/e2e/arm_resource_files/ingress.yaml → tests/e2e/cis_amd64_resource_files/ingress.yaml

-apiVersion: extensions/v1beta1
-kind: Ingress
-metadata:
- name: ingress
-spec:
- rules:
- - host: foo1.bar.com
-   http:
-     paths:
-     - path: /name.html
-       backend:
-         serviceName: nginx-ingress-svc
-         servicePort: 80
----
 apiVersion: v1
 kind: Service
 metadata:
@@ -20,29 +6,23 @@ metadata:
    k8s-app: nginx-app-ingress
 spec:
  ports:
- - port: 80
-   targetPort: 80
+ - port: 8080
+   targetPort: 8080
    protocol: TCP
    name: http
  selector:
    k8s-app: nginx-app-ingress
 ---
-apiVersion: v1
-kind: ReplicationController
+apiVersion: extensions/v1beta1
+kind: Ingress
 metadata:
- name: test-ingress
+  name: test-ingress
 spec:
- replicas: 2
- selector:
-   k8s-app: nginx-app-ingress
- template:
-   metadata:
-     labels:
-       k8s-app: nginx-app-ingress
-   spec:
-     terminationGracePeriodSeconds: 60
-     containers:
-     - name: testcontainer
-       image: shylajarancher19/shylajaarm64:v1.0
-       ports:
-       - containerPort: 80
+  rules:
+  - host: foo1.bar.com
+    http:
+      paths:
+      - backend:
+          serviceName: nginx-ingress-svc
+          servicePort: 8080
+        pathType: ImplementationSpecific

tests/e2e/arm_resource_files/loadbalancer.yaml → tests/e2e/cis_amd64_resource_files/loadbalancer.yaml

----
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -15,22 +14,23 @@ spec:
     spec:
       containers:
       - name: nginx
-        image: shylajarancher19/shylajaarm64:v1.0
+        image: ranchertest/mytestcontainer:unprivileged
         ports:
-        - containerPort: 80
+        - containerPort: 8080
 ---
 apiVersion: v1
 kind: Service
 metadata:
- name: nginx-loadbalancer-svc
- labels:
-   k8s-app: nginx-app-loadbalancer
+  labels:
+    k8s-app: nginx-app-loadbalancer
+  name: nginx-loadbalancer-svc
+  namespace: default
 spec:
- type: LoadBalancer
- ports:
- - port: 81
-   targetPort: 80
-   protocol: TCP
-   name: http
- selector:
-   k8s-app: nginx-app-loadbalancer
+  type: LoadBalancer
+  ports:
+  - port: 81
+    targetPort: 8080
+    protocol: TCP
+    name: http
+  selector:
+    k8s-app: nginx-app-loadbalancer

tests/e2e/arm_resource_files/nodeport.yaml → tests/e2e/cis_amd64_resource_files/nodeport.yaml

@@ -14,9 +14,9 @@ spec:
     spec:
       containers:
       - name: nginx
-        image: shylajarancher19/shylajaarm64:v1.0
+        image: ranchertest/mytestcontainer:unprivileged
         ports:
-        - containerPort: 80
+        - containerPort: 8080
 ---
 apiVersion: v1
 kind: Service
@@ -28,7 +28,7 @@ metadata:
 spec:
   type: NodePort
   ports:
-    - port: 80
+    - port: 8080
       nodePort: 30096
       name: http
   selector:

tests/e2e/cis_amd64_resource_files/secrets.yaml

+apiVersion: v1
+kind: Secret
+metadata:
+  name: e2e-secret1
+type: Opaque
+stringData:
+  config.yaml: |
+    key: "hello"
+    val: "world"
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: e2e-secret2
+type: Opaque
+stringData:
+  config.yaml: |
+    key: "good"
+    val: "day"
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: e2e-secret3
+type: Opaque
+stringData:
+  config.yaml: |
+    key: "top-secret"
+    val: "information"
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: e2e-secret4
+type: Opaque
+stringData:
+  config.yaml: |
+    key: "lock"
+    val: "key"
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: e2e-secret5
+type: Opaque
+stringData:
+  config.yaml: |
+    key: "last"
+    val: "call"
\ No newline at end of file

tests/e2e/dualstack/Vagrantfile

@@ -26,12 +26,13 @@ def provision(vm, roles, role_num, node_num)
     :libvirt__ipv6_address => "#{NETWORK6_PREFIX}::1",
     :libvirt__ipv6_prefix => "64"
     
-  vagrant_defaults = '../vagrantdefaults.rb'
-  load vagrant_defaults if File.exists?(vagrant_defaults)
+  scripts_location = Dir.exists?("./scripts") ? "./scripts" : "../scripts" 
+  vagrant_defaults = File.exists?("./vagrantdefaults.rb") ? "./vagrantdefaults.rb" : "../vagrantdefaults.rb"
+  load vagrant_defaults
   
   defaultOSConfigure(vm)
 
-  vm.provision "IPv6 Setup", type: "shell", path: "../scripts/ipv6.sh", args: [node_ip4, node_ip6, vm.box]
+  vm.provision "IPv6 Setup", type: "shell", path: scripts_location +"/ipv6.sh", args: [node_ip4, node_ip6, vm.box.to_s]
   install_type = getInstallType(vm, RELEASE_VERSION, GITHUB_BRANCH)
 
   vm.provision "Ping Check", type: "shell", inline: "ping -c 2 k3s.io"

tests/e2e/dualstack/dualstack_test.go

@@ -16,6 +16,7 @@ import (
 var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system")
 var serverCount = flag.Int("serverCount", 3, "number of server nodes")
 var agentCount = flag.Int("agentCount", 0, "number of agent nodes")
+var hardened = flag.Bool("hardened", false, "true or false")
 
 // Environment Variables Info:
 // E2E_RELEASE_VERSION=v1.23.1+k3s1 or nil for latest commit from master
@@ -129,7 +130,7 @@ var _ = Describe("Verify DualStack Configuration", func() {
 	})
 
 	It("Verifies ClusterIP Service", func() {
-		_, err := e2e.DeployWorkload("dualstack_clusterip.yaml", kubeConfigFile, false)
+		_, err := e2e.DeployWorkload("dualstack_clusterip.yaml", kubeConfigFile, *hardened)
 		Expect(err).NotTo(HaveOccurred())
 		Eventually(func() (string, error) {
 			cmd := "kubectl get pods -o=name -l k8s-app=nginx-app-clusterip --field-selector=status.phase=Running --kubeconfig=" + kubeConfigFile
@@ -157,7 +158,7 @@ var _ = Describe("Verify DualStack Configuration", func() {
 		}
 	})
 	It("Verifies Ingress", func() {
-		_, err := e2e.DeployWorkload("dualstack_ingress.yaml", kubeConfigFile, false)
+		_, err := e2e.DeployWorkload("dualstack_ingress.yaml", kubeConfigFile, *hardened)
 		Expect(err).NotTo(HaveOccurred(), "Ingress manifest not deployed")
 		cmd := "kubectl get ingress ds-ingress --kubeconfig=" + kubeConfigFile + " -o jsonpath=\"{.spec.rules[*].host}\""
 		hostName, err := e2e.RunCommand(cmd)
@@ -177,7 +178,7 @@ var _ = Describe("Verify DualStack Configuration", func() {
 	})
 
 	It("Verifies NodePort Service", func() {
-		_, err := e2e.DeployWorkload("dualstack_nodeport.yaml", kubeConfigFile, false)
+		_, err := e2e.DeployWorkload("dualstack_nodeport.yaml", kubeConfigFile, *hardened)
 		Expect(err).NotTo(HaveOccurred())
 		cmd := "kubectl get service ds-nodeport-svc --kubeconfig=" + kubeConfigFile + " --output jsonpath=\"{.spec.ports[0].nodePort}\""
 		nodeport, err := e2e.RunCommand(cmd)

tests/e2e/e2e_test_playbook.yaml

+---
+- name: Run Cluster validation tests
+  gather_facts: false
+  hosts: node
+  vars:
+    user: "{{ user }}"
+    key: "{{ key }}"
+    ip: "{{ ip }}"
+    dir: scripts
+  tasks:
+    - name : Print var
+      debug:
+        msg: |
+          user: {{ user }}
+          key: {{ key }}
+          ip: {{ ip }}
+          db: {{ db }}
+          nodeOS: {{ nodeOS }}
+          serverCount: {{ serverCount }}
+          agentCount: {{ agentCount }}
+
+    - name: build container image
+      command: chdir=k3s/tests/e2e /usr/local/bin/docker build -t k3s_nightly_build -f {{ dir }}/Dockerfile .
+
+    - name: Delete docker container
+      command: "/usr/local/bin/docker rm createcluster -f"
+      ignore_errors: True
+
+    - name: create docker container
+      command: "/usr/local/bin/docker run -d -it -v ~/config:/config --cap-add=NET_ADMIN --device /dev/net/tun --sysctl net.ipv6.conf.all.disable_ipv6=0  --name createcluster --privileged k3s_nightly_build"
+
+    - name: execute command in docker
+      shell: |
+        /usr/local/bin/docker exec -it createcluster /bin/bash ./run_tests.sh {{ key }} {{ user }} {{ ip }} {{ db }} {{ nodeOS }} {{ serverCount }} {{ agentCount }}

tests/e2e/scripts/Dockerfile

+FROM ubuntu:20.04
+ARG EXTERNAL_ENCODED_VPN
+ARG VPN_ENCODED_LOGIN
+
+RUN apt-get update && \
+    apt-get install -y curl bridge-utils iputils-ping openvpn openssh-client && \
+    mkdir -p /dev/net && \
+    mknod /dev/net/tun c 10 200 && \
+    chmod 600 /dev/net/tun
+
+RUN if [[ -z "$EXTERNAL_ENCODED_VPN" ]] ; then echo "no vpn provided" ;  \
+    else echo -n $EXTERNAL_ENCODED_VPN | base64 -di > external.ovpn && \
+    if [[ -z "$VPN_ENCODED_LOGIN" ]]; then echo "no passcode provided" ; \
+    else echo -n $VPN_ENCODED_LOGIN | base64 -di > authfile && \
+    sed -i 's/auth-user-pass/auth-user-pass authfile/g' external.ovpn; fi ; fi
+
+WORKDIR .
+COPY scripts/run_tests.sh .
+COPY scripts/init.sh .

tests/e2e/scripts/Jenkinsfile

+pipeline {
+    agent any
+    environment {
+	AWS_SSH_PEM_KEY = credentials('AWS_SSH_PEM_KEY')
+	VPN_ENCODED_LOGIN = credentials('VPN_ENCODED_LOGIN')
+	EXTERNAL_ENCODED_VPN = credentials('EXTERNAL_ENCODED_VPN')
+    }
+        
+    stages {
+        stage('Git Checkout') {
+            steps {
+                    git branch: 'master', url: 'https://github.com/k3s-io/k3s.git'
+                    script {
+                        dir("${WORKSPACE}/tests/e2e") {
+                        if (env.AWS_SSH_PEM_KEY && env.AWS_SSH_KEY_NAME) {
+                                    def decoded = new String(AWS_SSH_PEM_KEY.decodeBase64())
+                                    writeFile file: AWS_SSH_KEY_NAME, text: decoded
+                        }
+                    }
+                }
+            }
+        }
+        stage('Build Cluster'){
+            steps {
+                script {
+                        sh """
+                            cd ${WORKSPACE}/tests/e2e/
+                            /usr/bin/docker build -f ./scripts/Dockerfile --build-arg EXTERNAL_ENCODED_VPN="$EXTERNAL_ENCODED_VPN" \
+                            --build-arg VPN_ENCODED_LOGIN="$VPN_ENCODED_LOGIN" -t k3s_create_cluster .
+                            /usr/bin/docker run -d -it --cap-add=NET_ADMIN --device /dev/net/tun --sysctl net.ipv6.conf.all.disable_ipv6=0 \
+                            --name createcluster_${BUILD_NUMBER} --privileged k3s_create_cluster
+                            /usr/bin/docker cp ${AWS_SSH_KEY_NAME} createcluster_${BUILD_NUMBER}:/
+                            /usr/bin/docker exec createcluster_${BUILD_NUMBER} ./run_tests.sh ${AWS_SSH_KEY_NAME} ${USER} ${NODEIP} \
+                            ${NODEOS} ${SERVERCOUNT} ${AGENTCOUNT} ${DB} ${HARDENED}
+                        """
+                }
+            }
+        }
+	stage('Test Report') {
+            steps {
+                sh """
+                    /usr/bin/docker rm -f createcluster_${BUILD_NUMBER}
+                    /usr/bin/docker rmi -f k3s_create_cluster
+                """
+            }
+        }
+    }
+}

tests/e2e/scripts/hosts

+[node]
+localhost ansible_python_interpreter=/usr/local/opt/python@3.9/bin/python3.9
\ No newline at end of file

tests/e2e/scripts/init.sh

+#!/bin/sh
+# Script to install pre-requisites on vsphere node to run e2e test
+
+export PATH=$PATH:/usr/local/go/bin/:/usr/local/bin
+echo 'Installing Kubectl'
+curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" && \
+sudo mv kubectl /usr/local/bin/ && \
+chmod a+x /usr/local/bin/kubectl
+
+echo 'Installing jq'
+sudo apt-get -y install jq
+
+echo 'Installing Go'
+curl -L https://dl.google.com/go/go1.16.10.linux-amd64.tar.gz | tar xz
+sudo mv go /usr/local
+/usr/local/go/bin/go version
+go version
+
+echo 'Installing Virtualbox'
+sudo apt-get -y update && sudo apt-get -y upgrade
+sudo apt-get -y install virtualbox
+vboxmanage --version
+
+# Virtualbox >= 6.1.28 require `/etc/vbox/network.conf
+ls /etc/vbox 2>/dev/null || sudo mkdir -p /etc/vbox
+echo "* 10.0.0.0/8 192.168.0.0/16">>/etc/vbox/networks.conf
+
+echo 'Installing vagrant'
+sudo apt-get -y install -f unzip
+curl -O https://releases.hashicorp.com/vagrant/2.2.19/vagrant_2.2.19_linux_amd64.zip
+unzip vagrant_2.2.19_linux_amd64.zip
+sudo cp vagrant /usr/local/bin/
+vagrant --version
+sudo apt-get -y install libarchive-tools
+vagrant plugin install vagrant-k3s
+vagrant plugin install vagrant-reload
+
+echo 'Cloning repo'
+ls k3s 2>/dev/null || git clone https://github.com/k3s-io/k3s.git

tests/e2e/scripts/ipv6.sh

@@ -5,11 +5,17 @@ os=$3
 
 sysctl -w net.ipv6.conf.all.disable_ipv6=0
 sysctl -w net.ipv6.conf.eth1.accept_dad=0
+sysctl -w net.ipv6.conf.eth1.accept_ra=0
+sysctl -w net.ipv6.conf.eth1.forwarding=0
 
 if [ -z "${os##*ubuntu*}" ]; then
   netplan set ethernets.eth1.accept-ra=false
   netplan set ethernets.eth1.addresses=["$ip4_addr"/24,"$ip6_addr"/64]
   netplan apply
+elif [ -z "${os##*alpine*}" ]; then
+  iplink set eth1 down
+  iplink set eth1 up
+  ip -6 addr add "$ip6_addr"/64 dev eth1
 else
   ip -6 addr add "$ip6_addr"/64 dev eth1
 fi

tests/e2e/scripts/run_tests.sh

+#!/bin/bash
+servercount=${5:-3}
+agentcount=${6:-2}
+db=${7:-"etcd"}
+k3s_version=${k3s_version}
+k3s_channel=${k3s_channel:-"commit"}
+hardened=${8:-""}
+
+E2E_EXTERNAL_DB=$db && export E2E_EXTERNAL_DB
+
+eval openvpn --daemon --config external.ovpn &>/dev/null &
+sleep 10
+
+ssh -i "$1"  -o "StrictHostKeyChecking no" $2@$3 'cd k3s && git pull --rebase origin master'
+ssh -i "$1"  -o "StrictHostKeyChecking no" $2@$3 '/usr/local/go/bin/go get github.com/onsi/ginkgo/v2'
+ssh -i "$1"  -o "StrictHostKeyChecking no" $2@$3 '/usr/local/go/bin/go get github.com/onsi/gomega'
+ssh -i "$1"  -o "StrictHostKeyChecking no" $2@$3 '/usr/local/go/bin/go get github.com/k3s-io/k3s/tests/e2e'
+
+echo 'RUNNING CLUSTER VALIDATION TEST'
+ssh -i "$1"  -o "StrictHostKeyChecking no" $2@$3 'cd k3s/tests/e2e/validatecluster && vagrant destroy -f'
+ssh -i "$1"  -o "StrictHostKeyChecking no" $2@$3 "cd k3s/tests/e2e && E2E_HARDENED="$hardened" /usr/local/go/bin/go test -v validatecluster/validatecluster_test.go -nodeOS="$4" -serverCount=$((servercount)) -agentCount=$((agentcount)) -timeout=1h"
+
+echo 'RUNNING SECRETS ENCRYPTION TEST'
+ssh -i "$1"  -o "StrictHostKeyChecking no" $2@$3 'cd k3s/tests/e2e/secretsencryption && vagrant destroy -f'
+ssh -i "$1"  -o "StrictHostKeyChecking no" $2@$3 "cd k3s/tests/e2e && /usr/local/go/bin/go test -v secretsencryption/secretsencryption_test.go -nodeOS="$4" -serverCount=$((servercount)) -timeout=1h"
+
+E2E_RELEASE_VERSION=$k3s_version && export E2E_RELEASE_VERSION
+E2E_RELEASE_CHANNEL=$k3s_channel && export E2E_RELEASE_CHANNEL
+
+echo 'RUNNING CLUSTER UPGRADE TEST'
+ssh -i "$1"  -o "StrictHostKeyChecking no" $2@$3 'cd k3s/tests/e2e/upgradecluster && vagrant destroy -f'
+ssh -i "$1"  -o "StrictHostKeyChecking no" $2@$3 "cd k3s/tests/e2e && /usr/local/go/bin/go test -v upgradecluster/upgradecluster_test.go -nodeOS="$4" -serverCount=$((servercount)) -agentCount=$((agentcount))  -timeout=1h"

tests/e2e/secretsencryption/Vagrantfile

 ENV['VAGRANT_NO_PARALLEL'] = 'no'
-NODE_ROLES = (ENV['NODE_ROLES'] ||
+NODE_ROLES = (ENV['E2E_NODE_ROLES'] ||
   ["server-0", "server-1", "server-2"])
-NODE_BOXES = (ENV['NODE_BOXES'] ||
+NODE_BOXES = (ENV['E2E_NODE_BOXES'] ||
   ['generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004'])
-GITHUB_BRANCH = (ENV['GITHUB_BRANCH'] || "master")
-RELEASE_VERSION = (ENV['RELEASE_VERSION'] || "")
-NODE_CPUS = (ENV['NODE_CPUS'] || 2).to_i
-NODE_MEMORY = (ENV['NODE_MEMORY'] || 1024).to_i
+GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master")
+RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "")
+NODE_CPUS = (ENV['E2E_NODE_CPUS'] || 2).to_i
+NODE_MEMORY = (ENV['E2E_NODE_MEMORY'] || 1024).to_i
 # Virtualbox >= 6.1.28 require `/etc/vbox/network.conf` for expanded private networks 
 NETWORK_PREFIX = "10.10.10"
 install_type = ""

tests/e2e/secretsencryption/secretsencryption_test.go

@@ -6,7 +6,6 @@ import (
 	"os"
 	"strings"
 	"testing"
-	"time"
 
 	"github.com/k3s-io/k3s/tests/e2e"
 	. "github.com/onsi/ginkgo/v2"
@@ -16,14 +15,15 @@ import (
 // Valid nodeOS: generic/ubuntu2004, opensuse/Leap-15.3.x86_64, dweomer/microos.amd64
 var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system")
 var serverCount = flag.Int("serverCount", 3, "number of server nodes")
+var hardened = flag.Bool("hardened", false, "true or false")
 
 // Environment Variables Info:
 // E2E_RELEASE_VERSION=v1.23.1+k3s2 or nil for latest commit from master
 
-func Test_E2EClusterValidation(t *testing.T) {
+func Test_E2ESecretsEncryption(t *testing.T) {
 	RegisterFailHandler(Fail)
 	flag.Parse()
-	RunSpecs(t, "Create Cluster Test Suite")
+	RunSpecs(t, "Secrets Encryption Test Suite")
 }
 
 var (
@@ -71,7 +71,7 @@ var _ = Describe("Verify Secrets Encryption Rotation", func() {
 		})
 
 		It("Deploys several secrets", func() {
-			_, err := e2e.DeployWorkload("secrets.yaml", kubeConfigFile, false)
+			_, err := e2e.DeployWorkload("secrets.yaml", kubeConfigFile, *hardened)
 			Expect(err).NotTo(HaveOccurred(), "Secrets not deployed")
 		})
 
@@ -127,16 +127,19 @@ var _ = Describe("Verify Secrets Encryption Rotation", func() {
 					}
 				}
 			}, "420s", "5s").Should(Succeed())
+			_, _ = e2e.ParseNodes(kubeConfigFile, true)
 		})
 
 		It("Verifies encryption prepare stage", func() {
 			cmd := "sudo k3s secrets-encrypt status"
 			for _, nodeName := range serverNodeNames {
-				res, err := e2e.RunCmdOnNode(cmd, nodeName)
-				Expect(err).NotTo(HaveOccurred())
-				Expect(res).Should(ContainSubstring("Encryption Status: Enabled"))
-				Expect(res).Should(ContainSubstring("Current Rotation Stage: prepare"))
-				Expect(res).Should(ContainSubstring("Server Encryption Hashes: All hashes match"))
+				Eventually(func(g Gomega) {
+					res, err := e2e.RunCmdOnNode(cmd, nodeName)
+					g.Expect(err).NotTo(HaveOccurred())
+					g.Expect(res).Should(ContainSubstring("Encryption Status: Enabled"))
+					g.Expect(res).Should(ContainSubstring("Current Rotation Stage: prepare"))
+					g.Expect(res).Should(ContainSubstring("Server Encryption Hashes: All hashes match"))
+				}, "420s", "2s").Should(Succeed())
 			}
 		})
 
@@ -145,31 +148,34 @@ var _ = Describe("Verify Secrets Encryption Rotation", func() {
 			res, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0])
 			Expect(err).NotTo(HaveOccurred(), res)
 			for i, nodeName := range serverNodeNames {
-				cmd := "sudo k3s secrets-encrypt status"
-				res, err := e2e.RunCmdOnNode(cmd, nodeName)
-				Expect(err).NotTo(HaveOccurred(), res)
-				Expect(res).Should(ContainSubstring("Server Encryption Hashes: hash does not match"))
-				if i == 0 {
-					Expect(res).Should(ContainSubstring("Current Rotation Stage: rotate"))
-				} else {
-					Expect(res).Should(ContainSubstring("Current Rotation Stage: prepare"))
-				}
+				Eventually(func(g Gomega) {
+					cmd := "sudo k3s secrets-encrypt status"
+					res, err := e2e.RunCmdOnNode(cmd, nodeName)
+					g.Expect(err).NotTo(HaveOccurred(), res)
+					g.Expect(res).Should(ContainSubstring("Server Encryption Hashes: hash does not match"))
+					if i == 0 {
+						g.Expect(res).Should(ContainSubstring("Current Rotation Stage: rotate"))
+					} else {
+						g.Expect(res).Should(ContainSubstring("Current Rotation Stage: prepare"))
+					}
+				}, "420s", "2s").Should(Succeed())
 			}
 		})
 
 		It("Restarts K3s servers", func() {
 			Expect(e2e.RestartCluster(serverNodeNames)).To(Succeed())
-			time.Sleep(20 * time.Second)
 		})
 
 		It("Verifies encryption rotate stage", func() {
 			cmd := "sudo k3s secrets-encrypt status"
 			for _, nodeName := range serverNodeNames {
-				res, err := e2e.RunCmdOnNode(cmd, nodeName)
-				Expect(err).NotTo(HaveOccurred())
-				Expect(res).Should(ContainSubstring("Encryption Status: Enabled"))
-				Expect(res).Should(ContainSubstring("Current Rotation Stage: rotate"))
-				Expect(res).Should(ContainSubstring("Server Encryption Hashes: All hashes match"))
+				Eventually(func(g Gomega) {
+					res, err := e2e.RunCmdOnNode(cmd, nodeName)
+					g.Expect(err).NotTo(HaveOccurred())
+					g.Expect(res).Should(ContainSubstring("Encryption Status: Enabled"))
+					g.Expect(res).Should(ContainSubstring("Current Rotation Stage: rotate"))
+					g.Expect(res).Should(ContainSubstring("Server Encryption Hashes: All hashes match"))
+				}, "420s", "2s").Should(Succeed())
 			}
 		})
 
@@ -181,7 +187,7 @@ var _ = Describe("Verify Secrets Encryption Rotation", func() {
 			cmd = "sudo k3s secrets-encrypt status"
 			Eventually(func() (string, error) {
 				return e2e.RunCmdOnNode(cmd, serverNodeNames[0])
-			}, "30s", "5s").Should(ContainSubstring("Current Rotation Stage: reencrypt_finished"))
+			}, "180s", "5s").Should(ContainSubstring("Current Rotation Stage: reencrypt_finished"))
 
 			for _, nodeName := range serverNodeNames[1:] {
 				res, err := e2e.RunCmdOnNode(cmd, nodeName)
@@ -193,17 +199,18 @@ var _ = Describe("Verify Secrets Encryption Rotation", func() {
 
 		It("Restarts K3s Servers", func() {
 			Expect(e2e.RestartCluster(serverNodeNames)).To(Succeed())
-			time.Sleep(20 * time.Second)
 		})
 
 		It("Verifies Encryption Reencrypt Stage", func() {
 			cmd := "sudo k3s secrets-encrypt status"
 			for _, nodeName := range serverNodeNames {
-				res, err := e2e.RunCmdOnNode(cmd, nodeName)
-				Expect(err).NotTo(HaveOccurred())
-				Expect(res).Should(ContainSubstring("Encryption Status: Enabled"))
-				Expect(res).Should(ContainSubstring("Current Rotation Stage: reencrypt_finished"))
-				Expect(res).Should(ContainSubstring("Server Encryption Hashes: All hashes match"))
+				Eventually(func(g Gomega) {
+					res, err := e2e.RunCmdOnNode(cmd, nodeName)
+					g.Expect(err).NotTo(HaveOccurred())
+					g.Expect(res).Should(ContainSubstring("Encryption Status: Enabled"))
+					g.Expect(res).Should(ContainSubstring("Current Rotation Stage: reencrypt_finished"))
+					g.Expect(res).Should(ContainSubstring("Server Encryption Hashes: All hashes match"))
+				}, "420s", "2s").Should(Succeed())
 			}
 		})
 	})
@@ -220,28 +227,31 @@ var _ = Describe("Verify Secrets Encryption Rotation", func() {
 		cmd = "sudo k3s secrets-encrypt status"
 		Eventually(func() (string, error) {
 			return e2e.RunCmdOnNode(cmd, serverNodeNames[0])
-		}, "30s", "5s").Should(ContainSubstring("Current Rotation Stage: reencrypt_finished"))
+		}, "180s", "5s").Should(ContainSubstring("Current Rotation Stage: reencrypt_finished"))
 
 		for i, nodeName := range serverNodeNames {
-			res, err := e2e.RunCmdOnNode(cmd, nodeName)
-			Expect(err).NotTo(HaveOccurred(), res)
-			if i == 0 {
-				Expect(res).Should(ContainSubstring("Encryption Status: Disabled"))
-			} else {
-				Expect(res).Should(ContainSubstring("Encryption Status: Enabled"))
-			}
+			Eventually(func(g Gomega) {
+				res, err := e2e.RunCmdOnNode(cmd, nodeName)
+				g.Expect(err).NotTo(HaveOccurred(), res)
+				if i == 0 {
+					g.Expect(res).Should(ContainSubstring("Encryption Status: Disabled"))
+				} else {
+					g.Expect(res).Should(ContainSubstring("Encryption Status: Enabled"))
+				}
+			}, "420s", "2s").Should(Succeed())
 		}
 	})
 
 	It("Restarts K3s servers", func() {
 		Expect(e2e.RestartCluster(serverNodeNames)).To(Succeed())
-		time.Sleep(20 * time.Second)
 	})
 
 	It("Verifies encryption disabled on all nodes", func() {
 		cmd := "sudo k3s secrets-encrypt status"
 		for _, nodeName := range serverNodeNames {
-			Expect(e2e.RunCmdOnNode(cmd, nodeName)).Should(ContainSubstring("Encryption Status: Disabled"))
+			Eventually(func(g Gomega) {
+				g.Expect(e2e.RunCmdOnNode(cmd, nodeName)).Should(ContainSubstring("Encryption Status: Disabled"))
+			}, "420s", "2s").Should(Succeed())
 		}
 	})
 
@@ -257,19 +267,21 @@ var _ = Describe("Verify Secrets Encryption Rotation", func() {
 		cmd = "sudo k3s secrets-encrypt status"
 		Eventually(func() (string, error) {
 			return e2e.RunCmdOnNode(cmd, serverNodeNames[0])
-		}, "30s", "5s").Should(ContainSubstring("Current Rotation Stage: reencrypt_finished"))
+		}, "180s", "5s").Should(ContainSubstring("Current Rotation Stage: reencrypt_finished"))
 	})
 
 	It("Restarts K3s servers", func() {
 		Expect(e2e.RestartCluster(serverNodeNames)).To(Succeed())
-		time.Sleep(20 * time.Second)
 	})
 
 	It("Verifies encryption enabled on all nodes", func() {
 		cmd := "sudo k3s secrets-encrypt status"
 		for _, nodeName := range serverNodeNames {
-			Expect(e2e.RunCmdOnNode(cmd, nodeName)).Should(ContainSubstring("Encryption Status: Enabled"))
+			Eventually(func(g Gomega) {
+				g.Expect(e2e.RunCmdOnNode(cmd, nodeName)).Should(ContainSubstring("Encryption Status: Enabled"))
+			}, "420s", "2s").Should(Succeed())
 		}
+
 	})
 
 })

tests/e2e/splitserver/splitserver_test.go

@@ -18,6 +18,7 @@ var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system")
 var etcdCount = flag.Int("etcdCount", 1, "number of server nodes only deploying etcd")
 var controlPlaneCount = flag.Int("controlPlaneCount", 1, "number of server nodes acting as control plane")
 var agentCount = flag.Int("agentCount", 1, "number of agent nodes")
+var hardened = flag.Bool("hardened", false, "true or false")
 
 // Environment Variables Info:
 // E2E_RELEASE_VERSION=v1.23.1+k3s2 or nil for latest commit from master
@@ -111,7 +112,7 @@ var _ = Describe("Verify Create", func() {
 		})
 
 		It("Verifies ClusterIP Service", func() {
-			_, err := e2e.DeployWorkload("clusterip.yaml", kubeConfigFile, false)
+			_, err := e2e.DeployWorkload("clusterip.yaml", kubeConfigFile, *hardened)
 			Expect(err).NotTo(HaveOccurred(), "Cluster IP manifest not deployed")
 
 			cmd := "kubectl get pods -o=name -l k8s-app=nginx-app-clusterip --field-selector=status.phase=Running --kubeconfig=" + kubeConfigFile
@@ -129,7 +130,7 @@ var _ = Describe("Verify Create", func() {
 		})
 
 		It("Verifies NodePort Service", func() {
-			_, err := e2e.DeployWorkload("nodeport.yaml", kubeConfigFile, false)
+			_, err := e2e.DeployWorkload("nodeport.yaml", kubeConfigFile, *hardened)
 			Expect(err).NotTo(HaveOccurred(), "NodePort manifest not deployed")
 
 			for _, nodeName := range cpNodeNames {
@@ -151,7 +152,7 @@ var _ = Describe("Verify Create", func() {
 		})
 
 		It("Verifies LoadBalancer Service", func() {
-			_, err := e2e.DeployWorkload("loadbalancer.yaml", kubeConfigFile, false)
+			_, err := e2e.DeployWorkload("loadbalancer.yaml", kubeConfigFile, *hardened)
 			Expect(err).NotTo(HaveOccurred(), "Loadbalancer manifest not deployed")
 
 			for _, nodeName := range cpNodeNames {
@@ -174,7 +175,7 @@ var _ = Describe("Verify Create", func() {
 		})
 
 		It("Verifies Ingress", func() {
-			_, err := e2e.DeployWorkload("ingress.yaml", kubeConfigFile, false)
+			_, err := e2e.DeployWorkload("ingress.yaml", kubeConfigFile, *hardened)
 			Expect(err).NotTo(HaveOccurred(), "Ingress manifest not deployed")
 
 			for _, nodeName := range cpNodeNames {
@@ -187,7 +188,7 @@ var _ = Describe("Verify Create", func() {
 		})
 
 		It("Verifies Daemonset", func() {
-			_, err := e2e.DeployWorkload("daemonset.yaml", kubeConfigFile, false)
+			_, err := e2e.DeployWorkload("daemonset.yaml", kubeConfigFile, *hardened)
 			Expect(err).NotTo(HaveOccurred(), "Daemonset manifest not deployed")
 
 			Eventually(func(g Gomega) {
@@ -202,7 +203,7 @@ var _ = Describe("Verify Create", func() {
 		})
 
 		It("Verifies dns access", func() {
-			_, err := e2e.DeployWorkload("dnsutils.yaml", kubeConfigFile, false)
+			_, err := e2e.DeployWorkload("dnsutils.yaml", kubeConfigFile, *hardened)
 			Expect(err).NotTo(HaveOccurred(), "dnsutils manifest not deployed")
 
 			cmd := "kubectl get pods dnsutils --kubeconfig=" + kubeConfigFile

tests/e2e/testutils.go

@@ -38,7 +38,8 @@ func CountOfStringInSlice(str string, pods []Pod) int {
 	return count
 }
 
-func CreateCluster(nodeOS string, serverCount, agentCount int) ([]string, []string, error) {
+// genNodeEnvs generates the node and testing environment variables for vagrant up
+func genNodeEnvs(nodeOS string, serverCount, agentCount int) ([]string, []string, string) {
 	serverNodeNames := make([]string, serverCount)
 	for i := 0; i < serverCount; i++ {
 		serverNodeNames[i] = "server-" + strconv.Itoa(i)
@@ -47,12 +48,22 @@ func CreateCluster(nodeOS string, serverCount, agentCount int) ([]string, []stri
 	for i := 0; i < agentCount; i++ {
 		agentNodeNames[i] = "agent-" + strconv.Itoa(i)
 	}
-	nodeRoles := strings.Join(serverNodeNames, " ") + " " + strings.Join(agentNodeNames, " ")
 
+	nodeRoles := strings.Join(serverNodeNames, " ") + " " + strings.Join(agentNodeNames, " ")
 	nodeRoles = strings.TrimSpace(nodeRoles)
+
 	nodeBoxes := strings.Repeat(nodeOS+" ", serverCount+agentCount)
 	nodeBoxes = strings.TrimSpace(nodeBoxes)
 
+	nodeEnvs := fmt.Sprintf(`E2E_NODE_ROLES="%s" E2E_NODE_BOXES="%s"`, nodeRoles, nodeBoxes)
+
+	return serverNodeNames, agentNodeNames, nodeEnvs
+}
+
+func CreateCluster(nodeOS string, serverCount, agentCount int) ([]string, []string, error) {
+
+	serverNodeNames, agentNodeNames, nodeEnvs := genNodeEnvs(nodeOS, serverCount, agentCount)
+
 	var testOptions string
 	for _, env := range os.Environ() {
 		if strings.HasPrefix(env, "E2E_") {
@@ -60,19 +71,58 @@ func CreateCluster(nodeOS string, serverCount, agentCount int) ([]string, []stri
 		}
 	}
 
-	cmd := fmt.Sprintf(`E2E_NODE_ROLES="%s" E2E_NODE_BOXES="%s" %s vagrant up &> vagrant.log`, nodeRoles, nodeBoxes, testOptions)
+	cmd := fmt.Sprintf(`%s %s vagrant up &> vagrant.log`, nodeEnvs, testOptions)
 	fmt.Println(cmd)
 	if _, err := RunCommand(cmd); err != nil {
-		fmt.Println("Error Creating Cluster", err)
-		return nil, nil, err
+		return nil, nil, fmt.Errorf("failed creating cluster: %s: %v", cmd, err)
+	}
+	return serverNodeNames, agentNodeNames, nil
+}
+
+// CreateLocalCluster creates a cluster using the locally built k3s binary. The vagrant-scp plugin must be installed for
+// this function to work. The binary is deployed as an airgapped install of k3s on the VMs.
+// This is intended only for local testing puposes when writing a new E2E test.
+func CreateLocalCluster(nodeOS string, serverCount, agentCount int) ([]string, []string, error) {
+
+	serverNodeNames, agentNodeNames, nodeEnvs := genNodeEnvs(nodeOS, serverCount, agentCount)
+
+	var testOptions string
+	for _, env := range os.Environ() {
+		if strings.HasPrefix(env, "E2E_") {
+			testOptions += " " + env
+		}
+	}
+	testOptions += " E2E_RELEASE_VERSION=skip"
+
+	cmd := fmt.Sprintf(`%s vagrant up --no-provision &> vagrant.log`, nodeEnvs)
+	if _, err := RunCommand(cmd); err != nil {
+		return nil, nil, fmt.Errorf("failed creating nodes: %s: %v", cmd, err)
+	}
+
+	nodeRoles := append(serverNodeNames, agentNodeNames...)
+
+	for _, node := range nodeRoles {
+		cmd = fmt.Sprintf(`vagrant scp ../../../dist/artifacts/k3s  %s:/tmp/`, node)
+		if _, err := RunCommand(cmd); err != nil {
+			return nil, nil, fmt.Errorf("failed to scp k3s binary to %s: %v", node, err)
+		}
+		if _, err := RunCmdOnNode("sudo mv /tmp/k3s /usr/local/bin/", node); err != nil {
+			return nil, nil, err
+		}
+	}
+
+	cmd = fmt.Sprintf(`%s %s vagrant provision &>> vagrant.log`, nodeEnvs, testOptions)
+	if _, err := RunCommand(cmd); err != nil {
+		return nil, nil, fmt.Errorf("failed creating cluster: %s: %v", cmd, err)
 	}
+
 	return serverNodeNames, agentNodeNames, nil
 }
 
-func DeployWorkload(workload, kubeconfig string, arch bool) (string, error) {
+func DeployWorkload(workload, kubeconfig string, hardened bool) (string, error) {
 	resourceDir := "../amd64_resource_files"
-	if arch {
-		resourceDir = "../arm64_resource_files"
+	if hardened {
+		resourceDir = "../cis_amd64_resource_files"
 	}
 	files, err := ioutil.ReadDir(resourceDir)
 	if err != nil {
@@ -240,7 +290,11 @@ func RestartCluster(nodeNames []string) error {
 // RunCmdOnNode executes a command from within the given node
 func RunCmdOnNode(cmd string, nodename string) (string, error) {
 	runcmd := "vagrant ssh -c \"" + cmd + "\" " + nodename
-	return RunCommand(runcmd)
+	out, err := RunCommand(runcmd)
+	if err != nil {
+		return out, fmt.Errorf("failed to run command %s on node %s: %v", cmd, nodename, err)
+	}
+	return out, nil
 }
 
 // RunCommand executes a command on the host
@@ -252,7 +306,7 @@ func RunCommand(cmd string) (string, error) {
 
 func UpgradeCluster(serverNodeNames []string, agentNodeNames []string) error {
 	for _, nodeName := range serverNodeNames {
-		cmd := "RELEASE_CHANNEL=commit vagrant provision " + nodeName
+		cmd := "E2E_RELEASE_CHANNEL=commit vagrant provision " + nodeName
 		fmt.Println(cmd)
 		if out, err := RunCommand(cmd); err != nil {
 			fmt.Println("Error Upgrading Cluster", out)
@@ -260,7 +314,7 @@ func UpgradeCluster(serverNodeNames []string, agentNodeNames []string) error {
 		}
 	}
 	for _, nodeName := range agentNodeNames {
-		cmd := "RELEASE_CHANNEL=commit vagrant provision " + nodeName
+		cmd := "E2E_RELEASE_CHANNEL=commit vagrant provision " + nodeName
 		if _, err := RunCommand(cmd); err != nil {
 			fmt.Println("Error Upgrading Cluster", err)
 			return err

tests/e2e/upgradecluster/Vagrantfile

 ENV['VAGRANT_NO_PARALLEL'] = 'no'
-NODE_ROLES = (ENV['NODE_ROLES'] ||
+NODE_ROLES = (ENV['E2E_NODE_ROLES'] ||
   ["server-0", "server-1", "server-2", "agent-0", "agent-1"])
-NODE_BOXES = (ENV['NODE_BOXES'] ||
+NODE_BOXES = (ENV['E2E_NODE_BOXES'] ||
   ['generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004'])
-RELEASE_CHANNEL = (ENV['RELEASE_CHANNEL'] || "latest")
-RELEASE_VERSION = (ENV['RELEASE_VERSION'] || "")
-NODE_CPUS = (ENV['NODE_CPUS'] || 2).to_i
-NODE_MEMORY = (ENV['NODE_MEMORY'] || 1024).to_i
+RELEASE_CHANNEL = (ENV['E2E_RELEASE_CHANNEL'] || "latest")
+RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "")
+EXTERNAL_DB = (ENV['E2E_EXTERNAL_DB'] || "etcd")
+NODE_CPUS = (ENV['E2E_NODE_CPUS'] || 2).to_i
+NODE_MEMORY = (ENV['E2E_NODE_MEMORY'] || 1024).to_i
 # Virtualbox >= 6.1.28 require `/etc/vbox/network.conf` for expanded private networks 
 NETWORK_PREFIX = "10.10.10"
 install_type = ""
@@ -15,11 +16,13 @@ def provision(vm, role, role_num, node_num)
   vm.box = NODE_BOXES[node_num]
   vm.hostname = role
   # An expanded netmask is required to allow VM<-->VM communication, virtualbox defaults to /32
-  vm.network "private_network", ip: "#{NETWORK_PREFIX}.#{100+node_num}", netmask: "255.255.255.0"
+  node_ip = "#{NETWORK_PREFIX}.#{100+node_num}"
+  vm.network "private_network", ip: node_ip, netmask: "255.255.255.0"
+
+  scripts_location = Dir.exists?("./scripts") ? "./scripts" : "../scripts"
+  vagrant_defaults = File.exists?("./vagrantdefaults.rb") ? "./vagrantdefaults.rb" : "../vagrantdefaults.rb"
+  load vagrant_defaults
 
-  vagrant_defaults = '../vagrantdefaults.rb'
-  load vagrant_defaults if File.exists?(vagrant_defaults)
-  
   defaultOSConfigure(vm)
     
   if !RELEASE_VERSION.empty?
@@ -33,33 +36,100 @@ def provision(vm, role, role_num, node_num)
     install_type = "INSTALL_K3S_VERSION=$(cat\ /tmp/k3s_version)"
   end
 
+
+
   vm.provision "shell", inline: "ping -c 2 k3s.io"
+  db_type = getDBType(role, role_num, vm)
+
+
 
   if role.include?("server") && role_num == 0
-    vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s|
-      k3s.args = %W[server --cluster-init --node-external-ip=#{NETWORK_PREFIX}.100 --flannel-iface=eth1]
-      k3s.env = %W[K3S_KUBECONFIG_MODE=0644 K3S_TOKEN=vagrant #{install_type}]
+    vm.provision 'k3s-primary-server', type: 'k3s', run: 'once' do |k3s|
+      k3s.args = "server "
+      k3s.config = <<~YAML
+        token: vagrant
+        node-external-ip: #{NETWORK_PREFIX}.100
+        flannel-iface: eth1
+        tls-san: #{NETWORK_PREFIX}.100.nip.io
+        #{db_type}
+      YAML
+      k3s.env = %W[K3S_KUBECONFIG_MODE=0644 #{install_type}]
       k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321
     end
   elsif role.include?("server") && role_num != 0
-    vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s|
-      k3s.args = %W[server --server https://#{NETWORK_PREFIX}.100:6443 --flannel-iface=eth1]
+    vm.provision 'k3s-secondary-server', type: 'k3s', run: 'once' do |k3s|
+      k3s.args = "server"
+      k3s.config = <<~YAML
+        server: "https://#{NETWORK_PREFIX}.100:6443"
+        token: vagrant
+        node-external-ip: #{node_ip}
+        flannel-iface: eth1
+        #{db_type}
+      YAML
       k3s.env = %W[K3S_KUBECONFIG_MODE=0644 K3S_TOKEN=vagrant #{install_type}]
       k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321
     end
   end
-  if role.include?("agent")
-    vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s|
-      k3s.args = %W[agent --server https://#{NETWORK_PREFIX}.100:6443 --flannel-iface=eth1]
-      k3s.env = %W[K3S_KUBECONFIG_MODE=0644 K3S_TOKEN=vagrant #{install_type}]
+
+ if role.include?("agent")
+    vm.provision 'k3s-agent', type: 'k3s', run: 'once' do |k3s|
+      k3s.args = "agent"
+      k3s.config = <<~YAML
+        server: "https://#{NETWORK_PREFIX}.100:6443"
+        token: vagrant
+        node-external-ip: #{node_ip}
+        flannel-iface: eth1
+        #{db_type}
+      YAML
+      k3s.env = %W[K3S_KUBECONFIG_MODE=0644 #{install_type}]
       k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321
     end
   end
   if vm.box.to_s.include?("microos")
     vm.provision 'k3s-reload', type: 'reload', run: 'once'
+    if !EXTERNAL_DB.empty?
+      vm.provision "shell", inline: "docker start #{EXTERNAL_DB}"
+    end
+  end
+end
+
+
+def getDBType(role, role_num, vm)
+
+  if EXTERNAL_DB == "mysql"
+    if role.include?("server") && role_num == 0
+      dockerInstall(vm)
+      vm.provision "Start mysql", inline: "docker run -d -p 3306:3306 --name #{EXTERNAL_DB} -e MYSQL_ROOT_PASSWORD=e2e mysql:5.7"
+      vm.provision "shell", inline: "echo \"Wait for mysql to startup\"; sleep 10"
+      return "datastore-endpoint: 'mysql://root:e2e@tcp(#{NETWORK_PREFIX}.100:3306)/k3s'"
+    elsif role.include?("server") && role_num != 0
+      return "datastore-endpoint: 'mysql://root:e2e@tcp(#{NETWORK_PREFIX}.100:3306)/k3s'"
+    end
+
+  elsif EXTERNAL_DB == "postgres"
+    if role.include?("server") && role_num == 0
+      dockerInstall(vm)
+      vm.provision "Start postgres", inline: "docker run -d -p 5432:5432 --name #{EXTERNAL_DB} -e POSTGRES_PASSWORD=e2e  postgres:14-alpine"
+      vm.provision "shell", inline: "echo \"Wait for postgres to startup\"; sleep 10"
+      return "datastore-endpoint: 'postgres://postgres:e2e@#{NETWORK_PREFIX}.100:5432/k3s?sslmode=disable'"
+    elsif role.include?("server") && role_num != 0
+      return "datastore-endpoint: 'postgres://postgres:e2e@#{NETWORK_PREFIX}.100:5432/k3s?sslmode=disable'"
+    end
+
+  elsif ( EXTERNAL_DB == "" || EXTERNAL_DB == "etcd" )
+    if role.include?("server") && role_num == 0
+      return "cluster-init: true"
+    end
+  elsif ( EXTERNAL_DB == "none" )
+    # Use internal sqlite
+  else
+    puts "Unknown EXTERNAL_DB: " + EXTERNAL_DB
+    abort
   end
+  return ""
 end
 
+
 Vagrant.configure("2") do |config|
   config.vagrant.plugins = ["vagrant-k3s", "vagrant-reload"]
   # Default provider is libvirt, virtualbox is only provided as a backup
@@ -71,7 +141,7 @@ Vagrant.configure("2") do |config|
     v.cpus = NODE_CPUS
     v.memory = NODE_MEMORY
   end
-  
+
   if NODE_ROLES.kind_of?(String)
     NODE_ROLES = NODE_ROLES.split(" ", -1)
   end
@@ -79,7 +149,7 @@ Vagrant.configure("2") do |config|
     NODE_BOXES = NODE_BOXES.split(" ", -1)
   end
 
-  # Must iterate on the index, vagrant does not understand iterating 
+  # Must iterate on the index, vagrant does not understand iterating
   # over the node roles themselves
   NODE_ROLES.length.times do |i|
     name = NODE_ROLES[i]

tests/e2e/upgradecluster/upgradecluster_test.go

@@ -12,10 +12,13 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-// Valid nodeOS: generic/ubuntu2004, opensuse/Leap-15.3.x86_64, dweomer/microos.amd64
+// Valid nodeOS:
+// generic/ubuntu2004, generic/centos7, generic/rocky8
+// opensuse/Leap-15.3.x86_64, dweomer/microos.amd64
 var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system")
 var serverCount = flag.Int("serverCount", 3, "number of server nodes")
 var agentCount = flag.Int("agentCount", 2, "number of agent nodes")
+var hardened = flag.Bool("hardened", false, "true or false")
 
 // Environment Variables Info:
 // E2E_RELEASE_VERSION=v1.23.3+k3s1
@@ -75,7 +78,7 @@ var _ = Describe("Verify Upgrade", func() {
 		})
 
 		It("Verifies ClusterIP Service", func() {
-			_, err := e2e.DeployWorkload("clusterip.yaml", kubeConfigFile, false)
+			_, err := e2e.DeployWorkload("clusterip.yaml", kubeConfigFile, *hardened)
 
 			Expect(err).NotTo(HaveOccurred(), "Cluster IP manifest not deployed")
 
@@ -94,7 +97,7 @@ var _ = Describe("Verify Upgrade", func() {
 		})
 
 		It("Verifies NodePort Service", func() {
-			_, err := e2e.DeployWorkload("nodeport.yaml", kubeConfigFile, false)
+			_, err := e2e.DeployWorkload("nodeport.yaml", kubeConfigFile, *hardened)
 			Expect(err).NotTo(HaveOccurred(), "NodePort manifest not deployed")
 
 			for _, nodeName := range serverNodeNames {
@@ -117,7 +120,7 @@ var _ = Describe("Verify Upgrade", func() {
 		})
 
 		It("Verifies LoadBalancer Service", func() {
-			_, err := e2e.DeployWorkload("loadbalancer.yaml", kubeConfigFile, false)
+			_, err := e2e.DeployWorkload("loadbalancer.yaml", kubeConfigFile, *hardened)
 			Expect(err).NotTo(HaveOccurred(), "Loadbalancer manifest not deployed")
 			for _, nodeName := range serverNodeNames {
 				ip, _ := e2e.FetchNodeExternalIP(nodeName)
@@ -138,7 +141,7 @@ var _ = Describe("Verify Upgrade", func() {
 		})
 
 		It("Verifies Ingress", func() {
-			_, err := e2e.DeployWorkload("ingress.yaml", kubeConfigFile, false)
+			_, err := e2e.DeployWorkload("ingress.yaml", kubeConfigFile, *hardened)
 			Expect(err).NotTo(HaveOccurred(), "Ingress manifest not deployed")
 
 			for _, nodeName := range serverNodeNames {
@@ -151,13 +154,13 @@ var _ = Describe("Verify Upgrade", func() {
 		})
 
 		It("Verifies Daemonset", func() {
-			_, err := e2e.DeployWorkload("daemonset.yaml", kubeConfigFile, false)
+			_, err := e2e.DeployWorkload("daemonset.yaml", kubeConfigFile, *hardened)
 			Expect(err).NotTo(HaveOccurred(), "Daemonset manifest not deployed")
 
 			nodes, _ := e2e.ParseNodes(kubeConfigFile, false) //nodes :=
-			pods, _ := e2e.ParsePods(kubeConfigFile, false)
 
 			Eventually(func(g Gomega) {
+				pods, _ := e2e.ParsePods(kubeConfigFile, false)
 				count := e2e.CountOfStringInSlice("test-daemonset", pods)
 				fmt.Println("POD COUNT")
 				fmt.Println(count)
@@ -168,7 +171,7 @@ var _ = Describe("Verify Upgrade", func() {
 		})
 
 		It("Verifies dns access", func() {
-			_, err := e2e.DeployWorkload("dnsutils.yaml", kubeConfigFile, false)
+			_, err := e2e.DeployWorkload("dnsutils.yaml", kubeConfigFile, *hardened)
 			Expect(err).NotTo(HaveOccurred(), "dnsutils manifest not deployed")
 
 			Eventually(func() (string, error) {
@@ -183,7 +186,7 @@ var _ = Describe("Verify Upgrade", func() {
 		})
 
 		It("Verifies Local Path Provisioner storage ", func() {
-			_, err := e2e.DeployWorkload("local-path-provisioner.yaml", kubeConfigFile, false)
+			_, err := e2e.DeployWorkload("local-path-provisioner.yaml", kubeConfigFile, *hardened)
 			Expect(err).NotTo(HaveOccurred(), "local-path-provisioner manifest not deployed")
 			Eventually(func(g Gomega) {
 				cmd := "kubectl get pvc local-path-pvc --kubeconfig=" + kubeConfigFile
@@ -214,7 +217,7 @@ var _ = Describe("Verify Upgrade", func() {
 			Expect(err).NotTo(HaveOccurred())
 			fmt.Println(res)
 
-			_, err = e2e.DeployWorkload("local-path-provisioner.yaml", kubeConfigFile, false)
+			_, err = e2e.DeployWorkload("local-path-provisioner.yaml", kubeConfigFile, *hardened)
 			Expect(err).NotTo(HaveOccurred(), "local-path-provisioner manifest not deployed")
 
 			Eventually(func() (string, error) {
@@ -233,7 +236,7 @@ var _ = Describe("Verify Upgrade", func() {
 
 			// Check data after re-creation
 			Eventually(func() (string, error) {
-				cmd = "kubectl exec volume-test cat /data/test --kubeconfig=" + kubeConfigFile
+				cmd := "kubectl exec volume-test --kubeconfig=" + kubeConfigFile + " -- cat /data/test"
 				return e2e.RunCommand(cmd)
 			}, "180s", "2s").Should(ContainSubstring("local-path-test"))
 		})
@@ -343,9 +346,9 @@ var _ = Describe("Verify Upgrade", func() {
 
 		It("After upgrade verifies Daemonset", func() {
 			nodes, _ := e2e.ParseNodes(kubeConfigFile, false) //nodes :=
-			pods, _ := e2e.ParsePods(kubeConfigFile, false)
 
 			Eventually(func(g Gomega) {
+				pods, _ := e2e.ParsePods(kubeConfigFile, false)
 				count := e2e.CountOfStringInSlice("test-daemonset", pods)
 				fmt.Println("POD COUNT")
 				fmt.Println(count)
@@ -363,7 +366,7 @@ var _ = Describe("Verify Upgrade", func() {
 
 		It("After upgrade verify Local Path Provisioner storage ", func() {
 			Eventually(func() (string, error) {
-				cmd := "kubectl exec volume-test cat /data/test --kubeconfig=" + kubeConfigFile
+				cmd := "kubectl exec volume-test --kubeconfig=" + kubeConfigFile + " -- cat /data/test"
 				return e2e.RunCommand(cmd)
 			}, "180s", "2s").Should(ContainSubstring("local-path-test"))
 		})

tests/e2e/vagrantdefaults.rb

@@ -5,6 +5,13 @@ def defaultOSConfigure(vm)
     vm.provision "Install jq", type: "shell", inline: "apt install -y jq"
   elsif box.include?("Leap") || box.include?("Tumbleweed")
     vm.provision "Install jq", type: "shell", inline: "zypper install -y jq"
+    vm.provision "Install apparmor-parser", type: "shell", inline: "zypper install -y apparmor-parser"
+  elsif box.include?("rocky8") || box.include?("rocky9")
+    vm.provision "Install jq", type: "shell", inline: "dnf install -y jq"
+    vm.provision "Disable firewall", type: "shell", inline: "systemctl stop firewalld"
+  elsif box.include?("centos7")
+    vm.provision "Install jq", type: "shell", inline: "yum install -y jq"
+    vm.provision "Disable firewall", type: "shell", inline: "systemctl stop firewalld"
   elsif box.include?("alpine")
     vm.provision "Install tools", type: "shell", inline: "apk add jq coreutils"
   elsif box.include?("microos")
@@ -24,4 +31,24 @@ def getInstallType(vm, release_version, branch)
     vm.provision "Get latest commit", type: "shell", path: "../scripts/latest_commit.sh", args: [branch, "/tmp/k3s_commits"]
     return "INSTALL_K3S_COMMIT=$(head\ -n\ 1\ /tmp/k3s_commits)"
   end
-end
+end
\ No newline at end of file
+
+def dockerInstall(vm)
+  vm.provider "libvirt" do |v|
+    v.memory = NODE_MEMORY + 1024
+  end
+  vm.provider "virtualbox" do |v|
+    v.memory = NODE_MEMORY + 1024
+  end
+  if vm.box.to_s.include?("ubuntu")
+    vm.provision "shell", inline: "apt update; apt install -y docker.io"
+  end
+  if vm.box.to_s.include?("Leap")
+    vm.provision "shell", inline: "zypper install -y docker apparmor-parser"
+  end
+  if vm.box.to_s.include?("microos")
+    vm.provision "shell", inline: "transactional-update pkg install -y docker apparmor-parser"
+    vm.provision 'docker-reload', type: 'reload', run: 'once'
+    vm.provision "shell", inline: "systemctl enable --now docker"
+  end
+end

tests/e2e/validatecluster/Vagrantfile

@@ -111,7 +111,7 @@ def getDBType(role, role_num, vm)
   elsif EXTERNAL_DB == "postgres"
     if role.include?("server") && role_num == 0
       dockerInstall(vm)
-      vm.provision "Start postgres", inline: "docker run -d -p 5432:5432 --name #{EXTERNAL_DB} -e POSTGRES_PASSWORD=e2e  postgres:14-alpine"
+      vm.provision "Start postgres", type: "shell", inline: "docker run -d -p 5432:5432 --name postgres -e POSTGRES_PASSWORD=e2e  postgres:14-alpine"
       vm.provision "shell", inline: "echo \"Wait for postgres to startup\"; sleep 10"
       return "datastore-endpoint: 'postgres://postgres:e2e@#{NETWORK_PREFIX}.100:5432/k3s?sslmode=disable'"
     elsif role.include?("server") && role_num != 0
@@ -123,7 +123,7 @@ def getDBType(role, role_num, vm)
       return "cluster-init: true"
     end
   elsif ( EXTERNAL_DB == "none" )
-    # Use internal sqlite
+    # Use internal sqlite, only valid for single node clusters
   else
     puts "Unknown EXTERNAL_DB: " + EXTERNAL_DB
     abort

tests/e2e/validatecluster/validatecluster_test.go

@@ -12,14 +12,17 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-// Valid nodeOS: generic/ubuntu2004, opensuse/Leap-15.3.x86_64, dweomer/microos.amd64
+// Valid nodeOS:
+// generic/ubuntu2004, generic/centos7, generic/rocky8,
+// opensuse/Leap-15.3.x86_64, dweomer/microos.amd64
 var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system")
 var serverCount = flag.Int("serverCount", 3, "number of server nodes")
 var agentCount = flag.Int("agentCount", 2, "number of agent nodes")
+var hardened = flag.Bool("hardened", false, "true or false")
 
 // Environment Variables Info:
-// E2E_EXTERNAL_DB: mysql or postgres, nil for embedded etcd
-// E2E_RELEASE_VERSION=v1.23.1+k3s2 or nil for latest commit from master
+// E2E_EXTERNAL_DB: mysql, postgres, etcd (default: etcd)
+// E2E_RELEASE_VERSION=v1.23.1+k3s2 (default: latest commit from master)
 
 func Test_E2EClusterValidation(t *testing.T) {
 	RegisterFailHandler(Fail)
@@ -74,7 +77,7 @@ var _ = Describe("Verify Create", func() {
 		})
 
 		It("Verifies ClusterIP Service", func() {
-			_, err := e2e.DeployWorkload("clusterip.yaml", kubeConfigFile, false)
+			_, err := e2e.DeployWorkload("clusterip.yaml", kubeConfigFile, *hardened)
 			Expect(err).NotTo(HaveOccurred(), "Cluster IP manifest not deployed")
 
 			Eventually(func(g Gomega) {
@@ -91,14 +94,13 @@ var _ = Describe("Verify Create", func() {
 				Eventually(func(g Gomega) {
 					res, err := e2e.RunCmdOnNode(cmd, nodeName)
 					g.Expect(err).NotTo(HaveOccurred())
-					fmt.Println(res)
 					Expect(res).Should(ContainSubstring("test-clusterip"))
 				}, "120s", "10s").Should(Succeed())
 			}
 		})
 
 		It("Verifies NodePort Service", func() {
-			_, err := e2e.DeployWorkload("nodeport.yaml", kubeConfigFile, false)
+			_, err := e2e.DeployWorkload("nodeport.yaml", kubeConfigFile, *hardened)
 			Expect(err).NotTo(HaveOccurred(), "NodePort manifest not deployed")
 
 			for _, nodeName := range serverNodeNames {
@@ -118,15 +120,14 @@ var _ = Describe("Verify Create", func() {
 				fmt.Println(cmd)
 				Eventually(func(g Gomega) {
 					res, err := e2e.RunCommand(cmd)
-					Expect(err).NotTo(HaveOccurred())
-					fmt.Println(res)
+					g.Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd+" result: "+res)
 					g.Expect(res).Should(ContainSubstring("test-nodeport"))
 				}, "240s", "5s").Should(Succeed())
 			}
 		})
 
 		It("Verifies LoadBalancer Service", func() {
-			_, err := e2e.DeployWorkload("loadbalancer.yaml", kubeConfigFile, false)
+			_, err := e2e.DeployWorkload("loadbalancer.yaml", kubeConfigFile, *hardened)
 			Expect(err).NotTo(HaveOccurred(), "Loadbalancer manifest not deployed")
 
 			for _, nodeName := range serverNodeNames {
@@ -139,23 +140,21 @@ var _ = Describe("Verify Create", func() {
 				Eventually(func(g Gomega) {
 					cmd := "kubectl get pods -o=name -l k8s-app=nginx-app-loadbalancer --field-selector=status.phase=Running --kubeconfig=" + kubeConfigFile
 					res, err := e2e.RunCommand(cmd)
-					Expect(err).NotTo(HaveOccurred())
+					g.Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd+" result: "+res)
 					g.Expect(res).Should(ContainSubstring("test-loadbalancer"))
 				}, "240s", "5s").Should(Succeed())
 
 				Eventually(func(g Gomega) {
 					cmd = "curl -L --insecure http://" + ip + ":" + port + "/name.html"
-					fmt.Println(cmd)
 					res, err := e2e.RunCommand(cmd)
-					Expect(err).NotTo(HaveOccurred())
-					fmt.Println(res)
+					g.Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd+" result: "+res)
 					g.Expect(res).Should(ContainSubstring("test-loadbalancer"))
 				}, "240s", "5s").Should(Succeed())
 			}
 		})
 
 		It("Verifies Ingress", func() {
-			_, err := e2e.DeployWorkload("ingress.yaml", kubeConfigFile, false)
+			_, err := e2e.DeployWorkload("ingress.yaml", kubeConfigFile, *hardened)
 			Expect(err).NotTo(HaveOccurred(), "Ingress manifest not deployed")
 
 			for _, nodeName := range serverNodeNames {
@@ -165,21 +164,20 @@ var _ = Describe("Verify Create", func() {
 
 				Eventually(func(g Gomega) {
 					res, err := e2e.RunCommand(cmd)
-					g.Expect(err).NotTo(HaveOccurred())
-					fmt.Println(res)
+					g.Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd+" result: "+res)
 					g.Expect(res).Should(ContainSubstring("test-ingress"))
 				}, "240s", "5s").Should(Succeed())
 			}
 		})
 
 		It("Verifies Daemonset", func() {
-			_, err := e2e.DeployWorkload("daemonset.yaml", kubeConfigFile, false)
+			_, err := e2e.DeployWorkload("daemonset.yaml", kubeConfigFile, *hardened)
 			Expect(err).NotTo(HaveOccurred(), "Daemonset manifest not deployed")
 
 			nodes, _ := e2e.ParseNodes(kubeConfigFile, false)
-			pods, _ := e2e.ParsePods(kubeConfigFile, false)
 
 			Eventually(func(g Gomega) {
+				pods, _ := e2e.ParsePods(kubeConfigFile, false)
 				count := e2e.CountOfStringInSlice("test-daemonset", pods)
 				fmt.Println("POD COUNT")
 				fmt.Println(count)
@@ -190,34 +188,32 @@ var _ = Describe("Verify Create", func() {
 		})
 
 		It("Verifies dns access", func() {
-			_, err := e2e.DeployWorkload("dnsutils.yaml", kubeConfigFile, false)
+			_, err := e2e.DeployWorkload("dnsutils.yaml", kubeConfigFile, *hardened)
 			Expect(err).NotTo(HaveOccurred(), "dnsutils manifest not deployed")
 
 			Eventually(func(g Gomega) {
 				cmd := "kubectl get pods dnsutils --kubeconfig=" + kubeConfigFile
-				res, _ := e2e.RunCommand(cmd)
-				fmt.Println(res)
+				res, err := e2e.RunCommand(cmd)
+				g.Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd+" result: "+res)
 				g.Expect(res).Should(ContainSubstring("dnsutils"))
 			}, "420s", "2s").Should(Succeed())
 
 			Eventually(func(g Gomega) {
 				cmd := "kubectl --kubeconfig=" + kubeConfigFile + " exec -i -t dnsutils -- nslookup kubernetes.default"
-				fmt.Println(cmd)
-				res, _ := e2e.RunCommand(cmd)
-				fmt.Println(res)
+				res, err := e2e.RunCommand(cmd)
+				g.Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd+" result: "+res)
 				g.Expect(res).Should(ContainSubstring("kubernetes.default.svc.cluster.local"))
 			}, "420s", "2s").Should(Succeed())
 		})
 
 		It("Verifies Local Path Provisioner storage ", func() {
-			_, err := e2e.DeployWorkload("local-path-provisioner.yaml", kubeConfigFile, false)
+			_, err := e2e.DeployWorkload("local-path-provisioner.yaml", kubeConfigFile, *hardened)
 			Expect(err).NotTo(HaveOccurred(), "local-path-provisioner manifest not deployed")
 
 			Eventually(func(g Gomega) {
 				cmd := "kubectl get pvc local-path-pvc --kubeconfig=" + kubeConfigFile
 				res, err := e2e.RunCommand(cmd)
-				g.Expect(err).NotTo(HaveOccurred())
-				fmt.Println(res)
+				g.Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd+" result: "+res)
 				g.Expect(res).Should(ContainSubstring("local-path-pvc"))
 				g.Expect(res).Should(ContainSubstring("Bound"))
 			}, "420s", "2s").Should(Succeed())
@@ -225,8 +221,7 @@ var _ = Describe("Verify Create", func() {
 			Eventually(func(g Gomega) {
 				cmd := "kubectl get pod volume-test --kubeconfig=" + kubeConfigFile
 				res, err := e2e.RunCommand(cmd)
-				Expect(err).NotTo(HaveOccurred())
-				fmt.Println(res)
+				g.Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd+" result: "+res)
 				g.Expect(res).Should(ContainSubstring("volume-test"))
 				g.Expect(res).Should(ContainSubstring("Running"))
 			}, "420s", "2s").Should(Succeed())
@@ -234,36 +229,33 @@ var _ = Describe("Verify Create", func() {
 			cmd := "kubectl --kubeconfig=" + kubeConfigFile + " exec volume-test -- sh -c 'echo local-path-test > /data/test'"
 			_, err = e2e.RunCommand(cmd)
 			Expect(err).NotTo(HaveOccurred())
-			fmt.Println("Data stored in pvc: local-path-test")
 
 			cmd = "kubectl delete pod volume-test --kubeconfig=" + kubeConfigFile
 			res, err := e2e.RunCommand(cmd)
-			Expect(err).NotTo(HaveOccurred())
-			fmt.Println(res)
+			Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd+" result: "+res)
 
-			_, err = e2e.DeployWorkload("local-path-provisioner.yaml", kubeConfigFile, false)
+			_, err = e2e.DeployWorkload("local-path-provisioner.yaml", kubeConfigFile, *hardened)
 			Expect(err).NotTo(HaveOccurred(), "local-path-provisioner manifest not deployed")
 
 			Eventually(func(g Gomega) {
 				cmd := "kubectl get pods -o=name -l app=local-path-provisioner --field-selector=status.phase=Running -n kube-system --kubeconfig=" + kubeConfigFile
 				res, _ := e2e.RunCommand(cmd)
-				fmt.Println(res)
 				g.Expect(res).Should(ContainSubstring("local-path-provisioner"))
 			}, "420s", "2s").Should(Succeed())
 
 			Eventually(func(g Gomega) {
 				cmd := "kubectl get pod volume-test --kubeconfig=" + kubeConfigFile
 				res, err := e2e.RunCommand(cmd)
-				g.Expect(err).NotTo(HaveOccurred())
-				fmt.Println(res)
+				g.Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd+" result: "+res)
+
 				g.Expect(res).Should(ContainSubstring("volume-test"))
 				g.Expect(res).Should(ContainSubstring("Running"))
 			}, "420s", "2s").Should(Succeed())
 
 			Eventually(func(g Gomega) {
-				cmd = "kubectl exec volume-test cat /data/test --kubeconfig=" + kubeConfigFile
+				cmd := "kubectl exec volume-test --kubeconfig=" + kubeConfigFile + " -- cat /data/test"
 				res, err = e2e.RunCommand(cmd)
-				g.Expect(err).NotTo(HaveOccurred())
+				g.Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd+" result: "+res)
 				fmt.Println("Data after re-creation", res)
 				g.Expect(res).Should(ContainSubstring("local-path-test"))
 			}, "180s", "2s").Should(Succeed())

tests/install/centos-7/Vagrantfile

@@ -21,10 +21,10 @@ Vagrant.configure("2") do |config|
       k3s.env = ENV.select{|k,v| k.start_with?('K3S_') || k.start_with?('INSTALL_K3S_')}.merge({
         :INSTALL_K3S_NAME => 'server',
       })
-      k3s.config = {
-        :selinux => true,
-        :token => 'vagrant',
-      }
+      k3s.config = <<~YAML
+        selinux: true
+        token: 'vagrant'
+      YAML
       k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321
     end
     test.vm.provision "k3s-wait-for-node", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh|

tests/install/fedora-coreos/Vagrantfile

@@ -21,10 +21,10 @@ Vagrant.configure("2") do |config|
       k3s.env = ENV.select{|k,v| k.start_with?('K3S_') || k.start_with?('INSTALL_K3S_')}.merge({
         :INSTALL_K3S_NAME => 'server',
       })
-      k3s.config = {
-        :selinux => true,
-        :token => 'vagrant',
-      }
+      k3s.config = <<~YAML
+        selinux: true
+        token: 'vagrant'
+      YAML
       k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321
     end
     test.vm.provision "k3s-wait-for-node", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh|

tests/install/opensuse-leap/Vagrantfile

@@ -24,9 +24,9 @@ Vagrant.configure("2") do |config|
         :INSTALL_K3S_NAME => 'server',
         :INSTALL_K3S_SKIP_SELINUX_RPM => 'true',
       })
-      k3s.config = {
-        :token => 'vagrant',
-      }
+      k3s.config = <<~YAML
+        token: 'vagrant'
+      YAML
       k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321
     end
     test.vm.provision "k3s-wait-for-node", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh|
@@ -44,7 +44,7 @@ Vagrant.configure("2") do |config|
       sh.inline = <<~SHELL
         #!/usr/bin/env bash
         set -eu -o pipefail
-        time timeout 300 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/coredns 2>/dev/null); do sleep 5; done'
+        timeout 300 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/coredns); do sleep 5; done'
       SHELL
     end
     test.vm.provision "k3s-wait-for-local-storage", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh|

tests/install/opensuse-microos/Vagrantfile

@@ -22,10 +22,10 @@ Vagrant.configure("2") do |config|
       k3s.env = ENV.select{|k,v| k.start_with?('K3S_') || k.start_with?('INSTALL_K3S_')}.merge({
         :INSTALL_K3S_NAME => 'server',
       })
-      k3s.config = {
-        :selinux => true,
-        :token => 'vagrant',
-      }
+      k3s.config = <<~YAML
+        selinux: true
+        token: 'vagrant'
+      YAML
       k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321
     end
     # install.sh will move the snapshot forward when installing k3s-selinux policy, so, we reboot to pick that up

tests/install/rocky-8/Vagrantfile

@@ -22,10 +22,10 @@ Vagrant.configure("2") do |config|
       k3s.env = ENV.select{|k,v| k.start_with?('K3S_') || k.start_with?('INSTALL_K3S_')}.merge({
         :INSTALL_K3S_NAME => 'server',
       })
-      k3s.config = {
-        :selinux => true,
-        :token => 'vagrant',
-      }
+      k3s.config = <<~YAML
+        selinux: true
+        token: 'vagrant'
+      YAML
       k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321
     end
     test.vm.provision "k3s-wait-for-node", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh|

tests/install/ubuntu-focal/Vagrantfile

@@ -22,9 +22,9 @@ Vagrant.configure("2") do |config|
         :INSTALL_K3S_NAME => 'server',
         :INSTALL_K3S_SKIP_SELINUX_RPM => 'true',
       })
-      k3s.config = {
-        :token => 'vagrant',
-      }
+      k3s.config = <<~YAML
+        token: 'vagrant'
+      YAML
       k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321
     end
     test.vm.provision "k3s-wait-for-node", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh|

tests/integration/certrotation/certrotation_int_test.go

@@ -26,7 +26,7 @@ var _ = BeforeSuite(func() {
 	}
 })
 
-var _ = Describe("certificate rotation", func() {
+var _ = Describe("certificate rotation", Ordered, func() {
 	BeforeEach(func() {
 		if testutil.IsExistingServer() && !testutil.ServerArgsPresent(serverArgs) {
 			Skip("Test needs k3s server with: " + strings.Join(serverArgs, " "))
@@ -79,8 +79,11 @@ var _ = Describe("certificate rotation", func() {
 
 var _ = AfterSuite(func() {
 	if !testutil.IsExistingServer() {
+		if CurrentSpecReport().Failed() {
+			testutil.K3sDumpLog(server)
+		}
 		Expect(testutil.K3sKillServer(server)).To(Succeed())
-		Expect(testutil.K3sCleanup(testLock, "")).To(Succeed())
+		Expect(testutil.K3sCleanup(-1, "")).To(Succeed())
 		Expect(testutil.K3sKillServer(server2)).To(Succeed())
 		Expect(testutil.K3sCleanup(testLock, tmpdDataDir)).To(Succeed())
 	}

tests/integration/custometcdargs/custometcdargs_int_test.go

@@ -27,7 +27,7 @@ var _ = BeforeSuite(func() {
 	}
 })
 
-var _ = Describe("custom etcd args", func() {
+var _ = Describe("custom etcd args", Ordered, func() {
 	BeforeEach(func() {
 		if testutil.IsExistingServer() && !testutil.ServerArgsPresent(customEtcdArgsServerArgs) {
 			Skip("Test needs k3s server with: " + strings.Join(customEtcdArgsServerArgs, " "))
@@ -54,6 +54,9 @@ var _ = Describe("custom etcd args", func() {
 
 var _ = AfterSuite(func() {
 	if !testutil.IsExistingServer() {
+		if CurrentSpecReport().Failed() {
+			testutil.K3sDumpLog(customEtcdArgsServer)
+		}
 		Expect(testutil.K3sKillServer(customEtcdArgsServer)).To(Succeed())
 		Expect(testutil.K3sCleanup(testLock, "")).To(Succeed())
 	}

tests/integration/dualstack/dualstack_int_test.go

@@ -29,7 +29,7 @@ var _ = BeforeSuite(func() {
 	}
 })
 
-var _ = Describe("dual stack", func() {
+var _ = Describe("dual stack", Ordered, func() {
 	BeforeEach(func() {
 		if testutil.IsExistingServer() && !testutil.ServerArgsPresent(dualStackServerArgs) {
 			Skip("Test needs k3s server with: " + strings.Join(dualStackServerArgs, " "))
@@ -55,6 +55,9 @@ var _ = Describe("dual stack", func() {
 
 var _ = AfterSuite(func() {
 	if !testutil.IsExistingServer() && os.Getenv("CI") != "true" {
+		if CurrentSpecReport().Failed() {
+			testutil.K3sDumpLog(dualStackServer)
+		}
 		Expect(testutil.K3sKillServer(dualStackServer)).To(Succeed())
 		Expect(testutil.K3sCleanup(testLock, "")).To(Succeed())
 	}

tests/integration/etcdrestore/etcd_restore_int_test.go

@@ -24,7 +24,7 @@ var _ = BeforeSuite(func() {
 	}
 })
 
-var _ = Describe("etcd snapshot restore", func() {
+var _ = Describe("etcd snapshot restore", Ordered, func() {
 	BeforeEach(func() {
 		if testutil.IsExistingServer() && !testutil.ServerArgsPresent(restoreServerArgs) {
 			Skip("Test needs k3s server with: " + strings.Join(restoreServerArgs, " "))
@@ -108,6 +108,9 @@ var _ = Describe("etcd snapshot restore", func() {
 
 var _ = AfterSuite(func() {
 	if !testutil.IsExistingServer() {
+		if CurrentSpecReport().Failed() {
+			testutil.K3sDumpLog(server1)
+		}
 		Expect(testutil.K3sKillServer(server1)).To(Succeed())
 		Expect(testutil.K3sKillServer(server2)).To(Succeed())
 		Expect(testutil.K3sCleanup(testLock, tmpdDataDir)).To(Succeed())

tests/integration/etcdsnapshot/etcdsnapshot_int_test.go

@@ -25,7 +25,7 @@ var _ = BeforeSuite(func() {
 	}
 })
 
-var _ = Describe("etcd snapshots", func() {
+var _ = Describe("etcd snapshots", Ordered, func() {
 	BeforeEach(func() {
 		if testutil.IsExistingServer() && !testutil.ServerArgsPresent(serverArgs) {
 			Skip("Test needs k3s server with: " + strings.Join(serverArgs, " "))
@@ -115,6 +115,9 @@ var _ = Describe("etcd snapshots", func() {
 
 var _ = AfterSuite(func() {
 	if !testutil.IsExistingServer() {
+		if CurrentSpecReport().Failed() {
+			testutil.K3sDumpLog(server)
+		}
 		Expect(testutil.K3sKillServer(server)).To(Succeed())
 		Expect(testutil.K3sCleanup(testLock, "")).To(Succeed())
 	}

tests/integration/integration.go

@@ -6,6 +6,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"os/exec"
 	"os/user"
@@ -16,6 +17,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/vishvananda/netlink"
+	"golang.org/x/sys/unix"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
@@ -28,8 +30,8 @@ var existingServer = "False"
 const lockFile = "/tmp/k3s-test.lock"
 
 type K3sServer struct {
-	cmd     *exec.Cmd
-	scanner *bufio.Scanner
+	cmd *exec.Cmd
+	log *os.File
 }
 
 func findK3sExecutable() string {
@@ -160,12 +162,12 @@ func CheckDeployments(deployments []string) error {
 	return nil
 }
 
-func ParsePods() ([]corev1.Pod, error) {
+func ParsePods(opts metav1.ListOptions) ([]corev1.Pod, error) {
 	clientSet, err := k8sClient()
 	if err != nil {
 		return nil, err
 	}
-	pods, err := clientSet.CoreV1().Pods("").List(context.Background(), metav1.ListOptions{})
+	pods, err := clientSet.CoreV1().Pods("").List(context.Background(), opts)
 	if err != nil {
 		return nil, err
 	}
@@ -217,14 +219,20 @@ func K3sStartServer(inputArgs ...string) (*K3sServer, error) {
 	cmd := exec.Command(k3sBin, k3sCmd...)
 	// Give the server a new group id so we can kill it and its children later
 	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
-	cmdOut, _ := cmd.StderrPipe()
-	cmd.Stderr = os.Stderr
-	err := cmd.Start()
-	return &K3sServer{cmd, bufio.NewScanner(cmdOut)}, err
+	// Pipe output to a file for debugging later
+	f, err := os.Create("./k3log.txt")
+	if err != nil {
+		return nil, err
+	}
+	cmd.Stderr = f
+	err = cmd.Start()
+	return &K3sServer{cmd, f}, err
 }
 
 // K3sKillServer terminates the running K3s server and its children
 func K3sKillServer(server *K3sServer) error {
+	server.log.Close()
+	os.Remove(server.log.Name())
 	pgid, err := syscall.Getpgid(server.cmd.Process.Pid)
 	if err != nil {
 		if errors.Is(err, syscall.ESRCH) {
@@ -242,6 +250,10 @@ func K3sKillServer(server *K3sServer) error {
 	if _, err = server.cmd.Process.Wait(); err != nil {
 		return errors.Wrap(err, "failed to wait for k3s process exit")
 	}
+	//Unmount all the associated filesystems
+	unmountFolder("/run/k3s")
+	unmountFolder("/run/netns/cni-")
+
 	return nil
 }
 
@@ -278,6 +290,21 @@ func K3sCleanup(k3sTestLock int, dataDir string) error {
 	return nil
 }
 
+func K3sDumpLog(server *K3sServer) error {
+	server.log.Close()
+	log, err := os.Open(server.log.Name())
+	if err != nil {
+		return err
+	}
+	defer log.Close()
+	b, err := ioutil.ReadAll(log)
+	if err != nil {
+		return err
+	}
+	fmt.Printf("Server Log Dump:\n\n%s\n\n", b)
+	return nil
+}
+
 // RunCommand Runs command on the host
 func RunCommand(cmd string) (string, error) {
 	c := exec.Command("bash", "-c", cmd)
@@ -290,6 +317,32 @@ func RunCommand(cmd string) (string, error) {
 	return out.String(), nil
 }
 
+func unmountFolder(folder string) error {
+	mounts := []string{}
+	f, err := os.ReadFile("/proc/self/mounts")
+	if err != nil {
+		return err
+	}
+	for _, line := range strings.Split(string(f), "\n") {
+		columns := strings.Split(line, " ")
+		if len(columns) > 1 {
+			mounts = append(mounts, columns[1])
+		}
+	}
+
+	for _, mount := range mounts {
+		if strings.Contains(mount, folder) {
+			if err := unix.Unmount(mount, 0); err != nil {
+				return err
+			}
+			if err = os.Remove(mount); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
 func k8sClient() (*kubernetes.Clientset, error) {
 	config, err := clientcmd.BuildConfigFromFlags("", "/etc/rancher/k3s/k3s.yaml")
 	if err != nil {

tests/integration/localstorage/localstorage_int_test.go

@@ -84,6 +84,9 @@ var _ = Describe("local storage", func() {
 
 var _ = AfterSuite(func() {
 	if !testutil.IsExistingServer() {
+		if CurrentSpecReport().Failed() {
+			testutil.K3sDumpLog(localStorageServer)
+		}
 		Expect(testutil.K3sKillServer(localStorageServer)).To(Succeed())
 		Expect(testutil.K3sCleanup(testLock, "")).To(Succeed())
 	}

tests/integration/secretsencryption/secretsencryption_int_test.go

@@ -26,7 +26,7 @@ var _ = BeforeSuite(func() {
 	}
 })
 
-var _ = Describe("secrets encryption rotation", func() {
+var _ = Describe("secrets encryption rotation", Ordered, func() {
 	BeforeEach(func() {
 		if testutil.IsExistingServer() {
 			Skip("Test does not support running on existing k3s servers")
@@ -142,6 +142,9 @@ var _ = Describe("secrets encryption rotation", func() {
 
 var _ = AfterSuite(func() {
 	if !testutil.IsExistingServer() {
+		if CurrentSpecReport().Failed() {
+			testutil.K3sDumpLog(secretsEncryptionServer)
+		}
 		Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed())
 		Expect(testutil.K3sCleanup(testLock, secretsEncryptionDataDir)).To(Succeed())
 	}

tests/integration/startup/startup_int_test.go

 package integration
 
 import (
+	"os"
+	"path/filepath"
 	"testing"
 
 	testutil "github.com/k3s-io/k3s/tests/integration"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+
+	. "github.com/onsi/gomega/gstruct"
 	v1 "k8s.io/api/core/v1"
 )
 
@@ -22,7 +26,7 @@ var _ = BeforeSuite(func() {
 	Expect(err).ToNot(HaveOccurred())
 })
 
-var _ = Describe("startup tests", func() {
+var _ = Describe("startup tests", Ordered, func() {
 
 	When("a default server is created", func() {
 		It("is created with no arguments", func() {
@@ -33,10 +37,11 @@ var _ = Describe("startup tests", func() {
 		It("has the default pods deployed", func() {
 			Eventually(func() error {
 				return testutil.K3sDefaultDeployments()
-			}, "90s", "5s").Should(Succeed())
+			}, "120s", "5s").Should(Succeed())
 		})
 		It("dies cleanly", func() {
 			Expect(testutil.K3sKillServer(startupServer)).To(Succeed())
+			Expect(testutil.K3sCleanup(-1, "")).To(Succeed())
 		})
 	})
 	When("a etcd backed server is created", func() {
@@ -49,10 +54,11 @@ var _ = Describe("startup tests", func() {
 		It("has the default pods deployed", func() {
 			Eventually(func() error {
 				return testutil.K3sDefaultDeployments()
-			}, "90s", "5s").Should(Succeed())
+			}, "120s", "5s").Should(Succeed())
 		})
 		It("dies cleanly", func() {
 			Expect(testutil.K3sKillServer(startupServer)).To(Succeed())
+			Expect(testutil.K3sCleanup(-1, "")).To(Succeed())
 		})
 	})
 	When("a server without traefik is created", func() {
@@ -66,6 +72,9 @@ var _ = Describe("startup tests", func() {
 			Eventually(func() error {
 				return testutil.CheckDeployments([]string{"coredns", "local-path-provisioner", "metrics-server"})
 			}, "90s", "10s").Should(Succeed())
+			nodes, err := testutil.ParseNodes()
+			Expect(err).NotTo(HaveOccurred())
+			Expect(nodes).To(HaveLen(1))
 		})
 		It("dies cleanly", func() {
 			Expect(testutil.K3sKillServer(startupServer)).To(Succeed())
@@ -88,7 +97,7 @@ var _ = Describe("startup tests", func() {
 		It("has the node deployed with correct IPs", func() {
 			Eventually(func() error {
 				return testutil.K3sDefaultDeployments()
-			}, "90s", "10s").Should(Succeed())
+			}, "120s", "10s").Should(Succeed())
 
 			nodes, err := testutil.ParseNodes()
 			Expect(err).NotTo(HaveOccurred())
@@ -105,14 +114,86 @@ var _ = Describe("startup tests", func() {
 		})
 		It("dies cleanly", func() {
 			Expect(testutil.K3sKillServer(startupServer)).To(Succeed())
+			Expect(testutil.K3sCleanup(-1, "")).To(Succeed())
 			Expect(testutil.RunCommand("ip link del dummy2")).To(Equal(""))
 			Expect(testutil.RunCommand("ip link del dummy3")).To(Equal(""))
 		})
 	})
+	When("a server with different data-dir is created", func() {
+		var tempDir string
+		It("creates a temp directory", func() {
+			var err error
+			tempDir, err = os.MkdirTemp("", "k3s-data-dir")
+			Expect(err).ToNot(HaveOccurred())
+		})
+		It("is created with data-dir flag", func() {
+			var err error
+			startupServerArgs = []string{"--data-dir", tempDir}
+			startupServer, err = testutil.K3sStartServer(startupServerArgs...)
+			Expect(err).ToNot(HaveOccurred())
+		})
+		It("has the default pods deployed", func() {
+			Eventually(func() error {
+				return testutil.K3sDefaultDeployments()
+			}, "120s", "5s").Should(Succeed())
+			nodes, err := testutil.ParseNodes()
+			Expect(err).NotTo(HaveOccurred())
+			Expect(nodes).To(HaveLen(1))
+		})
+		It("has the correct files in the temp data-dir", func() {
+			_, err := os.Stat(filepath.Join(tempDir, "server", "tls", "server-ca.key"))
+			Expect(err).ToNot(HaveOccurred())
+			_, err = os.Stat(filepath.Join(tempDir, "server", "token"))
+			Expect(err).ToNot(HaveOccurred())
+			_, err = os.Stat(filepath.Join(tempDir, "agent", "client-kubelet.crt"))
+			Expect(err).ToNot(HaveOccurred())
+		})
+		It("dies cleanly", func() {
+			Expect(testutil.K3sKillServer(startupServer)).To(Succeed())
+			Expect(testutil.K3sCleanup(-1, tempDir)).To(Succeed())
+		})
+	})
+	When("a server with different node options is created", func() {
+		It("is created with node-name with-node-id, node-label and node-taint flags", func() {
+			var err error
+			startupServerArgs = []string{"--node-name", "customnoder", "--with-node-id", "--node-label", "foo=bar", "--node-taint", "alice=bob:PreferNoSchedule"}
+			startupServer, err = testutil.K3sStartServer(startupServerArgs...)
+			Expect(err).ToNot(HaveOccurred())
+		})
+		It("has the default pods deployed", func() {
+			Eventually(func() error {
+				return testutil.K3sDefaultDeployments()
+			}, "120s", "5s").Should(Succeed())
+			nodes, err := testutil.ParseNodes()
+			Expect(err).NotTo(HaveOccurred())
+			Expect(nodes).To(HaveLen(1))
+		})
+		var nodes []v1.Node
+		It("has a custom node name with id appended", func() {
+			var err error
+			nodes, err = testutil.ParseNodes()
+			Expect(err).NotTo(HaveOccurred())
+			Expect(nodes).To(HaveLen(1))
+			Expect(nodes[0].Name).To(MatchRegexp(`-[0-9a-f]*`))
+			Expect(nodes[0].Name).To(ContainSubstring("customnoder"))
+		})
+		It("has proper node labels and taints", func() {
+			Expect(nodes[0].ObjectMeta.Labels).To(MatchKeys(IgnoreExtras, Keys{
+				"foo": Equal("bar"),
+			}))
+			Expect(nodes[0].Spec.Taints).To(ContainElement(v1.Taint{Key: "alice", Value: "bob", Effect: v1.TaintEffectPreferNoSchedule}))
+		})
+		It("dies cleanly", func() {
+			Expect(testutil.K3sKillServer(startupServer)).To(Succeed())
+		})
+	})
 })
 
 var _ = AfterSuite(func() {
 	if !testutil.IsExistingServer() {
+		if CurrentSpecReport().Failed() {
+			testutil.K3sDumpLog(startupServer)
+		}
 		Expect(testutil.K3sCleanup(testLock, "")).To(Succeed())
 	}
 })

tests/snapshotter/btrfs/opensuse-leap/Vagrantfile

@@ -46,12 +46,16 @@ Vagrant.configure("2") do |config|
     test.vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s|
       k3s.args = %w[server --snapshotter=btrfs]
       k3s.env = %w[INSTALL_K3S_NAME=server INSTALL_K3S_SKIP_DOWNLOAD=true K3S_TOKEN=vagrant]
-      k3s.config = {
-        'disable' => %w[local-storage metrics-server servicelb traefik],
-        'disable-helm-controller' => true,
-        'disable-network-policy' => true,
-        'write-kubeconfig-mode' => '0644',
-      }
+      k3s.config = <<~YAML
+        disable:
+        - local-storage 
+        - metrics-server
+        - servicelb
+        - traefik
+        disable-helm-controller: true
+        disable-network-policy: true
+        write-kubeconfig-mode: '0644'
+      YAML
       k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321
     end
     test.vm.provision "k3s-wait-for-node", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh|

tests/terraform/testutils.go

@@ -38,8 +38,7 @@ var err error
 
 func checkError(e error) {
 	if e != nil {
-		log.Fatal(err)
-		panic(e)
+		log.Fatal(e)
 	}
 }