E2E secrets encryption test (#5144)

tests/e2e/amd64_resource_files/secrets.yaml

+apiVersion: v1
+kind: Secret
+metadata:
+  name: e2e-secret1
+type: Opaque
+stringData:
+  config.yaml: |
+    key: "hello"
+    val: "world"
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: e2e-secret2
+type: Opaque
+stringData:
+  config.yaml: |
+    key: "good"
+    val: "day"
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: e2e-secret3
+type: Opaque
+stringData:
+  config.yaml: |
+    key: "top-secret"
+    val: "information"
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: e2e-secret4
+type: Opaque
+stringData:
+  config.yaml: |
+    key: "lock"
+    val: "key"
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: e2e-secret5
+type: Opaque
+stringData:
+  config.yaml: |
+    key: "last"
+    val: "call"
\ No newline at end of file

tests/e2e/secretsencryption/Vagrantfile

+ENV['VAGRANT_NO_PARALLEL'] = 'no'
+NODE_ROLES = (ENV['NODE_ROLES'] ||
+  ["server-0", "server-1", "server-2"])
+NODE_BOXES = (ENV['NODE_BOXES'] ||
+  ['generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004'])
+GITHUB_BRANCH = (ENV['GITHUB_BRANCH'] || "master")
+RELEASE_VERSION = (ENV['RELEASE_VERSION'] || "")
+NODE_CPUS = (ENV['NODE_CPUS'] || 2).to_i
+NODE_MEMORY = (ENV['NODE_MEMORY'] || 1024).to_i
+# Virtualbox >= 6.1.28 require `/etc/vbox/network.conf` for expanded private networks 
+NETWORK_PREFIX = "10.10.10"
+install_type = ""
+
+def provision(vm, roles, role_num, node_num)
+  vm.box = NODE_BOXES[node_num]
+  vm.hostname = "#{roles[0]}-#{role_num}"
+  # An expanded netmask is required to allow VM<-->VM communication, virtualbox defaults to /32
+  vm.network "private_network", ip: "#{NETWORK_PREFIX}.#{100+node_num}", netmask: "255.255.255.0"
+
+  vagrant_defaults = '../vagrantdefaults.rb'
+  load vagrant_defaults if File.exists?(vagrant_defaults)
+  
+  defaultOSConfigure(vm)
+  
+  if !RELEASE_VERSION.empty?
+    install_type = "INSTALL_K3S_VERSION=#{RELEASE_VERSION}"
+  else
+    # Grabs the last 5 commit SHA's from the given branch, then purges any commits that do not have a passing CI build
+    vm.provision "shell", path: "../scripts/latest_commit.sh", args: [GITHUB_BRANCH, "/tmp/k3s_commits"]
+    install_type = "INSTALL_K3S_COMMIT=$(head\ -n\ 1\ /tmp/k3s_commits)"
+  end
+  vm.provision "shell", inline: "ping -c 2 k3s.io"
+  
+  if roles.include?("server") && role_num == 0
+    vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s|
+      k3s.args = %W[server --cluster-init --node-external-ip=#{NETWORK_PREFIX}.100 --flannel-iface=eth1 --secrets-encryption]
+      k3s.env = %W[K3S_KUBECONFIG_MODE=0644 K3S_TOKEN=vagrant #{install_type}]
+      k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321
+    end
+  elsif roles.include?("server") && role_num != 0
+    vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s|
+      k3s.args = %W[server --server https://#{NETWORK_PREFIX}.100:6443 --flannel-iface=eth1 --secrets-encryption]
+      k3s.env = %W[K3S_KUBECONFIG_MODE=0644 K3S_TOKEN=vagrant #{install_type}]
+      k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321
+    end
+  end
+  if vm.box.include?("microos")
+    vm.provision 'k3s-reload', type: 'reload', run: 'once'
+  end
+end
+
+Vagrant.configure("2") do |config|
+  config.vagrant.plugins = ["vagrant-k3s", "vagrant-reload"]
+  # Default provider is libvirt, virtualbox is only provided as a backup
+  config.vm.provider "libvirt" do |v|
+    v.cpus = NODE_CPUS
+    v.memory = NODE_MEMORY
+  end
+  config.vm.provider "virtualbox" do |v|
+    v.cpus = NODE_CPUS
+    v.memory = NODE_MEMORY
+  end
+  
+  if NODE_ROLES.kind_of?(String)
+    NODE_ROLES = NODE_ROLES.split(" ", -1)
+  end
+  if NODE_BOXES.kind_of?(String)
+    NODE_BOXES = NODE_BOXES.split(" ", -1)
+  end
+
+  # Must iterate on the index, vagrant does not understand iterating 
+  # over the node roles themselves
+  NODE_ROLES.length.times do |i|
+    name = NODE_ROLES[i]
+    config.vm.define name do |node|
+      roles = name.split("-", -1)
+      role_num = roles.pop.to_i
+      provision(node.vm, roles, role_num, i)
+    end
+  end
+end

tests/e2e/testutils.go

 package e2e
 
 import (
-	"bytes"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -40,15 +39,15 @@ func CountOfStringInSlice(str string, pods []Pod) int {
 }
 
 func CreateCluster(nodeOS string, serverCount int, agentCount int, installType string) ([]string, []string, error) {
-	serverNodeNames := make([]string, serverCount)
+	serverNodenames := make([]string, serverCount)
 	for i := 0; i < serverCount; i++ {
-		serverNodeNames[i] = "server-" + strconv.Itoa(i)
+		serverNodenames[i] = "server-" + strconv.Itoa(i)
 	}
-	agentNodeNames := make([]string, agentCount)
+	agentNodenames := make([]string, agentCount)
 	for i := 0; i < agentCount; i++ {
-		agentNodeNames[i] = "agent-" + strconv.Itoa(i)
+		agentNodenames[i] = "agent-" + strconv.Itoa(i)
 	}
-	nodeRoles := strings.Join(serverNodeNames, " ") + " " + strings.Join(agentNodeNames, " ")
+	nodeRoles := strings.Join(serverNodenames, " ") + " " + strings.Join(agentNodenames, " ")
 
 	nodeRoles = strings.TrimSpace(nodeRoles)
 	nodeBoxes := strings.Repeat(nodeOS+" ", serverCount+agentCount)
@@ -59,7 +58,7 @@ func CreateCluster(nodeOS string, serverCount int, agentCount int, installType s
 		fmt.Println("Error Creating Cluster", err)
 		return nil, nil, err
 	}
-	return serverNodeNames, agentNodeNames, nil
+	return serverNodenames, agentNodenames, nil
 }
 
 func DeployWorkload(workload, kubeconfig string, arch bool) (string, error) {
@@ -136,6 +135,18 @@ func GenKubeConfigFile(serverName string) (string, error) {
 	return kubeConfigFile, nil
 }
 
+func GetVagrantLog() string {
+	log, err := os.Open("vagrant.log")
+	if err != nil {
+		return err.Error()
+	}
+	bytes, err := ioutil.ReadAll(log)
+	if err != nil {
+		return err.Error()
+	}
+	return string(bytes)
+}
+
 func ParseNodes(kubeConfig string, print bool) ([]Node, error) {
 	nodes := make([]Node, 0, 10)
 	nodeList := ""
@@ -196,21 +207,28 @@ func ParsePods(kubeconfig string, print bool) ([]Pod, error) {
 	return pods, nil
 }
 
+// RestartCluster restarts the k3s service on each node given
+func RestartCluster(nodeNames []string) error {
+	for _, nodeName := range nodeNames {
+		cmd := "sudo systemctl restart k3s"
+		if _, err := RunCmdOnNode(cmd, nodeName); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // RunCmdOnNode executes a command from within the given node
 func RunCmdOnNode(cmd string, nodename string) (string, error) {
-	runcmd := "vagrant ssh -c " + cmd + " " + nodename
+	runcmd := "vagrant ssh -c \"" + cmd + "\" " + nodename
 	return RunCommand(runcmd)
 }
 
 // RunCommand executes a command on the host
 func RunCommand(cmd string) (string, error) {
 	c := exec.Command("bash", "-c", cmd)
-	var out bytes.Buffer
-	c.Stdout = &out
-	if err := c.Run(); err != nil {
-		return "", err
-	}
-	return out.String(), nil
+	out, err := c.CombinedOutput()
+	return string(out), err
 }
 
 func UpgradeCluster(serverNodenames []string, agentNodenames []string) error {

tests/e2e/upgradecluster/upgradecluster_test.go

@@ -37,7 +37,7 @@ var _ = Describe("Verify Upgrade", func() {
 		It("Starts up with no issues", func() {
 			var err error
 			serverNodenames, agentNodenames, err = e2e.CreateCluster(*nodeOS, *serverCount, *agentCount, *installType)
-			Expect(err).NotTo(HaveOccurred())
+			Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog())
 			fmt.Println("CLUSTER CONFIG")
 			fmt.Println("OS:", *nodeOS)
 			fmt.Println("Server Nodes:", serverNodenames)
@@ -83,7 +83,7 @@ var _ = Describe("Verify Upgrade", func() {
 			}, "240s", "5s").Should(ContainSubstring("test-clusterip"), "failed cmd: "+cmd)
 
 			clusterip, _ := e2e.FetchClusterIP(kubeConfigFile, "nginx-clusterip-svc")
-			cmd = "\"curl -L --insecure http://" + clusterip + "/name.html\""
+			cmd = "curl -L --insecure http://" + clusterip + "/name.html"
 			for _, nodeName := range serverNodenames {
 				Eventually(func() (string, error) {
 					return e2e.RunCmdOnNode(cmd, nodeName)
@@ -279,7 +279,7 @@ var _ = Describe("Verify Upgrade", func() {
 			}, "420s", "5s").Should(ContainSubstring("test-clusterip"))
 
 			clusterip, _ := e2e.FetchClusterIP(kubeConfigFile, "nginx-clusterip-svc")
-			cmd := "\"curl -L --insecure http://" + clusterip + "/name.html\""
+			cmd := "curl -L --insecure http://" + clusterip + "/name.html"
 			fmt.Println(cmd)
 			for _, nodeName := range serverNodenames {
 				Eventually(func() (string, error) {

tests/e2e/validatecluster/validatecluster_test.go

@@ -37,7 +37,7 @@ var _ = Describe("Verify Create", func() {
 		It("Starts up with no issues", func() {
 			var err error
 			serverNodenames, agentNodenames, err = e2e.CreateCluster(*nodeOS, *serverCount, *agentCount, *installType)
-			Expect(err).NotTo(HaveOccurred())
+			Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog)
 			fmt.Println("CLUSTER CONFIG")
 			fmt.Println("OS:", *nodeOS)
 			fmt.Println("Server Nodes:", serverNodenames)
@@ -84,7 +84,7 @@ var _ = Describe("Verify Create", func() {
 			}, "240s", "5s").Should(Succeed())
 
 			clusterip, _ := e2e.FetchClusterIP(kubeConfigFile, "nginx-clusterip-svc")
-			cmd := "\"curl -L --insecure http://" + clusterip + "/name.html\""
+			cmd := "curl -L --insecure http://" + clusterip + "/name.html"
 			fmt.Println(cmd)
 			for _, nodeName := range serverNodenames {
 				Eventually(func(g Gomega) {