Add nightly automation tests

tests/e2e/Dockerfile.build

+FROM golang:alpine
+
+ARG TERRAFORM_VERSION=0.12.10
+ENV TERRAFORM_VERSION=$TERRAFORM_VERSION
+
+
+RUN apk update && \
+    apk upgrade --update-cache --available && \
+    apk add curl git jq bash openssh unzip gcc g++ make ca-certificates && \
+    curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
+
+RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && \
+    chmod +x ./kubectl && \
+    mv ./kubectl /usr/local/bin
+RUN mkdir tmp && \
+    curl "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip" -o tmp/terraform.zip && \
+    unzip tmp/terraform.zip -d /usr/local/bin && \
+    chmod +x /usr/local/bin/terraform && \
+    rm -rf tmp
+
+WORKDIR $GOPATH/src/github.com/rancher/k3s-io/k3s/tests/e2e
+
+COPY . .
+
+RUN go get -u github.com/onsi/gomega
+RUN go get -u github.com/onsi/ginkgo

tests/e2e/amd64_resource_files/clusterip.yaml

+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: test-clusterip
+spec:
+  selector:
+    matchLabels:
+      k8s-app: nginx-app-clusterip
+  replicas: 2
+  template:
+    metadata:
+      labels:
+        k8s-app: nginx-app-clusterip
+    spec:
+      containers:
+      - name: nginx
+        image: ranchertest/mytestcontainer
+        ports:
+        - containerPort: 80
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    k8s-app: nginx-app-clusterip
+  name: nginx-clusterip-svc
+  namespace: default 
+spec:
+  type: ClusterIP
+  ports:
+    - port: 80
+  selector:
+    k8s-app: nginx-app-clusterip

tests/e2e/amd64_resource_files/daemonset.yaml

+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: test-daemonset
+spec:
+  selector:
+    matchLabels:
+      k8s-app: test-daemonset
+  template:
+    metadata:
+      labels:
+        k8s-app: test-daemonset
+    spec:
+      containers:
+        - name: webserver
+          image: nginx
+          ports:
+          - containerPort: 80

tests/e2e/amd64_resource_files/dnsutils.yaml

+apiVersion: v1
+kind: Pod
+metadata:
+  name: dnsutils
+  namespace: default
+spec:
+  containers:
+  - name: dnsutils
+    image: gcr.io/kubernetes-e2e-test-images/dnsutils:1.3
+    command:
+      - sleep
+      - "3600"
+    imagePullPolicy: IfNotPresent
+  restartPolicy: Always

tests/e2e/amd64_resource_files/ingress.yaml

+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+ name: ingress
+spec:
+ rules:
+ - host: foo1.bar.com
+   http:
+     paths:
+     - path: /name.html
+       backend:
+         serviceName: nginx-ingress-svc
+         servicePort: 80
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: nginx-ingress-svc
+ labels:
+   k8s-app: nginx-app-ingress
+spec:
+ ports:
+ - port: 80
+   targetPort: 80
+   protocol: TCP
+   name: http
+ selector:
+   k8s-app: nginx-app-ingress
+---
+apiVersion: v1
+kind: ReplicationController
+metadata:
+ name: test-ingress
+spec:
+ replicas: 2
+ selector:
+   k8s-app: nginx-app-ingress
+ template:
+   metadata:
+     labels:
+       k8s-app: nginx-app-ingress
+   spec:
+     terminationGracePeriodSeconds: 60
+     containers:
+     - name: testcontainer
+       image: ranchertest/mytestcontainer
+       ports:
+       - containerPort: 80

tests/e2e/amd64_resource_files/loadbalancer.yaml

+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: test-loadbalancer
+spec:
+  selector:
+    matchLabels:
+      k8s-app: nginx-app-loadbalancer
+  replicas: 2
+  template:
+    metadata:
+      labels:
+        k8s-app: nginx-app-loadbalancer
+    spec:
+      containers:
+      - name: nginx
+        image: ranchertest/mytestcontainer
+        ports:
+        - containerPort: 80
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: nginx-loadbalancer-svc
+ labels:
+   k8s-app: nginx-app-loadbalancer
+spec:
+ type: LoadBalancer
+ ports:
+ - port: 81
+   targetPort: 80
+   protocol: TCP
+   name: http
+ selector:
+   k8s-app: nginx-app-loadbalancer

tests/e2e/amd64_resource_files/local-path-provisioner.yaml

+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: local-path-pvc
+  namespace: default
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: local-path
+  resources:
+    requests:
+      storage: 500Mi
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: volume-test
+  namespace: default
+spec:
+  containers:
+  - name: volume-test
+    image: nginx:stable-alpine
+    imagePullPolicy: IfNotPresent
+    volumeMounts:
+    - name: volv
+      mountPath: /data
+    ports:
+    - containerPort: 80
+  volumes:
+  - name: volv
+    persistentVolumeClaim:
+      claimName: local-path-pvc

tests/e2e/amd64_resource_files/nodeport.yaml

+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: test-nodeport
+spec:
+  selector:
+    matchLabels:
+      k8s-app: nginx-app-nodeport
+  replicas: 2
+  template:
+    metadata:
+      labels:
+        k8s-app: nginx-app-nodeport
+    spec:
+      containers:
+      - name: nginx
+        image: ranchertest/mytestcontainer
+        ports:
+        - containerPort: 80
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    k8s-app: nginx-app-nodeport
+  name: nginx-nodeport-svc
+  namespace: default
+spec:
+  type: NodePort
+  ports:
+    - port: 80
+      nodePort: 30096
+      name: http
+  selector:
+    k8s-app: nginx-app-nodeport

tests/e2e/arm_resource_files/clusterip.yaml

+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: test-clusterip
+spec:
+  selector:
+    matchLabels:
+      k8s-app: nginx-app-clusterip
+  replicas: 2
+  template:
+    metadata:
+      labels:
+        k8s-app: nginx-app-clusterip
+    spec:
+      containers:
+      - name: nginx
+        image: shylajarancher19/shylajaarm64:v1.0
+        ports:
+        - containerPort: 80
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    k8s-app: nginx-app-clusterip
+  name: nginx-clusterip-svc
+  namespace: default 
+spec:
+  type: ClusterIP
+  ports:
+    - port: 80
+  selector:
+    k8s-app: nginx-app-clusterip

tests/e2e/arm_resource_files/daemonset.yaml

+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: test-daemonset
+spec:
+  selector:
+    matchLabels:
+      k8s-app: test-daemonset
+  template:
+    metadata:
+      labels:
+        k8s-app: test-daemonset
+    spec:
+      containers:
+        - name: webserver
+          image: nginx
+          ports:
+          - containerPort: 80

tests/e2e/arm_resource_files/ingress.yaml

+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+ name: ingress
+spec:
+ rules:
+ - host: foo1.bar.com
+   http:
+     paths:
+     - path: /name.html
+       backend:
+         serviceName: nginx-ingress-svc
+         servicePort: 80
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: nginx-ingress-svc
+ labels:
+   k8s-app: nginx-app-ingress
+spec:
+ ports:
+ - port: 80
+   targetPort: 80
+   protocol: TCP
+   name: http
+ selector:
+   k8s-app: nginx-app-ingress
+---
+apiVersion: v1
+kind: ReplicationController
+metadata:
+ name: test-ingress
+spec:
+ replicas: 2
+ selector:
+   k8s-app: nginx-app-ingress
+ template:
+   metadata:
+     labels:
+       k8s-app: nginx-app-ingress
+   spec:
+     terminationGracePeriodSeconds: 60
+     containers:
+     - name: testcontainer
+       image: shylajarancher19/shylajaarm64:v1.0
+       ports:
+       - containerPort: 80

tests/e2e/arm_resource_files/loadbalancer.yaml

+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: test-loadbalancer
+spec:
+  selector:
+    matchLabels:
+      k8s-app: nginx-app-loadbalancer
+  replicas: 2
+  template:
+    metadata:
+      labels:
+        k8s-app: nginx-app-loadbalancer
+    spec:
+      containers:
+      - name: nginx
+        image: shylajarancher19/shylajaarm64:v1.0
+        ports:
+        - containerPort: 80
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: nginx-loadbalancer-svc
+ labels:
+   k8s-app: nginx-app-loadbalancer
+spec:
+ type: LoadBalancer
+ ports:
+ - port: 81
+   targetPort: 80
+   protocol: TCP
+   name: http
+ selector:
+   k8s-app: nginx-app-loadbalancer

tests/e2e/arm_resource_files/local-path-provisioner.yaml

+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: local-path-pvc
+  namespace: default
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: local-path
+  resources:
+    requests:
+      storage: 500Mi
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: volume-test
+  namespace: default
+spec:
+  containers:
+  - name: volume-test
+    image: nginx:stable-alpine
+    imagePullPolicy: IfNotPresent
+    volumeMounts:
+    - name: volv
+      mountPath: /data
+    ports:
+    - containerPort: 80
+  volumes:
+  - name: volv
+    persistentVolumeClaim:
+      claimName: local-path-pvc

tests/e2e/arm_resource_files/nodeport.yaml

+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: test-nodeport
+spec:
+  selector:
+    matchLabels:
+      k8s-app: nginx-app-nodeport
+  replicas: 2
+  template:
+    metadata:
+      labels:
+        k8s-app: nginx-app-nodeport
+    spec:
+      containers:
+      - name: nginx
+        image: shylajarancher19/shylajaarm64:v1.0
+        ports:
+        - containerPort: 80
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    k8s-app: nginx-app-nodeport
+  name: nginx-nodeport-svc
+  namespace: default
+spec:
+  type: NodePort
+  ports:
+    - port: 80
+      nodePort: 30096
+      name: http
+  selector:
+    k8s-app: nginx-app-nodeport

tests/e2e/createcluster.go

+package e2e
+
+import (
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"path/filepath"
+	"testing"
+
+	"github.com/gruntwork-io/terratest/modules/terraform"
+)
+
+var destroy = flag.Bool("destroy", false, "a bool")
+var nodeOs = flag.String("node_os", "centos8", "a string")
+var externalDb = flag.String("external_db", "mysql", "a string")
+var arch = flag.String("arch", "amd64", "a string")
+var clusterType = flag.String("cluster_type", "etcd", "a string")
+var resourceName = flag.String("resource_name", "etcd", "a string")
+var sshuser = flag.String("sshuser", "ubuntu", "a string")
+var sshkey = flag.String("sshkey", "", "a string")
+
+var (
+	kubeconfig string
+	masterIPs  string
+	workerIPs  string
+)
+
+func DeployWorkloads(arch, Kubeconfig string) {
+	resource_dir := ""
+	if arch == "amd64" {
+		resource_dir = "./amd64_resource_files"
+	} else {
+		resource_dir = "./arm_resource_files"
+	}
+
+	files, err := ioutil.ReadDir(resource_dir)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	for _, f := range files {
+		workload := filepath.Join(resource_dir, f.Name())
+		_, _ = DeployWorkload(workload, Kubeconfig)
+	}
+}
+
+// nodeOs: ubuntu centos7 centos8 sles15
+// clusterType arm, etcd externaldb, if external_db var is not "" picks database from the vars file,
+// resourceName: name to resource created timestamp attached
+
+func BuildCluster(nodeOs, clusterType, externalDb, resourceName string, t *testing.T, destroy bool) (string, string, string) {
+
+	tDir := "./terraform/modules/k3scluster"
+	vDir := "/config/" + nodeOs + clusterType + ".tfvars"
+
+	if externalDb != "" {
+		vDir = "/config/" + nodeOs + externalDb + ".tfvars"
+	}
+
+	tfDir, _ := filepath.Abs(tDir)
+	varDir, _ := filepath.Abs(vDir)
+	TerraformOptions := &terraform.Options{
+		TerraformDir: tfDir,
+		VarFiles:     []string{varDir},
+		Vars: map[string]interface{}{
+			"cluster_type":  clusterType,
+			"resource_name": resourceName,
+			"external_db":   externalDb,
+		},
+	}
+
+	if destroy {
+		fmt.Printf("Cluster is being deleted")
+		terraform.Destroy(t, TerraformOptions)
+		return "", "", ""
+	}
+
+	fmt.Printf("Creating Cluster")
+	terraform.InitAndApply(t, TerraformOptions)
+	kubeconfig := terraform.Output(t, TerraformOptions, "kubeconfig") + "_kubeconfig"
+	masterIps := terraform.Output(t, TerraformOptions, "master_ips")
+	workerIps := terraform.Output(t, TerraformOptions, "worker_ips")
+	kubeconfigFile := "/config/" + kubeconfig
+	return kubeconfigFile, masterIps, workerIps
+}

tests/e2e/createcluster_test.go

+package e2e
+
+import (
+	"fmt"
+
+	. "github.com/onsi/ginkgo"
+	"github.com/onsi/ginkgo/reporters"
+	. "github.com/onsi/gomega"
+
+	"strings"
+	"testing"
+	"time"
+)
+
+func Test_E2EClusterCreateValidation(t *testing.T) {
+	junitReporter := reporters.NewJUnitReporter(fmt.Sprintf("/config/" + *resourceName + ".xml"))
+	RegisterFailHandler(Fail)
+	RunSpecsWithDefaultAndCustomReporters(t, "Test Suite", []Reporter{junitReporter})
+
+}
+
+var _ = Describe("Test:", func() {
+	Context("Build Cluster:", func() {
+		Context("Cluster Configuration: OS: "+*nodeOs+"  Cluster Type; "+*externalDb+" "+*clusterType, func() {
+
+			kubeconfig, masterIPs, workerIPs = BuildCluster(*nodeOs, *clusterType, *externalDb, *resourceName, &testing.T{}, *destroy)
+			defer GinkgoRecover()
+			if *destroy {
+				fmt.Printf("\nCluster is being Deleted\n")
+				return
+			}
+			fmt.Println("\nCLUSTER CONFIG:\nOS", *nodeOs, "BACKEND", *clusterType, *externalDb)
+			fmt.Printf("\nIPs:\n")
+			fmt.Println("Master Node IPS:", masterIPs)
+			fmt.Println("Worker Node IPS:", workerIPs)
+
+			fmt.Printf("\nFetching node status\n")
+			nodes := ParseNode(kubeconfig, true)
+			for _, config := range nodes {
+				Expect(config.Status).Should(Equal("Ready"), func() string { return config.Name })
+			}
+
+			fmt.Printf("\nFetching Pods status\n")
+			pods := ParsePod(kubeconfig, true)
+			for _, pod := range pods {
+				if strings.Contains(pod.Name, "helm-install") {
+					Expect(pod.Status).Should(Equal("Completed"), func() string { return pod.Name })
+				} else {
+					Expect(pod.Status).Should(Equal("Running"), func() string { return pod.Name })
+				}
+			}
+		})
+		Context("Validate Rebooting nodes", func() {
+			if *destroy {
+				return
+			}
+			defer GinkgoRecover()
+			nodeExternalIP := FetchNodeExternalIP(kubeconfig)
+			for _, ip := range nodeExternalIP {
+				fmt.Println("\nRebooting node: ", ip)
+				cmd := "ssh -i " + *sshkey + " -o \"StrictHostKeyChecking no\" " + *sshuser + "@" + ip + " sudo reboot"
+				_, _ = RunCommand(cmd)
+				time.Sleep(3 * time.Minute)
+				fmt.Println("\nNode and Pod Status after rebooting node: ", ip)
+				nodes := ParseNode(kubeconfig, true)
+				for _, config := range nodes {
+					Expect(config.Status).Should(Equal("Ready"), func() string { return config.Name })
+				}
+
+				pods := ParsePod(kubeconfig, true)
+				for _, pod := range pods {
+					if strings.Contains(pod.Name, "helm-install") {
+						Expect(pod.Status).Should(Equal("Completed"), func() string { return pod.Name })
+					} else {
+						Expect(pod.Status).Should(Equal("Running"), func() string { return pod.Name })
+					}
+				}
+			}
+		})
+
+		Context("Deploy workloads ", func() {
+			if *destroy {
+				return
+			}
+			defer GinkgoRecover()
+
+			It("Validate Cluster IP", func() {
+				if *destroy {
+					return
+				}
+				DeployWorkloads(*arch, kubeconfig)
+				fmt.Println("Validating ClusterIP")
+				clusterip := FetchClusterIP(kubeconfig, "nginx-clusterip-svc")
+				cmd := "curl -L --insecure http://" + clusterip + "/name.html"
+				fmt.Println(cmd)
+				//Fetch External IP to login to node and validate cluster ip
+				nodeExternalIP := FetchNodeExternalIP(kubeconfig)
+
+				for _, ip := range nodeExternalIP {
+					res := RunCmdOnNode(cmd, ip, *sshuser, *sshkey)
+					fmt.Println(res)
+					Expect(res).Should(ContainSubstring("test-clusterip"), func() string { return res })
+				}
+			})
+
+			It("Validate NodePort", func() {
+				if *destroy {
+					return
+				}
+				fmt.Println("Validating NodePort")
+				nodeExternalIP := FetchNodeExternalIP(kubeconfig)
+				cmd := "kubectl get service nginx-nodeport-svc --kubeconfig=" + kubeconfig + " --output jsonpath=\"{.spec.ports[0].nodePort}\""
+				nodeport, _ := RunCommand(cmd)
+				for _, nodeExternalIp := range nodeExternalIP {
+					cmd := "curl -L --insecure http://" + nodeExternalIp + ":" + nodeport + "/name.html"
+					fmt.Println(cmd)
+					res, _ := RunCommand(cmd)
+					fmt.Println(res)
+					Expect(res).Should(ContainSubstring("test-nodeport"), func() string { return res })
+				}
+			})
+
+			It("Validate LoadBalancer", func() {
+				if *destroy {
+					return
+				}
+				fmt.Println("Validating Service LoadBalancer")
+				nodeExternalIP := FetchNodeExternalIP(kubeconfig)
+				cmd := "kubectl get service nginx-loadbalancer-svc --kubeconfig=" + kubeconfig + " --output jsonpath=\"{.spec.ports[0].port}\""
+				port, _ := RunCommand(cmd)
+				for _, ip := range nodeExternalIP {
+					cmd = "curl -L --insecure http://" + ip + ":" + port + "/name.html"
+					fmt.Println(cmd)
+					res, _ := RunCommand(cmd)
+					fmt.Println(res)
+					Expect(res).Should(ContainSubstring("test-loadbalancer"), func() string { return res })
+				}
+			})
+
+			It("Validate Daemonset", func() {
+				if *destroy {
+					return
+				}
+				fmt.Println("Validating Daemonset")
+				nodes := ParseNode(kubeconfig, false)
+				pods := ParsePod(kubeconfig, false)
+				count := CountOfStringInSlice("test-daemonset", pods)
+				fmt.Println("POD COUNT")
+				fmt.Println(count)
+				fmt.Println("NODE COUNT")
+				fmt.Println(len(nodes))
+				Eventually(len(nodes)).Should((Equal(count)), "120s", "60s")
+			})
+
+			It("Validate Ingress", func() {
+				if *destroy {
+					return
+				}
+				fmt.Println("Validating Ingress")
+
+				ingressIps := FetchIngressIP(kubeconfig)
+				for _, ip := range ingressIps {
+					cmd := "curl  --header host:foo1.bar.com" + " http://" + ip + "/name.html"
+					fmt.Println(cmd)
+					//Access path from outside node
+					res, _ := RunCommand(cmd)
+					fmt.Println(res)
+					Eventually(res).Should((ContainSubstring("test-ingress")), "120s", "60s", func() string { return res })
+				}
+			})
+			It("Validate Local Path Provisioner storage ", func() {
+				if *destroy {
+					return
+				}
+				fmt.Println("Validating Local Path Provisioner")
+				cmd := "kubectl get pvc local-path-pvc --kubeconfig=" + kubeconfig
+				res, _ := RunCommand(cmd)
+				fmt.Println(res)
+				Eventually(res).Should((ContainSubstring("local-path-pvc")), "120s", "60s")
+				Eventually(res).Should((ContainSubstring("Bound")), "120s", "60s")
+
+				cmd = "kubectl get pod volume-test --kubeconfig=" + kubeconfig
+				res, _ = RunCommand(cmd)
+				fmt.Println(res)
+
+				Eventually(res).Should((ContainSubstring("volume-test")), "120s", "60s", func() string { return res })
+
+				cmd = "kubectl --kubeconfig=" + kubeconfig + " exec volume-test -- sh -c 'echo local-path-test > /data/test'"
+				fmt.Println(cmd)
+				res, _ = RunCommand(cmd)
+				fmt.Println(res)
+				fmt.Println("Data stored", res)
+
+				cmd = "kubectl delete pod volume-test --kubeconfig=" + kubeconfig
+				res, _ = RunCommand(cmd)
+				fmt.Println(res)
+				resource_dir := "./amd64_resource_files"
+				cmd = "kubectl apply -f " + resource_dir + "/local-path-provisioner.yaml --kubeconfig=" + kubeconfig
+				res, _ = RunCommand(cmd)
+				fmt.Println(res)
+
+				time.Sleep(1 * time.Minute)
+				cmd = "kubectl exec volume-test cat /data/test --kubeconfig=" + kubeconfig
+				res, _ = RunCommand(cmd)
+				fmt.Println("Data after re-creation", res)
+
+				Eventually(res).Should((ContainSubstring("local-path-test")), "120s", "60s", func() string { return res })
+			})
+			It("Validate DNS Resolution", func() {
+				if *destroy {
+					return
+				}
+				fmt.Println("Validating DNS Resolution")
+				cmd := "kubectl --kubeconfig=" + kubeconfig + " exec -i -t dnsutils -- nslookup kubernetes.default"
+				fmt.Println(cmd)
+				res, _ := RunCommand(cmd)
+				fmt.Println("Result", res)
+				Eventually(res).ShouldNot((ContainSubstring("nslookup")), "120s", "60s")
+			})
+		})
+	})
+
+})
+
+var _ = AfterSuite(func() {
+	kubeconfig, masterIPs, workerIPs = BuildCluster(*nodeOs, *clusterType, *externalDb, *resourceName, &testing.T{}, true)
+})

tests/e2e/go.mod

+module github.com/rancher/k3s/tests/e2e
+
+go 1.15
+
+require (
+	github.com/gruntwork-io/terratest v0.30.23
+	github.com/onsi/ginkgo v1.14.2
+	github.com/onsi/gomega v1.10.3
+	github.com/stretchr/testify v1.6.1 // indirect
+	golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a
+)

tests/e2e/scripts/build.sh

+/usr/local/bin/docker build -f ../Dockerfile.build -t k3s_create_cluster:v1.0 .

tests/e2e/terraform/modules/k3scluster/install_k3s_master.sh

+#!/bin/bash
+
+mkdir -p /etc/rancher/k3s
+cat << EOF >/etc/rancher/k3s/config.yaml
+write-kubeconfig-mode: "0644"
+tls-san:
+  - ${2}
+EOF
+
+if [[ -n "$8" ]] && [[ "$8" == *":"* ]]
+then
+   echo "$"
+   echo -e "$8" >> /etc/rancher/k3s/config.yaml
+   cat /etc/rancher/k3s/config.yaml
+fi
+
+if [ "${1}" = "rhel" ]
+then
+   subscription-manager register --auto-attach --username="${9}" --password="${10}"
+   subscription-manager repos --enable=rhel-7-server-extras-rpms
+fi
+
+export "${3}"="${4}"
+
+if [ "${5}" = "etcd" ]
+then
+   echo "CLUSTER TYPE  is etcd"
+   if [[ "$4" == *"v1.18"* ]] || [["$4" == *"v1.17"* ]] && [[ -n "$8" ]]
+   then
+       echo "curl -sfL https://get.k3s.io | INSTALL_K3S_TYPE='server' sh -s - --cluster-init --node-external-ip=${6} $8" >/tmp/master_cmd
+       curl -sfL https://get.k3s.io | INSTALL_K3S_TYPE='server' sh -s - --cluster-init --node-external-ip="${6}" "$8"
+   else
+       echo "curl -sfL https://get.k3s.io | INSTALL_K3S_TYPE='server' sh -s - --cluster-init --node-external-ip=${6}" >/tmp/master_cmd
+       curl -sfL https://get.k3s.io | INSTALL_K3S_TYPE='server' sh -s - --cluster-init --node-external-ip="${6}"
+   fi
+else
+   echo "CLUSTER TYPE is external db"
+   echo "$8"
+   if [[ "$4" == *"v1.18"* ]] || [[ "$4" == *"v1.17"* ]] && [[ -n "$8" ]]
+   then
+       echo "curl -sfL https://get.k3s.io | sh -s - server --node-external-ip=${6} --datastore-endpoint=\"${7}\" $8"  >/tmp/master_cmd
+       curl -sfL https://get.k3s.io | sh -s - server --node-external-ip="${6}" --datastore-endpoint="${7}" "$8"
+   else
+       echo "curl -sfL https://get.k3s.io | sh -s - server --node-external-ip=${6}  --datastore-endpoint=\"${7}\" "  >/tmp/master_cmd
+       curl -sfL https://get.k3s.io | sh -s - server --node-external-ip="${6}" --datastore-endpoint="${7}"
+   fi
+fi
+
+export PATH=$PATH:/usr/local/bin
+timeElapsed=0
+while ! $(kubectl get nodes >/dev/null 2>&1) && [[ $timeElapsed -lt 300 ]]
+do
+   sleep 5
+   timeElapsed=$(expr $timeElapsed + 5)
+done
+
+IFS=$'\n'
+timeElapsed=0
+sleep 10
+while [[ $timeElapsed -lt 420 ]]
+do
+   notready=false
+   for rec in $(kubectl get nodes)
+   do
+      if [[ "$rec" == *"NotReady"* ]]
+      then
+         notready=true
+      fi
+  done
+  if [[ $notready == false ]]
+  then
+     break
+  fi
+  sleep 20
+  timeElapsed=$(expr $timeElapsed + 20)
+done
+
+IFS=$'\n'
+timeElapsed=0
+while [[ $timeElapsed -lt 420 ]]
+do
+   helmPodsNR=false
+   systemPodsNR=false
+   for rec in $(kubectl get pods -A --no-headers)
+   do
+      if [[ "$rec" == *"helm-install"* ]] && [[ "$rec" != *"Completed"* ]]
+      then
+         helmPodsNR=true
+      elif [[ "$rec" != *"helm-install"* ]] && [[ "$rec" != *"Running"* ]]
+      then
+         systemPodsNR=true
+      else
+         echo ""
+      fi
+   done
+
+   if [[ $systemPodsNR == false ]] && [[ $helmPodsNR == false ]]
+   then
+      break
+   fi
+   sleep 20
+   timeElapsed=$(expr $timeElapsed + 20)
+done
+cat /etc/rancher/k3s/config.yaml> /tmp/joinflags
+cat /var/lib/rancher/k3s/server/node-token >/tmp/nodetoken
+cat /etc/rancher/k3s/k3s.yaml >/tmp/config

tests/e2e/terraform/modules/k3scluster/join_k3s_agent.sh

+#!/bin/bash
+# This script is used to join one or more nodes as agents
+
+mkdir -p /etc/rancher/k3s
+cat <<EOF >>/etc/rancher/k3s/config.yaml
+server: https://${4}:6443
+token:  "${5}"
+EOF
+
+if [[ ! -z "$7" ]] && [[ "$7" == *":"* ]]
+then
+   echo -e "$7" >> /etc/rancher/k3s/config.yaml
+   cat /etc/rancher/k3s/config.yaml
+fi
+
+if [ ${1} = "rhel" ]
+then
+    subscription-manager register --auto-attach --username=${8} --password=${9}
+    subscription-manager repos --enable=rhel-7-server-extras-rpms
+fi
+
+export "${2}"="${3}"
+  if [[ "$3" == *"v1.18"* ]] || [["$3" == *"v1.17"* ]] && [[ -n "$7" ]]
+then
+  echo "curl -sfL https://get.k3s.io | sh -s - agent --node-external-ip=${6} $7" >/tmp/agent_cmd
+curl -sfL https://get.k3s.io | sh -s - agent --node-external-ip=${6} ${7}
+  else
+
+echo "curl -sfL https://get.k3s.io | sh -s - agent --node-external-ip=${6}" >/tmp/agent_cmd
+curl -sfL https://get.k3s.io | sh -s - agent --node-external-ip=${6}
+fi

tests/e2e/terraform/modules/k3scluster/join_k3s_master.sh

+#!/bin/bash
+# This script is used to join one or more nodes as masters
+
+mkdir -p /etc/rancher/k3s
+cat <<EOF >>/etc/rancher/k3s/config.yaml
+write-kubeconfig-mode: "0644"
+tls-san:
+  - ${2}
+EOF
+
+if [[ -n "${10}" ]] && [[ "${10}" == *":"* ]]
+then
+   echo -e "${10}" >> /etc/rancher/k3s/config.yaml
+   cat /etc/rancher/k3s/config.yaml
+fi
+
+if [ "${1}" = "rhel" ]
+then
+   subscription-manager register --auto-attach --username="${11}" --password="${12}"
+   subscription-manager repos --enable=rhel-7-server-extras-rpms
+fi
+
+export "${3}"="${4}"
+
+if [ "${5}" = "etcd" ]
+then
+   if [[ "$4" == *"v1.18"* ]] || [["$4" == *"v1.17"* ]] && [[ -n "$10" ]]
+   then
+      echo "curl -sfL https://get.k3s.io | INSTALL_K3S_TYPE='server' sh -s - --server https://\"${7}\":6443 --token \"${8}\" --node-external-ip=\"${6}\" ${10}" >/tmp/master_cmd
+      curl -sfL https://get.k3s.io | INSTALL_K3S_TYPE='server' sh -s - --server https://"${7}":6443 --token "${8}" --node-external-ip="${6} ${10}"
+   else
+        echo "curl -sfL https://get.k3s.io | INSTALL_K3S_TYPE='server' sh -s - --server https://\"${7}\":6443 --token \"${8}\" --node-external-ip=\"${6}\"" >/tmp/master_cmd
+       curl -sfL https://get.k3s.io | INSTALL_K3S_TYPE='server' sh -s - --server https://"${7}":6443 --token "${8}" --node-external-ip="${6}"
+   fi
+else
+  if [[ "$4" == *"v1.18"* ]] || [["$4" == *"v1.17"* ]] && [[ -n "$10" ]]
+  then
+      echo "curl -sfL https://get.k3s.io | INSTALL_K3S_TYPE='server' sh -s - --node-external-ip=\"${6}\" --datastore-endpoint=\"${9}\" ${10}" >/tmp/master_cmd
+      curl -sfL https://get.k3s.io | INSTALL_K3S_TYPE='server' sh -s - --node-external-ip="${6}" --token="${8}" --datastore-endpoint="${9} ${10}"
+   else
+      echo "curl -sfL https://get.k3s.io | INSTALL_K3S_TYPE='server' sh -s - --node-external-ip=\"${6}\" --token \"${8}\" --datastore-endpoint=\"${9}\"" >/tmp/master_cmd
+      curl -sfL https://get.k3s.io | INSTALL_K3S_TYPE='server' sh -s - --node-external-ip="${6}" --token="${8}" --datastore-endpoint="${9}"
+    fi
+fi

tests/e2e/terraform/modules/k3scluster/main.tf

+module "master" {
+   source="./master"
+   aws_ami=var.aws_ami
+   aws_user=var.aws_user
+   key_name=var.key_name
+   no_of_server_nodes=var.no_of_server_nodes
+   k3s_version=var.k3s_version
+   install_mode=var.install_mode
+   region=var.region
+   vpc_id=var.vpc_id
+   subnets=var.subnets
+   qa_space=var.qa_space
+   ec2_instance_class=var.ec2_instance_class
+   access_key=var.access_key
+   cluster_type=var.cluster_type
+   server_flags=var.server_flags
+   availability_zone=var.availability_zone
+   sg_id=var.sg_id
+   resource_name=var.resource_name
+   node_os=var.node_os
+   username=var.username
+   password=var.password
+   db_username=var.db_username
+   db_password=var.db_password
+   db_group_name=var.db_group_name
+   external_db=var.external_db
+   instance_class=var.instance_class
+   external_db_version=var.external_db_version
+   engine_mode=var.engine_mode
+   environment=var.environment
+}
+module "worker" {
+   source="./worker"
+   dependency = module.master
+   aws_ami=var.aws_ami
+   aws_user=var.aws_user
+   key_name=var.key_name
+   no_of_worker_nodes=var.no_of_worker_nodes
+   k3s_version=var.k3s_version
+   install_mode=var.install_mode
+   region=var.region
+   vpc_id=var.vpc_id
+   subnets=var.subnets
+   ec2_instance_class=var.ec2_instance_class
+   access_key=var.access_key
+   worker_flags=var.worker_flags
+   availability_zone=var.availability_zone
+   sg_id=var.sg_id
+   resource_name=var.resource_name
+   node_os=var.node_os
+   username=var.username
+   password=var.password
+}

tests/e2e/terraform/modules/k3scluster/master/outputs.tf

+output "Route53_info" {
+  value       = aws_route53_record.aws_route53.*
+  description = "List of DNS records"
+}
+
+output "master_ips" {
+  value = join("," , aws_instance.master.*.public_ip,aws_instance.master2-ha.*.public_ip)
+  description = "The public IP of the AWS node"
+}
+
+output "kubeconfig" {
+  value = var.resource_name
+  description = "kubeconfig of the cluster created"
+}

tests/e2e/terraform/modules/k3scluster/master/providers.tf

+provider "aws" {
+    region = "${var.region}"
+}

tests/e2e/terraform/modules/k3scluster/master/variables.tf

+variable "aws_ami" {}
+variable "aws_user" {}
+variable "region" {}
+variable "access_key" {}
+variable "vpc_id" {}
+variable "subnets" {}
+variable "availability_zone" {}
+variable "sg_id" {}
+variable "qa_space" {}
+variable "ec2_instance_class" {}
+variable "resource_name" {}
+variable "key_name" {}
+
+variable "external_db" {}
+variable "external_db_version" {}
+variable "instance_class" {}
+
+variable "db_group_name" {}
+variable "username" {}
+variable "password" {}
+variable "k3s_version" {}
+variable "no_of_server_nodes" {}
+variable "server_flags" {}
+
+variable "cluster_type" {}
+variable "node_os" {}
+variable "db_username" {}
+variable "db_password" {}
+variable "environment" {}
+variable "engine_mode" {}
+variable "install_mode" {}

tests/e2e/terraform/modules/k3scluster/outputs.tf

+output "master_ips" {
+  value       = module.master.master_ips
+  description = "The public IP of the AWS node"
+}
+
+output "worker_ips" {
+  value       = module.worker.worker_ips
+  description = "The public IP of the AWS node"
+}
+
+output "kubeconfig" {
+  value = module.master.kubeconfig
+  description = "kubeconfig of the cluster created"
+}

tests/e2e/terraform/modules/k3scluster/providers.tf

+provider "aws" {
+    region = "${var.region}"
+}

tests/e2e/terraform/modules/k3scluster/variables.tf

+#variable "db" {}
+variable "no_of_worker_nodes" {}
+variable "aws_ami" {}
+variable "aws_user" {}
+variable "region" {}
+variable "access_key" {}
+variable "vpc_id" {}
+variable "subnets" {}
+variable "qa_space" {}
+variable "resource_name" {}
+variable "key_name" {}
+variable "external_db" {}
+variable "external_db_version" {}
+variable "instance_class" {}
+variable "ec2_instance_class" {}
+variable "db_group_name" {}
+variable "username" {}
+variable "password" {}
+variable "k3s_version" {}
+variable "no_of_server_nodes" {}
+variable "server_flags" {}
+variable "worker_flags" {}
+variable "availability_zone" {}
+variable "sg_id" {}
+variable "cluster_type" {}
+variable "node_os" {}
+variable "db_username" {}
+variable "db_password" {}
+variable "environment" {}
+variable "engine_mode" {}
+variable "install_mode" {}

tests/e2e/terraform/modules/k3scluster/worker/instances_worker.tf

+resource "aws_instance" "worker" {
+  depends_on = [
+    var.dependency
+  ]
+  ami                    = var.aws_ami
+  instance_type          = var.ec2_instance_class
+  count                  = var.no_of_worker_nodes
+  connection {
+    type                 = "ssh"
+    user                 = var.aws_user
+    host                 = self.public_ip
+    private_key          = file(var.access_key)
+  }
+  subnet_id              = var.subnets
+  availability_zone      = var.availability_zone
+  vpc_security_group_ids = [var.sg_id]
+  key_name               = var.key_name
+  tags = {
+    Name                 = "${var.resource_name}-worker"
+  }
+  provisioner "file" {
+    source = "join_k3s_agent.sh"
+    destination = "/tmp/join_k3s_agent.sh"
+  }
+  provisioner "remote-exec" {
+    inline = [
+      "chmod +x /tmp/join_k3s_agent.sh",
+      "sudo /tmp/join_k3s_agent.sh ${var.node_os} ${var.install_mode} ${var.k3s_version} ${local.master_ip} ${local.node_token} ${self.public_ip} \"${var.worker_flags}\" ${var.username} ${var.password} ",
+    ]
+  }
+}
+
+data "local_file" "master_ip" {
+  depends_on = [var.dependency]
+  filename = "/tmp/${var.resource_name}_master_ip"
+}
+
+locals {
+  master_ip = trimspace(data.local_file.master_ip.content)
+}
+
+data "local_file" "token" {
+  depends_on = [var.dependency]
+  filename = "/tmp/${var.resource_name}_nodetoken"
+}
+
+locals {
+  node_token = trimspace(data.local_file.token.content)
+}

tests/e2e/terraform/modules/k3scluster/worker/outputs.tf

+output "Registration_address" {
+  value = "${data.local_file.master_ip.content}"
+}
+
+output "master_node_token" {
+  value = "${data.local_file.token.content}"
+}
+
+output "worker_ips" {
+  value = join("," , aws_instance.worker.*.public_ip)
+  description = "The public IP of the AWS node"
+}

tests/e2e/terraform/modules/k3scluster/worker/providers.tf

+provider "aws" {
+    region = "${var.region}"
+}

tests/e2e/terraform/modules/k3scluster/worker/variables.tf

+variable "dependency" {
+  type    = any
+  default = null
+}
+variable "region" {}
+variable "aws_ami" {}
+variable "aws_user" {}
+variable "vpc_id" {}
+variable "subnets" {}
+variable "resource_name" {}
+variable "access_key" {}
+variable "k3s_version" {}
+variable "no_of_worker_nodes" {}
+variable "worker_flags" {}
+variable "ec2_instance_class" {}
+variable "availability_zone" {}
+variable "sg_id" {}
+variable "username" {}
+variable "password" {}
+variable "node_os" {}
+variable "install_mode" {}
+variable "key_name" {}

tests/e2e/testutils.go

+package e2e
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os/exec"
+	"strings"
+	"time"
+
+	"golang.org/x/crypto/ssh"
+)
+
+type Node struct {
+	Name       string
+	Status     string
+	Roles      string
+	InternalIP string
+	ExternalIP string
+}
+
+type Pod struct {
+	NameSpace string
+	Name      string
+	Ready     string
+	Status    string
+	Restarts  string
+	NodeIP    string
+	Node      string
+}
+
+var config *ssh.ClientConfig
+var SSHKEY string
+var SSHUSER string
+var err error
+
+func checkError(e error) {
+	if e != nil {
+		log.Fatal(err)
+		panic(e)
+	}
+}
+
+func publicKey(path string) ssh.AuthMethod {
+	key, err := ioutil.ReadFile(path)
+	if err != nil {
+		panic(err)
+	}
+	signer, err := ssh.ParsePrivateKey(key)
+	if err != nil {
+		panic(err)
+	}
+	return ssh.PublicKeys(signer)
+}
+
+func ConfigureSSH(host string, SSHUser string, SSHKey string) *ssh.Client {
+	config = &ssh.ClientConfig{
+		User: SSHUser,
+		Auth: []ssh.AuthMethod{
+			publicKey(SSHKey),
+		},
+		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
+	}
+	conn, err := ssh.Dial("tcp", host, config)
+	checkError(err)
+	return conn
+}
+
+func runsshCommand(cmd string, conn *ssh.Client) string {
+	session, err := conn.NewSession()
+	if err != nil {
+		panic(err)
+	}
+	defer session.Close()
+	var stdoutBuf bytes.Buffer
+	var stderrBuf bytes.Buffer
+	session.Stdout = &stdoutBuf
+	session.Stderr = &stderrBuf
+
+	if err := session.Run(cmd); err != nil {
+		log.Println(session.Stdout)
+		log.Fatal("Error on command execution", err.Error())
+	}
+	return fmt.Sprintf("%s", stdoutBuf.String())
+}
+
+//Runs command passed from within the node ServerIP
+func RunCmdOnNode(cmd string, ServerIP string, SSHUser string, SSHKey string) string {
+	Server := ServerIP + ":22"
+	conn := ConfigureSSH(Server, SSHUser, SSHKey)
+	res := runsshCommand(cmd, conn)
+	res = strings.TrimSpace(res)
+	return res
+}
+
+// RunCommand Runs command on the cluster accessing the cluster through kubeconfig file
+func RunCommand(cmd string) (string, error) {
+	c := exec.Command("bash", "-c", cmd)
+	time.Sleep(10 * time.Second)
+	var out bytes.Buffer
+	c.Stdout = &out
+	err := c.Run()
+	if err != nil {
+		return "", errors.New(fmt.Sprintf("%s", err))
+	}
+	return out.String(), nil
+}
+
+//Used to count the pods using prefix passed in the list of pods
+func CountOfStringInSlice(str string, pods []Pod) int {
+	count := 0
+	for _, pod := range pods {
+		if strings.Contains(pod.Name, str) {
+			count++
+		}
+	}
+	return count
+}
+
+func DeployWorkload(workload string, kubeconfig string) (string, error) {
+	cmd := "kubectl apply -f " + workload + " --kubeconfig=" + kubeconfig
+	return RunCommand(cmd)
+}
+
+func FetchClusterIP(kubeconfig string, servicename string) string {
+	cmd := "kubectl get svc " + servicename + " -o jsonpath='{.spec.clusterIP}' --kubeconfig=" + kubeconfig
+	res, _ := RunCommand(cmd)
+	return res
+}
+
+func FetchNodeExternalIP(kubeconfig string) []string {
+	cmd := "kubectl get node --output=jsonpath='{range .items[*]} { .status.addresses[?(@.type==\"ExternalIP\")].address}' --kubeconfig=" + kubeconfig
+	time.Sleep(10 * time.Second)
+	res, _ := RunCommand(cmd)
+	nodeExternalIP := strings.Trim(res, " ")
+	nodeExternalIPs := strings.Split(nodeExternalIP, " ")
+	return nodeExternalIPs
+}
+func FetchIngressIP(kubeconfig string) []string {
+	cmd := "kubectl get ing  ingress  -o jsonpath='{.status.loadBalancer.ingress[*].ip}' --kubeconfig=" + kubeconfig
+	time.Sleep(10 * time.Second)
+	res, _ := RunCommand(cmd)
+
+	ingressIp := strings.Trim(res, " ")
+	ingressIps := strings.Split(ingressIp, " ")
+	return ingressIps
+}
+
+func ParseNode(kubeconfig string, printres bool) []Node {
+	nodes := make([]Node, 0, 10)
+	var node Node
+	timeElapsed := 0
+	nodeList := ""
+	time.Sleep(60 * time.Second)
+	for timeElapsed < 420 {
+		notReady := false
+		cmd := "kubectl get nodes --no-headers -o wide -A --kubeconfig=" + kubeconfig
+		res, _ := RunCommand(cmd)
+		res = strings.TrimSpace(res)
+		nodeList = res
+		split := strings.Split(res, "\n")
+		for _, rec := range split {
+			fields := strings.Fields(string(rec))
+			node.Name = fields[0]
+			node.Status = fields[1]
+			node.Roles = fields[2]
+			node.InternalIP = fields[5]
+			node.ExternalIP = fields[6]
+			nodes = append(nodes, node)
+			if node.Status != "Ready" {
+				notReady = true
+				break
+			}
+		}
+		if notReady == false {
+			break
+		}
+		time.Sleep(5 * time.Second)
+		timeElapsed = timeElapsed + 10
+	}
+	if printres {
+		fmt.Println(nodeList)
+	}
+	return nodes
+}
+
+func ParsePod(kubeconfig string, printres bool) []Pod {
+	pods := make([]Pod, 0, 10)
+	var pod Pod
+	timeElapsed := 0
+	time.Sleep(60 * time.Second)
+	podList := ""
+	for timeElapsed < 420 {
+		helmPodsNR := false
+		systemPodsNR := false
+		cmd := "kubectl get pods -o wide --no-headers -A --kubeconfig=" + kubeconfig
+		res, _ := RunCommand(cmd)
+		res = strings.TrimSpace(res)
+		podList = res
+
+		split := strings.Split(res, "\n")
+		for _, rec := range split {
+			fields := strings.Fields(string(rec))
+			pod.NameSpace = fields[0]
+			pod.Name = fields[1]
+			pod.Ready = fields[2]
+			pod.Status = fields[3]
+			pod.Restarts = fields[4]
+			pod.NodeIP = fields[6]
+			pod.Node = fields[7]
+			pods = append(pods, pod)
+			if strings.HasPrefix(pod.Name, "helm-install") && pod.Status != "Completed" {
+				helmPodsNR = true
+				break
+			} else if !strings.HasPrefix(pod.Name, "helm-install") && pod.Status != "Running" {
+
+				systemPodsNR = true
+				break
+			}
+			time.Sleep(10 * time.Second)
+			timeElapsed = timeElapsed + 10
+		}
+		if systemPodsNR == false && helmPodsNR == false {
+			break
+		}
+	}
+	if printres {
+		fmt.Println(podList)
+	}
+	return pods
+}