Refactor automation using terraform (#5268)

go.mod

@@ -80,6 +80,7 @@ require (
 	github.com/google/uuid v1.3.0
 	github.com/gorilla/mux v1.8.0
 	github.com/gorilla/websocket v1.4.2
+	github.com/gruntwork-io/terratest v0.40.6
 	github.com/k3s-io/helm-controller v0.12.0
 	github.com/k3s-io/kine v0.8.1
 	github.com/klauspost/compress v1.14.2

tests/terraform/Dockerfile.build

+FROM golang:alpine
+
+ARG TERRAFORM_VERSION=0.12.10
+ENV TERRAFORM_VERSION=$TERRAFORM_VERSION
+
+
+RUN apk update && \
+    apk upgrade --update-cache --available && \
+    apk add curl git jq bash openssh unzip gcc g++ make ca-certificates && \
+
+RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && \
+    chmod +x ./kubectl && \
+    mv ./kubectl /usr/local/bin
+RUN mkdir tmp && \
+    curl "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip" -o tmp/terraform.zip && \
+    unzip tmp/terraform.zip -d /usr/local/bin && \
+    chmod +x /usr/local/bin/terraform && \
+    rm -rf tmp
+
+WORKDIR $GOPATH/src/github.com/k3s-io/k3s
+
+COPY . .
+RUN go get github.com/gruntwork-io/terratest/modules/terraform
+RUN go get -u github.com/onsi/gomega
+RUN go get -u github.com/onsi/ginkgo/v2
+RUN go get -u golang.org/x/crypto/...

tests/terraform/amd64_resource_files/clusterip.yaml

+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: test-clusterip
+spec:
+  selector:
+    matchLabels:
+      k8s-app: nginx-app-clusterip
+  replicas: 2
+  template:
+    metadata:
+      labels:
+        k8s-app: nginx-app-clusterip
+    spec:
+      containers:
+      - name: nginx
+        image: ranchertest/mytestcontainer
+        ports:
+        - containerPort: 80
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    k8s-app: nginx-app-clusterip
+  name: nginx-clusterip-svc
+  namespace: default 
+spec:
+  type: ClusterIP
+  ports:
+    - port: 80
+  selector:
+    k8s-app: nginx-app-clusterip

tests/terraform/amd64_resource_files/daemonset.yaml

+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: test-daemonset
+spec:
+  selector:
+    matchLabels:
+      k8s-app: test-daemonset
+  template:
+    metadata:
+      labels:
+        k8s-app: test-daemonset
+    spec:
+      containers:
+        - name: webserver
+          image: nginx
+          ports:
+          - containerPort: 80

tests/terraform/amd64_resource_files/dnsutils.yaml

+apiVersion: v1
+kind: Pod
+metadata:
+  name: dnsutils
+  namespace: default
+spec:
+  containers:
+  - name: dnsutils
+    image: gcr.io/kubernetes-e2e-test-images/dnsutils:1.3
+    command:
+      - sleep
+      - "3600"
+    imagePullPolicy: IfNotPresent
+  restartPolicy: Always

tests/terraform/amd64_resource_files/ingress.yaml

+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: test-ingress
+spec:
+  rules:
+  - host: foo1.bar.com
+    http:
+      paths:
+      - backend:
+          service:
+            name: nginx-ingress-svc
+            port:
+              number: 80
+        path: /
+        pathType: ImplementationSpecific
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: nginx-ingress-svc
+ labels:
+   k8s-app: nginx-app-ingress
+spec:
+ ports:
+ - port: 80
+   targetPort: 80
+   protocol: TCP
+   name: http
+ selector:
+   k8s-app: nginx-app-ingress
+---
+apiVersion: v1
+kind: ReplicationController
+metadata:
+ name: test-ingress
+spec:
+ replicas: 2
+ selector:
+   k8s-app: nginx-app-ingress
+ template:
+   metadata:
+     labels:
+       k8s-app: nginx-app-ingress
+   spec:
+     terminationGracePeriodSeconds: 60
+     containers:
+     - name: testcontainer
+       image: ranchertest/mytestcontainer
+       ports:
+       - containerPort: 80

tests/terraform/amd64_resource_files/loadbalancer.yaml

+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: test-loadbalancer
+spec:
+  selector:
+    matchLabels:
+      k8s-app: nginx-app-loadbalancer
+  replicas: 2
+  template:
+    metadata:
+      labels:
+        k8s-app: nginx-app-loadbalancer
+    spec:
+      containers:
+      - name: nginx
+        image: ranchertest/mytestcontainer
+        ports:
+        - containerPort: 80
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: nginx-loadbalancer-svc
+ labels:
+   k8s-app: nginx-app-loadbalancer
+spec:
+ type: LoadBalancer
+ ports:
+ - port: 81
+   targetPort: 80
+   protocol: TCP
+   name: http
+ selector:
+   k8s-app: nginx-app-loadbalancer

tests/terraform/amd64_resource_files/local-path-provisioner.yaml

+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: local-path-pvc
+  namespace: default
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: local-path
+  resources:
+    requests:
+      storage: 500Mi
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: volume-test
+  namespace: default
+spec:
+  containers:
+  - name: volume-test
+    image: nginx:stable-alpine
+    imagePullPolicy: IfNotPresent
+    volumeMounts:
+    - name: volv
+      mountPath: /data
+    ports:
+    - containerPort: 80
+  volumes:
+  - name: volv
+    persistentVolumeClaim:
+      claimName: local-path-pvc

tests/terraform/amd64_resource_files/nodeport.yaml

+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: test-nodeport
+spec:
+  selector:
+    matchLabels:
+      k8s-app: nginx-app-nodeport
+  replicas: 2
+  template:
+    metadata:
+      labels:
+        k8s-app: nginx-app-nodeport
+    spec:
+      containers:
+      - name: nginx
+        image: ranchertest/mytestcontainer
+        ports:
+        - containerPort: 80
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    k8s-app: nginx-app-nodeport
+  name: nginx-nodeport-svc
+  namespace: default
+spec:
+  type: NodePort
+  ports:
+    - port: 80
+      nodePort: 30096
+      name: http
+  selector:
+    k8s-app: nginx-app-nodeport

tests/terraform/amd64_resource_files/secrets.yaml

+apiVersion: v1
+kind: Secret
+metadata:
+  name: e2e-secret1
+type: Opaque
+stringData:
+  config.yaml: |
+    key: "hello"
+    val: "world"
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: e2e-secret2
+type: Opaque
+stringData:
+  config.yaml: |
+    key: "good"
+    val: "day"
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: e2e-secret3
+type: Opaque
+stringData:
+  config.yaml: |
+    key: "top-secret"
+    val: "information"
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: e2e-secret4
+type: Opaque
+stringData:
+  config.yaml: |
+    key: "lock"
+    val: "key"
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: e2e-secret5
+type: Opaque
+stringData:
+  config.yaml: |
+    key: "last"
+    val: "call"

tests/terraform/arm_resource_files/clusterip.yaml

+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: test-clusterip
+spec:
+  selector:
+    matchLabels:
+      k8s-app: nginx-app-clusterip
+  replicas: 2
+  template:
+    metadata:
+      labels:
+        k8s-app: nginx-app-clusterip
+    spec:
+      containers:
+      - name: nginx
+        image: shylajarancher19/shylajaarm64:v1.0
+        ports:
+        - containerPort: 80
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    k8s-app: nginx-app-clusterip
+  name: nginx-clusterip-svc
+  namespace: default 
+spec:
+  type: ClusterIP
+  ports:
+    - port: 80
+  selector:
+    k8s-app: nginx-app-clusterip

tests/terraform/arm_resource_files/daemonset.yaml

+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: test-daemonset
+spec:
+  selector:
+    matchLabels:
+      k8s-app: test-daemonset
+  template:
+    metadata:
+      labels:
+        k8s-app: test-daemonset
+    spec:
+      containers:
+        - name: webserver
+          image: nginx
+          ports:
+          - containerPort: 80

tests/terraform/arm_resource_files/ingress.yaml

+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+ name: ingress
+spec:
+ rules:
+ - host: foo1.bar.com
+   http:
+     paths:
+     - path: /name.html
+       backend:
+         serviceName: nginx-ingress-svc
+         servicePort: 80
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: nginx-ingress-svc
+ labels:
+   k8s-app: nginx-app-ingress
+spec:
+ ports:
+ - port: 80
+   targetPort: 80
+   protocol: TCP
+   name: http
+ selector:
+   k8s-app: nginx-app-ingress
+---
+apiVersion: v1
+kind: ReplicationController
+metadata:
+ name: test-ingress
+spec:
+ replicas: 2
+ selector:
+   k8s-app: nginx-app-ingress
+ template:
+   metadata:
+     labels:
+       k8s-app: nginx-app-ingress
+   spec:
+     terminationGracePeriodSeconds: 60
+     containers:
+     - name: testcontainer
+       image: shylajarancher19/shylajaarm64:v1.0
+       ports:
+       - containerPort: 80

tests/terraform/arm_resource_files/loadbalancer.yaml

+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: test-loadbalancer
+spec:
+  selector:
+    matchLabels:
+      k8s-app: nginx-app-loadbalancer
+  replicas: 2
+  template:
+    metadata:
+      labels:
+        k8s-app: nginx-app-loadbalancer
+    spec:
+      containers:
+      - name: nginx
+        image: shylajarancher19/shylajaarm64:v1.0
+        ports:
+        - containerPort: 80
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: nginx-loadbalancer-svc
+ labels:
+   k8s-app: nginx-app-loadbalancer
+spec:
+ type: LoadBalancer
+ ports:
+ - port: 81
+   targetPort: 80
+   protocol: TCP
+   name: http
+ selector:
+   k8s-app: nginx-app-loadbalancer

tests/terraform/arm_resource_files/local-path-provisioner.yaml

+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: local-path-pvc
+  namespace: default
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: local-path
+  resources:
+    requests:
+      storage: 500Mi
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: volume-test
+  namespace: default
+spec:
+  containers:
+  - name: volume-test
+    image: nginx:stable-alpine
+    imagePullPolicy: IfNotPresent
+    volumeMounts:
+    - name: volv
+      mountPath: /data
+    ports:
+    - containerPort: 80
+  volumes:
+  - name: volv
+    persistentVolumeClaim:
+      claimName: local-path-pvc

tests/terraform/arm_resource_files/nodeport.yaml

+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: test-nodeport
+spec:
+  selector:
+    matchLabels:
+      k8s-app: nginx-app-nodeport
+  replicas: 2
+  template:
+    metadata:
+      labels:
+        k8s-app: nginx-app-nodeport
+    spec:
+      containers:
+      - name: nginx
+        image: shylajarancher19/shylajaarm64:v1.0
+        ports:
+        - containerPort: 80
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    k8s-app: nginx-app-nodeport
+  name: nginx-nodeport-svc
+  namespace: default
+spec:
+  type: NodePort
+  ports:
+    - port: 80
+      nodePort: 30096
+      name: http
+  selector:
+    k8s-app: nginx-app-nodeport

tests/terraform/createcluster.go

+package e2e
+
+import (
+	"flag"
+	"fmt"
+	"path/filepath"
+	"testing"
+
+	"github.com/gruntwork-io/terratest/modules/terraform"
+)
+
+var destroy = flag.Bool("destroy", false, "a bool")
+var nodeOs = flag.String("node_os", "centos8", "a string")
+var externalDb = flag.String("external_db", "mysql", "a string")
+var arch = flag.String("arch", "amd64", "a string")
+var clusterType = flag.String("cluster_type", "etcd", "a string")
+var resourceName = flag.String("resource_name", "etcd", "a string")
+var sshuser = flag.String("sshuser", "ubuntu", "a string")
+var sshkey = flag.String("sshkey", "", "a string")
+
+var (
+	kubeConfigFile string
+	masterIPs      string
+	workerIPs      string
+)
+
+func BuildCluster(nodeOs, clusterType, externalDb, resourceName string, t *testing.T, destroy bool) (string, string, string, error) {
+
+	tDir := "./modules/k3scluster"
+	vDir := "/config/" + nodeOs + clusterType + ".tfvars"
+
+	if externalDb != "" {
+		vDir = "/config/" + nodeOs + externalDb + ".tfvars"
+	}
+
+	tfDir, _ := filepath.Abs(tDir)
+	if err != nil {
+		return "", "", "", err
+	}
+	varDir, _ := filepath.Abs(vDir)
+	if err != nil {
+		return "", "", "", err
+	}
+	TerraformOptions := &terraform.Options{
+		TerraformDir: tfDir,
+		VarFiles:     []string{varDir},
+		Vars: map[string]interface{}{
+			"cluster_type":  clusterType,
+			"resource_name": resourceName,
+			"external_db":   externalDb,
+		},
+	}
+
+	if destroy {
+		fmt.Printf("Cluster is being deleted")
+		terraform.Destroy(t, TerraformOptions)
+		return "", "", "", err
+	}
+
+	fmt.Printf("Creating Cluster")
+	terraform.InitAndApply(t, TerraformOptions)
+	kubeconfig := terraform.Output(t, TerraformOptions, "kubeconfig") + "_kubeconfig"
+	masterIPs := terraform.Output(t, TerraformOptions, "master_ips")
+	workerIPs := terraform.Output(t, TerraformOptions, "worker_ips")
+	kubeconfigFile := "/config/" + kubeconfig
+	return kubeconfigFile, masterIPs, workerIPs, err
+}

tests/terraform/jenkinsfile

+pipeline {
+    agent any
+        environment {
+            setupResultsOut = "setup-results.xml"
+            testResultsOut = "results.xml"
+
+            AWS_ACCESS_KEY_ID     = credentials('AWS_ACCESS_KEY_ID')
+            AWS_SECRET_ACCESS_KEY = credentials('AWS_SECRET_ACCESS_KEY')
+            AWS_SSH_PEM_KEY = credentials('AWS_SSH_PEM_KEY')
+        }
+
+        stages {
+            stage('Git Checkout') {
+                steps {
+                    git branch: 'add_automation_using_tf', url: 'https://github.com/ShylajaDevadiga/k3s.git'
+                    script {
+                        dir("${WORKSPACE}/tests/terraform") {
+                            if (env.AWS_SSH_PEM_KEY && env.AWS_SSH_KEY_NAME) {
+                                    def decoded = new String(AWS_SSH_PEM_KEY.decodeBase64())
+                                    writeFile file: AWS_SSH_KEY_NAME, text: decoded
+                            }
+                        }
+                    }
+                }
+            }
+
+            stage('Configure') {
+                steps {
+                    sh """
+                    set -e -x
+                    echo 'aws_ami="${env.AWS_AMI}"
+                        aws_user="${env.AWS_USER}"
+                        region="${env.REGION}"
+                        vpc_id="${env.VPC_ID}"
+                        subnets="${env.SUBNETS}"
+                        qa_space="${env.QA_SPACE}"
+                        ec2_instance_class="${env.EC2_INSTANCE_CLASS}"
+                        access_key="/config/$AWS_SSH_KEY_NAME"
+                        no_of_worker_nodes="${env.NO_OF_WORKER_NODES}"
+                        key_name="jenkins-rke-validation"
+                        server_flags="${env.SERVER_FLAGS}"
+                        worker_flags="${env.WORKER_FLAGS}"
+                        k3s_version="${env.K3S_VERSION}"
+                        availability_zone="${env.AVAILABILITY_ZONE}"
+                        sg_id="${env.SG_ID}"
+                        install_mode="${env.INSTALL_MODE}"
+                        resource_name="${env.RESOURCE_NAME}"
+                        no_of_server_nodes="${env.NO_OF_SERVER_NODES}"
+                        username="${env.RHEL_USERNAME}"
+                        password="${env.RHEL_PASSWORD}"
+                        db_username="${env.DB_USERNAME}"
+                        db_password="${env.DB_PASSWORD}"
+                        node_os="${env.NODE_OS}"
+                        environment="${env.ENVIRONMENT}"
+                        engine_mode="${env.ENGINE_MODE}"
+                        external_db="${env.EXTERNAL_DB}"
+                        external_db_version="${env.EXTERNAL_DB_VERSION}"
+                        instance_class="${env.DB_INSTANCE_CLASS}"
+                        db_group_name="${env.DB_GROUP_NAME}"
+                        cluster_type="${env.CLUSTER_TYPE}"
+                        create_lb=${env.CREATE_LB}
+                        ' >${WORKSPACE}/tests/terraform/${env.NODE_OS}${env.EXTERNAL_DB}".tfvars"
+                    """
+                }
+            }
+            stage('Build Cluster') {
+                steps {
+                    sh """
+
+                        /usr/bin/docker build -f tests/terraform/Dockerfile.build -t k3s_create_cluster .
+
+                        /usr/bin/docker run -d --name ${RESOURCE_NAME}_${BUILD_NUMBER} -v ${WORKSPACE}/tests/terraform:/config \
+                        -t -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
+                        k3s_create_cluster
+
+                        /usr/bin/docker cp "${WORKSPACE}/tests/terraform/${NODE_OS}${EXTERNAL_DB}".tfvars "${RESOURCE_NAME}_${BUILD_NUMBER}":/config
+
+                        /usr/bin/docker cp "${WORKSPACE}/tests/terraform/$AWS_SSH_KEY_NAME" "${RESOURCE_NAME}_${BUILD_NUMBER}":/config
+
+                        /usr/bin/docker exec ${RESOURCE_NAME}_${BUILD_NUMBER} /usr/local/go/bin/go test -v tests/terraform/createcluster.go \
+                        tests/terraform/createcluster_test.go tests/terraform/testutils.go -v \
+                        -timeout=2h -node_os=${NODE_OS} \
+                        -cluster_type=${CLUSTER_TYPE} -external_db=${EXTERNAL_DB} -resource_name=${RESOURCE_NAME} \
+                        -sshuser=${AWS_USER} -sshkey="/config/${AWS_SSH_KEY_NAME}" -destroy=false -arch=${ARCH}
+
+                    """
+                }
+            }
+
+            stage('Test Report') {
+                steps {
+                    sh """
+                        /usr/bin/docker rm -f ${RESOURCE_NAME}_${BUILD_NUMBER}
+                        /usr/bin/docker rmi -f k3s_create_cluster
+                    """
+                }
+            }
+        }
+    }
\ No newline at end of file

tests/terraform/modules/k3scluster/install_k3s_master.sh

+#!/bin/bash
+
+mkdir -p /etc/rancher/k3s
+cat << EOF >/etc/rancher/k3s/config.yaml
+write-kubeconfig-mode: "0644"
+tls-san:
+  - ${2}
+EOF
+
+if [[ -n "$8" ]] && [[ "$8" == *":"* ]]
+then
+   echo "$"
+   echo -e "$8" >> /etc/rancher/k3s/config.yaml
+   cat /etc/rancher/k3s/config.yaml
+fi
+
+if [ "${1}" = "rhel" ]
+then
+   subscription-manager register --auto-attach --username="${9}" --password="${10}"
+   subscription-manager repos --enable=rhel-7-server-extras-rpms
+fi
+
+export "${3}"="${4}"
+
+if [ "${5}" = "etcd" ]
+then
+   echo "CLUSTER TYPE  is etcd"
+   if [[ "$4" == *"v1.18"* ]] || [["$4" == *"v1.17"* ]] && [[ -n "$8" ]]
+   then
+       echo "curl -sfL https://get.k3s.io | INSTALL_K3S_TYPE='server' sh -s - --cluster-init --node-external-ip=${6} $8" >/tmp/master_cmd
+       curl -sfL https://get.k3s.io | INSTALL_K3S_TYPE='server' sh -s - --cluster-init --node-external-ip="${6}" "$8"
+   else
+       echo "curl -sfL https://get.k3s.io | INSTALL_K3S_TYPE='server' sh -s - --cluster-init --node-external-ip=${6}" >/tmp/master_cmd
+       curl -sfL https://get.k3s.io | INSTALL_K3S_TYPE='server' sh -s - --cluster-init --node-external-ip="${6}"
+   fi
+else
+   echo "CLUSTER TYPE is external db"
+   echo "$8"
+   if [[ "$4" == *"v1.18"* ]] || [[ "$4" == *"v1.17"* ]] && [[ -n "$8" ]]
+   then
+       echo "curl -sfL https://get.k3s.io | sh -s - server --node-external-ip=${6} --datastore-endpoint=\"${7}\" $8"  >/tmp/master_cmd
+       curl -sfL https://get.k3s.io | sh -s - server --node-external-ip="${6}" --datastore-endpoint="${7}" "$8"
+   else
+       echo "curl -sfL https://get.k3s.io | sh -s - server --node-external-ip=${6}  --datastore-endpoint=\"${7}\" "  >/tmp/master_cmd
+       curl -sfL https://get.k3s.io | sh -s - server --node-external-ip="${6}" --datastore-endpoint="${7}"
+   fi
+fi
+
+export PATH=$PATH:/usr/local/bin
+timeElapsed=0
+while ! $(kubectl get nodes >/dev/null 2>&1) && [[ $timeElapsed -lt 300 ]]
+do
+   sleep 5
+   timeElapsed=$(expr $timeElapsed + 5)
+done
+
+IFS=$'\n'
+timeElapsed=0
+sleep 10
+while [[ $timeElapsed -lt 420 ]]
+do
+   notready=false
+   for rec in $(kubectl get nodes)
+   do
+      if [[ "$rec" == *"NotReady"* ]]
+      then
+         notready=true
+      fi
+  done
+  if [[ $notready == false ]]
+  then
+     break
+  fi
+  sleep 20
+  timeElapsed=$(expr $timeElapsed + 20)
+done
+
+IFS=$'\n'
+timeElapsed=0
+while [[ $timeElapsed -lt 420 ]]
+do
+   helmPodsNR=false
+   systemPodsNR=false
+   for rec in $(kubectl get pods -A --no-headers)
+   do
+      if [[ "$rec" == *"helm-install"* ]] && [[ "$rec" != *"Completed"* ]]
+      then
+         helmPodsNR=true
+      elif [[ "$rec" != *"helm-install"* ]] && [[ "$rec" != *"Running"* ]]
+      then
+         systemPodsNR=true
+      else
+         echo ""
+      fi
+   done
+
+   if [[ $systemPodsNR == false ]] && [[ $helmPodsNR == false ]]
+   then
+      break
+   fi
+   sleep 20
+   timeElapsed=$(expr $timeElapsed + 20)
+done
+cat /etc/rancher/k3s/config.yaml> /tmp/joinflags
+cat /var/lib/rancher/k3s/server/node-token >/tmp/nodetoken
+cat /etc/rancher/k3s/k3s.yaml >/tmp/config

tests/terraform/modules/k3scluster/join_k3s_agent.sh

+#!/bin/bash
+# This script is used to join one or more nodes as agents
+
+mkdir -p /etc/rancher/k3s
+cat <<EOF >>/etc/rancher/k3s/config.yaml
+server: https://${4}:6443
+token:  "${5}"
+EOF
+
+if [[ ! -z "$7" ]] && [[ "$7" == *":"* ]]
+then
+   echo -e "$7" >> /etc/rancher/k3s/config.yaml
+   cat /etc/rancher/k3s/config.yaml
+fi
+
+if [ ${1} = "rhel" ]
+then
+    subscription-manager register --auto-attach --username=${8} --password=${9}
+    subscription-manager repos --enable=rhel-7-server-extras-rpms
+fi
+
+export "${2}"="${3}"
+  if [[ "$3" == *"v1.18"* ]] || [["$3" == *"v1.17"* ]] && [[ -n "$7" ]]
+then
+  echo "curl -sfL https://get.k3s.io | sh -s - agent --node-external-ip=${6} $7" >/tmp/agent_cmd
+curl -sfL https://get.k3s.io | sh -s - agent --node-external-ip=${6} ${7}
+  else
+
+echo "curl -sfL https://get.k3s.io | sh -s - agent --node-external-ip=${6}" >/tmp/agent_cmd
+curl -sfL https://get.k3s.io | sh -s - agent --node-external-ip=${6}
+fi

tests/terraform/modules/k3scluster/join_k3s_master.sh

+#!/bin/bash
+# This script is used to join one or more nodes as masters
+
+mkdir -p /etc/rancher/k3s
+cat <<EOF >>/etc/rancher/k3s/config.yaml
+write-kubeconfig-mode: "0644"
+tls-san:
+  - ${2}
+EOF
+
+if [[ -n "${10}" ]] && [[ "${10}" == *":"* ]]
+then
+   echo -e "${10}" >> /etc/rancher/k3s/config.yaml
+   cat /etc/rancher/k3s/config.yaml
+fi
+
+if [ "${1}" = "rhel" ]
+then
+   subscription-manager register --auto-attach --username="${11}" --password="${12}"
+   subscription-manager repos --enable=rhel-7-server-extras-rpms
+fi
+
+export "${3}"="${4}"
+
+if [ "${5}" = "etcd" ]
+then
+   if [[ "$4" == *"v1.18"* ]] || [["$4" == *"v1.17"* ]] && [[ -n "$10" ]]
+   then
+      echo "curl -sfL https://get.k3s.io | INSTALL_K3S_TYPE='server' sh -s - --server https://\"${7}\":6443 --token \"${8}\" --node-external-ip=\"${6}\" ${10}" >/tmp/master_cmd
+      curl -sfL https://get.k3s.io | INSTALL_K3S_TYPE='server' sh -s - --server https://"${7}":6443 --token "${8}" --node-external-ip="${6} ${10}"
+   else
+        echo "curl -sfL https://get.k3s.io | INSTALL_K3S_TYPE='server' sh -s - --server https://\"${7}\":6443 --token \"${8}\" --node-external-ip=\"${6}\"" >/tmp/master_cmd
+       curl -sfL https://get.k3s.io | INSTALL_K3S_TYPE='server' sh -s - --server https://"${7}":6443 --token "${8}" --node-external-ip="${6}"
+   fi
+else
+  if [[ "$4" == *"v1.18"* ]] || [["$4" == *"v1.17"* ]] && [[ -n "$10" ]]
+  then
+      echo "curl -sfL https://get.k3s.io | INSTALL_K3S_TYPE='server' sh -s - --node-external-ip=\"${6}\" --datastore-endpoint=\"${9}\" ${10}" >/tmp/master_cmd
+      curl -sfL https://get.k3s.io | INSTALL_K3S_TYPE='server' sh -s - --node-external-ip="${6}" --token="${8}" --datastore-endpoint="${9} ${10}"
+   else
+      echo "curl -sfL https://get.k3s.io | INSTALL_K3S_TYPE='server' sh -s - --node-external-ip=\"${6}\" --token \"${8}\" --datastore-endpoint=\"${9}\"" >/tmp/master_cmd
+      curl -sfL https://get.k3s.io | INSTALL_K3S_TYPE='server' sh -s - --node-external-ip="${6}" --token="${8}" --datastore-endpoint="${9}"
+    fi
+fi

tests/terraform/modules/k3scluster/main.tf

+module "master" {
+   source="./master"
+   aws_ami=var.aws_ami
+   aws_user=var.aws_user
+   key_name=var.key_name
+   no_of_server_nodes=var.no_of_server_nodes
+   k3s_version=var.k3s_version
+   install_mode=var.install_mode
+   region=var.region
+   vpc_id=var.vpc_id
+   subnets=var.subnets
+   qa_space=var.qa_space
+   ec2_instance_class=var.ec2_instance_class
+   access_key=var.access_key
+   cluster_type=var.cluster_type
+   server_flags=var.server_flags
+   availability_zone=var.availability_zone
+   sg_id=var.sg_id
+   resource_name=var.resource_name
+   node_os=var.node_os
+   username=var.username
+   password=var.password
+   db_username=var.db_username
+   db_password=var.db_password
+   db_group_name=var.db_group_name
+   external_db=var.external_db
+   instance_class=var.instance_class
+   external_db_version=var.external_db_version
+   engine_mode=var.engine_mode
+   environment=var.environment
+   create_lb=var.create_lb
+}
+module "worker" {
+   source="./worker"
+   dependency = module.master
+   aws_ami=var.aws_ami
+   aws_user=var.aws_user
+   key_name=var.key_name
+   no_of_worker_nodes=var.no_of_worker_nodes
+   k3s_version=var.k3s_version
+   install_mode=var.install_mode
+   region=var.region
+   vpc_id=var.vpc_id
+   subnets=var.subnets
+   ec2_instance_class=var.ec2_instance_class
+   access_key=var.access_key
+   worker_flags=var.worker_flags
+   availability_zone=var.availability_zone
+   sg_id=var.sg_id
+   resource_name=var.resource_name
+   node_os=var.node_os
+   username=var.username
+   password=var.password
+}

tests/terraform/modules/k3scluster/master/outputs.tf

+output "Route53_info" {
+  value       = aws_route53_record.aws_route53.*
+  description = "List of DNS records"
+}
+
+output "master_ips" {
+  value = join("," , aws_instance.master.*.public_ip,aws_instance.master2-ha.*.public_ip)
+  description = "The public IP of the AWS node"
+}
+
+output "kubeconfig" {
+  value = var.resource_name
+  description = "kubeconfig of the cluster created"
+}

tests/terraform/modules/k3scluster/master/providers.tf

+provider "aws" {
+    region = "${var.region}"
+}

tests/terraform/modules/k3scluster/master/variables.tf

+variable "aws_ami" {}
+variable "aws_user" {}
+variable "region" {}
+variable "access_key" {}
+variable "vpc_id" {}
+variable "subnets" {}
+variable "availability_zone" {}
+variable "sg_id" {}
+variable "qa_space" {}
+variable "ec2_instance_class" {}
+variable "resource_name" {}
+variable "key_name" {}
+
+variable "external_db" {}
+variable "external_db_version" {}
+variable "instance_class" {}
+
+variable "db_group_name" {}
+variable "username" {}
+variable "password" {}
+variable "k3s_version" {}
+variable "no_of_server_nodes" {}
+variable "server_flags" {}
+
+variable "cluster_type" {}
+variable "node_os" {}
+variable "db_username" {}
+variable "db_password" {}
+variable "environment" {}
+variable "engine_mode" {}
+variable "install_mode" {}
+
+variable "create_lb" {
+  description = "Create Network Load Balancer if set to true"
+  type = bool
+}

tests/terraform/modules/k3scluster/outputs.tf

+output "master_ips" {
+  value       = module.master.master_ips
+  description = "The public IP of the AWS node"
+}
+
+output "worker_ips" {
+  value       = module.worker.worker_ips
+  description = "The public IP of the AWS node"
+}
+
+output "kubeconfig" {
+  value = module.master.kubeconfig
+  description = "kubeconfig of the cluster created"
+}

tests/terraform/modules/k3scluster/providers.tf

+provider "aws" {
+    region = "${var.region}"
+}

tests/terraform/modules/k3scluster/variables.tf

+#variable "db" {}
+variable "no_of_worker_nodes" {}
+variable "aws_ami" {}
+variable "aws_user" {}
+variable "region" {}
+variable "access_key" {}
+variable "vpc_id" {}
+variable "subnets" {}
+variable "qa_space" {}
+variable "resource_name" {}
+variable "key_name" {}
+variable "external_db" {}
+variable "external_db_version" {}
+variable "instance_class" {}
+variable "ec2_instance_class" {}
+variable "db_group_name" {}
+variable "username" {}
+variable "password" {}
+variable "k3s_version" {}
+variable "no_of_server_nodes" {}
+variable "server_flags" {}
+variable "worker_flags" {}
+variable "availability_zone" {}
+variable "sg_id" {}
+variable "cluster_type" {}
+variable "node_os" {}
+variable "db_username" {}
+variable "db_password" {}
+variable "environment" {}
+variable "engine_mode" {}
+variable "install_mode" {}
+variable "create_lb" {
+  description = "Create Network Load Balancer if set to true"
+  type = bool
+}
\ No newline at end of file

tests/terraform/modules/k3scluster/worker/instances_worker.tf

+resource "aws_instance" "worker" {
+  depends_on = [
+    var.dependency
+  ]
+  ami                    = var.aws_ami
+  instance_type          = var.ec2_instance_class
+  count                  = var.no_of_worker_nodes
+  connection {
+    type                 = "ssh"
+    user                 = var.aws_user
+    host                 = self.public_ip
+    private_key          = file(var.access_key)
+  }
+  subnet_id              = var.subnets
+  availability_zone      = var.availability_zone
+  vpc_security_group_ids = [var.sg_id]
+  key_name               = var.key_name
+  tags = {
+    Name                 = "${var.resource_name}-worker"
+  }
+  provisioner "file" {
+    source = "join_k3s_agent.sh"
+    destination = "/tmp/join_k3s_agent.sh"
+  }
+  provisioner "remote-exec" {
+    inline = [
+      "chmod +x /tmp/join_k3s_agent.sh",
+      "sudo /tmp/join_k3s_agent.sh ${var.node_os} ${var.install_mode} ${var.k3s_version} ${local.master_ip} ${local.node_token} ${self.public_ip} \"${var.worker_flags}\" ${var.username} ${var.password} ",
+    ]
+  }
+}
+
+data "local_file" "master_ip" {
+  depends_on = [var.dependency]
+  filename = "/tmp/${var.resource_name}_master_ip"
+}
+
+locals {
+  master_ip = trimspace(data.local_file.master_ip.content)
+}
+
+data "local_file" "token" {
+  depends_on = [var.dependency]
+  filename = "/tmp/${var.resource_name}_nodetoken"
+}
+
+locals {
+  node_token = trimspace(data.local_file.token.content)
+}

tests/terraform/modules/k3scluster/worker/outputs.tf

+output "Registration_address" {
+  value = "${data.local_file.master_ip.content}"
+}
+
+output "master_node_token" {
+  value = "${data.local_file.token.content}"
+}
+
+output "worker_ips" {
+  value = join("," , aws_instance.worker.*.public_ip)
+  description = "The public IP of the AWS node"
+}

tests/terraform/modules/k3scluster/worker/providers.tf

+provider "aws" {
+    region = "${var.region}"
+}

tests/terraform/modules/k3scluster/worker/variables.tf

+variable "dependency" {
+  type    = any
+  default = null
+}
+variable "region" {}
+variable "aws_ami" {}
+variable "aws_user" {}
+variable "vpc_id" {}
+variable "subnets" {}
+variable "resource_name" {}
+variable "access_key" {}
+variable "k3s_version" {}
+variable "no_of_worker_nodes" {}
+variable "worker_flags" {}
+variable "ec2_instance_class" {}
+variable "availability_zone" {}
+variable "sg_id" {}
+variable "username" {}
+variable "password" {}
+variable "node_os" {}
+variable "install_mode" {}
+variable "key_name" {}

tests/terraform/scripts/build.sh

+/usr/local/bin/docker build -f Dockerfile.build -t k3s_create_cluster .

tests/terraform/testutils.go

+package e2e
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"golang.org/x/crypto/ssh"
+)
+
+type Node struct {
+	Name       string
+	Status     string
+	Roles      string
+	InternalIP string
+	ExternalIP string
+}
+
+type Pod struct {
+	NameSpace string
+	Name      string
+	Ready     string
+	Status    string
+	Restarts  string
+	NodeIP    string
+	Node      string
+}
+
+var config *ssh.ClientConfig
+var SSHKEY string
+var SSHUSER string
+var err error
+
+func checkError(e error) {
+	if e != nil {
+		log.Fatal(err)
+		panic(e)
+	}
+}
+
+func publicKey(path string) ssh.AuthMethod {
+	key, err := ioutil.ReadFile(path)
+	if err != nil {
+		panic(err)
+	}
+	signer, err := ssh.ParsePrivateKey(key)
+	if err != nil {
+		panic(err)
+	}
+	return ssh.PublicKeys(signer)
+}
+
+func ConfigureSSH(host string, SSHUser string, SSHKey string) *ssh.Client {
+	config = &ssh.ClientConfig{
+		User: SSHUser,
+		Auth: []ssh.AuthMethod{
+			publicKey(SSHKey),
+		},
+		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
+	}
+	conn, err := ssh.Dial("tcp", host, config)
+	checkError(err)
+	return conn
+}
+
+func runsshCommand(cmd string, conn *ssh.Client) (string, error) {
+	session, err := conn.NewSession()
+	if err != nil {
+		panic(err)
+	}
+	defer session.Close()
+	var stdoutBuf bytes.Buffer
+	var stderrBuf bytes.Buffer
+	session.Stdout = &stdoutBuf
+	session.Stderr = &stderrBuf
+	return fmt.Sprintf("%s", stdoutBuf.String()), err
+}
+
+// nodeOs: ubuntu centos7 centos8 sles15
+// clusterType arm, etcd externaldb, if external_db var is not "" picks database from the vars file,
+// resourceName: name to resource created timestamp attached
+
+// RunCmdOnNode executes a command from within the given node
+func RunCmdOnNode(cmd string, ServerIP string, SSHUser string, SSHKey string) (string, error) {
+	Server := ServerIP + ":22"
+	fmt.Println(Server, SSHUser, SSHKey)
+	conn := ConfigureSSH(Server, SSHUser, SSHKey)
+	res, err := runsshCommand(cmd, conn)
+	res = strings.TrimSpace(res)
+	return res, err
+}
+
+// RunCommand executes a command on the host
+func RunCommand(cmd string) (string, error) {
+	c := exec.Command("bash", "-c", cmd)
+	out, err := c.CombinedOutput()
+	return string(out), err
+}
+
+//Used to count the pods using prefix passed in the list of pods
+func CountOfStringInSlice(str string, pods []Pod) int {
+	count := 0
+	for _, pod := range pods {
+		if strings.Contains(pod.Name, str) {
+			count++
+		}
+	}
+	return count
+}
+
+func DeployWorkload(workload, kubeconfig string, arch bool) (string, error) {
+	resourceDir := "./amd64_resource_files"
+	if arch {
+		resourceDir = "./arm64_resource_files"
+	}
+	files, err := ioutil.ReadDir(resourceDir)
+	if err != nil {
+		err = fmt.Errorf("%s : Unable to read resource manifest file for %s", err, workload)
+		return "", err
+	}
+	fmt.Println("\nDeploying", workload)
+	for _, f := range files {
+		filename := filepath.Join(resourceDir, f.Name())
+		if strings.TrimSpace(f.Name()) == workload {
+			cmd := "kubectl apply -f " + filename + " --kubeconfig=" + kubeconfig
+			fmt.Println(cmd)
+			return RunCommand(cmd)
+		}
+	}
+	return "", nil
+}
+
+func FetchClusterIP(kubeconfig string, servicename string) (string, error) {
+	cmd := "kubectl get svc " + servicename + " -o jsonpath='{.spec.clusterIP}' --kubeconfig=" + kubeconfig
+	fmt.Println(cmd)
+	return RunCommand(cmd)
+}
+
+func FetchNodeExternalIP(kubeconfig string) []string {
+	cmd := "kubectl get node --output=jsonpath='{range .items[*]} { .status.addresses[?(@.type==\"ExternalIP\")].address}' --kubeconfig=" + kubeconfig
+	time.Sleep(10 * time.Second)
+	res, _ := RunCommand(cmd)
+	nodeExternalIP := strings.Trim(res, " ")
+	nodeExternalIPs := strings.Split(nodeExternalIP, " ")
+	return nodeExternalIPs
+}
+
+func FetchIngressIP(kubeconfig string) ([]string, error) {
+	cmd := "kubectl get ingress -o jsonpath='{.items[0].status.loadBalancer.ingress[*].ip}' --kubeconfig=" + kubeconfig
+	fmt.Println(cmd)
+	res, err := RunCommand(cmd)
+	if err != nil {
+		return nil, err
+	}
+	ingressIP := strings.Trim(res, " ")
+	fmt.Println(ingressIP)
+	ingressIPs := strings.Split(ingressIP, " ")
+	return ingressIPs, nil
+}
+
+func ParseNodes(kubeConfig string, print bool) ([]Node, error) {
+	nodes := make([]Node, 0, 10)
+	nodeList := ""
+
+	cmd := "kubectl get nodes --no-headers -o wide -A --kubeconfig=" + kubeConfig
+	res, err := RunCommand(cmd)
+
+	if err != nil {
+		return nil, err
+	}
+	nodeList = strings.TrimSpace(res)
+	split := strings.Split(nodeList, "\n")
+	for _, rec := range split {
+		if strings.TrimSpace(rec) != "" {
+			fields := strings.Fields(rec)
+			node := Node{
+				Name:       fields[0],
+				Status:     fields[1],
+				Roles:      fields[2],
+				InternalIP: fields[5],
+				ExternalIP: fields[6],
+			}
+			nodes = append(nodes, node)
+		}
+	}
+	if print {
+		fmt.Println(nodeList)
+	}
+	return nodes, nil
+}
+
+func ParsePods(kubeconfig string, print bool) ([]Pod, error) {
+	pods := make([]Pod, 0, 10)
+	podList := ""
+
+	cmd := "kubectl get pods -o wide --no-headers -A --kubeconfig=" + kubeconfig
+	res, _ := RunCommand(cmd)
+	res = strings.TrimSpace(res)
+	podList = res
+
+	split := strings.Split(res, "\n")
+	for _, rec := range split {
+		fields := strings.Fields(string(rec))
+		pod := Pod{
+			NameSpace: fields[0],
+			Name:      fields[1],
+			Ready:     fields[2],
+			Status:    fields[3],
+			Restarts:  fields[4],
+			NodeIP:    fields[6],
+			Node:      fields[7],
+		}
+		pods = append(pods, pod)
+	}
+	if print {
+		fmt.Println(podList)
+	}
+	return pods, nil
+}