Remove terraform test package (#8136)

tests/TESTING.md

@@ -6,6 +6,7 @@ Testing in K3s comes in 5 forms:
 - [Smoke](#smoke-tests)
 - [Performance](#performance)
 - [End-to-End (E2E)](#end-to-end-e2e-tests)
+- [Distros-test-framework](#distros-test-framework)
 
 This document will explain *when* each test should be written and *how* each test should be
 generated, formatted, and run.
@@ -146,6 +147,13 @@ See [e2e/README.md](./e2e/README.md) for more info.
 
 ___
 
+## Distros Test Framework
+
+The acceptance tests from distros test framework are a customizable way to create clusters and perform validations on them such that the requirements of specific features and functions can be validated.
+
+See [distros-test-framework/README](https://github.com/rancher/distros-test-framework#readme) for more info.
+___
+
 ## Contributing New Or Updated Tests
 
 We gladly accept new and updated tests of all types. If you wish to create

tests/terraform/.golangci.yaml

-linters:
-  enable:
-    - gofmt
-    - govet
-    - revive
-    - gosec
-    - megacheck
-    - misspell
-    - unparam
-    - exportloopref
-    - nlreturn
-    - nestif
-    - dupl
-    - gci
-    - ginkgolinter
-
-linters-settings:
-  govet:
-    check-shadowing: true
-    check-tests: true
-
-  nestif:
-    min-complexity: 4
-
-  revive:
-    confidence: 0.8
-    severity: warning
-    ignore-generated-header: true
-    rules:
-      - name: line-length-limit
-        arguments: [100]
-      - name: cognitive-complexity
-        arguments: [10]
-      - name: empty-lines
-      - name: empty-block
-      - name: bare-return
-      - name: blank-imports
-      - name: confusing-naming
-      - name: confusing-results
-      - name: context-as-argument
-      - name: duplicated-imports
-      - name: early-return
-      - name: empty-block
-      - name: empty-lines
-      - name: error-naming
-      - name: error-return
-      - name: error-strings
-      - name: errorf
-      - name: exported
-      - name: flag-parameter
-      - name: get-return
-      - name: if-return
-      - name: increment-decrement
-      - name: indent-error-flow
-      - name: import-shadowing
-      - name: modifies-parameter
-      - name: modifies-value-receiver
-      - name: range
-      - name: range-val-in-closure
-      - name: range-val-address
-      - name: receiver-naming
-      - name: string-of-int
-      - name: struct-tag
-      - name: superfluous-else
-      - name: time-naming
-      - name: var-declaration
-      - name: unconditional-recursion
-      - name: unexported-naming
-      - name: unexported-return
-      - name: unhandled-error
-        arguments: ["fmt.Printf", "builder.WriteString"]
-      - name: unnecessary-stmt
-      - name: unreachable-code
-      - name: unused-parameter
-      - name: unused-receiver
-
-issues:
-  exclude-rules:
-    - linters: [typecheck]
-      text: "command-line-arguments"
\ No newline at end of file

tests/terraform/Dockerfile.build

-FROM golang:alpine
-
-ARG TF_VERSION=1.4.0
-ENV TERRAFORM_VERSION $TF_VERSION
-
-RUN apk update && \
-    apk upgrade --update-cache --available && \
-    apk add --no-cache curl git jq bash openssh unzip gcc g++ make ca-certificates && \
-    if [ "$(uname -m)" = "aarch64" ]; then \
-        KUBE_ARCH="linux/arm64" && \
-        TF_ARCH="linux_arm64"; \
-    else \
-        KUBE_ARCH="linux/amd64" && \
-        TF_ARCH="linux_amd64"; \
-    fi && \
-    curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/${KUBE_ARCH}/kubectl" && \
-    chmod +x ./kubectl && \
-    mv ./kubectl /usr/local/bin && \
-    mkdir tmp && \
-    curl "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_${TF_ARCH}.zip" -o tmp/terraform.zip && \
-    unzip tmp/terraform.zip -d /usr/local/bin && \
-    chmod +x /usr/local/bin/terraform && \
-    rm -rf tmp
-
-WORKDIR $GOPATH/src/github.com/k3s-io/k3s
-
-COPY . .
\ No newline at end of file

tests/terraform/Makefile

-
-##========================= Terraform Tests =========================#
-include ./config.mk
-
-TAGNAME ?= default
-tf-up:
-	@cd ../.. && docker build . -q -f ./tests/terraform/Dockerfile.build -t k3s-tf-${TAGNAME}
-
-.PHONY: tf-run
-tf-run:
-	@docker run -d --name k3s-tf-test${IMGNAME} -t \
-      -e AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID}" \
-      -e AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY}" \
-      -v ${ACCESS_KEY_LOCAL}:/go/src/github.com/k3s-io/k3s/tests/terraform/modules/k3scluster/config/.ssh/aws_key.pem \
-      k3s-tf-${TAGNAME} sh -c 'cd ./tests/terraform ; \
-      				  if [ -n "${ARGNAME}" ]; then \
-                         go test -v -timeout=45m \
-                           ./${TESTDIR}/... \
-                           -"${ARGNAME}"="${ARGVALUE}"; \
-                       elif [ -z "${TESTDIR}" ]; then \
-                         go test -v -timeout=45m \
-                           ./createcluster/...; \
-                       else \
-                         go test -v -timeout=45m \
-                           ./${TESTDIR}/...; \
-                       fi'
-
-.PHONY: tf-logs
-tf-logs:
-	@docker logs -f k3s-tf-test${IMGNAME}
-
-.PHONY: tf-down
-tf-down:
-	@echo "Removing containers and images"
-	@docker stop $$(docker ps -a -q --filter="name=k3s-tf*")
-	@docker rm $$(docker ps -a -q --filter="name=k3s-tf*") ; \
-	 docker rmi --force $$(docker images -q --filter="reference=k3s-tf*")
-
-tf-clean:
-	@./scripts/delete_resources.sh
-
-.PHONY: tf-complete
-tf-complete: tf-clean tf-down tf-remove-state tf-up tf-run
-
-
-#========================= Run terraform tests locally =========================#
-
-.PHONY: tf-create
-tf-create:
-	@go test -timeout=45m -v ./createcluster/...
-
-.PHONY: tf-upgrade
-tf-upgrade:
-	@go test -timeout=45m -v ./upgradecluster/... -${ARGNAME}=${ARGVALUE}
-
-.PHONY: tf-remove-state
-tf-remove-state:
-	@rm -rf ./modules/k3scluster/.terraform
-	@rm -rf ./modules/k3scluster/.terraform.lock.hcl ./modules/k3scluster/terraform.tfstate ./modules/k3scluster/terraform.tfstate.backup
-
-.PHONY: tf-test-suite
-tf-test-suite:
-	@make tf-remove-state && make tf-create ; sleep 5 && \
-	  make tf-remove-state && make tf-upgrade ${ARGNAME}=${ARGVALUE}
-
-.PHONY: tf-test-suite-same-cluster
-tf-test-suite-same-cluster:
-	@make tf-create ; sleep 5 && make v ${ARGNAME}=${ARGVALUE}
-
-
-#========================= TestCode Static Quality Check =========================#
-.PHONY: vet-lint                   ## Run locally only inside Tests package
-vet-lint:
-	@echo "Running go vet and lint"
-	@go vet ./${TESTDIR} && golangci-lint run --tests
\ No newline at end of file

tests/terraform/README.md

-# Terraform (TF) Tests
-
-Terraform (TF) tests are an additional form of End-to-End (E2E) tests that cover multi-node K3s configuration and administration: install, update, teardown, etc. across a wide range of operating systems. Terraform tests are used as part of K3s quality assurance (QA) to bring up clusters with different configurations on demand, perform specific functionality tests, and keep them up and running to perform some exploratory tests in real-world scenarios.
-
-## Framework 
-TF tests utilize [Ginkgo](https://onsi.github.io/ginkgo/) and [Gomega](https://onsi.github.io/gomega/) like the e2e tests. They rely on [Terraform](https://www.terraform.io/) to provide the underlying cluster configuration. 
-
-## Format
-
-- All TF tests should be placed under `tests/terraform/<TEST_NAME>`.
-- All TF test functions should be named: `Test_TF<TEST_NAME>`. 
-
-See the [create cluster test](../tests/terraform/createcluster_test.go) as an example.
-
-## Running
-
-- Before running the tests, you should create local.tfvars file in `./tests/terraform/modules/k3scluster/config/local.tfvars`. There is some information there to get you started, but the empty variables should be filled in appropriately per your AWS environment.
-
-
-
-- For running tests with "etcd" cluster type, you should add the value "etcd" to the variable "cluster_type" , also you need have those variables at least empty:
-```
-- external_db       
-- external_db_version
-- instance_class  
-- db_group_name
-```
-
-- For running with external db you need the same variables above filled in with the correct data and also cluster_type= ""
-
-
-All TF tests can be run with:
-```bash
-go test -timeout=60m ./tests/terraform/... -run TF
-```
-Tests can be run individually with:
-```bash
-go test -timeout=30m ./tests/terraform/createcluster/createcluster.go ./tests/terraform/createcluster/createcluster_test.go
-# OR
-go test -v -timeout=30m ./tests/terraform/... -run TFClusterCreateValidation
-# example with vars:
-go test -timeout=30m -v ./tests/terraform/createcluster.go ./tests/terraform/createcluster_test.go -node_os=ubuntu -aws_ami=ami-02f3416038bdb17fb -cluster_type=etcd -resource_name=localrun1 -sshuser=ubuntu -sshkey="key-name" -destroy=false
-
-```
-Test Flags:
-```
-- ${upgradeVersion} version to upgrade to
-```
-We can also run tests through the Makefile through tests' directory:
-
-- On the first run with make and docker please delete your .terraform folder, terraform.tfstate and terraform.hcl.lock file
-
-```bash
-Args:
-*All args are optional and can be used with:
-
-`$make tf-run`         `$make tf-logs`,
-`$make vet-lint`       `$make tf-complete`, 
-`$make tf-upgrade`     `$make tf-test-suite-same-cluster`,
-`$make tf-test-suite`
-
-
-- ${IMGNAME}     append any string to the end of image name
-- ${TAGNAME}     append any string to the end of tag name
-- ${ARGNAME}     name of the arg to pass to the test
-- ${ARGVALUE}    value of the arg to pass to the test
-- ${TESTDIR}     path to the test directory 
-
-Commands:
-$ make tf-up                         # create the image from Dockerfile.build
-$ make tf-run                        # runs all tests if no flags or args provided
-$ make tf-down                       # removes the image
-$ make tf-clean                      # removes instances and resources created by tests
-$ make tf-logs                       # prints logs from container the tests
-$ make tf-complete                   # clean resources + remove images + run tests
-$ make tf-create                     # runs create cluster test locally
-$ make tf-upgrade                    # runs upgrade cluster test locally
-$ make tf-test-suite-same-cluster    # runs all tests locally in sequence using the same state    
-$ make tf-remove-state               # removes terraform state dir and files
-$ make tf-test-suite                 # runs all tests locally in sequence not using the same state
-$ make vet-lint                      # runs go vet and go lint
-
-      
-Examples:
-$ make tf-up TAGNAME=ubuntu
-$ make tf-run IMGNAME=2 TAGNAME=ubuntu TESTDIR=upgradecluster ARGNAME=upgradeVersion ARGVALUE=v1.26.2+k3s1
-$ make tf-run TESTDIR=upgradecluster
-$ make tf-logs IMGNAME=1
-$ make vet-lint TESTDIR=upgradecluster
-```
-
-
-# Running tests in parallel:
-- You can play around and have a lot of different test combinations like:
-```
-- Build docker image with different TAGNAME="OS`s" + with different configurations( resource_name, node_os, versions, install type, nodes and etc) and have unique "IMGNAMES"
-
-- And in the meanwhile run also locally with different configuration while your dockers TAGNAME and IMGNAMES are running
-```
-
-# In between tests:
-- If you want to run with same cluster do not delete ./tests/terraform/modules/terraform.tfstate + .terraform.lock.hcl file after each test.
-
-- if you want to use new resources then make sure to delete the ./tests/terraform/modules/terraform.tfstate + .terraform.lock.hcl file if you want to create a new cluster.
-
-
-# Common Issues:
-
-- Issues related to terraform plugin please also delete the modules/.terraform folder
-- In mac m1 maybe you need also to go to rke2/tests/terraform/modules and run `terraform init` to download the plugins
-
-
-
-
-# Reporting:
-Additionally, to generate junit reporting for the tests, the Ginkgo CLI is used. Installation instructions can be found [here.](https://onsi.github.io/ginkgo/#getting-started)  
-
-To run the all TF tests and generate JUnit testing reports:
-```
-ginkgo --junit-report=result.xml ./tests/terraform/...
-```
-
-Note: The `go test` default timeout is 10 minutes, thus the `-timeout` flag should be used. The `ginkgo` default timeout is 1 hour, no timeout flag is needed.
-
-# Debugging
-The cluster and VMs can be retained after a test by passing `-destroy=false`. 
-To focus individual runs on specific test clauses, you can prefix with `F`. For example, in the [create cluster test](../tests/terraform/createcluster_test.go), you can update the initial creation to be: `FIt("Starts up with no issues", func() {` in order to focus the run on only that clause.
\ No newline at end of file

tests/terraform/amd64_resource_files/clusterip.yaml

-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: test-clusterip
-spec:
-  selector:
-    matchLabels:
-      k8s-app: nginx-app-clusterip
-  replicas: 2
-  template:
-    metadata:
-      labels:
-        k8s-app: nginx-app-clusterip
-    spec:
-      containers:
-      - name: nginx
-        image: ranchertest/mytestcontainer
-        ports:
-        - containerPort: 80
----
-apiVersion: v1
-kind: Service
-metadata:
-  labels:
-    k8s-app: nginx-app-clusterip
-  name: nginx-clusterip-svc
-  namespace: default 
-spec:
-  type: ClusterIP
-  ports:
-    - port: 80
-  selector:
-    k8s-app: nginx-app-clusterip

tests/terraform/amd64_resource_files/daemonset.yaml

-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: test-daemonset
-spec:
-  selector:
-    matchLabels:
-      k8s-app: test-daemonset
-  template:
-    metadata:
-      labels:
-        k8s-app: test-daemonset
-    spec:
-      containers:
-        - name: webserver
-          image: nginx
-          ports:
-          - containerPort: 80

tests/terraform/amd64_resource_files/dnsutils.yaml

-apiVersion: v1
-kind: Pod
-metadata:
-  name: dnsutils
-  namespace: default
-spec:
-  containers:
-  - name: dnsutils
-    image: gcr.io/kubernetes-e2e-test-images/dnsutils:1.3
-    command:
-      - sleep
-      - "3600"
-    imagePullPolicy: IfNotPresent
-  restartPolicy: Always

tests/terraform/amd64_resource_files/ingress.yaml

-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: test-ingress
-spec:
-  rules:
-  - host: foo1.bar.com
-    http:
-      paths:
-      - backend:
-          service:
-            name: nginx-ingress-svc
-            port:
-              number: 80
-        path: /
-        pathType: ImplementationSpecific
----
-apiVersion: v1
-kind: Service
-metadata:
- name: nginx-ingress-svc
- labels:
-   k8s-app: nginx-app-ingress
-spec:
- ports:
- - port: 80
-   targetPort: 80
-   protocol: TCP
-   name: http
- selector:
-   k8s-app: nginx-app-ingress
----
-apiVersion: v1
-kind: ReplicationController
-metadata:
- name: test-ingress
-spec:
- replicas: 2
- selector:
-   k8s-app: nginx-app-ingress
- template:
-   metadata:
-     labels:
-       k8s-app: nginx-app-ingress
-   spec:
-     terminationGracePeriodSeconds: 60
-     containers:
-     - name: testcontainer
-       image: ranchertest/mytestcontainer
-       ports:
-       - containerPort: 80

tests/terraform/amd64_resource_files/loadbalancer.yaml

----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: test-loadbalancer
-spec:
-  selector:
-    matchLabels:
-      k8s-app: nginx-app-loadbalancer
-  replicas: 2
-  template:
-    metadata:
-      labels:
-        k8s-app: nginx-app-loadbalancer
-    spec:
-      containers:
-      - name: nginx
-        image: ranchertest/mytestcontainer
-        ports:
-        - containerPort: 80
----
-apiVersion: v1
-kind: Service
-metadata:
- name: nginx-loadbalancer-svc
- labels:
-   k8s-app: nginx-app-loadbalancer
-spec:
- type: LoadBalancer
- ports:
- - port: 81
-   targetPort: 80
-   protocol: TCP
-   name: http
- selector:
-   k8s-app: nginx-app-loadbalancer

tests/terraform/amd64_resource_files/local-path-provisioner.yaml

-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: local-path-pvc
-  namespace: default
-spec:
-  accessModes:
-    - ReadWriteOnce
-  storageClassName: local-path
-  resources:
-    requests:
-      storage: 500Mi
----
-apiVersion: v1
-kind: Pod
-metadata:
-  name: volume-test
-  namespace: default
-spec:
-  containers:
-  - name: volume-test
-    image: nginx:stable-alpine
-    imagePullPolicy: IfNotPresent
-    volumeMounts:
-    - name: volv
-      mountPath: /data
-    ports:
-    - containerPort: 80
-  volumes:
-  - name: volv
-    persistentVolumeClaim:
-      claimName: local-path-pvc

tests/terraform/amd64_resource_files/nodeport.yaml

-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: test-nodeport
-spec:
-  selector:
-    matchLabels:
-      k8s-app: nginx-app-nodeport
-  replicas: 2
-  template:
-    metadata:
-      labels:
-        k8s-app: nginx-app-nodeport
-    spec:
-      containers:
-      - name: nginx
-        image: ranchertest/mytestcontainer
-        ports:
-        - containerPort: 80
----
-apiVersion: v1
-kind: Service
-metadata:
-  labels:
-    k8s-app: nginx-app-nodeport
-  name: nginx-nodeport-svc
-  namespace: default
-spec:
-  type: NodePort
-  ports:
-    - port: 80
-      nodePort: 30096
-      name: http
-  selector:
-    k8s-app: nginx-app-nodeport

tests/terraform/amd64_resource_files/secrets.yaml

-apiVersion: v1
-kind: Secret
-metadata:
-  name: e2e-secret1
-type: Opaque
-stringData:
-  config.yaml: |
-    key: "hello"
-    val: "world"
----
-apiVersion: v1
-kind: Secret
-metadata:
-  name: e2e-secret2
-type: Opaque
-stringData:
-  config.yaml: |
-    key: "good"
-    val: "day"
----
-apiVersion: v1
-kind: Secret
-metadata:
-  name: e2e-secret3
-type: Opaque
-stringData:
-  config.yaml: |
-    key: "top-secret"
-    val: "information"
----
-apiVersion: v1
-kind: Secret
-metadata:
-  name: e2e-secret4
-type: Opaque
-stringData:
-  config.yaml: |
-    key: "lock"
-    val: "key"
----
-apiVersion: v1
-kind: Secret
-metadata:
-  name: e2e-secret5
-type: Opaque
-stringData:
-  config.yaml: |
-    key: "last"
-    val: "call"

tests/terraform/arm_resource_files/clusterip.yaml

-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: test-clusterip
-spec:
-  selector:
-    matchLabels:
-      k8s-app: nginx-app-clusterip
-  replicas: 2
-  template:
-    metadata:
-      labels:
-        k8s-app: nginx-app-clusterip
-    spec:
-      containers:
-      - name: nginx
-        image: shylajarancher19/shylajaarm64:v1.0
-        ports:
-        - containerPort: 80
----
-apiVersion: v1
-kind: Service
-metadata:
-  labels:
-    k8s-app: nginx-app-clusterip
-  name: nginx-clusterip-svc
-  namespace: default 
-spec:
-  type: ClusterIP
-  ports:
-    - port: 80
-  selector:
-    k8s-app: nginx-app-clusterip

tests/terraform/arm_resource_files/daemonset.yaml

-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: test-daemonset
-spec:
-  selector:
-    matchLabels:
-      k8s-app: test-daemonset
-  template:
-    metadata:
-      labels:
-        k8s-app: test-daemonset
-    spec:
-      containers:
-        - name: webserver
-          image: nginx
-          ports:
-          - containerPort: 80

tests/terraform/arm_resource_files/ingress.yaml

-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: test-ingress
-spec:
-  rules:
-    - host: foo1.bar.com
-      http:
-        paths:
-          - backend:
-              service:
-                name: nginx-ingress-svc
-                port:
-                  number: 80
-            path: /
-            pathType: ImplementationSpecific
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: nginx-ingress-svc
-  labels:
-    k8s-app: nginx-app-ingress
-spec:
-  ports:
-    - port: 80
-      targetPort: 80
-      protocol: TCP
-      name: http
-  selector:
-    k8s-app: nginx-app-ingress
----
-apiVersion: v1
-kind: ReplicationController
-metadata:
-  name: test-ingress
-spec:
-  replicas: 2
-  selector:
-    k8s-app: nginx-app-ingress
-  template:
-    metadata:
-      labels:
-        k8s-app: nginx-app-ingress
-    spec:
-      terminationGracePeriodSeconds: 60
-      containers:
-        - name: testcontainer
-          image: shylajarancher19/shylajaarm64:v1.0
-          ports:
-            - containerPort: 80

tests/terraform/arm_resource_files/loadbalancer.yaml

----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: test-loadbalancer
-spec:
-  selector:
-    matchLabels:
-      k8s-app: nginx-app-loadbalancer
-  replicas: 2
-  template:
-    metadata:
-      labels:
-        k8s-app: nginx-app-loadbalancer
-    spec:
-      containers:
-      - name: nginx
-        image: shylajarancher19/shylajaarm64:v1.0
-        ports:
-        - containerPort: 80
----
-apiVersion: v1
-kind: Service
-metadata:
- name: nginx-loadbalancer-svc
- labels:
-   k8s-app: nginx-app-loadbalancer
-spec:
- type: LoadBalancer
- ports:
- - port: 81
-   targetPort: 80
-   protocol: TCP
-   name: http
- selector:
-   k8s-app: nginx-app-loadbalancer

tests/terraform/arm_resource_files/local-path-provisioner.yaml

-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: local-path-pvc
-  namespace: default
-spec:
-  accessModes:
-    - ReadWriteOnce
-  storageClassName: local-path
-  resources:
-    requests:
-      storage: 500Mi
----
-apiVersion: v1
-kind: Pod
-metadata:
-  name: volume-test
-  namespace: default
-spec:
-  containers:
-  - name: volume-test
-    image: nginx:stable-alpine
-    imagePullPolicy: IfNotPresent
-    volumeMounts:
-    - name: volv
-      mountPath: /data
-    ports:
-    - containerPort: 80
-  volumes:
-  - name: volv
-    persistentVolumeClaim:
-      claimName: local-path-pvc

tests/terraform/arm_resource_files/nodeport.yaml

-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: test-nodeport
-spec:
-  selector:
-    matchLabels:
-      k8s-app: nginx-app-nodeport
-  replicas: 2
-  template:
-    metadata:
-      labels:
-        k8s-app: nginx-app-nodeport
-    spec:
-      containers:
-      - name: nginx
-        image: shylajarancher19/shylajaarm64:v1.0
-        ports:
-        - containerPort: 80
----
-apiVersion: v1
-kind: Service
-metadata:
-  labels:
-    k8s-app: nginx-app-nodeport
-  name: nginx-nodeport-svc
-  namespace: default
-spec:
-  type: NodePort
-  ports:
-    - port: 80
-      nodePort: 30096
-      name: http
-  selector:
-    k8s-app: nginx-app-nodeport

tests/terraform/config.mk

-SHELL := /bin/bash
-
-LOCAL_TFVARS_PATH := modules/k3scluster/config/local.tfvars
-
-ifeq ($(wildcard ${LOCAL_TFVARS_PATH}),)
-  RESOURCE_NAME :=
-else
-  export RESOURCE_NAME := $(shell sed -n 's/resource_name *= *"\([^"]*\)"/\1/p' ${LOCAL_TFVARS_PATH})
-endif
-
-export ACCESS_KEY_LOCAL
-export AWS_ACCESS_KEY_ID
-export AWS_SECRET_ACCESS_KEY
\ No newline at end of file

tests/terraform/createcluster/createcluster.go

-package createcluster
-
-import (
-	"fmt"
-	"strconv"
-
-	"path/filepath"
-	"testing"
-
-	"github.com/gruntwork-io/terratest/modules/terraform"
-	tf "github.com/k3s-io/k3s/tests/terraform"
-)
-
-var (
-	KubeConfigFile   string
-	MasterIPs        string
-	WorkerIPs        string
-	NumServers       int
-	NumWorkers       int
-	AwsUser          string
-	AccessKey        string
-	RenderedTemplate string
-	ExternalDb       string
-	ClusterType      string
-	TfVarsPath       = "/tests/terraform/modules/k3scluster/config/local.tfvars"
-	modulesPath      = "/tests/terraform/modules/k3scluster"
-)
-
-func BuildCluster(t *testing.T, destroy bool) (string, error) {
-	basepath := tf.GetBasepath()
-	tfDir, err := filepath.Abs(basepath + modulesPath)
-	if err != nil {
-		return "", err
-	}
-	varDir, err := filepath.Abs(basepath + TfVarsPath)
-	if err != nil {
-		return "", err
-	}
-	TerraformOptions := &terraform.Options{
-		TerraformDir: tfDir,
-		VarFiles:     []string{varDir},
-	}
-
-	NumServers, err = strconv.Atoi(terraform.GetVariableAsStringFromVarFile(t, varDir,
-		"no_of_server_nodes"))
-	if err != nil {
-		return "", err
-	}
-
-	NumWorkers, err = strconv.Atoi(terraform.GetVariableAsStringFromVarFile(t, varDir,
-		"no_of_worker_nodes"))
-	if err != nil {
-		return "", err
-	}
-
-	ClusterType = terraform.GetVariableAsStringFromVarFile(t, varDir, "cluster_type")
-	ExternalDb = terraform.GetVariableAsStringFromVarFile(t, varDir, "external_db")
-	AwsUser = terraform.GetVariableAsStringFromVarFile(t, varDir, "aws_user")
-	AccessKey = terraform.GetVariableAsStringFromVarFile(t, varDir, "access_key")
-
-	if destroy {
-		fmt.Printf("Cluster is being deleted")
-		terraform.Destroy(t, TerraformOptions)
-		return "cluster destroyed", err
-	}
-
-	fmt.Printf("Creating Cluster")
-
-	terraform.InitAndApply(t, TerraformOptions)
-	KubeConfigFile = "/tmp/" + terraform.Output(t, TerraformOptions, "kubeconfig") + "_kubeconfig"
-	MasterIPs = terraform.Output(t, TerraformOptions, "master_ips")
-	WorkerIPs = terraform.Output(t, TerraformOptions, "worker_ips")
-	RenderedTemplate = terraform.Output(t, TerraformOptions, "rendered_template")
-
-	return "cluster created", err
-}

tests/terraform/go.mod

-module github.com/k3s-io/k3s/tests/terraform
-
-go 1.19
-
-require (
-	github.com/gruntwork-io/terratest v0.41.15
-	github.com/onsi/ginkgo/v2 v2.9.0
-	github.com/onsi/gomega v1.27.2
-	golang.org/x/crypto v0.7.0
-)
-
-require (
-	cloud.google.com/go v0.107.0 // indirect
-	cloud.google.com/go/compute v1.15.1 // indirect
-	cloud.google.com/go/compute/metadata v0.2.3 // indirect
-	cloud.google.com/go/iam v0.8.0 // indirect
-	cloud.google.com/go/storage v1.27.0 // indirect
-	github.com/agext/levenshtein v1.2.3 // indirect
-	github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect
-	github.com/aws/aws-sdk-go v1.44.122 // indirect
-	github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/go-logr/logr v1.2.3 // indirect
-	github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
-	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/google/go-cmp v0.5.9 // indirect
-	github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect
-	github.com/google/uuid v1.3.0 // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect
-	github.com/googleapis/gax-go/v2 v2.7.0 // indirect
-	github.com/hashicorp/errwrap v1.0.0 // indirect
-	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
-	github.com/hashicorp/go-getter v1.7.0 // indirect
-	github.com/hashicorp/go-multierror v1.1.0 // indirect
-	github.com/hashicorp/go-safetemp v1.0.0 // indirect
-	github.com/hashicorp/go-version v1.6.0 // indirect
-	github.com/hashicorp/hcl/v2 v2.9.1 // indirect
-	github.com/hashicorp/terraform-json v0.13.0 // indirect
-	github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a // indirect
-	github.com/jmespath/go-jmespath v0.4.0 // indirect
-	github.com/klauspost/compress v1.15.11 // indirect
-	github.com/mattn/go-zglob v0.0.2-0.20190814121620-e3c945676326 // indirect
-	github.com/mitchellh/go-homedir v1.1.0 // indirect
-	github.com/mitchellh/go-testing-interface v1.14.1 // indirect
-	github.com/mitchellh/go-wordwrap v1.0.1 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/stretchr/testify v1.8.1 // indirect
-	github.com/tmccombs/hcl2json v0.3.3 // indirect
-	github.com/ulikunitz/xz v0.5.10 // indirect
-	github.com/zclconf/go-cty v1.9.1 // indirect
-	go.opencensus.io v0.24.0 // indirect
-	golang.org/x/net v0.8.0 // indirect
-	golang.org/x/oauth2 v0.4.0 // indirect
-	golang.org/x/sys v0.6.0 // indirect
-	golang.org/x/text v0.8.0 // indirect
-	golang.org/x/tools v0.6.0 // indirect
-	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
-	google.golang.org/api v0.103.0 // indirect
-	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect
-	google.golang.org/grpc v1.53.0 // indirect
-	google.golang.org/protobuf v1.28.1 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
-)

tests/terraform/jenkinsfile

-pipeline {
-    agent any
-        environment {
-            setupResultsOut = "setup-results.xml"
-            testResultsOut = "results.xml"
-
-            AWS_ACCESS_KEY_ID     = credentials('AWS_ACCESS_KEY_ID')
-            AWS_SECRET_ACCESS_KEY = credentials('AWS_SECRET_ACCESS_KEY')
-            AWS_SSH_PEM_KEY = credentials('AWS_SSH_PEM_KEY')
-        }
-
-        stages {
-            stage('Git Checkout') {
-                steps {
-                    git branch: 'master', url: 'https://github.com/k3s-io/k3s.git'
-                    script {
-                        dir("${WORKSPACE}/tests/terraform") {
-                            if (env.AWS_SSH_PEM_KEY && env.AWS_SSH_KEY_NAME) {
-                                    def decoded = new String(AWS_SSH_PEM_KEY.decodeBase64())
-                                    writeFile file: AWS_SSH_KEY_NAME, text: decoded
-                            }
-                        }
-                    }
-                }
-            }
-
-            stage('Configure') {
-                steps {
-                    sh """
-                    set -e -x
-                    echo 'aws_ami="${env.AWS_AMI}"
-                        aws_user="${env.AWS_USER}"
-                        region="${env.REGION}"
-                        vpc_id="${env.VPC_ID}"
-                        subnets="${env.SUBNETS}"
-                        qa_space="${env.QA_SPACE}"
-                        ec2_instance_class="${env.AWS_INSTANCE_TYPE}"
-                        access_key="/config/$AWS_SSH_KEY_NAME"
-                        no_of_worker_nodes="${env.NO_OF_WORKER_NODES}"
-                        key_name="jenkins-rke-validation"
-                        server_flags="${env.K3S_SERVER_FLAGS}"
-                        worker_flags="${env.K3S_WORKER_FLAGS}"
-                        k3s_version="${env.K3S_VERSION}"
-                        availability_zone="${env.AVAILABILITY_ZONE}"
-                        sg_id="${env.SG_ID}"
-                        install_mode="${env.K3S_INSTALL_MODE}"
-                        resource_name="${env.RESOURCE_NAME}"
-                        no_of_server_nodes="${env.NO_OF_SERVER_NODES}"
-                        username="${env.RHEL_USERNAME}"
-                        password="${env.RHEL_PASSWORD}"
-                        db_username="${env.DB_USERNAME}"
-                        db_password="${env.DB_PASSWORD}"
-                        node_os="${env.NODE_OS}"
-                        environment="${env.DB_ENVIRONMENT}"
-                        engine_mode="${env.DB_ENGINE_MODE}"
-                        external_db="${env.EXTERNAL_DB}"
-                        external_db_version="${env.EXTERNAL_DB_VERSION}"
-                        instance_class="${env.DB_INSTANCE_CLASS}"
-                        db_group_name="${env.DB_GROUP_NAME}"
-                        cluster_type="${env.CLUSTER_TYPE}"
-                        create_lb=${env.CREATE_LB}
-                        ' >${WORKSPACE}/tests/terraform/${env.NODE_OS}${env.EXTERNAL_DB}".tfvars"
-                    """
-                }
-            }
-            stage('Build Cluster') {
-                steps {
-                    sh """
-
-                        /usr/bin/docker build -f tests/terraform/Dockerfile.build -t k3s_create_cluster .
-
-                        /usr/bin/docker run -d --name ${RESOURCE_NAME}_${BUILD_NUMBER} -v ${WORKSPACE}/tests/terraform:/config \
-                        -t -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
-                        k3s_create_cluster
-
-                        /usr/bin/docker cp "${WORKSPACE}/tests/terraform/${NODE_OS}${EXTERNAL_DB}".tfvars "${RESOURCE_NAME}_${BUILD_NUMBER}":/config
-
-                        /usr/bin/docker cp "${WORKSPACE}/tests/terraform/$AWS_SSH_KEY_NAME" "${RESOURCE_NAME}_${BUILD_NUMBER}":/config
-
-                        /usr/bin/docker exec ${RESOURCE_NAME}_${BUILD_NUMBER} /usr/local/go/bin/go test -v tests/terraform/createcluster.go \
-                        tests/terraform/createcluster_test.go tests/terraform/testutils.go \
-                        -timeout=1h -node_os=${NODE_OS} \
-                        -cluster_type=${CLUSTER_TYPE} -external_db=${EXTERNAL_DB} -resource_name=${RESOURCE_NAME} \
-                        -sshuser=${AWS_USER} -sshkey="/config/${AWS_SSH_KEY_NAME}" -destroy=false -arch=${ARCH}
-
-                    """
-                }
-            }
-
-            stage('Test Report') {
-                steps {
-                    sh """
-                        /usr/bin/docker rm -f ${RESOURCE_NAME}_${BUILD_NUMBER}
-                        /usr/bin/docker rmi -f k3s_create_cluster
-                    """
-                }
-            }
-        }
-    }
\ No newline at end of file

tests/terraform/modules/k3scluster/config/local.tfvars

-region = "us-east-2"
-qa_space = ""
-create_lb = false
-
-external_db_version = "5.7"
-instance_class = "db.t2.micro"
-db_group_name = "mysql5.7"
-engine_mode = "provisioned"
-db_username = ""
-db_password = ""
-
-username = ""
-password = ""
-
-ec2_instance_class = "t3a.medium"
-vpc_id = ""
-subnets = ""
-availability_zone = "us-east-2a"
-sg_id = ""
-
-no_of_server_nodes = 2
-no_of_worker_nodes = 1
-server_flags = "token: test"
-worker_flags = "token: test"
-
-k3s_version = "v1.23.8+k3s2"
-install_mode = "INSTALL_K3S_VERSION"
-environment = "local"
\ No newline at end of file

tests/terraform/modules/k3scluster/install_k3s_master.sh

-#!/bin/bash
-
-mkdir -p /etc/rancher/k3s
-cat << EOF >/etc/rancher/k3s/config.yaml
-write-kubeconfig-mode: "0644"
-tls-san:
-  - ${2}
-EOF
-
-if [[ -n "$8" ]] && [[ "$8" == *":"* ]]
-then
-   echo "$"
-   echo -e "$8" >> /etc/rancher/k3s/config.yaml
-   cat /etc/rancher/k3s/config.yaml
-fi
-
-if [ "${1}" = "rhel" ]
-then
-   subscription-manager register --auto-attach --username="${9}" --password="${10}"
-   subscription-manager repos --enable=rhel-7-server-extras-rpms
-fi
-
-export "${3}"="${4}"
-
-if [ "${5}" = "etcd" ]
-then
-   echo "CLUSTER TYPE  is etcd"
-   if [[ "$4" == *"v1.18"* ]] || [[ "$4" == *"v1.17"* ]] && [[ -n "$8" ]]
-   then
-       echo "curl -sfL https://get.k3s.io | INSTALL_K3S_TYPE='server' sh -s - --cluster-init --node-external-ip=${6} $8" >/tmp/master_cmd
-       curl -sfL https://get.k3s.io | INSTALL_K3S_TYPE='server' sh -s - --cluster-init --node-external-ip="${6}" "$8"
-   else
-       echo "curl -sfL https://get.k3s.io | INSTALL_K3S_TYPE='server' sh -s - --cluster-init --node-external-ip=${6}" >/tmp/master_cmd
-       curl -sfL https://get.k3s.io | INSTALL_K3S_TYPE='server' sh -s - --cluster-init --node-external-ip="${6}"
-   fi
-else
-   echo "CLUSTER TYPE is external db"
-   if [[ "$4" == *"v1.18"* ]] || [[ "$4" == *"v1.17"* ]] && [[ -n "$8" ]]
-   then
-       echo "curl -sfL https://get.k3s.io | sh -s - server --node-external-ip=${6} --datastore-endpoint=\"${7}\" $8"  >/tmp/master_cmd
-       curl -sfL https://get.k3s.io | sh -s - server --node-external-ip="${6}" --datastore-endpoint="${7}" "$8"
-   else
-       echo "curl -sfL https://get.k3s.io | sh -s - server --node-external-ip=${6}  --datastore-endpoint=\"${7}\" "  >/tmp/master_cmd
-       curl -sfL https://get.k3s.io | sh -s - server --node-external-ip="${6}" --datastore-endpoint="${7}"
-   fi
-fi
-
-export PATH=$PATH:/usr/local/bin
-timeElapsed=0
-while ! $(kubectl get nodes >/dev/null 2>&1) && [[ $timeElapsed -lt 300 ]]
-do
-   sleep 5
-   timeElapsed=$(expr $timeElapsed + 5)
-done
-
-IFS=$'\n'
-timeElapsed=0
-sleep 10
-while [[ $timeElapsed -lt 420 ]]
-do
-   notready=false
-   for rec in $(kubectl get nodes)
-   do
-      if [[ "$rec" == *"NotReady"* ]]
-      then
-         notready=true
-      fi
-  done
-  if [[ $notready == false ]]
-  then
-     break
-  fi
-  sleep 20
-  timeElapsed=$(expr $timeElapsed + 20)
-done
-
-IFS=$'\n'
-timeElapsed=0
-while [[ $timeElapsed -lt 420 ]]
-do
-   helmPodsNR=false
-   systemPodsNR=false
-   for rec in $(kubectl get pods -A --no-headers)
-   do
-      if [[ "$rec" == *"helm-install"* ]] && [[ "$rec" != *"Completed"* ]]
-      then
-         helmPodsNR=true
-      elif [[ "$rec" != *"helm-install"* ]] && [[ "$rec" != *"Running"* ]]
-      then
-         systemPodsNR=true
-      else
-         echo ""
-      fi
-   done
-
-   if [[ $systemPodsNR == false ]] && [[ $helmPodsNR == false ]]
-   then
-      break
-   fi
-   sleep 20
-   timeElapsed=$(expr $timeElapsed + 20)
-done
-cat /etc/rancher/k3s/config.yaml> /tmp/joinflags
-cat /var/lib/rancher/k3s/server/node-token >/tmp/nodetoken
-cat /etc/rancher/k3s/k3s.yaml >/tmp/config

tests/terraform/modules/k3scluster/join_k3s_agent.sh

-#!/bin/bash
-# This script is used to join one or more nodes as agents
-
-mkdir -p /etc/rancher/k3s
-cat <<EOF >>/etc/rancher/k3s/config.yaml
-server: https://${4}:6443
-token:  "${5}"
-EOF
-
-if [[ ! -z "$7" ]] && [[ "$7" == *":"* ]]
-then
-   echo -e "$7" >> /etc/rancher/k3s/config.yaml
-   cat /etc/rancher/k3s/config.yaml
-fi
-
-if [ ${1} = "rhel" ]
-then
-    subscription-manager register --auto-attach --username=${8} --password=${9}
-    subscription-manager repos --enable=rhel-7-server-extras-rpms
-fi
-
-export "${2}"="${3}"
-  if [[ "$3" == *"v1.18"* ]] || [["$3" == *"v1.17"* ]] && [[ -n "$7" ]]
-then
-  echo "curl -sfL https://get.k3s.io | sh -s - agent --node-external-ip=${6} $7" >/tmp/agent_cmd
-curl -sfL https://get.k3s.io | sh -s - agent --node-external-ip=${6} ${7}
-  else
-
-echo "curl -sfL https://get.k3s.io | sh -s - agent --node-external-ip=${6}" >/tmp/agent_cmd
-curl -sfL https://get.k3s.io | sh -s - agent --node-external-ip=${6}
-fi

tests/terraform/modules/k3scluster/join_k3s_master.sh

-#!/bin/bash
-# This script is used to join one or more nodes as masters
-
-mkdir -p /etc/rancher/k3s
-cat <<EOF >>/etc/rancher/k3s/config.yaml
-write-kubeconfig-mode: "0644"
-tls-san:
-  - ${2}
-EOF
-
-if [[ -n "${10}" ]] && [[ "${10}" == *":"* ]]
-then
-   echo -e "${10}" >> /etc/rancher/k3s/config.yaml
-   cat /etc/rancher/k3s/config.yaml
-fi
-
-if [ "${1}" = "rhel" ]
-then
-   subscription-manager register --auto-attach --username="${11}" --password="${12}"
-   subscription-manager repos --enable=rhel-7-server-extras-rpms
-fi
-
-export "${3}"="${4}"
-
-if [ "${5}" = "etcd" ]
-then
-   if [[ "$4" == *"v1.18"* ]] || [["$4" == *"v1.17"* ]] && [[ -n "$10" ]]
-   then
-      echo "curl -sfL https://get.k3s.io | INSTALL_K3S_TYPE='server' sh -s - --server https://\"${7}\":6443 --token \"${8}\" --node-external-ip=\"${6}\" ${10}" >/tmp/master_cmd
-      curl -sfL https://get.k3s.io | INSTALL_K3S_TYPE='server' sh -s - --server https://"${7}":6443 --token "${8}" --node-external-ip="${6} ${10}"
-   else
-        echo "curl -sfL https://get.k3s.io | INSTALL_K3S_TYPE='server' sh -s - --server https://\"${7}\":6443 --token \"${8}\" --node-external-ip=\"${6}\"" >/tmp/master_cmd
-       curl -sfL https://get.k3s.io | INSTALL_K3S_TYPE='server' sh -s - --server https://"${7}":6443 --token "${8}" --node-external-ip="${6}"
-   fi
-else
-  if [[ "$4" == *"v1.18"* ]] || [["$4" == *"v1.17"* ]] && [[ -n "$10" ]]
-  then
-      echo "curl -sfL https://get.k3s.io | INSTALL_K3S_TYPE='server' sh -s - --node-external-ip=\"${6}\" --datastore-endpoint=\"${9}\" ${10}" >/tmp/master_cmd
-      curl -sfL https://get.k3s.io | INSTALL_K3S_TYPE='server' sh -s - --node-external-ip="${6}" --token="${8}" --datastore-endpoint="${9} ${10}"
-   else
-      echo "curl -sfL https://get.k3s.io | INSTALL_K3S_TYPE='server' sh -s - --node-external-ip=\"${6}\" --token \"${8}\" --datastore-endpoint=\"${9}\"" >/tmp/master_cmd
-      curl -sfL https://get.k3s.io | INSTALL_K3S_TYPE='server' sh -s - --node-external-ip="${6}" --token="${8}" --datastore-endpoint="${9}"
-    fi
-fi

tests/terraform/modules/k3scluster/main.tf

-module "master" {
-   source="./master"
-   aws_ami=var.aws_ami
-   aws_user=var.aws_user
-   key_name=var.key_name
-   no_of_server_nodes=var.no_of_server_nodes
-   k3s_version=var.k3s_version
-   install_mode=var.install_mode
-   region=var.region
-   vpc_id=var.vpc_id
-   subnets=var.subnets
-   qa_space=var.qa_space
-   ec2_instance_class=var.ec2_instance_class
-   access_key=var.access_key
-   cluster_type=var.cluster_type
-   server_flags=var.server_flags
-   availability_zone=var.availability_zone
-   sg_id=var.sg_id
-   resource_name=var.resource_name
-   node_os=var.node_os
-   username=var.username
-   password=var.password
-   db_username=var.db_username
-   db_password=var.db_password
-   db_group_name=var.db_group_name
-   external_db=var.external_db
-   instance_class=var.instance_class
-   external_db_version=var.external_db_version
-   engine_mode=var.engine_mode
-   environment=var.environment
-   create_lb=var.create_lb
-}
-module "worker" {
-   source="./worker"
-   dependency = module.master
-   aws_ami=var.aws_ami
-   aws_user=var.aws_user
-   key_name=var.key_name
-   no_of_worker_nodes=var.no_of_worker_nodes
-   k3s_version=var.k3s_version
-   install_mode=var.install_mode
-   region=var.region
-   vpc_id=var.vpc_id
-   subnets=var.subnets
-   ec2_instance_class=var.ec2_instance_class
-   access_key=var.access_key
-   worker_flags=var.worker_flags
-   availability_zone=var.availability_zone
-   sg_id=var.sg_id
-   resource_name=var.resource_name
-   node_os=var.node_os
-   username=var.username
-   password=var.password
-}

tests/terraform/modules/k3scluster/master/outputs.tf

-output "Route53_info" {
-  value       = aws_route53_record.aws_route53.*
-  description = "List of DNS records"
-}
-
-output "master_ips" {
-  value = join("," , aws_instance.master.*.public_ip,aws_instance.master2-ha.*.public_ip)
-  description = "The public IP of the AWS node"
-}
-
-output "kubeconfig" {
-  value = var.resource_name
-  description = "kubeconfig of the cluster created"
-}
-
-output "rendered_template" {
-  value = data.template_file.test.rendered
-}
\ No newline at end of file

tests/terraform/modules/k3scluster/master/providers.tf

-provider "aws" {
-    region = "${var.region}"
-}

tests/terraform/modules/k3scluster/master/variables.tf

-variable "aws_ami" {}
-variable "aws_user" {}
-variable "region" {}
-variable "access_key" {}
-variable "vpc_id" {}
-variable "subnets" {}
-variable "availability_zone" {}
-variable "sg_id" {}
-variable "qa_space" {}
-variable "ec2_instance_class" {}
-variable "resource_name" {}
-variable "key_name" {}
-variable "external_db" {}
-variable "external_db_version" {}
-variable "instance_class" {}
-variable "db_group_name" {}
-variable "username" {}
-variable "password" {}
-variable "k3s_version" {}
-variable "no_of_server_nodes" {}
-variable "server_flags" {}
-
-variable "cluster_type" {}
-variable "node_os" {}
-variable "db_username" {}
-variable "db_password" {}
-variable "environment" {}
-variable "engine_mode" {}
-variable "install_mode" {}
-
-variable "create_lb" {
-  description = "Create Network Load Balancer if set to true"
-  type = bool
-}

tests/terraform/modules/k3scluster/outputs.tf

-output "master_ips" {
-  value       = module.master.master_ips
-  description = "The public IP of the AWS node"
-}
-
-output "worker_ips" {
-  value       = module.worker.worker_ips
-  description = "The public IP of the AWS node"
-}
-
-output "kubeconfig" {
-  value = module.master.kubeconfig
-  description = "kubeconfig of the cluster created"
-}
-
-output "rendered_template" {
-  value = module.master.rendered_template
-}
\ No newline at end of file

tests/terraform/modules/k3scluster/providers.tf

-provider "aws" {
-    region = "${var.region}"
-}

tests/terraform/modules/k3scluster/variables.tf

-#variable "db" {}
-variable "no_of_worker_nodes" {}
-variable "aws_ami" {}
-variable "aws_user" {}
-variable "region" {}
-variable "access_key" {}
-variable "vpc_id" {}
-variable "subnets" {}
-variable "qa_space" {}
-variable "resource_name" {}
-variable "key_name" {}
-variable "external_db" {}
-variable "external_db_version" {}
-variable "instance_class" {}
-variable "ec2_instance_class" {}
-variable "db_group_name" {}
-variable "username" {}
-variable "password" {}
-variable "k3s_version" {}
-variable "no_of_server_nodes" {}
-variable "server_flags" {}
-variable "worker_flags" {}
-variable "availability_zone" {}
-variable "sg_id" {}
-variable "cluster_type" {}
-variable "node_os" {}
-variable "db_username" {}
-variable "db_password" {}
-variable "environment" {}
-variable "engine_mode" {}
-variable "install_mode" {}
-variable "create_lb" {
-  description = "Create Network Load Balancer if set to true"
-  type = bool
-}
\ No newline at end of file

tests/terraform/modules/k3scluster/worker/instances_worker.tf

-resource "aws_instance" "worker" {
-  depends_on = [
-    var.dependency
-  ]
-  ami                    = var.aws_ami
-  instance_type          = var.ec2_instance_class
-  count                  = var.no_of_worker_nodes
-  connection {
-    type                 = "ssh"
-    user                 = var.aws_user
-    host                 = self.public_ip
-    private_key          = file(var.access_key)
-  }
-  subnet_id              = var.subnets
-  availability_zone      = var.availability_zone
-  vpc_security_group_ids = [var.sg_id]
-  key_name               = var.key_name
-  tags = {
-    Name                 = "${var.resource_name}-worker"
-  }
-  provisioner "file" {
-    source = "join_k3s_agent.sh"
-    destination = "/tmp/join_k3s_agent.sh"
-  }
-  provisioner "remote-exec" {
-    inline = [
-      "chmod +x /tmp/join_k3s_agent.sh",
-      "sudo /tmp/join_k3s_agent.sh ${var.node_os} ${var.install_mode} ${var.k3s_version} ${local.master_ip} ${local.node_token} ${self.public_ip} \"${var.worker_flags}\" ${var.username} ${var.password} ",
-    ]
-  }
-}
-
-data "local_file" "master_ip" {
-  depends_on = [var.dependency]
-  filename = "/tmp/${var.resource_name}_master_ip"
-}
-
-locals {
-  master_ip = trimspace(data.local_file.master_ip.content)
-}
-
-data "local_file" "token" {
-  depends_on = [var.dependency]
-  filename = "/tmp/${var.resource_name}_nodetoken"
-}
-
-locals {
-  node_token = trimspace(data.local_file.token.content)
-}

tests/terraform/modules/k3scluster/worker/outputs.tf

-output "Registration_address" {
-  value = "${data.local_file.master_ip.content}"
-}
-
-output "master_node_token" {
-  value = "${data.local_file.token.content}"
-}
-
-output "worker_ips" {
-  value = join("," , aws_instance.worker.*.public_ip)
-  description = "The public IP of the AWS node"
-}

tests/terraform/modules/k3scluster/worker/providers.tf

-provider "aws" {
-    region = "${var.region}"
-}

tests/terraform/modules/k3scluster/worker/variables.tf

-variable "dependency" {
-  type    = any
-  default = null
-}
-variable "region" {}
-variable "aws_ami" {}
-variable "aws_user" {}
-variable "vpc_id" {}
-variable "subnets" {}
-variable "resource_name" {}
-variable "access_key" {}
-variable "k3s_version" {}
-variable "no_of_worker_nodes" {}
-variable "worker_flags" {}
-variable "ec2_instance_class" {}
-variable "availability_zone" {}
-variable "sg_id" {}
-variable "username" {}
-variable "password" {}
-variable "node_os" {}
-variable "install_mode" {}
-variable "key_name" {}

tests/terraform/scripts/delete_resources.sh

-#!/bin/bash
-
-#Get resource name from tfvarslocal && change name to make more sense in this context
-RESOURCE_NAME=$(grep resource_name <modules/k3scluster/config/local.tfvars | cut -d= -f2 | tr -d ' "')
-NAME_PREFIX="$RESOURCE_NAME"
-
-#Terminate the instances
-echo "Terminating resources for $NAME_PREFIX if still up and running"
-# shellcheck disable=SC2046
-aws ec2 terminate-instances --instance-ids $(aws ec2 describe-instances \
-  --filters "Name=tag:Name,Values=${NAME_PREFIX}*" \
-  "Name=instance-state-name,Values=running" --query \
-  'Reservations[].Instances[].InstanceId' --output text) > /dev/null 2>&1
-
-
-#Search for DB instances and delete them
-INSTANCES=$(aws rds describe-db-instances --query "DBInstances[?starts_with(DBInstanceIdentifier,
-'${NAME_PREFIX}')].DBInstanceIdentifier" --output text 2> /dev/null)
-for instance in $INSTANCES; do
-  aws rds delete-db-instance --db-instance-identifier "$instance" --skip-final-snapshot > /dev/null 2>&1
-done
-
-
-#Search for DB clusters and delete them
-CLUSTERS=$(aws rds describe-db-clusters --query "DBClusters[?starts_with(DBClusterIdentifier,
- '${NAME_PREFIX}')].DBClusterIdentifier" --output text 2> /dev/null)
-for cluster in $CLUSTERS; do
-  aws rds delete-db-cluster --db-cluster-identifier "$cluster" --skip-final-snapshot > /dev/null 2>&1
-  aws rds wait db-cluster-deleted --db-cluster-identifier "$cluster"
-done
-
-
-#Get the list of load balancer ARNs
-LB_ARN_LIST=$(aws elbv2 describe-load-balancers \
-  --query "LoadBalancers[?starts_with(LoadBalancerName, '${NAME_PREFIX}') && Type=='network'].LoadBalancerArn" \
-  --output text)
-
-
-#Loop through the load balancer ARNs and delete the load balancers
-for LB_ARN in $LB_ARN_LIST; do
-  echo "Deleting load balancer $LB_ARN"
-  aws elbv2 delete-load-balancer --load-balancer-arn "$LB_ARN"
-done
-
-#Get the list of target group ARNs
-TG_ARN_LIST=$(aws elbv2 describe-target-groups \
-  --query "TargetGroups[?starts_with(TargetGroupName, '${NAME_PREFIX}') && Protocol=='TCP'].TargetGroupArn" \
-  --output text)
-
-
-#Loop through the target group ARNs and delete the target groups
-for TG_ARN in $TG_ARN_LIST; do
-  echo "Deleting target group $TG_ARN"
-  aws elbv2 delete-target-group --target-group-arn "$TG_ARN"
-done
-
-
-#Get the ID and recordName with lower case of the hosted zone that contains the Route 53 record sets
-NAME_PREFIX_LOWER=$(echo "$NAME_PREFIX" | tr '[:upper:]' '[:lower:]')
-R53_ZONE_ID=$(aws route53 list-hosted-zones-by-name --dns-name "${NAME_PREFIX}." \
-  --query "HostedZones[0].Id" --output text)
-R53_RECORD=$(aws route53 list-resource-record-sets \
-  --hosted-zone-id "${R53_ZONE_ID}" \
-  --query "ResourceRecordSets[?starts_with(Name, '${NAME_PREFIX_LOWER}.') && Type == 'CNAME'].Name" \
-  --output text)
-
-
-#Get ResourceRecord Value
-RECORD_VALUE=$(aws route53 list-resource-record-sets \
-  --hosted-zone-id "${R53_ZONE_ID}" \
-  --query "ResourceRecordSets[?starts_with(Name, '${NAME_PREFIX_LOWER}.') \
-    && Type == 'CNAME'].ResourceRecords[0].Value" --output text)
-
-
-#Delete Route53 record
-if [[ "$R53_RECORD" == "${NAME_PREFIX_LOWER}."* ]]; then
-  echo "Deleting Route53 record ${R53_RECORD}"
-  CHANGE_STATUS=$(aws route53 change-resource-record-sets --hosted-zone-id "${R53_ZONE_ID}" \
-    --change-batch '{"Changes": [
-            {
-                "Action": "DELETE",
-                "ResourceRecordSet": {
-                    "Name": "'"${R53_RECORD}"'",
-                    "Type": "CNAME",
-                    "TTL": 300,
-                    "ResourceRecords": [
-                        {
-                            "Value": "'"${RECORD_VALUE}"'"
-                        }
-                    ]
-                }
-            }
-        ]
-    }')
-  STATUS_ID=$(echo "$CHANGE_STATUS" | jq -r '.ChangeInfo.Id')
-  #Get status from the change
-  aws route53 wait resource-record-sets-changed --id "$STATUS_ID"
-  echo "Successfully deleted Route53 record ${R53_RECORD}: status: ${STATUS_ID}"
-else
-  echo "No Route53 record found"
-fi
\ No newline at end of file

tests/terraform/testutils.go

-package terraform
-
-import (
-	"bytes"
-	"fmt"
-	"log"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"runtime"
-	"strings"
-	"time"
-
-	"golang.org/x/crypto/ssh"
-)
-
-var config *ssh.ClientConfig
-
-type Node struct {
-	Name       string
-	Status     string
-	Roles      string
-	Version    string
-	InternalIP string
-	ExternalIP string
-}
-
-type Pod struct {
-	NameSpace string
-	Name      string
-	Ready     string
-	Status    string
-	Restarts  string
-	NodeIP    string
-	Node      string
-}
-
-func GetBasepath() string {
-	_, b, _, _ := runtime.Caller(0)
-	return filepath.Join(filepath.Dir(b), "../..")
-}
-
-func checkError(e error) {
-	if e != nil {
-		log.Fatal(e)
-	}
-}
-
-func publicKey(path string) ssh.AuthMethod {
-	key, err := os.ReadFile(path)
-	if err != nil {
-		panic(err)
-	}
-	signer, err := ssh.ParsePrivateKey(key)
-	if err != nil {
-		panic(err)
-	}
-	return ssh.PublicKeys(signer)
-}
-
-func ConfigureSSH(host string, SSHUser string, SSHKey string) *ssh.Client {
-	config = &ssh.ClientConfig{
-		User: SSHUser,
-		Auth: []ssh.AuthMethod{
-			publicKey(SSHKey),
-		},
-		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
-	}
-	conn, err := ssh.Dial("tcp", host, config)
-	checkError(err)
-	return conn
-}
-
-func runsshCommand(cmd string, conn *ssh.Client) (string, error) {
-	session, err := conn.NewSession()
-	if err != nil {
-		panic(err)
-	}
-	defer session.Close()
-	var stdoutBuf bytes.Buffer
-	var stderrBuf bytes.Buffer
-	session.Stdout = &stdoutBuf
-	session.Stderr = &stderrBuf
-	if err := session.Run(cmd); err != nil {
-		log.Println(session.Stdout)
-		log.Fatal("Error on command execution", err.Error())
-	}
-	return fmt.Sprintf("%s", stdoutBuf.String()), err
-}
-
-// RunCmdOnNode executes a command from within the given node
-func RunCmdOnNode(cmd string, ServerIP string, SSHUser string, SSHKey string) (string, error) {
-	Server := ServerIP + ":22"
-	conn := ConfigureSSH(Server, SSHUser, SSHKey)
-	res, err := runsshCommand(cmd, conn)
-	res = strings.TrimSpace(res)
-	return res, err
-}
-
-// RunCommand executes a command on the host
-func RunCommand(cmd string) (string, error) {
-	c := exec.Command("bash", "-c", cmd)
-	out, err := c.CombinedOutput()
-	return string(out), err
-}
-
-// CountOfStringInSlice Used to count the pods using prefix passed in the list of pods
-func CountOfStringInSlice(str string, pods []Pod) int {
-	count := 0
-	for _, pod := range pods {
-		if strings.Contains(pod.Name, str) {
-			count++
-		}
-	}
-	return count
-}
-
-// DeployWorkload deploys the workloads on the cluster from resource manifest files
-func DeployWorkload(workload, kubeconfig string, arch string) (string, error) {
-	resourceDir := GetBasepath() + "/tests/terraform/amd64_resource_files"
-	if arch == "arm64" {
-		resourceDir = GetBasepath() + "/tests/terraform/arm_resource_files"
-	}
-	files, err := os.ReadDir(resourceDir)
-	if err != nil {
-		err = fmt.Errorf("%s : Unable to read resource manifest file for %s", err, workload)
-		return "", err
-	}
-	fmt.Println("\nDeploying", workload)
-	for _, f := range files {
-		filename := filepath.Join(resourceDir, f.Name())
-		if strings.TrimSpace(f.Name()) == workload {
-			cmd := "kubectl apply -f " + filename + " --kubeconfig=" + kubeconfig
-			fmt.Println(cmd)
-			return RunCommand(cmd)
-		}
-	}
-	return "", nil
-}
-
-func FetchClusterIP(kubeconfig string, servicename string) (string, error) {
-	cmd := "kubectl get svc " + servicename + " -o jsonpath='{.spec.clusterIP}' --kubeconfig=" + kubeconfig
-	fmt.Println(cmd)
-	return RunCommand(cmd)
-}
-
-func FetchNodeExternalIP(kubeconfig string) []string {
-	cmd := "kubectl get node --output=jsonpath='{range .items[*]} { .status.addresses[?(@.type==\"ExternalIP\")].address}' --kubeconfig=" + kubeconfig
-	time.Sleep(10 * time.Second)
-	res, _ := RunCommand(cmd)
-	nodeExternalIP := strings.Trim(res, " ")
-	nodeExternalIPs := strings.Split(nodeExternalIP, " ")
-	return nodeExternalIPs
-}
-
-func FetchIngressIP(kubeconfig string) ([]string, error) {
-	cmd := "kubectl get ingress -o jsonpath='{.items[0].status.loadBalancer.ingress[*].ip}' --kubeconfig=" + kubeconfig
-	fmt.Println(cmd)
-	res, err := RunCommand(cmd)
-	if err != nil {
-		return nil, err
-	}
-	ingressIP := strings.Trim(res, " ")
-	fmt.Println(ingressIP)
-	ingressIPs := strings.Split(ingressIP, " ")
-	return ingressIPs, nil
-}
-
-// ParseNodes parses the nodes from the kubectl get nodes command
-// and returns a list of nodes
-func ParseNodes(kubeConfig string, print bool) ([]Node, error) {
-	nodes := make([]Node, 0, 10)
-	nodeList := ""
-
-	cmd := "kubectl get nodes --no-headers -o wide -A --kubeconfig=" + kubeConfig
-	res, err := RunCommand(cmd)
-
-	if err != nil {
-		return nil, err
-	}
-	nodeList = strings.TrimSpace(res)
-	split := strings.Split(nodeList, "\n")
-	for _, rec := range split {
-		if strings.TrimSpace(rec) != "" {
-			fields := strings.Fields(rec)
-			node := Node{
-				Name:       fields[0],
-				Status:     fields[1],
-				Roles:      fields[2],
-				Version:    fields[4],
-				InternalIP: fields[5],
-				ExternalIP: fields[6],
-			}
-			nodes = append(nodes, node)
-		}
-	}
-	if print {
-		fmt.Println(nodeList)
-	}
-	return nodes, nil
-}
-
-// ParsePods parses the pods from the kubectl get pods command
-// and returns a list of pods
-func ParsePods(kubeconfig string, print bool) ([]Pod, error) {
-	pods := make([]Pod, 0, 10)
-	podList := ""
-
-	cmd := "kubectl get pods -o wide --no-headers -A --kubeconfig=" + kubeconfig
-	res, _ := RunCommand(cmd)
-	res = strings.TrimSpace(res)
-	podList = res
-
-	split := strings.Split(res, "\n")
-	for _, rec := range split {
-		fields := strings.Fields(string(rec))
-		pod := Pod{
-			NameSpace: fields[0],
-			Name:      fields[1],
-			Ready:     fields[2],
-			Status:    fields[3],
-			Restarts:  fields[4],
-			NodeIP:    fields[6],
-			Node:      fields[7],
-		}
-		pods = append(pods, pod)
-	}
-	if print {
-		fmt.Println(podList)
-	}
-	return pods, nil
-}
-
-func PrintFileContents(f ...string) error {
-	for _, file := range f {
-		content, err := os.ReadFile(file)
-		if err != nil {
-			return err
-		}
-		fmt.Println(string(content) + "\n")
-	}
-
-	return nil
-}