Add make commands to terraform automation and fix external dbs related issue (#7159)

.gitignore

@@ -29,3 +29,7 @@ __pycache__
 /tests/.vscode
 /sonobuoy-output
 *.tmp
+config/local.tfvars
+*.terraform
+*.tfstate
+.terraform.lock.hcl

tests/terraform/.golangci.yaml

+linters:
+  enable:
+    - gofmt
+    - govet
+    - revive
+    - gosec
+    - megacheck
+    - misspell
+    - unparam
+    - exportloopref
+    - nlreturn
+    - nestif
+    - dupl
+    - gci
+    - ginkgolinter
+
+linters-settings:
+  govet:
+    check-shadowing: true
+    check-tests: true
+
+  nestif:
+    min-complexity: 4
+
+  revive:
+    confidence: 0.8
+    severity: warning
+    ignore-generated-header: true
+    rules:
+      - name: line-length-limit
+        arguments: [100]
+      - name: cognitive-complexity
+        arguments: [10]
+      - name: empty-lines
+      - name: empty-block
+      - name: bare-return
+      - name: blank-imports
+      - name: confusing-naming
+      - name: confusing-results
+      - name: context-as-argument
+      - name: duplicated-imports
+      - name: early-return
+      - name: empty-block
+      - name: empty-lines
+      - name: error-naming
+      - name: error-return
+      - name: error-strings
+      - name: errorf
+      - name: exported
+      - name: flag-parameter
+      - name: get-return
+      - name: if-return
+      - name: increment-decrement
+      - name: indent-error-flow
+      - name: import-shadowing
+      - name: modifies-parameter
+      - name: modifies-value-receiver
+      - name: range
+      - name: range-val-in-closure
+      - name: range-val-address
+      - name: receiver-naming
+      - name: string-of-int
+      - name: struct-tag
+      - name: superfluous-else
+      - name: time-naming
+      - name: var-declaration
+      - name: unconditional-recursion
+      - name: unexported-naming
+      - name: unexported-return
+      - name: unhandled-error
+        arguments: ["fmt.Printf", "builder.WriteString"]
+      - name: unnecessary-stmt
+      - name: unreachable-code
+      - name: unused-parameter
+      - name: unused-receiver
+
+issues:
+  exclude-rules:
+    - linters: [typecheck]
+      text: "command-line-arguments"
\ No newline at end of file

tests/terraform/Dockerfile.build

 FROM golang:alpine
 
-ARG TERRAFORM_VERSION=0.12.10
-ENV TERRAFORM_VERSION=$TERRAFORM_VERSION
-
+ARG TF_VERSION=1.4.0
+ENV TERRAFORM_VERSION $TF_VERSION
 
 RUN apk update && \
     apk upgrade --update-cache --available && \
-    apk add curl git jq bash openssh unzip gcc g++ make ca-certificates && \
-    curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" && \
+    apk add --no-cache curl git jq bash openssh unzip gcc g++ make ca-certificates && \
+    if [ "$(uname -m)" = "aarch64" ]; then \
+        KUBE_ARCH="linux/arm64" && \
+        TF_ARCH="linux_arm64"; \
+    else \
+        KUBE_ARCH="linux/amd64" && \
+        TF_ARCH="linux_amd64"; \
+    fi && \
+    curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/${KUBE_ARCH}/kubectl" && \
     chmod +x ./kubectl && \
     mv ./kubectl /usr/local/bin && \
     mkdir tmp && \
-    curl "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip" -o tmp/terraform.zip && \
+    curl "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_${TF_ARCH}.zip" -o tmp/terraform.zip && \
     unzip tmp/terraform.zip -d /usr/local/bin && \
     chmod +x /usr/local/bin/terraform && \
     rm -rf tmp
 
 WORKDIR $GOPATH/src/github.com/k3s-io/k3s
 
-COPY . .
-RUN go get github.com/gruntwork-io/terratest/modules/terraform
-RUN go get -u github.com/onsi/gomega
-RUN go get -u github.com/onsi/ginkgo/v2
-RUN go get -u golang.org/x/crypto/...
-RUN go get -u github.com/Thatooine/go-test-html-report
+COPY . .
\ No newline at end of file

tests/terraform/Makefile

+
+##========================= Terraform Tests =========================#
+include ./config.mk
+
+TAGNAME ?= default
+tf-up:
+	@cd ../.. && docker build . -q -f ./tests/terraform/Dockerfile.build -t k3s-tf-${TAGNAME}
+
+.PHONY: tf-run
+tf-run:
+	@docker run -d --name k3s-tf-test${IMGNAME} -t \
+      -e AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID}" \
+      -e AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY}" \
+      -v ${ACCESS_KEY_LOCAL}:/go/src/github.com/k3s-io/k3s/tests/terraform/modules/k3scluster/config/.ssh/aws_key.pem \
+      k3s-tf-${TAGNAME} sh -c 'cd ./tests/terraform ; \
+      				  if [ -n "${ARGNAME}" ]; then \
+                         go test -v -timeout=45m \
+                           ./${TESTDIR}/... \
+                           -"${ARGNAME}"="${ARGVALUE}"; \
+                       elif [ -z "${TESTDIR}" ]; then \
+                         go test -v -timeout=45m \
+                           ./createcluster/...; \
+                       else \
+                         go test -v -timeout=45m \
+                           ./${TESTDIR}/...; \
+                       fi'
+
+.PHONY: tf-logs
+tf-logs:
+	@docker logs -f k3s-tf-test${IMGNAME}
+
+.PHONY: tf-down
+tf-down:
+	@echo "Removing containers and images"
+	@docker stop $$(docker ps -a -q --filter="name=k3s-tf*")
+	@docker rm $$(docker ps -a -q --filter="name=k3s-tf*") ; \
+	 docker rmi --force $$(docker images -q --filter="reference=k3s-tf*")
+
+tf-clean:
+	@./scripts/delete_resources.sh
+
+.PHONY: tf-complete
+tf-complete: tf-clean tf-down tf-remove-state tf-up tf-run
+
+
+#========================= Run terraform tests locally =========================#
+
+.PHONY: tf-create
+tf-create:
+	@go test -timeout=45m -v ./createcluster/...
+
+.PHONY: tf-upgrade
+tf-upgrade:
+	@go test -timeout=45m -v ./upgradecluster/... -${ARGNAME}=${ARGVALUE}
+
+.PHONY: tf-remove-state
+tf-remove-state:
+	@rm -rf ./modules/k3scluster/.terraform
+	@rm -rf ./modules/k3scluster/.terraform.lock.hcl ./modules/k3scluster/terraform.tfstate ./modules/k3scluster/terraform.tfstate.backup
+
+.PHONY: tf-test-suite
+tf-test-suite:
+	@make tf-remove-state && make tf-create ; sleep 5 && \
+	  make tf-remove-state && make tf-upgrade ${ARGNAME}=${ARGVALUE}
+
+.PHONY: tf-test-suite-same-cluster
+tf-test-suite-same-cluster:
+	@make tf-create ; sleep 5 && make v ${ARGNAME}=${ARGVALUE}
+
+
+#========================= TestCode Static Quality Check =========================#
+.PHONY: vet-lint                   ## Run locally only inside Tests package
+vet-lint:
+	@echo "Running go vet and lint"
+	@go vet ./${TESTDIR} && golangci-lint run --tests
\ No newline at end of file

tests/terraform/README.md

@@ -14,11 +14,24 @@ See the [create cluster test](../tests/terraform/createcluster_test.go) as an ex
 
 ## Running
 
-Before running the tests, it's best to create a tfvars file in `./tests/terraform/modules/k3scluster/config/local.tfvars`. There is some information there to get you started, but the empty variables should be filled in appropriately per your AWS environment.
+- Before running the tests, you should creat local.tfvars file in `./tests/terraform/modules/k3scluster/config/local.tfvars`. There is some information there to get you started, but the empty variables should be filled in appropriately per your AWS environment.
+
+
+
+- For running tests with "etcd" cluster type, you should add the value "etcd" to the variable "cluster_type" , also you need have those variables at least empty:
+```
+- external_db       
+- external_db_version
+- instance_class  
+- db_group_name
+```
+
+- For running with external db you need the same variables above filled in with the correct data and also cluster_type= ""
+
 
 All TF tests can be run with:
 ```bash
-go test -timeout=60m ./tests/terrfaorm/... -run TF
+go test -timeout=60m ./tests/terraform/... -run TF
 ```
 Tests can be run individually with:
 ```bash
@@ -27,10 +40,79 @@ go test -timeout=30m ./tests/terraform/createcluster/createcluster.go ./tests/te
 go test -v -timeout=30m ./tests/terraform/... -run TFClusterCreateValidation
 # example with vars:
 go test -timeout=30m -v ./tests/terraform/createcluster.go ./tests/terraform/createcluster_test.go -node_os=ubuntu -aws_ami=ami-02f3416038bdb17fb -cluster_type=etcd -resource_name=localrun1 -sshuser=ubuntu -sshkey="key-name" -destroy=false
+
 ```
+Test Flags:
+```
+- ${upgradeVersion} version to upgrade to
+```
+We can also run tests through the Makefile through tests' directory:
+
+- On the first run with make and docker please delete your .terraform folder, terraform.tfstate and terraform.hcl.lock file
+
+```bash
+Args:
+*All args are optional and can be used with:
+
+`$make tf-run`         `$make tf-logs`,
+`$make vet-lint`       `$make tf-complete`, 
+`$make tf-upgrade`     `$make tf-test-suite-same-cluster`,
+`$make tf-test-suite`
+
+
+- ${IMGNAME}     append any string to the end of image name
+- ${TAGNAME}     append any string to the end of tag name
+- ${ARGNAME}     name of the arg to pass to the test
+- ${ARGVALUE}    value of the arg to pass to the test
+- ${TESTDIR}     path to the test directory 
+
+Commands:
+$ make tf-up                         # create the image from Dockerfile.build
+$ make tf-run                        # runs all tests if no flags or args provided
+$ make tf-down                       # removes the image
+$ make tf-clean                      # removes instances and resources created by tests
+$ make tf-logs                       # prints logs from container the tests
+$ make tf-complete                   # clean resources + remove images + run tests
+$ make tf-create                     # runs create cluster test locally
+$ make tf-upgrade                    # runs upgrade cluster test locally
+$ make tf-test-suite-same-cluster    # runs all tests locally in sequence using the same state    
+$ make tf-remove-state               # removes terraform state dir and files
+$ make tf-test-suite                 # runs all tests locally in sequence not using the same state
+$ make vet-lint                      # runs go vet and go lint
+
+      
+Examples:
+$ make tf-up TAGNAME=ubuntu
+$ make tf-run IMGNAME=2 TAGNAME=ubuntu TESTDIR=upgradecluster ARGNAME=upgradeVersion ARGVALUE=v1.26.2+k3s1
+$ make tf-run TESTDIR=upgradecluster
+$ make tf-logs IMGNAME=1
+$ make vet-lint TESTDIR=upgradecluster
+```
+
+
+# Running tests in parallel:
+- You can play around and have a lot of different test combinations like:
+```
+- Build docker image with different TAGNAME="OS`s" + with different configurations( resource_name, node_os, versions, install type, nodes and etc) and have unique "IMGNAMES"
+
+- And in the meanwhile run also locally with different configuration while your dockers TAGNAME and IMGNAMES are running
+```
+
+# In between tests:
+- If you want to run with same cluster do not delete ./tests/terraform/modules/terraform.tfstate + .terraform.lock.hcl file after each test.
+
+- if you want to use new resources then make sure to delete the ./tests/terraform/modules/terraform.tfstate + .terraform.lock.hcl file if you want to create a new cluster.
+
+
+# Common Issues:
+
+- Issues related to terraform plugin please also delete the modules/.terraform folder
+- In mac m1 maybe you need also to go to rke2/tests/terraform/modules and run `terraform init` to download the plugins
+
+
 
-In between tests, if the cluster is not destroyed, then make sure to delete the ./tests/terraform/terraform.tfstate file if you want to create a new cluster.
 
+# Reporting:
 Additionally, to generate junit reporting for the tests, the Ginkgo CLI is used. Installation instructions can be found [here.](https://onsi.github.io/ginkgo/#getting-started)  
 
 To run the all TF tests and generate JUnit testing reports:

tests/terraform/config.mk

+SHELL := /bin/bash
+
+LOCAL_TFVARS_PATH := modules/k3scluster/config/local.tfvars
+
+ifeq ($(wildcard ${LOCAL_TFVARS_PATH}),)
+  RESOURCE_NAME :=
+else
+  export RESOURCE_NAME := $(shell sed -n 's/resource_name *= *"\([^"]*\)"/\1/p' ${LOCAL_TFVARS_PATH})
+endif
+
+export ACCESS_KEY_LOCAL
+export AWS_ACCESS_KEY_ID
+export AWS_SECRET_ACCESS_KEY
\ No newline at end of file

tests/terraform/createcluster/createcluster.go

@@ -2,6 +2,7 @@ package createcluster
 
 import (
 	"fmt"
+	"strconv"
 
 	"path/filepath"
 	"testing"
@@ -11,59 +12,52 @@ import (
 )
 
 var (
-	KubeConfigFile string
-	MasterIPs      string
-	WorkerIPs      string
+	KubeConfigFile   string
+	MasterIPs        string
+	WorkerIPs        string
+	NumServers       int
+	NumWorkers       int
+	AwsUser          string
+	AccessKey        string
+	RenderedTemplate string
+	ExternalDb       string
+	ClusterType      string
+	TfVarsPath       = "/tests/terraform/modules/k3scluster/config/local.tfvars"
+	modulesPath      = "/tests/terraform/modules/k3scluster"
 )
 
-type options struct {
-	nodeOs       string
-	awsAmi       string
-	clusterType  string
-	resourceName string
-	externalDb   string
-	sshuser      string
-	sshkey       string
-	accessKey    string
-	serverNodes  int
-	workerNodes  int
-}
-
-func ClusterOptions(os ...ClusterOption) map[string]interface{} {
-	opts := options{}
-	for _, o := range os {
-		opts = o(opts)
-	}
-	return map[string]interface{}{
-		"node_os":            opts.nodeOs,
-		"aws_ami":            opts.awsAmi,
-		"cluster_type":       opts.clusterType,
-		"resource_name":      opts.resourceName,
-		"external_db":        opts.externalDb,
-		"aws_user":           opts.sshuser,
-		"key_name":           opts.sshkey,
-		"access_key":         opts.accessKey,
-		"no_of_server_nodes": opts.serverNodes,
-		"no_of_worker_nodes": opts.workerNodes,
-	}
-}
-
-func BuildCluster(t *testing.T, tfVarsPath string, destroy bool, terraformVars map[string]interface{}) (string, error) {
+func BuildCluster(t *testing.T, destroy bool) (string, error) {
 	basepath := tf.GetBasepath()
-	tfDir, err := filepath.Abs(basepath + "/tests/terraform/modules/k3scluster")
+	tfDir, err := filepath.Abs(basepath + modulesPath)
 	if err != nil {
 		return "", err
 	}
-	varDir, err := filepath.Abs(basepath + tfVarsPath)
+	varDir, err := filepath.Abs(basepath + TfVarsPath)
 	if err != nil {
 		return "", err
 	}
 	TerraformOptions := &terraform.Options{
 		TerraformDir: tfDir,
 		VarFiles:     []string{varDir},
-		Vars:         terraformVars,
 	}
 
+	NumServers, err = strconv.Atoi(terraform.GetVariableAsStringFromVarFile(t, varDir,
+		"no_of_server_nodes"))
+	if err != nil {
+		return "", err
+	}
+
+	NumWorkers, err = strconv.Atoi(terraform.GetVariableAsStringFromVarFile(t, varDir,
+		"no_of_worker_nodes"))
+	if err != nil {
+		return "", err
+	}
+
+	ClusterType = terraform.GetVariableAsStringFromVarFile(t, varDir, "cluster_type")
+	ExternalDb = terraform.GetVariableAsStringFromVarFile(t, varDir, "external_db")
+	AwsUser = terraform.GetVariableAsStringFromVarFile(t, varDir, "aws_user")
+	AccessKey = terraform.GetVariableAsStringFromVarFile(t, varDir, "access_key")
+
 	if destroy {
 		fmt.Printf("Cluster is being deleted")
 		terraform.Destroy(t, TerraformOptions)
@@ -71,72 +65,12 @@ func BuildCluster(t *testing.T, tfVarsPath string, destroy bool, terraformVars m
 	}
 
 	fmt.Printf("Creating Cluster")
+
 	terraform.InitAndApply(t, TerraformOptions)
 	KubeConfigFile = "/tmp/" + terraform.Output(t, TerraformOptions, "kubeconfig") + "_kubeconfig"
 	MasterIPs = terraform.Output(t, TerraformOptions, "master_ips")
 	WorkerIPs = terraform.Output(t, TerraformOptions, "worker_ips")
-	return "cluster created", err
-}
+	RenderedTemplate = terraform.Output(t, TerraformOptions, "rendered_template")
 
-type ClusterOption func(o options) options
-
-func NodeOs(n string) ClusterOption {
-	return func(o options) options {
-		o.nodeOs = n
-		return o
-	}
-}
-func AwsAmi(n string) ClusterOption {
-	return func(o options) options {
-		o.awsAmi = n
-		return o
-	}
-}
-func ClusterType(n string) ClusterOption {
-	return func(o options) options {
-		o.clusterType = n
-		return o
-	}
-}
-func ResourceName(n string) ClusterOption {
-	return func(o options) options {
-		o.resourceName = n
-		return o
-	}
-}
-func ExternalDb(n string) ClusterOption {
-	return func(o options) options {
-		o.externalDb = n
-		return o
-	}
-}
-func Sshuser(n string) ClusterOption {
-	return func(o options) options {
-		o.sshuser = n
-		return o
-	}
-}
-func Sshkey(n string) ClusterOption {
-	return func(o options) options {
-		o.sshkey = n
-		return o
-	}
-}
-func AccessKey(n string) ClusterOption {
-	return func(o options) options {
-		o.accessKey = n
-		return o
-	}
-}
-func ServerNodes(n int) ClusterOption {
-	return func(o options) options {
-		o.serverNodes = n
-		return o
-	}
-}
-func WorkerNodes(n int) ClusterOption {
-	return func(o options) options {
-		o.workerNodes = n
-		return o
-	}
+	return "cluster created", err
 }

tests/terraform/modules/k3scluster/install_k3s_master.sh

@@ -25,7 +25,7 @@ export "${3}"="${4}"
 if [ "${5}" = "etcd" ]
 then
    echo "CLUSTER TYPE  is etcd"
-   if [[ "$4" == *"v1.18"* ]] || [["$4" == *"v1.17"* ]] && [[ -n "$8" ]]
+   if [[ "$4" == *"v1.18"* ]] || [[ "$4" == *"v1.17"* ]] && [[ -n "$8" ]]
    then
        echo "curl -sfL https://get.k3s.io | INSTALL_K3S_TYPE='server' sh -s - --cluster-init --node-external-ip=${6} $8" >/tmp/master_cmd
        curl -sfL https://get.k3s.io | INSTALL_K3S_TYPE='server' sh -s - --cluster-init --node-external-ip="${6}" "$8"
@@ -35,7 +35,6 @@ then
    fi
 else
    echo "CLUSTER TYPE is external db"
-   echo "$8"
    if [[ "$4" == *"v1.18"* ]] || [[ "$4" == *"v1.17"* ]] && [[ -n "$8" ]]
    then
        echo "curl -sfL https://get.k3s.io | sh -s - server --node-external-ip=${6} --datastore-endpoint=\"${7}\" $8"  >/tmp/master_cmd

tests/terraform/modules/k3scluster/master/outputs.tf

@@ -12,3 +12,7 @@ output "kubeconfig" {
   value = var.resource_name
   description = "kubeconfig of the cluster created"
 }
+
+output "rendered_template" {
+  value = data.template_file.test.rendered
+}
\ No newline at end of file

tests/terraform/modules/k3scluster/master/variables.tf

@@ -10,11 +10,9 @@ variable "qa_space" {}
 variable "ec2_instance_class" {}
 variable "resource_name" {}
 variable "key_name" {}
-
 variable "external_db" {}
 variable "external_db_version" {}
 variable "instance_class" {}
-
 variable "db_group_name" {}
 variable "username" {}
 variable "password" {}

tests/terraform/modules/k3scluster/outputs.tf

@@ -12,3 +12,7 @@ output "kubeconfig" {
   value = module.master.kubeconfig
   description = "kubeconfig of the cluster created"
 }
+
+output "rendered_template" {
+  value = module.master.rendered_template
+}
\ No newline at end of file

tests/terraform/scripts/delete_resources.sh

+#!/bin/bash
+
+#Get resource name from tfvarslocal && change name to make more sense in this context
+RESOURCE_NAME=$(grep resource_name <modules/k3scluster/config/local.tfvars | cut -d= -f2 | tr -d ' "')
+NAME_PREFIX="$RESOURCE_NAME"
+
+#Terminate the instances
+echo "Terminating resources for $NAME_PREFIX if still up and running"
+# shellcheck disable=SC2046
+aws ec2 terminate-instances --instance-ids $(aws ec2 describe-instances \
+  --filters "Name=tag:Name,Values=${NAME_PREFIX}*" \
+  "Name=instance-state-name,Values=running" --query \
+  'Reservations[].Instances[].InstanceId' --output text) > /dev/null 2>&1
+
+
+#Search for DB instances and delete them
+INSTANCES=$(aws rds describe-db-instances --query "DBInstances[?starts_with(DBInstanceIdentifier,
+'${NAME_PREFIX}')].DBInstanceIdentifier" --output text 2> /dev/null)
+for instance in $INSTANCES; do
+  aws rds delete-db-instance --db-instance-identifier "$instance" --skip-final-snapshot > /dev/null 2>&1
+done
+
+
+#Search for DB clusters and delete them
+CLUSTERS=$(aws rds describe-db-clusters --query "DBClusters[?starts_with(DBClusterIdentifier,
+ '${NAME_PREFIX}')].DBClusterIdentifier" --output text 2> /dev/null)
+for cluster in $CLUSTERS; do
+  aws rds delete-db-cluster --db-cluster-identifier "$cluster" --skip-final-snapshot > /dev/null 2>&1
+  aws rds wait db-cluster-deleted --db-cluster-identifier "$cluster"
+done
+
+
+#Get the list of load balancer ARNs
+LB_ARN_LIST=$(aws elbv2 describe-load-balancers \
+  --query "LoadBalancers[?starts_with(LoadBalancerName, '${NAME_PREFIX}') && Type=='network'].LoadBalancerArn" \
+  --output text)
+
+
+#Loop through the load balancer ARNs and delete the load balancers
+for LB_ARN in $LB_ARN_LIST; do
+  echo "Deleting load balancer $LB_ARN"
+  aws elbv2 delete-load-balancer --load-balancer-arn "$LB_ARN"
+done
+
+#Get the list of target group ARNs
+TG_ARN_LIST=$(aws elbv2 describe-target-groups \
+  --query "TargetGroups[?starts_with(TargetGroupName, '${NAME_PREFIX}') && Protocol=='TCP'].TargetGroupArn" \
+  --output text)
+
+
+#Loop through the target group ARNs and delete the target groups
+for TG_ARN in $TG_ARN_LIST; do
+  echo "Deleting target group $TG_ARN"
+  aws elbv2 delete-target-group --target-group-arn "$TG_ARN"
+done
+
+
+#Get the ID and recordName with lower case of the hosted zone that contains the Route 53 record sets
+NAME_PREFIX_LOWER=$(echo "$NAME_PREFIX" | tr '[:upper:]' '[:lower:]')
+R53_ZONE_ID=$(aws route53 list-hosted-zones-by-name --dns-name "${NAME_PREFIX}." \
+  --query "HostedZones[0].Id" --output text)
+R53_RECORD=$(aws route53 list-resource-record-sets \
+  --hosted-zone-id "${R53_ZONE_ID}" \
+  --query "ResourceRecordSets[?starts_with(Name, '${NAME_PREFIX_LOWER}.') && Type == 'CNAME'].Name" \
+  --output text)
+
+
+#Get ResourceRecord Value
+RECORD_VALUE=$(aws route53 list-resource-record-sets \
+  --hosted-zone-id "${R53_ZONE_ID}" \
+  --query "ResourceRecordSets[?starts_with(Name, '${NAME_PREFIX_LOWER}.') \
+    && Type == 'CNAME'].ResourceRecords[0].Value" --output text)
+
+
+#Delete Route53 record
+if [[ "$R53_RECORD" == "${NAME_PREFIX_LOWER}."* ]]; then
+  echo "Deleting Route53 record ${R53_RECORD}"
+  CHANGE_STATUS=$(aws route53 change-resource-record-sets --hosted-zone-id "${R53_ZONE_ID}" \
+    --change-batch '{"Changes": [
+            {
+                "Action": "DELETE",
+                "ResourceRecordSet": {
+                    "Name": "'"${R53_RECORD}"'",
+                    "Type": "CNAME",
+                    "TTL": 300,
+                    "ResourceRecords": [
+                        {
+                            "Value": "'"${RECORD_VALUE}"'"
+                        }
+                    ]
+                }
+            }
+        ]
+    }')
+  STATUS_ID=$(echo "$CHANGE_STATUS" | jq -r '.ChangeInfo.Id')
+  #Get status from the change
+  aws route53 wait resource-record-sets-changed --id "$STATUS_ID"
+  echo "Successfully deleted Route53 record ${R53_RECORD}: status: ${STATUS_ID}"
+else
+  echo "No Route53 record found"
+fi
\ No newline at end of file

tests/terraform/testutils.go

@@ -14,6 +14,8 @@ import (
 	"golang.org/x/crypto/ssh"
 )
 
+var config *ssh.ClientConfig
+
 type Node struct {
 	Name       string
 	Status     string
@@ -33,11 +35,6 @@ type Pod struct {
 	Node      string
 }
 
-var config *ssh.ClientConfig
-var SSHKEY string
-var SSHUSER string
-var err error
-
 func GetBasepath() string {
 	_, b, _, _ := runtime.Caller(0)
 	return filepath.Join(filepath.Dir(b), "../..")
@@ -91,10 +88,6 @@ func runsshCommand(cmd string, conn *ssh.Client) (string, error) {
 	return fmt.Sprintf("%s", stdoutBuf.String()), err
 }
 
-// nodeOs: ubuntu centos7 centos8 sles15
-// clusterType arm, etcd externaldb, if external_db var is not "" picks database from the vars file,
-// resourceName: name to resource created timestamp attached
-
 // RunCmdOnNode executes a command from within the given node
 func RunCmdOnNode(cmd string, ServerIP string, SSHUser string, SSHKey string) (string, error) {
 	Server := ServerIP + ":22"
@@ -122,6 +115,7 @@ func CountOfStringInSlice(str string, pods []Pod) int {
 	return count
 }
 
+// DeployWorkload deploys the workloads on the cluster from resource manifest files
 func DeployWorkload(workload, kubeconfig string, arch string) (string, error) {
 	resourceDir := GetBasepath() + "/tests/terraform/amd64_resource_files"
 	if arch == "arm64" {
@@ -172,6 +166,8 @@ func FetchIngressIP(kubeconfig string) ([]string, error) {
 	return ingressIPs, nil
 }
 
+// ParseNodes parses the nodes from the kubectl get nodes command
+// and returns a list of nodes
 func ParseNodes(kubeConfig string, print bool) ([]Node, error) {
 	nodes := make([]Node, 0, 10)
 	nodeList := ""
@@ -204,6 +200,8 @@ func ParseNodes(kubeConfig string, print bool) ([]Node, error) {
 	return nodes, nil
 }
 
+// ParsePods parses the pods from the kubectl get pods command
+// and returns a list of pods
 func ParsePods(kubeconfig string, print bool) ([]Pod, error) {
 	pods := make([]Pod, 0, 10)
 	podList := ""
@@ -232,3 +230,15 @@ func ParsePods(kubeconfig string, print bool) ([]Pod, error) {
 	}
 	return pods, nil
 }
+
+func PrintFileContents(f ...string) error {
+	for _, file := range f {
+		content, err := os.ReadFile(file)
+		if err != nil {
+			return err
+		}
+		fmt.Println(string(content) + "\n")
+	}
+
+	return nil
+}