Skip to content

K
k3s
Unverified Commit 97c69546 authored by ShylajaDevadiga's avatar ShylajaDevadiga Committed by GitHub
Browse files
Options

add arm tests and upgrade tests (#5526)

Signed-off-by: 's avatarShylaja Devadiga <shylaja@rancher.com>
parent a693071c
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 81 additions and 73 deletions
tests/terraform/Dockerfile.build
View file @ 97c69546
......@@ -19,7 +19,8 @@ RUN apk update && \
WORKDIR $GOPATH/src/github.com/k3s-io/k3s
COPY . .
RUN go install github.com/gruntwork-io/terratest/modules/terraform
RUN go install -u github.com/onsi/gomega
RUN go install -u github.com/onsi/ginkgo/v2
RUN go install -u golang.org/x/crypto/...
RUN go get github.com/gruntwork-io/terratest/modules/terraform
RUN go get -u github.com/onsi/gomega
RUN go get -u github.com/onsi/ginkgo/v2
RUN go get -u golang.org/x/crypto/...
RUN go get -u github.com/Thatooine/go-test-html-report
tests/terraform/arm_resource_files/ingress.yaml
View file @ 97c69546
apiVersion: extensions/v1beta1
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: ingress
name: test-ingress
spec:
rules:
- host: foo1.bar.com
http:
paths:
- path: /name.html
backend:
serviceName: nginx-ingress-svc
servicePort: 80
rules:
- host: foo1.bar.com
http:
paths:
- backend:
service:
name: nginx-ingress-svc
port:
number: 80
path: /
pathType: ImplementationSpecific
---
apiVersion: v1
kind: Service
metadata:
name: nginx-ingress-svc
labels:
k8s-app: nginx-app-ingress
name: nginx-ingress-svc
labels:
k8s-app: nginx-app-ingress
spec:
ports:
- port: 80
targetPort: 80
protocol: TCP
name: http
selector:
k8s-app: nginx-app-ingress
ports:
- port: 80
targetPort: 80
protocol: TCP
name: http
selector:
k8s-app: nginx-app-ingress
---
apiVersion: v1
kind: ReplicationController
metadata:
name: test-ingress
name: test-ingress
spec:
replicas: 2
selector:
k8s-app: nginx-app-ingress
template:
metadata:
labels:
k8s-app: nginx-app-ingress
spec:
terminationGracePeriodSeconds: 60
containers:
- name: testcontainer
image: shylajarancher19/shylajaarm64:v1.0
ports:
- containerPort: 80
replicas: 2
selector:
k8s-app: nginx-app-ingress
template:
metadata:
labels:
k8s-app: nginx-app-ingress
spec:
terminationGracePeriodSeconds: 60
containers:
- name: testcontainer
image: shylajarancher19/shylajaarm64:v1.0
ports:
- containerPort: 80
tests/terraform/createcluster.go
View file @ 97c69546
......@@ -3,6 +3,7 @@ package e2e
import (
"flag"
"fmt"
"path/filepath"
"testing"
......@@ -17,6 +18,7 @@ var clusterType = flag.String("cluster_type", "etcd", "a string")
var resourceName = flag.String("resource_name", "etcd", "a string")
var sshuser = flag.String("sshuser", "ubuntu", "a string")
var sshkey = flag.String("sshkey", "", "a string")
var failed = false
var (
kubeConfigFile string
......@@ -24,8 +26,7 @@ var (
workerIPs string
)
func BuildCluster(nodeOs, clusterType, externalDb, resourceName string, t *testing.T, destroy bool) (string, string, string, error) {
func BuildCluster(nodeOs, clusterType, externalDb, resourceName string, t *testing.T, destroy bool, arch string) (string, string, string, error) {
tDir := "./modules/k3scluster"
vDir := "/config/" + nodeOs + clusterType + ".tfvars"
......
tests/terraform/createcluster_test.go
View file @ 97c69546
......@@ -14,19 +14,16 @@ import (
func Test_E2EClusterCreateValidation(t *testing.T) {
RegisterFailHandler(Fail)
flag.Parse()
RunSpecs(t, "Create Cluster Test Suite")
}
var _ = Describe("Test:", func() {
Context("Build Cluster:", func() {
It("Starts up with no issues", func() {
kubeConfigFile, masterIPs, workerIPs, err = BuildCluster(*nodeOs, *clusterType, *externalDb, *resourceName, &testing.T{}, *destroy)
kubeConfigFile, masterIPs, workerIPs, err = BuildCluster(*nodeOs, *clusterType, *externalDb, *resourceName, &testing.T{}, *destroy, *arch)
Expect(err).NotTo(HaveOccurred())
defer GinkgoRecover()
if *destroy {
fmt.Printf("\nCluster is being Deleted\n")
return
}
fmt.Println("\nCLUSTER CONFIG:\nOS", *nodeOs, "BACKEND", *clusterType, *externalDb)
fmt.Printf("\nIPs:\n")
fmt.Println("Server Node IPS:", masterIPs)
......@@ -62,7 +59,7 @@ var _ = Describe("Test:", func() {
})
It("Verifies ClusterIP Service", func() {
_, err := DeployWorkload("clusterip.yaml", kubeConfigFile, false)
_, err := DeployWorkload("clusterip.yaml", kubeConfigFile, *arch)
Expect(err).NotTo(HaveOccurred(), "Cluster IP manifest not deployed")
Eventually(func(g Gomega) {
......@@ -81,13 +78,13 @@ var _ = Describe("Test:", func() {
res, err := RunCmdOnNode(cmd, ip, *sshuser, *sshkey)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(res).Should(ContainSubstring("test-clusterip"))
fmt.Println(res)
}, "420s", "10s").Should(Succeed())
}
})
It("Verifies NodePort Service", func() {
_, err := DeployWorkload("nodeport.yaml", kubeConfigFile, false)
_, err := DeployWorkload("nodeport.yaml", kubeConfigFile, *arch)
Expect(err).NotTo(HaveOccurred(), "NodePort manifest not deployed")
nodeExternalIP := FetchNodeExternalIP(kubeConfigFile)
cmd := "kubectl get service nginx-nodeport-svc --kubeconfig=" + kubeConfigFile + " --output jsonpath=\"{.spec.ports[0].nodePort}\""
......@@ -109,12 +106,13 @@ var _ = Describe("Test:", func() {
g.Expect(err).NotTo(HaveOccurred())
fmt.Println(res)
g.Expect(res).Should(ContainSubstring("test-nodeport"))
fmt.Println(res)
}, "240s", "5s").Should(Succeed())
}
})
It("Verifies LoadBalancer Service", func() {
_, err := DeployWorkload("loadbalancer.yaml", kubeConfigFile, false)
_, err := DeployWorkload("loadbalancer.yaml", kubeConfigFile, *arch)
Expect(err).NotTo(HaveOccurred(), "Loadbalancer manifest not deployed")
nodeExternalIP := FetchNodeExternalIP(kubeConfigFile)
cmd := "kubectl get service nginx-loadbalancer-svc --kubeconfig=" + kubeConfigFile + " --output jsonpath=\"{.spec.ports[0].port}\""
......@@ -136,12 +134,13 @@ var _ = Describe("Test:", func() {
g.Expect(err).NotTo(HaveOccurred())
fmt.Println(res)
g.Expect(res).Should(ContainSubstring("test-loadbalancer"))
fmt.Println(res)
}, "240s", "5s").Should(Succeed())
}
})
It("Verifies Ingress", func() {
_, err := DeployWorkload("ingress.yaml", kubeConfigFile, false)
_, err := DeployWorkload("ingress.yaml", kubeConfigFile, *arch)
Expect(err).NotTo(HaveOccurred(), "Ingress manifest not deployed")
Eventually(func(g Gomega) {
......@@ -162,12 +161,13 @@ var _ = Describe("Test:", func() {
res, err := RunCommand(cmd)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(res).Should(ContainSubstring("test-ingress"))
fmt.Println(res)
}, "240s", "5s").Should(Succeed())
}
})
It("Verifies Daemonset", func() {
_, err := DeployWorkload("daemonset.yaml", kubeConfigFile, false)
_, err := DeployWorkload("daemonset.yaml", kubeConfigFile, *arch)
Expect(err).NotTo(HaveOccurred(), "Daemonset manifest not deployed")
nodes, _ := ParseNodes(kubeConfigFile, false)
......@@ -184,7 +184,7 @@ var _ = Describe("Test:", func() {
})
It("Verifies Local Path Provisioner storage ", func() {
_, err := DeployWorkload("local-path-provisioner.yaml", kubeConfigFile, false)
_, err := DeployWorkload("local-path-provisioner.yaml", kubeConfigFile, *arch)
Expect(err).NotTo(HaveOccurred(), "local-path-provisioner manifest not deployed")
Eventually(func(g Gomega) {
......@@ -200,7 +200,6 @@ var _ = Describe("Test:", func() {
cmd := "kubectl get pod volume-test --kubeconfig=" + kubeConfigFile
res, err := RunCommand(cmd)
Expect(err).NotTo(HaveOccurred())
fmt.Println(res)
g.Expect(res).Should(ContainSubstring("volume-test"))
g.Expect(res).Should(ContainSubstring("Running"))
}, "420s", "2s").Should(Succeed())
......@@ -215,7 +214,7 @@ var _ = Describe("Test:", func() {
Expect(err).NotTo(HaveOccurred())
fmt.Println(res)
_, err = DeployWorkload("local-path-provisioner.yaml", kubeConfigFile, false)
_, err = DeployWorkload("local-path-provisioner.yaml", kubeConfigFile, *arch)
Expect(err).NotTo(HaveOccurred(), "local-path-provisioner manifest not deployed")
Eventually(func(g Gomega) {
......@@ -244,7 +243,7 @@ var _ = Describe("Test:", func() {
})
It("Verifies dns access", func() {
_, err := DeployWorkload("dnsutils.yaml", kubeConfigFile, false)
_, err := DeployWorkload("dnsutils.yaml", kubeConfigFile, *arch)
Expect(err).NotTo(HaveOccurred(), "dnsutils manifest not deployed")
Eventually(func(g Gomega) {
......@@ -260,15 +259,11 @@ var _ = Describe("Test:", func() {
res, _ := RunCommand(cmd)
fmt.Println(res)
g.Expect(res).Should(ContainSubstring("kubernetes.default.svc.cluster.local"))
fmt.Println(res)
}, "420s", "2s").Should(Succeed())
})
It("Validate Rebooting nodes", func() {
if *destroy {
return
}
defer GinkgoRecover()
nodeExternalIP := FetchNodeExternalIP(kubeConfigFile)
for _, ip := range nodeExternalIP {
fmt.Println("\nRebooting node: ", ip)
......@@ -303,16 +298,23 @@ var _ = Describe("Test:", func() {
})
})
var failed = false
var _ = AfterEach(func() {
failed = failed || CurrentGinkgoTestDescription().Failed
})
var _ = BeforeEach(func() {
failed = failed || CurrentGinkgoTestDescription().Failed
if *destroy {
fmt.Printf("\nCluster is being Deleted\n")
Skip("Cluster is being Deleted")
}
})
var _ = AfterSuite(func() {
if failed {
fmt.Println("FAILED!")
} else {
kubeConfigFile, masterIPs, workerIPs, err = BuildCluster(*nodeOs, *clusterType, *externalDb, *resourceName, &testing.T{}, true)
kubeConfigFile, masterIPs, workerIPs, err = BuildCluster(*nodeOs, *clusterType, *externalDb, *resourceName, &testing.T{}, true, *arch)
if err != nil {
fmt.Println("Error Destroying Cluster", err)
}
......
tests/terraform/jenkinsfile
View file @ 97c69546
......@@ -34,16 +34,16 @@ pipeline {
vpc_id="${env.VPC_ID}"
subnets="${env.SUBNETS}"
qa_space="${env.QA_SPACE}"
ec2_instance_class="${env.EC2_INSTANCE_CLASS}"
ec2_instance_class="${env.AWS_INSTANCE_TYPE}"
access_key="/config/$AWS_SSH_KEY_NAME"
no_of_worker_nodes="${env.NO_OF_WORKER_NODES}"
key_name="jenkins-rke-validation"
server_flags="${env.SERVER_FLAGS}"
worker_flags="${env.WORKER_FLAGS}"
server_flags="${env.K3S_SERVER_FLAGS}"
worker_flags="${env.K3S_WORKER_FLAGS}"
k3s_version="${env.K3S_VERSION}"
availability_zone="${env.AVAILABILITY_ZONE}"
sg_id="${env.SG_ID}"
install_mode="${env.INSTALL_MODE}"
install_mode="${env.K3S_INSTALL_MODE}"
resource_name="${env.RESOURCE_NAME}"
no_of_server_nodes="${env.NO_OF_SERVER_NODES}"
username="${env.RHEL_USERNAME}"
......@@ -51,8 +51,8 @@ pipeline {
db_username="${env.DB_USERNAME}"
db_password="${env.DB_PASSWORD}"
node_os="${env.NODE_OS}"
environment="${env.ENVIRONMENT}"
engine_mode="${env.ENGINE_MODE}"
environment="${env.DB_ENVIRONMENT}"
engine_mode="${env.DB_ENGINE_MODE}"
external_db="${env.EXTERNAL_DB}"
external_db_version="${env.EXTERNAL_DB_VERSION}"
instance_class="${env.DB_INSTANCE_CLASS}"
......
tests/terraform/testutils.go
View file @ 97c69546
......@@ -17,6 +17,7 @@ type Node struct {
Name string
Status string
Roles string
Version string
InternalIP string
ExternalIP string
}
......@@ -92,7 +93,6 @@ func runsshCommand(cmd string, conn *ssh.Client) (string, error) {
// RunCmdOnNode executes a command from within the given node
func RunCmdOnNode(cmd string, ServerIP string, SSHUser string, SSHKey string) (string, error) {
Server := ServerIP + ":22"
fmt.Println(Server, SSHUser, SSHKey)
conn := ConfigureSSH(Server, SSHUser, SSHKey)
res, err := runsshCommand(cmd, conn)
res = strings.TrimSpace(res)
......@@ -117,10 +117,10 @@ func CountOfStringInSlice(str string, pods []Pod) int {
return count
}
func DeployWorkload(workload, kubeconfig string, arch bool) (string, error) {
func DeployWorkload(workload, kubeconfig string, arch string) (string, error) {
resourceDir := "./amd64_resource_files"
if arch {
resourceDir = "./arm64_resource_files"
if arch == "arm64" {
resourceDir = "./arm_resource_files"
}
files, err := ioutil.ReadDir(resourceDir)
if err != nil {
......@@ -186,6 +186,7 @@ func ParseNodes(kubeConfig string, print bool) ([]Node, error) {
Name: fields[0],
Status: fields[1],
Roles: fields[2],
Version: fields[4],
InternalIP: fields[5],
ExternalIP: fields[6],
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment