Migrate svcpolicies E2E test to docker

.github/workflows/e2e.yaml

@@ -160,7 +160,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        dtest: [autoimport, basics, bootstraptoken, cacerts, etcd, hardened, lazypull, skew, secretsencryption, snapshotrestore, token, upgrade]
+        dtest: [autoimport, basics, bootstraptoken, cacerts, etcd, hardened, lazypull, skew, secretsencryption, snapshotrestore, svcpoliciesandfirewall, token, upgrade]
         arch: [amd64, arm64]
         exclude:
           - dtest: autoimport
@@ -169,6 +169,8 @@ jobs:
             arch: arm64
           - dtest: snapshotrestore
             arch: arm64
+          - dtest: svcpoliciesandfirewall
+            arch: arm64
     runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }}
     env:
       CHANNEL: ${{ needs.build-go-tests.outputs.channel }}
@@ -209,7 +211,7 @@ jobs:
         cd ./tests/docker/${{ matrix.dtest }}
         
         # These tests use rancher/systemd-node and have different flags.
-        CI_TESTS="autoimport hardened secretsencryption snapshotrestore token"
+        CI_TESTS="autoimport hardened secretsencryption snapshotrestore svcpoliciesandfirewall token"
         if [ ${{ matrix.dtest }} = "upgrade" ] || [ ${{ matrix.dtest }} = "skew" ]; then
           ./${{ matrix.dtest }}.test -test.timeout=0 -test.v -ginkgo.v -k3sImage=$K3S_IMAGE -channel=$CHANNEL
         elif [[ $CI_TESTS =~ ${{ matrix.dtest }} ]]; then

tests/client.go

@@ -62,6 +62,23 @@ func ParseNodes(kubeconfigFile string) ([]corev1.Node, error) {
 	return nodes.Items, nil
 }
 
+// Returns all internal IPs of the nodes in the cluster as map[node][ip]
+func GetInternalIPs(kubeconfigFile string) (map[string]string, error) {
+	nodes, err := ParseNodes(kubeconfigFile)
+	if err != nil {
+		return nil, err
+	}
+	ips := make(map[string]string)
+	for _, node := range nodes {
+		for _, address := range node.Status.Addresses {
+			if address.Type == corev1.NodeInternalIP {
+				ips[node.Name] = address.Address
+			}
+		}
+	}
+	return ips, nil
+}
+
 func ParsePods(kubeconfigFile string) ([]corev1.Pod, error) {
 	clientSet, err := K8sClient(kubeconfigFile)
 	if err != nil {

tests/docker/resources/loadbalancer-allTraffic.yaml

+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: nginx-config
+data:
+  default.conf: |
+    server {
+        listen 80;
+        location /ip {
+            return 200 "$remote_addr\n";
+        }
+         # Default location block to serve the default "Welcome to nginx" page
+        location / {
+            root /usr/share/nginx/html;
+            index index.html;
+        }
+    }
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: test-loadbalancer
+spec:
+  selector:
+    matchLabels:
+      k8s-app: nginx-app-loadbalancer
+  replicas: 2
+  template:
+    metadata:
+      labels:
+        k8s-app: nginx-app-loadbalancer
+    spec:
+      containers:
+      - name: nginx
+        image: ranchertest/mytestcontainer
+        ports:
+        - containerPort: 80
+        volumeMounts:
+        - name: nginx-config-volume
+          mountPath: /etc/nginx/conf.d
+      volumes:
+      - name: nginx-config-volume
+        configMap:
+          name: nginx-config
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: nginx-loadbalancer-svc
+ labels:
+   k8s-app: nginx-app-loadbalancer
+spec:
+ type: LoadBalancer
+ ports:
+ - port: 81
+   targetPort: 80
+   protocol: TCP
+   name: http
+ selector:
+   k8s-app: nginx-app-loadbalancer

tests/docker/resources/pod_client.yaml

+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: client
+  name: client-deployment
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: client
+  template:
+    metadata:
+      labels:
+        app: client
+    spec:
+      containers:
+      - image: ranchertest/mytestcontainer
+        imagePullPolicy: Always
+        name: client-curl
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+          - labelSelector:
+              matchExpressions:
+              - key: app
+                operator: In
+                values:
+                - client
+            topologyKey: kubernetes.io/hostname
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: client-curl
+  labels:
+    app: client
+    service: client-curl
+spec:
+  type: ClusterIP
+  selector:
+    app: client
+  ports:
+  - port: 8080

tests/docker/test-helpers.go

 package docker
 
 import (
+	"encoding/json"
 	"fmt"
 	"math/rand"
 	"net"
@@ -608,6 +609,34 @@ func (config TestConfig) DeployWorkload(workload string) (string, error) {
 	return "", nil
 }
 
+type svcExternalIP struct {
+	IP     string `json:"ip"`
+	IPMode string `json:"ipMode"`
+}
+
+// FetchExternalIPs fetches the external IPs of a service
+func FetchExternalIPs(kubeconfig string, servicename string) ([]string, error) {
+	var externalIPs []string
+	cmd := "kubectl get svc " + servicename + " -o jsonpath='{.status.loadBalancer.ingress}' --kubeconfig=" + kubeconfig
+	output, err := RunCommand(cmd)
+	if err != nil {
+		return externalIPs, err
+	}
+
+	var svcExternalIPs []svcExternalIP
+	err = json.Unmarshal([]byte(output), &svcExternalIPs)
+	if err != nil {
+		return externalIPs, fmt.Errorf("error unmarshalling JSON: %v", err)
+	}
+
+	// Iterate over externalIPs and append each IP to the ips slice
+	for _, ipEntry := range svcExternalIPs {
+		externalIPs = append(externalIPs, ipEntry.IP)
+	}
+
+	return externalIPs, nil
+}
+
 // RestartCluster restarts the k3s service on each node given
 func RestartCluster(nodes []DockerNode) error {
 	for _, node := range nodes {

tests/e2e/svcpoliciesandfirewall/Vagrantfile

-ENV['VAGRANT_NO_PARALLEL'] = 'no'
-NODE_ROLES = (ENV['E2E_NODE_ROLES'] ||
-  ["server-0", "agent-0" ])
-NODE_BOXES = (ENV['E2E_NODE_BOXES'] ||
-  ['bento/ubuntu-24.04', 'bento/ubuntu-24.04'])
-GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master")
-RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "")
-GOCOVER = (ENV['E2E_GOCOVER'] || "")
-NODE_CPUS = (ENV['E2E_NODE_CPUS'] || 2).to_i
-NODE_MEMORY = (ENV['E2E_NODE_MEMORY'] || 2048).to_i
-NETWORK4_PREFIX = "10.10.10"
-install_type = ""
-
-def provision(vm, role, role_num, node_num)
-  vm.box = NODE_BOXES[node_num]
-  vm.hostname = role
-  node_ip4 = "#{NETWORK4_PREFIX}.#{100+node_num}"
-  vm.network "private_network", :ip => node_ip4, :netmask => "255.255.255.0"
-    
-  scripts_location = Dir.exist?("./scripts") ? "./scripts" : "../scripts" 
-  vagrant_defaults = File.exist?("./vagrantdefaults.rb") ? "./vagrantdefaults.rb" : "../vagrantdefaults.rb"
-  load vagrant_defaults
-  
-  defaultOSConfigure(vm)
-  addCoverageDir(vm, role, GOCOVER)
-  install_type = getInstallType(vm, RELEASE_VERSION, GITHUB_BRANCH)
-  
-  if role.include?("server") && role_num == 0
-    vm.provision :k3s, run: 'once' do |k3s|
-      k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321
-      k3s.args = "server "
-      k3s.config = <<~YAML
-        node-ip: #{node_ip4}
-        token: vagrant
-      YAML
-      k3s.env = ["K3S_KUBECONFIG_MODE=0644", install_type]
-    end
-  end
-  if role.include?("agent")
-    vm.provision :k3s, run: 'once' do |k3s|
-      k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321
-      k3s.args = "agent "
-      k3s.config = <<~YAML
-        server: https://#{NETWORK4_PREFIX}.100:6443
-        token: vagrant
-        node-ip: #{node_ip4}
-      YAML
-      k3s.env = ["K3S_KUBECONFIG_MODE=0644", install_type]
-    end
-  end
-end
-
-Vagrant.configure("2") do |config|
-  config.vagrant.plugins = ["vagrant-k3s", "vagrant-reload", "vagrant-libvirt", "vagrant-scp"]
-  config.vm.provider "libvirt" do |v|
-    v.cpus = NODE_CPUS
-    v.memory = NODE_MEMORY
-    # We replicate the default prefix, but add a timestamp to enable parallel runs and cleanup of old VMs
-    v.default_prefix = File.basename(Dir.getwd) + "_" + Time.now.to_i.to_s + "_"
-  end
-  
-  if NODE_ROLES.kind_of?(String)
-    NODE_ROLES = NODE_ROLES.split(" ", -1)
-  end
-  if NODE_BOXES.kind_of?(String)
-    NODE_BOXES = NODE_BOXES.split(" ", -1)
-  end
-
-  NODE_ROLES.each_with_index do |role, i|
-    role_num = role.split("-", -1).pop.to_i
-    config.vm.define role do |node|
-      provision(node.vm, role, role_num, i)
-    end
-  end
-end