Added option to deploy hardened k3s (#5415)

tests/e2e/amd64_resource_files/hardened_psp.yaml

+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: default-network-dns-policy
+  namespace: kube-system
+spec:
+  ingress:
+  - ports:
+    - port: 53
+      protocol: TCP
+    - port: 53
+      protocol: UDP
+  podSelector:
+    matchLabels:
+      k8s-app: kube-dns
+  policyTypes:
+  - Ingress
+---
+apiVersion: policy/v1beta1
+kind: PodSecurityPolicy
+metadata:
+  name: cis1.5-compliant-psp
+spec:
+  privileged: false
+  allowPrivilegeEscalation: false
+  requiredDropCapabilities:
+    - ALL
+  volumes:
+    - 'configMap'
+    - 'emptyDir'
+    - 'projected'
+    - 'secret'
+    - 'downwardAPI'
+    - 'persistentVolumeClaim'
+  hostNetwork: false
+  hostIPC: false
+  hostPID: false
+  runAsUser:
+    rule: 'MustRunAsNonRoot'
+  seLinux:
+    rule: 'RunAsAny'
+  supplementalGroups:
+    rule: 'MustRunAs'
+    ranges:
+      - min: 1
+        max: 65535
+  fsGroup:
+    rule: 'MustRunAs'
+    ranges:
+      - min: 1
+        max: 65535
+  readOnlyRootFilesystem: false
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: psp:restricted
+  labels:
+    addonmanager.kubernetes.io/mode: EnsureExists
+rules:
+- apiGroups: ['extensions']
+  resources: ['podsecuritypolicies']
+  verbs:     ['use']
+  resourceNames:
+  - cis1.5-compliant-psp
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: default:restricted
+  labels:
+    addonmanager.kubernetes.io/mode: EnsureExists
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: psp:restricted
+subjects:
+- kind: Group
+  name: system:authenticated
+  apiGroup: rbac.authorization.k8s.io
+---
+kind: NetworkPolicy
+apiVersion: networking.k8s.io/v1
+metadata:
+  name: intra-namespace
+  namespace: kube-system
+spec:
+  podSelector: {}
+  ingress:
+    - from:
+      - namespaceSelector:
+          matchLabels:
+            name: kube-system
+---
+kind: NetworkPolicy
+apiVersion: networking.k8s.io/v1
+metadata:
+  name: intra-namespace
+  namespace: default
+spec:
+  podSelector: {}
+  ingress:
+    - from:
+      - namespaceSelector:
+          matchLabels:
+            name: default
+---
+kind: NetworkPolicy
+apiVersion: networking.k8s.io/v1
+metadata:
+  name: intra-namespace
+  namespace: kube-public
+spec:
+  podSelector: {}
+  ingress:
+    - from:
+      - namespaceSelector:
+          matchLabels:
+            name: kube-public
+---
+apiVersion: policy/v1beta1
+kind: PodSecurityPolicy
+metadata:
+  name: system-unrestricted-psp
+spec:
+  allowPrivilegeEscalation: true
+  allowedCapabilities:
+  - '*'
+  fsGroup:
+    rule: RunAsAny
+  hostIPC: true
+  hostNetwork: true
+  hostPID: true
+  hostPorts:
+  - max: 65535
+    min: 0
+  privileged: true
+  runAsUser:
+    rule: RunAsAny
+  seLinux:
+    rule: RunAsAny
+  supplementalGroups:
+    rule: RunAsAny
+  volumes:
+  - '*'
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: system-unrestricted-node-psp-rolebinding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system-unrestricted-psp-role
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: Group
+  name: system:nodes
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: system-unrestricted-psp-role
+rules:
+- apiGroups:
+  - policy
+  resourceNames:
+  - system-unrestricted-psp
+  resources:
+  - podsecuritypolicies
+  verbs:
+  - use
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: system-unrestricted-svc-acct-psp-rolebinding
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system-unrestricted-psp-role
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: Group
+  name: system:serviceaccounts
+---
+# Reference https://rancher.com/docs/k3s/latest/en/security/hardening_guide/#networkpolicies
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: allow-all-svclbtraefik-ingress
+  namespace: kube-system
+spec:
+  podSelector:
+    matchLabels:
+      app: svclb-traefik
+  ingress:
+  - {}
+  policyTypes:
+  - Ingress
+---
+# Reference https://rancher.com/docs/k3s/latest/en/security/hardening_guide/#networkpolicies
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: allow-all-metrics-server
+  namespace: kube-system
+spec:
+  podSelector:
+    matchLabels:
+      k8s-app: metrics-server
+  ingress:
+  - {}
+  policyTypes:
+  - Ingress
+---

tests/e2e/scripts/harden.sh

+#!/bin/bash
+
+echo "vm.panic_on_oom=0
+vm.overcommit_memory=1
+kernel.panic=10
+kernel.panic_on_oops=1
+kernel.keys.root_maxbytes=25000000
+" >> /etc/sysctl.d/90-kubelet.conf
+sysctl -p /etc/sysctl.d/90-kubelet.conf
\ No newline at end of file

tests/e2e/splitserver/Vagrantfile

@@ -40,7 +40,7 @@ def provision(vm, role, role_num, node_num)
       k3s.args = "server"
       k3s.config = <<~YAML
         cluster-init: true
-        note-external-ip: #{NETWORK_PREFIX}.100
+        node-external-ip: #{NETWORK_PREFIX}.100
         flannel-iface: eth1
         disable-apiserver: true
         disable-controller-manager: true

tests/e2e/vagrantdefaults.rb

 def defaultOSConfigure(vm)
-    if vm.box.include?("ubuntu2004")
-      vm.provision "shell", inline: "systemd-resolve --set-dns=8.8.8.8 --interface=eth0"
-      vm.provision "shell", inline: "apt install -y jq"
-    end
-    if vm.box.include?("Leap")
-      vm.provision "shell", inline: "zypper install -y jq"
-    end
-    if vm.box.include?("microos")
-      vm.provision "shell", inline: "transactional-update pkg install -y jq"
+    if vm.box.include?("generic/ubuntu")
+      vm.provision "Set DNS", type: "shell", inline: "systemd-resolve --set-dns=8.8.8.8 --interface=eth0"
+      vm.provision "Install jq", type: "shell", inline: "apt install -y jq"
+    elsif vm.box.include?("Leap")
+      vm.provision "Install jq", type: "shell", inline: "zypper install -y jq"
+    elsif vm.box.include?("microos")
+      vm.provision "Install jq", type: "shell", inline: "transactional-update pkg install -y jq"
       vm.provision 'reload', run: 'once'
     end 
 end
\ No newline at end of file

tests/e2e/validatecluster/Vagrantfile

@@ -5,19 +5,21 @@ NODE_BOXES = (ENV['E2E_NODE_BOXES'] ||
   ['generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004'])
 GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master")
 RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "")
-EXTERNAL_DB = (ENV['E2E_EXTERNAL_DB'] || "mysql")
+EXTERNAL_DB = (ENV['E2E_EXTERNAL_DB'] || "etcd")
+HARDENED = (ENV['E2E_HARDENED'] || "")
 NODE_CPUS = (ENV['E2E_NODE_CPUS'] || 2).to_i
 NODE_MEMORY = (ENV['E2E_NODE_MEMORY'] || 1024).to_i
 # Virtualbox >= 6.1.28 require `/etc/vbox/network.conf` for expanded private networks 
 NETWORK_PREFIX = "10.10.10"
 install_type = ""
-db_type = ""
+hardened_arg = ""
 
 def provision(vm, role, role_num, node_num)
   vm.box = NODE_BOXES[node_num]
   vm.hostname = role
   # An expanded netmask is required to allow VM<-->VM communication, virtualbox defaults to /32
-  vm.network "private_network", ip: "#{NETWORK_PREFIX}.#{100+node_num}", netmask: "255.255.255.0"
+  node_ip = "#{NETWORK_PREFIX}.#{100+node_num}"
+  vm.network "private_network", ip: node_ip, netmask: "24"
 
   vagrant_defaults = '../vagrantdefaults.rb'
   load vagrant_defaults if File.exists?(vagrant_defaults)
@@ -29,51 +31,60 @@ def provision(vm, role, role_num, node_num)
   else
     # Grabs the last 5 commit SHA's from the given branch, then purges any commits that do not have a passing CI build
     # MicroOS requires it not be in a /tmp/ or other root system folder
-    vm.provision "shell", path: "../scripts/latest_commit.sh", args: [GITHUB_BRANCH, "/home/vagrant/k3s_commits"]
+    vm.provision "Get latest commit", type: "shell", path: "../scripts/latest_commit.sh", args: [GITHUB_BRANCH, "/home/vagrant/k3s_commits"]
     install_type = "INSTALL_K3S_COMMIT=$(head\ -n\ 1\ /home/vagrant/k3s_commits)"
   end
   vm.provision "shell", inline: "ping -c 2 k3s.io"
   
+  db_type = getDBType(role, role_num, vm)
+
+  if !HARDENED.empty?
+    vm.provision "Set kernel parameters", type: "shell", path: "../scripts/harden.sh"
+    hardened_arg = "protect-kernel-defaults: true\nkube-apiserver-arg: \"enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount\""
+  end
+
   if role.include?("server") && role_num == 0
-    if EXTERNAL_DB == "mysql"
-      dockerInstall(vm)
-      vm.provision "shell", inline: "docker run -d -p 3306:3306 --name #{EXTERNAL_DB} -e MYSQL_ROOT_PASSWORD=e2e mysql:5.7"
-      vm.provision "shell", inline: "echo \"Wait for mysql to startup\"; sleep 10"
-      db_type = "--datastore-endpoint='mysql://root:e2e@tcp(#{NETWORK_PREFIX}.100:3306)/k3s'"
-    elsif EXTERNAL_DB == "postgres"
-      dockerInstall(vm)
-      vm.provision "shell", inline: "docker run -d -p 5432:5432 --name #{EXTERNAL_DB} -e POSTGRES_PASSWORD=e2e  postgres:14-alpine"
-      vm.provision "shell", inline: "echo \"Wait for postgres to startup\"; sleep 10"
-      db_type = "--datastore-endpoint='postgres://postgres:e2e@#{NETWORK_PREFIX}.100:5432/k3s?sslmode=disable'"
-    elsif EXTERNAL_DB == "" || EXTERNAL_DB == "etcd"
-      db_type = "--cluster-init"
-    else
-      puts "Unknown EXTERNAL_DB: " + EXTERNAL_DB
-      abort
-    end
-    
-    vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s|
-      k3s.args = "server #{db_type} --node-external-ip=#{NETWORK_PREFIX}.100 --flannel-iface=eth1"
-      k3s.env = %W[K3S_KUBECONFIG_MODE=0644 K3S_TOKEN=vagrant #{install_type}]
+    vm.provision 'k3s-primary-server', type: 'k3s', run: 'once' do |k3s|
+      k3s.args = "server "
+      k3s.config = <<~YAML
+        token: vagrant
+        node-external-ip: #{NETWORK_PREFIX}.100
+        flannel-iface: eth1
+        #{db_type}
+        #{hardened_arg}
+      YAML
+      k3s.env = %W[K3S_KUBECONFIG_MODE=0644 #{install_type}]
       k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321
     end
+  
   elsif role.include?("server") && role_num != 0
-    if EXTERNAL_DB == "mysql"
-      db_type = "--datastore-endpoint='mysql://root:e2e@tcp(#{NETWORK_PREFIX}.100:3306)/k3s'"
-    elsif EXTERNAL_DB == "postgres"
-      db_type = "--datastore-endpoint='postgres://postgres:e2e@#{NETWORK_PREFIX}.100:5432/k3s?sslmode=disable'"
-    end
-
-    vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s|
-      k3s.args = "server #{db_type} --server https://#{NETWORK_PREFIX}.100:6443 --flannel-iface=eth1"
+    vm.provision 'k3s-secondary-server', type: 'k3s', run: 'once' do |k3s|
+      k3s.args = "server"
+      k3s.config = <<~YAML
+        server: "https://#{NETWORK_PREFIX}.100:6443"
+        token: vagrant
+        node-external-ip: #{node_ip}
+        flannel-iface: eth1
+        #{db_type}
+        #{hardened_arg}
+      YAML
       k3s.env = %W[K3S_KUBECONFIG_MODE=0644 K3S_TOKEN=vagrant #{install_type}]
       k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321
     end
   end
+  
   if role.include?("agent")
-    vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s|
-      k3s.args = %W[agent #{db_type} --server https://#{NETWORK_PREFIX}.100:6443 --flannel-iface=eth1]
-      k3s.env = %W[K3S_KUBECONFIG_MODE=0644 K3S_TOKEN=vagrant #{install_type}]
+    vm.provision 'k3s-agent', type: 'k3s', run: 'once' do |k3s|
+      k3s.args = "agent"
+      k3s.config = <<~YAML
+        server: "https://#{NETWORK_PREFIX}.100:6443"
+        token: vagrant
+        node-external-ip: #{node_ip}
+        flannel-iface: eth1
+        #{db_type}
+        #{hardened_arg}
+      YAML
+      k3s.env = %W[K3S_KUBECONFIG_MODE=0644 #{install_type}]
       k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321
     end
   end
@@ -85,6 +96,39 @@ def provision(vm, role, role_num, node_num)
   end
 end
 
+def getDBType(role, role_num, vm)
+  
+  if EXTERNAL_DB == "mysql"
+    if role.include?("server") && role_num == 0
+      dockerInstall(vm)
+      vm.provision "Start mysql", inline: "docker run -d -p 3306:3306 --name #{EXTERNAL_DB} -e MYSQL_ROOT_PASSWORD=e2e mysql:5.7"
+      vm.provision "shell", inline: "echo \"Wait for mysql to startup\"; sleep 10"
+      return "datastore-endpoint: 'mysql://root:e2e@tcp(#{NETWORK_PREFIX}.100:3306)/k3s'"
+    elsif role.include?("server") && role_num != 0
+      return "datastore-endpoint: 'mysql://root:e2e@tcp(#{NETWORK_PREFIX}.100:3306)/k3s'"
+    end
+
+  elsif EXTERNAL_DB == "postgres"
+    if role.include?("server") && role_num == 0
+      dockerInstall(vm)
+      vm.provision "Start postgres", inline: "docker run -d -p 5432:5432 --name #{EXTERNAL_DB} -e POSTGRES_PASSWORD=e2e  postgres:14-alpine"
+      vm.provision "shell", inline: "echo \"Wait for postgres to startup\"; sleep 10"
+      return "datastore-endpoint: 'postgres://postgres:e2e@#{NETWORK_PREFIX}.100:5432/k3s?sslmode=disable'"
+    elsif role.include?("server") && role_num != 0
+      return "datastore-endpoint: 'postgres://postgres:e2e@#{NETWORK_PREFIX}.100:5432/k3s?sslmode=disable'"
+    end
+
+  elsif ( EXTERNAL_DB == "" || EXTERNAL_DB == "etcd" )
+    if role.include?("server") && role_num == 0
+      return "cluster-init: true"
+    end
+  else
+    puts "Unknown EXTERNAL_DB: " + EXTERNAL_DB
+    abort
+  end
+  return ""
+end
+
 def dockerInstall(vm)
   vm.provider "libvirt" do |v|
     v.memory = NODE_MEMORY + 1024
@@ -92,7 +136,7 @@ def dockerInstall(vm)
   vm.provider "virtualbox" do |v|
     v.memory = NODE_MEMORY + 1024
   end
-  if vm.box.include?("ubuntu2004")
+  if vm.box.include?("ubuntu")
     vm.provision "shell", inline: "apt install -y docker.io"
   end
   if vm.box.include?("Leap")
@@ -105,6 +149,7 @@ def dockerInstall(vm)
   end
 end
 
+
 Vagrant.configure("2") do |config|
   config.vagrant.plugins = ["vagrant-k3s", "vagrant-reload"]
   # Default provider is libvirt, virtualbox is only provided as a backup