[Backport 1.21] Update embedded kube-router (#3557) (#3595)

go.mod

@@ -125,6 +125,7 @@ require (
 	k8s.io/controller-manager v0.21.2
 	k8s.io/cri-api v0.21.2
 	k8s.io/klog v1.0.0
+	k8s.io/klog/v2 v2.8.0
 	k8s.io/kubectl v0.21.2
 	k8s.io/kubernetes v1.21.2
 	k8s.io/utils v0.0.0-20201110183641-67b214c5f920

pkg/agent/netpol/namespace.go

 // Apache License v2.0 (copyright Cloud Native Labs & Rancher Labs)
-// - modified from https://github.com/cloudnativelabs/kube-router/blob/ee9f6d890d10609284098229fa1e283ab5d83b93/pkg/controllers/netpol/namespace.go
+// - modified from https://github.com/cloudnativelabs/kube-router/blob/73b1b03b32c5755b240f6c077bb097abe3888314/pkg/controllers/netpol/namespace.go
 
 // +build !windows
 
@@ -10,7 +10,7 @@ import (
 
 	api "k8s.io/api/core/v1"
 	"k8s.io/client-go/tools/cache"
-	glog "k8s.io/klog"
+	"k8s.io/klog/v2"
 )
 
 func (npc *NetworkPolicyController) newNamespaceEventHandler() cache.ResourceEventHandler {
@@ -32,7 +32,7 @@ func (npc *NetworkPolicyController) newNamespaceEventHandler() cache.ResourceEve
 					return
 				}
 			default:
-				glog.Errorf("unexpected object type: %v", obj)
+				klog.Errorf("unexpected object type: %v", obj)
 			}
 		},
 	}
@@ -42,7 +42,7 @@ func (npc *NetworkPolicyController) handleNamespaceAdd(obj *api.Namespace) {
 	if obj.Labels == nil {
 		return
 	}
-	glog.V(2).Infof("Received update for namespace: %s", obj.Name)
+	klog.V(2).Infof("Received update for namespace: %s", obj.Name)
 
 	npc.RequestFullSync()
 }
@@ -51,7 +51,7 @@ func (npc *NetworkPolicyController) handleNamespaceUpdate(oldObj, newObj *api.Na
 	if reflect.DeepEqual(oldObj.Labels, newObj.Labels) {
 		return
 	}
-	glog.V(2).Infof("Received update for namespace: %s", newObj.Name)
+	klog.V(2).Infof("Received update for namespace: %s", newObj.Name)
 
 	npc.RequestFullSync()
 }
@@ -60,7 +60,7 @@ func (npc *NetworkPolicyController) handleNamespaceDelete(obj *api.Namespace) {
 	if obj.Labels == nil {
 		return
 	}
-	glog.V(2).Infof("Received namespace: %s delete event", obj.Name)
+	klog.V(2).Infof("Received namespace: %s delete event", obj.Name)
 
 	npc.RequestFullSync()
 }

pkg/agent/netpol/netpol.go

+// Apache License v2.0 (copyright Cloud Native Labs & Rancher Labs)
+// - modified from https://github.com/cloudnativelabs/kube-router/blob/73b1b03b32c5755b240f6c077bb097abe3888314/pkg/controllers/netpol.go
+
 // +build !windows
 
 package netpol
 
 import (
 	"context"
+	"sync"
 
 	"github.com/rancher/k3s/pkg/agent/netpol/utils"
 	"github.com/rancher/k3s/pkg/daemons/config"
@@ -48,7 +52,7 @@ func Run(ctx context.Context, nodeConfig *config.Node) error {
 	informerFactory.Start(stopCh)
 	informerFactory.WaitForCacheSync(stopCh)
 
-	npc, err := NewNetworkPolicyController(client, nodeConfig, podInformer, npInformer, nsInformer)
+	npc, err := NewNetworkPolicyController(client, nodeConfig, podInformer, npInformer, nsInformer, &sync.Mutex{})
 	if err != nil {
 		return err
 	}

pkg/agent/netpol/network_policy_controller_test.go

 // Apache License v2.0 (copyright Cloud Native Labs & Rancher Labs)
-// - modified from https://github.com/cloudnativelabs/kube-router/blob/ee9f6d890d10609284098229fa1e283ab5d83b93/pkg/controllers/network_policy_controller_test.go
+// - modified from https://github.com/cloudnativelabs/kube-router/blob/73b1b03b32c5755b240f6c077bb097abe3888314/pkg/controllers/network_policy_controller_test.go
 
 // +build !windows
 
 package netpol
 
 import (
+	"bytes"
 	"context"
+	"fmt"
 	"net"
 	"strings"
+	"sync"
 	"testing"
 	"time"
 
+	"github.com/rancher/k3s/pkg/daemons/config"
 	netv1 "k8s.io/api/networking/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/client-go/tools/cache"
 
 	v1 "k8s.io/api/core/v1"
@@ -23,8 +28,6 @@ import (
 	"k8s.io/client-go/informers"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/fake"
-
-	"github.com/rancher/k3s/pkg/daemons/config"
 )
 
 // newFakeInformersFromClient creates the different informers used in the uneventful network policy controller
@@ -213,6 +216,7 @@ type tNetpolTestCase struct {
 	targetPods   tPodNamespaceMap
 	inSourcePods tPodNamespaceMap
 	outDestPods  tPodNamespaceMap
+	expectedRule string
 }
 
 // tGetNotTargetedPods finds set of pods that should not be targeted by netpol selectors
@@ -417,6 +421,182 @@ func TestNewNetworkPolicySelectors(t *testing.T) {
 	}
 }
 
+func TestNetworkPolicyBuilder(t *testing.T) {
+	port, port1 := intstr.FromInt(30000), intstr.FromInt(34000)
+	ingressPort := intstr.FromInt(37000)
+	endPort, endPort1 := int32(31000), int32(35000)
+	testCases := []tNetpolTestCase{
+		{
+			name: "Simple Egress Destination Port",
+			netpol: tNetpol{name: "simple-egress", namespace: "nsA",
+				podSelector: metav1.LabelSelector{
+					MatchExpressions: []metav1.LabelSelectorRequirement{
+						{
+							Key:      "app",
+							Operator: "In",
+							Values:   []string{"a"},
+						},
+					},
+				},
+				egress: []netv1.NetworkPolicyEgressRule{
+					{
+						Ports: []netv1.NetworkPolicyPort{
+							{
+								Port: &port,
+							},
+						},
+					},
+				},
+			},
+			expectedRule: "-A KUBE-NWPLCY-QHFGOTFJZFXUJVTH -m comment --comment \"rule to ACCEPT traffic from source pods to all destinations selected by policy name: simple-egress namespace nsA\" --dport 30000 -j MARK --set-xmark 0x10000/0x10000 \n" +
+				"-A KUBE-NWPLCY-QHFGOTFJZFXUJVTH -m comment --comment \"rule to ACCEPT traffic from source pods to all destinations selected by policy name: simple-egress namespace nsA\" --dport 30000 -m mark --mark 0x10000/0x10000 -j RETURN \n",
+		},
+		{
+			name: "Simple Ingress/Egress Destination Port",
+			netpol: tNetpol{name: "simple-ingress-egress", namespace: "nsA",
+				podSelector: metav1.LabelSelector{
+					MatchExpressions: []metav1.LabelSelectorRequirement{
+						{
+							Key:      "app",
+							Operator: "In",
+							Values:   []string{"a"},
+						},
+					},
+				},
+				egress: []netv1.NetworkPolicyEgressRule{
+					{
+						Ports: []netv1.NetworkPolicyPort{
+							{
+								Port: &port,
+							},
+						},
+					},
+				},
+				ingress: []netv1.NetworkPolicyIngressRule{
+					{
+						Ports: []netv1.NetworkPolicyPort{
+							{
+								Port: &ingressPort,
+							},
+						},
+					},
+				},
+			},
+			expectedRule: "-A KUBE-NWPLCY-KO52PWL34ABMMBI7 -m comment --comment \"rule to ACCEPT traffic from source pods to all destinations selected by policy name: simple-ingress-egress namespace nsA\" --dport 30000 -j MARK --set-xmark 0x10000/0x10000 \n" +
+				"-A KUBE-NWPLCY-KO52PWL34ABMMBI7 -m comment --comment \"rule to ACCEPT traffic from source pods to all destinations selected by policy name: simple-ingress-egress namespace nsA\" --dport 30000 -m mark --mark 0x10000/0x10000 -j RETURN \n" +
+				"-A KUBE-NWPLCY-KO52PWL34ABMMBI7 -m comment --comment \"rule to ACCEPT traffic from all sources to dest pods selected by policy name: simple-ingress-egress namespace nsA\" --dport 37000 -j MARK --set-xmark 0x10000/0x10000 \n" +
+				"-A KUBE-NWPLCY-KO52PWL34ABMMBI7 -m comment --comment \"rule to ACCEPT traffic from all sources to dest pods selected by policy name: simple-ingress-egress namespace nsA\" --dport 37000 -m mark --mark 0x10000/0x10000 -j RETURN \n",
+		},
+		{
+			name: "Simple Egress Destination Port Range",
+			netpol: tNetpol{name: "simple-egress-pr", namespace: "nsA",
+				podSelector: metav1.LabelSelector{
+					MatchExpressions: []metav1.LabelSelectorRequirement{
+						{
+							Key:      "app",
+							Operator: "In",
+							Values:   []string{"a"},
+						},
+					},
+				},
+				egress: []netv1.NetworkPolicyEgressRule{
+					{
+						Ports: []netv1.NetworkPolicyPort{
+							{
+								Port:    &port,
+								EndPort: &endPort,
+							},
+							{
+								Port:    &port1,
+								EndPort: &endPort1,
+							},
+						},
+					},
+				},
+			},
+			expectedRule: "-A KUBE-NWPLCY-SQYQ7PVNG6A6Q3DU -m comment --comment \"rule to ACCEPT traffic from source pods to all destinations selected by policy name: simple-egress-pr namespace nsA\" --dport 30000:31000 -j MARK --set-xmark 0x10000/0x10000 \n" +
+				"-A KUBE-NWPLCY-SQYQ7PVNG6A6Q3DU -m comment --comment \"rule to ACCEPT traffic from source pods to all destinations selected by policy name: simple-egress-pr namespace nsA\" --dport 30000:31000 -m mark --mark 0x10000/0x10000 -j RETURN \n" +
+				"-A KUBE-NWPLCY-SQYQ7PVNG6A6Q3DU -m comment --comment \"rule to ACCEPT traffic from source pods to all destinations selected by policy name: simple-egress-pr namespace nsA\" --dport 34000:35000 -j MARK --set-xmark 0x10000/0x10000 \n" +
+				"-A KUBE-NWPLCY-SQYQ7PVNG6A6Q3DU -m comment --comment \"rule to ACCEPT traffic from source pods to all destinations selected by policy name: simple-egress-pr namespace nsA\" --dport 34000:35000 -m mark --mark 0x10000/0x10000 -j RETURN \n",
+		},
+		{
+			name: "Port > EndPort (invalid condition, should drop endport)",
+			netpol: tNetpol{name: "invalid-endport", namespace: "nsA",
+				podSelector: metav1.LabelSelector{
+					MatchExpressions: []metav1.LabelSelectorRequirement{
+						{
+							Key:      "app",
+							Operator: "In",
+							Values:   []string{"a"},
+						},
+					},
+				},
+				egress: []netv1.NetworkPolicyEgressRule{
+					{
+						Ports: []netv1.NetworkPolicyPort{
+							{
+								Port:    &port1,
+								EndPort: &endPort,
+							},
+						},
+					},
+				},
+			},
+			expectedRule: "-A KUBE-NWPLCY-2A4DPWPR5REBS66I -m comment --comment \"rule to ACCEPT traffic from source pods to all destinations selected by policy name: invalid-endport namespace nsA\" --dport 34000 -j MARK --set-xmark 0x10000/0x10000 \n" +
+				"-A KUBE-NWPLCY-2A4DPWPR5REBS66I -m comment --comment \"rule to ACCEPT traffic from source pods to all destinations selected by policy name: invalid-endport namespace nsA\" --dport 34000 -m mark --mark 0x10000/0x10000 -j RETURN \n",
+		},
+	}
+
+	client := fake.NewSimpleClientset(&v1.NodeList{Items: []v1.Node{*newFakeNode("node", "10.10.10.10")}})
+	informerFactory, podInformer, nsInformer, netpolInformer := newFakeInformersFromClient(client)
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	informerFactory.Start(ctx.Done())
+	cache.WaitForCacheSync(ctx.Done(), podInformer.HasSynced)
+	krNetPol, _ := newUneventfulNetworkPolicyController(podInformer, netpolInformer, nsInformer)
+	tCreateFakePods(t, podInformer, nsInformer)
+	for _, test := range testCases {
+		test.netpol.createFakeNetpol(t, netpolInformer)
+		netpols, err := krNetPol.buildNetworkPoliciesInfo()
+		if err != nil {
+			t.Errorf("Problems building policies: %s", err)
+		}
+		for _, np := range netpols {
+			fmt.Printf(np.policyType)
+			if np.policyType == "egress" || np.policyType == "both" {
+				err = krNetPol.processEgressRules(np, "", nil, "1")
+				if err != nil {
+					t.Errorf("Error syncing the rules: %s", err)
+				}
+			}
+			if np.policyType == "ingress" || np.policyType == "both" {
+				err = krNetPol.processIngressRules(np, "", nil, "1")
+				if err != nil {
+					t.Errorf("Error syncing the rules: %s", err)
+				}
+			}
+		}
+
+		if !bytes.Equal([]byte(test.expectedRule), krNetPol.filterTableRules.Bytes()) {
+			t.Errorf("Invalid rule %s created:\nExpected:\n%s \nGot:\n%s", test.name, test.expectedRule, krNetPol.filterTableRules.String())
+		}
+		key := fmt.Sprintf("%s/%s", test.netpol.namespace, test.netpol.name)
+		obj, exists, err := krNetPol.npLister.GetByKey(key)
+		if err != nil {
+			t.Errorf("Failed to get Netpol from store: %s", err)
+		}
+		if exists {
+			err = krNetPol.npLister.Delete(obj)
+			if err != nil {
+				t.Errorf("Failed to remove Netpol from store: %s", err)
+			}
+		}
+		krNetPol.filterTableRules.Reset()
+
+	}
+
+}
+
 func TestNetworkPolicyController(t *testing.T) {
 	testCases := []tNetPolConfigTestCase{
 		{
@@ -429,7 +609,7 @@ func TestNetworkPolicyController(t *testing.T) {
 			"Missing nodename fails appropriately",
 			newMinimalNodeConfig("", "", "", nil),
 			true,
-			"Failed to identify the node by NODE_NAME, hostname or --hostname-override",
+			"failed to identify the node by NODE_NAME, hostname or --hostname-override",
 		},
 		{
 			"Test good cluster CIDR (using single IP with a /32)",
@@ -466,7 +646,7 @@ func TestNetworkPolicyController(t *testing.T) {
 	_, podInformer, nsInformer, netpolInformer := newFakeInformersFromClient(client)
 	for _, test := range testCases {
 		t.Run(test.name, func(t *testing.T) {
-			_, err := NewNetworkPolicyController(client, test.config, podInformer, netpolInformer, nsInformer)
+			_, err := NewNetworkPolicyController(client, test.config, podInformer, netpolInformer, nsInformer, &sync.Mutex{})
 			if err == nil && test.expectError {
 				t.Error("This config should have failed, but it was successful instead")
 			} else if err != nil {

pkg/agent/netpol/utils.go

+// Apache License v2.0 (copyright Cloud Native Labs & Rancher Labs)
+// - modified from https://github.com/cloudnativelabs/kube-router/blob/73b1b03b32c5755b240f6c077bb097abe3888314/pkg/controllers/netpol/utils.go
+
+package netpol
+
+import (
+	"fmt"
+	"reflect"
+	"regexp"
+	"strconv"
+
+	api "k8s.io/api/core/v1"
+)
+
+const (
+	PodCompleted api.PodPhase = "Completed"
+)
+
+// isPodUpdateNetPolRelevant checks the attributes that we care about for building NetworkPolicies on the host and if it
+// finds a relevant change, it returns true otherwise it returns false. The things we care about for NetworkPolicies:
+// 1) Is the phase of the pod changing? (matters for catching completed, succeeded, or failed jobs)
+// 2) Is the pod IP changing? (changes how the network policy is applied to the host)
+// 3) Is the pod's host IP changing? (should be caught in the above, with the CNI kube-router runs with but we check this as well for sanity)
+// 4) Is a pod's label changing? (potentially changes which NetworkPolicies select this pod)
+func isPodUpdateNetPolRelevant(oldPod, newPod *api.Pod) bool {
+	return newPod.Status.Phase != oldPod.Status.Phase ||
+		newPod.Status.PodIP != oldPod.Status.PodIP ||
+		!reflect.DeepEqual(newPod.Status.PodIPs, oldPod.Status.PodIPs) ||
+		newPod.Status.HostIP != oldPod.Status.HostIP ||
+		!reflect.DeepEqual(newPod.Labels, oldPod.Labels)
+}
+
+func isNetPolActionable(pod *api.Pod) bool {
+	return !isFinished(pod) && pod.Status.PodIP != "" && !pod.Spec.HostNetwork
+}
+
+func isFinished(pod *api.Pod) bool {
+	switch pod.Status.Phase {
+	case api.PodFailed, api.PodSucceeded, PodCompleted:
+		return true
+	}
+	return false
+}
+
+func validateNodePortRange(nodePortOption string) (string, error) {
+	nodePortValidator := regexp.MustCompile(`^([0-9]+)[:-]([0-9]+)$`)
+	if matched := nodePortValidator.MatchString(nodePortOption); !matched {
+		return "", fmt.Errorf("failed to parse node port range given: '%s' please see specification in help text", nodePortOption)
+	}
+	matches := nodePortValidator.FindStringSubmatch(nodePortOption)
+	if len(matches) != 3 {
+		return "", fmt.Errorf("could not parse port number from range given: '%s'", nodePortOption)
+	}
+	port1, err := strconv.ParseUint(matches[1], 10, 16)
+	if err != nil {
+		return "", fmt.Errorf("could not parse first port number from range given: '%s'", nodePortOption)
+	}
+	port2, err := strconv.ParseUint(matches[2], 10, 16)
+	if err != nil {
+		return "", fmt.Errorf("could not parse second port number from range given: '%s'", nodePortOption)
+	}
+	if port1 >= port2 {
+		return "", fmt.Errorf("port 1 is greater than or equal to port 2 in range given: '%s'", nodePortOption)
+	}
+	return fmt.Sprintf("%d:%d", port1, port2), nil
+}

pkg/agent/netpol/utils/ipset.go

 // Apache License v2.0 (copyright Cloud Native Labs & Rancher Labs)
-// - modified from https://github.com/cloudnativelabs/kube-router/blob/ee9f6d890d10609284098229fa1e283ab5d83b93/pkg/utils/ipset.go
+// - modified from https://github.com/cloudnativelabs/kube-router/blob/73b1b03b32c5755b240f6c077bb097abe3888314/pkg/utils/ipset.go
 
 // +build !windows
 
@@ -7,15 +7,18 @@ package utils
 
 import (
 	"bytes"
+	"crypto/sha1"
+	"encoding/base32"
 	"errors"
 	"fmt"
 	"os/exec"
+	"sort"
 	"strings"
 )
 
 var (
 	// Error returned when ipset binary is not found.
-	errIpsetNotFound = errors.New("Ipset utility not found")
+	errIpsetNotFound = errors.New("ipset utility not found")
 )
 
 const (
@@ -82,6 +85,9 @@ const (
 	OptionNoMatch = "nomatch"
 	// OptionForceAdd All hash set types support the optional forceadd parameter when creating a set. When sets created with this option become full the next addition to the set may succeed and evict a random entry from the set.
 	OptionForceAdd = "forceadd"
+
+	// tmpIPSetPrefix Is the prefix added to temporary ipset names used in the atomic swap operations during ipset restore. You should never see these on your system because they only exist during the restore.
+	tmpIPSetPrefix = "TMP-"
 )
 
 // IPSet represent ipset sets managed by.
@@ -181,7 +187,7 @@ func (ipset *IPSet) Create(setName string, createOptions ...string) (*Set, error
 	// Determine if set with the same name is already active on the system
 	setIsActive, err := ipset.Sets[setName].IsActive()
 	if err != nil {
-		return nil, fmt.Errorf("Failed to determine if ipset set %s exists: %s",
+		return nil, fmt.Errorf("failed to determine if ipset set %s exists: %s",
 			setName, err)
 	}
 
@@ -193,20 +199,20 @@ func (ipset *IPSet) Create(setName string, createOptions ...string) (*Set, error
 			args = append(args, createOptions...)
 			args = append(args, "family", "inet6")
 			if _, err := ipset.run(args...); err != nil {
-				return nil, fmt.Errorf("Failed to create ipset set on system: %s", err)
+				return nil, fmt.Errorf("failed to create ipset set on system: %s", err)
 			}
 		} else {
 			_, err := ipset.run(append([]string{"create", "-exist", setName},
 				createOptions...)...)
 			if err != nil {
-				return nil, fmt.Errorf("Failed to create ipset set on system: %s", err)
+				return nil, fmt.Errorf("failed to create ipset set on system: %s", err)
 			}
 		}
 	}
 	return ipset.Sets[setName], nil
 }
 
-// Adds a given Set to an IPSet
+// Add a given Set to an IPSet
 func (ipset *IPSet) Add(set *Set) error {
 	_, err := ipset.Create(set.Name, set.Options...)
 	if err != nil {
@@ -226,6 +232,22 @@ func (ipset *IPSet) Add(set *Set) error {
 	return nil
 }
 
+// RefreshSet add/update internal Sets with a Set of entries but does not run restore command
+func (ipset *IPSet) RefreshSet(setName string, entriesWithOptions [][]string, setType string) {
+	if ipset.Get(setName) == nil {
+		ipset.Sets[setName] = &Set{
+			Name:    setName,
+			Options: []string{setType, OptionTimeout, "0"},
+			Parent:  ipset,
+		}
+	}
+	entries := make([]*Entry, len(entriesWithOptions))
+	for i, entry := range entriesWithOptions {
+		entries[i] = &Entry{Set: ipset.Sets[setName], Options: entry}
+	}
+	ipset.Get(setName).Entries = entries
+}
+
 // Add a given entry to the set. If the -exist option is specified, ipset
 // ignores if the entry already added to the set.
 // Note: if you need to add multiple entries (e.g., in a loop), use BatchAdd instead,
@@ -243,7 +265,7 @@ func (set *Set) Add(addOptions ...string) (*Entry, error) {
 	return entry, nil
 }
 
-// Adds given entries (with their options) to the set.
+// BatchAdd given entries (with their options) to the set.
 // For multiple items, this is much faster than Add().
 func (set *Set) BatchAdd(addOptions [][]string) error {
 	newEntries := make([]*Entry, len(addOptions))
@@ -389,14 +411,59 @@ func parseIPSetSave(ipset *IPSet, result string) map[string]*Set {
 // create KUBE-DST-3YNVZWWGX3UQQ4VQ hash:ip family inet hashsize 1024 maxelem 65536 timeout 0
 // add KUBE-DST-3YNVZWWGX3UQQ4VQ 100.96.1.6 timeout 0
 func buildIPSetRestore(ipset *IPSet) string {
-	ipSetRestore := ""
-	for _, set := range ipset.Sets {
-		ipSetRestore += fmt.Sprintf("create %s %s\n", set.Name, strings.Join(set.Options[:], " "))
+	setNames := make([]string, 0, len(ipset.Sets))
+	for setName := range ipset.Sets {
+		// we need setNames in some consistent order so that we can unit-test this method has a predictable output:
+		setNames = append(setNames, setName)
+	}
+
+	sort.Strings(setNames)
+
+	tmpSets := map[string]string{}
+	ipSetRestore := &strings.Builder{}
+	for _, setName := range setNames {
+		set := ipset.Sets[setName]
+		setOptions := strings.Join(set.Options, " ")
+
+		tmpSetName := tmpSets[setOptions]
+		if tmpSetName == "" {
+			// create a temporary set per unique set-options:
+			hash := sha1.Sum([]byte("tmp:" + setOptions))
+			tmpSetName = tmpIPSetPrefix + base32.StdEncoding.EncodeToString(hash[:10])
+			ipSetRestore.WriteString(fmt.Sprintf("create %s %s\n", tmpSetName, setOptions))
+			// just in case we are starting up after a crash, we should flush the TMP ipset to be safe if it
+			// already existed, so we do not pollute other ipsets:
+			ipSetRestore.WriteString(fmt.Sprintf("flush %s\n", tmpSetName))
+			tmpSets[setOptions] = tmpSetName
+		}
+
 		for _, entry := range set.Entries {
-			ipSetRestore += fmt.Sprintf("add %s %s\n", set.Name, strings.Join(entry.Options[:], " "))
+			// add entries to the tmp set:
+			ipSetRestore.WriteString(fmt.Sprintf("add %s %s\n", tmpSetName, strings.Join(entry.Options, " ")))
 		}
+
+		// now create the actual IPSet (this is a noop if it already exists, because we run with -exists):
+		ipSetRestore.WriteString(fmt.Sprintf("create %s %s\n", set.Name, setOptions))
+
+		// now that both exist, we can swap them:
+		ipSetRestore.WriteString(fmt.Sprintf("swap %s %s\n", tmpSetName, set.Name))
+
+		// empty the tmp set (which is actually the old one now):
+		ipSetRestore.WriteString(fmt.Sprintf("flush %s\n", tmpSetName))
 	}
-	return ipSetRestore
+
+	setsToDestroy := make([]string, 0, len(tmpSets))
+	for _, tmpSetName := range tmpSets {
+		setsToDestroy = append(setsToDestroy, tmpSetName)
+	}
+	// need to destroy the sets in a predictable order for unit test!
+	sort.Strings(setsToDestroy)
+	for _, tmpSetName := range setsToDestroy {
+		// finally, destroy the tmp sets.
+		ipSetRestore.WriteString(fmt.Sprintf("destroy %s\n", tmpSetName))
+	}
+
+	return ipSetRestore.String()
 }
 
 // Save the given set, or all sets if none is given to stdout in a format that
@@ -489,7 +556,7 @@ func (set *Set) Refresh(entries []string, extraOptions ...string) error {
 	return set.RefreshWithBuiltinOptions(entriesWithOptions)
 }
 
-// Refresh a Set with new entries with built-in options.
+// RefreshWithBuiltinOptions refresh a Set with new entries with built-in options.
 func (set *Set) RefreshWithBuiltinOptions(entries [][]string) error {
 	var err error
 

pkg/agent/netpol/utils/ipset_test.go

+// Apache License v2.0 (copyright Cloud Native Labs & Rancher Labs)
+// - modified from https://github.com/cloudnativelabs/kube-router/blob/73b1b03b32c5755b240f6c077bb097abe3888314/pkg/utils/ipset_test.go
+
+package utils
+
+import "testing"
+
+func Test_buildIPSetRestore(t *testing.T) {
+	type args struct {
+		ipset *IPSet
+	}
+	tests := []struct {
+		name string
+		args args
+		want string
+	}{
+		{
+			name: "simple-restore",
+			args: args{
+				ipset: &IPSet{Sets: map[string]*Set{
+					"foo": {
+						Name:    "foo",
+						Options: []string{"hash:ip", "yolo", "things", "12345"},
+						Entries: []*Entry{
+							{Options: []string{"1.2.3.4"}},
+						},
+					},
+					"google-dns-servers": {
+						Name:    "google-dns-servers",
+						Options: []string{"hash:ip", "lol"},
+						Entries: []*Entry{
+							{Options: []string{"4.4.4.4"}},
+							{Options: []string{"8.8.8.8"}},
+						},
+					},
+					// this one and the one above share the same exact options -- and therefore will reuse the same
+					// tmp ipset:
+					"more-ip-addresses": {
+						Name:    "google-dns-servers",
+						Options: []string{"hash:ip", "lol"},
+						Entries: []*Entry{
+							{Options: []string{"5.5.5.5"}},
+							{Options: []string{"6.6.6.6"}},
+						},
+					},
+				}},
+			},
+			want: "create TMP-7NOTZDOMLXBX6DAJ hash:ip yolo things 12345\n" +
+				"flush TMP-7NOTZDOMLXBX6DAJ\n" +
+				"add TMP-7NOTZDOMLXBX6DAJ 1.2.3.4\n" +
+				"create foo hash:ip yolo things 12345\n" +
+				"swap TMP-7NOTZDOMLXBX6DAJ foo\n" +
+				"flush TMP-7NOTZDOMLXBX6DAJ\n" +
+				"create TMP-XD7BSSQZELS7TP35 hash:ip lol\n" +
+				"flush TMP-XD7BSSQZELS7TP35\n" +
+				"add TMP-XD7BSSQZELS7TP35 4.4.4.4\n" +
+				"add TMP-XD7BSSQZELS7TP35 8.8.8.8\n" +
+				"create google-dns-servers hash:ip lol\n" +
+				"swap TMP-XD7BSSQZELS7TP35 google-dns-servers\n" +
+				"flush TMP-XD7BSSQZELS7TP35\n" +
+				"add TMP-XD7BSSQZELS7TP35 5.5.5.5\n" +
+				"add TMP-XD7BSSQZELS7TP35 6.6.6.6\n" +
+				"create google-dns-servers hash:ip lol\n" +
+				"swap TMP-XD7BSSQZELS7TP35 google-dns-servers\n" +
+				"flush TMP-XD7BSSQZELS7TP35\n" +
+				"destroy TMP-7NOTZDOMLXBX6DAJ\n" +
+				"destroy TMP-XD7BSSQZELS7TP35\n",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := buildIPSetRestore(tt.args.ipset); got != tt.want {
+				t.Errorf("buildIPSetRestore() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

pkg/agent/netpol/utils/iptables.go

+// Apache License v2.0 (copyright Cloud Native Labs & Rancher Labs)
+// - modified from https://github.com/cloudnativelabs/kube-router/blob/73b1b03b32c5755b240f6c077bb097abe3888314/pkg/utils/iptables.go
+
+package utils
+
+import (
+	"bytes"
+	"fmt"
+	"os/exec"
+	"strings"
+)
+
+var hasWait bool
+
+func init() {
+	path, err := exec.LookPath("iptables-restore")
+	if err != nil {
+		return
+	}
+	args := []string{"iptables-restore", "--help"}
+	cmd := exec.Cmd{
+		Path: path,
+		Args: args,
+	}
+	cmdOutput, err := cmd.CombinedOutput()
+	if err != nil {
+		return
+	}
+	hasWait = strings.Contains(string(cmdOutput), "wait")
+}
+
+// SaveInto calls `iptables-save` for given table and stores result in a given buffer.
+func SaveInto(table string, buffer *bytes.Buffer) error {
+	path, err := exec.LookPath("iptables-save")
+	if err != nil {
+		return err
+	}
+	stderrBuffer := bytes.NewBuffer(nil)
+	args := []string{"iptables-save", "-t", table}
+	cmd := exec.Cmd{
+		Path:   path,
+		Args:   args,
+		Stdout: buffer,
+		Stderr: stderrBuffer,
+	}
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("%v (%s)", err, stderrBuffer)
+	}
+	return nil
+}
+
+// Restore runs `iptables-restore` passing data through []byte.
+func Restore(table string, data []byte) error {
+	path, err := exec.LookPath("iptables-restore")
+	if err != nil {
+		return err
+	}
+	var args []string
+	if hasWait {
+		args = []string{"iptables-restore", "--wait", "-T", table}
+	} else {
+		args = []string{"iptables-restore", "-T", table}
+	}
+	cmd := exec.Cmd{
+		Path:  path,
+		Args:  args,
+		Stdin: bytes.NewBuffer(data),
+	}
+	b, err := cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("%v (%s)", err, b)
+	}
+
+	return nil
+}

pkg/agent/netpol/utils/node.go

 // Apache License v2.0 (copyright Cloud Native Labs & Rancher Labs)
-// - modified from https://github.com/cloudnativelabs/kube-router/blob/ee9f6d890d10609284098229fa1e283ab5d83b93/pkg/utils/node.go
+// - modified from https://github.com/cloudnativelabs/kube-router/blob/73b1b03b32c5755b240f6c077bb097abe3888314/pkg/utils/node.go
 
 // +build !windows
 
@@ -43,7 +43,7 @@ func GetNodeObject(clientset kubernetes.Interface, hostnameOverride string) (*ap
 		}
 	}
 
-	return nil, fmt.Errorf("Failed to identify the node by NODE_NAME, hostname or --hostname-override")
+	return nil, fmt.Errorf("failed to identify the node by NODE_NAME, hostname or --hostname-override")
 }
 
 // GetNodeIP returns the most valid external facing IP address for a node.

pkg/cluster/encrypt.go

@@ -30,7 +30,7 @@ func keyHash(passphrase string) string {
 }
 
 // encrypt encrypts a byte slice using aes+gcm with a pbkdf2 key derived from the passphrase and a random salt.
-// It returns a byte slice containing the salt and base64-encoded cyphertext.
+// It returns a byte slice containing the salt and base64-encoded ciphertext.
 func encrypt(passphrase string, plaintext []byte) ([]byte, error) {
 	salt, err := token.Random(8)
 	if err != nil {
@@ -59,7 +59,7 @@ func encrypt(passphrase string, plaintext []byte) ([]byte, error) {
 }
 
 // decrypt attempts to decrypt the byte slice using the supplied passphrase.
-// The input byte slice should be the cyphertext output from the encrypt function.
+// The input byte slice should be the ciphertext output from the encrypt function.
 func decrypt(passphrase string, ciphertext []byte) ([]byte, error) {
 	parts := strings.SplitN(string(ciphertext), ":", 2)
 	if len(parts) != 2 {

vendor/modules.txt

@@ -2140,6 +2140,7 @@ k8s.io/heapster/metrics/api/v1/types
 ## explicit
 k8s.io/klog
 # k8s.io/klog/v2 v2.8.0
+## explicit
 k8s.io/klog/v2
 # k8s.io/kube-aggregator v0.18.0 => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-aggregator v1.21.2-k3s1
 k8s.io/kube-aggregator/pkg/apis/apiregistration