implement ipvs mode of kube-proxy

build/debian-iptables/Dockerfile

@@ -21,4 +21,5 @@ CROSS_BUILD_COPY qemu-ARCH-static /usr/bin/
 RUN clean-install \
     iptables \
     ebtables \
-    conntrack
+    conntrack \
+    module-init-tools

cmd/kube-proxy/app/BUILD

@@ -18,6 +18,7 @@ go_library(
         "//pkg/apis/componentconfig/v1alpha1:go_default_library",
         "//pkg/client/clientset_generated/internalclientset:go_default_library",
         "//pkg/client/informers/informers_generated/internalversion:go_default_library",
+        "//pkg/features:go_default_library",
         "//pkg/kubectl/cmd/util:go_default_library",
         "//pkg/kubelet/qos:go_default_library",
         "//pkg/master/ports:go_default_library",
@@ -25,11 +26,13 @@ go_library(
         "//pkg/proxy/config:go_default_library",
         "//pkg/proxy/healthcheck:go_default_library",
         "//pkg/proxy/iptables:go_default_library",
+        "//pkg/proxy/ipvs:go_default_library",
         "//pkg/proxy/userspace:go_default_library",
         "//pkg/proxy/winuserspace:go_default_library",
         "//pkg/util/configz:go_default_library",
         "//pkg/util/dbus:go_default_library",
         "//pkg/util/iptables:go_default_library",
+        "//pkg/util/ipvs:go_default_library",
         "//pkg/util/mount:go_default_library",
         "//pkg/util/netsh:go_default_library",
         "//pkg/util/node:go_default_library",

cmd/kube-proxy/app/server.go

@@ -58,11 +58,13 @@ import (
 	proxyconfig "k8s.io/kubernetes/pkg/proxy/config"
 	"k8s.io/kubernetes/pkg/proxy/healthcheck"
 	"k8s.io/kubernetes/pkg/proxy/iptables"
+	"k8s.io/kubernetes/pkg/proxy/ipvs"
 	"k8s.io/kubernetes/pkg/proxy/userspace"
 	"k8s.io/kubernetes/pkg/proxy/winuserspace"
 	"k8s.io/kubernetes/pkg/util/configz"
 	utildbus "k8s.io/kubernetes/pkg/util/dbus"
 	utiliptables "k8s.io/kubernetes/pkg/util/iptables"
+	utilipvs "k8s.io/kubernetes/pkg/util/ipvs"
 	utilnetsh "k8s.io/kubernetes/pkg/util/netsh"
 	utilnode "k8s.io/kubernetes/pkg/util/node"
 	"k8s.io/kubernetes/pkg/util/oom"
@@ -76,17 +78,19 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
+	"k8s.io/kubernetes/pkg/features"
 )
 
 const (
 	proxyModeUserspace = "userspace"
 	proxyModeIPTables  = "iptables"
+	proxyModeIPVS      = "ipvs"
 )
 
 // checkKnownProxyMode returns true if proxyMode is valid.
 func checkKnownProxyMode(proxyMode string) bool {
 	switch proxyMode {
-	case "", proxyModeUserspace, proxyModeIPTables:
+	case "", proxyModeUserspace, proxyModeIPTables, proxyModeIPVS:
 		return true
 	}
 	return false
@@ -122,7 +126,8 @@ type Options struct {
 func AddFlags(options *Options, fs *pflag.FlagSet) {
 	fs.StringVar(&options.ConfigFile, "config", options.ConfigFile, "The path to the configuration file.")
 	fs.StringVar(&options.WriteConfigTo, "write-config-to", options.WriteConfigTo, "If set, write the default configuration values to this file and exit.")
-	fs.BoolVar(&options.CleanupAndExit, "cleanup-iptables", options.CleanupAndExit, "If true, cleanup iptables rules and exit.")
+	fs.MarkDeprecated("cleanup-iptables", "This flag is replaced by cleanup-proxyrules.")
+	fs.BoolVar(&options.CleanupAndExit, "cleanup", options.CleanupAndExit, "If true cleanup iptables and ipvs rules and exit.")
 
 	// All flags below here are deprecated and will eventually be removed.
 
@@ -137,10 +142,12 @@ func AddFlags(options *Options, fs *pflag.FlagSet) {
 	fs.StringVar(&options.config.ClientConnection.KubeConfigFile, "kubeconfig", options.config.ClientConnection.KubeConfigFile, "Path to kubeconfig file with authorization information (the master location is set by the master flag).")
 	fs.Var(componentconfig.PortRangeVar{Val: &options.config.PortRange}, "proxy-port-range", "Range of host ports (beginPort-endPort, inclusive) that may be consumed in order to proxy service traffic. If unspecified (0-0) then ports will be randomly chosen.")
 	fs.StringVar(&options.config.HostnameOverride, "hostname-override", options.config.HostnameOverride, "If non-empty, will use this string as identification instead of the actual hostname.")
-	fs.Var(&options.config.Mode, "proxy-mode", "Which proxy mode to use: 'userspace' (older) or 'iptables' (faster). If blank, use the best-available proxy (currently iptables).  If the iptables proxy is selected, regardless of how, but the system's kernel or iptables versions are insufficient, this always falls back to the userspace proxy.")
+	fs.Var(&options.config.Mode, "proxy-mode", "Which proxy mode to use: 'userspace' (older) or 'iptables' (faster) or 'ipvs'(experimental). If blank, use the best-available proxy (currently iptables).  If the iptables proxy is selected, regardless of how, but the system's kernel or iptables versions are insufficient, this always falls back to the userspace proxy.")
 	fs.Int32Var(options.config.IPTables.MasqueradeBit, "iptables-masquerade-bit", utilpointer.Int32PtrDerefOr(options.config.IPTables.MasqueradeBit, 14), "If using the pure iptables proxy, the bit of the fwmark space to mark packets requiring SNAT with.  Must be within the range [0, 31].")
 	fs.DurationVar(&options.config.IPTables.SyncPeriod.Duration, "iptables-sync-period", options.config.IPTables.SyncPeriod.Duration, "The maximum interval of how often iptables rules are refreshed (e.g. '5s', '1m', '2h22m').  Must be greater than 0.")
 	fs.DurationVar(&options.config.IPTables.MinSyncPeriod.Duration, "iptables-min-sync-period", options.config.IPTables.MinSyncPeriod.Duration, "The minimum interval of how often the iptables rules can be refreshed as endpoints and services change (e.g. '5s', '1m', '2h22m').")
+	fs.DurationVar(&options.config.IPVS.SyncPeriod.Duration, "ipvs-sync-period", options.config.IPVS.SyncPeriod.Duration, "The maximum interval of how often ipvs rules are refreshed (e.g. '5s', '1m', '2h22m').  Must be greater than 0.")
+	fs.DurationVar(&options.config.IPVS.MinSyncPeriod.Duration, "ipvs-min-sync-period", options.config.IPVS.MinSyncPeriod.Duration, "The minimum interval of how often the ipvs rules can be refreshed as endpoints and services change (e.g. '5s', '1m', '2h22m').")
 	fs.DurationVar(&options.config.ConfigSyncPeriod.Duration, "config-sync-period", options.config.ConfigSyncPeriod.Duration, "How often configuration from the apiserver is refreshed.  Must be greater than 0.")
 	fs.BoolVar(&options.config.IPTables.MasqueradeAll, "masquerade-all", options.config.IPTables.MasqueradeAll, "If using the pure iptables proxy, SNAT everything (this not commonly needed)")
 	fs.StringVar(&options.config.ClusterCIDR, "cluster-cidr", options.config.ClusterCIDR, "The CIDR range of pods in the cluster. It is used to bridge traffic coming from outside of the cluster. If not provided, no off-cluster bridging will be performed.")
@@ -161,7 +168,7 @@ func AddFlags(options *Options, fs *pflag.FlagSet) {
 		options.config.Conntrack.TCPCloseWaitTimeout.Duration,
 		"NAT timeout for TCP connections in the CLOSE_WAIT state")
 	fs.BoolVar(&options.config.EnableProfiling, "profiling", options.config.EnableProfiling, "If true enables profiling via web interface on /debug/pprof handler.")
-
+	fs.StringVar(&options.config.IPVS.Scheduler, "ipvs-scheduler", options.config.IPVS.Scheduler, "The ipvs scheduler type when proxy mode is ipvs")
 	utilfeature.DefaultFeatureGate.AddFlag(fs)
 }
 
@@ -187,7 +194,7 @@ func NewOptions() (*Options, error) {
 // Complete completes all the required options.
 func (o *Options) Complete() error {
 	if len(o.ConfigFile) == 0 && len(o.WriteConfigTo) == 0 {
-		glog.Warning("WARNING: all flags other than --config, --write-config-to, and --cleanup-iptables are deprecated. Please begin using a config file ASAP.")
+		glog.Warning("WARNING: all flags other than --config, --write-config-to, and --cleanup are deprecated. Please begin using a config file ASAP.")
 		o.applyDeprecatedHealthzPortToConfig()
 	}
 
@@ -363,6 +370,8 @@ type ProxyServer struct {
 	Client                 clientset.Interface
 	EventClient            v1core.EventsGetter
 	IptInterface           utiliptables.Interface
+	IpvsInterface          utilipvs.Interface
+	execer                 exec.Interface
 	Proxier                proxy.ProxyProvider
 	Broadcaster            record.EventBroadcaster
 	Recorder               record.EventRecorder
@@ -435,6 +444,7 @@ func NewProxyServer(config *componentconfig.KubeProxyConfiguration, cleanupAndEx
 
 	var netshInterface utilnetsh.Interface
 	var iptInterface utiliptables.Interface
+	var ipvsInterface utilipvs.Interface
 	var dbus utildbus.Interface
 
 	// Create a iptables utils.
@@ -445,11 +455,12 @@ func NewProxyServer(config *componentconfig.KubeProxyConfiguration, cleanupAndEx
 	} else {
 		dbus = utildbus.New()
 		iptInterface = utiliptables.New(execer, dbus, protocol)
+		ipvsInterface = utilipvs.New(execer)
 	}
 
 	// We omit creation of pretty much everything if we run in cleanup mode
 	if cleanupAndExit {
-		return &ProxyServer{IptInterface: iptInterface, CleanupAndExit: cleanupAndExit}, nil
+		return &ProxyServer{IptInterface: iptInterface, IpvsInterface: ipvsInterface, CleanupAndExit: cleanupAndExit}, nil
 	}
 
 	client, eventClient, err := createClients(config.ClientConnection, master)
@@ -517,9 +528,40 @@ func NewProxyServer(config *componentconfig.KubeProxyConfiguration, cleanupAndEx
 		serviceEventHandler = proxierIPTables
 		endpointsEventHandler = proxierIPTables
 		// No turning back. Remove artifacts that might still exist from the userspace Proxier.
-		glog.V(0).Info("Tearing down userspace rules.")
+		glog.V(0).Info("Tearing down inactive rules.")
+		// TODO this has side effects that should only happen when Run() is invoked.
+		userspace.CleanupLeftovers(iptInterface)
+		// IPVS Proxier will generate some iptables rules,
+		// need to clean them before switching to other proxy mode.
+		ipvs.CleanupLeftovers(execer, ipvsInterface, iptInterface)
+	} else if proxyMode == proxyModeIPVS {
+		glog.V(0).Info("Using ipvs Proxier.")
+		proxierIPVS, err := ipvs.NewProxier(
+			iptInterface,
+			ipvsInterface,
+			utilsysctl.New(),
+			execer,
+			config.IPVS.SyncPeriod.Duration,
+			config.IPVS.MinSyncPeriod.Duration,
+			config.IPTables.MasqueradeAll,
+			int(*config.IPTables.MasqueradeBit),
+			config.ClusterCIDR,
+			hostname,
+			getNodeIP(client, hostname),
+			recorder,
+			healthzServer,
+			config.IPVS.Scheduler,
+		)
+		if err != nil {
+			return nil, fmt.Errorf("unable to create proxier: %v", err)
+		}
+		proxier = proxierIPVS
+		serviceEventHandler = proxierIPVS
+		endpointsEventHandler = proxierIPVS
+		glog.V(0).Info("Tearing down inactive rules.")
 		// TODO this has side effects that should only happen when Run() is invoked.
 		userspace.CleanupLeftovers(iptInterface)
+		iptables.CleanupLeftovers(iptInterface)
 	} else {
 		glog.V(0).Info("Using userspace Proxier.")
 		if goruntime.GOOS == "windows" {
@@ -566,11 +608,14 @@ func NewProxyServer(config *componentconfig.KubeProxyConfiguration, cleanupAndEx
 			serviceEventHandler = proxierUserspace
 			proxier = proxierUserspace
 		}
-		// Remove artifacts from the pure-iptables Proxier, if not on Windows.
+		// Remove artifacts from the iptables and ipvs Proxier, if not on Windows.
 		if goruntime.GOOS != "windows" {
-			glog.V(0).Info("Tearing down pure-iptables proxy rules.")
+			glog.V(0).Info("Tearing down inactive rules.")
 			// TODO this has side effects that should only happen when Run() is invoked.
 			iptables.CleanupLeftovers(iptInterface)
+			// IPVS Proxier will generate some iptables rules,
+			// need to clean them before switching to other proxy mode.
+			ipvs.CleanupLeftovers(execer, ipvsInterface, iptInterface)
 		}
 	}
 
@@ -583,6 +628,8 @@ func NewProxyServer(config *componentconfig.KubeProxyConfiguration, cleanupAndEx
 		Client:                 client,
 		EventClient:            eventClient,
 		IptInterface:           iptInterface,
+		IpvsInterface:          ipvsInterface,
+		execer:                 execer,
 		Proxier:                proxier,
 		Broadcaster:            eventBroadcaster,
 		Recorder:               recorder,
@@ -607,6 +654,7 @@ func (s *ProxyServer) Run() error {
 	if s.CleanupAndExit {
 		encounteredError := userspace.CleanupLeftovers(s.IptInterface)
 		encounteredError = iptables.CleanupLeftovers(s.IptInterface) || encounteredError
+		encounteredError = ipvs.CleanupLeftovers(s.execer, s.IpvsInterface, s.IptInterface) || encounteredError
 		if encounteredError {
 			return errors.New("encountered an error while tearing down rules.")
 		}
@@ -754,10 +802,38 @@ func getProxyMode(proxyMode string, iptver iptables.IPTablesVersioner, kcompat i
 		return proxyModeUserspace
 	}
 
-	if len(proxyMode) > 0 && proxyMode != proxyModeIPTables {
-		glog.Warningf("Flag proxy-mode=%q unknown, assuming iptables proxy", proxyMode)
+	if len(proxyMode) > 0 && proxyMode == proxyModeIPTables {
+		return tryIPTablesProxy(iptver, kcompat)
+	}
+
+	if utilfeature.DefaultFeatureGate.Enabled(features.SupportIPVSProxyMode) {
+		if proxyMode == proxyModeIPVS {
+			return tryIPVSProxy(iptver, kcompat)
+		} else {
+			glog.Warningf("Can't use ipvs proxier, trying iptables proxier")
+			return tryIPTablesProxy(iptver, kcompat)
+		}
+	}
+	glog.Warningf("Flag proxy-mode=%q unknown, assuming iptables proxy", proxyMode)
+	return tryIPTablesProxy(iptver, kcompat)
+}
+
+func tryIPVSProxy(iptver iptables.IPTablesVersioner, kcompat iptables.KernelCompatTester) string {
+	// guaranteed false on error, error only necessary for debugging
+	// IPVS Proxier relies on iptables
+	useIPVSProxy, err := ipvs.CanUseIPVSProxier()
+	if err != nil {
+		utilruntime.HandleError(fmt.Errorf("can't determine whether to use ipvs proxy, using userspace proxier: %v", err))
+		return proxyModeUserspace
+	}
+	if useIPVSProxy {
+		return proxyModeIPVS
 	}
 
+	// TODO: Check ipvs version
+
+	// Try to fallback to iptables before falling back to userspace
+	glog.V(1).Infof("Can't use ipvs proxier, trying iptables proxier")
 	return tryIPTablesProxy(iptver, kcompat)
 }
 

cmd/kube-proxy/app/server_test.go

@@ -287,6 +287,9 @@ iptables:
   masqueradeBit: 17
   minSyncPeriod: 10s
   syncPeriod: 60s
+ipvs:
+  minSyncPeriod: 10s
+  syncPeriod: 60s
 kind: KubeProxyConfiguration
 metricsBindAddress: "%s"
 mode: "iptables"
@@ -347,12 +350,17 @@ udpTimeoutMilliseconds: 123ms
 				MinSyncPeriod: metav1.Duration{Duration: 10 * time.Second},
 				SyncPeriod:    metav1.Duration{Duration: 60 * time.Second},
 			},
+			IPVS: componentconfig.KubeProxyIPVSConfiguration{
+				MinSyncPeriod: metav1.Duration{Duration: 10 * time.Second},
+				SyncPeriod:    metav1.Duration{Duration: 60 * time.Second},
+			},
 			MetricsBindAddress: tc.metricsBindAddress,
 			Mode:               "iptables",
-			OOMScoreAdj:        utilpointer.Int32Ptr(17),
-			PortRange:          "2-7",
-			ResourceContainer:  "/foo",
-			UDPIdleTimeout:     metav1.Duration{Duration: 123 * time.Millisecond},
+			// TODO: IPVS
+			OOMScoreAdj:       utilpointer.Int32Ptr(17),
+			PortRange:         "2-7",
+			ResourceContainer: "/foo",
+			UDPIdleTimeout:    metav1.Duration{Duration: 123 * time.Millisecond},
 		}
 
 		options, err := NewOptions()

pkg/apis/componentconfig/types.go

@@ -52,6 +52,19 @@ type KubeProxyIPTablesConfiguration struct {
 	MinSyncPeriod metav1.Duration
 }
 
+// KubeProxyIPVSConfiguration contains ipvs-related configuration
+// details for the Kubernetes proxy server.
+type KubeProxyIPVSConfiguration struct {
+	// syncPeriod is the period that ipvs rules are refreshed (e.g. '5s', '1m',
+	// '2h22m').  Must be greater than 0.
+	SyncPeriod metav1.Duration
+	// minSyncPeriod is the minimum period that ipvs rules are refreshed (e.g. '5s', '1m',
+	// '2h22m').
+	MinSyncPeriod metav1.Duration
+	// ipvs scheduler
+	Scheduler string
+}
+
 // KubeProxyConntrackConfiguration contains conntrack settings for
 // the Kubernetes proxy server.
 type KubeProxyConntrackConfiguration struct {
@@ -112,6 +125,8 @@ type KubeProxyConfiguration struct {
 	ClientConnection ClientConnectionConfiguration
 	// iptables contains iptables-related configuration options.
 	IPTables KubeProxyIPTablesConfiguration
+	// ipvs contains ipvs-related configuration options.
+	IPVS KubeProxyIPVSConfiguration
 	// oomScoreAdj is the oom-score-adj value for kube-proxy process. Values must be within
 	// the range [-1000, 1000]
 	OOMScoreAdj *int32

pkg/apis/componentconfig/v1alpha1/types.go

@@ -52,6 +52,19 @@ type KubeProxyIPTablesConfiguration struct {
 	MinSyncPeriod metav1.Duration `json:"minSyncPeriod"`
 }
 
+// KubeProxyIPVSConfiguration contains ipvs-related configuration
+// details for the Kubernetes proxy server.
+type KubeProxyIPVSConfiguration struct {
+	// syncPeriod is the period that ipvs rules are refreshed (e.g. '5s', '1m',
+	// '2h22m').  Must be greater than 0.
+	SyncPeriod metav1.Duration `json:"syncPeriod"`
+	// minSyncPeriod is the minimum period that ipvs rules are refreshed (e.g. '5s', '1m',
+	// '2h22m').
+	MinSyncPeriod metav1.Duration `json:"minSyncPeriod"`
+	// ipvs scheduler
+	Scheduler string `json:"scheduler"`
+}
+
 // KubeProxyConntrackConfiguration contains conntrack settings for
 // the Kubernetes proxy server.
 type KubeProxyConntrackConfiguration struct {
@@ -112,6 +125,8 @@ type KubeProxyConfiguration struct {
 	ClientConnection ClientConnectionConfiguration `json:"clientConnection"`
 	// iptables contains iptables-related configuration options.
 	IPTables KubeProxyIPTablesConfiguration `json:"iptables"`
+	// ipvs contains ipvs-related configuration options.
+	IPVS KubeProxyIPVSConfiguration `json:"ipvs"`
 	// oomScoreAdj is the oom-score-adj value for kube-proxy process. Values must be within
 	// the range [-1000, 1000]
 	OOMScoreAdj *int32 `json:"oomScoreAdj"`

pkg/apis/componentconfig/v1alpha1/zz_generated.conversion.go

@@ -44,6 +44,8 @@ func RegisterConversions(scheme *runtime.Scheme) error {
 		Convert_componentconfig_KubeProxyConntrackConfiguration_To_v1alpha1_KubeProxyConntrackConfiguration,
 		Convert_v1alpha1_KubeProxyIPTablesConfiguration_To_componentconfig_KubeProxyIPTablesConfiguration,
 		Convert_componentconfig_KubeProxyIPTablesConfiguration_To_v1alpha1_KubeProxyIPTablesConfiguration,
+		Convert_v1alpha1_KubeProxyIPVSConfiguration_To_componentconfig_KubeProxyIPVSConfiguration,
+		Convert_componentconfig_KubeProxyIPVSConfiguration_To_v1alpha1_KubeProxyIPVSConfiguration,
 		Convert_v1alpha1_KubeSchedulerConfiguration_To_componentconfig_KubeSchedulerConfiguration,
 		Convert_componentconfig_KubeSchedulerConfiguration_To_v1alpha1_KubeSchedulerConfiguration,
 		Convert_v1alpha1_LeaderElectionConfiguration_To_componentconfig_LeaderElectionConfiguration,
@@ -93,6 +95,9 @@ func autoConvert_v1alpha1_KubeProxyConfiguration_To_componentconfig_KubeProxyCon
 	if err := Convert_v1alpha1_KubeProxyIPTablesConfiguration_To_componentconfig_KubeProxyIPTablesConfiguration(&in.IPTables, &out.IPTables, s); err != nil {
 		return err
 	}
+	if err := Convert_v1alpha1_KubeProxyIPVSConfiguration_To_componentconfig_KubeProxyIPVSConfiguration(&in.IPVS, &out.IPVS, s); err != nil {
+		return err
+	}
 	out.OOMScoreAdj = (*int32)(unsafe.Pointer(in.OOMScoreAdj))
 	out.Mode = componentconfig.ProxyMode(in.Mode)
 	out.PortRange = in.PortRange
@@ -124,6 +129,9 @@ func autoConvert_componentconfig_KubeProxyConfiguration_To_v1alpha1_KubeProxyCon
 	if err := Convert_componentconfig_KubeProxyIPTablesConfiguration_To_v1alpha1_KubeProxyIPTablesConfiguration(&in.IPTables, &out.IPTables, s); err != nil {
 		return err
 	}
+	if err := Convert_componentconfig_KubeProxyIPVSConfiguration_To_v1alpha1_KubeProxyIPVSConfiguration(&in.IPVS, &out.IPVS, s); err != nil {
+		return err
+	}
 	out.OOMScoreAdj = (*int32)(unsafe.Pointer(in.OOMScoreAdj))
 	out.Mode = ProxyMode(in.Mode)
 	out.PortRange = in.PortRange
@@ -195,6 +203,30 @@ func Convert_componentconfig_KubeProxyIPTablesConfiguration_To_v1alpha1_KubeProx
 	return autoConvert_componentconfig_KubeProxyIPTablesConfiguration_To_v1alpha1_KubeProxyIPTablesConfiguration(in, out, s)
 }
 
+func autoConvert_v1alpha1_KubeProxyIPVSConfiguration_To_componentconfig_KubeProxyIPVSConfiguration(in *KubeProxyIPVSConfiguration, out *componentconfig.KubeProxyIPVSConfiguration, s conversion.Scope) error {
+	out.SyncPeriod = in.SyncPeriod
+	out.MinSyncPeriod = in.MinSyncPeriod
+	out.Scheduler = in.Scheduler
+	return nil
+}
+
+// Convert_v1alpha1_KubeProxyIPVSConfiguration_To_componentconfig_KubeProxyIPVSConfiguration is an autogenerated conversion function.
+func Convert_v1alpha1_KubeProxyIPVSConfiguration_To_componentconfig_KubeProxyIPVSConfiguration(in *KubeProxyIPVSConfiguration, out *componentconfig.KubeProxyIPVSConfiguration, s conversion.Scope) error {
+	return autoConvert_v1alpha1_KubeProxyIPVSConfiguration_To_componentconfig_KubeProxyIPVSConfiguration(in, out, s)
+}
+
+func autoConvert_componentconfig_KubeProxyIPVSConfiguration_To_v1alpha1_KubeProxyIPVSConfiguration(in *componentconfig.KubeProxyIPVSConfiguration, out *KubeProxyIPVSConfiguration, s conversion.Scope) error {
+	out.SyncPeriod = in.SyncPeriod
+	out.MinSyncPeriod = in.MinSyncPeriod
+	out.Scheduler = in.Scheduler
+	return nil
+}
+
+// Convert_componentconfig_KubeProxyIPVSConfiguration_To_v1alpha1_KubeProxyIPVSConfiguration is an autogenerated conversion function.
+func Convert_componentconfig_KubeProxyIPVSConfiguration_To_v1alpha1_KubeProxyIPVSConfiguration(in *componentconfig.KubeProxyIPVSConfiguration, out *KubeProxyIPVSConfiguration, s conversion.Scope) error {
+	return autoConvert_componentconfig_KubeProxyIPVSConfiguration_To_v1alpha1_KubeProxyIPVSConfiguration(in, out, s)
+}
+
 func autoConvert_v1alpha1_KubeSchedulerConfiguration_To_componentconfig_KubeSchedulerConfiguration(in *KubeSchedulerConfiguration, out *componentconfig.KubeSchedulerConfiguration, s conversion.Scope) error {
 	out.Port = int32(in.Port)
 	out.Address = in.Address

pkg/apis/componentconfig/v1alpha1/zz_generated.deepcopy.go

@@ -53,6 +53,10 @@ func RegisterDeepCopies(scheme *runtime.Scheme) error {
 			return nil
 		}, InType: reflect.TypeOf(&KubeProxyIPTablesConfiguration{})},
 		conversion.GeneratedDeepCopyFunc{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error {
+			in.(*KubeProxyIPVSConfiguration).DeepCopyInto(out.(*KubeProxyIPVSConfiguration))
+			return nil
+		}, InType: reflect.TypeOf(&KubeProxyIPVSConfiguration{})},
+		conversion.GeneratedDeepCopyFunc{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error {
 			in.(*KubeSchedulerConfiguration).DeepCopyInto(out.(*KubeSchedulerConfiguration))
 			return nil
 		}, InType: reflect.TypeOf(&KubeSchedulerConfiguration{})},
@@ -85,6 +89,7 @@ func (in *KubeProxyConfiguration) DeepCopyInto(out *KubeProxyConfiguration) {
 	out.TypeMeta = in.TypeMeta
 	out.ClientConnection = in.ClientConnection
 	in.IPTables.DeepCopyInto(&out.IPTables)
+	out.IPVS = in.IPVS
 	if in.OOMScoreAdj != nil {
 		in, out := &in.OOMScoreAdj, &out.OOMScoreAdj
 		if *in == nil {
@@ -165,6 +170,24 @@ func (in *KubeProxyIPTablesConfiguration) DeepCopy() *KubeProxyIPTablesConfigura
 }
 
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *KubeProxyIPVSConfiguration) DeepCopyInto(out *KubeProxyIPVSConfiguration) {
+	*out = *in
+	out.SyncPeriod = in.SyncPeriod
+	out.MinSyncPeriod = in.MinSyncPeriod
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeProxyIPVSConfiguration.
+func (in *KubeProxyIPVSConfiguration) DeepCopy() *KubeProxyIPVSConfiguration {
+	if in == nil {
+		return nil
+	}
+	out := new(KubeProxyIPVSConfiguration)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *KubeSchedulerConfiguration) DeepCopyInto(out *KubeSchedulerConfiguration) {
 	*out = *in
 	out.TypeMeta = in.TypeMeta

pkg/apis/componentconfig/zz_generated.deepcopy.go

@@ -65,6 +65,10 @@ func RegisterDeepCopies(scheme *runtime.Scheme) error {
 			return nil
 		}, InType: reflect.TypeOf(&KubeProxyIPTablesConfiguration{})},
 		conversion.GeneratedDeepCopyFunc{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error {
+			in.(*KubeProxyIPVSConfiguration).DeepCopyInto(out.(*KubeProxyIPVSConfiguration))
+			return nil
+		}, InType: reflect.TypeOf(&KubeProxyIPVSConfiguration{})},
+		conversion.GeneratedDeepCopyFunc{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error {
 			in.(*KubeSchedulerConfiguration).DeepCopyInto(out.(*KubeSchedulerConfiguration))
 			return nil
 		}, InType: reflect.TypeOf(&KubeSchedulerConfiguration{})},
@@ -206,6 +210,7 @@ func (in *KubeProxyConfiguration) DeepCopyInto(out *KubeProxyConfiguration) {
 	out.TypeMeta = in.TypeMeta
 	out.ClientConnection = in.ClientConnection
 	in.IPTables.DeepCopyInto(&out.IPTables)
+	out.IPVS = in.IPVS
 	if in.OOMScoreAdj != nil {
 		in, out := &in.OOMScoreAdj, &out.OOMScoreAdj
 		if *in == nil {
@@ -286,6 +291,24 @@ func (in *KubeProxyIPTablesConfiguration) DeepCopy() *KubeProxyIPTablesConfigura
 }
 
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *KubeProxyIPVSConfiguration) DeepCopyInto(out *KubeProxyIPVSConfiguration) {
+	*out = *in
+	out.SyncPeriod = in.SyncPeriod
+	out.MinSyncPeriod = in.MinSyncPeriod
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeProxyIPVSConfiguration.
+func (in *KubeProxyIPVSConfiguration) DeepCopy() *KubeProxyIPVSConfiguration {
+	if in == nil {
+		return nil
+	}
+	out := new(KubeProxyIPVSConfiguration)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *KubeSchedulerConfiguration) DeepCopyInto(out *KubeSchedulerConfiguration) {
 	*out = *in
 	out.TypeMeta = in.TypeMeta

pkg/features/kube_features.go

@@ -127,6 +127,12 @@ const (
 	// Taint nodes based on their condition status for 'NetworkUnavailable',
 	// 'MemoryPressure', 'OutOfDisk' and 'DiskPressure'.
 	TaintNodesByCondition utilfeature.Feature = "TaintNodesByCondition"
+
+	// owner: @haibinxie
+	// alpha: v1.8
+	//
+	// Implement IPVS-based in-cluster service load balancing
+	SupportIPVSProxyMode utilfeature.Feature = "SupportIPVSProxyMode"
 )
 
 func init() {
@@ -164,4 +170,5 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS
 	// inherited features from apiextensions-apiserver, relisted here to get a conflict if it is changed
 	// unintentionally on either side:
 	apiextensionsfeatures.CustomResourceValidation: {Default: false, PreRelease: utilfeature.Alpha},
+	SupportIPVSProxyMode:                           {Default: false, PreRelease: utilfeature.Alpha},
 }

pkg/proxy/BUILD

@@ -28,6 +28,7 @@ filegroup(
         "//pkg/proxy/config:all-srcs",
         "//pkg/proxy/healthcheck:all-srcs",
         "//pkg/proxy/iptables:all-srcs",
+        "//pkg/proxy/ipvs:all-srcs",
         "//pkg/proxy/userspace:all-srcs",
         "//pkg/proxy/util:all-srcs",
         "//pkg/proxy/winuserspace:all-srcs",

pkg/proxy/ipvs/BUILD

+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = select({
+        "@io_bazel_rules_go//go/platform:linux_amd64": [
+            "proxier_test.go",
+        ],
+        "//conditions:default": [],
+    }),
+    library = ":go_default_library",
+    tags = ["automanaged"],
+    deps = select({
+        "@io_bazel_rules_go//go/platform:linux_amd64": [
+            "//pkg/api:go_default_library",
+            "//pkg/proxy:go_default_library",
+            "//pkg/proxy/util:go_default_library",
+            "//pkg/util/iptables:go_default_library",
+            "//pkg/util/iptables/testing:go_default_library",
+            "//pkg/util/ipvs:go_default_library",
+            "//pkg/util/ipvs/testing:go_default_library",
+            "//vendor/github.com/davecgh/go-spew/spew:go_default_library",
+            "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+            "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+            "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
+            "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+            "//vendor/k8s.io/utils/exec:go_default_library",
+            "//vendor/k8s.io/utils/exec/testing:go_default_library",
+        ],
+        "//conditions:default": [],
+    }),
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["proxier.go"],
+    tags = ["automanaged"],
+    deps = [
+        "//pkg/api:go_default_library",
+        "//pkg/api/helper:go_default_library",
+        "//pkg/api/service:go_default_library",
+        "//pkg/features:go_default_library",
+        "//pkg/proxy:go_default_library",
+        "//pkg/proxy/healthcheck:go_default_library",
+        "//pkg/proxy/util:go_default_library",
+        "//pkg/util/iptables:go_default_library",
+        "//pkg/util/ipvs:go_default_library",
+        "//pkg/util/sysctl:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
+        "//vendor/k8s.io/client-go/tools/record:go_default_library",
+        "//vendor/k8s.io/client-go/util/flowcontrol:go_default_library",
+        "//vendor/k8s.io/utils/exec:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

pkg/proxy/ipvs/proxier.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package ipvs
+
+//
+// NOTE: this needs to be tested in e2e since it uses ipvs for everything.
+//
+
+import (
+	"bytes"
+	"fmt"
+	"net"
+	"reflect"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/golang/glog"
+
+	clientv1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/sets"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	"k8s.io/client-go/tools/record"
+	"k8s.io/client-go/util/flowcontrol"
+	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/api/helper"
+	apiservice "k8s.io/kubernetes/pkg/api/service"
+	"k8s.io/kubernetes/pkg/features"
+	"k8s.io/kubernetes/pkg/proxy"
+	"k8s.io/kubernetes/pkg/proxy/healthcheck"
+	utilproxy "k8s.io/kubernetes/pkg/proxy/util"
+	utiliptables "k8s.io/kubernetes/pkg/util/iptables"
+	utilipvs "k8s.io/kubernetes/pkg/util/ipvs"
+	utilsysctl "k8s.io/kubernetes/pkg/util/sysctl"
+	utilexec "k8s.io/utils/exec"
+)
+
+const (
+	// kubeServicesChain is the services portal chain
+	kubeServicesChain utiliptables.Chain = "KUBE-SERVICES"
+
+	// kubePostroutingChain is the kubernetes postrouting chain
+	kubePostroutingChain utiliptables.Chain = "KUBE-POSTROUTING"
+
+	// KubeMarkMasqChain is the mark-for-masquerade chain
+	KubeMarkMasqChain utiliptables.Chain = "KUBE-MARK-MASQ"
+
+	// KubeMarkDropChain is the mark-for-drop chain
+	KubeMarkDropChain utiliptables.Chain = "KUBE-MARK-DROP"
+)
+
+const (
+	// DefaultScheduler is the default ipvs scheduler algorithm - round robin.
+	DefaultScheduler = "rr"
+	// DefaultDummyDevice is the default dummy interface where ipvs service address will bind to it.
+	DefaultDummyDevice = "kube-ipvs0"
+)
+
+var ipvsModules = []string{
+	"ip_vs",
+	"ip_vs_rr",
+	"ip_vs_wrr",
+	"ip_vs_sh",
+	"nf_conntrack_ipv4",
+}
+
+// In IPVS proxy mode, the following flags need to be setted
+const sysctlRouteLocalnet = "net/ipv4/conf/all/route_localnet"
+const sysctlBridgeCallIPTables = "net/bridge/bridge-nf-call-iptables"
+const sysctlVSConnTrack = "net/ipv4/vs/conntrack"
+const sysctlForward = "net/ipv4/ip_forward"
+
+// Proxier is an ipvs based proxy for connections between a localhost:lport
+// and services that provide the actual backends.
+type Proxier struct {
+	// endpointsChanges and serviceChanges contains all changes to endpoints and
+	// services that happened since last syncProxyRules call. For a single object,
+	// changes are accumulated, i.e. previous is state from before all of them,
+	// current is state after applying all of those.
+	endpointsChanges endpointsChangeMap
+	serviceChanges   serviceChangeMap
+
+	mu           sync.Mutex // protects the following fields
+	serviceMap   proxyServiceMap
+	endpointsMap proxyEndpointsMap
+	portsMap     map[utilproxy.LocalPort]utilproxy.Closeable
+	// endpointsSynced and servicesSynced are set to true when corresponding
+	// objects are synced after startup. This is used to avoid updating ipvs rules
+	// with some partial data after kube-proxy restart.
+	endpointsSynced bool
+	servicesSynced  bool
+
+	throttle flowcontrol.RateLimiter
+
+	// These are effectively const and do not need the mutex to be held.
+	syncPeriod     time.Duration
+	minSyncPeriod  time.Duration
+	iptables       utiliptables.Interface
+	ipvs           utilipvs.Interface
+	exec           utilexec.Interface
+	masqueradeAll  bool
+	masqueradeMark string
+	clusterCIDR    string
+	hostname       string
+	nodeIP         net.IP
+	portMapper     utilproxy.PortOpener
+	recorder       record.EventRecorder
+	healthChecker  healthcheck.Server
+	healthzServer  healthcheck.HealthzUpdater
+	ipvsScheduler  string
+	// Added as a member to the struct to allow injection for testing.
+	ipGetter IPGetter
+}
+
+// IPGetter helps get node network interface IP
+type IPGetter interface {
+	NodeIPs() ([]net.IP, error)
+}
+
+type realIPGetter struct{}
+
+func (r *realIPGetter) NodeIPs() (ips []net.IP, err error) {
+	interfaces, err := net.Interfaces()
+	if err != nil {
+		return nil, err
+	}
+	for i := range interfaces {
+		name := interfaces[i].Name
+		// We assume node ip bind to eth{x}
+		if !strings.HasPrefix(name, "eth") {
+			continue
+		}
+		intf, err := net.InterfaceByName(name)
+		if err != nil {
+			continue
+		}
+		addrs, err := intf.Addrs()
+		if err != nil {
+			continue
+		}
+		for _, a := range addrs {
+			if ipnet, ok := a.(*net.IPNet); ok {
+				if ipnet.IP.To4() != nil {
+					ips = append(ips, ipnet.IP.To4())
+				}
+			}
+		}
+	}
+	return
+}
+
+// Proxier implements ProxyProvider
+var _ proxy.ProxyProvider = &Proxier{}
+
+// NewProxier returns a new Proxier given an iptables and ipvs Interface instance.
+// Because of the iptables and ipvs logic, it is assumed that there is only a single Proxier active on a machine.
+// An error will be returned if it fails to update or acquire the initial lock.
+// Once a proxier is created, it will keep iptables and ipvs rules up to date in the background and
+// will not terminate if a particular iptables or ipvs call fails.
+func NewProxier(ipt utiliptables.Interface, ipvs utilipvs.Interface,
+	sysctl utilsysctl.Interface,
+	exec utilexec.Interface,
+	syncPeriod time.Duration,
+	minSyncPeriod time.Duration,
+	masqueradeAll bool,
+	masqueradeBit int,
+	clusterCIDR string,
+	hostname string,
+	nodeIP net.IP,
+	recorder record.EventRecorder,
+	healthzServer healthcheck.HealthzUpdater,
+	scheduler string,
+) (*Proxier, error) {
+	// check valid user input
+	if minSyncPeriod > syncPeriod {
+		return nil, fmt.Errorf("min-sync (%v) must be < sync(%v)", minSyncPeriod, syncPeriod)
+	}
+
+	// Set the route_localnet sysctl we need for
+	if err := sysctl.SetSysctl(sysctlRouteLocalnet, 1); err != nil {
+		return nil, fmt.Errorf("can't set sysctl %s: %v", sysctlRouteLocalnet, err)
+	}
+
+	// Proxy needs br_netfilter and bridge-nf-call-iptables=1 when containers
+	// are connected to a Linux bridge (but not SDN bridges).  Until most
+	// plugins handle this, log when config is missing
+	if val, err := sysctl.GetSysctl(sysctlBridgeCallIPTables); err == nil && val != 1 {
+		glog.Infof("missing br-netfilter module or unset sysctl br-nf-call-iptables; proxy may not work as intended")
+	}
+
+	// Set the conntrack sysctl we need for
+	if err := sysctl.SetSysctl(sysctlVSConnTrack, 1); err != nil {
+		return nil, fmt.Errorf("can't set sysctl %s: %v", sysctlVSConnTrack, err)
+	}
+
+	// Set the ip_forward sysctl we need for
+	if err := sysctl.SetSysctl(sysctlForward, 1); err != nil {
+		return nil, fmt.Errorf("can't set sysctl %s: %v", sysctlForward, err)
+	}
+
+	// Generate the masquerade mark to use for SNAT rules.
+	if masqueradeBit < 0 || masqueradeBit > 31 {
+		return nil, fmt.Errorf("invalid iptables-masquerade-bit %v not in [0, 31]", masqueradeBit)
+	}
+	masqueradeValue := 1 << uint(masqueradeBit)
+	masqueradeMark := fmt.Sprintf("%#08x/%#08x", masqueradeValue, masqueradeValue)
+
+	if nodeIP == nil {
+		glog.Warningf("invalid nodeIP, initializing kube-proxy with 127.0.0.1 as nodeIP")
+		nodeIP = net.ParseIP("127.0.0.1")
+	}
+
+	if len(clusterCIDR) == 0 {
+		glog.Warningf("clusterCIDR not specified, unable to distinguish between internal and external traffic")
+	}
+
+	if len(scheduler) == 0 {
+		glog.Warningf("IPVS scheduler not specified, use %s by default", DefaultScheduler)
+		scheduler = DefaultScheduler
+	}
+
+	healthChecker := healthcheck.NewServer(hostname, recorder, nil, nil) // use default implementations of deps
+
+	var throttle flowcontrol.RateLimiter
+	// Defaulting back to not limit sync rate when minSyncPeriod is 0.
+	if minSyncPeriod != 0 {
+		syncsPerSecond := float32(time.Second) / float32(minSyncPeriod)
+		// The average use case will process 2 updates in short succession
+		throttle = flowcontrol.NewTokenBucketRateLimiter(syncsPerSecond, 2)
+	}
+
+	return &Proxier{
+		portsMap:         make(map[utilproxy.LocalPort]utilproxy.Closeable),
+		serviceMap:       make(proxyServiceMap),
+		serviceChanges:   newServiceChangeMap(),
+		endpointsMap:     make(proxyEndpointsMap),
+		endpointsChanges: newEndpointsChangeMap(),
+		syncPeriod:       syncPeriod,
+		minSyncPeriod:    minSyncPeriod,
+		throttle:         throttle,
+		iptables:         ipt,
+		masqueradeAll:    masqueradeAll,
+		masqueradeMark:   masqueradeMark,
+		exec:             exec,
+		clusterCIDR:      clusterCIDR,
+		hostname:         hostname,
+		nodeIP:           nodeIP,
+		portMapper:       &listenPortOpener{},
+		recorder:         recorder,
+		healthChecker:    healthChecker,
+		healthzServer:    healthzServer,
+		ipvs:             ipvs,
+		ipvsScheduler:    scheduler,
+		ipGetter:         &realIPGetter{},
+	}, nil
+}
+
+type proxyServiceMap map[proxy.ServicePortName]*serviceInfo
+
+// internal struct for string service information
+type serviceInfo struct {
+	clusterIP                net.IP
+	port                     int
+	protocol                 api.Protocol
+	nodePort                 int
+	loadBalancerStatus       api.LoadBalancerStatus
+	sessionAffinityType      api.ServiceAffinity
+	stickyMaxAgeSeconds      int
+	externalIPs              []string
+	loadBalancerSourceRanges []string
+	onlyNodeLocalEndpoints   bool
+	healthCheckNodePort      int
+}
+
+// <serviceMap> is updated by this function (based on the given changes).
+// <changes> map is cleared after applying them.
+func updateServiceMap(
+	serviceMap proxyServiceMap,
+	changes *serviceChangeMap) (syncRequired bool, hcServices map[types.NamespacedName]uint16, staleServices sets.String) {
+	syncRequired = false
+	staleServices = sets.NewString()
+
+	for _, change := range changes.items {
+		mergeSyncRequired, existingPorts := serviceMap.mergeService(change.current)
+		unmergeSyncRequired := serviceMap.unmergeService(change.previous, existingPorts, staleServices)
+		syncRequired = syncRequired || mergeSyncRequired || unmergeSyncRequired
+	}
+	changes.items = make(map[types.NamespacedName]*serviceChange)
+
+	// TODO: If this will appear to be computationally expensive, consider
+	// computing this incrementally similarly to serviceMap.
+	hcServices = make(map[types.NamespacedName]uint16)
+	for svcPort, info := range serviceMap {
+		if info.healthCheckNodePort != 0 {
+			hcServices[svcPort.NamespacedName] = uint16(info.healthCheckNodePort)
+		}
+	}
+
+	return syncRequired, hcServices, staleServices
+}
+
+// returns a new serviceInfo struct
+func newServiceInfo(serviceName proxy.ServicePortName, port *api.ServicePort, service *api.Service) *serviceInfo {
+	onlyNodeLocalEndpoints := false
+	if utilfeature.DefaultFeatureGate.Enabled(features.ExternalTrafficLocalOnly) &&
+		apiservice.RequestsOnlyLocalTraffic(service) {
+		onlyNodeLocalEndpoints = true
+	}
+	var stickyMaxAgeSeconds int
+	if service.Spec.SessionAffinity == api.ServiceAffinityClientIP {
+		stickyMaxAgeSeconds = int(*service.Spec.SessionAffinityConfig.ClientIP.TimeoutSeconds)
+	}
+	info := &serviceInfo{
+		clusterIP: net.ParseIP(service.Spec.ClusterIP),
+		port:      int(port.Port),
+		protocol:  port.Protocol,
+		nodePort:  int(port.NodePort),
+		// Deep-copy in case the service instance changes
+		loadBalancerStatus:       *helper.LoadBalancerStatusDeepCopy(&service.Status.LoadBalancer),
+		sessionAffinityType:      service.Spec.SessionAffinity,
+		stickyMaxAgeSeconds:      stickyMaxAgeSeconds,
+		externalIPs:              make([]string, len(service.Spec.ExternalIPs)),
+		loadBalancerSourceRanges: make([]string, len(service.Spec.LoadBalancerSourceRanges)),
+		onlyNodeLocalEndpoints:   onlyNodeLocalEndpoints,
+	}
+	copy(info.loadBalancerSourceRanges, service.Spec.LoadBalancerSourceRanges)
+	copy(info.externalIPs, service.Spec.ExternalIPs)
+
+	if apiservice.NeedsHealthCheck(service) {
+		p := service.Spec.HealthCheckNodePort
+		if p == 0 {
+			glog.Errorf("Service %q has no healthcheck nodeport", serviceName)
+		} else {
+			info.healthCheckNodePort = int(p)
+		}
+	}
+
+	return info
+}
+
+func (sm *proxyServiceMap) mergeService(service *api.Service) (bool, sets.String) {
+	if service == nil {
+		return false, nil
+	}
+	svcName := types.NamespacedName{Namespace: service.Namespace, Name: service.Name}
+	if utilproxy.ShouldSkipService(svcName, service) {
+		return false, nil
+	}
+	syncRequired := false
+	existingPorts := sets.NewString()
+	for i := range service.Spec.Ports {
+		servicePort := &service.Spec.Ports[i]
+		serviceName := proxy.ServicePortName{NamespacedName: svcName, Port: servicePort.Name}
+		existingPorts.Insert(servicePort.Name)
+		info := newServiceInfo(serviceName, servicePort, service)
+		oldInfo, exists := (*sm)[serviceName]
+		equal := reflect.DeepEqual(info, oldInfo)
+		if exists {
+			glog.V(1).Infof("Adding new service %q at %s:%d/%s", serviceName, info.clusterIP, servicePort.Port, servicePort.Protocol)
+		} else if !equal {
+			glog.V(1).Infof("Updating existing service %q at %s:%d/%s", serviceName, info.clusterIP, servicePort.Port, servicePort.Protocol)
+		}
+		if !equal {
+			(*sm)[serviceName] = info
+			syncRequired = true
+		}
+	}
+	return syncRequired, existingPorts
+}
+
+// <staleServices> are modified by this function with detected stale services.
+func (sm *proxyServiceMap) unmergeService(service *api.Service, existingPorts, staleServices sets.String) bool {
+	if service == nil {
+		return false
+	}
+	svcName := types.NamespacedName{Namespace: service.Namespace, Name: service.Name}
+	if utilproxy.ShouldSkipService(svcName, service) {
+		return false
+	}
+	syncRequired := false
+	for i := range service.Spec.Ports {
+		servicePort := &service.Spec.Ports[i]
+		if existingPorts.Has(servicePort.Name) {
+			continue
+		}
+		serviceName := proxy.ServicePortName{NamespacedName: svcName, Port: servicePort.Name}
+		info, exists := (*sm)[serviceName]
+		if exists {
+			glog.V(1).Infof("Removing service %q", serviceName)
+			if info.protocol == api.ProtocolUDP {
+				staleServices.Insert(info.clusterIP.String())
+			}
+			delete(*sm, serviceName)
+			syncRequired = true
+		} else {
+			glog.Errorf("Service %q removed, but doesn't exists", serviceName)
+		}
+	}
+	return syncRequired
+}
+
+type serviceChangeMap struct {
+	sync.Mutex
+	items map[types.NamespacedName]*serviceChange
+}
+
+type serviceChange struct {
+	previous *api.Service
+	current  *api.Service
+}
+
+func newServiceChangeMap() serviceChangeMap {
+	return serviceChangeMap{
+		items: make(map[types.NamespacedName]*serviceChange),
+	}
+}
+
+func (scm *serviceChangeMap) update(namespacedName *types.NamespacedName, previous, current *api.Service) {
+	scm.Lock()
+	defer scm.Unlock()
+
+	change, exists := scm.items[*namespacedName]
+	if !exists {
+		change = &serviceChange{}
+		change.previous = previous
+		scm.items[*namespacedName] = change
+	}
+	change.current = current
+}
+
+// internal struct for endpoints information
+type endpointsInfo struct {
+	endpoint string // TODO: should be an endpointString type
+	isLocal  bool
+}
+
+func (e *endpointsInfo) String() string {
+	return fmt.Sprintf("%v", *e)
+}
+
+type endpointServicePair struct {
+	endpoint        string
+	servicePortName proxy.ServicePortName
+}
+
+type proxyEndpointsMap map[proxy.ServicePortName][]*endpointsInfo
+
+type endpointsChange struct {
+	previous *api.Endpoints
+	current  *api.Endpoints
+}
+
+type endpointsChangeMap struct {
+	sync.Mutex
+	items map[types.NamespacedName]*endpointsChange
+}
+
+// <staleEndpoints> are modified by this function with detected stale
+// connections.
+func detectStaleConnections(oldEndpointsMap, newEndpointsMap proxyEndpointsMap, staleEndpoints map[endpointServicePair]bool) {
+	for svcPort, epList := range oldEndpointsMap {
+		for _, ep := range epList {
+			stale := true
+			for i := range newEndpointsMap[svcPort] {
+				if *newEndpointsMap[svcPort][i] == *ep {
+					stale = false
+					break
+				}
+			}
+			if stale {
+				glog.V(4).Infof("Stale endpoint %v -> %v", svcPort, ep.endpoint)
+				staleEndpoints[endpointServicePair{endpoint: ep.endpoint, servicePortName: svcPort}] = true
+			}
+		}
+	}
+}
+
+// <endpointsMap> is updated by this function (based on the given changes).
+// <changes> map is cleared after applying them.
+func updateEndpointsMap(
+	endpointsMap proxyEndpointsMap,
+	changes *endpointsChangeMap,
+	hostname string) (syncRequired bool, hcEndpoints map[types.NamespacedName]int, staleSet map[endpointServicePair]bool) {
+	syncRequired = false
+	staleSet = make(map[endpointServicePair]bool)
+	for _, change := range changes.items {
+		oldEndpointsMap := endpointsToEndpointsMap(change.previous, hostname)
+		newEndpointsMap := endpointsToEndpointsMap(change.current, hostname)
+		if !reflect.DeepEqual(oldEndpointsMap, newEndpointsMap) {
+			endpointsMap.unmerge(oldEndpointsMap)
+			endpointsMap.merge(newEndpointsMap)
+			detectStaleConnections(oldEndpointsMap, newEndpointsMap, staleSet)
+			syncRequired = true
+		}
+	}
+	changes.items = make(map[types.NamespacedName]*endpointsChange)
+
+	if !utilfeature.DefaultFeatureGate.Enabled(features.ExternalTrafficLocalOnly) {
+		return
+	}
+
+	// TODO: If this will appear to be computationally expensive, consider
+	// computing this incrementally similarly to endpointsMap.
+	hcEndpoints = make(map[types.NamespacedName]int)
+	localIPs := getLocalIPs(endpointsMap)
+	for nsn, ips := range localIPs {
+		hcEndpoints[nsn] = len(ips)
+	}
+
+	return syncRequired, hcEndpoints, staleSet
+}
+
+// Translates single Endpoints object to proxyEndpointsMap.
+// This function is used for incremental updated of endpointsMap.
+//
+// NOTE: endpoints object should NOT be modified.
+func endpointsToEndpointsMap(endpoints *api.Endpoints, hostname string) proxyEndpointsMap {
+	if endpoints == nil {
+		return nil
+	}
+
+	endpointsMap := make(proxyEndpointsMap)
+	// We need to build a map of portname -> all ip:ports for that
+	// portname.  Explode Endpoints.Subsets[*] into this structure.
+	for i := range endpoints.Subsets {
+		ss := &endpoints.Subsets[i]
+		for i := range ss.Ports {
+			port := &ss.Ports[i]
+			if port.Port == 0 {
+				glog.Warningf("ignoring invalid endpoint port %s", port.Name)
+				continue
+			}
+			svcPort := proxy.ServicePortName{
+				NamespacedName: types.NamespacedName{Namespace: endpoints.Namespace, Name: endpoints.Name},
+				Port:           port.Name,
+			}
+			for i := range ss.Addresses {
+				addr := &ss.Addresses[i]
+				if addr.IP == "" {
+					glog.Warningf("ignoring invalid endpoint port %s with empty host", port.Name)
+					continue
+				}
+				epInfo := &endpointsInfo{
+					endpoint: net.JoinHostPort(addr.IP, strconv.Itoa(int(port.Port))),
+					isLocal:  addr.NodeName != nil && *addr.NodeName == hostname,
+				}
+				endpointsMap[svcPort] = append(endpointsMap[svcPort], epInfo)
+			}
+			if glog.V(3) {
+				newEPList := []string{}
+				for _, ep := range endpointsMap[svcPort] {
+					newEPList = append(newEPList, ep.endpoint)
+				}
+				glog.Infof("Setting endpoints for %q to %+v", svcPort, newEPList)
+			}
+		}
+	}
+	return endpointsMap
+}
+
+func newEndpointsChangeMap() endpointsChangeMap {
+	return endpointsChangeMap{
+		items: make(map[types.NamespacedName]*endpointsChange),
+	}
+}
+
+func (ecm *endpointsChangeMap) Update(namespacedName *types.NamespacedName, previous, current *api.Endpoints) {
+	ecm.Lock()
+	defer ecm.Unlock()
+
+	change, exists := ecm.items[*namespacedName]
+	if !exists {
+		change = &endpointsChange{}
+		change.previous = previous
+		ecm.items[*namespacedName] = change
+	}
+	change.current = current
+}
+
+func (em proxyEndpointsMap) merge(other proxyEndpointsMap) {
+	for svcPort := range other {
+		em[svcPort] = other[svcPort]
+	}
+}
+
+func (em proxyEndpointsMap) unmerge(other proxyEndpointsMap) {
+	for svcPort := range other {
+		delete(em, svcPort)
+	}
+}
+
+// CanUseIPVSProxier returns true if we can use the ipvs Proxier.
+// This is determined by checking if all the required kernel modules are loaded. It may
+// return an error if it fails to get the kernel modules information without error, in which
+// case it will also return false.
+func CanUseIPVSProxier() (bool, error) {
+	// Find out loaded kernel modules
+	out, err := utilexec.New().Command("cut", "-f1", "-d", " ", "/proc/modules").CombinedOutput()
+	if err != nil {
+		return false, err
+	}
+
+	mods := strings.Split(string(out), "\n")
+	wantModules := sets.NewString()
+	loadModules := sets.NewString()
+	wantModules.Insert(ipvsModules...)
+	loadModules.Insert(mods...)
+	modules := wantModules.Difference(loadModules).List()
+	if len(modules) != 0 {
+		return false, fmt.Errorf("Failed to load kernel modules: %v", modules)
+	}
+	return true, nil
+}
+
+// TODO: make it simpler.
+// CleanupIptablesLeftovers removes all iptables rules and chains created by the Proxier
+// It returns true if an error was encountered. Errors are logged.
+func cleanupIptablesLeftovers(ipt utiliptables.Interface) (encounteredError bool) {
+	// Unlink the services chain.
+	args := []string{
+		"-m", "comment", "--comment", "kubernetes service portals",
+		"-j", string(kubeServicesChain),
+	}
+	tableChainsWithJumpServices := []struct {
+		table utiliptables.Table
+		chain utiliptables.Chain
+	}{
+		{utiliptables.TableNAT, utiliptables.ChainOutput},
+		{utiliptables.TableNAT, utiliptables.ChainPrerouting},
+	}
+	for _, tc := range tableChainsWithJumpServices {
+		if err := ipt.DeleteRule(tc.table, tc.chain, args...); err != nil {
+			if !utiliptables.IsNotFoundError(err) {
+				glog.Errorf("Error removing pure-iptables proxy rule: %v", err)
+				encounteredError = true
+			}
+		}
+	}
+
+	// Unlink the postrouting chain.
+	args = []string{
+		"-m", "comment", "--comment", "kubernetes postrouting rules",
+		"-j", string(kubePostroutingChain),
+	}
+	if err := ipt.DeleteRule(utiliptables.TableNAT, utiliptables.ChainPostrouting, args...); err != nil {
+		if !utiliptables.IsNotFoundError(err) {
+			glog.Errorf("Error removing ipvs Proxier iptables rule: %v", err)
+			encounteredError = true
+		}
+	}
+
+	// Flush and remove all of our chains.
+	iptablesData := bytes.NewBuffer(nil)
+	if err := ipt.SaveInto(utiliptables.TableNAT, iptablesData); err != nil {
+		glog.Errorf("Failed to execute iptables-save for %s: %v", utiliptables.TableNAT, err)
+		encounteredError = true
+	} else {
+		existingNATChains := utiliptables.GetChainLines(utiliptables.TableNAT, iptablesData.Bytes())
+		natChains := bytes.NewBuffer(nil)
+		natRules := bytes.NewBuffer(nil)
+		writeLine(natChains, "*nat")
+		// Start with chains we know we need to remove.
+		for _, chain := range []utiliptables.Chain{kubeServicesChain, kubePostroutingChain, KubeMarkMasqChain} {
+			if _, found := existingNATChains[chain]; found {
+				chainString := string(chain)
+				writeLine(natChains, existingNATChains[chain]) // flush
+				writeLine(natRules, "-X", chainString)         // delete
+			}
+		}
+		writeLine(natRules, "COMMIT")
+		natLines := append(natChains.Bytes(), natRules.Bytes()...)
+		// Write it.
+		err = ipt.Restore(utiliptables.TableNAT, natLines, utiliptables.NoFlushTables, utiliptables.RestoreCounters)
+		if err != nil {
+			glog.Errorf("Failed to execute iptables-restore for %s: %v", utiliptables.TableNAT, err)
+			encounteredError = true
+		}
+	}
+	return encounteredError
+}
+
+// CleanupLeftovers clean up all ipvs and iptables rules created by ipvs Proxier.
+func CleanupLeftovers(execer utilexec.Interface, ipvs utilipvs.Interface, ipt utiliptables.Interface) (encounteredError bool) {
+	// Return immediately when ipvs interface is nil - Probably initialization failed in somewhere.
+	if ipvs == nil {
+		return true
+	}
+	encounteredError = false
+	// Currently we assume only ipvs proxier will create ipvs rules, ipvs proxier will flush all ipvs rules when clean up.
+	// Users do this operation should be with caution.
+	err := ipvs.Flush()
+	if err != nil {
+		encounteredError = true
+	}
+	// Delete dummy interface created by ipvs Proxier.
+	err = deleteDummyDevice(execer, DefaultDummyDevice)
+	if err != nil {
+		encounteredError = true
+	}
+	// Clear iptables created by ipvs Proxier.
+	encounteredError = cleanupIptablesLeftovers(ipt) || encounteredError
+	return encounteredError
+}
+
+// Sync is called to immediately synchronize the proxier state to iptables
+func (proxier *Proxier) Sync() {
+	proxier.syncProxyRules(syncReasonForce)
+}
+
+// SyncLoop runs periodic work.  This is expected to run as a goroutine or as the main loop of the app.  It does not return.
+func (proxier *Proxier) SyncLoop() {
+	t := time.NewTicker(proxier.syncPeriod)
+	defer t.Stop()
+	// Update healthz timestamp at beginning in case Sync() never succeeds.
+	if proxier.healthzServer != nil {
+		proxier.healthzServer.UpdateTimestamp()
+	}
+	for {
+		<-t.C
+		glog.V(6).Infof("Periodic sync")
+		proxier.Sync()
+	}
+}
+
+// OnServiceAdd is called whenever creation of new service object is observed.
+func (proxier *Proxier) OnServiceAdd(service *api.Service) {
+	namespacedName := types.NamespacedName{Namespace: service.Namespace, Name: service.Name}
+	proxier.serviceChanges.update(&namespacedName, nil, service)
+
+	proxier.syncProxyRules(syncReasonServices)
+}
+
+// OnServiceUpdate is called whenever modification of an existing service object is observed.
+func (proxier *Proxier) OnServiceUpdate(oldService, service *api.Service) {
+	namespacedName := types.NamespacedName{Namespace: service.Namespace, Name: service.Name}
+	proxier.serviceChanges.update(&namespacedName, oldService, service)
+
+	proxier.syncProxyRules(syncReasonServices)
+}
+
+// OnServiceDelete is called whenever deletion of an existing service object is observed.
+func (proxier *Proxier) OnServiceDelete(service *api.Service) {
+	namespacedName := types.NamespacedName{Namespace: service.Namespace, Name: service.Name}
+	proxier.serviceChanges.update(&namespacedName, service, nil)
+
+	proxier.syncProxyRules(syncReasonServices)
+}
+
+// OnServiceSynced is called once all the initial even handlers were called and the state is fully propagated to local cache.
+func (proxier *Proxier) OnServiceSynced() {
+	proxier.mu.Lock()
+	proxier.servicesSynced = true
+	proxier.mu.Unlock()
+
+	proxier.syncProxyRules(syncReasonServices)
+}
+
+// OnEndpointsAdd is called whenever creation of new endpoints object is observed.
+func (proxier *Proxier) OnEndpointsAdd(endpoints *api.Endpoints) {
+	namespacedName := types.NamespacedName{Namespace: endpoints.Namespace, Name: endpoints.Name}
+	proxier.endpointsChanges.Update(&namespacedName, nil, endpoints)
+
+	proxier.syncProxyRules(syncReasonEndpoints)
+}
+
+// OnEndpointsUpdate is called whenever modification of an existing endpoints object is observed.
+func (proxier *Proxier) OnEndpointsUpdate(oldEndpoints, endpoints *api.Endpoints) {
+	namespacedName := types.NamespacedName{Namespace: endpoints.Namespace, Name: endpoints.Name}
+	proxier.endpointsChanges.Update(&namespacedName, oldEndpoints, endpoints)
+
+	proxier.syncProxyRules(syncReasonEndpoints)
+}
+
+// OnEndpointsDelete is called whenever deletion of an existing endpoints object is observed.
+func (proxier *Proxier) OnEndpointsDelete(endpoints *api.Endpoints) {
+	namespacedName := types.NamespacedName{Namespace: endpoints.Namespace, Name: endpoints.Name}
+	proxier.endpointsChanges.Update(&namespacedName, endpoints, nil)
+
+	proxier.syncProxyRules(syncReasonEndpoints)
+}
+
+// OnEndpointsSynced is called once all the initial event handlers were called and the state is fully propagated to local cache.
+func (proxier *Proxier) OnEndpointsSynced() {
+	proxier.mu.Lock()
+	proxier.endpointsSynced = true
+	proxier.mu.Unlock()
+
+	proxier.syncProxyRules(syncReasonEndpoints)
+}
+
+type syncReason string
+
+const syncReasonServices syncReason = "ServicesUpdate"
+const syncReasonEndpoints syncReason = "EndpointsUpdate"
+const syncReasonForce syncReason = "Force"
+
+// This is where all of the ipvs calls happen.
+// assumes proxier.mu is held
+func (proxier *Proxier) syncProxyRules(reason syncReason) {
+	proxier.mu.Lock()
+	defer proxier.mu.Unlock()
+
+	if proxier.throttle != nil {
+		proxier.throttle.Accept()
+	}
+	start := time.Now()
+	defer func() {
+		glog.V(4).Infof("syncProxyRules(%s) took %v", reason, time.Since(start))
+	}()
+	// don't sync rules till we've received services and endpoints
+	if !proxier.endpointsSynced || !proxier.servicesSynced {
+		glog.V(2).Info("Not syncing ipvs rules until Services and Endpoints have been received from master")
+		return
+	}
+
+	// Figure out the new services we need to activate.
+	proxier.serviceChanges.Lock()
+	serviceSyncRequired, hcServices, staleServices := updateServiceMap(
+		proxier.serviceMap, &proxier.serviceChanges)
+	proxier.serviceChanges.Unlock()
+
+	// If this was called because of a services update, but nothing actionable has changed, skip it.
+	if reason == syncReasonServices && !serviceSyncRequired {
+		glog.V(3).Infof("Skipping ipvs sync because nothing changed")
+		return
+	}
+
+	proxier.endpointsChanges.Lock()
+	endpointsSyncRequired, hcEndpoints, staleEndpoints := updateEndpointsMap(
+		proxier.endpointsMap, &proxier.endpointsChanges, proxier.hostname)
+	proxier.endpointsChanges.Unlock()
+
+	// If this was called because of an endpoints update, but nothing actionable has changed, skip it.
+	if reason == syncReasonEndpoints && !endpointsSyncRequired {
+		glog.V(3).Infof("Skipping ipvs sync because nothing changed")
+		return
+	}
+
+	glog.V(3).Infof("Syncing ipvs Proxier rules")
+
+	// TODO: UT output result
+	// Begin install iptables
+	// Get iptables-save output so we can check for existing chains and rules.
+	// This will be a map of chain name to chain with rules as stored in iptables-save/iptables-restore
+	existingNATChains := make(map[utiliptables.Chain]string)
+	iptablesData := bytes.NewBuffer(nil)
+	err := proxier.iptables.SaveInto(utiliptables.TableNAT, iptablesData)
+	if err != nil { // if we failed to get any rules
+		glog.Errorf("Failed to execute iptables-save, syncing all rules: %v", err)
+	} else { // otherwise parse the output
+		existingNATChains = utiliptables.GetChainLines(utiliptables.TableNAT, iptablesData.Bytes())
+	}
+	natChains := bytes.NewBuffer(nil)
+	natRules := bytes.NewBuffer(nil)
+	// Write table headers.
+	writeLine(natChains, "*nat")
+	// Make sure we keep stats for the top-level chains, if they existed
+	// (which most should have because we created them above).
+	if chain, ok := existingNATChains[kubePostroutingChain]; ok {
+		writeLine(natChains, chain)
+	} else {
+		writeLine(natChains, utiliptables.MakeChainLine(kubePostroutingChain))
+	}
+	// Install the kubernetes-specific postrouting rules. We use a whole chain for
+	// this so that it is easier to flush and change, for example if the mark
+	// value should ever change.
+	writeLine(natRules, []string{
+		"-A", string(kubePostroutingChain),
+		"-m", "comment", "--comment", `"kubernetes service traffic requiring SNAT"`,
+		"-m", "mark", "--mark", proxier.masqueradeMark,
+		"-j", "MASQUERADE",
+	}...)
+
+	if chain, ok := existingNATChains[KubeMarkMasqChain]; ok {
+		writeLine(natChains, chain)
+	} else {
+		writeLine(natChains, utiliptables.MakeChainLine(KubeMarkMasqChain))
+	}
+	// Install the kubernetes-specific masquerade mark rule. We use a whole chain for
+	// this so that it is easier to flush and change, for example if the mark
+	// value should ever change.
+	writeLine(natRules, []string{
+		"-A", string(KubeMarkMasqChain),
+		"-j", "MARK", "--set-xmark", proxier.masqueradeMark,
+	}...)
+	// End install iptables
+
+	// make sure dummy interface exists in the system where ipvs Proxier will bind service address on it
+	_, err = ensureDummyDevice(proxier.exec, DefaultDummyDevice)
+	if err != nil {
+		glog.Errorf("Failed to create dummy interface: %s, error: %v", DefaultDummyDevice, err)
+		return
+	}
+
+	// Accumulate the set of local ports that we will be holding open once this update is complete
+	replacementPortsMap := map[utilproxy.LocalPort]utilproxy.Closeable{}
+	// activeIPVSServices represents IPVS service successfully created in this round of sync
+	activeIPVSServices := map[string]bool{}
+	// currentIPVSServices represent IPVS services listed from the system
+	currentIPVSServices := make(map[string]*utilipvs.VirtualServer)
+
+	// Build IPVS rules for each service.
+	for svcName, svcInfo := range proxier.serviceMap {
+		protocol := strings.ToLower(string(svcInfo.protocol))
+		// Precompute svcNameString; with many services the many calls
+		// to ServicePortName.String() show up in CPU profiles.
+		svcNameString := svcName.String()
+
+		// Capture the clusterIP.
+		serv := &utilipvs.VirtualServer{
+			Address:   svcInfo.clusterIP,
+			Port:      uint16(svcInfo.port),
+			Protocol:  string(svcInfo.protocol),
+			Scheduler: proxier.ipvsScheduler,
+		}
+		// Set session affinity flag and timeout for IPVS service
+		var flags utilipvs.ServiceFlags
+		if svcInfo.sessionAffinityType == api.ServiceAffinityClientIP {
+			serv.Flags |= utilipvs.FlagPersistent
+			serv.Timeout = uint32(svcInfo.stickyMaxAgeSeconds)
+		}
+		// We need to bind ClusterIP to dummy interface, so set `bindAddr` parameter to `true` in syncService()
+		if err := proxier.syncService(svcNameString, serv, true); err == nil {
+			activeIPVSServices[serv.String()] = true
+			if err := proxier.syncEndpoint(svcName, svcInfo.onlyNodeLocalEndpoints, serv); err != nil {
+				glog.Errorf("Failed to sync endpoint for service: %v, err: %v", serv, err)
+			}
+		} else {
+			glog.Errorf("Failed to sync service: %v, err: %v", serv, err)
+		}
+		// Install masquerade rules if 'masqueradeAll' or 'clusterCIDR' is specified.
+		args := []string{
+			"-A", string(kubeServicesChain),
+			"-m", "comment", "--comment", fmt.Sprintf(`"%s cluster IP"`, svcNameString),
+			"-m", protocol, "-p", protocol,
+			"-d", fmt.Sprintf("%s/32", svcInfo.clusterIP.String()),
+			"--dport", strconv.Itoa(svcInfo.port),
+		}
+		if proxier.masqueradeAll {
+			err = proxier.linkKubeServiceChain(existingNATChains, natChains)
+			if err != nil {
+				glog.Errorf("Failed to link KUBE-SERVICES chain: %v", err)
+			}
+			writeLine(natRules, append(args, "-j", string(KubeMarkMasqChain))...)
+		} else if len(proxier.clusterCIDR) > 0 {
+			// This masquerades off-cluster traffic to a service VIP.  The idea
+			// is that you can establish a static route for your Service range,
+			// routing to any node, and that node will bridge into the Service
+			// for you.  Since that might bounce off-node, we masquerade here.
+			// If/when we support "Local" policy for VIPs, we should update this.
+			err = proxier.linkKubeServiceChain(existingNATChains, natChains)
+			if err != nil {
+				glog.Errorf("Failed to link KUBE-SERVICES chain: %v", err)
+			}
+			writeLine(natRules, append(args, "! -s", proxier.clusterCIDR, "-j", string(KubeMarkMasqChain))...)
+		}
+
+		// Capture externalIPs.
+		for _, externalIP := range svcInfo.externalIPs {
+			if local, err := utilproxy.IsLocalIP(externalIP); err != nil {
+				glog.Errorf("can't determine if IP is local, assuming not: %v", err)
+			} else if local {
+				lp := utilproxy.LocalPort{
+					Description: "externalIP for " + svcNameString,
+					IP:          externalIP,
+					Port:        svcInfo.port,
+					Protocol:    protocol,
+				}
+				if proxier.portsMap[lp] != nil {
+					glog.V(4).Infof("Port %s was open before and is still needed", lp.String())
+					replacementPortsMap[lp] = proxier.portsMap[lp]
+				} else {
+					socket, err := proxier.portMapper.OpenLocalPort(&lp)
+					if err != nil {
+						msg := fmt.Sprintf("can't open %s, skipping this externalIP: %v", lp.String(), err)
+
+						proxier.recorder.Eventf(
+							&clientv1.ObjectReference{
+								Kind:      "Node",
+								Name:      proxier.hostname,
+								UID:       types.UID(proxier.hostname),
+								Namespace: "",
+							}, api.EventTypeWarning, err.Error(), msg)
+						glog.Error(msg)
+						continue
+					}
+					replacementPortsMap[lp] = socket
+				}
+			} // We're holding the port, so it's OK to install IPVS rules.
+
+			serv := &utilipvs.VirtualServer{
+				Address:   net.ParseIP(externalIP),
+				Port:      uint16(svcInfo.port),
+				Protocol:  string(svcInfo.protocol),
+				Flags:     flags,
+				Scheduler: proxier.ipvsScheduler,
+			}
+			if svcInfo.sessionAffinityType == api.ServiceAffinityClientIP {
+				serv.Timeout = uint32(svcInfo.stickyMaxAgeSeconds)
+			}
+			// There is no need to bind externalIP to dummy interface, so set parameter `bindAddr` to `false`.
+			if err := proxier.syncService(svcNameString, serv, false); err == nil {
+				activeIPVSServices[serv.String()] = true
+				if err := proxier.syncEndpoint(svcName, svcInfo.onlyNodeLocalEndpoints, serv); err != nil {
+					glog.Errorf("Failed to sync endpoint for service: %v, err: %v", serv, err)
+				}
+			} else {
+				glog.Errorf("Failed to sync service: %v, err: %v", serv, err)
+			}
+		}
+
+		// Capture load-balancer ingress.
+		for _, ingress := range svcInfo.loadBalancerStatus.Ingress {
+			if ingress.IP != "" {
+				if len(svcInfo.loadBalancerSourceRanges) != 0 {
+					err = proxier.linkKubeServiceChain(existingNATChains, natChains)
+					if err != nil {
+						glog.Errorf("Failed to link KUBE-SERVICES chain: %v", err)
+					}
+					// The service firewall rules are created based on ServiceSpec.loadBalancerSourceRanges field.
+					// This currently works for loadbalancers that preserves source ips.
+					// For loadbalancers which direct traffic to service NodePort, the firewall rules will not apply.
+					args := []string{
+						"-A", string(kubeServicesChain),
+						"-m", "comment", "--comment", fmt.Sprintf(`"%s loadbalancer IP"`, svcNameString),
+						"-m", string(svcInfo.protocol), "-p", string(svcInfo.protocol),
+						"-d", fmt.Sprintf("%s/32", ingress.IP),
+						"--dport", fmt.Sprintf("%d", svcInfo.port),
+					}
+
+					allowFromNode := false
+					for _, src := range svcInfo.loadBalancerSourceRanges {
+						writeLine(natRules, append(args, "-s", src, "-j", "ACCEPT")...)
+						// ignore error because it has been validated
+						_, cidr, _ := net.ParseCIDR(src)
+						if cidr.Contains(proxier.nodeIP) {
+							allowFromNode = true
+						}
+					}
+					// generally, ip route rule was added to intercept request to loadbalancer vip from the
+					// loadbalancer's backend hosts. In this case, request will not hit the loadbalancer but loop back directly.
+					// Need to add the following rule to allow request on host.
+					if allowFromNode {
+						writeLine(natRules, append(args, "-s", fmt.Sprintf("%s/32", ingress.IP), "-j", "ACCEPT")...)
+					}
+
+					// If the packet was able to reach the end of firewall chain, then it did not get DNATed.
+					// It means the packet cannot go through the firewall, then DROP it.
+					writeLine(natRules, append(args, "-j", string(KubeMarkDropChain))...)
+				}
+
+				serv := &utilipvs.VirtualServer{
+					Address:   net.ParseIP(ingress.IP),
+					Port:      uint16(svcInfo.port),
+					Protocol:  string(svcInfo.protocol),
+					Flags:     flags,
+					Scheduler: proxier.ipvsScheduler,
+				}
+				if svcInfo.sessionAffinityType == api.ServiceAffinityClientIP {
+					serv.Timeout = uint32(svcInfo.stickyMaxAgeSeconds)
+				}
+				// There is no need to bind LB ingress.IP to dummy interface, so set parameter `bindAddr` to `false`.
+				if err := proxier.syncService(svcNameString, serv, false); err == nil {
+					activeIPVSServices[serv.String()] = true
+					if err := proxier.syncEndpoint(svcName, svcInfo.onlyNodeLocalEndpoints, serv); err != nil {
+						glog.Errorf("Failed to sync endpoint for service: %v, err: %v", serv, err)
+					}
+				} else {
+					glog.Errorf("Failed to sync service: %v, err: %v", serv, err)
+				}
+			}
+		}
+
+		if svcInfo.nodePort != 0 {
+			lp := utilproxy.LocalPort{
+				Description: "nodePort for " + svcNameString,
+				IP:          "",
+				Port:        svcInfo.nodePort,
+				Protocol:    protocol,
+			}
+			if proxier.portsMap[lp] != nil {
+				glog.V(4).Infof("Port %s was open before and is still needed", lp.String())
+				replacementPortsMap[lp] = proxier.portsMap[lp]
+			} else {
+				socket, err := proxier.portMapper.OpenLocalPort(&lp)
+				if err != nil {
+					glog.Errorf("can't open %s, skipping this nodePort: %v", lp.String(), err)
+					continue
+				}
+				if lp.Protocol == "udp" {
+					utilproxy.ClearUDPConntrackForPort(proxier.exec, lp.Port)
+				}
+				replacementPortsMap[lp] = socket
+			} // We're holding the port, so it's OK to install ipvs rules.
+
+			// Build ipvs kernel routes for each node ip address
+			nodeIPs, err := proxier.ipGetter.NodeIPs()
+			if err != nil {
+				glog.Errorf("Failed to get node IP, err: %v", err)
+			} else {
+				for _, nodeIP := range nodeIPs {
+					serv := &utilipvs.VirtualServer{
+						Address:   nodeIP,
+						Port:      uint16(svcInfo.nodePort),
+						Protocol:  string(svcInfo.protocol),
+						Flags:     flags,
+						Scheduler: proxier.ipvsScheduler,
+					}
+					if svcInfo.sessionAffinityType == api.ServiceAffinityClientIP {
+						serv.Timeout = uint32(svcInfo.stickyMaxAgeSeconds)
+					}
+					// There is no need to bind Node IP to dummy interface, so set parameter `bindAddr` to `false`.
+					if err := proxier.syncService(svcNameString, serv, false); err == nil {
+						activeIPVSServices[serv.String()] = true
+						if err := proxier.syncEndpoint(svcName, svcInfo.onlyNodeLocalEndpoints, serv); err != nil {
+							glog.Errorf("Failed to sync endpoint for service: %v, err: %v", serv, err)
+						}
+					} else {
+						glog.Errorf("Failed to sync service: %v, err: %v", serv, err)
+					}
+				}
+			}
+		}
+	}
+
+	// Write the end-of-table markers.
+	writeLine(natRules, "COMMIT")
+
+	// Sync iptables rules.
+	// NOTE: NoFlushTables is used so we don't flush non-kubernetes chains in the table.
+	natLines := append(natChains.Bytes(), natRules.Bytes()...)
+	lines := natLines
+
+	glog.V(3).Infof("Restoring iptables rules: %s", lines)
+	err = proxier.iptables.RestoreAll(lines, utiliptables.NoFlushTables, utiliptables.RestoreCounters)
+	if err != nil {
+		glog.Errorf("Failed to execute iptables-restore: %v\nRules:\n%s", err, lines)
+		// Revert new local ports.
+		utilproxy.RevertPorts(replacementPortsMap, proxier.portsMap)
+		return
+	}
+
+	// Close old local ports and save new ones.
+	for k, v := range proxier.portsMap {
+		if replacementPortsMap[k] == nil {
+			v.Close()
+		}
+	}
+	proxier.portsMap = replacementPortsMap
+
+	// Clean up legacy IPVS services
+	appliedSvcs, err := proxier.ipvs.GetVirtualServers()
+	if err == nil {
+		for _, appliedSvc := range appliedSvcs {
+			currentIPVSServices[appliedSvc.String()] = appliedSvc
+		}
+	} else {
+		glog.Errorf("Failed to get ipvs service, err: %v", err)
+	}
+	proxier.cleanLegacyService(activeIPVSServices, currentIPVSServices)
+
+	// Update healthz timestamp if it is periodic sync.
+	if proxier.healthzServer != nil && reason == syncReasonForce {
+		proxier.healthzServer.UpdateTimestamp()
+	}
+
+	// Update healthchecks.  The endpoints list might include services that are
+	// not "OnlyLocal", but the services list will not, and the healthChecker
+	// will just drop those endpoints.
+	if err := proxier.healthChecker.SyncServices(hcServices); err != nil {
+		glog.Errorf("Error syncing healtcheck services: %v", err)
+	}
+	if err := proxier.healthChecker.SyncEndpoints(hcEndpoints); err != nil {
+		glog.Errorf("Error syncing healthcheck endpoints: %v", err)
+	}
+
+	// Finish housekeeping.
+	// TODO: these could be made more consistent.
+	for _, svcIP := range staleServices.List() {
+		if err := utilproxy.ClearUDPConntrackForIP(proxier.exec, svcIP); err != nil {
+			glog.Errorf("Failed to delete stale service IP %s connections, error: %v", svcIP, err)
+		}
+	}
+	proxier.deleteEndpointConnections(staleEndpoints)
+}
+
+// After a UDP endpoint has been removed, we must flush any pending conntrack entries to it, or else we
+// risk sending more traffic to it, all of which will be lost (because UDP).
+// This assumes the proxier mutex is held
+func (proxier *Proxier) deleteEndpointConnections(connectionMap map[endpointServicePair]bool) {
+	for epSvcPair := range connectionMap {
+		if svcInfo, ok := proxier.serviceMap[epSvcPair.servicePortName]; ok && svcInfo.protocol == api.ProtocolUDP {
+			endpointIP := epSvcPair.endpoint[0:strings.Index(epSvcPair.endpoint, ":")]
+			err := utilproxy.ClearUDPConntrackForPeers(proxier.exec, svcInfo.clusterIP.String(), endpointIP)
+			if err != nil {
+				glog.Errorf("Failed to delete %s endpoint connections, error: %v", epSvcPair.servicePortName.String(), err)
+			}
+		}
+	}
+}
+
+func (proxier *Proxier) syncService(svcName string, vs *utilipvs.VirtualServer, bindAddr bool) error {
+	appliedVirtualServer, _ := proxier.ipvs.GetVirtualServer(vs)
+	if appliedVirtualServer == nil || !appliedVirtualServer.Equal(vs) {
+		if appliedVirtualServer == nil {
+			// IPVS service is not found, create a new service
+			glog.V(3).Infof("Adding new service %q %s:%d/%s", svcName, vs.Address, vs.Port, vs.Protocol)
+			if err := proxier.ipvs.AddVirtualServer(vs); err != nil {
+				glog.Errorf("Failed to add IPVS service %q: %v", svcName, err)
+				return err
+			}
+		} else {
+			// IPVS service was changed, update the existing one
+			// During updates, service VIP will not go down
+			glog.V(3).Infof("IPVS service %s was changed", svcName)
+			if err := proxier.ipvs.UpdateVirtualServer(appliedVirtualServer); err != nil {
+				glog.Errorf("Failed to update IPVS service, err:%v", err)
+				return err
+			}
+		}
+	}
+
+	// bind service address to dummy interface even if service not changed,
+	// in case that service IP was removed by other processes
+	if bindAddr {
+		_, err := proxier.ipvs.EnsureVirtualServerAddressBind(vs, DefaultDummyDevice)
+		if err != nil {
+			glog.Errorf("Failed to bind service address to dummy device %q: %v", svcName, err)
+			return err
+		}
+	}
+	return nil
+}
+
+func (proxier *Proxier) syncEndpoint(svcPortName proxy.ServicePortName, onlyNodeLocalEndpoints bool, vs *utilipvs.VirtualServer) error {
+	appliedVirtualServer, err := proxier.ipvs.GetVirtualServer(vs)
+	if err != nil || appliedVirtualServer == nil {
+		glog.Errorf("Failed to get IPVS service, error: %v", err)
+		return err
+	}
+
+	// curEndpoints represents IPVS destiantions listed from current system.
+	curEndpoints := sets.NewString()
+	// newEndpoints represents Endpoints watched from API Server.
+	newEndpoints := sets.NewString()
+
+	curDests, err := proxier.ipvs.GetRealServers(appliedVirtualServer)
+	if err != nil {
+		glog.Errorf("Failed to list IPVS destinations, error: %v", err)
+		return err
+	}
+	for _, des := range curDests {
+		curEndpoints.Insert(des.String())
+	}
+
+	for _, eps := range proxier.endpointsMap[svcPortName] {
+		if !onlyNodeLocalEndpoints || onlyNodeLocalEndpoints && eps.isLocal {
+			newEndpoints.Insert(eps.endpoint)
+		}
+	}
+
+	if !curEndpoints.Equal(newEndpoints) {
+		// Create new endpoints
+		for _, ep := range newEndpoints.Difference(curEndpoints).List() {
+			ip, port, err := net.SplitHostPort(ep)
+			if err != nil {
+				glog.Errorf("Failed to parse endpoint: %v, error: %v", ep, err)
+				continue
+			}
+			portNum, err := strconv.Atoi(port)
+			if err != nil {
+				glog.Errorf("Failed to parse endpoint port %s, error: %v", port, err)
+				continue
+			}
+
+			newDest := &utilipvs.RealServer{
+				Address: net.ParseIP(ip),
+				Port:    uint16(portNum),
+				Weight:  1,
+			}
+			err = proxier.ipvs.AddRealServer(appliedVirtualServer, newDest)
+			if err != nil {
+				glog.Errorf("Failed to add destination: %v, error: %v", newDest, err)
+				continue
+			}
+		}
+		// Delete old endpoints
+		for _, ep := range curEndpoints.Difference(newEndpoints).List() {
+			ip, port, err := net.SplitHostPort(ep)
+			if err != nil {
+				glog.Errorf("Failed to parse endpoint: %v, error: %v", ep, err)
+				continue
+			}
+			portNum, err := strconv.Atoi(port)
+			if err != nil {
+				glog.Errorf("Failed to parse endpoint port %s, error: %v", port, err)
+				continue
+			}
+
+			delDest := &utilipvs.RealServer{
+				Address: net.ParseIP(ip),
+				Port:    uint16(portNum),
+			}
+			err = proxier.ipvs.DeleteRealServer(appliedVirtualServer, delDest)
+			if err != nil {
+				glog.Errorf("Failed to delete destination: %v, error: %v", delDest, err)
+				continue
+			}
+		}
+	}
+	return nil
+}
+
+func (proxier *Proxier) cleanLegacyService(atciveServices map[string]bool, currentServices map[string]*utilipvs.VirtualServer) {
+	for cS := range currentServices {
+		if !atciveServices[cS] {
+			svc := currentServices[cS]
+			err := proxier.ipvs.DeleteVirtualServer(svc)
+			if err != nil {
+				glog.Errorf("Failed to delete service, error: %v", err)
+			}
+			err = proxier.ipvs.UnbindVirtualServerAddress(svc, DefaultDummyDevice)
+			if err != nil {
+				glog.Errorf("Failed to unbind service from dummy interface, error: %v", err)
+			}
+		}
+	}
+}
+
+// linkKubeServiceChain will Create chain KUBE-SERVICES and link the chin in PREROUTING and OUTPUT
+// If not specify masqueradeAll or clusterCIDR or LB source range, won't create them.
+
+// Chain PREROUTING (policy ACCEPT)
+// target            prot opt source               destination
+// KUBE-SERVICES     all  --  0.0.0.0/0            0.0.0.0/0
+
+// Chain OUTPUT (policy ACCEPT)
+// target            prot opt source               destination
+// KUBE-SERVICES     all  --  0.0.0.0/0            0.0.0.0/0
+
+// Chain KUBE-SERVICES (2 references)
+func (proxier *Proxier) linkKubeServiceChain(existingNATChains map[utiliptables.Chain]string, natChains *bytes.Buffer) error {
+	if _, err := proxier.iptables.EnsureChain(utiliptables.TableNAT, kubeServicesChain); err != nil {
+		return fmt.Errorf("Failed to ensure that %s chain %s exists: %v", utiliptables.TableNAT, kubeServicesChain, err)
+	}
+	tableChainsNeedJumpServices := []struct {
+		table utiliptables.Table
+		chain utiliptables.Chain
+	}{
+		{utiliptables.TableNAT, utiliptables.ChainOutput},
+		{utiliptables.TableNAT, utiliptables.ChainPrerouting},
+	}
+	comment := "kubernetes service portals"
+	args := []string{"-m", "comment", "--comment", comment, "-j", string(kubeServicesChain)}
+	for _, tc := range tableChainsNeedJumpServices {
+		if _, err := proxier.iptables.EnsureRule(utiliptables.Prepend, tc.table, tc.chain, args...); err != nil {
+			return fmt.Errorf("Failed to ensure that %s chain %s jumps to %s: %v", tc.table, tc.chain, kubeServicesChain, err)
+		}
+	}
+
+	// equal to `iptables -t nat -N KUBE-SERVICES`
+	// write `:KUBE-SERVICES - [0:0]` in nat table
+	if chain, ok := existingNATChains[kubeServicesChain]; ok {
+		writeLine(natChains, chain)
+	} else {
+		writeLine(natChains, utiliptables.MakeChainLine(kubeServicesChain))
+	}
+	return nil
+}
+
+// Join all words with spaces, terminate with newline and write to buff.
+func writeLine(buf *bytes.Buffer, words ...string) {
+	buf.WriteString(strings.Join(words, " ") + "\n")
+}
+
+func getLocalIPs(endpointsMap proxyEndpointsMap) map[types.NamespacedName]sets.String {
+	localIPs := make(map[types.NamespacedName]sets.String)
+	for svcPort := range endpointsMap {
+		for _, ep := range endpointsMap[svcPort] {
+			if ep.isLocal {
+				nsn := svcPort.NamespacedName
+				if localIPs[nsn] == nil {
+					localIPs[nsn] = sets.NewString()
+				}
+				ip := strings.Split(ep.endpoint, ":")[0] // just the IP part
+				localIPs[nsn].Insert(ip)
+			}
+		}
+	}
+	return localIPs
+}
+
+// listenPortOpener opens ports by calling bind() and listen().
+type listenPortOpener struct{}
+
+// OpenLocalPort holds the given local port open.
+func (l *listenPortOpener) OpenLocalPort(lp *utilproxy.LocalPort) (utilproxy.Closeable, error) {
+	return openLocalPort(lp)
+}
+
+func openLocalPort(lp *utilproxy.LocalPort) (utilproxy.Closeable, error) {
+	// For ports on node IPs, open the actual port and hold it, even though we
+	// use iptables to redirect traffic.
+	// This ensures a) that it's safe to use that port and b) that (a) stays
+	// true.  The risk is that some process on the node (e.g. sshd or kubelet)
+	// is using a port and we give that same port out to a Service.  That would
+	// be bad because iptables would silently claim the traffic but the process
+	// would never know.
+	// NOTE: We should not need to have a real listen()ing socket - bind()
+	// should be enough, but I can't figure out a way to e2e test without
+	// it.  Tools like 'ss' and 'netstat' do not show sockets that are
+	// bind()ed but not listen()ed, and at least the default debian netcat
+	// has no way to avoid about 10 seconds of retries.
+	var socket utilproxy.Closeable
+	switch lp.Protocol {
+	case "tcp":
+		listener, err := net.Listen("tcp", net.JoinHostPort(lp.IP, strconv.Itoa(lp.Port)))
+		if err != nil {
+			return nil, err
+		}
+		socket = listener
+	case "udp":
+		addr, err := net.ResolveUDPAddr("udp", net.JoinHostPort(lp.IP, strconv.Itoa(lp.Port)))
+		if err != nil {
+			return nil, err
+		}
+		conn, err := net.ListenUDP("udp", addr)
+		if err != nil {
+			return nil, err
+		}
+		socket = conn
+	default:
+		return nil, fmt.Errorf("unknown protocol %q", lp.Protocol)
+	}
+	glog.V(2).Infof("Opened local port %s", lp.String())
+	return socket, nil
+}
+
+const cmdIP = "ip"
+
+func ensureDummyDevice(execer utilexec.Interface, dummyDev string) (exist bool, err error) {
+	args := []string{"link", "add", dummyDev, "type", "dummy"}
+	out, err := execer.Command(cmdIP, args...).CombinedOutput()
+	if err != nil {
+		// "exit status code 2" will be returned if the device already exists
+		if ee, ok := err.(utilexec.ExitError); ok {
+			if ee.Exited() && ee.ExitStatus() == 2 {
+				return true, nil
+			}
+		}
+		return false, fmt.Errorf("error creating dummy interface %q: %v: %s", dummyDev, err, out)
+	}
+	return false, nil
+}
+
+func deleteDummyDevice(execer utilexec.Interface, dummyDev string) error {
+	args := []string{"link", "del", dummyDev}
+	out, err := execer.Command(cmdIP, args...).CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("error deleting dummy interface %q: %v: %s", dummyDev, err, out)
+	}
+	return nil
+}
+
+// ipvs Proxier fall back on iptables when it needs to do SNAT for engress packets
+// It will only operate iptables *nat table.
+// Create and link the kube postrouting chain for SNAT packets.
+// Chain POSTROUTING (policy ACCEPT)
+// target     prot opt source               destination
+// KUBE-POSTROUTING  all  --  0.0.0.0/0            0.0.0.0/0            /* kubernetes postrouting rules *
+// Maintain by kubelet network sync loop
+
+// *nat
+// :KUBE-POSTROUTING - [0:0]
+// Chain KUBE-POSTROUTING (1 references)
+// target     prot opt source               destination
+// MASQUERADE  all  --  0.0.0.0/0            0.0.0.0/0            /* kubernetes service traffic requiring SNAT */ mark match 0x4000/0x4000
+
+// :KUBE-MARK-MASQ - [0:0]
+// Chain KUBE-MARK-MASQ (0 references)
+// target     prot opt source               destination
+// MARK       all  --  0.0.0.0/0            0.0.0.0/0            MARK or 0x4000

pkg/proxy/ipvs/proxier_test.go

+// +build linux
+
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package ipvs
+
+import (
+	"net"
+	"reflect"
+	"testing"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/proxy"
+	"k8s.io/utils/exec"
+	fakeexec "k8s.io/utils/exec/testing"
+
+	"k8s.io/apimachinery/pkg/util/sets"
+	proxyutil "k8s.io/kubernetes/pkg/proxy/util"
+	utiliptables "k8s.io/kubernetes/pkg/util/iptables"
+	iptablestest "k8s.io/kubernetes/pkg/util/iptables/testing"
+	utilipvs "k8s.io/kubernetes/pkg/util/ipvs"
+	ipvstest "k8s.io/kubernetes/pkg/util/ipvs/testing"
+
+	"github.com/davecgh/go-spew/spew"
+)
+
+const testHostname = "test-hostname"
+
+type fakeIPGetter struct {
+	nodeIPs []net.IP
+}
+
+func (f *fakeIPGetter) NodeIPs() ([]net.IP, error) {
+	return f.nodeIPs, nil
+}
+
+type fakeHealthChecker struct {
+	services  map[types.NamespacedName]uint16
+	Endpoints map[types.NamespacedName]int
+}
+
+func newFakeHealthChecker() *fakeHealthChecker {
+	return &fakeHealthChecker{
+		services:  map[types.NamespacedName]uint16{},
+		Endpoints: map[types.NamespacedName]int{},
+	}
+}
+
+// fakePortOpener implements portOpener.
+type fakePortOpener struct {
+	openPorts []*proxyutil.LocalPort
+}
+
+// OpenLocalPort fakes out the listen() and bind() used by syncProxyRules
+// to lock a local port.
+func (f *fakePortOpener) OpenLocalPort(lp *proxyutil.LocalPort) (proxyutil.Closeable, error) {
+	f.openPorts = append(f.openPorts, lp)
+	return nil, nil
+}
+
+func (fake *fakeHealthChecker) SyncServices(newServices map[types.NamespacedName]uint16) error {
+	fake.services = newServices
+	return nil
+}
+
+func (fake *fakeHealthChecker) SyncEndpoints(newEndpoints map[types.NamespacedName]int) error {
+	fake.Endpoints = newEndpoints
+	return nil
+}
+
+func NewFakeProxier(ipt utiliptables.Interface, ipvs utilipvs.Interface, nodeIPs []net.IP) *Proxier {
+	fcmd := fakeexec.FakeCmd{
+		CombinedOutputScript: []fakeexec.FakeCombinedOutputAction{
+			func() ([]byte, error) { return []byte("dummy device have been created"), nil },
+		},
+	}
+	fexec := &fakeexec.FakeExec{
+		CommandScript: []fakeexec.FakeCommandAction{
+			func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
+		},
+		LookPathFunc: func(cmd string) (string, error) { return cmd, nil },
+	}
+	return &Proxier{
+		exec:             fexec,
+		serviceMap:       make(proxyServiceMap),
+		serviceChanges:   newServiceChangeMap(),
+		endpointsMap:     make(proxyEndpointsMap),
+		endpointsChanges: newEndpointsChangeMap(),
+		iptables:         ipt,
+		ipvs:             ipvs,
+		clusterCIDR:      "10.0.0.0/24",
+		hostname:         testHostname,
+		portsMap:         make(map[proxyutil.LocalPort]proxyutil.Closeable),
+		portMapper:       &fakePortOpener{[]*proxyutil.LocalPort{}},
+		healthChecker:    newFakeHealthChecker(),
+		ipvsScheduler:    DefaultScheduler,
+		ipGetter:         &fakeIPGetter{nodeIPs: nodeIPs},
+	}
+}
+
+func makeNSN(namespace, name string) types.NamespacedName {
+	return types.NamespacedName{Namespace: namespace, Name: name}
+}
+
+func makeServiceMap(proxier *Proxier, allServices ...*api.Service) {
+	for i := range allServices {
+		proxier.OnServiceAdd(allServices[i])
+	}
+
+	proxier.mu.Lock()
+	defer proxier.mu.Unlock()
+	proxier.servicesSynced = true
+}
+
+func makeTestService(namespace, name string, svcFunc func(*api.Service)) *api.Service {
+	svc := &api.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:        name,
+			Namespace:   namespace,
+			Annotations: map[string]string{},
+		},
+		Spec:   api.ServiceSpec{},
+		Status: api.ServiceStatus{},
+	}
+	svcFunc(svc)
+	return svc
+}
+
+func makeEndpointsMap(proxier *Proxier, allEndpoints ...*api.Endpoints) {
+	for i := range allEndpoints {
+		proxier.OnEndpointsAdd(allEndpoints[i])
+	}
+
+	proxier.mu.Lock()
+	defer proxier.mu.Unlock()
+	proxier.endpointsSynced = true
+}
+
+func makeTestEndpoints(namespace, name string, eptFunc func(*api.Endpoints)) *api.Endpoints {
+	ept := &api.Endpoints{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: namespace,
+		},
+	}
+	eptFunc(ept)
+	return ept
+}
+
+func TestNodePort(t *testing.T) {
+	ipt := iptablestest.NewFake()
+	ipvs := ipvstest.NewFake()
+	nodeIP := net.ParseIP("100.101.102.103")
+	fp := NewFakeProxier(ipt, ipvs, []net.IP{nodeIP})
+	svcIP := "10.20.30.41"
+	svcPort := 80
+	svcNodePort := 3001
+	svcPortName := proxy.ServicePortName{
+		NamespacedName: makeNSN("ns1", "svc1"),
+		Port:           "p80",
+	}
+
+	makeServiceMap(fp,
+		makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *api.Service) {
+			svc.Spec.Type = "NodePort"
+			svc.Spec.ClusterIP = svcIP
+			svc.Spec.Ports = []api.ServicePort{{
+				Name:     svcPortName.Port,
+				Port:     int32(svcPort),
+				Protocol: api.ProtocolTCP,
+				NodePort: int32(svcNodePort),
+			}}
+		}),
+	)
+	epIP := "10.180.0.1"
+	makeEndpointsMap(fp,
+		makeTestEndpoints(svcPortName.Namespace, svcPortName.Name, func(ept *api.Endpoints) {
+			ept.Subsets = []api.EndpointSubset{{
+				Addresses: []api.EndpointAddress{{
+					IP: epIP,
+				}},
+				Ports: []api.EndpointPort{{
+					Name: svcPortName.Port,
+					Port: int32(svcPort),
+				}},
+			}}
+		}),
+	)
+
+	fp.syncProxyRules(syncReasonForce)
+
+	// Check ipvs service and destinations
+	services, err := ipvs.GetVirtualServers()
+	if err != nil {
+		t.Errorf("Failed to get ipvs services, err: %v", err)
+	}
+	if len(services) != 2 {
+		t.Errorf("Expect 2 ipvs services, got %d", len(services))
+	}
+	found := false
+	for _, svc := range services {
+		if svc.Address.Equal(nodeIP) && svc.Port == uint16(svcNodePort) && svc.Protocol == string(api.ProtocolTCP) {
+			found = true
+			destinations, err := ipvs.GetRealServers(svc)
+			if err != nil {
+				t.Errorf("Failed to get ipvs destinations, err: %v", err)
+			}
+			for _, dest := range destinations {
+				if dest.Address.To4().String() != epIP || dest.Port != uint16(svcPort) {
+					t.Errorf("service Endpoint mismatch ipvs service destination")
+				}
+			}
+			break
+		}
+	}
+	if !found {
+		t.Errorf("Expect node port type service, got none")
+	}
+}
+
+func TestNodePortNoEndpoint(t *testing.T) {
+	ipt := iptablestest.NewFake()
+	ipvs := ipvstest.NewFake()
+	nodeIP := net.ParseIP("100.101.102.103")
+	fp := NewFakeProxier(ipt, ipvs, []net.IP{nodeIP})
+	svcIP := "10.20.30.41"
+	svcPort := 80
+	svcNodePort := 3001
+	svcPortName := proxy.ServicePortName{
+		NamespacedName: makeNSN("ns1", "svc1"),
+		Port:           "p80",
+	}
+
+	makeServiceMap(fp,
+		makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *api.Service) {
+			svc.Spec.Type = "NodePort"
+			svc.Spec.ClusterIP = svcIP
+			svc.Spec.Ports = []api.ServicePort{{
+				Name:     svcPortName.Port,
+				Port:     int32(svcPort),
+				Protocol: api.ProtocolTCP,
+				NodePort: int32(svcNodePort),
+			}}
+		}),
+	)
+	makeEndpointsMap(fp)
+
+	fp.syncProxyRules(syncReasonForce)
+
+	// Check ipvs service and destinations
+	services, err := ipvs.GetVirtualServers()
+	if err != nil {
+		t.Errorf("Failed to get ipvs services, err: %v", err)
+	}
+	if len(services) != 2 {
+		t.Errorf("Expect 2 ipvs services, got %d", len(services))
+	}
+	found := false
+	for _, svc := range services {
+		if svc.Address.Equal(nodeIP) && svc.Port == uint16(svcNodePort) && svc.Protocol == string(api.ProtocolTCP) {
+			found = true
+			destinations, _ := ipvs.GetRealServers(svc)
+			if len(destinations) != 0 {
+				t.Errorf("Unexpected %d destinations, expect 0 destinations", len(destinations))
+			}
+			break
+		}
+	}
+	if !found {
+		t.Errorf("Expect node port type service, got none")
+	}
+}
+
+func TestClusterIPNoEndpoint(t *testing.T) {
+	ipt := iptablestest.NewFake()
+	ipvs := ipvstest.NewFake()
+	fp := NewFakeProxier(ipt, ipvs, nil)
+	svcIP := "10.20.30.41"
+	svcPort := 80
+	svcPortName := proxy.ServicePortName{
+		NamespacedName: makeNSN("ns1", "svc1"),
+		Port:           "p80",
+	}
+
+	makeServiceMap(fp,
+		makeTestService(svcPortName.Namespace, svcPortName.Namespace, func(svc *api.Service) {
+			svc.Spec.ClusterIP = svcIP
+			svc.Spec.Ports = []api.ServicePort{{
+				Name:     svcPortName.Port,
+				Port:     int32(svcPort),
+				Protocol: api.ProtocolTCP,
+			}}
+		}),
+	)
+	makeEndpointsMap(fp)
+	fp.syncProxyRules(syncReasonForce)
+
+	// check ipvs service and destinations
+	services, err := ipvs.GetVirtualServers()
+	if err != nil {
+		t.Errorf("Failed to get ipvs services, err: %v", err)
+	}
+	if len(services) != 1 {
+		t.Errorf("Expect 1 ipvs services, got %d", len(services))
+	} else {
+		if services[0].Address.To4().String() != svcIP || services[0].Port != uint16(svcPort) && services[0].Protocol == string(api.ProtocolTCP) {
+			t.Errorf("Unexpected mismatch service")
+		} else {
+			destinations, _ := ipvs.GetRealServers(services[0])
+			if len(destinations) != 0 {
+				t.Errorf("Unexpected %d destinations, expect 0 destinations", len(destinations))
+			}
+		}
+	}
+}
+
+func TestClusterIP(t *testing.T) {
+	ipt := iptablestest.NewFake()
+	ipvs := ipvstest.NewFake()
+	fp := NewFakeProxier(ipt, ipvs, nil)
+	svcIP := "10.20.30.41"
+	svcPort := 80
+	svcPortName := proxy.ServicePortName{
+		NamespacedName: makeNSN("ns1", "svc1"),
+		Port:           "p80",
+	}
+
+	makeServiceMap(fp,
+		makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *api.Service) {
+			svc.Spec.ClusterIP = svcIP
+			svc.Spec.Ports = []api.ServicePort{{
+				Name:     svcPortName.Port,
+				Port:     int32(svcPort),
+				Protocol: api.ProtocolTCP,
+			}}
+		}),
+	)
+
+	epIP := "10.180.0.1"
+	makeEndpointsMap(fp,
+		makeTestEndpoints(svcPortName.Namespace, svcPortName.Name, func(ept *api.Endpoints) {
+			ept.Subsets = []api.EndpointSubset{{
+				Addresses: []api.EndpointAddress{{
+					IP: epIP,
+				}},
+				Ports: []api.EndpointPort{{
+					Name: svcPortName.Port,
+					Port: int32(svcPort),
+				}},
+			}}
+		}),
+	)
+
+	fp.syncProxyRules(syncReasonForce)
+
+	// check ipvs service and destinations
+	services, err := ipvs.GetVirtualServers()
+	if err != nil {
+		t.Errorf("Failed to get ipvs services, err: %v", err)
+	}
+	if len(services) != 1 {
+		t.Errorf("Expect 1 ipvs services, got %d", len(services))
+	} else {
+		if services[0].Address.To4().String() != svcIP || services[0].Port != uint16(svcPort) && services[0].Protocol == string(api.ProtocolTCP) {
+			t.Errorf("Unexpected mismatch service")
+		} else {
+			destinations, _ := ipvs.GetRealServers(services[0])
+			if len(destinations) != 1 {
+				t.Errorf("Unexpected %d destinations, expect 0 destinations", len(destinations))
+			} else if destinations[0].Address.To4().String() != epIP || destinations[0].Port != uint16(svcPort) {
+				t.Errorf("Unexpected mismatch destinations")
+			}
+		}
+	}
+}
+
+func TestExternalIPsNoEndpoint(t *testing.T) {
+	ipt := iptablestest.NewFake()
+	ipvs := ipvstest.NewFake()
+	fp := NewFakeProxier(ipt, ipvs, nil)
+	svcIP := "10.20.30.41"
+	svcPort := 80
+	svcExternalIPs := "50.60.70.81"
+	svcPortName := proxy.ServicePortName{
+		NamespacedName: makeNSN("ns1", "svc1"),
+		Port:           "p80",
+	}
+
+	makeServiceMap(fp,
+		makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *api.Service) {
+			svc.Spec.Type = "ClusterIP"
+			svc.Spec.ClusterIP = svcIP
+			svc.Spec.ExternalIPs = []string{svcExternalIPs}
+			svc.Spec.Ports = []api.ServicePort{{
+				Name:       svcPortName.Port,
+				Port:       int32(svcPort),
+				Protocol:   api.ProtocolTCP,
+				TargetPort: intstr.FromInt(svcPort),
+			}}
+		}),
+	)
+
+	makeEndpointsMap(fp)
+
+	fp.syncProxyRules(syncReasonForce)
+
+	// check ipvs service and destinations
+	services, err := ipvs.GetVirtualServers()
+	if err != nil {
+		t.Errorf("Failed to get ipvs services, err: %v", err)
+	}
+	if len(services) != 2 {
+		t.Errorf("Expect 2 ipvs services, got %d", len(services))
+	}
+	found := false
+	for _, svc := range services {
+		if svc.Address.To4().String() == svcExternalIPs && svc.Port == uint16(svcPort) && svc.Protocol == string(api.ProtocolTCP) {
+			found = true
+			destinations, _ := ipvs.GetRealServers(svc)
+			if len(destinations) != 0 {
+				t.Errorf("Unexpected %d destinations, expect 0 destinations", len(destinations))
+			}
+			break
+		}
+	}
+	if !found {
+		t.Errorf("Expect external ip type service, got none")
+	}
+}
+
+func TestExternalIPs(t *testing.T) {
+	ipt := iptablestest.NewFake()
+	ipvs := ipvstest.NewFake()
+	fp := NewFakeProxier(ipt, ipvs, nil)
+	svcIP := "10.20.30.41"
+	svcPort := 80
+	svcExternalIPs := "50.60.70.81"
+	svcPortName := proxy.ServicePortName{
+		NamespacedName: makeNSN("ns1", "svc1"),
+		Port:           "p80",
+	}
+
+	makeServiceMap(fp,
+		makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *api.Service) {
+			svc.Spec.Type = "ClusterIP"
+			svc.Spec.ClusterIP = svcIP
+			svc.Spec.ExternalIPs = []string{svcExternalIPs}
+			svc.Spec.Ports = []api.ServicePort{{
+				Name:       svcPortName.Port,
+				Port:       int32(svcPort),
+				Protocol:   api.ProtocolTCP,
+				TargetPort: intstr.FromInt(svcPort),
+			}}
+		}),
+	)
+
+	epIP := "10.180.0.1"
+	makeEndpointsMap(fp,
+		makeTestEndpoints(svcPortName.Namespace, svcPortName.Name, func(ept *api.Endpoints) {
+			ept.Subsets = []api.EndpointSubset{{
+				Addresses: []api.EndpointAddress{{
+					IP: epIP,
+				}},
+				Ports: []api.EndpointPort{{
+					Name: svcPortName.Port,
+					Port: int32(svcPort),
+				}},
+			}}
+		}),
+	)
+
+	fp.syncProxyRules(syncReasonForce)
+
+	// check ipvs service and destinations
+	services, err := ipvs.GetVirtualServers()
+	if err != nil {
+		t.Errorf("Failed to get ipvs services, err: %v", err)
+	}
+	if len(services) != 2 {
+		t.Errorf("Expect 2 ipvs services, got %d", len(services))
+	}
+	found := false
+	for _, svc := range services {
+		if svc.Address.To4().String() == svcExternalIPs && svc.Port == uint16(svcPort) && svc.Protocol == string(api.ProtocolTCP) {
+			found = true
+			destinations, _ := ipvs.GetRealServers(svc)
+			for _, dest := range destinations {
+				if dest.Address.To4().String() != epIP || dest.Port != uint16(svcPort) {
+					t.Errorf("service Endpoint mismatch ipvs service destination")
+				}
+			}
+			break
+		}
+	}
+	if !found {
+		t.Errorf("Expect external ip type service, got none")
+	}
+}
+
+func TestLoadBalancer(t *testing.T) {
+	ipt := iptablestest.NewFake()
+	ipvs := ipvstest.NewFake()
+	fp := NewFakeProxier(ipt, ipvs, nil)
+	svcIP := "10.20.30.41"
+	svcPort := 80
+	svcNodePort := 3001
+	svcLBIP := "1.2.3.4"
+	svcPortName := proxy.ServicePortName{
+		NamespacedName: makeNSN("ns1", "svc1"),
+		Port:           "p80",
+	}
+
+	makeServiceMap(fp,
+		makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *api.Service) {
+			svc.Spec.Type = "LoadBalancer"
+			svc.Spec.ClusterIP = svcIP
+			svc.Spec.Ports = []api.ServicePort{{
+				Name:     svcPortName.Port,
+				Port:     int32(svcPort),
+				Protocol: api.ProtocolTCP,
+				NodePort: int32(svcNodePort),
+			}}
+			svc.Status.LoadBalancer.Ingress = []api.LoadBalancerIngress{{
+				IP: svcLBIP,
+			}}
+		}),
+	)
+
+	epIP := "10.180.0.1"
+	makeEndpointsMap(fp,
+		makeTestEndpoints(svcPortName.Namespace, svcPortName.Name, func(ept *api.Endpoints) {
+			ept.Subsets = []api.EndpointSubset{{
+				Addresses: []api.EndpointAddress{{
+					IP: epIP,
+				}},
+				Ports: []api.EndpointPort{{
+					Name: svcPortName.Port,
+					Port: int32(svcPort),
+				}},
+			}}
+		}),
+	)
+
+	fp.syncProxyRules(syncReasonForce)
+}
+
+func strPtr(s string) *string {
+	return &s
+}
+
+func TestOnlyLocalNodePorts(t *testing.T) {
+	ipt := iptablestest.NewFake()
+	ipvs := ipvstest.NewFake()
+	nodeIP := net.ParseIP("100.101.102.103")
+	fp := NewFakeProxier(ipt, ipvs, []net.IP{nodeIP})
+	svcIP := "10.20.30.41"
+	svcPort := 80
+	svcNodePort := 3001
+	svcPortName := proxy.ServicePortName{
+		NamespacedName: makeNSN("ns1", "svc1"),
+		Port:           "p80",
+	}
+
+	makeServiceMap(fp,
+		makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *api.Service) {
+			svc.Spec.Type = "NodePort"
+			svc.Spec.ClusterIP = svcIP
+			svc.Spec.Ports = []api.ServicePort{{
+				Name:     svcPortName.Port,
+				Port:     int32(svcPort),
+				Protocol: api.ProtocolTCP,
+				NodePort: int32(svcNodePort),
+			}}
+			svc.Spec.ExternalTrafficPolicy = api.ServiceExternalTrafficPolicyTypeLocal
+		}),
+	)
+
+	epIP1 := "10.180.0.1"
+	epIP2 := "10.180.2.1"
+	makeEndpointsMap(fp,
+		makeTestEndpoints(svcPortName.Namespace, svcPortName.Name, func(ept *api.Endpoints) {
+			ept.Subsets = []api.EndpointSubset{{
+				Addresses: []api.EndpointAddress{{
+					IP:       epIP1,
+					NodeName: nil,
+				}, {
+					IP:       epIP2,
+					NodeName: strPtr(testHostname),
+				}},
+				Ports: []api.EndpointPort{{
+					Name: svcPortName.Port,
+					Port: int32(svcPort),
+				}},
+			}}
+		}),
+	)
+
+	fp.syncProxyRules(syncReasonForce)
+
+	// Expect 2 services and 1 destination
+	services, err := ipvs.GetVirtualServers()
+	if err != nil {
+		t.Errorf("Failed to get ipvs services, err: %v", err)
+	}
+	if len(services) != 2 {
+		t.Errorf("Expect 2 ipvs services, got %d", len(services))
+	}
+	found := false
+	for _, svc := range services {
+		if svc.Address.Equal(nodeIP) && svc.Port == uint16(svcNodePort) && svc.Protocol == string(api.ProtocolTCP) {
+			found = true
+			destinations, err := ipvs.GetRealServers(svc)
+			if err != nil {
+				t.Errorf("Failed to get ipvs destinations, err: %v", err)
+			}
+			if len(destinations) != 1 {
+				t.Errorf("Expect 1 ipvs destination, got %d", len(destinations))
+			} else {
+				if destinations[0].Address.To4().String() != epIP2 || destinations[0].Port != uint16(svcPort) {
+					t.Errorf("service Endpoint mismatch ipvs service destination")
+				}
+			}
+			break
+		}
+	}
+	if !found {
+		t.Errorf("Expect node port type service, got none")
+	}
+}
+
+// NO help
+func TestOnlyLocalLoadBalancing(t *testing.T) {
+	ipt := iptablestest.NewFake()
+	ipvs := ipvstest.NewFake()
+	fp := NewFakeProxier(ipt, ipvs, nil)
+	svcIP := "10.20.30.41"
+	svcPort := 80
+	svcNodePort := 3001
+	svcLBIP := "1.2.3.4"
+	svcPortName := proxy.ServicePortName{
+		NamespacedName: makeNSN("ns1", "svc1"),
+		Port:           "p80",
+	}
+
+	makeServiceMap(fp,
+		makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *api.Service) {
+			svc.Spec.Type = "LoadBalancer"
+			svc.Spec.ClusterIP = svcIP
+			svc.Spec.Ports = []api.ServicePort{{
+				Name:     svcPortName.Port,
+				Port:     int32(svcPort),
+				Protocol: api.ProtocolTCP,
+				NodePort: int32(svcNodePort),
+			}}
+			svc.Status.LoadBalancer.Ingress = []api.LoadBalancerIngress{{
+				IP: svcLBIP,
+			}}
+			svc.Spec.ExternalTrafficPolicy = api.ServiceExternalTrafficPolicyTypeLocal
+		}),
+	)
+
+	epIP1 := "10.180.0.1"
+	epIP2 := "10.180.2.1"
+	makeEndpointsMap(fp,
+		makeTestEndpoints(svcPortName.Namespace, svcPortName.Name, func(ept *api.Endpoints) {
+			ept.Subsets = []api.EndpointSubset{{
+				Addresses: []api.EndpointAddress{{
+					IP:       epIP1,
+					NodeName: nil,
+				}, {
+					IP:       epIP2,
+					NodeName: strPtr(testHostname),
+				}},
+				Ports: []api.EndpointPort{{
+					Name: svcPortName.Port,
+					Port: int32(svcPort),
+				}},
+			}}
+		}),
+	)
+
+	fp.syncProxyRules(syncReasonForce)
+}
+
+func addTestPort(array []api.ServicePort, name string, protocol api.Protocol, port, nodeport int32, targetPort int) []api.ServicePort {
+	svcPort := api.ServicePort{
+		Name:       name,
+		Protocol:   protocol,
+		Port:       port,
+		NodePort:   nodeport,
+		TargetPort: intstr.FromInt(targetPort),
+	}
+	return append(array, svcPort)
+}
+
+func TestBuildServiceMapAddRemove(t *testing.T) {
+	ipt := iptablestest.NewFake()
+	ipvs := ipvstest.NewFake()
+	fp := NewFakeProxier(ipt, ipvs, nil)
+
+	services := []*api.Service{
+		makeTestService("somewhere-else", "cluster-ip", func(svc *api.Service) {
+			svc.Spec.Type = api.ServiceTypeClusterIP
+			svc.Spec.ClusterIP = "172.16.55.4"
+			svc.Spec.Ports = addTestPort(svc.Spec.Ports, "something", "UDP", 1234, 4321, 0)
+			svc.Spec.Ports = addTestPort(svc.Spec.Ports, "somethingelse", "UDP", 1235, 5321, 0)
+		}),
+		makeTestService("somewhere-else", "node-port", func(svc *api.Service) {
+			svc.Spec.Type = api.ServiceTypeNodePort
+			svc.Spec.ClusterIP = "172.16.55.10"
+			svc.Spec.Ports = addTestPort(svc.Spec.Ports, "blahblah", "UDP", 345, 678, 0)
+			svc.Spec.Ports = addTestPort(svc.Spec.Ports, "moreblahblah", "TCP", 344, 677, 0)
+		}),
+		makeTestService("somewhere", "load-balancer", func(svc *api.Service) {
+			svc.Spec.Type = api.ServiceTypeLoadBalancer
+			svc.Spec.ClusterIP = "172.16.55.11"
+			svc.Spec.LoadBalancerIP = "5.6.7.8"
+			svc.Spec.Ports = addTestPort(svc.Spec.Ports, "foobar", "UDP", 8675, 30061, 7000)
+			svc.Spec.Ports = addTestPort(svc.Spec.Ports, "baz", "UDP", 8676, 30062, 7001)
+			svc.Status.LoadBalancer = api.LoadBalancerStatus{
+				Ingress: []api.LoadBalancerIngress{
+					{IP: "10.1.2.4"},
+				},
+			}
+		}),
+		makeTestService("somewhere", "only-local-load-balancer", func(svc *api.Service) {
+			svc.Spec.Type = api.ServiceTypeLoadBalancer
+			svc.Spec.ClusterIP = "172.16.55.12"
+			svc.Spec.LoadBalancerIP = "5.6.7.8"
+			svc.Spec.Ports = addTestPort(svc.Spec.Ports, "foobar2", "UDP", 8677, 30063, 7002)
+			svc.Spec.Ports = addTestPort(svc.Spec.Ports, "baz", "UDP", 8678, 30064, 7003)
+			svc.Status.LoadBalancer = api.LoadBalancerStatus{
+				Ingress: []api.LoadBalancerIngress{
+					{IP: "10.1.2.3"},
+				},
+			}
+			svc.Spec.ExternalTrafficPolicy = api.ServiceExternalTrafficPolicyTypeLocal
+			svc.Spec.HealthCheckNodePort = 345
+		}),
+	}
+
+	for i := range services {
+		fp.OnServiceAdd(services[i])
+	}
+	_, hcPorts, staleUDPServices := updateServiceMap(fp.serviceMap, &fp.serviceChanges)
+	if len(fp.serviceMap) != 8 {
+		t.Errorf("expected service map length 8, got %v", fp.serviceMap)
+	}
+
+	// The only-local-loadbalancer ones get added
+	if len(hcPorts) != 1 {
+		t.Errorf("expected 1 healthcheck port, got %v", hcPorts)
+	} else {
+		nsn := makeNSN("somewhere", "only-local-load-balancer")
+		if port, found := hcPorts[nsn]; !found || port != 345 {
+			t.Errorf("expected healthcheck port [%q]=345: got %v", nsn, hcPorts)
+		}
+	}
+
+	if len(staleUDPServices) != 0 {
+		// Services only added, so nothing stale yet
+		t.Errorf("expected stale UDP services length 0, got %d", len(staleUDPServices))
+	}
+
+	// Remove some stuff
+	// oneService is a modification of services[0] with removed first port.
+	oneService := makeTestService("somewhere-else", "cluster-ip", func(svc *api.Service) {
+		svc.Spec.Type = api.ServiceTypeClusterIP
+		svc.Spec.ClusterIP = "172.16.55.4"
+		svc.Spec.Ports = addTestPort(svc.Spec.Ports, "somethingelse", "UDP", 1235, 5321, 0)
+	})
+
+	fp.OnServiceUpdate(services[0], oneService)
+	fp.OnServiceDelete(services[1])
+	fp.OnServiceDelete(services[2])
+	fp.OnServiceDelete(services[3])
+
+	_, hcPorts, staleUDPServices = updateServiceMap(fp.serviceMap, &fp.serviceChanges)
+	if len(fp.serviceMap) != 1 {
+		t.Errorf("expected service map length 1, got %v", fp.serviceMap)
+	}
+
+	if len(hcPorts) != 0 {
+		t.Errorf("expected 0 healthcheck ports, got %v", hcPorts)
+	}
+
+	// All services but one were deleted. While you'd expect only the ClusterIPs
+	// from the three deleted services here, we still have the ClusterIP for
+	// the not-deleted service, because one of it's ServicePorts was deleted.
+	expectedStaleUDPServices := []string{"172.16.55.10", "172.16.55.4", "172.16.55.11", "172.16.55.12"}
+	if len(staleUDPServices) != len(expectedStaleUDPServices) {
+		t.Errorf("expected stale UDP services length %d, got %v", len(expectedStaleUDPServices), staleUDPServices.List())
+	}
+	for _, ip := range expectedStaleUDPServices {
+		if !staleUDPServices.Has(ip) {
+			t.Errorf("expected stale UDP service service %s", ip)
+		}
+	}
+}
+
+func TestBuildServiceMapServiceHeadless(t *testing.T) {
+	ipt := iptablestest.NewFake()
+	ipvs := ipvstest.NewFake()
+	fp := NewFakeProxier(ipt, ipvs, nil)
+
+	makeServiceMap(fp,
+		makeTestService("somewhere-else", "headless", func(svc *api.Service) {
+			svc.Spec.Type = api.ServiceTypeClusterIP
+			svc.Spec.ClusterIP = api.ClusterIPNone
+			svc.Spec.Ports = addTestPort(svc.Spec.Ports, "rpc", "UDP", 1234, 0, 0)
+		}),
+	)
+
+	// Headless service should be ignored
+	_, hcPorts, staleUDPServices := updateServiceMap(fp.serviceMap, &fp.serviceChanges)
+	if len(fp.serviceMap) != 0 {
+		t.Errorf("expected service map length 0, got %d", len(fp.serviceMap))
+	}
+
+	// No proxied services, so no healthchecks
+	if len(hcPorts) != 0 {
+		t.Errorf("expected healthcheck ports length 0, got %d", len(hcPorts))
+	}
+
+	if len(staleUDPServices) != 0 {
+		t.Errorf("expected stale UDP services length 0, got %d", len(staleUDPServices))
+	}
+}
+
+func TestBuildServiceMapServiceTypeExternalName(t *testing.T) {
+	ipt := iptablestest.NewFake()
+	ipvs := ipvstest.NewFake()
+	fp := NewFakeProxier(ipt, ipvs, nil)
+
+	makeServiceMap(fp,
+		makeTestService("somewhere-else", "external-name", func(svc *api.Service) {
+			svc.Spec.Type = api.ServiceTypeExternalName
+			svc.Spec.ClusterIP = "172.16.55.4" // Should be ignored
+			svc.Spec.ExternalName = "foo2.bar.com"
+			svc.Spec.Ports = addTestPort(svc.Spec.Ports, "blah", "UDP", 1235, 5321, 0)
+		}),
+	)
+
+	_, hcPorts, staleUDPServices := updateServiceMap(fp.serviceMap, &fp.serviceChanges)
+	if len(fp.serviceMap) != 0 {
+		t.Errorf("expected service map length 0, got %v", fp.serviceMap)
+	}
+	// No proxied services, so no healthchecks
+	if len(hcPorts) != 0 {
+		t.Errorf("expected healthcheck ports length 0, got %v", hcPorts)
+	}
+	if len(staleUDPServices) != 0 {
+		t.Errorf("expected stale UDP services length 0, got %v", staleUDPServices)
+	}
+}
+
+func TestBuildServiceMapServiceUpdate(t *testing.T) {
+	ipt := iptablestest.NewFake()
+	ipvs := ipvstest.NewFake()
+	fp := NewFakeProxier(ipt, ipvs, nil)
+
+	servicev1 := makeTestService("somewhere", "some-service", func(svc *api.Service) {
+		svc.Spec.Type = api.ServiceTypeClusterIP
+		svc.Spec.ClusterIP = "172.16.55.4"
+		svc.Spec.Ports = addTestPort(svc.Spec.Ports, "something", "UDP", 1234, 4321, 0)
+		svc.Spec.Ports = addTestPort(svc.Spec.Ports, "somethingelse", "TCP", 1235, 5321, 0)
+	})
+	servicev2 := makeTestService("somewhere", "some-service", func(svc *api.Service) {
+		svc.Spec.Type = api.ServiceTypeLoadBalancer
+		svc.Spec.ClusterIP = "172.16.55.4"
+		svc.Spec.LoadBalancerIP = "5.6.7.8"
+		svc.Spec.Ports = addTestPort(svc.Spec.Ports, "something", "UDP", 1234, 4321, 7002)
+		svc.Spec.Ports = addTestPort(svc.Spec.Ports, "somethingelse", "TCP", 1235, 5321, 7003)
+		svc.Status.LoadBalancer = api.LoadBalancerStatus{
+			Ingress: []api.LoadBalancerIngress{
+				{IP: "10.1.2.3"},
+			},
+		}
+		svc.Spec.ExternalTrafficPolicy = api.ServiceExternalTrafficPolicyTypeLocal
+		svc.Spec.HealthCheckNodePort = 345
+	})
+
+	fp.OnServiceAdd(servicev1)
+
+	syncRequired, hcPorts, staleUDPServices := updateServiceMap(fp.serviceMap, &fp.serviceChanges)
+	if !syncRequired {
+		t.Errorf("expected sync required, got %t", syncRequired)
+	}
+	if len(fp.serviceMap) != 2 {
+		t.Errorf("expected service map length 2, got %v", fp.serviceMap)
+	}
+	if len(hcPorts) != 0 {
+		t.Errorf("expected healthcheck ports length 0, got %v", hcPorts)
+	}
+	if len(staleUDPServices) != 0 {
+		// Services only added, so nothing stale yet
+		t.Errorf("expected stale UDP services length 0, got %d", len(staleUDPServices))
+	}
+
+	// Change service to load-balancer
+	fp.OnServiceUpdate(servicev1, servicev2)
+	syncRequired, hcPorts, staleUDPServices = updateServiceMap(fp.serviceMap, &fp.serviceChanges)
+	if !syncRequired {
+		t.Errorf("expected sync required, got %t", syncRequired)
+	}
+	if len(fp.serviceMap) != 2 {
+		t.Errorf("expected service map length 2, got %v", fp.serviceMap)
+	}
+	if len(hcPorts) != 1 {
+		t.Errorf("expected healthcheck ports length 1, got %v", hcPorts)
+	}
+	if len(staleUDPServices) != 0 {
+		t.Errorf("expected stale UDP services length 0, got %v", staleUDPServices.List())
+	}
+
+	// No change; make sure the service map stays the same and there are
+	// no health-check changes
+	fp.OnServiceUpdate(servicev2, servicev2)
+	syncRequired, hcPorts, staleUDPServices = updateServiceMap(fp.serviceMap, &fp.serviceChanges)
+	if syncRequired {
+		t.Errorf("not expected sync required, got %t", syncRequired)
+	}
+	if len(fp.serviceMap) != 2 {
+		t.Errorf("expected service map length 2, got %v", fp.serviceMap)
+	}
+	if len(hcPorts) != 1 {
+		t.Errorf("expected healthcheck ports length 1, got %v", hcPorts)
+	}
+	if len(staleUDPServices) != 0 {
+		t.Errorf("expected stale UDP services length 0, got %v", staleUDPServices.List())
+	}
+
+	// And back to ClusterIP
+	fp.OnServiceUpdate(servicev2, servicev1)
+	syncRequired, hcPorts, staleUDPServices = updateServiceMap(fp.serviceMap, &fp.serviceChanges)
+	if !syncRequired {
+		t.Errorf("expected sync required, got %t", syncRequired)
+	}
+	if len(fp.serviceMap) != 2 {
+		t.Errorf("expected service map length 2, got %v", fp.serviceMap)
+	}
+	if len(hcPorts) != 0 {
+		t.Errorf("expected healthcheck ports length 0, got %v", hcPorts)
+	}
+	if len(staleUDPServices) != 0 {
+		// Services only added, so nothing stale yet
+		t.Errorf("expected stale UDP services length 0, got %d", len(staleUDPServices))
+	}
+}
+
+func TestSessionAffinity(t *testing.T) {
+	ipt := iptablestest.NewFake()
+	ipvs := ipvstest.NewFake()
+	nodeIP := net.ParseIP("100.101.102.103")
+	fp := NewFakeProxier(ipt, ipvs, []net.IP{nodeIP})
+	svcIP := "10.20.30.41"
+	svcPort := 80
+	svcNodePort := 3001
+	svcExternalIPs := "50.60.70.81"
+	svcPortName := proxy.ServicePortName{
+		NamespacedName: makeNSN("ns1", "svc1"),
+		Port:           "p80",
+	}
+	timeoutSeconds := api.DefaultClientIPServiceAffinitySeconds
+
+	makeServiceMap(fp,
+		makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *api.Service) {
+			svc.Spec.Type = "NodePort"
+			svc.Spec.ClusterIP = svcIP
+			svc.Spec.ExternalIPs = []string{svcExternalIPs}
+			svc.Spec.SessionAffinity = api.ServiceAffinityClientIP
+			svc.Spec.SessionAffinityConfig = &api.SessionAffinityConfig{
+				ClientIP: &api.ClientIPConfig{
+					TimeoutSeconds: &timeoutSeconds,
+				},
+			}
+			svc.Spec.Ports = []api.ServicePort{{
+				Name:     svcPortName.Port,
+				Port:     int32(svcPort),
+				Protocol: api.ProtocolTCP,
+				NodePort: int32(svcNodePort),
+			}}
+		}),
+	)
+	makeEndpointsMap(fp)
+
+	fp.syncProxyRules(syncReasonForce)
+
+	// check ipvs service and destinations
+	services, err := ipvs.GetVirtualServers()
+	if err != nil {
+		t.Errorf("Failed to get ipvs services, err: %v", err)
+	}
+	for _, svc := range services {
+		if svc.Timeout != uint32(api.DefaultClientIPServiceAffinitySeconds) {
+			t.Errorf("Unexpected mismatch ipvs service session affinity timeout: %d, expected: %d", svc.Timeout, api.DefaultClientIPServiceAffinitySeconds)
+		}
+	}
+}
+
+func makeServicePortName(ns, name, port string) proxy.ServicePortName {
+	return proxy.ServicePortName{
+		NamespacedName: makeNSN(ns, name),
+		Port:           port,
+	}
+}
+
+func Test_updateEndpointsMap(t *testing.T) {
+	var nodeName = "host"
+
+	unnamedPort := func(ept *api.Endpoints) {
+		ept.Subsets = []api.EndpointSubset{{
+			Addresses: []api.EndpointAddress{{
+				IP: "1.1.1.1",
+			}},
+			Ports: []api.EndpointPort{{
+				Port: 11,
+			}},
+		}}
+	}
+	unnamedPortLocal := func(ept *api.Endpoints) {
+		ept.Subsets = []api.EndpointSubset{{
+			Addresses: []api.EndpointAddress{{
+				IP:       "1.1.1.1",
+				NodeName: &nodeName,
+			}},
+			Ports: []api.EndpointPort{{
+				Port: 11,
+			}},
+		}}
+	}
+	namedPortLocal := func(ept *api.Endpoints) {
+		ept.Subsets = []api.EndpointSubset{{
+			Addresses: []api.EndpointAddress{{
+				IP:       "1.1.1.1",
+				NodeName: &nodeName,
+			}},
+			Ports: []api.EndpointPort{{
+				Name: "p11",
+				Port: 11,
+			}},
+		}}
+	}
+	namedPort := func(ept *api.Endpoints) {
+		ept.Subsets = []api.EndpointSubset{{
+			Addresses: []api.EndpointAddress{{
+				IP: "1.1.1.1",
+			}},
+			Ports: []api.EndpointPort{{
+				Name: "p11",
+				Port: 11,
+			}},
+		}}
+	}
+	namedPortRenamed := func(ept *api.Endpoints) {
+		ept.Subsets = []api.EndpointSubset{{
+			Addresses: []api.EndpointAddress{{
+				IP: "1.1.1.1",
+			}},
+			Ports: []api.EndpointPort{{
+				Name: "p11-2",
+				Port: 11,
+			}},
+		}}
+	}
+	namedPortRenumbered := func(ept *api.Endpoints) {
+		ept.Subsets = []api.EndpointSubset{{
+			Addresses: []api.EndpointAddress{{
+				IP: "1.1.1.1",
+			}},
+			Ports: []api.EndpointPort{{
+				Name: "p11",
+				Port: 22,
+			}},
+		}}
+	}
+	namedPortsLocalNoLocal := func(ept *api.Endpoints) {
+		ept.Subsets = []api.EndpointSubset{{
+			Addresses: []api.EndpointAddress{{
+				IP: "1.1.1.1",
+			}, {
+				IP:       "1.1.1.2",
+				NodeName: &nodeName,
+			}},
+			Ports: []api.EndpointPort{{
+				Name: "p11",
+				Port: 11,
+			}, {
+				Name: "p12",
+				Port: 12,
+			}},
+		}}
+	}
+	multipleSubsets := func(ept *api.Endpoints) {
+		ept.Subsets = []api.EndpointSubset{{
+			Addresses: []api.EndpointAddress{{
+				IP: "1.1.1.1",
+			}},
+			Ports: []api.EndpointPort{{
+				Name: "p11",
+				Port: 11,
+			}},
+		}, {
+			Addresses: []api.EndpointAddress{{
+				IP: "1.1.1.2",
+			}},
+			Ports: []api.EndpointPort{{
+				Name: "p12",
+				Port: 12,
+			}},
+		}}
+	}
+	multipleSubsetsWithLocal := func(ept *api.Endpoints) {
+		ept.Subsets = []api.EndpointSubset{{
+			Addresses: []api.EndpointAddress{{
+				IP: "1.1.1.1",
+			}},
+			Ports: []api.EndpointPort{{
+				Name: "p11",
+				Port: 11,
+			}},
+		}, {
+			Addresses: []api.EndpointAddress{{
+				IP:       "1.1.1.2",
+				NodeName: &nodeName,
+			}},
+			Ports: []api.EndpointPort{{
+				Name: "p12",
+				Port: 12,
+			}},
+		}}
+	}
+	multipleSubsetsMultiplePortsLocal := func(ept *api.Endpoints) {
+		ept.Subsets = []api.EndpointSubset{{
+			Addresses: []api.EndpointAddress{{
+				IP:       "1.1.1.1",
+				NodeName: &nodeName,
+			}},
+			Ports: []api.EndpointPort{{
+				Name: "p11",
+				Port: 11,
+			}, {
+				Name: "p12",
+				Port: 12,
+			}},
+		}, {
+			Addresses: []api.EndpointAddress{{
+				IP: "1.1.1.3",
+			}},
+			Ports: []api.EndpointPort{{
+				Name: "p13",
+				Port: 13,
+			}},
+		}}
+	}
+	multipleSubsetsIPsPorts1 := func(ept *api.Endpoints) {
+		ept.Subsets = []api.EndpointSubset{{
+			Addresses: []api.EndpointAddress{{
+				IP: "1.1.1.1",
+			}, {
+				IP:       "1.1.1.2",
+				NodeName: &nodeName,
+			}},
+			Ports: []api.EndpointPort{{
+				Name: "p11",
+				Port: 11,
+			}, {
+				Name: "p12",
+				Port: 12,
+			}},
+		}, {
+			Addresses: []api.EndpointAddress{{
+				IP: "1.1.1.3",
+			}, {
+				IP:       "1.1.1.4",
+				NodeName: &nodeName,
+			}},
+			Ports: []api.EndpointPort{{
+				Name: "p13",
+				Port: 13,
+			}, {
+				Name: "p14",
+				Port: 14,
+			}},
+		}}
+	}
+	multipleSubsetsIPsPorts2 := func(ept *api.Endpoints) {
+		ept.Subsets = []api.EndpointSubset{{
+			Addresses: []api.EndpointAddress{{
+				IP: "2.2.2.1",
+			}, {
+				IP:       "2.2.2.2",
+				NodeName: &nodeName,
+			}},
+			Ports: []api.EndpointPort{{
+				Name: "p21",
+				Port: 21,
+			}, {
+				Name: "p22",
+				Port: 22,
+			}},
+		}}
+	}
+	complexBefore1 := func(ept *api.Endpoints) {
+		ept.Subsets = []api.EndpointSubset{{
+			Addresses: []api.EndpointAddress{{
+				IP: "1.1.1.1",
+			}},
+			Ports: []api.EndpointPort{{
+				Name: "p11",
+				Port: 11,
+			}},
+		}}
+	}
+	complexBefore2 := func(ept *api.Endpoints) {
+		ept.Subsets = []api.EndpointSubset{{
+			Addresses: []api.EndpointAddress{{
+				IP:       "2.2.2.2",
+				NodeName: &nodeName,
+			}, {
+				IP:       "2.2.2.22",
+				NodeName: &nodeName,
+			}},
+			Ports: []api.EndpointPort{{
+				Name: "p22",
+				Port: 22,
+			}},
+		}, {
+			Addresses: []api.EndpointAddress{{
+				IP:       "2.2.2.3",
+				NodeName: &nodeName,
+			}},
+			Ports: []api.EndpointPort{{
+				Name: "p23",
+				Port: 23,
+			}},
+		}}
+	}
+	complexBefore4 := func(ept *api.Endpoints) {
+		ept.Subsets = []api.EndpointSubset{{
+			Addresses: []api.EndpointAddress{{
+				IP:       "4.4.4.4",
+				NodeName: &nodeName,
+			}, {
+				IP:       "4.4.4.5",
+				NodeName: &nodeName,
+			}},
+			Ports: []api.EndpointPort{{
+				Name: "p44",
+				Port: 44,
+			}},
+		}, {
+			Addresses: []api.EndpointAddress{{
+				IP:       "4.4.4.6",
+				NodeName: &nodeName,
+			}},
+			Ports: []api.EndpointPort{{
+				Name: "p45",
+				Port: 45,
+			}},
+		}}
+	}
+	complexAfter1 := func(ept *api.Endpoints) {
+		ept.Subsets = []api.EndpointSubset{{
+			Addresses: []api.EndpointAddress{{
+				IP: "1.1.1.1",
+			}, {
+				IP: "1.1.1.11",
+			}},
+			Ports: []api.EndpointPort{{
+				Name: "p11",
+				Port: 11,
+			}},
+		}, {
+			Addresses: []api.EndpointAddress{{
+				IP: "1.1.1.2",
+			}},
+			Ports: []api.EndpointPort{{
+				Name: "p12",
+				Port: 12,
+			}, {
+				Name: "p122",
+				Port: 122,
+			}},
+		}}
+	}
+	complexAfter3 := func(ept *api.Endpoints) {
+		ept.Subsets = []api.EndpointSubset{{
+			Addresses: []api.EndpointAddress{{
+				IP: "3.3.3.3",
+			}},
+			Ports: []api.EndpointPort{{
+				Name: "p33",
+				Port: 33,
+			}},
+		}}
+	}
+	complexAfter4 := func(ept *api.Endpoints) {
+		ept.Subsets = []api.EndpointSubset{{
+			Addresses: []api.EndpointAddress{{
+				IP:       "4.4.4.4",
+				NodeName: &nodeName,
+			}},
+			Ports: []api.EndpointPort{{
+				Name: "p44",
+				Port: 44,
+			}},
+		}}
+	}
+
+	testCases := []struct {
+		// previousEndpoints and currentEndpoints are used to call appropriate
+		// handlers OnEndpoints* (based on whether corresponding values are nil
+		// or non-nil) and must be of equal length.
+		previousEndpoints    []*api.Endpoints
+		currentEndpoints     []*api.Endpoints
+		oldEndpoints         map[proxy.ServicePortName][]*endpointsInfo
+		expectedResult       map[proxy.ServicePortName][]*endpointsInfo
+		expectedStale        []endpointServicePair
+		expectedHealthchecks map[types.NamespacedName]int
+	}{{
+		// Case[0]: nothing
+		oldEndpoints:         map[proxy.ServicePortName][]*endpointsInfo{},
+		expectedResult:       map[proxy.ServicePortName][]*endpointsInfo{},
+		expectedStale:        []endpointServicePair{},
+		expectedHealthchecks: map[types.NamespacedName]int{},
+	}, {
+		// Case[1]: no change, unnamed port
+		previousEndpoints: []*api.Endpoints{
+			makeTestEndpoints("ns1", "ep1", unnamedPort),
+		},
+		currentEndpoints: []*api.Endpoints{
+			makeTestEndpoints("ns1", "ep1", unnamedPort),
+		},
+		oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{
+			makeServicePortName("ns1", "ep1", ""): {
+				{"1.1.1.1:11", false},
+			},
+		},
+		expectedResult: map[proxy.ServicePortName][]*endpointsInfo{
+			makeServicePortName("ns1", "ep1", ""): {
+				{"1.1.1.1:11", false},
+			},
+		},
+		expectedStale:        []endpointServicePair{},
+		expectedHealthchecks: map[types.NamespacedName]int{},
+	}, {
+		// Case[2]: no change, named port, local
+		previousEndpoints: []*api.Endpoints{
+			makeTestEndpoints("ns1", "ep1", namedPortLocal),
+		},
+		currentEndpoints: []*api.Endpoints{
+			makeTestEndpoints("ns1", "ep1", namedPortLocal),
+		},
+		oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{
+			makeServicePortName("ns1", "ep1", "p11"): {
+				{"1.1.1.1:11", true},
+			},
+		},
+		expectedResult: map[proxy.ServicePortName][]*endpointsInfo{
+			makeServicePortName("ns1", "ep1", "p11"): {
+				{"1.1.1.1:11", true},
+			},
+		},
+		expectedStale: []endpointServicePair{},
+		expectedHealthchecks: map[types.NamespacedName]int{
+			makeNSN("ns1", "ep1"): 1,
+		},
+	}, {
+		// Case[3]: no change, multiple subsets
+		previousEndpoints: []*api.Endpoints{
+			makeTestEndpoints("ns1", "ep1", multipleSubsets),
+		},
+		currentEndpoints: []*api.Endpoints{
+			makeTestEndpoints("ns1", "ep1", multipleSubsets),
+		},
+		oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{
+			makeServicePortName("ns1", "ep1", "p11"): {
+				{"1.1.1.1:11", false},
+			},
+			makeServicePortName("ns1", "ep1", "p12"): {
+				{"1.1.1.2:12", false},
+			},
+		},
+		expectedResult: map[proxy.ServicePortName][]*endpointsInfo{
+			makeServicePortName("ns1", "ep1", "p11"): {
+				{"1.1.1.1:11", false},
+			},
+			makeServicePortName("ns1", "ep1", "p12"): {
+				{"1.1.1.2:12", false},
+			},
+		},
+		expectedStale:        []endpointServicePair{},
+		expectedHealthchecks: map[types.NamespacedName]int{},
+	}, {
+		// Case[4]: no change, multiple subsets, multiple ports, local
+		previousEndpoints: []*api.Endpoints{
+			makeTestEndpoints("ns1", "ep1", multipleSubsetsMultiplePortsLocal),
+		},
+		currentEndpoints: []*api.Endpoints{
+			makeTestEndpoints("ns1", "ep1", multipleSubsetsMultiplePortsLocal),
+		},
+		oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{
+			makeServicePortName("ns1", "ep1", "p11"): {
+				{"1.1.1.1:11", true},
+			},
+			makeServicePortName("ns1", "ep1", "p12"): {
+				{"1.1.1.1:12", true},
+			},
+			makeServicePortName("ns1", "ep1", "p13"): {
+				{"1.1.1.3:13", false},
+			},
+		},
+		expectedResult: map[proxy.ServicePortName][]*endpointsInfo{
+			makeServicePortName("ns1", "ep1", "p11"): {
+				{"1.1.1.1:11", true},
+			},
+			makeServicePortName("ns1", "ep1", "p12"): {
+				{"1.1.1.1:12", true},
+			},
+			makeServicePortName("ns1", "ep1", "p13"): {
+				{"1.1.1.3:13", false},
+			},
+		},
+		expectedStale: []endpointServicePair{},
+		expectedHealthchecks: map[types.NamespacedName]int{
+			makeNSN("ns1", "ep1"): 1,
+		},
+	}, {
+		// Case[5]: no change, multiple Endpoints, subsets, IPs, and ports
+		previousEndpoints: []*api.Endpoints{
+			makeTestEndpoints("ns1", "ep1", multipleSubsetsIPsPorts1),
+			makeTestEndpoints("ns2", "ep2", multipleSubsetsIPsPorts2),
+		},
+		currentEndpoints: []*api.Endpoints{
+			makeTestEndpoints("ns1", "ep1", multipleSubsetsIPsPorts1),
+			makeTestEndpoints("ns2", "ep2", multipleSubsetsIPsPorts2),
+		},
+		oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{
+			makeServicePortName("ns1", "ep1", "p11"): {
+				{"1.1.1.1:11", false},
+				{"1.1.1.2:11", true},
+			},
+			makeServicePortName("ns1", "ep1", "p12"): {
+				{"1.1.1.1:12", false},
+				{"1.1.1.2:12", true},
+			},
+			makeServicePortName("ns1", "ep1", "p13"): {
+				{"1.1.1.3:13", false},
+				{"1.1.1.4:13", true},
+			},
+			makeServicePortName("ns1", "ep1", "p14"): {
+				{"1.1.1.3:14", false},
+				{"1.1.1.4:14", true},
+			},
+			makeServicePortName("ns2", "ep2", "p21"): {
+				{"2.2.2.1:21", false},
+				{"2.2.2.2:21", true},
+			},
+			makeServicePortName("ns2", "ep2", "p22"): {
+				{"2.2.2.1:22", false},
+				{"2.2.2.2:22", true},
+			},
+		},
+		expectedResult: map[proxy.ServicePortName][]*endpointsInfo{
+			makeServicePortName("ns1", "ep1", "p11"): {
+				{"1.1.1.1:11", false},
+				{"1.1.1.2:11", true},
+			},
+			makeServicePortName("ns1", "ep1", "p12"): {
+				{"1.1.1.1:12", false},
+				{"1.1.1.2:12", true},
+			},
+			makeServicePortName("ns1", "ep1", "p13"): {
+				{"1.1.1.3:13", false},
+				{"1.1.1.4:13", true},
+			},
+			makeServicePortName("ns1", "ep1", "p14"): {
+				{"1.1.1.3:14", false},
+				{"1.1.1.4:14", true},
+			},
+			makeServicePortName("ns2", "ep2", "p21"): {
+				{"2.2.2.1:21", false},
+				{"2.2.2.2:21", true},
+			},
+			makeServicePortName("ns2", "ep2", "p22"): {
+				{"2.2.2.1:22", false},
+				{"2.2.2.2:22", true},
+			},
+		},
+		expectedStale: []endpointServicePair{},
+		expectedHealthchecks: map[types.NamespacedName]int{
+			makeNSN("ns1", "ep1"): 2,
+			makeNSN("ns2", "ep2"): 1,
+		},
+	}, {
+		// Case[6]: add an Endpoints
+		previousEndpoints: []*api.Endpoints{
+			nil,
+		},
+		currentEndpoints: []*api.Endpoints{
+			makeTestEndpoints("ns1", "ep1", unnamedPortLocal),
+		},
+		oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{},
+		expectedResult: map[proxy.ServicePortName][]*endpointsInfo{
+			makeServicePortName("ns1", "ep1", ""): {
+				{"1.1.1.1:11", true},
+			},
+		},
+		expectedStale: []endpointServicePair{},
+		expectedHealthchecks: map[types.NamespacedName]int{
+			makeNSN("ns1", "ep1"): 1,
+		},
+	}, {
+		// Case[7]: remove an Endpoints
+		previousEndpoints: []*api.Endpoints{
+			makeTestEndpoints("ns1", "ep1", unnamedPortLocal),
+		},
+		currentEndpoints: []*api.Endpoints{
+			nil,
+		},
+		oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{
+			makeServicePortName("ns1", "ep1", ""): {
+				{"1.1.1.1:11", true},
+			},
+		},
+		expectedResult: map[proxy.ServicePortName][]*endpointsInfo{},
+		expectedStale: []endpointServicePair{{
+			endpoint:        "1.1.1.1:11",
+			servicePortName: makeServicePortName("ns1", "ep1", ""),
+		}},
+		expectedHealthchecks: map[types.NamespacedName]int{},
+	}, {
+		// Case[8]: add an IP and port
+		previousEndpoints: []*api.Endpoints{
+			makeTestEndpoints("ns1", "ep1", namedPort),
+		},
+		currentEndpoints: []*api.Endpoints{
+			makeTestEndpoints("ns1", "ep1", namedPortsLocalNoLocal),
+		},
+		oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{
+			makeServicePortName("ns1", "ep1", "p11"): {
+				{"1.1.1.1:11", false},
+			},
+		},
+		expectedResult: map[proxy.ServicePortName][]*endpointsInfo{
+			makeServicePortName("ns1", "ep1", "p11"): {
+				{"1.1.1.1:11", false},
+				{"1.1.1.2:11", true},
+			},
+			makeServicePortName("ns1", "ep1", "p12"): {
+				{"1.1.1.1:12", false},
+				{"1.1.1.2:12", true},
+			},
+		},
+		expectedStale: []endpointServicePair{},
+		expectedHealthchecks: map[types.NamespacedName]int{
+			makeNSN("ns1", "ep1"): 1,
+		},
+	}, {
+		// Case[9]: remove an IP and port
+		previousEndpoints: []*api.Endpoints{
+			makeTestEndpoints("ns1", "ep1", namedPortsLocalNoLocal),
+		},
+		currentEndpoints: []*api.Endpoints{
+			makeTestEndpoints("ns1", "ep1", namedPort),
+		},
+		oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{
+			makeServicePortName("ns1", "ep1", "p11"): {
+				{"1.1.1.1:11", false},
+				{"1.1.1.2:11", true},
+			},
+			makeServicePortName("ns1", "ep1", "p12"): {
+				{"1.1.1.1:12", false},
+				{"1.1.1.2:12", true},
+			},
+		},
+		expectedResult: map[proxy.ServicePortName][]*endpointsInfo{
+			makeServicePortName("ns1", "ep1", "p11"): {
+				{"1.1.1.1:11", false},
+			},
+		},
+		expectedStale: []endpointServicePair{{
+			endpoint:        "1.1.1.2:11",
+			servicePortName: makeServicePortName("ns1", "ep1", "p11"),
+		}, {
+			endpoint:        "1.1.1.1:12",
+			servicePortName: makeServicePortName("ns1", "ep1", "p12"),
+		}, {
+			endpoint:        "1.1.1.2:12",
+			servicePortName: makeServicePortName("ns1", "ep1", "p12"),
+		}},
+		expectedHealthchecks: map[types.NamespacedName]int{},
+	}, {
+		// Case[10]: add a subset
+		previousEndpoints: []*api.Endpoints{
+			makeTestEndpoints("ns1", "ep1", namedPort),
+		},
+		currentEndpoints: []*api.Endpoints{
+			makeTestEndpoints("ns1", "ep1", multipleSubsetsWithLocal),
+		},
+		oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{
+			makeServicePortName("ns1", "ep1", "p11"): {
+				{"1.1.1.1:11", false},
+			},
+		},
+		expectedResult: map[proxy.ServicePortName][]*endpointsInfo{
+			makeServicePortName("ns1", "ep1", "p11"): {
+				{"1.1.1.1:11", false},
+			},
+			makeServicePortName("ns1", "ep1", "p12"): {
+				{"1.1.1.2:12", true},
+			},
+		},
+		expectedStale: []endpointServicePair{},
+		expectedHealthchecks: map[types.NamespacedName]int{
+			makeNSN("ns1", "ep1"): 1,
+		},
+	}, {
+		// Case[11]: remove a subset
+		previousEndpoints: []*api.Endpoints{
+			makeTestEndpoints("ns1", "ep1", multipleSubsets),
+		},
+		currentEndpoints: []*api.Endpoints{
+			makeTestEndpoints("ns1", "ep1", namedPort),
+		},
+		oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{
+			makeServicePortName("ns1", "ep1", "p11"): {
+				{"1.1.1.1:11", false},
+			},
+			makeServicePortName("ns1", "ep1", "p12"): {
+				{"1.1.1.2:12", false},
+			},
+		},
+		expectedResult: map[proxy.ServicePortName][]*endpointsInfo{
+			makeServicePortName("ns1", "ep1", "p11"): {
+				{"1.1.1.1:11", false},
+			},
+		},
+		expectedStale: []endpointServicePair{{
+			endpoint:        "1.1.1.2:12",
+			servicePortName: makeServicePortName("ns1", "ep1", "p12"),
+		}},
+		expectedHealthchecks: map[types.NamespacedName]int{},
+	}, {
+		// Case[12]: rename a port
+		previousEndpoints: []*api.Endpoints{
+			makeTestEndpoints("ns1", "ep1", namedPort),
+		},
+		currentEndpoints: []*api.Endpoints{
+			makeTestEndpoints("ns1", "ep1", namedPortRenamed),
+		},
+		oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{
+			makeServicePortName("ns1", "ep1", "p11"): {
+				{"1.1.1.1:11", false},
+			},
+		},
+		expectedResult: map[proxy.ServicePortName][]*endpointsInfo{
+			makeServicePortName("ns1", "ep1", "p11-2"): {
+				{"1.1.1.1:11", false},
+			},
+		},
+		expectedStale: []endpointServicePair{{
+			endpoint:        "1.1.1.1:11",
+			servicePortName: makeServicePortName("ns1", "ep1", "p11"),
+		}},
+		expectedHealthchecks: map[types.NamespacedName]int{},
+	}, {
+		// Case[13]: renumber a port
+		previousEndpoints: []*api.Endpoints{
+			makeTestEndpoints("ns1", "ep1", namedPort),
+		},
+		currentEndpoints: []*api.Endpoints{
+			makeTestEndpoints("ns1", "ep1", namedPortRenumbered),
+		},
+		oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{
+			makeServicePortName("ns1", "ep1", "p11"): {
+				{"1.1.1.1:11", false},
+			},
+		},
+		expectedResult: map[proxy.ServicePortName][]*endpointsInfo{
+			makeServicePortName("ns1", "ep1", "p11"): {
+				{"1.1.1.1:22", false},
+			},
+		},
+		expectedStale: []endpointServicePair{{
+			endpoint:        "1.1.1.1:11",
+			servicePortName: makeServicePortName("ns1", "ep1", "p11"),
+		}},
+		expectedHealthchecks: map[types.NamespacedName]int{},
+	}, {
+		// Case[14]: complex add and remove
+		previousEndpoints: []*api.Endpoints{
+			makeTestEndpoints("ns1", "ep1", complexBefore1),
+			makeTestEndpoints("ns2", "ep2", complexBefore2),
+			nil,
+			makeTestEndpoints("ns4", "ep4", complexBefore4),
+		},
+		currentEndpoints: []*api.Endpoints{
+			makeTestEndpoints("ns1", "ep1", complexAfter1),
+			nil,
+			makeTestEndpoints("ns3", "ep3", complexAfter3),
+			makeTestEndpoints("ns4", "ep4", complexAfter4),
+		},
+		oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{
+			makeServicePortName("ns1", "ep1", "p11"): {
+				{"1.1.1.1:11", false},
+			},
+			makeServicePortName("ns2", "ep2", "p22"): {
+				{"2.2.2.2:22", true},
+				{"2.2.2.22:22", true},
+			},
+			makeServicePortName("ns2", "ep2", "p23"): {
+				{"2.2.2.3:23", true},
+			},
+			makeServicePortName("ns4", "ep4", "p44"): {
+				{"4.4.4.4:44", true},
+				{"4.4.4.5:44", true},
+			},
+			makeServicePortName("ns4", "ep4", "p45"): {
+				{"4.4.4.6:45", true},
+			},
+		},
+		expectedResult: map[proxy.ServicePortName][]*endpointsInfo{
+			makeServicePortName("ns1", "ep1", "p11"): {
+				{"1.1.1.1:11", false},
+				{"1.1.1.11:11", false},
+			},
+			makeServicePortName("ns1", "ep1", "p12"): {
+				{"1.1.1.2:12", false},
+			},
+			makeServicePortName("ns1", "ep1", "p122"): {
+				{"1.1.1.2:122", false},
+			},
+			makeServicePortName("ns3", "ep3", "p33"): {
+				{"3.3.3.3:33", false},
+			},
+			makeServicePortName("ns4", "ep4", "p44"): {
+				{"4.4.4.4:44", true},
+			},
+		},
+		expectedStale: []endpointServicePair{{
+			endpoint:        "2.2.2.2:22",
+			servicePortName: makeServicePortName("ns2", "ep2", "p22"),
+		}, {
+			endpoint:        "2.2.2.22:22",
+			servicePortName: makeServicePortName("ns2", "ep2", "p22"),
+		}, {
+			endpoint:        "2.2.2.3:23",
+			servicePortName: makeServicePortName("ns2", "ep2", "p23"),
+		}, {
+			endpoint:        "4.4.4.5:44",
+			servicePortName: makeServicePortName("ns4", "ep4", "p44"),
+		}, {
+			endpoint:        "4.4.4.6:45",
+			servicePortName: makeServicePortName("ns4", "ep4", "p45"),
+		}},
+		expectedHealthchecks: map[types.NamespacedName]int{
+			makeNSN("ns4", "ep4"): 1,
+		},
+	}}
+
+	for tci, tc := range testCases {
+		ipt := iptablestest.NewFake()
+		ipvs := ipvstest.NewFake()
+		fp := NewFakeProxier(ipt, ipvs, nil)
+		fp.hostname = nodeName
+
+		// First check that after adding all previous versions of Endpoints,
+		// the fp.oldEndpoints is as we expect.
+		for i := range tc.previousEndpoints {
+			if tc.previousEndpoints[i] != nil {
+				fp.OnEndpointsAdd(tc.previousEndpoints[i])
+			}
+		}
+		updateEndpointsMap(fp.endpointsMap, &fp.endpointsChanges, fp.hostname)
+		compareEndpointsMaps(t, tci, fp.endpointsMap, tc.oldEndpoints)
+
+		// Now let's call appropriate handlers to get to state we want to be.
+		if len(tc.previousEndpoints) != len(tc.currentEndpoints) {
+			t.Fatalf("[%d] different lengths of previous and current Endpoints", tci)
+			continue
+		}
+
+		for i := range tc.previousEndpoints {
+			prev, curr := tc.previousEndpoints[i], tc.currentEndpoints[i]
+			switch {
+			case prev == nil:
+				fp.OnEndpointsAdd(curr)
+			case curr == nil:
+				fp.OnEndpointsDelete(prev)
+			default:
+				fp.OnEndpointsUpdate(prev, curr)
+			}
+		}
+		_, hcEndpoints, stale := updateEndpointsMap(fp.endpointsMap, &fp.endpointsChanges, fp.hostname)
+		newMap := fp.endpointsMap
+		compareEndpointsMaps(t, tci, newMap, tc.expectedResult)
+		if len(stale) != len(tc.expectedStale) {
+			t.Errorf("[%d] expected %d stale, got %d: %v", tci, len(tc.expectedStale), len(stale), stale)
+		}
+		for _, x := range tc.expectedStale {
+			if stale[x] != true {
+				t.Errorf("[%d] expected stale[%v], but didn't find it: %v", tci, x, stale)
+			}
+		}
+		if !reflect.DeepEqual(hcEndpoints, tc.expectedHealthchecks) {
+			t.Errorf("[%d] expected healthchecks %v, got %v", tci, tc.expectedHealthchecks, hcEndpoints)
+		}
+	}
+}
+
+func compareEndpointsMaps(t *testing.T, tci int, newMap, expected map[proxy.ServicePortName][]*endpointsInfo) {
+	if len(newMap) != len(expected) {
+		t.Errorf("[%d] expected %d results, got %d: %v", tci, len(expected), len(newMap), newMap)
+	}
+	for x := range expected {
+		if len(newMap[x]) != len(expected[x]) {
+			t.Errorf("[%d] expected %d Endpoints for %v, got %d", tci, len(expected[x]), x, len(newMap[x]))
+		} else {
+			for i := range expected[x] {
+				if *(newMap[x][i]) != *(expected[x][i]) {
+					t.Errorf("[%d] expected new[%v][%d] to be %v, got %v", tci, x, i, expected[x][i], newMap[x][i])
+				}
+			}
+		}
+	}
+}
+
+func Test_getLocalIPs(t *testing.T) {
+	testCases := []struct {
+		endpointsMap map[proxy.ServicePortName][]*endpointsInfo
+		expected     map[types.NamespacedName]sets.String
+	}{{
+		// Case[0]: nothing
+		endpointsMap: map[proxy.ServicePortName][]*endpointsInfo{},
+		expected:     map[types.NamespacedName]sets.String{},
+	}, {
+		// Case[1]: unnamed port
+		endpointsMap: map[proxy.ServicePortName][]*endpointsInfo{
+			makeServicePortName("ns1", "ep1", ""): {
+				{"1.1.1.1:11", false},
+			},
+		},
+		expected: map[types.NamespacedName]sets.String{},
+	}, {
+		// Case[2]: unnamed port local
+		endpointsMap: map[proxy.ServicePortName][]*endpointsInfo{
+			makeServicePortName("ns1", "ep1", ""): {
+				{"1.1.1.1:11", true},
+			},
+		},
+		expected: map[types.NamespacedName]sets.String{
+			{Namespace: "ns1", Name: "ep1"}: sets.NewString("1.1.1.1"),
+		},
+	}, {
+		// Case[3]: named local and non-local ports for the same IP.
+		endpointsMap: map[proxy.ServicePortName][]*endpointsInfo{
+			makeServicePortName("ns1", "ep1", "p11"): {
+				{"1.1.1.1:11", false},
+				{"1.1.1.2:11", true},
+			},
+			makeServicePortName("ns1", "ep1", "p12"): {
+				{"1.1.1.1:12", false},
+				{"1.1.1.2:12", true},
+			},
+		},
+		expected: map[types.NamespacedName]sets.String{
+			{Namespace: "ns1", Name: "ep1"}: sets.NewString("1.1.1.2"),
+		},
+	}, {
+		// Case[4]: named local and non-local ports for different IPs.
+		endpointsMap: map[proxy.ServicePortName][]*endpointsInfo{
+			makeServicePortName("ns1", "ep1", "p11"): {
+				{"1.1.1.1:11", false},
+			},
+			makeServicePortName("ns2", "ep2", "p22"): {
+				{"2.2.2.2:22", true},
+				{"2.2.2.22:22", true},
+			},
+			makeServicePortName("ns2", "ep2", "p23"): {
+				{"2.2.2.3:23", true},
+			},
+			makeServicePortName("ns4", "ep4", "p44"): {
+				{"4.4.4.4:44", true},
+				{"4.4.4.5:44", false},
+			},
+			makeServicePortName("ns4", "ep4", "p45"): {
+				{"4.4.4.6:45", true},
+			},
+		},
+		expected: map[types.NamespacedName]sets.String{
+			{Namespace: "ns2", Name: "ep2"}: sets.NewString("2.2.2.2", "2.2.2.22", "2.2.2.3"),
+			{Namespace: "ns4", Name: "ep4"}: sets.NewString("4.4.4.4", "4.4.4.6"),
+		},
+	}}
+
+	for tci, tc := range testCases {
+		// outputs
+		localIPs := getLocalIPs(tc.endpointsMap)
+
+		if !reflect.DeepEqual(localIPs, tc.expected) {
+			t.Errorf("[%d] expected %#v, got %#v", tci, tc.expected, localIPs)
+		}
+	}
+}
+
+// This is a coarse test, but it offers some modicum of confidence as the code is evolved.
+func Test_endpointsToEndpointsMap(t *testing.T) {
+	testCases := []struct {
+		newEndpoints *api.Endpoints
+		expected     map[proxy.ServicePortName][]*endpointsInfo
+	}{{
+		// Case[0]: nothing
+		newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *api.Endpoints) {}),
+		expected:     map[proxy.ServicePortName][]*endpointsInfo{},
+	}, {
+		// Case[1]: no changes, unnamed port
+		newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *api.Endpoints) {
+			ept.Subsets = []api.EndpointSubset{
+				{
+					Addresses: []api.EndpointAddress{{
+						IP: "1.1.1.1",
+					}},
+					Ports: []api.EndpointPort{{
+						Name: "",
+						Port: 11,
+					}},
+				},
+			}
+		}),
+		expected: map[proxy.ServicePortName][]*endpointsInfo{
+			makeServicePortName("ns1", "ep1", ""): {
+				{"1.1.1.1:11", false},
+			},
+		},
+	}, {
+		// Case[2]: no changes, named port
+		newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *api.Endpoints) {
+			ept.Subsets = []api.EndpointSubset{
+				{
+					Addresses: []api.EndpointAddress{{
+						IP: "1.1.1.1",
+					}},
+					Ports: []api.EndpointPort{{
+						Name: "port",
+						Port: 11,
+					}},
+				},
+			}
+		}),
+		expected: map[proxy.ServicePortName][]*endpointsInfo{
+			makeServicePortName("ns1", "ep1", "port"): {
+				{"1.1.1.1:11", false},
+			},
+		},
+	}, {
+		// Case[3]: new port
+		newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *api.Endpoints) {
+			ept.Subsets = []api.EndpointSubset{
+				{
+					Addresses: []api.EndpointAddress{{
+						IP: "1.1.1.1",
+					}},
+					Ports: []api.EndpointPort{{
+						Port: 11,
+					}},
+				},
+			}
+		}),
+		expected: map[proxy.ServicePortName][]*endpointsInfo{
+			makeServicePortName("ns1", "ep1", ""): {
+				{"1.1.1.1:11", false},
+			},
+		},
+	}, {
+		// Case[4]: remove port
+		newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *api.Endpoints) {}),
+		expected:     map[proxy.ServicePortName][]*endpointsInfo{},
+	}, {
+		// Case[5]: new IP and port
+		newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *api.Endpoints) {
+			ept.Subsets = []api.EndpointSubset{
+				{
+					Addresses: []api.EndpointAddress{{
+						IP: "1.1.1.1",
+					}, {
+						IP: "2.2.2.2",
+					}},
+					Ports: []api.EndpointPort{{
+						Name: "p1",
+						Port: 11,
+					}, {
+						Name: "p2",
+						Port: 22,
+					}},
+				},
+			}
+		}),
+		expected: map[proxy.ServicePortName][]*endpointsInfo{
+			makeServicePortName("ns1", "ep1", "p1"): {
+				{"1.1.1.1:11", false},
+				{"2.2.2.2:11", false},
+			},
+			makeServicePortName("ns1", "ep1", "p2"): {
+				{"1.1.1.1:22", false},
+				{"2.2.2.2:22", false},
+			},
+		},
+	}, {
+		// Case[6]: remove IP and port
+		newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *api.Endpoints) {
+			ept.Subsets = []api.EndpointSubset{
+				{
+					Addresses: []api.EndpointAddress{{
+						IP: "1.1.1.1",
+					}},
+					Ports: []api.EndpointPort{{
+						Name: "p1",
+						Port: 11,
+					}},
+				},
+			}
+		}),
+		expected: map[proxy.ServicePortName][]*endpointsInfo{
+			makeServicePortName("ns1", "ep1", "p1"): {
+				{"1.1.1.1:11", false},
+			},
+		},
+	}, {
+		// Case[7]: rename port
+		newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *api.Endpoints) {
+			ept.Subsets = []api.EndpointSubset{
+				{
+					Addresses: []api.EndpointAddress{{
+						IP: "1.1.1.1",
+					}},
+					Ports: []api.EndpointPort{{
+						Name: "p2",
+						Port: 11,
+					}},
+				},
+			}
+		}),
+		expected: map[proxy.ServicePortName][]*endpointsInfo{
+			makeServicePortName("ns1", "ep1", "p2"): {
+				{"1.1.1.1:11", false},
+			},
+		},
+	}, {
+		// Case[8]: renumber port
+		newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *api.Endpoints) {
+			ept.Subsets = []api.EndpointSubset{
+				{
+					Addresses: []api.EndpointAddress{{
+						IP: "1.1.1.1",
+					}},
+					Ports: []api.EndpointPort{{
+						Name: "p1",
+						Port: 22,
+					}},
+				},
+			}
+		}),
+		expected: map[proxy.ServicePortName][]*endpointsInfo{
+			makeServicePortName("ns1", "ep1", "p1"): {
+				{"1.1.1.1:22", false},
+			},
+		},
+	}}
+
+	for tci, tc := range testCases {
+		// outputs
+		newEndpoints := endpointsToEndpointsMap(tc.newEndpoints, "host")
+
+		if len(newEndpoints) != len(tc.expected) {
+			t.Errorf("[%d] expected %d new, got %d: %v", tci, len(tc.expected), len(newEndpoints), spew.Sdump(newEndpoints))
+		}
+		for x := range tc.expected {
+			if len(newEndpoints[x]) != len(tc.expected[x]) {
+				t.Errorf("[%d] expected %d endpoints for %v, got %d", tci, len(tc.expected[x]), x, len(newEndpoints[x]))
+			} else {
+				for i := range newEndpoints[x] {
+					if *(newEndpoints[x][i]) != *(tc.expected[x][i]) {
+						t.Errorf("[%d] expected new[%v][%d] to be %v, got %v", tci, x, i, tc.expected[x][i], *(newEndpoints[x][i]))
+					}
+				}
+			}
+		}
+	}
+}
+
+func Test_ensureDummyDevice(t *testing.T) {
+	fcmd := fakeexec.FakeCmd{
+		CombinedOutputScript: []fakeexec.FakeCombinedOutputAction{
+			// Success.
+			func() ([]byte, error) { return []byte{}, nil },
+			// Exists.
+			func() ([]byte, error) { return nil, &fakeexec.FakeExitError{Status: 2} },
+		},
+	}
+	fexec := fakeexec.FakeExec{
+		CommandScript: []fakeexec.FakeCommandAction{
+			func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
+			func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
+		},
+	}
+	// Success.
+	exists, err := ensureDummyDevice(&fexec, DefaultDummyDevice)
+	if err != nil {
+		t.Errorf("expected success, got %v", err)
+	}
+	if exists {
+		t.Errorf("expected exists = false")
+	}
+	if fcmd.CombinedOutputCalls != 1 {
+		t.Errorf("expected 1 CombinedOutput() calls, got %d", fcmd.CombinedOutputCalls)
+	}
+	if !sets.NewString(fcmd.CombinedOutputLog[0]...).HasAll("ip", "link", "add", "kube-ipvs0", "type", "dummy") {
+		t.Errorf("wrong CombinedOutput() log, got %s", fcmd.CombinedOutputLog[0])
+	}
+	// Exists.
+	exists, err = ensureDummyDevice(&fexec, DefaultDummyDevice)
+	if err != nil {
+		t.Errorf("expected success, got %v", err)
+	}
+	if !exists {
+		t.Errorf("expected exists = true")
+	}
+}
+
+func Test_deleteDummyDevice(t *testing.T) {
+	fcmd := fakeexec.FakeCmd{
+		CombinedOutputScript: []fakeexec.FakeCombinedOutputAction{
+			// Success.
+			func() ([]byte, error) { return []byte{}, nil },
+			// Failure.
+			func() ([]byte, error) { return nil, &fakeexec.FakeExitError{Status: 1} },
+		},
+	}
+	fexec := fakeexec.FakeExec{
+		CommandScript: []fakeexec.FakeCommandAction{
+			func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
+			func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
+		},
+	}
+	// Success.
+	err := deleteDummyDevice(&fexec, DefaultDummyDevice)
+	if err != nil {
+		t.Errorf("expected success, got %v", err)
+	}
+	if fcmd.CombinedOutputCalls != 1 {
+		t.Errorf("expected 1 CombinedOutput() calls, got %d", fcmd.CombinedOutputCalls)
+	}
+	if !sets.NewString(fcmd.CombinedOutputLog[0]...).HasAll("ip", "link", "del", "kube-ipvs0") {
+		t.Errorf("wrong CombinedOutput() log, got %s", fcmd.CombinedOutputLog[0])
+	}
+	// Failure.
+	err = deleteDummyDevice(&fexec, DefaultDummyDevice)
+	if err == nil {
+		t.Errorf("expected failure")
+	}
+}

vendor/github.com/docker/libnetwork/ipvs/BUILD

@@ -4,18 +4,21 @@ go_library(
     name = "go_default_library",
     srcs = select({
         "@io_bazel_rules_go//go/platform:linux_amd64": [
-            "addr_linux.go",
-            "link_linux.go",
-            "nl_linux.go",
-            "route_linux.go",
-            "tc_linux.go",
-            "xfrm_linux.go",
-            "xfrm_policy_linux.go",
-            "xfrm_state_linux.go",
+            "constants.go",
+            "ipvs.go",
+            "netlink.go",
         ],
         "//conditions:default": [],
     }),
     visibility = ["//visibility:public"],
+    deps = select({
+        "@io_bazel_rules_go//go/platform:linux_amd64": [
+            "//vendor/github.com/Sirupsen/logrus:go_default_library",
+            "//vendor/github.com/vishvananda/netlink/nl:go_default_library",
+            "//vendor/github.com/vishvananda/netns:go_default_library",
+        ],
+        "//conditions:default": [],
+    }),
 )
 
 filegroup(