implement ipvs mode of kube-proxy

build/debian-iptables/Dockerfile

@@ -21,4 +21,5 @@ CROSS_BUILD_COPY qemu-ARCH-static /usr/bin/
 RUN clean-install \
     iptables \
     ebtables \
-    conntrack
+    conntrack \
+    module-init-tools

cmd/kube-proxy/app/BUILD

@@ -18,6 +18,7 @@ go_library(
         "//pkg/apis/componentconfig/v1alpha1:go_default_library",
         "//pkg/client/clientset_generated/internalclientset:go_default_library",
         "//pkg/client/informers/informers_generated/internalversion:go_default_library",
+        "//pkg/features:go_default_library",
         "//pkg/kubectl/cmd/util:go_default_library",
         "//pkg/kubelet/qos:go_default_library",
         "//pkg/master/ports:go_default_library",
@@ -25,11 +26,13 @@ go_library(
         "//pkg/proxy/config:go_default_library",
         "//pkg/proxy/healthcheck:go_default_library",
         "//pkg/proxy/iptables:go_default_library",
+        "//pkg/proxy/ipvs:go_default_library",
         "//pkg/proxy/userspace:go_default_library",
         "//pkg/proxy/winuserspace:go_default_library",
         "//pkg/util/configz:go_default_library",
         "//pkg/util/dbus:go_default_library",
         "//pkg/util/iptables:go_default_library",
+        "//pkg/util/ipvs:go_default_library",
         "//pkg/util/mount:go_default_library",
         "//pkg/util/netsh:go_default_library",
         "//pkg/util/node:go_default_library",

cmd/kube-proxy/app/server_test.go

@@ -287,6 +287,9 @@ iptables:
   masqueradeBit: 17
   minSyncPeriod: 10s
   syncPeriod: 60s
+ipvs:
+  minSyncPeriod: 10s
+  syncPeriod: 60s
 kind: KubeProxyConfiguration
 metricsBindAddress: "%s"
 mode: "iptables"
@@ -347,12 +350,17 @@ udpTimeoutMilliseconds: 123ms
 				MinSyncPeriod: metav1.Duration{Duration: 10 * time.Second},
 				SyncPeriod:    metav1.Duration{Duration: 60 * time.Second},
 			},
+			IPVS: componentconfig.KubeProxyIPVSConfiguration{
+				MinSyncPeriod: metav1.Duration{Duration: 10 * time.Second},
+				SyncPeriod:    metav1.Duration{Duration: 60 * time.Second},
+			},
 			MetricsBindAddress: tc.metricsBindAddress,
 			Mode:               "iptables",
-			OOMScoreAdj:        utilpointer.Int32Ptr(17),
-			PortRange:          "2-7",
-			ResourceContainer:  "/foo",
-			UDPIdleTimeout:     metav1.Duration{Duration: 123 * time.Millisecond},
+			// TODO: IPVS
+			OOMScoreAdj:       utilpointer.Int32Ptr(17),
+			PortRange:         "2-7",
+			ResourceContainer: "/foo",
+			UDPIdleTimeout:    metav1.Duration{Duration: 123 * time.Millisecond},
 		}
 
 		options, err := NewOptions()

pkg/apis/componentconfig/types.go

@@ -52,6 +52,19 @@ type KubeProxyIPTablesConfiguration struct {
 	MinSyncPeriod metav1.Duration
 }
 
+// KubeProxyIPVSConfiguration contains ipvs-related configuration
+// details for the Kubernetes proxy server.
+type KubeProxyIPVSConfiguration struct {
+	// syncPeriod is the period that ipvs rules are refreshed (e.g. '5s', '1m',
+	// '2h22m').  Must be greater than 0.
+	SyncPeriod metav1.Duration
+	// minSyncPeriod is the minimum period that ipvs rules are refreshed (e.g. '5s', '1m',
+	// '2h22m').
+	MinSyncPeriod metav1.Duration
+	// ipvs scheduler
+	Scheduler string
+}
+
 // KubeProxyConntrackConfiguration contains conntrack settings for
 // the Kubernetes proxy server.
 type KubeProxyConntrackConfiguration struct {
@@ -112,6 +125,8 @@ type KubeProxyConfiguration struct {
 	ClientConnection ClientConnectionConfiguration
 	// iptables contains iptables-related configuration options.
 	IPTables KubeProxyIPTablesConfiguration
+	// ipvs contains ipvs-related configuration options.
+	IPVS KubeProxyIPVSConfiguration
 	// oomScoreAdj is the oom-score-adj value for kube-proxy process. Values must be within
 	// the range [-1000, 1000]
 	OOMScoreAdj *int32

pkg/apis/componentconfig/v1alpha1/types.go

@@ -52,6 +52,19 @@ type KubeProxyIPTablesConfiguration struct {
 	MinSyncPeriod metav1.Duration `json:"minSyncPeriod"`
 }
 
+// KubeProxyIPVSConfiguration contains ipvs-related configuration
+// details for the Kubernetes proxy server.
+type KubeProxyIPVSConfiguration struct {
+	// syncPeriod is the period that ipvs rules are refreshed (e.g. '5s', '1m',
+	// '2h22m').  Must be greater than 0.
+	SyncPeriod metav1.Duration `json:"syncPeriod"`
+	// minSyncPeriod is the minimum period that ipvs rules are refreshed (e.g. '5s', '1m',
+	// '2h22m').
+	MinSyncPeriod metav1.Duration `json:"minSyncPeriod"`
+	// ipvs scheduler
+	Scheduler string `json:"scheduler"`
+}
+
 // KubeProxyConntrackConfiguration contains conntrack settings for
 // the Kubernetes proxy server.
 type KubeProxyConntrackConfiguration struct {
@@ -112,6 +125,8 @@ type KubeProxyConfiguration struct {
 	ClientConnection ClientConnectionConfiguration `json:"clientConnection"`
 	// iptables contains iptables-related configuration options.
 	IPTables KubeProxyIPTablesConfiguration `json:"iptables"`
+	// ipvs contains ipvs-related configuration options.
+	IPVS KubeProxyIPVSConfiguration `json:"ipvs"`
 	// oomScoreAdj is the oom-score-adj value for kube-proxy process. Values must be within
 	// the range [-1000, 1000]
 	OOMScoreAdj *int32 `json:"oomScoreAdj"`

pkg/apis/componentconfig/v1alpha1/zz_generated.conversion.go

@@ -44,6 +44,8 @@ func RegisterConversions(scheme *runtime.Scheme) error {
 		Convert_componentconfig_KubeProxyConntrackConfiguration_To_v1alpha1_KubeProxyConntrackConfiguration,
 		Convert_v1alpha1_KubeProxyIPTablesConfiguration_To_componentconfig_KubeProxyIPTablesConfiguration,
 		Convert_componentconfig_KubeProxyIPTablesConfiguration_To_v1alpha1_KubeProxyIPTablesConfiguration,
+		Convert_v1alpha1_KubeProxyIPVSConfiguration_To_componentconfig_KubeProxyIPVSConfiguration,
+		Convert_componentconfig_KubeProxyIPVSConfiguration_To_v1alpha1_KubeProxyIPVSConfiguration,
 		Convert_v1alpha1_KubeSchedulerConfiguration_To_componentconfig_KubeSchedulerConfiguration,
 		Convert_componentconfig_KubeSchedulerConfiguration_To_v1alpha1_KubeSchedulerConfiguration,
 		Convert_v1alpha1_LeaderElectionConfiguration_To_componentconfig_LeaderElectionConfiguration,
@@ -93,6 +95,9 @@ func autoConvert_v1alpha1_KubeProxyConfiguration_To_componentconfig_KubeProxyCon
 	if err := Convert_v1alpha1_KubeProxyIPTablesConfiguration_To_componentconfig_KubeProxyIPTablesConfiguration(&in.IPTables, &out.IPTables, s); err != nil {
 		return err
 	}
+	if err := Convert_v1alpha1_KubeProxyIPVSConfiguration_To_componentconfig_KubeProxyIPVSConfiguration(&in.IPVS, &out.IPVS, s); err != nil {
+		return err
+	}
 	out.OOMScoreAdj = (*int32)(unsafe.Pointer(in.OOMScoreAdj))
 	out.Mode = componentconfig.ProxyMode(in.Mode)
 	out.PortRange = in.PortRange
@@ -124,6 +129,9 @@ func autoConvert_componentconfig_KubeProxyConfiguration_To_v1alpha1_KubeProxyCon
 	if err := Convert_componentconfig_KubeProxyIPTablesConfiguration_To_v1alpha1_KubeProxyIPTablesConfiguration(&in.IPTables, &out.IPTables, s); err != nil {
 		return err
 	}
+	if err := Convert_componentconfig_KubeProxyIPVSConfiguration_To_v1alpha1_KubeProxyIPVSConfiguration(&in.IPVS, &out.IPVS, s); err != nil {
+		return err
+	}
 	out.OOMScoreAdj = (*int32)(unsafe.Pointer(in.OOMScoreAdj))
 	out.Mode = ProxyMode(in.Mode)
 	out.PortRange = in.PortRange
@@ -195,6 +203,30 @@ func Convert_componentconfig_KubeProxyIPTablesConfiguration_To_v1alpha1_KubeProx
 	return autoConvert_componentconfig_KubeProxyIPTablesConfiguration_To_v1alpha1_KubeProxyIPTablesConfiguration(in, out, s)
 }
 
+func autoConvert_v1alpha1_KubeProxyIPVSConfiguration_To_componentconfig_KubeProxyIPVSConfiguration(in *KubeProxyIPVSConfiguration, out *componentconfig.KubeProxyIPVSConfiguration, s conversion.Scope) error {
+	out.SyncPeriod = in.SyncPeriod
+	out.MinSyncPeriod = in.MinSyncPeriod
+	out.Scheduler = in.Scheduler
+	return nil
+}
+
+// Convert_v1alpha1_KubeProxyIPVSConfiguration_To_componentconfig_KubeProxyIPVSConfiguration is an autogenerated conversion function.
+func Convert_v1alpha1_KubeProxyIPVSConfiguration_To_componentconfig_KubeProxyIPVSConfiguration(in *KubeProxyIPVSConfiguration, out *componentconfig.KubeProxyIPVSConfiguration, s conversion.Scope) error {
+	return autoConvert_v1alpha1_KubeProxyIPVSConfiguration_To_componentconfig_KubeProxyIPVSConfiguration(in, out, s)
+}
+
+func autoConvert_componentconfig_KubeProxyIPVSConfiguration_To_v1alpha1_KubeProxyIPVSConfiguration(in *componentconfig.KubeProxyIPVSConfiguration, out *KubeProxyIPVSConfiguration, s conversion.Scope) error {
+	out.SyncPeriod = in.SyncPeriod
+	out.MinSyncPeriod = in.MinSyncPeriod
+	out.Scheduler = in.Scheduler
+	return nil
+}
+
+// Convert_componentconfig_KubeProxyIPVSConfiguration_To_v1alpha1_KubeProxyIPVSConfiguration is an autogenerated conversion function.
+func Convert_componentconfig_KubeProxyIPVSConfiguration_To_v1alpha1_KubeProxyIPVSConfiguration(in *componentconfig.KubeProxyIPVSConfiguration, out *KubeProxyIPVSConfiguration, s conversion.Scope) error {
+	return autoConvert_componentconfig_KubeProxyIPVSConfiguration_To_v1alpha1_KubeProxyIPVSConfiguration(in, out, s)
+}
+
 func autoConvert_v1alpha1_KubeSchedulerConfiguration_To_componentconfig_KubeSchedulerConfiguration(in *KubeSchedulerConfiguration, out *componentconfig.KubeSchedulerConfiguration, s conversion.Scope) error {
 	out.Port = int32(in.Port)
 	out.Address = in.Address

pkg/apis/componentconfig/v1alpha1/zz_generated.deepcopy.go

@@ -53,6 +53,10 @@ func RegisterDeepCopies(scheme *runtime.Scheme) error {
 			return nil
 		}, InType: reflect.TypeOf(&KubeProxyIPTablesConfiguration{})},
 		conversion.GeneratedDeepCopyFunc{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error {
+			in.(*KubeProxyIPVSConfiguration).DeepCopyInto(out.(*KubeProxyIPVSConfiguration))
+			return nil
+		}, InType: reflect.TypeOf(&KubeProxyIPVSConfiguration{})},
+		conversion.GeneratedDeepCopyFunc{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error {
 			in.(*KubeSchedulerConfiguration).DeepCopyInto(out.(*KubeSchedulerConfiguration))
 			return nil
 		}, InType: reflect.TypeOf(&KubeSchedulerConfiguration{})},
@@ -85,6 +89,7 @@ func (in *KubeProxyConfiguration) DeepCopyInto(out *KubeProxyConfiguration) {
 	out.TypeMeta = in.TypeMeta
 	out.ClientConnection = in.ClientConnection
 	in.IPTables.DeepCopyInto(&out.IPTables)
+	out.IPVS = in.IPVS
 	if in.OOMScoreAdj != nil {
 		in, out := &in.OOMScoreAdj, &out.OOMScoreAdj
 		if *in == nil {
@@ -165,6 +170,24 @@ func (in *KubeProxyIPTablesConfiguration) DeepCopy() *KubeProxyIPTablesConfigura
 }
 
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *KubeProxyIPVSConfiguration) DeepCopyInto(out *KubeProxyIPVSConfiguration) {
+	*out = *in
+	out.SyncPeriod = in.SyncPeriod
+	out.MinSyncPeriod = in.MinSyncPeriod
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeProxyIPVSConfiguration.
+func (in *KubeProxyIPVSConfiguration) DeepCopy() *KubeProxyIPVSConfiguration {
+	if in == nil {
+		return nil
+	}
+	out := new(KubeProxyIPVSConfiguration)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *KubeSchedulerConfiguration) DeepCopyInto(out *KubeSchedulerConfiguration) {
 	*out = *in
 	out.TypeMeta = in.TypeMeta

pkg/apis/componentconfig/zz_generated.deepcopy.go

@@ -65,6 +65,10 @@ func RegisterDeepCopies(scheme *runtime.Scheme) error {
 			return nil
 		}, InType: reflect.TypeOf(&KubeProxyIPTablesConfiguration{})},
 		conversion.GeneratedDeepCopyFunc{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error {
+			in.(*KubeProxyIPVSConfiguration).DeepCopyInto(out.(*KubeProxyIPVSConfiguration))
+			return nil
+		}, InType: reflect.TypeOf(&KubeProxyIPVSConfiguration{})},
+		conversion.GeneratedDeepCopyFunc{Fn: func(in interface{}, out interface{}, c *conversion.Cloner) error {
 			in.(*KubeSchedulerConfiguration).DeepCopyInto(out.(*KubeSchedulerConfiguration))
 			return nil
 		}, InType: reflect.TypeOf(&KubeSchedulerConfiguration{})},
@@ -206,6 +210,7 @@ func (in *KubeProxyConfiguration) DeepCopyInto(out *KubeProxyConfiguration) {
 	out.TypeMeta = in.TypeMeta
 	out.ClientConnection = in.ClientConnection
 	in.IPTables.DeepCopyInto(&out.IPTables)
+	out.IPVS = in.IPVS
 	if in.OOMScoreAdj != nil {
 		in, out := &in.OOMScoreAdj, &out.OOMScoreAdj
 		if *in == nil {
@@ -286,6 +291,24 @@ func (in *KubeProxyIPTablesConfiguration) DeepCopy() *KubeProxyIPTablesConfigura
 }
 
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *KubeProxyIPVSConfiguration) DeepCopyInto(out *KubeProxyIPVSConfiguration) {
+	*out = *in
+	out.SyncPeriod = in.SyncPeriod
+	out.MinSyncPeriod = in.MinSyncPeriod
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeProxyIPVSConfiguration.
+func (in *KubeProxyIPVSConfiguration) DeepCopy() *KubeProxyIPVSConfiguration {
+	if in == nil {
+		return nil
+	}
+	out := new(KubeProxyIPVSConfiguration)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *KubeSchedulerConfiguration) DeepCopyInto(out *KubeSchedulerConfiguration) {
 	*out = *in
 	out.TypeMeta = in.TypeMeta

pkg/features/kube_features.go

@@ -127,6 +127,12 @@ const (
 	// Taint nodes based on their condition status for 'NetworkUnavailable',
 	// 'MemoryPressure', 'OutOfDisk' and 'DiskPressure'.
 	TaintNodesByCondition utilfeature.Feature = "TaintNodesByCondition"
+
+	// owner: @haibinxie
+	// alpha: v1.8
+	//
+	// Implement IPVS-based in-cluster service load balancing
+	SupportIPVSProxyMode utilfeature.Feature = "SupportIPVSProxyMode"
 )
 
 func init() {
@@ -164,4 +170,5 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS
 	// inherited features from apiextensions-apiserver, relisted here to get a conflict if it is changed
 	// unintentionally on either side:
 	apiextensionsfeatures.CustomResourceValidation: {Default: false, PreRelease: utilfeature.Alpha},
+	SupportIPVSProxyMode:                           {Default: false, PreRelease: utilfeature.Alpha},
 }

pkg/proxy/BUILD

@@ -28,6 +28,7 @@ filegroup(
         "//pkg/proxy/config:all-srcs",
         "//pkg/proxy/healthcheck:all-srcs",
         "//pkg/proxy/iptables:all-srcs",
+        "//pkg/proxy/ipvs:all-srcs",
         "//pkg/proxy/userspace:all-srcs",
         "//pkg/proxy/util:all-srcs",
         "//pkg/proxy/winuserspace:all-srcs",

pkg/proxy/ipvs/BUILD

+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = select({
+        "@io_bazel_rules_go//go/platform:linux_amd64": [
+            "proxier_test.go",
+        ],
+        "//conditions:default": [],
+    }),
+    library = ":go_default_library",
+    tags = ["automanaged"],
+    deps = select({
+        "@io_bazel_rules_go//go/platform:linux_amd64": [
+            "//pkg/api:go_default_library",
+            "//pkg/proxy:go_default_library",
+            "//pkg/proxy/util:go_default_library",
+            "//pkg/util/iptables:go_default_library",
+            "//pkg/util/iptables/testing:go_default_library",
+            "//pkg/util/ipvs:go_default_library",
+            "//pkg/util/ipvs/testing:go_default_library",
+            "//vendor/github.com/davecgh/go-spew/spew:go_default_library",
+            "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+            "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+            "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
+            "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+            "//vendor/k8s.io/utils/exec:go_default_library",
+            "//vendor/k8s.io/utils/exec/testing:go_default_library",
+        ],
+        "//conditions:default": [],
+    }),
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["proxier.go"],
+    tags = ["automanaged"],
+    deps = [
+        "//pkg/api:go_default_library",
+        "//pkg/api/helper:go_default_library",
+        "//pkg/api/service:go_default_library",
+        "//pkg/features:go_default_library",
+        "//pkg/proxy:go_default_library",
+        "//pkg/proxy/healthcheck:go_default_library",
+        "//pkg/proxy/util:go_default_library",
+        "//pkg/util/iptables:go_default_library",
+        "//pkg/util/ipvs:go_default_library",
+        "//pkg/util/sysctl:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
+        "//vendor/k8s.io/client-go/tools/record:go_default_library",
+        "//vendor/k8s.io/client-go/util/flowcontrol:go_default_library",
+        "//vendor/k8s.io/utils/exec:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

vendor/github.com/docker/libnetwork/ipvs/BUILD

@@ -4,18 +4,21 @@ go_library(
     name = "go_default_library",
     srcs = select({
         "@io_bazel_rules_go//go/platform:linux_amd64": [
-            "addr_linux.go",
-            "link_linux.go",
-            "nl_linux.go",
-            "route_linux.go",
-            "tc_linux.go",
-            "xfrm_linux.go",
-            "xfrm_policy_linux.go",
-            "xfrm_state_linux.go",
+            "constants.go",
+            "ipvs.go",
+            "netlink.go",
         ],
         "//conditions:default": [],
     }),
     visibility = ["//visibility:public"],
+    deps = select({
+        "@io_bazel_rules_go//go/platform:linux_amd64": [
+            "//vendor/github.com/Sirupsen/logrus:go_default_library",
+            "//vendor/github.com/vishvananda/netlink/nl:go_default_library",
+            "//vendor/github.com/vishvananda/netns:go_default_library",
+        ],
+        "//conditions:default": [],
+    }),
 )
 
 filegroup(