Scale kube-proxy conntrack limits by cores

cmd/kube-proxy/app/options/options.go

@@ -83,6 +83,9 @@ func (s *ProxyServerConfig) AddFlags(fs *pflag.FlagSet) {
 	fs.Float32Var(&s.KubeAPIQPS, "kube-api-qps", s.KubeAPIQPS, "QPS to use while talking with kubernetes apiserver")
 	fs.Int32Var(&s.KubeAPIBurst, "kube-api-burst", s.KubeAPIBurst, "Burst to use while talking with kubernetes apiserver")
 	fs.DurationVar(&s.UDPIdleTimeout.Duration, "udp-timeout", s.UDPIdleTimeout.Duration, "How long an idle UDP connection will be kept open (e.g. '250ms', '2s').  Must be greater than 0. Only applicable for proxy-mode=userspace")
-	fs.Int32Var(&s.ConntrackMax, "conntrack-max", s.ConntrackMax, "Maximum number of NAT connections to track (0 to leave as-is)")
+	fs.Int32Var(&s.ConntrackMax, "conntrack-max", s.ConntrackMax,
+		"Maximum number of NAT connections to track (0 to leave as-is).")
+	fs.Int32Var(&s.ConntrackMaxPerCore, "conntrack-max-per-core", s.ConntrackMaxPerCore,
+		"Maximum number of NAT connections to track per CPU core (0 to leave as-is). This is only considered if conntrack-max is 0.")
 	fs.DurationVar(&s.ConntrackTCPEstablishedTimeout.Duration, "conntrack-tcp-timeout-established", s.ConntrackTCPEstablishedTimeout.Duration, "Idle timeout for established TCP connections (0 to leave as-is)")
 }

cmd/kube-proxy/app/server.go

@@ -24,6 +24,7 @@ import (
 	"net"
 	"net/http"
 	_ "net/http/pprof"
+	"runtime"
 	"strconv"
 	"time"
 
@@ -298,8 +299,12 @@ func (s *ProxyServer) Run() error {
 
 	// Tune conntrack, if requested
 	if s.Conntracker != nil {
-		if s.Config.ConntrackMax > 0 {
-			err := s.Conntracker.SetMax(int(s.Config.ConntrackMax))
+		max, err := getConntrackMax(s.Config)
+		if err != nil {
+			return err
+		}
+		if max > 0 {
+			err := s.Conntracker.SetMax(max)
 			if err != nil {
 				if err != readOnlySysFSError {
 					return err
@@ -329,6 +334,18 @@ func (s *ProxyServer) Run() error {
 	return nil
 }
 
+func getConntrackMax(config *options.ProxyServerConfig) (int, error) {
+	if config.ConntrackMax > 0 && config.ConntrackMaxPerCore > 0 {
+		return -1, fmt.Errorf("invalid config: ConntrackMax and ConntrackMaxPerCore are mutually exclusive")
+	}
+	if config.ConntrackMax > 0 {
+		return int(config.ConntrackMax), nil
+	} else if config.ConntrackMaxPerCore > 0 {
+		return (int(config.ConntrackMaxPerCore) * runtime.NumCPU()), nil
+	}
+	return 0, nil
+}
+
 type nodeGetter interface {
 	Get(hostname string) (*api.Node, error)
 }

cmd/kube-proxy/app/server_test.go

@@ -19,12 +19,14 @@ package app
 import (
 	"fmt"
 	"runtime"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 
 	"k8s.io/kubernetes/cmd/kube-proxy/app/options"
 	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/apis/componentconfig"
 	"k8s.io/kubernetes/pkg/util/iptables"
 )
 
@@ -268,7 +270,6 @@ func Test_getProxyMode(t *testing.T) {
 // Config and iptinterface are not nil when CleanupAndExit is true.
 // To avoid proxy crash: https://github.com/kubernetes/kubernetes/pull/14736
 func TestProxyServerWithCleanupAndExit(t *testing.T) {
-
 	// creates default config
 	config := options.NewProxyConfig()
 
@@ -284,3 +285,51 @@ func TestProxyServerWithCleanupAndExit(t *testing.T) {
 	assert.NotNil(t, proxyserver.Config)
 	assert.NotNil(t, proxyserver.IptInterface)
 }
+
+func TestGetConntrackMax(t *testing.T) {
+	ncores := runtime.NumCPU()
+	testCases := []struct {
+		config   componentconfig.KubeProxyConfiguration
+		expected int
+		err      string
+	}{
+		{
+			config:   componentconfig.KubeProxyConfiguration{},
+			expected: 0,
+		},
+		{
+			config: componentconfig.KubeProxyConfiguration{
+				ConntrackMax: 12345,
+			},
+			expected: 12345,
+		},
+		{
+			config: componentconfig.KubeProxyConfiguration{
+				ConntrackMax:        12345,
+				ConntrackMaxPerCore: 67890,
+			},
+			expected: -1,
+			err:      "mutually exclusive",
+		},
+		{
+			config: componentconfig.KubeProxyConfiguration{
+				ConntrackMaxPerCore: 67890, // use this if other is 0
+			},
+			expected: 67890 * ncores,
+		},
+	}
+
+	for i, tc := range testCases {
+		cfg := options.ProxyServerConfig{KubeProxyConfiguration: tc.config}
+		x, e := getConntrackMax(&cfg)
+		if e != nil {
+			if tc.err == "" {
+				t.Errorf("[%d] unexpected error: %v", i, e)
+			} else if !strings.Contains(e.Error(), tc.err) {
+				t.Errorf("[%d] expected an error containing %q: %v", i, tc.err, e)
+			}
+		} else if x != tc.expected {
+			t.Errorf("[%d] expected %d, got %d", i, tc.expected, x)
+		}
+	}
+}

hack/verify-flags/known-flags.txt

@@ -74,6 +74,7 @@ config-sync-period
 configure-cbr0
 configure-cloud-routes
 conntrack-max
+conntrack-max-per-core
 conntrack-tcp-timeout-established
 consumer-port
 consumer-service-name

pkg/apis/componentconfig/types.go

@@ -65,10 +65,15 @@ type KubeProxyConfiguration struct {
 	// udpIdleTimeout is how long an idle UDP connection will be kept open (e.g. '250ms', '2s').
 	// Must be greater than 0. Only applicable for proxyMode=userspace.
 	UDPIdleTimeout unversioned.Duration `json:"udpTimeoutMilliseconds"`
-	// conntrackMax is the maximum number of NAT connections to track (0 to leave as-is)")
+	// conntrackMax is the maximum number of NAT connections to track (0 to
+	// leave as-is).  This takes precendence over conntrackMaxPerCore.
 	ConntrackMax int32 `json:"conntrackMax"`
-	// conntrackTCPEstablishedTimeout is how long an idle UDP connection will be kept open
-	// (e.g. '250ms', '2s').  Must be greater than 0. Only applicable for proxyMode is Userspace
+	// conntrackMaxPerCore is the maximum number of NAT connections to track
+	// per CPU core (0 to leave as-is).  This value is only considered if
+	// conntrackMax == 0.
+	ConntrackMaxPerCore int32 `json:"conntrackMaxPerCore"`
+	// conntrackTCPEstablishedTimeout is how long an idle TCP connection will be kept open
+	// (e.g. '250ms', '2s').  Must be greater than 0.
 	ConntrackTCPEstablishedTimeout unversioned.Duration `json:"conntrackTCPEstablishedTimeout"`
 }
 

pkg/apis/componentconfig/v1alpha1/defaults.go

@@ -80,8 +80,12 @@ func SetDefaults_KubeProxyConfiguration(obj *KubeProxyConfiguration) {
 	if obj.UDPIdleTimeout == zero {
 		obj.UDPIdleTimeout = unversioned.Duration{Duration: 250 * time.Millisecond}
 	}
+	// If ConntrackMax is set, respect it.
 	if obj.ConntrackMax == 0 {
-		obj.ConntrackMax = 256 * 1024 // 4x default (64k)
+		// If ConntrackMax is *not* set, use per-core scaling.
+		if obj.ConntrackMaxPerCore == 0 {
+			obj.ConntrackMaxPerCore = 32 * 1024
+		}
 	}
 	if obj.IPTablesMasqueradeBit == nil {
 		temp := int32(14)

pkg/apis/componentconfig/v1alpha1/types.go

@@ -62,10 +62,15 @@ type KubeProxyConfiguration struct {
 	// udpIdleTimeout is how long an idle UDP connection will be kept open (e.g. '250ms', '2s').
 	// Must be greater than 0. Only applicable for proxyMode=userspace.
 	UDPIdleTimeout unversioned.Duration `json:"udpTimeoutMilliseconds"`
-	// conntrackMax is the maximum number of NAT connections to track (0 to leave as-is)")
+	// conntrackMax is the maximum number of NAT connections to track (0 to
+	// leave as-is).  This takes precendence over conntrackMaxPerCore.
 	ConntrackMax int32 `json:"conntrackMax"`
-	// conntrackTCPEstablishedTimeout is how long an idle UDP connection will be kept open
-	// (e.g. '250ms', '2s').  Must be greater than 0. Only applicable for proxyMode is Userspace
+	// conntrackMaxPerCore is the maximum number of NAT connections to track
+	// per CPU core (0 to leave as-is).  This value is only considered if
+	// conntrackMax == 0.
+	ConntrackMaxPerCore int32 `json:"conntrackMaxPerCore"`
+	// conntrackTCPEstablishedTimeout is how long an idle TCP connection will be kept open
+	// (e.g. '250ms', '2s').  Must be greater than 0.
 	ConntrackTCPEstablishedTimeout unversioned.Duration `json:"conntrackTCPEstablishedTimeout"`
 }
 

pkg/apis/componentconfig/v1alpha1/zz_generated.conversion.go

@@ -64,6 +64,7 @@ func autoConvert_v1alpha1_KubeProxyConfiguration_To_componentconfig_KubeProxyCon
 	out.ResourceContainer = in.ResourceContainer
 	out.UDPIdleTimeout = in.UDPIdleTimeout
 	out.ConntrackMax = in.ConntrackMax
+	out.ConntrackMaxPerCore = in.ConntrackMaxPerCore
 	out.ConntrackTCPEstablishedTimeout = in.ConntrackTCPEstablishedTimeout
 	return nil
 }
@@ -92,6 +93,7 @@ func autoConvert_componentconfig_KubeProxyConfiguration_To_v1alpha1_KubeProxyCon
 	out.ResourceContainer = in.ResourceContainer
 	out.UDPIdleTimeout = in.UDPIdleTimeout
 	out.ConntrackMax = in.ConntrackMax
+	out.ConntrackMaxPerCore = in.ConntrackMaxPerCore
 	out.ConntrackTCPEstablishedTimeout = in.ConntrackTCPEstablishedTimeout
 	return nil
 }
@@ -234,6 +236,9 @@ func autoConvert_v1alpha1_KubeletConfiguration_To_componentconfig_KubeletConfigu
 	out.RuntimeCgroups = in.RuntimeCgroups
 	out.SystemCgroups = in.SystemCgroups
 	out.CgroupRoot = in.CgroupRoot
+	if err := api.Convert_Pointer_bool_To_bool(&in.CgroupsPerQOS, &out.CgroupsPerQOS, s); err != nil {
+		return err
+	}
 	out.ContainerRuntime = in.ContainerRuntime
 	out.RuntimeRequestTimeout = in.RuntimeRequestTimeout
 	out.RktPath = in.RktPath
@@ -391,6 +396,9 @@ func autoConvert_componentconfig_KubeletConfiguration_To_v1alpha1_KubeletConfigu
 	out.CloudProvider = in.CloudProvider
 	out.CloudConfigFile = in.CloudConfigFile
 	out.KubeletCgroups = in.KubeletCgroups
+	if err := api.Convert_bool_To_Pointer_bool(&in.CgroupsPerQOS, &out.CgroupsPerQOS, s); err != nil {
+		return err
+	}
 	out.RuntimeCgroups = in.RuntimeCgroups
 	out.SystemCgroups = in.SystemCgroups
 	out.CgroupRoot = in.CgroupRoot

pkg/apis/componentconfig/v1alpha1/zz_generated.deepcopy.go

@@ -71,6 +71,7 @@ func DeepCopy_v1alpha1_KubeProxyConfiguration(in interface{}, out interface{}, c
 		out.ResourceContainer = in.ResourceContainer
 		out.UDPIdleTimeout = in.UDPIdleTimeout
 		out.ConntrackMax = in.ConntrackMax
+		out.ConntrackMaxPerCore = in.ConntrackMaxPerCore
 		out.ConntrackTCPEstablishedTimeout = in.ConntrackTCPEstablishedTimeout
 		return nil
 	}
@@ -242,6 +243,13 @@ func DeepCopy_v1alpha1_KubeletConfiguration(in interface{}, out interface{}, c *
 		out.RuntimeCgroups = in.RuntimeCgroups
 		out.SystemCgroups = in.SystemCgroups
 		out.CgroupRoot = in.CgroupRoot
+		if in.CgroupsPerQOS != nil {
+			in, out := &in.CgroupsPerQOS, &out.CgroupsPerQOS
+			*out = new(bool)
+			**out = **in
+		} else {
+			out.CgroupsPerQOS = nil
+		}
 		out.ContainerRuntime = in.ContainerRuntime
 		out.RuntimeRequestTimeout = in.RuntimeRequestTimeout
 		out.RktPath = in.RktPath

pkg/apis/componentconfig/zz_generated.deepcopy.go

@@ -150,6 +150,7 @@ func DeepCopy_componentconfig_KubeProxyConfiguration(in interface{}, out interfa
 		out.ResourceContainer = in.ResourceContainer
 		out.UDPIdleTimeout = in.UDPIdleTimeout
 		out.ConntrackMax = in.ConntrackMax
+		out.ConntrackMaxPerCore = in.ConntrackMaxPerCore
 		out.ConntrackTCPEstablishedTimeout = in.ConntrackTCPEstablishedTimeout
 		return nil
 	}
@@ -250,6 +251,7 @@ func DeepCopy_componentconfig_KubeletConfiguration(in interface{}, out interface
 		out.CloudProvider = in.CloudProvider
 		out.CloudConfigFile = in.CloudConfigFile
 		out.KubeletCgroups = in.KubeletCgroups
+		out.CgroupsPerQOS = in.CgroupsPerQOS
 		out.RuntimeCgroups = in.RuntimeCgroups
 		out.SystemCgroups = in.SystemCgroups
 		out.CgroupRoot = in.CgroupRoot