Merge pull request #4450 from olljanat/support-ipv6-only

pkg/agent/config/config.go

@@ -425,7 +425,6 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 	nodeConfig.AgentConfig.ClusterDomain = controlConfig.ClusterDomain
 	nodeConfig.AgentConfig.ResolvConf = locateOrGenerateResolvConf(envInfo)
 	nodeConfig.AgentConfig.ClientCA = clientCAFile
-	nodeConfig.AgentConfig.ListenAddress = "0.0.0.0"
 	nodeConfig.AgentConfig.KubeConfigKubelet = kubeconfigKubelet
 	nodeConfig.AgentConfig.KubeConfigKubeProxy = kubeconfigKubeproxy
 	nodeConfig.AgentConfig.KubeConfigK3sController = kubeconfigK3sController
@@ -466,18 +465,20 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 	nodeConfig.Certificate = servingCert
 
 	nodeConfig.AgentConfig.NodeIPs = nodeIPs
-	nodeIP, err := util.GetFirst4(nodeIPs)
+	nodeIP, listenAddress, _, err := util.GetFirstIP(nodeIPs)
 	if err != nil {
-		return nil, errors.Wrap(err, "cannot configure IPv4 node-ip")
+		return nil, errors.Wrap(err, "cannot configure IPv4/IPv6 node-ip")
 	}
 	nodeConfig.AgentConfig.NodeIP = nodeIP.String()
+	nodeConfig.AgentConfig.ListenAddress = listenAddress
 	nodeConfig.AgentConfig.NodeExternalIPs = nodeExternalIPs
 
 	// if configured, set NodeExternalIP to the first IPv4 address, for legacy clients
+	// unless only IPv6 address given
 	if len(nodeConfig.AgentConfig.NodeExternalIPs) > 0 {
-		nodeExternalIP, err := util.GetFirst4(nodeConfig.AgentConfig.NodeExternalIPs)
+		nodeExternalIP, _, _, err := util.GetFirstIP(nodeConfig.AgentConfig.NodeExternalIPs)
 		if err != nil {
-			return nil, errors.Wrap(err, "cannot configure IPv4 node-external-ip")
+			return nil, errors.Wrap(err, "cannot configure IPv4/IPv6 node-external-ip")
 		}
 		nodeConfig.AgentConfig.NodeExternalIP = nodeExternalIP.String()
 	}

pkg/agent/run.go

@@ -59,8 +59,10 @@ func run(ctx context.Context, cfg cmds.Agent, proxy proxy.Proxy) error {
 	if err != nil {
 		return errors.Wrap(err, "failed to validate node-ip")
 	}
+	serviceIPv6 := utilsnet.IsIPv6CIDR(nodeConfig.AgentConfig.ServiceCIDR)
+	clusterIPv6 := utilsnet.IsIPv6CIDR(nodeConfig.AgentConfig.ClusterCIDR)
 
-	enableIPv6 := dualCluster || dualService || dualNode
+	enableIPv6 := dualCluster || dualService || dualNode || serviceIPv6 || clusterIPv6
 	conntrackConfig, err := getConntrackConfig(nodeConfig)
 	if err != nil {
 		return errors.Wrap(err, "failed to validate kube-proxy conntrack configuration")

pkg/cli/server/server.go

@@ -200,17 +200,17 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
 
 	if serverConfig.ControlConfig.PrivateIP == "" && len(cmds.AgentConfig.NodeIP) != 0 {
 		// ignoring the error here is fine since etcd will fall back to the interface's IPv4 address
-		serverConfig.ControlConfig.PrivateIP, _ = util.GetFirst4String(cmds.AgentConfig.NodeIP)
+		serverConfig.ControlConfig.PrivateIP, _, _ = util.GetFirstString(cmds.AgentConfig.NodeIP)
 	}
 
 	// if not set, try setting advertise-ip from agent node-external-ip
 	if serverConfig.ControlConfig.AdvertiseIP == "" && len(cmds.AgentConfig.NodeExternalIP) != 0 {
-		serverConfig.ControlConfig.AdvertiseIP, _ = util.GetFirst4String(cmds.AgentConfig.NodeExternalIP)
+		serverConfig.ControlConfig.AdvertiseIP, _, _ = util.GetFirstString(cmds.AgentConfig.NodeExternalIP)
 	}
 
 	// if not set, try setting advertise-ip from agent node-ip
 	if serverConfig.ControlConfig.AdvertiseIP == "" && len(cmds.AgentConfig.NodeIP) != 0 {
-		serverConfig.ControlConfig.AdvertiseIP, _ = util.GetFirst4String(cmds.AgentConfig.NodeIP)
+		serverConfig.ControlConfig.AdvertiseIP, _, _ = util.GetFirstString(cmds.AgentConfig.NodeIP)
 	}
 
 	// if we ended up with any advertise-ips, ensure they're added to the SAN list;
@@ -228,14 +228,19 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
 		return err
 	}
 	serverConfig.ControlConfig.ServerNodeName = nodeName
-	serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, "127.0.0.1", "localhost", nodeName)
+	serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, "127.0.0.1", "::1", "localhost", nodeName)
 	for _, ip := range nodeIPs {
 		serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, ip.String())
 	}
 
 	// configure ClusterIPRanges
+	_, _, IPv6only, _ := util.GetFirstIP(nodeIPs)
 	if len(cmds.ServerConfig.ClusterCIDR) == 0 {
-		cmds.ServerConfig.ClusterCIDR.Set("10.42.0.0/16")
+		clusterCIDR := "10.42.0.0/16"
+		if IPv6only {
+			clusterCIDR = "fd:42::/56"
+		}
+		cmds.ServerConfig.ClusterCIDR.Set(clusterCIDR)
 	}
 	for _, cidr := range cmds.ServerConfig.ClusterCIDR {
 		for _, v := range strings.Split(cidr, ",") {
@@ -248,15 +253,20 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
 	}
 
 	// set ClusterIPRange to the first IPv4 block, for legacy clients
-	clusterIPRange, err := util.GetFirst4Net(serverConfig.ControlConfig.ClusterIPRanges)
+	// unless only IPv6 range given
+	clusterIPRange, err := util.GetFirstNet(serverConfig.ControlConfig.ClusterIPRanges)
 	if err != nil {
-		return errors.Wrap(err, "cannot configure IPv4 cluster-cidr")
+		return errors.Wrap(err, "cannot configure IPv4/IPv6 cluster-cidr")
 	}
 	serverConfig.ControlConfig.ClusterIPRange = clusterIPRange
 
 	// configure ServiceIPRanges
 	if len(cmds.ServerConfig.ServiceCIDR) == 0 {
-		cmds.ServerConfig.ServiceCIDR.Set("10.43.0.0/16")
+		serviceCIDR := "10.43.0.0/16"
+		if IPv6only {
+			serviceCIDR = "fd:43::/112"
+		}
+		cmds.ServerConfig.ServiceCIDR.Set(serviceCIDR)
 	}
 	for _, cidr := range cmds.ServerConfig.ServiceCIDR {
 		for _, v := range strings.Split(cidr, ",") {
@@ -269,9 +279,10 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
 	}
 
 	// set ServiceIPRange to the first IPv4 block, for legacy clients
-	serviceIPRange, err := util.GetFirst4Net(serverConfig.ControlConfig.ServiceIPRanges)
+	// unless only IPv6 range given
+	serviceIPRange, err := util.GetFirstNet(serverConfig.ControlConfig.ServiceIPRanges)
 	if err != nil {
-		return errors.Wrap(err, "cannot configure IPv4 service-cidr")
+		return errors.Wrap(err, "cannot configure IPv4/IPv6 service-cidr")
 	}
 	serverConfig.ControlConfig.ServiceIPRange = serviceIPRange
 
@@ -289,7 +300,8 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
 
 	// If cluster-dns CLI arg is not set, we set ClusterDNS address to be the first IPv4 ServiceCIDR network + 10,
 	// i.e. when you set service-cidr to 192.168.0.0/16 and don't provide cluster-dns, it will be set to 192.168.0.10
-	// If there are no IPv4 ServiceCIDRs, an error will be raised.
+	// If there are no IPv4 ServiceCIDRs, an IPv6 ServiceCIDRs will be used.
+	// If neither of IPv4 or IPv6 are found an error is raised.
 	if len(cmds.ServerConfig.ClusterDNS) == 0 {
 		clusterDNS, err := utilsnet.GetIndexedIP(serverConfig.ControlConfig.ServiceIPRange, 10)
 		if err != nil {
@@ -308,9 +320,10 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
 			}
 		}
 		// Set ClusterDNS to the first IPv4 address, for legacy clients
-		clusterDNS, err := util.GetFirst4(serverConfig.ControlConfig.ClusterDNSs)
+		// unless only IPv6 range given
+		clusterDNS, _, _, err := util.GetFirstIP(serverConfig.ControlConfig.ClusterDNSs)
 		if err != nil {
-			return errors.Wrap(err, "cannot configure IPv4 cluster-dns address")
+			return errors.Wrap(err, "cannot configure IPv4/IPv6 cluster-dns address")
 		}
 		serverConfig.ControlConfig.ClusterDNS = clusterDNS
 	}
@@ -446,6 +459,9 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
 	ip := serverConfig.ControlConfig.BindAddress
 	if ip == "" {
 		ip = "127.0.0.1"
+		if IPv6only {
+			ip = "[::1]"
+		}
 	}
 
 	url := fmt.Sprintf("https://%s:%d", ip, serverConfig.ControlConfig.SupervisorPort)
@@ -513,6 +529,16 @@ func validateNetworkConfiguration(serverConfig server.Config) error {
 		return errors.New("dual-stack cluster-dns is not supported")
 	}
 
+	IPv6OnlyService, _ := util.IsIPv6OnlyCIDRs(serverConfig.ControlConfig.ServiceIPRanges)
+	if IPv6OnlyService {
+		if serverConfig.ControlConfig.DisableNPC == false {
+			return errors.New("network policy enforcement is not compatible with IPv6 only operation; server must be restarted with --disable-network-policy")
+		}
+		if serverConfig.ControlConfig.FlannelBackend != config.FlannelBackendNone {
+			return errors.New("Flannel is not compatible with IPv6 only operation; server must be restarted with --flannel-backend=none")
+		}
+	}
+
 	return nil
 }
 

pkg/daemons/agent/agent_linux.go

@@ -40,9 +40,14 @@ func checkRuntimeEndpoint(cfg *config.Agent, argsMap map[string]string) {
 }
 
 func kubeProxyArgs(cfg *config.Agent) map[string]string {
+	bindAddress := "127.0.0.1"
+	_, IPv6only, _ := util.GetFirstString([]string{cfg.NodeIP})
+	if IPv6only {
+		bindAddress = "::1"
+	}
 	argsMap := map[string]string{
 		"proxy-mode":                        "iptables",
-		"healthz-bind-address":              "127.0.0.1",
+		"healthz-bind-address":              bindAddress,
 		"kubeconfig":                        cfg.KubeConfigKubeProxy,
 		"cluster-cidr":                      util.JoinIPNets(cfg.ClusterCIDRs),
 		"conntrack-max-per-core":            "0",
@@ -56,8 +61,13 @@ func kubeProxyArgs(cfg *config.Agent) map[string]string {
 }
 
 func kubeletArgs(cfg *config.Agent) map[string]string {
+	bindAddress := "127.0.0.1"
+	_, IPv6only, _ := util.GetFirstString([]string{cfg.NodeIP})
+	if IPv6only {
+		bindAddress = "::1"
+	}
 	argsMap := map[string]string{
-		"healthz-bind-address":     "127.0.0.1",
+		"healthz-bind-address":     bindAddress,
 		"read-only-port":           "0",
 		"cluster-domain":           cfg.ClusterDomain,
 		"kubeconfig":               cfg.KubeConfigKubelet,

pkg/daemons/agent/agent_windows.go

@@ -28,9 +28,14 @@ func checkRuntimeEndpoint(cfg *config.Agent, argsMap map[string]string) {
 }
 
 func kubeProxyArgs(cfg *config.Agent) map[string]string {
+	bindAddress := "127.0.0.1"
+	_, IPv6only, _ := util.GetFirstString(cfg.NodeIP)
+	if IPv6only {
+		bindAddress = "::1"
+	}
 	argsMap := map[string]string{
 		"proxy-mode":           "kernelspace",
-		"healthz-bind-address": "127.0.0.1",
+		"healthz-bind-address": bindAddress,
 		"kubeconfig":           cfg.KubeConfigKubeProxy,
 		"cluster-cidr":         util.JoinIPNets(cfg.ClusterCIDRs),
 	}
@@ -46,8 +51,13 @@ func kubeProxyArgs(cfg *config.Agent) map[string]string {
 }
 
 func kubeletArgs(cfg *config.Agent) map[string]string {
+	bindAddress := "127.0.0.1"
+	_, IPv6only, _ := util.GetFirstString(cfg.NodeIP)
+	if IPv6only {
+		bindAddress = "::1"
+	}
 	argsMap := map[string]string{
-		"healthz-bind-address":         "127.0.0.1",
+		"healthz-bind-address":         bindAddress,
 		"read-only-port":               "0",
 		"cluster-domain":               cfg.ClusterDomain,
 		"kubeconfig":                   cfg.KubeConfigKubelet,

pkg/util/net.go

@@ -80,6 +80,46 @@ func JoinIP4Nets(elems []*net.IPNet) string {
 	return strings.Join(strs, ",")
 }
 
+// GetFirst6 returns the first IPv6 address from the list of IP addresses.
+// If no IPv6 addresses are found, an error is raised.
+func GetFirst6(elems []net.IP) (net.IP, error) {
+	for _, elem := range elems {
+		if elem == nil || elem.To16() == nil {
+			continue
+		}
+		return elem, nil
+	}
+	return nil, errors.New("no IPv6 address found")
+}
+
+// GetFirst6Net returns the first IPv4 network from the list of IP networks.
+// If no IPv6 addresses are found, an error is raised.
+func GetFirst6Net(elems []*net.IPNet) (*net.IPNet, error) {
+	for _, elem := range elems {
+		if elem == nil || elem.IP.To16() == nil {
+			continue
+		}
+		return elem, nil
+	}
+	return nil, errors.New("no IPv6 CIDRs found")
+}
+
+// GetFirst6String returns the first IPv6 address from a list of IP address strings.
+// If no IPv6 addresses are found, an error is raised.
+func GetFirst6String(elems []string) (string, error) {
+	ips := []net.IP{}
+	for _, elem := range elems {
+		for _, v := range strings.Split(elem, ",") {
+			ips = append(ips, net.ParseIP(v))
+		}
+	}
+	ip, err := GetFirst6(ips)
+	if err != nil {
+		return "", err
+	}
+	return ip.String(), nil
+}
+
 // JoinIP6Nets stringifies and joins a list of IPv6 networks with commas.
 func JoinIP6Nets(elems []*net.IPNet) string {
 	var strs []string
@@ -141,3 +181,78 @@ func ParseStringSliceToIPs(s cli.StringSlice) ([]net.IP, error) {
 
 	return ips, nil
 }
+
+// GetFirstIP returns the first IPv4 address from the list of IP addresses.
+// If no IPv4 addresses are found, returns the first IPv6 address
+// if neither of IPv4 or IPv6 are found an error is raised.
+// Additionally matching listen address and IP version is returned.
+func GetFirstIP(nodeIPs []net.IP) (net.IP, string, bool, error) {
+	nodeIP, err := GetFirst4(nodeIPs)
+	ListenAddress := "0.0.0.0"
+	IPv6only := false
+	if err != nil {
+		nodeIP, err = GetFirst6(nodeIPs)
+		if err != nil {
+			return nil, "", false, err
+		}
+		ListenAddress = "::"
+		IPv6only = true
+	}
+	return nodeIP, ListenAddress, IPv6only, nil
+}
+
+// GetFirstNet returns the first IPv4 network from the list of IP networks.
+// If no IPv4 addresses are found, returns the first IPv6 address
+// if neither of IPv4 or IPv6 are found an error is raised.
+func GetFirstNet(elems []*net.IPNet) (*net.IPNet, error) {
+	serviceIPRange, err := GetFirst4Net(elems)
+	if err != nil {
+		serviceIPRange, err = GetFirst6Net(elems)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return serviceIPRange, nil
+}
+
+// GetFirstString returns the first IP4 address from a list of IP address strings.
+// If no IPv4 addresses are found, returns the first IPv6 address
+// if neither of IPv4 or IPv6 are found an error is raised.
+func GetFirstString(elems []string) (string, bool, error) {
+	ip, err := GetFirst4String(elems)
+	IPv6only := false
+	if err != nil {
+		ip, err = GetFirst6String(elems)
+		if err != nil {
+			return "", false, err
+		}
+		IPv6only = true
+	}
+	return ip, IPv6only, nil
+}
+
+// IsIPv6OnlyCIDRs returns if
+// - all are valid cidrs
+// - at least one cidr from v6 family is found
+// - v4 family cidr is not found
+func IsIPv6OnlyCIDRs(cidrs []*net.IPNet) (bool, error) {
+	v4Found := false
+	v6Found := false
+	for _, cidr := range cidrs {
+		if cidr == nil {
+			return false, fmt.Errorf("cidr %v is invalid", cidr)
+		}
+
+		if v4Found && v6Found {
+			continue
+		}
+
+		if cidr.IP != nil && cidr.IP.To4() == nil {
+			v6Found = true
+			continue
+		}
+		v4Found = true
+	}
+
+	return !v4Found && v6Found, nil
+}