Merge pull request #309 from galal-hussein/generic_flags

pkg/agent/config/config.go

@@ -249,6 +249,9 @@ func get(envInfo *cmds.Agent) (*config.Node, error) {
 	os.Setenv("NODE_NAME", nodeConfig.AgentConfig.NodeName)
 	v1beta1.KubeletSocket = filepath.Join(envInfo.DataDir, "kubelet/device-plugins/kubelet.sock")
 
+	nodeConfig.AgentConfig.ExtraKubeletArgs = envInfo.ExtraKubeletArgs
+	nodeConfig.AgentConfig.ExtraKubeProxyArgs = envInfo.ExtraKubeProxyArgs
+
 	return nodeConfig, nil
 }
 

pkg/cli/cmds/agent.go

@@ -21,6 +21,8 @@ type Agent struct {
 	NoFlannel                bool
 	Debug                    bool
 	AgentShared
+	ExtraKubeletArgs   cli.StringSlice
+	ExtraKubeProxyArgs cli.StringSlice
 }
 
 type AgentShared struct {
@@ -62,6 +64,16 @@ var (
 		EnvVar:      "K3S_RESOLV_CONF",
 		Destination: &AgentConfig.ResolvConf,
 	}
+	ExtraKubeletArgs = cli.StringSliceFlag{
+		Name:  "kubelet-arg",
+		Usage: "(agent) Customized flag for kubelet process",
+		Value: &AgentConfig.ExtraKubeletArgs,
+	}
+	ExtraKubeProxyArgs = cli.StringSliceFlag{
+		Name:  "kube-proxy-arg",
+		Usage: "(agent) Customized flag for kube-proxy process",
+		Value: &AgentConfig.ExtraKubeProxyArgs,
+	}
 )
 
 func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command {
@@ -107,6 +119,8 @@ func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command {
 			NodeIPFlag,
 			CRIEndpointFlag,
 			ResolvConfFlag,
+			ExtraKubeletArgs,
+			ExtraKubeProxyArgs,
 		},
 	}
 }

pkg/cli/cmds/server.go

@@ -5,19 +5,22 @@ import (
 )
 
 type Server struct {
-	Log              string
-	ClusterCIDR      string
-	ClusterSecret    string
-	ServiceCIDR      string
-	ClusterDNS       string
-	HTTPSPort        int
-	HTTPPort         int
-	DataDir          string
-	DisableAgent     bool
-	KubeConfigOutput string
-	KubeConfigMode   string
-	KnownIPs         cli.StringSlice
-	BindAddress      string
+	Log                 string
+	ClusterCIDR         string
+	ClusterSecret       string
+	ServiceCIDR         string
+	ClusterDNS          string
+	HTTPSPort           int
+	HTTPPort            int
+	DataDir             string
+	DisableAgent        bool
+	KubeConfigOutput    string
+	KubeConfigMode      string
+	KnownIPs            cli.StringSlice
+	BindAddress         string
+	ExtraAPIArgs        cli.StringSlice
+	ExtraSchedulerArgs  cli.StringSlice
+	ExtraControllerArgs cli.StringSlice
 }
 
 var ServerConfig Server
@@ -106,12 +109,29 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command {
 				Usage: "Add additional hostname or IP as a Subject Alternative Name in the TLS cert",
 				Value: &ServerConfig.KnownIPs,
 			},
+			cli.StringSliceFlag{
+				Name:  "kube-apiserver-arg",
+				Usage: "Customized flag for kube-apiserver process",
+				Value: &ServerConfig.ExtraAPIArgs,
+			},
+			cli.StringSliceFlag{
+				Name:  "kube-scheduler-arg",
+				Usage: "Customized flag for kube-scheduler process",
+				Value: &ServerConfig.ExtraSchedulerArgs,
+			},
+			cli.StringSliceFlag{
+				Name:  "kube-controller-arg",
+				Usage: "Customized flag for kube-controller-manager process",
+				Value: &ServerConfig.ExtraControllerArgs,
+			},
 			NodeIPFlag,
 			NodeNameFlag,
 			DockerFlag,
 			FlannelFlag,
 			CRIEndpointFlag,
 			ResolvConfFlag,
+			ExtraKubeletArgs,
+			ExtraKubeProxyArgs,
 		},
 	}
 }

pkg/cli/server/server.go

@@ -83,6 +83,9 @@ func run(app *cli.Context, cfg *cmds.Server) error {
 	serverConfig.TLSConfig.HTTPPort = cfg.HTTPPort
 	serverConfig.TLSConfig.KnownIPs = knownIPs(cfg.KnownIPs)
 	serverConfig.TLSConfig.BindAddress = cfg.BindAddress
+	serverConfig.ControlConfig.ExtraAPIArgs = cfg.ExtraAPIArgs
+	serverConfig.ControlConfig.ExtraControllerArgs = cfg.ExtraControllerArgs
+	serverConfig.ControlConfig.ExtraSchedulerAPIArgs = cfg.ExtraSchedulerArgs
 
 	_, serverConfig.ControlConfig.ClusterIPRange, err = net2.ParseCIDR(cfg.ClusterCIDR)
 	if err != nil {

pkg/daemons/agent/agent.go

@@ -28,14 +28,14 @@ func Agent(config *config.Agent) error {
 	return nil
 }
 
-func kubeProxy(config *config.Agent) {
-	args := []string{
-		"--proxy-mode", "iptables",
-		"--healthz-bind-address", "127.0.0.1",
-		"--kubeconfig", config.KubeConfig,
-		"--cluster-cidr", config.ClusterCIDR.String(),
-	}
-	args = append(args, config.ExtraKubeletArgs...)
+func kubeProxy(cfg *config.Agent) {
+	argsMap := map[string]string{
+		"proxy-mode":           "iptables",
+		"healthz-bind-address": "127.0.0.1",
+		"kubeconfig":           cfg.KubeConfig,
+		"cluster-cidr":         cfg.ClusterCIDR.String(),
+	}
+	args := config.GetArgsList(argsMap, cfg.ExtraKubeProxyArgs)
 
 	command := app2.NewProxyCommand()
 	command.SetArgs(args)
@@ -50,63 +50,64 @@ func kubelet(cfg *config.Agent) {
 	logs.InitLogs()
 	defer logs.FlushLogs()
 
-	args := []string{
-		"--healthz-bind-address", "127.0.0.1",
-		"--read-only-port", "0",
-		"--allow-privileged=true",
-		"--cluster-domain", "cluster.local",
-		"--kubeconfig", cfg.KubeConfig,
-		"--eviction-hard", "imagefs.available<5%,nodefs.available<5%",
-		"--eviction-minimum-reclaim", "imagefs.available=10%,nodefs.available=10%",
-		"--fail-swap-on=false",
-		//"--cgroup-root", "/k3s",
-		"--cgroup-driver", "cgroupfs",
+	argsMap := map[string]string{
+		"healthz-bind-address":     "127.0.0.1",
+		"read-only-port":           "0",
+		"allow-privileged":         "true",
+		"cluster-domain":           "cluster.local",
+		"kubeconfig":               cfg.KubeConfig,
+		"eviction-hard":            "imagefs.available<5%,nodefs.available<5%",
+		"eviction-minimum-reclaim": "imagefs.available=10%,nodefs.available=10%",
+		"fail-swap-on":             "false",
+		//"cgroup-root": "/k3s",
+		"cgroup-driver": "cgroupfs",
 	}
 	if cfg.RootDir != "" {
-		args = append(args, "--root-dir", cfg.RootDir)
-		args = append(args, "--cert-dir", filepath.Join(cfg.RootDir, "pki"))
-		args = append(args, "--seccomp-profile-root", filepath.Join(cfg.RootDir, "seccomp"))
+		argsMap["root-dir"] = cfg.RootDir
+		argsMap["cert-dir"] = filepath.Join(cfg.RootDir, "pki")
+		argsMap["seccomp-profile-root"] = filepath.Join(cfg.RootDir, "seccomp")
 	}
 	if cfg.CNIConfDir != "" {
-		args = append(args, "--cni-conf-dir", cfg.CNIConfDir)
+		argsMap["cni-conf-dir"] = cfg.CNIConfDir
 	}
 	if cfg.CNIBinDir != "" {
-		args = append(args, "--cni-bin-dir", cfg.CNIBinDir)
+		argsMap["cni-bin-dir"] = cfg.CNIBinDir
 	}
 	if len(cfg.ClusterDNS) > 0 {
-		args = append(args, "--cluster-dns", cfg.ClusterDNS.String())
+		argsMap["cluster-dns"] = cfg.ClusterDNS.String()
 	}
 	if cfg.ResolvConf != "" {
-		args = append(args, "--resolv-conf", cfg.ResolvConf)
+		argsMap["resolv-conf"] = cfg.ResolvConf
 	}
 	if cfg.RuntimeSocket != "" {
-		args = append(args, "--container-runtime", "remote")
-		args = append(args, "--container-runtime-endpoint", cfg.RuntimeSocket)
-		args = append(args, "--serialize-image-pulls=false")
+		argsMap["container-runtime"] = "remote"
+		argsMap["container-runtime-endpoint"] = cfg.RuntimeSocket
+		argsMap["serialize-image-pulls"] = "false"
 	}
 	if cfg.ListenAddress != "" {
-		args = append(args, "--address", cfg.ListenAddress)
+		argsMap["address"] = cfg.ListenAddress
 	}
 	if cfg.CACertPath != "" {
-		args = append(args, "--anonymous-auth=false", "--client-ca-file", cfg.CACertPath)
+		argsMap["anonymous-auth"] = "false"
+		argsMap["client-ca-file"] = cfg.CACertPath
 	}
 	if cfg.NodeName != "" {
-		args = append(args, "--hostname-override", cfg.NodeName)
+		argsMap["hostname-override"] = cfg.NodeName
 	}
 	defaultIP, err := net.ChooseHostInterface()
 	if err != nil || defaultIP.String() != cfg.NodeIP {
-		args = append(args, "--node-ip", cfg.NodeIP)
+		argsMap["node-ip"] = cfg.NodeIP
 	}
 	root, hasCFS := checkCgroups()
 	if !hasCFS {
 		logrus.Warn("Disabling CPU quotas due to missing cpu.cfs_period_us")
-		args = append(args, "--cpu-cfs-quota=false")
+		argsMap["cpu-cfs-quota"] = "false"
 	}
 	if root != "" {
-		args = append(args, "--runtime-cgroups", root)
-		args = append(args, "--kubelet-cgroups", root)
+		argsMap["runtime-cgroups"] = root
+		argsMap["kubelet-cgroups"] = root
 	}
-	args = append(args, cfg.ExtraKubeletArgs...)
+	args := config.GetArgsList(argsMap, cfg.ExtraKubeletArgs)
 
 	command.SetArgs(args)
 

pkg/daemons/config/types.go

@@ -2,6 +2,7 @@ package config
 
 import (
 	"crypto/tls"
+	"fmt"
 	"net"
 	"net/http"
 	"strings"
@@ -106,3 +107,21 @@ func (a ArgString) String() string {
 	}
 	return b.String()
 }
+
+func GetArgsList(argsMap map[string]string, extraArgs []string) []string {
+	// add extra args to args map to override any default option
+	for _, arg := range extraArgs {
+		splitArg := strings.Split(arg, "=")
+		if len(splitArg) < 2 {
+			argsMap[splitArg[0]] = "true"
+			continue
+		}
+		argsMap[splitArg[0]] = splitArg[1]
+	}
+	var args []string
+	for arg, value := range argsMap {
+		cmd := fmt.Sprintf("--%s=%s", arg, value)
+		args = append(args, cmd)
+	}
+	return args
+}

pkg/daemons/control/server.go

@@ -95,20 +95,22 @@ func Server(ctx context.Context, cfg *config.Control) error {
 }
 
 func controllerManager(cfg *config.Control, runtime *config.ControlRuntime) {
-	args := []string{
-		"--kubeconfig", runtime.KubeConfigSystem,
-		"--service-account-private-key-file", runtime.ServiceKey,
-		"--allocate-node-cidrs",
-		"--cluster-cidr", cfg.ClusterIPRange.String(),
-		"--root-ca-file", runtime.TokenCA,
-		"--port", "10252",
-		"--bind-address", "127.0.0.1",
-		"--secure-port", "0",
+	argsMap := map[string]string{
+		"kubeconfig":                       runtime.KubeConfigSystem,
+		"service-account-private-key-file": runtime.ServiceKey,
+		"allocate-node-cidrs":              "true",
+		"cluster-cidr":                     cfg.ClusterIPRange.String(),
+		"root-ca-file":                     runtime.TokenCA,
+		"port":                             "10252",
+		"bind-address":                     "127.0.0.1",
+		"secure-port":                      "0",
 	}
 	if cfg.NoLeaderElect {
-		args = append(args, "--leader-elect=false")
+		argsMap["leader-elect"] = "false"
 	}
-	args = append(args, cfg.ExtraControllerArgs...)
+
+	args := config.GetArgsList(argsMap, cfg.ExtraControllerArgs)
+
 	command := cmapp.NewControllerManagerCommand()
 	command.SetArgs(args)
 
@@ -119,16 +121,17 @@ func controllerManager(cfg *config.Control, runtime *config.ControlRuntime) {
 }
 
 func scheduler(cfg *config.Control, runtime *config.ControlRuntime) {
-	args := []string{
-		"--kubeconfig", runtime.KubeConfigSystem,
-		"--port", "10251",
-		"--bind-address", "127.0.0.1",
-		"--secure-port", "0",
+	argsMap := map[string]string{
+		"kubeconfig":   runtime.KubeConfigSystem,
+		"port":         "10251",
+		"bind-address": "127.0.0.1",
+		"secure-port":  "0",
 	}
 	if cfg.NoLeaderElect {
-		args = append(args, "--leader-elect=false")
+		argsMap["leader-elect"] = "false"
 	}
-	args = append(args, cfg.ExtraSchedulerAPIArgs...)
+	args := config.GetArgsList(argsMap, cfg.ExtraSchedulerAPIArgs)
+
 	command := sapp.NewSchedulerCommand()
 	command.SetArgs(args)
 
@@ -139,19 +142,18 @@ func scheduler(cfg *config.Control, runtime *config.ControlRuntime) {
 }
 
 func apiServer(ctx context.Context, cfg *config.Control, runtime *config.ControlRuntime) (authenticator.Request, http.Handler, error) {
-	var args []string
-
+	argsMap := make(map[string]string)
 	if len(cfg.ETCDEndpoints) > 0 {
-		args = append(args, "--storage-backend", "etcd3")
-		args = append(args, "--etcd-servers", strings.Join(cfg.ETCDEndpoints, ","))
+		argsMap["storage-backend"] = "etcd3"
+		argsMap["etcd-servers"] = strings.Join(cfg.ETCDEndpoints, ",")
 		if cfg.ETCDKeyFile != "" {
-			args = append(args, "--etcd-keyfile", cfg.ETCDKeyFile)
+			argsMap["etcd-keyfile"] = cfg.ETCDKeyFile
 		}
 		if cfg.ETCDCAFile != "" {
-			args = append(args, "--etcd-cafile", cfg.ETCDCAFile)
+			argsMap["etcd-cafile"] = cfg.ETCDCAFile
 		}
 		if cfg.ETCDCertFile != "" {
-			args = append(args, "--etcd-certfile", cfg.ETCDCertFile)
+			argsMap["etcd-certfile"] = cfg.ETCDCertFile
 		}
 	}
 
@@ -159,27 +161,28 @@ func apiServer(ctx context.Context, cfg *config.Control, runtime *config.Control
 	os.MkdirAll(certDir, 0700)
 
 	// TODO: sqlite doesn't need the watch cache, but etcd does, so make this dynamic
-	args = append(args, "--watch-cache=false")
-	args = append(args, "--cert-dir", certDir)
-	args = append(args, "--allow-privileged=true")
-	args = append(args, "--authorization-mode", strings.Join([]string{modes.ModeNode, modes.ModeRBAC}, ","))
-	args = append(args, "--service-account-signing-key-file", runtime.ServiceKey)
-	args = append(args, "--service-cluster-ip-range", cfg.ServiceIPRange.String())
-	args = append(args, "--advertise-port", strconv.Itoa(cfg.AdvertisePort))
-	args = append(args, "--advertise-address", localhostIP.String())
-	args = append(args, "--insecure-port", "0")
-	args = append(args, "--secure-port", strconv.Itoa(cfg.ListenPort))
-	args = append(args, "--bind-address", localhostIP.String())
-	args = append(args, "--tls-cert-file", runtime.TLSCert)
-	args = append(args, "--tls-private-key-file", runtime.TLSKey)
-	args = append(args, "--service-account-key-file", runtime.ServiceKey)
-	args = append(args, "--service-account-issuer", "k3s")
-	args = append(args, "--api-audiences", "unknown")
-	args = append(args, "--basic-auth-file", runtime.PasswdFile)
-	args = append(args, "--kubelet-client-certificate", runtime.NodeCert)
-	args = append(args, "--kubelet-client-key", runtime.NodeKey)
-
-	args = append(args, cfg.ExtraAPIArgs...)
+	argsMap["watch-cache"] = "false"
+	argsMap["cert-dir"] = certDir
+	argsMap["allow-privileged"] = "true"
+	argsMap["authorization-mode"] = strings.Join([]string{modes.ModeNode, modes.ModeRBAC}, ",")
+	argsMap["service-account-signing-key-file"] = runtime.ServiceKey
+	argsMap["service-cluster-ip-range"] = cfg.ServiceIPRange.String()
+	argsMap["advertise-port"] = strconv.Itoa(cfg.AdvertisePort)
+	argsMap["advertise-address"] = localhostIP.String()
+	argsMap["insecure-port"] = "0"
+	argsMap["secure-port"] = strconv.Itoa(cfg.ListenPort)
+	argsMap["bind-address"] = localhostIP.String()
+	argsMap["tls-cert-file"] = runtime.TLSCert
+	argsMap["tls-private-key-file"] = runtime.TLSKey
+	argsMap["service-account-key-file"] = runtime.ServiceKey
+	argsMap["service-account-issuer"] = "k3s"
+	argsMap["api-audiences"] = "unknown"
+	argsMap["basic-auth-file"] = runtime.PasswdFile
+	argsMap["kubelet-client-certificate"] = runtime.NodeCert
+	argsMap["kubelet-client-key"] = runtime.NodeKey
+
+	args := config.GetArgsList(argsMap, cfg.ExtraAPIArgs)
+
 	command := app.NewAPIServerCommand(ctx.Done())
 	command.SetArgs(args)