Merge pull request #1013 from erikwilson/node-cleanup

pkg/agent/config/config.go

@@ -82,7 +82,7 @@ func getNodeNamedCrt(nodeName, nodePasswordFile string) HTTPRequester {
 		defer resp.Body.Close()
 
 		if resp.StatusCode == http.StatusForbidden {
-			return nil, fmt.Errorf("Node password rejected, contents of '%s' may not match server passwd entry", nodePasswordFile)
+			return nil, fmt.Errorf("Node password rejected, duplicate hostname or contents of '%s' may not match server node-passwd entry, try enabling a unique node name with the --with-node-id flag", nodePasswordFile)
 		}
 
 		if resp.StatusCode != http.StatusOK {
@@ -93,6 +93,20 @@ func getNodeNamedCrt(nodeName, nodePasswordFile string) HTTPRequester {
 	}
 }
 
+func ensureNodeID(nodeIDFile string) (string, error) {
+	if _, err := os.Stat(nodeIDFile); err == nil {
+		id, err := ioutil.ReadFile(nodeIDFile)
+		return strings.TrimSpace(string(id)), err
+	}
+	id := make([]byte, 4, 4)
+	_, err := cryptorand.Read(id)
+	if err != nil {
+		return "", err
+	}
+	nodeID := hex.EncodeToString(id)
+	return nodeID, ioutil.WriteFile(nodeIDFile, []byte(nodeID+"\n"), 0644)
+}
+
 func ensureNodePassword(nodePasswordFile string) (string, error) {
 	if _, err := os.Stat(nodePasswordFile); err == nil {
 		password, err := ioutil.ReadFile(nodePasswordFile)
@@ -107,6 +121,21 @@ func ensureNodePassword(nodePasswordFile string) (string, error) {
 	return nodePassword, ioutil.WriteFile(nodePasswordFile, []byte(nodePassword+"\n"), 0600)
 }
 
+func upgradeOldNodePasswordPath(oldNodePasswordFile, newNodePasswordFile string) {
+	password, err := ioutil.ReadFile(oldNodePasswordFile)
+	if err != nil {
+		return
+	}
+	if err := ioutil.WriteFile(newNodePasswordFile, password, 0600); err != nil {
+		logrus.Warnf("Unable to write password file: %v", err)
+		return
+	}
+	if err := os.Remove(oldNodePasswordFile); err != nil {
+		logrus.Warnf("Unable to remove old password file: %v", err)
+		return
+	}
+}
+
 func getServingCert(nodeName, servingCertFile, servingKeyFile, nodePasswordFile string, info *clientaccess.Info) (*tls.Certificate, error) {
 	servingCert, err := Request("/v1-k3s/serving-kubelet.crt", info, getNodeNamedCrt(nodeName, nodePasswordFile))
 	if err != nil {
@@ -244,11 +273,6 @@ func get(envInfo *cmds.Agent) (*config.Node, error) {
 		return nil, err
 	}
 
-	nodeName, nodeIP, err := getHostnameAndIP(*envInfo)
-	if err != nil {
-		return nil, err
-	}
-
 	hostLocal, err := exec.LookPath("host-local")
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to find host-local")
@@ -274,14 +298,40 @@ func get(envInfo *cmds.Agent) (*config.Node, error) {
 
 	servingKubeletCert := filepath.Join(envInfo.DataDir, "serving-kubelet.crt")
 	servingKubeletKey := filepath.Join(envInfo.DataDir, "serving-kubelet.key")
-	nodePasswordFile := filepath.Join(envInfo.DataDir, "node-password.txt")
-	servingCert, err := getServingCert(nodeName, servingKubeletCert, servingKubeletKey, nodePasswordFile, info)
+
+	nodePasswordRoot := "/"
+	if envInfo.Rootless {
+		nodePasswordRoot = envInfo.DataDir
+	}
+	nodeConfigPath := filepath.Join(nodePasswordRoot, "etc", "rancher", "node")
+	if err := os.MkdirAll(nodeConfigPath, 0755); err != nil {
+		return nil, err
+	}
+
+	oldNodePasswordFile := filepath.Join(envInfo.DataDir, "node-password.txt")
+	newNodePasswordFile := filepath.Join(nodeConfigPath, "password")
+	upgradeOldNodePasswordPath(oldNodePasswordFile, newNodePasswordFile)
+
+	nodeName, nodeIP, err := getHostnameAndIP(*envInfo)
+	if err != nil {
+		return nil, err
+	}
+
+	if envInfo.WithNodeID {
+		nodeID, err := ensureNodeID(filepath.Join(nodeConfigPath, "id"))
+		if err != nil {
+			return nil, err
+		}
+		nodeName += "-" + nodeID
+	}
+
+	servingCert, err := getServingCert(nodeName, servingKubeletCert, servingKubeletKey, newNodePasswordFile, info)
 	if err != nil {
 		return nil, err
 	}
 
 	clientKubeletCert := filepath.Join(envInfo.DataDir, "client-kubelet.crt")
-	if err := getNodeNamedHostFile(clientKubeletCert, nodeName, nodePasswordFile, info); err != nil {
+	if err := getNodeNamedHostFile(clientKubeletCert, nodeName, newNodePasswordFile, info); err != nil {
 		return nil, err
 	}
 
@@ -334,6 +384,7 @@ func get(envInfo *cmds.Agent) (*config.Node, error) {
 	nodeConfig.Images = filepath.Join(envInfo.DataDir, "images")
 	nodeConfig.AgentConfig.NodeIP = nodeIP
 	nodeConfig.AgentConfig.NodeName = nodeName
+	nodeConfig.AgentConfig.NodeConfigPath = nodeConfigPath
 	nodeConfig.AgentConfig.NodeExternalIP = envInfo.NodeExternalIP
 	nodeConfig.AgentConfig.ServingKubeletCert = servingKubeletCert
 	nodeConfig.AgentConfig.ServingKubeletKey = servingKubeletKey

pkg/cli/cmds/agent.go

@@ -26,6 +26,7 @@ type Agent struct {
 	Debug                    bool
 	Rootless                 bool
 	RootlessAlreadyUnshared  bool
+	WithNodeID               bool
 	AgentShared
 	ExtraKubeletArgs   cli.StringSlice
 	ExtraKubeProxyArgs cli.StringSlice
@@ -57,6 +58,11 @@ var (
 		EnvVar:      "K3S_NODE_NAME",
 		Destination: &AgentConfig.NodeName,
 	}
+	WithNodeIDFlag = cli.BoolFlag{
+		Name:        "with-node-id",
+		Usage:       "(agent/node) Append id to node name",
+		Destination: &AgentConfig.WithNodeID,
+	}
 	DockerFlag = cli.BoolFlag{
 		Name:        "docker",
 		Usage:       "(agent/runtime) Use docker instead of containerd",

pkg/cli/cmds/server.go

@@ -210,6 +210,7 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command {
 				Destination: &ServerConfig.DisableNPC,
 			},
 			NodeNameFlag,
+			WithNodeIDFlag,
 			NodeLabels,
 			NodeTaints,
 			DockerFlag,

pkg/daemons/agent/agent.go

@@ -14,8 +14,8 @@ import (
 	"github.com/sirupsen/logrus"
 	"k8s.io/apimachinery/pkg/util/net"
 	"k8s.io/component-base/logs"
-	app2 "k8s.io/kubernetes/cmd/kube-proxy/app"
-	"k8s.io/kubernetes/cmd/kubelet/app"
+	proxy "k8s.io/kubernetes/cmd/kube-proxy/app"
+	kubelet "k8s.io/kubernetes/cmd/kubelet/app"
 	"k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes"
 
 	_ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration
@@ -25,34 +25,37 @@ import (
 func Agent(config *config.Agent) error {
 	rand.Seed(time.Now().UTC().UnixNano())
 
-	kubelet(config)
-	kubeProxy(config)
+	logs.InitLogs()
+	defer logs.FlushLogs()
+
+	startKubelet(config)
+	startKubeProxy(config)
 
 	return nil
 }
 
-func kubeProxy(cfg *config.Agent) {
+func startKubeProxy(cfg *config.Agent) {
 	argsMap := map[string]string{
 		"proxy-mode":           "iptables",
 		"healthz-bind-address": "127.0.0.1",
 		"kubeconfig":           cfg.KubeConfigKubeProxy,
 		"cluster-cidr":         cfg.ClusterCIDR.String(),
 	}
-	args := config.GetArgsList(argsMap, cfg.ExtraKubeProxyArgs)
+	if cfg.NodeName != "" {
+		argsMap["hostname-override"] = cfg.NodeName
+	}
 
-	command := app2.NewProxyCommand()
+	args := config.GetArgsList(argsMap, cfg.ExtraKubeProxyArgs)
+	command := proxy.NewProxyCommand()
 	command.SetArgs(args)
+
 	go func() {
-		err := command.Execute()
-		logrus.Fatalf("kube-proxy exited: %v", err)
+		logrus.Infof("Running kube-proxy %s", config.ArgString(args))
+		logrus.Fatalf("kube-proxy exited: %v", command.Execute())
 	}()
 }
 
-func kubelet(cfg *config.Agent) {
-	command := app.NewKubeletCommand(context.Background().Done())
-	logs.InitLogs()
-	defer logs.FlushLogs()
-
+func startKubelet(cfg *config.Agent) {
 	argsMap := map[string]string{
 		"healthz-bind-address":     "127.0.0.1",
 		"read-only-port":           "0",
@@ -146,6 +149,7 @@ func kubelet(cfg *config.Agent) {
 	}
 
 	args := config.GetArgsList(argsMap, cfg.ExtraKubeletArgs)
+	command := kubelet.NewKubeletCommand(context.Background().Done())
 	command.SetArgs(args)
 
 	go func() {

pkg/daemons/config/types.go

@@ -48,6 +48,7 @@ type Containerd struct {
 
 type Agent struct {
 	NodeName                string
+	NodeConfigPath          string
 	ServingKubeletCert      string
 	ServingKubeletKey       string
 	ClusterCIDR             net.IPNet

pkg/passwd/passwd.go

@@ -44,8 +44,8 @@ func Read(file string) (*Passwd, error) {
 		if err != nil {
 			return nil, err
 		}
-		if len(record) < 3 {
-			return nil, fmt.Errorf("password file '%s' must have at least 3 columns (password, user name, user uid), found %d", file, len(record))
+		if len(record) < 2 {
+			return nil, fmt.Errorf("password file '%s' must have at least 2 columns (password, name), found %d", file, len(record))
 		}
 		e := entry{
 			pass: record[0],