Add embedded etcd support

go.mod

@@ -110,6 +110,8 @@ require (
 	github.com/spf13/pflag v1.0.5
 	github.com/tchap/go-patricia v2.3.0+incompatible // indirect
 	github.com/urfave/cli v1.22.2
+	// e694b7bb0875 is v3.4.7
+	go.etcd.io/etcd v0.5.0-alpha.5.0.20200401174654-e694b7bb0875
 	golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975
 	golang.org/x/net v0.0.0-20191204025024-5ee1b9f4859a
 	golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e

pkg/cluster/etcd.go

+// +build !no_etcd
+
+package cluster
+
+import (
+	"github.com/rancher/k3s/pkg/cluster/managed"
+	"github.com/rancher/k3s/pkg/etcd"
+)
+
+func init() {
+	managed.RegisterDriver(&etcd.ETCD{})
+}

pkg/daemons/config/types.go

 package config
 
 import (
+	"context"
 	"crypto/tls"
 	"fmt"
 	"net"

pkg/daemons/control/server.go

@@ -19,6 +19,8 @@ import (
 	"text/template"
 	"time"
 
+	"k8s.io/apimachinery/pkg/util/sets"
+
 	"github.com/pkg/errors"
 	certutil "github.com/rancher/dynamiclistener/cert"
 	"github.com/rancher/k3s/pkg/clientaccess"
@@ -209,7 +211,7 @@ func apiServer(ctx context.Context, cfg *config.Control, runtime *config.Control
 	args := config.GetArgsList(argsMap, cfg.ExtraAPIArgs)
 
 	logrus.Infof("Running kube-apiserver %s", config.ArgString(args))
-	return executor.APIServer(ctx, args)
+	return executor.APIServer(ctx, runtime.ETCDReady, args)
 }
 
 func defaults(config *config.Control) {
@@ -963,6 +965,19 @@ func waitForAPIServerInBackground(ctx context.Context, runtime *config.ControlRu
 
 	go func() {
 		defer close(done)
+
+	etcdLoop:
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-runtime.ETCDReady:
+				break etcdLoop
+			case <-time.After(30 * time.Second):
+				logrus.Infof("Waiting for etcd server to become available")
+			}
+		}
+
 		logrus.Infof("Waiting for API server to become available")
 		for {
 			select {

pkg/daemons/executor/embed.go

@@ -45,7 +45,8 @@ func (Embedded) KubeProxy(args []string) error {
 	return nil
 }
 
-func (Embedded) APIServer(ctx context.Context, args []string) (authenticator.Request, http.Handler, error) {
+func (Embedded) APIServer(ctx context.Context, etcdReady <-chan struct{}, args []string) (authenticator.Request, http.Handler, error) {
+	<-etcdReady
 	command := app.NewAPIServerCommand(ctx.Done())
 	command.SetArgs(args)
 

pkg/daemons/executor/etcd.go

+// +build !no_embedded_executor
+
+package executor
+
+import (
+	"github.com/sirupsen/logrus"
+	"go.etcd.io/etcd/embed"
+)
+
+func (e Embedded) CurrentETCDOptions() (InitialOptions, error) {
+	return InitialOptions{}, nil
+}
+
+func (e Embedded) ETCD(args ETCDConfig) error {
+	configFile, err := args.ToConfigFile()
+	if err != nil {
+		return err
+	}
+	cfg, err := embed.ConfigFromFile(configFile)
+	if err != nil {
+		return err
+	}
+	etcd, err := embed.StartEtcd(cfg)
+	if err != nil {
+		return nil
+	}
+
+	go func() {
+		err := <-etcd.Err()
+		logrus.Fatalf("etcd exited: %v", err)
+	}()
+	return nil
+}

pkg/daemons/executor/executor.go

@@ -2,22 +2,76 @@ package executor
 
 import (
 	"context"
+	"io/ioutil"
 	"net/http"
+	"os"
+	"path/filepath"
+
+	"sigs.k8s.io/yaml"
 
 	"k8s.io/apiserver/pkg/authentication/authenticator"
 )
 
+var (
+	executor Executor
+)
+
 type Executor interface {
 	Kubelet(args []string) error
 	KubeProxy(args []string) error
-	APIServer(ctx context.Context, args []string) (authenticator.Request, http.Handler, error)
+	APIServer(ctx context.Context, etcdReady <-chan struct{}, args []string) (authenticator.Request, http.Handler, error)
 	Scheduler(apiReady <-chan struct{}, args []string) error
 	ControllerManager(apiReady <-chan struct{}, args []string) error
+	CurrentETCDOptions() (InitialOptions, error)
+	ETCD(args ETCDConfig) error
 }
 
-var (
-	executor Executor
-)
+type ETCDConfig struct {
+	InitialOptions      `json:",inline"`
+	Name                string      `json:"name,omitempty"`
+	ListenClientURLs    string      `json:"listen-client-urls,omitempty"`
+	ListenMetricsURLs   string      `json:"listen-metrics-urls,omitempty"`
+	ListenPeerURLs      string      `json:"listen-peer-urls,omitempty"`
+	AdvertiseClientURLs string      `json:"advertise-client-urls,omitempty"`
+	DataDir             string      `json:"data-dir,omitempty"`
+	SnapshotCount       int         `json:"snapshot-count,omitempty"`
+	ServerTrust         ServerTrust `json:"client-transport-security"`
+	PeerTrust           PeerTrust   `json:"peer-transport-security"`
+	ForceNewCluster     bool        `json:"force-new-cluster,omitempty"`
+}
+
+type ServerTrust struct {
+	CertFile       string `json:"cert-file"`
+	KeyFile        string `json:"key-file"`
+	ClientCertAuth bool   `json:"client-cert-auth"`
+	TrustedCAFile  string `json:"trusted-ca-file"`
+}
+
+type PeerTrust struct {
+	CertFile       string `json:"cert-file"`
+	KeyFile        string `json:"key-file"`
+	ClientCertAuth bool   `json:"client-cert-auth"`
+	TrustedCAFile  string `json:"trusted-ca-file"`
+}
+
+type InitialOptions struct {
+	AdvertisePeerURL string `json:"initial-advertise-peer-urls,omitempty"`
+	Cluster          string `json:"initial-cluster,omitempty"`
+	State            string `json:"initial-cluster-state,omitempty"`
+}
+
+func (e ETCDConfig) ToConfigFile() (string, error) {
+	confFile := filepath.Join(e.DataDir, "config")
+	bytes, err := yaml.Marshal(&e)
+	if err != nil {
+		return "", err
+	}
+
+	if err := os.MkdirAll(e.DataDir, 0700); err != nil {
+		return "", err
+	}
+	return confFile, ioutil.WriteFile(confFile, bytes, 0600)
+}
 
 func Set(driver Executor) {
 	executor = driver
@@ -31,8 +85,8 @@ func KubeProxy(args []string) error {
 	return executor.KubeProxy(args)
 }
 
-func APIServer(ctx context.Context, args []string) (authenticator.Request, http.Handler, error) {
-	return executor.APIServer(ctx, args)
+func APIServer(ctx context.Context, etcdReady <-chan struct{}, args []string) (authenticator.Request, http.Handler, error) {
+	return executor.APIServer(ctx, etcdReady, args)
 }
 
 func Scheduler(apiReady <-chan struct{}, args []string) error {
@@ -42,3 +96,11 @@ func Scheduler(apiReady <-chan struct{}, args []string) error {
 func ControllerManager(apiReady <-chan struct{}, args []string) error {
 	return executor.ControllerManager(apiReady, args)
 }
+
+func CurrentETCDOptions() (InitialOptions, error) {
+	return executor.CurrentETCDOptions()
+}
+
+func ETCD(args ETCDConfig) error {
+	return executor.ETCD(args)
+}

pkg/etcd/controller.go

+package etcd
+
+import (
+	"context"
+	"os"
+	"time"
+
+	controllerv1 "github.com/rancher/wrangler-api/pkg/generated/controllers/core/v1"
+	"github.com/sirupsen/logrus"
+	v1 "k8s.io/api/core/v1"
+)
+
+const (
+	nodeID      = "etcd.k3s.cattle.io/node-name"
+	nodeAddress = "etcd.k3s.cattle.io/node-address"
+	master      = "node-role.kubernetes.io/master"
+	etcdRole    = "node-role.kubernetes.io/etcd"
+)
+
+type NodeControllerGetter func() controllerv1.NodeController
+
+func Register(ctx context.Context, etcd *ETCD, nodes controllerv1.NodeController) {
+	h := &handler{
+		etcd:           etcd,
+		nodeController: nodes,
+		ctx:            ctx,
+	}
+	nodes.OnChange(ctx, "managed-etcd-controller", h.sync)
+	nodes.OnRemove(ctx, "managed-etcd-controller", h.onRemove)
+}
+
+type handler struct {
+	etcd           *ETCD
+	nodeController controllerv1.NodeController
+	ctx            context.Context
+}
+
+func (h *handler) sync(key string, node *v1.Node) (*v1.Node, error) {
+	if node == nil {
+		return nil, nil
+	}
+
+	nodeName := os.Getenv("NODE_NAME")
+	if nodeName == "" {
+		logrus.Debug("waiting for node to be assigned for etcd controller")
+		h.nodeController.EnqueueAfter(key, 5*time.Second)
+		return node, nil
+	}
+
+	if key == nodeName {
+		return h.handleSelf(node)
+	}
+
+	return node, nil
+}
+
+func (h *handler) handleSelf(node *v1.Node) (*v1.Node, error) {
+	if node.Annotations[nodeID] == h.etcd.name &&
+		node.Annotations[nodeAddress] == h.etcd.address &&
+		node.Labels[etcdRole] == "true" &&
+		node.Labels[master] == "true" {
+		return node, nil
+	}
+
+	node = node.DeepCopy()
+	if node.Annotations == nil {
+		node.Annotations = map[string]string{}
+	}
+	node.Annotations[nodeID] = h.etcd.name
+	node.Annotations[nodeAddress] = h.etcd.address
+	node.Labels[etcdRole] = "true"
+	node.Labels[master] = "true"
+
+	return h.nodeController.Update(node)
+}
+
+func (h *handler) onRemove(key string, node *v1.Node) (*v1.Node, error) {
+	if _, ok := node.Labels[etcdRole]; !ok {
+		return node, nil
+	}
+
+	id := node.Annotations[nodeID]
+	address := node.Annotations[nodeAddress]
+	if address == "" {
+		return node, nil
+	}
+
+	return node, h.etcd.removePeer(h.ctx, id, address)
+}