[Release-1.25] Clear remove annotations on cluster reset (#8589)

pkg/cli/etcdsnapshot/etcd_snapshot.go

@@ -64,7 +64,7 @@ func commandSetup(app *cli.Context, cfg *cmds.Server, sc *server.Config) error {
 	sc.ControlConfig.Runtime.ETCDServerCA = filepath.Join(dataDir, "tls", "etcd", "server-ca.crt")
 	sc.ControlConfig.Runtime.ClientETCDCert = filepath.Join(dataDir, "tls", "etcd", "client.crt")
 	sc.ControlConfig.Runtime.ClientETCDKey = filepath.Join(dataDir, "tls", "etcd", "client.key")
-	sc.ControlConfig.Runtime.KubeConfigSupervisor = filepath.Join(dataDir, "cred", "supervisor.kubeconfig")
+	sc.ControlConfig.Runtime.KubeConfigAdmin = filepath.Join(dataDir, "cred", "admin.kubeconfig")
 
 	return nil
 }
@@ -97,11 +97,11 @@ func save(app *cli.Context, cfg *cmds.Server) error {
 
 	ctx := signals.SetupSignalContext()
 	e := etcd.NewETCD()
-	if err := e.SetControlConfig(ctx, &serverConfig.ControlConfig); err != nil {
+	if err := e.SetControlConfig(&serverConfig.ControlConfig); err != nil {
 		return err
 	}
 
-	initialized, err := e.IsInitialized(ctx, &serverConfig.ControlConfig)
+	initialized, err := e.IsInitialized()
 	if err != nil {
 		return err
 	}
@@ -115,7 +115,7 @@ func save(app *cli.Context, cfg *cmds.Server) error {
 		return err
 	}
 
-	sc, err := server.NewContext(ctx, serverConfig.ControlConfig.Runtime.KubeConfigSupervisor)
+	sc, err := server.NewContext(ctx, serverConfig.ControlConfig.Runtime.KubeConfigAdmin, false)
 	if err != nil {
 		return err
 	}
@@ -145,11 +145,11 @@ func delete(app *cli.Context, cfg *cmds.Server) error {
 
 	ctx := signals.SetupSignalContext()
 	e := etcd.NewETCD()
-	if err := e.SetControlConfig(ctx, &serverConfig.ControlConfig); err != nil {
+	if err := e.SetControlConfig(&serverConfig.ControlConfig); err != nil {
 		return err
 	}
 
-	sc, err := server.NewContext(ctx, serverConfig.ControlConfig.Runtime.KubeConfigSupervisor)
+	sc, err := server.NewContext(ctx, serverConfig.ControlConfig.Runtime.KubeConfigAdmin, false)
 	if err != nil {
 		return err
 	}
@@ -185,7 +185,7 @@ func list(app *cli.Context, cfg *cmds.Server) error {
 
 	ctx := signals.SetupSignalContext()
 	e := etcd.NewETCD()
-	if err := e.SetControlConfig(ctx, &serverConfig.ControlConfig); err != nil {
+	if err := e.SetControlConfig(&serverConfig.ControlConfig); err != nil {
 		return err
 	}
 
@@ -251,11 +251,11 @@ func prune(app *cli.Context, cfg *cmds.Server) error {
 
 	ctx := signals.SetupSignalContext()
 	e := etcd.NewETCD()
-	if err := e.SetControlConfig(ctx, &serverConfig.ControlConfig); err != nil {
+	if err := e.SetControlConfig(&serverConfig.ControlConfig); err != nil {
 		return err
 	}
 
-	sc, err := server.NewContext(ctx, serverConfig.ControlConfig.Runtime.KubeConfigSupervisor)
+	sc, err := server.NewContext(ctx, serverConfig.ControlConfig.Runtime.KubeConfigAdmin, false)
 	if err != nil {
 		return err
 	}

pkg/cluster/bootstrap.go

@@ -82,7 +82,7 @@ func (c *Cluster) shouldBootstrapLoad(ctx context.Context) (bool, bool, error) {
 	if c.managedDB != nil {
 		c.config.Runtime.HTTPBootstrap = true
 
-		isInitialized, err := c.managedDB.IsInitialized(ctx, c.config)
+		isInitialized, err := c.managedDB.IsInitialized()
 		if err != nil {
 			return false, false, err
 		}
@@ -430,7 +430,7 @@ func (c *Cluster) Snapshot(ctx context.Context, config *config.Control) error {
 	if c.managedDB == nil {
 		return errors.New("unable to perform etcd snapshot on non-etcd system")
 	}
-	return c.managedDB.Snapshot(ctx, config)
+	return c.managedDB.Snapshot(ctx)
 }
 
 // compareConfig verifies that the config of the joining control plane node coincides with the cluster's config
@@ -503,7 +503,7 @@ func (c *Cluster) reconcileEtcd(ctx context.Context) error {
 	}()
 
 	e := etcd.NewETCD()
-	if err := e.SetControlConfig(reconcileCtx, c.config); err != nil {
+	if err := e.SetControlConfig(c.config); err != nil {
 		return err
 	}
 	if err := e.StartEmbeddedTemporary(reconcileCtx); err != nil {

pkg/cluster/https.go

@@ -15,7 +15,6 @@ import (
 
 	"github.com/gorilla/mux"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
-	"github.com/k3s-io/k3s/pkg/etcd"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/rancher/dynamiclistener"
 	"github.com/rancher/dynamiclistener/factory"
@@ -33,9 +32,14 @@ import (
 // and will sync the certs into the Kubernetes datastore, with a local disk cache.
 func (c *Cluster) newListener(ctx context.Context) (net.Listener, http.Handler, error) {
 	if c.managedDB != nil {
-		if _, err := os.Stat(etcd.ResetFile(c.config)); err == nil {
-			// delete the dynamic listener file if it exists after restoration to fix restoration
-			// on fresh nodes
+		resetDone, err := c.managedDB.IsReset()
+		if err != nil {
+			return nil, nil, err
+		}
+		if resetDone {
+			// delete the dynamic listener TLS secret cache after restoring,
+			// to ensure that dynamiclistener doesn't sync the old secret over the top
+			// of whatever was just restored.
 			os.Remove(filepath.Join(c.config.DataDir, "tls/dynamic-cert.json"))
 		}
 	}
@@ -104,8 +108,8 @@ func (c *Cluster) initClusterAndHTTPS(ctx context.Context) error {
 		return err
 	}
 
-	// Config the cluster database and allow it to add additional request handlers
-	handler, err = c.initClusterDB(ctx, handler)
+	// Register database request handlers and controller callbacks
+	handler, err = c.registerDBHandlers(handler)
 	if err != nil {
 		return err
 	}

pkg/cluster/managed.go

@@ -9,14 +9,12 @@ import (
 	"net/http"
 	"net/url"
 	"os"
-	"strings"
 	"time"
 
 	"github.com/k3s-io/k3s/pkg/cluster/managed"
 	"github.com/k3s-io/k3s/pkg/etcd"
 	"github.com/k3s-io/k3s/pkg/nodepassword"
 	"github.com/k3s-io/k3s/pkg/version"
-	"github.com/k3s-io/kine/pkg/endpoint"
 	"github.com/sirupsen/logrus"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 )
@@ -57,75 +55,64 @@ func (c *Cluster) start(ctx context.Context) error {
 	if c.managedDB == nil {
 		return nil
 	}
-	resetFile := etcd.ResetFile(c.config)
 	rebootstrap := func() error {
 		return c.storageBootstrap(ctx)
 	}
 
+	resetDone, err := c.managedDB.IsReset()
+	if err != nil {
+		return err
+	}
+
 	if c.config.ClusterReset {
 		// If we're restoring from a snapshot, don't check the reset-flag - just reset and restore.
 		if c.config.ClusterResetRestorePath != "" {
 			return c.managedDB.Reset(ctx, rebootstrap)
 		}
+
 		// If the reset-flag doesn't exist, reset. This will create the reset-flag if it succeeds.
-		if _, err := os.Stat(resetFile); err != nil {
-			if !os.IsNotExist(err) {
-				return err
-			}
+		if !resetDone {
 			return c.managedDB.Reset(ctx, rebootstrap)
 		}
+
 		// The reset-flag exists, ask the user to remove it if they want to reset again.
-		return fmt.Errorf("Managed etcd cluster membership was previously reset, please remove the cluster-reset flag and start %s normally. If you need to perform another cluster reset, you must first manually delete the %s file", version.Program, resetFile)
+		return fmt.Errorf("Managed etcd cluster membership was previously reset, please remove the cluster-reset flag and start %s normally. "+
+			"If you need to perform another cluster reset, you must first manually delete the file at %s", version.Program, c.managedDB.ResetFile())
 	}
 
-	// The reset-flag exists but we're not resetting; remove it
-	if _, err := os.Stat(resetFile); err == nil {
-		// Before removing reset file we need to delete the node passwd secret in case the node
+	if resetDone {
+		// If the cluster was reset, we need to delete the node passwd secret in case the node
 		// password from the previously restored snapshot differs from the current password on disk.
 		c.config.Runtime.ClusterControllerStarts["node-password-secret-cleanup"] = c.deleteNodePasswdSecret
-		os.Remove(resetFile)
 	}
 
+	// Starting the managed database will clear the reset-flag if set
 	return c.managedDB.Start(ctx, c.clientAccessInfo)
 }
 
-// initClusterDB registers routes for database info with the http request handler
-func (c *Cluster) initClusterDB(ctx context.Context, handler http.Handler) (http.Handler, error) {
+// registerDBHandlers registers routes for database info with the http request handler
+func (c *Cluster) registerDBHandlers(handler http.Handler) (http.Handler, error) {
 	if c.managedDB == nil {
 		return handler, nil
 	}
 
-	if !strings.HasPrefix(c.config.Datastore.Endpoint, c.managedDB.EndpointName()+"://") {
-		c.config.Datastore = endpoint.Config{
-			Endpoint: c.managedDB.EndpointName(),
-		}
-	}
-
-	return c.managedDB.Register(ctx, c.config, handler)
+	return c.managedDB.Register(handler)
 }
 
 // assignManagedDriver assigns a driver based on a number of different configuration variables.
 // If a driver has been initialized it is used.
-// If the configured endpoint matches the name of a driver, that driver is used.
 // If no specific endpoint has been requested and creating or joining has been requested,
 // we use the default driver.
 // If none of the above are true, no managed driver is assigned.
 func (c *Cluster) assignManagedDriver(ctx context.Context) error {
 	// Check all managed drivers for an initialized database on disk; use one if found
 	for _, driver := range managed.Registered() {
-		if ok, err := driver.IsInitialized(ctx, c.config); err != nil {
+		if err := driver.SetControlConfig(c.config); err != nil {
 			return err
-		} else if ok {
-			c.managedDB = driver
-			return nil
 		}
-	}
-
-	// This is needed to allow downstreams to override driver selection logic by
-	// setting ServerConfig.Datastore.Endpoint such that it will match a driver's EndpointName
-	endpointType := strings.SplitN(c.config.Datastore.Endpoint, ":", 2)[0]
-	for _, driver := range managed.Registered() {
-		if endpointType == driver.EndpointName() {
+		if ok, err := driver.IsInitialized(); err != nil {
+			return err
+		} else if ok {
 			c.managedDB = driver
 			return nil
 		}
@@ -133,12 +120,7 @@ func (c *Cluster) assignManagedDriver(ctx context.Context) error {
 
 	// If we have been asked to initialize or join a cluster, do so using the default managed database.
 	if c.config.Datastore.Endpoint == "" && (c.config.ClusterInit || (c.config.Token != "" && c.config.JoinURL != "")) {
-		for _, driver := range managed.Registered() {
-			if driver.EndpointName() == managed.Default() {
-				c.managedDB = driver
-				return nil
-			}
-		}
+		c.managedDB = managed.Default()
 	}
 
 	return nil

pkg/cluster/managed/drivers.go

@@ -9,19 +9,21 @@ import (
 )
 
 var (
-	defaultDriver string
-	drivers       []Driver
+	drivers []Driver
 )
 
 type Driver interface {
-	IsInitialized(ctx context.Context, config *config.Control) (bool, error)
-	Register(ctx context.Context, config *config.Control, handler http.Handler) (http.Handler, error)
+	SetControlConfig(config *config.Control) error
+	IsInitialized() (bool, error)
+	Register(handler http.Handler) (http.Handler, error)
 	Reset(ctx context.Context, reboostrap func() error) error
+	IsReset() (bool, error)
+	ResetFile() string
 	Start(ctx context.Context, clientAccessInfo *clientaccess.Info) error
 	Test(ctx context.Context) error
 	Restore(ctx context.Context) error
 	EndpointName() string
-	Snapshot(ctx context.Context, config *config.Control) error
+	Snapshot(ctx context.Context) error
 	ReconcileSnapshotData(ctx context.Context) error
 	GetMembersClientURLs(ctx context.Context) ([]string, error)
 	RemoveSelf(ctx context.Context) error
@@ -35,9 +37,6 @@ func Registered() []Driver {
 	return drivers
 }
 
-func Default() string {
-	if defaultDriver == "" && len(drivers) == 1 {
-		return drivers[0].EndpointName()
-	}
-	return defaultDriver
+func Default() Driver {
+	return drivers[0]
 }

pkg/etcd/etcd_test.go

@@ -118,7 +118,11 @@ func Test_UnitETCD_IsInitialized(t *testing.T) {
 				t.Errorf("Prep for ETCD.IsInitialized() failed = %v", err)
 				return
 			}
-			got, err := e.IsInitialized(tt.args.ctx, tt.args.config)
+			if err := e.SetControlConfig(tt.args.config); err != nil {
+				t.Errorf("ETCD.SetControlConfig() failed= %v", err)
+				return
+			}
+			got, err := e.IsInitialized()
 			if (err != nil) != tt.wantErr {
 				t.Errorf("ETCD.IsInitialized() error = %v, wantErr %v", err, tt.wantErr)
 				return
@@ -170,17 +174,17 @@ func Test_UnitETCD_Register(t *testing.T) {
 				if err := testutil.GenerateRuntime(cnf); err != nil {
 					return err
 				}
-				if err := os.MkdirAll(DBDir(cnf), 0700); err != nil {
+				if err := os.MkdirAll(dbDir(cnf), 0700); err != nil {
 					return err
 				}
-				tombstoneFile := filepath.Join(DBDir(cnf), "tombstone")
+				tombstoneFile := filepath.Join(dbDir(cnf), "tombstone")
 				if _, err := os.Create(tombstoneFile); err != nil {
 					return err
 				}
 				return nil
 			},
 			teardown: func(cnf *config.Control) error {
-				tombstoneFile := filepath.Join(DBDir(cnf), "tombstone")
+				tombstoneFile := filepath.Join(dbDir(cnf), "tombstone")
 				os.Remove(tombstoneFile)
 				testutil.CleanupDataDir(cnf)
 				return nil
@@ -196,7 +200,11 @@ func Test_UnitETCD_Register(t *testing.T) {
 				t.Errorf("Setup for ETCD.Register() failed = %v", err)
 				return
 			}
-			_, err := e.Register(tt.args.ctx, tt.args.config, tt.args.handler)
+			if err := e.SetControlConfig(tt.args.config); err != nil {
+				t.Errorf("ETCD.SetControlConfig() failed = %v", err)
+				return
+			}
+			_, err := e.Register(tt.args.handler)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("ETCD.Register() error = %v, wantErr %v", err, tt.wantErr)
 				return
@@ -244,17 +252,13 @@ func Test_UnitETCD_Start(t *testing.T) {
 				ctxInfo.ctx, ctxInfo.cancel = context.WithCancel(context.Background())
 				e.config.EtcdDisableSnapshots = true
 				testutil.GenerateRuntime(e.config)
-				client, err := GetClient(ctxInfo.ctx, e.config)
-				e.client = client
-
-				return err
+				return nil
 			},
 			teardown: func(e *ETCD, ctxInfo *contextInfo) error {
 				// RemoveSelf will fail with a specific error, but it still does cleanup for testing purposes
 				if err := e.RemoveSelf(ctxInfo.ctx); err != nil && err.Error() != etcdserver.ErrNotEnoughStartedMembers.Error() {
 					return err
 				}
-				e.client.Close()
 				ctxInfo.cancel()
 				time.Sleep(10 * time.Second)
 				testutil.CleanupDataDir(e.config)
@@ -275,17 +279,13 @@ func Test_UnitETCD_Start(t *testing.T) {
 			setup: func(e *ETCD, ctxInfo *contextInfo) error {
 				ctxInfo.ctx, ctxInfo.cancel = context.WithCancel(context.Background())
 				testutil.GenerateRuntime(e.config)
-				client, err := GetClient(ctxInfo.ctx, e.config)
-				e.client = client
-
-				return err
+				return nil
 			},
 			teardown: func(e *ETCD, ctxInfo *contextInfo) error {
 				// RemoveSelf will fail with a specific error, but it still does cleanup for testing purposes
 				if err := e.RemoveSelf(ctxInfo.ctx); err != nil && err.Error() != etcdserver.ErrNotEnoughStartedMembers.Error() {
 					return err
 				}
-				e.client.Close()
 				ctxInfo.cancel()
 				time.Sleep(5 * time.Second)
 				testutil.CleanupDataDir(e.config)
@@ -308,11 +308,6 @@ func Test_UnitETCD_Start(t *testing.T) {
 				if err := testutil.GenerateRuntime(e.config); err != nil {
 					return err
 				}
-				client, err := GetClient(ctxInfo.ctx, e.config)
-				if err != nil {
-					return err
-				}
-				e.client = client
 				return os.MkdirAll(walDir(e.config), 0700)
 			},
 			teardown: func(e *ETCD, ctxInfo *contextInfo) error {
@@ -320,7 +315,6 @@ func Test_UnitETCD_Start(t *testing.T) {
 				if err := e.RemoveSelf(ctxInfo.ctx); err != nil && err.Error() != etcdserver.ErrNotEnoughStartedMembers.Error() {
 					return err
 				}
-				e.client.Close()
 				ctxInfo.cancel()
 				time.Sleep(5 * time.Second)
 				testutil.CleanupDataDir(e.config)

pkg/etcd/member_controller.go

@@ -7,8 +7,10 @@ import (
 
 	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/k3s/pkg/version"
+	"github.com/pkg/errors"
 	controllerv1 "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
 	"github.com/sirupsen/logrus"
+	"go.etcd.io/etcd/api/v3/v3rpc/rpctypes"
 	v1 "k8s.io/api/core/v1"
 )
 
@@ -49,52 +51,58 @@ func (e *etcdMemberHandler) sync(key string, node *v1.Node) (*v1.Node, error) {
 	node = node.DeepCopy()
 
 	if removalRequested, ok := node.Annotations[removalAnnotation]; ok {
+		// removal requires node name and address annotations; fail if either are not found
+		name, ok := node.Annotations[NodeNameAnnotation]
+		if !ok {
+			return node, fmt.Errorf("node name annotation for node %s not found", key)
+		}
+		address, ok := node.Annotations[NodeAddressAnnotation]
+		if !ok {
+			return node, fmt.Errorf("node address annotation for node %s not found", key)
+		}
+		lf := logrus.Fields{"name": name, "address": address}
+
+		// Check to see if the node was previously removed from the cluster
 		if removed, ok := node.Annotations[removedNodeNameAnnotation]; ok {
-			// check to see if removed is true. if it is, nothing to do.
-			if currentNodeName, ok := node.Annotations[NodeNameAnnotation]; ok {
-				if currentNodeName != removed {
-					// If the current node name is not the same as the removed node name, reset the tainted annotation and removed node name
-					logrus.Infof("Resetting removed node flag as removed node name (did not match current node name")
-					delete(node.Annotations, removedNodeNameAnnotation)
-					node.Annotations[removalAnnotation] = "false"
-					return e.nodeController.Update(node)
-				}
-				// this is the case where the current node name matches the removed node name. We have already removed the
-				// node, so no need to perform any action. Fallthrough to the non-op below.
+			if removed != name {
+				// If the current node name is not the same as the removed node name, clear the removal annotations,
+				// as this indicates that the node has been re-added with a new name.
+				logrus.WithFields(lf).Info("Resetting removed node flag as removed node name does not match current node name")
+				delete(node.Annotations, removedNodeNameAnnotation)
+				delete(node.Annotations, removalAnnotation)
+				return e.nodeController.Update(node)
 			}
-			// This is the edge case where the removed annotation exists, but there is not a current node name annotation.
-			// This should be a non-op, as we can't remove the node anyway.
-			logrus.Debugf("etcd member %s was already marked via annotations as removed", key)
+			// Current node name matches removed node name; don't need to do anything
 			return node, nil
 		}
-		if strings.ToLower(removalRequested) == "true" {
-			// remove the member.
-			name, ok := node.Annotations[NodeNameAnnotation]
-			if !ok {
-				return node, fmt.Errorf("node name annotation for node %s not found", key)
-			}
-			address, ok := node.Annotations[NodeAddressAnnotation]
-			if !ok {
-				return node, fmt.Errorf("node address annotation for node %s not found", key)
-			}
 
-			logrus.Debugf("removing etcd member from cluster name: %s address: %s", name, address)
+		if strings.ToLower(removalRequested) == "true" {
+			// Removal requested, attempt to remove it from the cluster
+			logrus.WithFields(lf).Info("Removing etcd member from cluster due to remove annotation")
 			if err := e.etcd.RemovePeer(e.ctx, name, address, true); err != nil {
+				// etcd will reject the removal if this is the only voting member; abort the removal by removing
+				// the annotation if this is the case. The requesting controller can re-request removal by setting
+				// the annotation again, once there are more cluster members.
+				if errors.Is(err, rpctypes.ErrMemberNotEnoughStarted) {
+					logrus.WithFields(lf).Errorf("etcd member removal rejected, clearing remove annotation: %v", err)
+					delete(node.Annotations, removalAnnotation)
+					return e.nodeController.Update(node)
+				}
 				return node, err
 			}
-			logrus.Debugf("etcd member removal successful for name: %s address: %s", name, address)
-			// Set the removed node name annotation and clean up the other etcd node annotations.
-			// These will be set if the tombstone file is then created and the etcd member is re-added, to their new
-			// respective values.
+
+			logrus.WithFields(lf).Info("etcd emember removed successfully")
+			// Set the removed node name annotation and delete the etcd name and address annotations.
+			// These will be re-set to their new value when the member rejoins the cluster.
 			node.Annotations[removedNodeNameAnnotation] = name
 			delete(node.Annotations, NodeNameAnnotation)
 			delete(node.Annotations, NodeAddressAnnotation)
 			return e.nodeController.Update(node)
 		}
-		// In the event that we had an unexpected removal value, simply return.
+		// In the event that we had an unexpected removal annotation value, simply return.
 		// Fallthrough to the non-op below.
 	}
-	// This is a non-op, as we don't have a tainted annotation to worry about.
+	// This is a non-op, as we don't have a deleted annotation to worry about.
 	return node, nil
 }
 
@@ -103,19 +111,13 @@ func (e *etcdMemberHandler) onRemove(key string, node *v1.Node) (*v1.Node, error
 		logrus.Debugf("Node %s was not labeled etcd node, skipping etcd member removal", key)
 		return node, nil
 	}
-	logrus.Infof("Removing etcd member %s from cluster", key)
-	if removalRequested, ok := node.Annotations[removalAnnotation]; ok {
-		if strings.ToLower(removalRequested) == "true" {
-			if removedNodeName, ok := node.Annotations[removedNodeNameAnnotation]; ok {
-				if len(removedNodeName) > 0 {
-					// If we received a node to delete that has already been removed via annotation, it will be missing
-					// the corresponding node name and address annotations.
-					logrus.Infof("etcd member %s was already removed as member name %s via annotation from the cluster", key, removedNodeName)
-					return node, nil
-				}
-			}
-		}
+
+	if removedNodeName, ok := node.Annotations[removedNodeNameAnnotation]; ok && len(removedNodeName) > 0 {
+		logrus.Debugf("Node %s was already removed from the cluster, skipping etcd member removal", key)
+		return node, nil
 	}
+
+	// removal requires node name and address annotations; fail if either are not found
 	name, ok := node.Annotations[NodeNameAnnotation]
 	if !ok {
 		return node, fmt.Errorf("node name annotation for node %s not found", key)
@@ -124,5 +126,8 @@ func (e *etcdMemberHandler) onRemove(key string, node *v1.Node) (*v1.Node, error
 	if !ok {
 		return node, fmt.Errorf("node address annotation for node %s not found", key)
 	}
+	lf := logrus.Fields{"name": name, "address": address}
+
+	logrus.WithFields(lf).Info("Removing etcd member from cluster due to node delete")
 	return node, e.etcd.RemovePeer(e.ctx, name, address, true)
 }

pkg/etcd/metadata_controller.go

@@ -3,12 +3,16 @@ package etcd
 import (
 	"context"
 	"os"
+	"sync"
 	"time"
 
 	"github.com/k3s-io/k3s/pkg/util"
 	controllerv1 "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
 	"github.com/sirupsen/logrus"
 	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/client-go/util/retry"
 )
 
 func registerMetadataHandlers(ctx context.Context, etcd *ETCD) {
@@ -17,6 +21,7 @@ func registerMetadataHandlers(ctx context.Context, etcd *ETCD) {
 		etcd:           etcd,
 		nodeController: nodes,
 		ctx:            ctx,
+		once:           &sync.Once{},
 	}
 
 	logrus.Infof("Starting managed etcd node metadata controller")
@@ -27,9 +32,11 @@ type metadataHandler struct {
 	etcd           *ETCD
 	nodeController controllerv1.NodeController
 	ctx            context.Context
+	once           *sync.Once
 }
 
 func (m *metadataHandler) sync(key string, node *v1.Node) (*v1.Node, error) {
+
 	if node == nil {
 		return nil, nil
 	}
@@ -48,6 +55,43 @@ func (m *metadataHandler) sync(key string, node *v1.Node) (*v1.Node, error) {
 	return node, nil
 }
 
+// checkReset ensures that member removal annotations are cleared when the cluster is reset.
+// This is done here instead of in the member controller, as the member removal controller is
+// not guaranteed to run on the node that was reset.
+func (m *metadataHandler) checkReset() {
+	if resetDone, _ := m.etcd.IsReset(); resetDone {
+		labelSelector := labels.Set{util.ETCDRoleLabelKey: "true"}.String()
+		nodes, err := m.nodeController.List(metav1.ListOptions{LabelSelector: labelSelector})
+		if err != nil {
+			logrus.Errorf("Failed to list etcd nodes: %v", err)
+			return
+		}
+		for _, n := range nodes.Items {
+			node := &n
+			err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
+				_, remove := node.Annotations[removalAnnotation]
+				_, removed := node.Annotations[removedNodeNameAnnotation]
+				if remove || removed {
+					node = node.DeepCopy()
+					delete(node.Annotations, removalAnnotation)
+					delete(node.Annotations, removedNodeNameAnnotation)
+					node, err = m.nodeController.Update(node)
+					return err
+				}
+				return nil
+			})
+			if err != nil {
+				logrus.Errorf("Failed to clear removal annotations from node %s after cluster reset: %v", node.Name, err)
+			} else {
+				logrus.Infof("Cleared etcd member removal annotations from node %s after cluster reset", node.Name)
+			}
+		}
+		if err := m.etcd.clearReset(); err != nil {
+			logrus.Errorf("Failed to delete etcd cluster-reset file: %v", err)
+		}
+	}
+}
+
 func (m *metadataHandler) handleSelf(node *v1.Node) (*v1.Node, error) {
 	if m.etcd.config.DisableETCD {
 		if node.Annotations[NodeNameAnnotation] == "" &&
@@ -71,6 +115,8 @@ func (m *metadataHandler) handleSelf(node *v1.Node) (*v1.Node, error) {
 		return m.nodeController.Update(node)
 	}
 
+	m.once.Do(m.checkReset)
+
 	if node.Annotations[NodeNameAnnotation] == m.etcd.name &&
 		node.Annotations[NodeAddressAnnotation] == m.etcd.address &&
 		node.Labels[util.ETCDRoleLabelKey] == "true" {

pkg/server/context.go

@@ -38,21 +38,26 @@ func (c *Context) Start(ctx context.Context) error {
 	return start.All(ctx, 5, c.K3s, c.Helm, c.Apps, c.Auth, c.Batch, c.Core)
 }
 
-func NewContext(ctx context.Context, cfg string) (*Context, error) {
+func NewContext(ctx context.Context, cfg string, forServer bool) (*Context, error) {
 	restConfig, err := clientcmd.BuildConfigFromFlags("", cfg)
 	if err != nil {
 		return nil, err
 	}
 	restConfig.UserAgent = util.GetUserAgent(version.Program + "-supervisor")
 
-	if err := crds(ctx, restConfig); err != nil {
-		return nil, errors.Wrap(err, "failed to register CRDs")
-	}
-
 	k8s, err := kubernetes.NewForConfig(restConfig)
 	if err != nil {
 		return nil, err
 	}
+
+	var recorder record.EventRecorder
+	if forServer {
+		recorder = util.BuildControllerEventRecorder(k8s, version.Program+"-supervisor", metav1.NamespaceAll)
+		if err := crds(ctx, restConfig); err != nil {
+			return nil, errors.Wrap(err, "failed to register CRDs")
+		}
+	}
+
 	return &Context{
 		K3s:   k3s.NewFactoryFromConfigOrDie(restConfig),
 		Helm:  helm.NewFactoryFromConfigOrDie(restConfig),
@@ -61,7 +66,7 @@ func NewContext(ctx context.Context, cfg string) (*Context, error) {
 		Apps:  apps.NewFactoryFromConfigOrDie(restConfig),
 		Batch: batch.NewFactoryFromConfigOrDie(restConfig),
 		Core:  core.NewFactoryFromConfigOrDie(restConfig),
-		Event: util.BuildControllerEventRecorder(k8s, version.Program+"-supervisor", metav1.NamespaceAll),
+		Event: recorder,
 	}, nil
 }
 

pkg/server/server.go

@@ -98,7 +98,7 @@ func startOnAPIServerReady(ctx context.Context, config *Config) {
 func runControllers(ctx context.Context, config *Config) error {
 	controlConfig := &config.ControlConfig
 
-	sc, err := NewContext(ctx, controlConfig.Runtime.KubeConfigSupervisor)
+	sc, err := NewContext(ctx, controlConfig.Runtime.KubeConfigSupervisor, true)
 	if err != nil {
 		return errors.Wrap(err, "failed to create new server context")
 	}