[Engine-1.21] Secrets-encryption rotation (#4656)

.github/workflows/integration.yaml

@@ -45,15 +45,8 @@ jobs:
     - name: Run Integration Tests
       run: | 
         chmod +x ./dist/artifacts/k3s
-        go test -coverpkg=./... -coverprofile=coverage.out ./pkg/... -run Integration
-        go tool cover -func coverage.out
+        go test ./pkg/... ./tests/integration/... -run Integration
     - name: On Failure, Launch Debug Session
       if: ${{ failure() }}
       uses: mxschmitt/action-tmate@v3
       timeout-minutes: 5
-    - name: Upload Results To Codecov
-      uses: codecov/codecov-action@v1
-      with:
-        files: ./coverage.out
-        flags: inttests # optional
-        verbose: true # optional (default = false)

cmd/encrypt/main.go

+package main
+
+import (
+	"context"
+	"errors"
+	"os"
+
+	"github.com/rancher/k3s/pkg/cli/cmds"
+	"github.com/rancher/k3s/pkg/cli/secretsencrypt"
+	"github.com/rancher/k3s/pkg/configfilearg"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+func main() {
+	app := cmds.NewApp()
+	app.Commands = []cli.Command{
+		cmds.NewSecretsEncryptCommand(cli.ShowAppHelp,
+			cmds.NewSecretsEncryptSubcommands(
+				secretsencrypt.Status,
+				secretsencrypt.Enable,
+				secretsencrypt.Disable,
+				secretsencrypt.Prepare,
+				secretsencrypt.Rotate,
+				secretsencrypt.Reencrypt),
+		),
+	}
+
+	if err := app.Run(configfilearg.MustParse(os.Args)); err != nil && !errors.Is(err, context.Canceled) {
+		logrus.Fatal(err)
+	}
+}

cmd/k3s/main.go

@@ -35,6 +35,7 @@ func main() {
 	}
 
 	etcdsnapshotCommand := internalCLIAction(version.Program+"-"+cmds.EtcdSnapshotCommand, dataDir, os.Args)
+	secretsencryptCommand := internalCLIAction(version.Program+"-"+cmds.SecretsEncryptCommand, dataDir, os.Args)
 	certCommand := internalCLIAction(version.Program+"-"+cmds.CertCommand, dataDir, os.Args)
 
 	// Handle subcommand invocation (k3s server, k3s crictl, etc)
@@ -53,6 +54,15 @@ func main() {
 				etcdsnapshotCommand,
 				etcdsnapshotCommand),
 		),
+		cmds.NewSecretsEncryptCommand(secretsencryptCommand,
+			cmds.NewSecretsEncryptSubcommands(
+				secretsencryptCommand,
+				secretsencryptCommand,
+				secretsencryptCommand,
+				secretsencryptCommand,
+				secretsencryptCommand,
+				secretsencryptCommand),
+		),
 		cmds.NewCertCommand(
 			cmds.NewCertSubcommands(
 				certCommand),

cmd/server/main.go

@@ -15,6 +15,7 @@ import (
 	"github.com/rancher/k3s/pkg/cli/ctr"
 	"github.com/rancher/k3s/pkg/cli/etcdsnapshot"
 	"github.com/rancher/k3s/pkg/cli/kubectl"
+	"github.com/rancher/k3s/pkg/cli/secretsencrypt"
 	"github.com/rancher/k3s/pkg/cli/server"
 	"github.com/rancher/k3s/pkg/configfilearg"
 	"github.com/rancher/k3s/pkg/containerd"
@@ -53,6 +54,15 @@ func main() {
 				etcdsnapshot.Prune,
 				etcdsnapshot.Run),
 		),
+		cmds.NewSecretsEncryptCommand(cli.ShowAppHelp,
+			cmds.NewSecretsEncryptSubcommands(
+				secretsencrypt.Status,
+				secretsencrypt.Enable,
+				secretsencrypt.Disable,
+				secretsencrypt.Prepare,
+				secretsencrypt.Rotate,
+				secretsencrypt.Reencrypt),
+		),
 		cmds.NewCertCommand(
 			cmds.NewCertSubcommands(
 				cert.Run),

main.go

@@ -17,6 +17,7 @@ import (
 	"github.com/rancher/k3s/pkg/cli/crictl"
 	"github.com/rancher/k3s/pkg/cli/etcdsnapshot"
 	"github.com/rancher/k3s/pkg/cli/kubectl"
+	"github.com/rancher/k3s/pkg/cli/secretsencrypt"
 	"github.com/rancher/k3s/pkg/cli/server"
 	"github.com/rancher/k3s/pkg/configfilearg"
 	"github.com/sirupsen/logrus"
@@ -37,6 +38,15 @@ func main() {
 				etcdsnapshot.Prune,
 				etcdsnapshot.Run),
 		),
+		cmds.NewSecretsEncryptCommand(cli.ShowAppHelp,
+			cmds.NewSecretsEncryptSubcommands(
+				secretsencrypt.Status,
+				secretsencrypt.Enable,
+				secretsencrypt.Disable,
+				secretsencrypt.Prepare,
+				secretsencrypt.Rotate,
+				secretsencrypt.Reencrypt),
+		),
 		cmds.NewCertCommand(
 			cmds.NewCertSubcommands(
 				cert.Run),

pkg/cli/cmds/agent.go

@@ -55,9 +55,15 @@ type AgentShared struct {
 }
 
 var (
-	appName     = filepath.Base(os.Args[0])
-	AgentConfig Agent
-	NodeIPFlag  = cli.StringSliceFlag{
+	appName        = filepath.Base(os.Args[0])
+	AgentConfig    Agent
+	AgentTokenFlag = cli.StringFlag{
+		Name:        "token,t",
+		Usage:       "(cluster) Token to use for authentication",
+		EnvVar:      version.ProgramUpper + "_TOKEN",
+		Destination: &AgentConfig.Token,
+	}
+	NodeIPFlag = cli.StringSliceFlag{
 		Name:  "node-ip,i",
 		Usage: "(agent/networking) IPv4/IPv6 addresses to advertise for node",
 		Value: &AgentConfig.NodeIP,
@@ -216,12 +222,7 @@ func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command {
 			VModule,
 			LogFile,
 			AlsoLogToStderr,
-			cli.StringFlag{
-				Name:        "token,t",
-				Usage:       "(cluster) Token to use for authentication",
-				EnvVar:      version.ProgramUpper + "_TOKEN",
-				Destination: &AgentConfig.Token,
-			},
+			AgentTokenFlag,
 			cli.StringFlag{
 				Name:        "token-file",
 				Usage:       "(cluster) Token file to use for authentication",

pkg/cli/cmds/etcd_snapshot.go

@@ -20,11 +20,7 @@ var EtcdSnapshotFlags = []cli.Flag{
 		EnvVar:      version.ProgramUpper + "_NODE_NAME",
 		Destination: &AgentConfig.NodeName,
 	},
-	cli.StringFlag{
-		Name:        "data-dir,d",
-		Usage:       "(data) Folder to hold state default /var/lib/rancher/" + version.Program + " or ${HOME}/.rancher/" + version.Program + " if not root",
-		Destination: &ServerConfig.DataDir,
-	},
+	DataDirFlag,
 	&cli.StringFlag{
 		Name:        "dir,etcd-snapshot-dir",
 		Usage:       "(db) Directory to save etcd on-demand snapshot. (default: ${data-dir}/db/snapshots)",

pkg/cli/cmds/secrets_encrypt.go

+package cmds
+
+import (
+	"github.com/urfave/cli"
+)
+
+const SecretsEncryptCommand = "secrets-encrypt"
+
+var EncryptFlags = []cli.Flag{
+	DataDirFlag,
+	ServerToken,
+}
+
+func NewSecretsEncryptCommand(action func(*cli.Context) error, subcommands []cli.Command) cli.Command {
+	return cli.Command{
+		Name:            SecretsEncryptCommand,
+		Usage:           "Control secrets encryption and keys rotation",
+		SkipFlagParsing: false,
+		SkipArgReorder:  true,
+		Action:          action,
+		Subcommands:     subcommands,
+	}
+}
+
+func NewSecretsEncryptSubcommands(status, enable, disable, prepare, rotate, reencrypt func(ctx *cli.Context) error) []cli.Command {
+	return []cli.Command{
+		{
+			Name:            "status",
+			Usage:           "Print current status of secrets encryption",
+			SkipFlagParsing: false,
+			SkipArgReorder:  true,
+			Action:          status,
+			Flags:           EncryptFlags,
+		},
+		{
+			Name:            "enable",
+			Usage:           "Enable secrets encryption",
+			SkipFlagParsing: false,
+			SkipArgReorder:  true,
+			Action:          enable,
+			Flags:           EncryptFlags,
+		},
+		{
+			Name:            "disable",
+			Usage:           "Disable secrets encryption",
+			SkipFlagParsing: false,
+			SkipArgReorder:  true,
+			Action:          disable,
+			Flags:           EncryptFlags,
+		},
+		{
+			Name:            "prepare",
+			Usage:           "Prepare for encryption keys rotation",
+			SkipFlagParsing: false,
+			SkipArgReorder:  true,
+			Action:          prepare,
+			Flags: append(EncryptFlags, &cli.BoolFlag{
+				Name:        "f,force",
+				Usage:       "Force preparation.",
+				Destination: &ServerConfig.EncryptForce,
+			}),
+		},
+		{
+			Name:            "rotate",
+			Usage:           "Rotate secrets encryption keys",
+			SkipFlagParsing: false,
+			SkipArgReorder:  true,
+			Action:          rotate,
+			Flags: append(EncryptFlags, &cli.BoolFlag{
+				Name:        "f,force",
+				Usage:       "Force key rotation.",
+				Destination: &ServerConfig.EncryptForce,
+			}),
+		},
+		{
+			Name:            "reencrypt",
+			Usage:           "Reencrypt all data with new encryption key",
+			SkipFlagParsing: false,
+			SkipArgReorder:  true,
+			Action:          reencrypt,
+			Flags: append(EncryptFlags,
+				&cli.BoolFlag{
+					Name:        "f,force",
+					Usage:       "Force secrets reencryption.",
+					Destination: &ServerConfig.EncryptForce,
+				},
+				&cli.BoolFlag{
+					Name:        "skip",
+					Usage:       "Skip removing old key",
+					Destination: &ServerConfig.EncryptSkip,
+				}),
+		},
+	}
+}

pkg/cli/cmds/server.go

@@ -74,6 +74,8 @@ type Server struct {
 	ClusterReset             bool
 	ClusterResetRestorePath  string
 	EncryptSecrets           bool
+	EncryptForce             bool
+	EncryptSkip              bool
 	SystemDefaultRegistry    string
 	StartupHooks             []StartupHook
 	EtcdSnapshotName         string
@@ -97,7 +99,18 @@ type Server struct {
 
 var (
 	ServerConfig Server
-	ClusterCIDR  = cli.StringSliceFlag{
+	DataDirFlag  = cli.StringFlag{
+		Name:        "data-dir,d",
+		Usage:       "(data) Folder to hold state default /var/lib/rancher/" + version.Program + " or ${HOME}/.rancher/" + version.Program + " if not root",
+		Destination: &ServerConfig.DataDir,
+	}
+	ServerToken = cli.StringFlag{
+		Name:        "token,t",
+		Usage:       "(cluster) Shared secret used to join a server or agent to a cluster",
+		Destination: &ServerConfig.Token,
+		EnvVar:      version.ProgramUpper + "_TOKEN",
+	}
+	ClusterCIDR = cli.StringSliceFlag{
 		Name:  "cluster-cidr",
 		Usage: "(networking) IPv4/IPv6 network CIDRs to use for pod IPs (default: 10.42.0.0/16)",
 		Value: &ServerConfig.ClusterCIDR,
@@ -179,11 +192,7 @@ var ServerFlags = []cli.Flag{
 		Usage: "(listener) Add additional hostnames or IPv4/IPv6 addresses as Subject Alternative Names on the server TLS cert",
 		Value: &ServerConfig.TLSSan,
 	},
-	cli.StringFlag{
-		Name:        "data-dir,d",
-		Usage:       "(data) Folder to hold state default /var/lib/rancher/" + version.Program + " or ${HOME}/.rancher/" + version.Program + " if not root",
-		Destination: &ServerConfig.DataDir,
-	},
+	DataDirFlag,
 	ClusterCIDR,
 	ServiceCIDR,
 	ServiceNodePortRange,
@@ -195,12 +204,7 @@ var ServerFlags = []cli.Flag{
 		Destination: &ServerConfig.FlannelBackend,
 		Value:       "vxlan",
 	},
-	cli.StringFlag{
-		Name:        "token,t",
-		Usage:       "(cluster) Shared secret used to join a server or agent to a cluster",
-		Destination: &ServerConfig.Token,
-		EnvVar:      version.ProgramUpper + "_TOKEN",
-	},
+	ServerToken,
 	cli.StringFlag{
 		Name:        "token-file",
 		Usage:       "(cluster) File containing the cluster-secret/token",

pkg/cli/secretsencrypt/secrets_encrypt.go

+package secretsencrypt
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"text/tabwriter"
+
+	"github.com/erikdubbelboer/gspt"
+	"github.com/rancher/k3s/pkg/cli/cmds"
+	"github.com/rancher/k3s/pkg/clientaccess"
+	"github.com/rancher/k3s/pkg/daemons/config"
+	"github.com/rancher/k3s/pkg/secretsencrypt"
+	"github.com/rancher/k3s/pkg/server"
+	"github.com/rancher/k3s/pkg/version"
+	"github.com/urfave/cli"
+	"k8s.io/utils/pointer"
+)
+
+func commandPrep(app *cli.Context, cfg *cmds.Server) (config.Control, *clientaccess.Info, error) {
+	var controlConfig config.Control
+	var err error
+	// hide process arguments from ps output, since they may contain
+	// database credentials or other secrets.
+	gspt.SetProcTitle(os.Args[0] + " encrypt")
+
+	controlConfig.DataDir, err = server.ResolveDataDir(cfg.DataDir)
+	if err != nil {
+		return controlConfig, nil, err
+	}
+	if cfg.ServerURL == "" {
+		cfg.ServerURL = "https://127.0.0.1:6443"
+	}
+
+	if cfg.Token == "" {
+		fp := filepath.Join(controlConfig.DataDir, "token")
+		tokenByte, err := ioutil.ReadFile(fp)
+		if err != nil {
+			return controlConfig, nil, err
+		}
+		controlConfig.Token = string(bytes.TrimRight(tokenByte, "\n"))
+	} else {
+		controlConfig.Token = cfg.Token
+	}
+	controlConfig.EncryptForce = cfg.EncryptForce
+	controlConfig.EncryptSkip = cfg.EncryptSkip
+	info, err := clientaccess.ParseAndValidateTokenForUser(cmds.ServerConfig.ServerURL, controlConfig.Token, "node")
+	if err != nil {
+		return controlConfig, nil, err
+	}
+	return controlConfig, info, nil
+}
+
+func Enable(app *cli.Context) error {
+	var err error
+	if err = cmds.InitLogging(); err != nil {
+		return err
+	}
+	_, info, err := commandPrep(app, &cmds.ServerConfig)
+	if err != nil {
+		return err
+	}
+	b, err := json.Marshal(server.EncryptionRequest{Enable: pointer.Bool(true)})
+	if err != nil {
+		return err
+	}
+	if err = info.Put("/v1-"+version.Program+"/encrypt/config", b); err != nil {
+		return err
+	}
+	fmt.Println("secrets-encryption enabled")
+	return nil
+}
+
+func Disable(app *cli.Context) error {
+
+	if err := cmds.InitLogging(); err != nil {
+		return err
+	}
+	_, info, err := commandPrep(app, &cmds.ServerConfig)
+	if err != nil {
+		return err
+	}
+	b, err := json.Marshal(server.EncryptionRequest{Enable: pointer.Bool(false)})
+	if err != nil {
+		return err
+	}
+	if err = info.Put("/v1-"+version.Program+"/encrypt/config", b); err != nil {
+		return err
+	}
+	fmt.Println("secrets-encryption disabled")
+	return nil
+}
+
+func Status(app *cli.Context) error {
+	if err := cmds.InitLogging(); err != nil {
+		return err
+	}
+	_, info, err := commandPrep(app, &cmds.ServerConfig)
+	if err != nil {
+		return err
+	}
+	data, err := info.Get("/v1-" + version.Program + "/encrypt/status")
+	if err != nil {
+		return err
+	}
+	status := server.EncryptionState{}
+	if err := json.Unmarshal(data, &status); err != nil {
+		return err
+	}
+
+	if status.Enable == nil {
+		fmt.Println("Encryption Status: Disabled, no configuration file found")
+		return nil
+	}
+
+	var statusOutput string
+	if *status.Enable {
+		statusOutput += "Encryption Status: Enabled\n"
+	} else {
+		statusOutput += "Encryption Status: Disabled\n"
+	}
+	statusOutput += fmt.Sprintln("Current Rotation Stage:", status.Stage)
+
+	if status.HashMatch {
+		statusOutput += fmt.Sprintln("Server Encryption Hashes: All hashes match")
+	} else {
+		statusOutput += fmt.Sprintf("Server Encryption Hashes: %s\n", status.HashError)
+	}
+
+	var tabBuffer bytes.Buffer
+	w := tabwriter.NewWriter(&tabBuffer, 0, 0, 2, ' ', 0)
+	fmt.Fprintf(w, "\n")
+	fmt.Fprintf(w, "Active\tKey Type\tName\n")
+	fmt.Fprintf(w, "------\t--------\t----\n")
+	if status.ActiveKey != "" {
+		fmt.Fprintf(w, " *\t%s\t%s\n", "AES-CBC", status.ActiveKey)
+	}
+	for _, k := range status.InactiveKeys {
+		fmt.Fprintf(w, "\t%s\t%s\n", "AES-CBC", k)
+	}
+	w.Flush()
+	fmt.Println(statusOutput + tabBuffer.String())
+	return nil
+}
+
+func Prepare(app *cli.Context) error {
+	var err error
+	if err = cmds.InitLogging(); err != nil {
+		return err
+	}
+	controlConfig, info, err := commandPrep(app, &cmds.ServerConfig)
+	if err != nil {
+		return err
+	}
+	b, err := json.Marshal(server.EncryptionRequest{
+		Stage: pointer.StringPtr(secretsencrypt.EncryptionPrepare),
+		Force: controlConfig.EncryptForce,
+	})
+	if err != nil {
+		return err
+	}
+	if err = info.Put("/v1-"+version.Program+"/encrypt/config", b); err != nil {
+		return err
+	}
+	fmt.Println("prepare completed successfully")
+	return nil
+}
+
+func Rotate(app *cli.Context) error {
+	if err := cmds.InitLogging(); err != nil {
+		return err
+	}
+	controlConfig, info, err := commandPrep(app, &cmds.ServerConfig)
+	if err != nil {
+		return err
+	}
+	b, err := json.Marshal(server.EncryptionRequest{
+		Stage: pointer.StringPtr(secretsencrypt.EncryptionRotate),
+		Force: controlConfig.EncryptForce,
+	})
+	if err != nil {
+		return err
+	}
+	if err = info.Put("/v1-"+version.Program+"/encrypt/config", b); err != nil {
+		return err
+	}
+	fmt.Println("rotate completed successfully")
+	return nil
+}
+
+func Reencrypt(app *cli.Context) error {
+	var err error
+	if err = cmds.InitLogging(); err != nil {
+		return err
+	}
+	controlConfig, info, err := commandPrep(app, &cmds.ServerConfig)
+	if err != nil {
+		return err
+	}
+	b, err := json.Marshal(server.EncryptionRequest{
+		Stage: pointer.StringPtr(secretsencrypt.EncryptionReencryptActive),
+		Force: controlConfig.EncryptForce,
+		Skip:  controlConfig.EncryptSkip,
+	})
+	if err != nil {
+		return err
+	}
+	if err = info.Put("/v1-"+version.Program+"/encrypt/config", b); err != nil {
+		return err
+	}
+	fmt.Println("reencryption started")
+	return nil
+}

pkg/clientaccess/token.go

 package clientaccess
 
 import (
+	"bytes"
 	"crypto/sha256"
 	"crypto/tls"
 	"crypto/x509"
@@ -186,6 +187,16 @@ func (i *Info) Get(path string) ([]byte, error) {
 	return get(u.String(), GetHTTPClient(i.CACerts), i.Username, i.Password)
 }
 
+// Put makes a request to a subpath of info's BaseURL
+func (i *Info) Put(path string, body []byte) error {
+	u, err := url.Parse(i.BaseURL)
+	if err != nil {
+		return err
+	}
+	u.Path = path
+	return put(u.String(), body, GetHTTPClient(i.CACerts), i.Username, i.Password)
+}
+
 // setServer sets the BaseURL and CACerts fields of the Info by connecting to the server
 // and storing the CA bundle.
 func (i *Info) setServer(server string) error {
@@ -288,6 +299,32 @@ func get(u string, client *http.Client, username, password string) ([]byte, erro
 	return ioutil.ReadAll(resp.Body)
 }
 
+// put makes a request to a url using a provided client, username, and password
+// only an error is returned
+func put(u string, body []byte, client *http.Client, username, password string) error {
+	req, err := http.NewRequest(http.MethodPut, u, bytes.NewBuffer(body))
+	if err != nil {
+		return err
+	}
+
+	if username != "" {
+		req.SetBasicAuth(username, password)
+	}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	respBody, _ := ioutil.ReadAll(resp.Body)
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("%s: %s %s", u, resp.Status, string(respBody))
+	}
+
+	return nil
+}
+
 func FormatToken(token, certFile string) (string, error) {
 	if len(token) == 0 {
 		return token, nil

pkg/cluster/bootstrap.go

@@ -188,17 +188,18 @@ func (c *Cluster) shouldBootstrapLoad(ctx context.Context) (bool, bool, error) {
 		if err != nil {
 			return false, false, err
 		}
-
 		if isInitialized {
-			// If the database is initialized we skip bootstrapping; if the user wants to rejoin a
-			// cluster they need to delete the database.
-			logrus.Infof("Managed %s cluster bootstrap already complete and initialized", c.managedDB.EndpointName())
 			// This is a workaround for an issue that can be caused by terminating the cluster bootstrap before
 			// etcd is promoted from learner. Odds are we won't need this info, and we don't want to fail startup
 			// due to failure to retrieve it as this will break cold cluster restart, so we ignore any errors.
 			if c.config.JoinURL != "" && c.config.Token != "" {
 				c.clientAccessInfo, _ = clientaccess.ParseAndValidateTokenForUser(c.config.JoinURL, c.config.Token, "server")
+				logrus.Infof("Joining %s cluster already initialized, forcing reconciliation", c.managedDB.EndpointName())
+				return true, true, nil
 			}
+			// If the database is initialized we skip bootstrapping; if the user wants to rejoin a
+			// cluster they need to delete the database.
+			logrus.Infof("Managed %s cluster bootstrap already complete and initialized", c.managedDB.EndpointName())
 			return false, true, nil
 		} else if c.config.JoinURL == "" {
 			// Not initialized, not joining - must be initializing (cluster-init)
@@ -353,7 +354,7 @@ func (c *Cluster) ReconcileBootstrapData(ctx context.Context, buf io.ReadSeeker,
 		if ec != nil {
 			etcdConfig = *ec
 		} else {
-			etcdConfig = c.etcdConfig
+			etcdConfig = c.EtcdConfig
 		}
 
 		storageClient, err := client.New(etcdConfig)
@@ -366,7 +367,7 @@ func (c *Cluster) ReconcileBootstrapData(ctx context.Context, buf io.ReadSeeker,
 
 	RETRY:
 		for {
-			value, err = c.getBootstrapKeyFromStorage(ctx, storageClient, normalizedToken, token)
+			value, c.saveBootstrap, err = getBootstrapKeyFromStorage(ctx, storageClient, normalizedToken, token)
 			if err != nil {
 				if strings.Contains(err.Error(), "not supported for learner") {
 					for range ticker.C {

pkg/cluster/bootstrap_test.go

@@ -130,7 +130,7 @@ func TestCluster_certDirsExist(t *testing.T) {
 				config:           tt.fields.config,
 				runtime:          tt.fields.runtime,
 				managedDB:        tt.fields.managedDB,
-				etcdConfig:       tt.fields.etcdConfig,
+				EtcdConfig:       tt.fields.etcdConfig,
 				storageStarted:   tt.fields.storageStarted,
 				saveBootstrap:    tt.fields.saveBootstrap,
 			}
@@ -240,7 +240,7 @@ func TestCluster_Snapshot(t *testing.T) {
 				config:           tt.fields.config,
 				runtime:          tt.fields.runtime,
 				managedDB:        tt.fields.managedDB,
-				etcdConfig:       tt.fields.etcdConfig,
+				EtcdConfig:       tt.fields.etcdConfig,
 				joining:          tt.fields.joining,
 				storageStarted:   tt.fields.storageStarted,
 				saveBootstrap:    tt.fields.saveBootstrap,

pkg/cluster/cluster.go

@@ -20,7 +20,7 @@ type Cluster struct {
 	config           *config.Control
 	runtime          *config.ControlRuntime
 	managedDB        managed.Driver
-	etcdConfig       endpoint.ETCDConfig
+	EtcdConfig       endpoint.ETCDConfig
 	joining          bool
 	storageStarted   bool
 	saveBootstrap    bool
@@ -85,7 +85,7 @@ func (c *Cluster) Start(ctx context.Context) (<-chan struct{}, error) {
 
 	// if necessary, store bootstrap data to datastore
 	if c.saveBootstrap {
-		if err := c.save(ctx, false); err != nil {
+		if err := Save(ctx, c.config, c.EtcdConfig, false); err != nil {
 			return nil, err
 		}
 	}
@@ -99,7 +99,7 @@ func (c *Cluster) Start(ctx context.Context) (<-chan struct{}, error) {
 			for {
 				select {
 				case <-ready:
-					if err := c.save(ctx, false); err != nil {
+					if err := Save(ctx, c.config, c.EtcdConfig, false); err != nil {
 						panic(err)
 					}
 
@@ -139,7 +139,7 @@ func (c *Cluster) startStorage(ctx context.Context) error {
 	// Persist the returned etcd configuration. We decide if we're doing leader election for embedded controllers
 	// based on what the kine wrapper tells us about the datastore. Single-node datastores like sqlite don't require
 	// leader election, while basically all others (etcd, external database, etc) do since they allow multiple servers.
-	c.etcdConfig = etcdConfig
+	c.EtcdConfig = etcdConfig
 	c.config.Datastore.Config = etcdConfig.TLSConfig
 	c.config.Datastore.Endpoint = strings.Join(etcdConfig.Endpoints, ",")
 	c.config.NoLeaderElect = !etcdConfig.LeaderElect

pkg/cluster/storage.go

@@ -10,23 +10,25 @@ import (
 	"strings"
 
 	"github.com/k3s-io/kine/pkg/client"
+	"github.com/k3s-io/kine/pkg/endpoint"
 	"github.com/rancher/k3s/pkg/bootstrap"
 	"github.com/rancher/k3s/pkg/clientaccess"
+	"github.com/rancher/k3s/pkg/daemons/config"
 	"github.com/sirupsen/logrus"
 )
 
-// save writes the current ControlRuntimeBootstrap data to the datastore. This contains a complete
+// Save writes the current ControlRuntimeBootstrap data to the datastore. This contains a complete
 // snapshot of the cluster's CA certs and keys, encryption passphrases, etc - encrypted with the join token.
 // This is used when bootstrapping a cluster from a managed database or external etcd cluster.
 // This is NOT used with embedded etcd, which bootstraps over HTTP.
-func (c *Cluster) save(ctx context.Context, override bool) error {
+func Save(ctx context.Context, config *config.Control, etcdConfig endpoint.ETCDConfig, override bool) error {
 	buf := &bytes.Buffer{}
-	if err := bootstrap.ReadFromDisk(buf, &c.runtime.ControlRuntimeBootstrap); err != nil {
+	if err := bootstrap.ReadFromDisk(buf, &config.Runtime.ControlRuntimeBootstrap); err != nil {
 		return err
 	}
-	token := c.config.Token
+	token := config.Token
 	if token == "" {
-		tokenFromFile, err := readTokenFromFile(c.runtime.ServerToken, c.runtime.ServerCA, c.config.DataDir)
+		tokenFromFile, err := readTokenFromFile(config.Runtime.ServerToken, config.Runtime.ServerCA, config.DataDir)
 		if err != nil {
 			return err
 		}
@@ -42,12 +44,12 @@ func (c *Cluster) save(ctx context.Context, override bool) error {
 		return err
 	}
 
-	storageClient, err := client.New(c.etcdConfig)
+	storageClient, err := client.New(etcdConfig)
 	if err != nil {
 		return err
 	}
 
-	if _, err := c.getBootstrapKeyFromStorage(ctx, storageClient, normalizedToken, token); err != nil {
+	if _, _, err = getBootstrapKeyFromStorage(ctx, storageClient, normalizedToken, token); err != nil {
 		return err
 	}
 
@@ -55,7 +57,7 @@ func (c *Cluster) save(ctx context.Context, override bool) error {
 		if err.Error() == "key exists" {
 			logrus.Warn("bootstrap key already exists")
 			if override {
-				bsd, err := c.bootstrapKeyData(ctx, storageClient)
+				bsd, err := bootstrapKeyData(ctx, storageClient)
 				if err != nil {
 					return err
 				}
@@ -74,7 +76,7 @@ func (c *Cluster) save(ctx context.Context, override bool) error {
 
 // bootstrapKeyData lists keys stored in the datastore with the prefix "/bootstrap", and
 // will return the first such key. It will return an error if not exactly one key is found.
-func (c *Cluster) bootstrapKeyData(ctx context.Context, storageClient client.Client) (*client.Value, error) {
+func bootstrapKeyData(ctx context.Context, storageClient client.Client) (*client.Value, error) {
 	bootstrapList, err := storageClient.List(ctx, "/bootstrap", 0)
 	if err != nil {
 		return nil, err
@@ -96,7 +98,7 @@ func (c *Cluster) storageBootstrap(ctx context.Context) error {
 		return err
 	}
 
-	storageClient, err := client.New(c.etcdConfig)
+	storageClient, err := client.New(c.EtcdConfig)
 	if err != nil {
 		return err
 	}
@@ -119,7 +121,8 @@ func (c *Cluster) storageBootstrap(ctx context.Context) error {
 		return err
 	}
 
-	value, err := c.getBootstrapKeyFromStorage(ctx, storageClient, normalizedToken, token)
+	value, saveBootstrap, err := getBootstrapKeyFromStorage(ctx, storageClient, normalizedToken, token)
+	c.saveBootstrap = saveBootstrap
 	if err != nil {
 		return err
 	}
@@ -139,38 +142,37 @@ func (c *Cluster) storageBootstrap(ctx context.Context) error {
 // hashed with empty string and will check for any key that is hashed by different token than the one
 // passed to it, it will return error if it finds a key that is hashed with different token and will return
 // value if it finds the key hashed by passed token or empty string
-func (c *Cluster) getBootstrapKeyFromStorage(ctx context.Context, storageClient client.Client, normalizedToken, oldToken string) (*client.Value, error) {
+func getBootstrapKeyFromStorage(ctx context.Context, storageClient client.Client, normalizedToken, oldToken string) (*client.Value, bool, error) {
 	emptyStringKey := storageKey("")
 	tokenKey := storageKey(normalizedToken)
 	bootstrapList, err := storageClient.List(ctx, "/bootstrap", 0)
 	if err != nil {
-		return nil, err
+		return nil, false, err
 	}
 	if len(bootstrapList) == 0 {
-		c.saveBootstrap = true
-		return nil, nil
+		return nil, true, nil
 	}
 	if len(bootstrapList) > 1 {
 		logrus.Warn("found multiple bootstrap keys in storage")
 	}
 	// check for empty string key and for old token format with k10 prefix
-	if err := c.migrateOldTokens(ctx, bootstrapList, storageClient, emptyStringKey, tokenKey, normalizedToken, oldToken); err != nil {
-		return nil, err
+	if err := migrateOldTokens(ctx, bootstrapList, storageClient, emptyStringKey, tokenKey, normalizedToken, oldToken); err != nil {
+		return nil, false, err
 	}
 
 	// getting the list of bootstrap again after migrating the empty key
 	bootstrapList, err = storageClient.List(ctx, "/bootstrap", 0)
 	if err != nil {
-		return nil, err
+		return nil, false, err
 	}
 	for _, bootstrapKV := range bootstrapList {
 		// ensure bootstrap is stored in the current token's key
 		if string(bootstrapKV.Key) == tokenKey {
-			return &bootstrapKV, nil
+			return &bootstrapKV, false, nil
 		}
 	}
 
-	return nil, errors.New("bootstrap data already found and encrypted with different token")
+	return nil, false, errors.New("bootstrap data already found and encrypted with different token")
 }
 
 // readTokenFromFile will attempt to get the token from <data-dir>/token if it the file not found
@@ -209,7 +211,7 @@ func normalizeToken(token string) (string, error) {
 // migrateOldTokens will list all keys that has prefix /bootstrap and will check for key that is
 // hashed with empty string and keys that is hashed with old token format before normalizing
 // then migrate those and resave only with the normalized token
-func (c *Cluster) migrateOldTokens(ctx context.Context, bootstrapList []client.Value, storageClient client.Client, emptyStringKey, tokenKey, token, oldToken string) error {
+func migrateOldTokens(ctx context.Context, bootstrapList []client.Value, storageClient client.Client, emptyStringKey, tokenKey, token, oldToken string) error {
 	oldTokenKey := storageKey(oldToken)
 
 	for _, bootstrapKV := range bootstrapList {

pkg/daemons/config/types.go

@@ -151,6 +151,8 @@ type Control struct {
 	ClusterReset             bool
 	ClusterResetRestorePath  string
 	EncryptSecrets           bool
+	EncryptForce             bool
+	EncryptSkip              bool
 	TLSMinVersion            uint16
 	TLSCipherSuites          []uint16
 	EtcdSnapshotName         string
@@ -193,6 +195,7 @@ type ControlRuntimeBootstrap struct {
 	RequestHeaderCAKey string
 	IPSECKey           string
 	EncryptionConfig   string
+	EncryptionHash     string
 }
 
 type ControlRuntime struct {
@@ -249,7 +252,8 @@ type ControlRuntime struct {
 	ClientETCDCert           string
 	ClientETCDKey            string
 
-	Core *core.Factory
+	Core       *core.Factory
+	EtcdConfig endpoint.ETCDConfig
 }
 
 type ArgString []string

pkg/daemons/control/deps/deps.go

@@ -3,8 +3,10 @@ package deps
 import (
 	"crypto"
 	cryptorand "crypto/rand"
+	"crypto/sha256"
 	"crypto/x509"
 	b64 "encoding/base64"
+	"encoding/hex"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -147,6 +149,7 @@ func CreateRuntimeCertFiles(config *config.Control, runtime *config.ControlRunti
 
 	if config.EncryptSecrets {
 		runtime.EncryptionConfig = filepath.Join(config.DataDir, "cred", "encryption-config.json")
+		runtime.EncryptionHash = filepath.Join(config.DataDir, "cred", "encryption-state.json")
 	}
 }
 
@@ -169,7 +172,7 @@ func GenServerDeps(config *config.Control, runtime *config.ControlRuntime) error
 		return err
 	}
 
-	if err := genEncryptionConfig(config, runtime); err != nil {
+	if err := genEncryptionConfigAndState(config, runtime); err != nil {
 		return err
 	}
 
@@ -653,7 +656,7 @@ func expired(certFile string, pool *x509.CertPool) bool {
 	return certutil.IsCertExpired(certificates[0], config.CertificateRenewDays)
 }
 
-func genEncryptionConfig(controlConfig *config.Control, runtime *config.ControlRuntime) error {
+func genEncryptionConfigAndState(controlConfig *config.Control, runtime *config.ControlRuntime) error {
 	if !controlConfig.EncryptSecrets {
 		return nil
 	}
@@ -694,9 +697,14 @@ func genEncryptionConfig(controlConfig *config.Control, runtime *config.ControlR
 			},
 		},
 	}
-	jsonfile, err := json.Marshal(encConfig)
+	b, err := json.Marshal(encConfig)
 	if err != nil {
 		return err
 	}
-	return ioutil.WriteFile(runtime.EncryptionConfig, jsonfile, 0600)
+	if err := ioutil.WriteFile(runtime.EncryptionConfig, b, 0600); err != nil {
+		return err
+	}
+	encryptionConfigHash := sha256.Sum256(b)
+	ann := "start-" + hex.EncodeToString(encryptionConfigHash[:])
+	return ioutil.WriteFile(controlConfig.Runtime.EncryptionHash, []byte(ann), 0600)
 }

pkg/daemons/control/server.go

@@ -257,6 +257,7 @@ func prepare(ctx context.Context, config *config.Control, runtime *config.Contro
 	}
 
 	runtime.ETCDReady = ready
+	runtime.EtcdConfig = cluster.EtcdConfig
 	return nil
 }
 

pkg/deploy/controller.go

@@ -18,10 +18,10 @@ import (
 	"github.com/rancher/k3s/pkg/agent/util"
 	apisv1 "github.com/rancher/k3s/pkg/apis/k3s.cattle.io/v1"
 	controllersv1 "github.com/rancher/k3s/pkg/generated/controllers/k3s.cattle.io/v1"
+	pkgutil "github.com/rancher/k3s/pkg/util"
 	"github.com/rancher/wrangler/pkg/apply"
 	"github.com/rancher/wrangler/pkg/merr"
 	"github.com/rancher/wrangler/pkg/objectset"
-	"github.com/rancher/wrangler/pkg/schemes"
 	"github.com/sirupsen/logrus"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
@@ -31,7 +31,6 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	yamlDecoder "k8s.io/apimachinery/pkg/util/yaml"
 	"k8s.io/client-go/kubernetes"
-	typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1"
 	"k8s.io/client-go/tools/record"
 )
 
@@ -74,12 +73,7 @@ type watcher struct {
 
 // start calls listFiles at regular intervals to trigger application of manifests that have changed on disk.
 func (w *watcher) start(ctx context.Context, client kubernetes.Interface) {
-	nodeName := os.Getenv("NODE_NAME")
-	broadcaster := record.NewBroadcaster()
-	broadcaster.StartLogging(logrus.Infof)
-	broadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: client.CoreV1().Events(metav1.NamespaceSystem)})
-	w.recorder = broadcaster.NewRecorder(schemes.All, corev1.EventSource{Component: ControllerName, Host: nodeName})
-
+	w.recorder = pkgutil.BuildControllerEventRecorder(client, ControllerName)
 	force := true
 	for {
 		if err := w.listFiles(force); err == nil {

pkg/secretsencrypt/config.go

+package secretsencrypt
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+
+	"github.com/rancher/k3s/pkg/daemons/config"
+	"github.com/rancher/k3s/pkg/version"
+	corev1 "k8s.io/api/core/v1"
+
+	"github.com/sirupsen/logrus"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	apiserverconfigv1 "k8s.io/apiserver/pkg/apis/config/v1"
+)
+
+const (
+	EncryptionStart             string = "start"
+	EncryptionPrepare           string = "prepare"
+	EncryptionRotate            string = "rotate"
+	EncryptionReencryptRequest  string = "reencrypt_request"
+	EncryptionReencryptActive   string = "reencrypt_active"
+	EncryptionReencryptFinished string = "reencrypt_finished"
+)
+
+var EncryptionHashAnnotation = version.Program + ".io/encryption-config-hash"
+
+func GetEncryptionProviders(runtime *config.ControlRuntime) ([]apiserverconfigv1.ProviderConfiguration, error) {
+	curEncryptionByte, err := ioutil.ReadFile(runtime.EncryptionConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	curEncryption := apiserverconfigv1.EncryptionConfiguration{}
+	if err = json.Unmarshal(curEncryptionByte, &curEncryption); err != nil {
+		return nil, err
+	}
+	return curEncryption.Resources[0].Providers, nil
+}
+
+func GetEncryptionKeys(runtime *config.ControlRuntime) ([]apiserverconfigv1.Key, error) {
+
+	providers, err := GetEncryptionProviders(runtime)
+	if err != nil {
+		return nil, err
+	}
+	if len(providers) > 2 {
+		return nil, fmt.Errorf("more than 2 providers (%d) found in secrets encryption", len(providers))
+	}
+
+	var curKeys []apiserverconfigv1.Key
+	for _, p := range providers {
+		if p.AESCBC != nil {
+			curKeys = append(curKeys, p.AESCBC.Keys...)
+		}
+		if p.AESGCM != nil || p.KMS != nil || p.Secretbox != nil {
+			return nil, fmt.Errorf("non-standard encryption keys found")
+		}
+	}
+	return curKeys, nil
+}
+
+func WriteEncryptionConfig(runtime *config.ControlRuntime, keys []apiserverconfigv1.Key, enable bool) error {
+
+	// Placing the identity provider first disables encryption
+	var providers []apiserverconfigv1.ProviderConfiguration
+	if enable {
+		providers = []apiserverconfigv1.ProviderConfiguration{
+			{
+				AESCBC: &apiserverconfigv1.AESConfiguration{
+					Keys: keys,
+				},
+			},
+			{
+				Identity: &apiserverconfigv1.IdentityConfiguration{},
+			},
+		}
+	} else {
+		providers = []apiserverconfigv1.ProviderConfiguration{
+			{
+				Identity: &apiserverconfigv1.IdentityConfiguration{},
+			},
+			{
+				AESCBC: &apiserverconfigv1.AESConfiguration{
+					Keys: keys,
+				},
+			},
+		}
+	}
+
+	encConfig := apiserverconfigv1.EncryptionConfiguration{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "EncryptionConfiguration",
+			APIVersion: "apiserver.config.k8s.io/v1",
+		},
+		Resources: []apiserverconfigv1.ResourceConfiguration{
+			{
+				Resources: []string{"secrets"},
+				Providers: providers,
+			},
+		},
+	}
+	jsonfile, err := json.Marshal(encConfig)
+	if err != nil {
+		return err
+	}
+	return ioutil.WriteFile(runtime.EncryptionConfig, jsonfile, 0600)
+}
+
+func GenEncryptionConfigHash(runtime *config.ControlRuntime) (string, error) {
+	curEncryptionByte, err := ioutil.ReadFile(runtime.EncryptionConfig)
+	if err != nil {
+		return "", err
+	}
+	encryptionConfigHash := sha256.Sum256(curEncryptionByte)
+	return hex.EncodeToString(encryptionConfigHash[:]), nil
+}
+
+// GenReencryptHash generates a sha256 hash fom the existing secrets keys and
+// a new key based on the input arguments.
+func GenReencryptHash(runtime *config.ControlRuntime, keyName string) (string, error) {
+
+	keys, err := GetEncryptionKeys(runtime)
+	if err != nil {
+		return "", err
+	}
+	newKey := apiserverconfigv1.Key{
+		Name:   keyName,
+		Secret: "12345",
+	}
+	keys = append(keys, newKey)
+	b, err := json.Marshal(keys)
+	if err != nil {
+		return "", err
+	}
+	hash := sha256.Sum256(b)
+	return hex.EncodeToString(hash[:]), nil
+}
+
+func getEncryptionHashFile(runtime *config.ControlRuntime) (string, error) {
+	curEncryptionByte, err := ioutil.ReadFile(runtime.EncryptionHash)
+	if err != nil {
+		return "", err
+	}
+	return string(curEncryptionByte), nil
+}
+
+func BootstrapEncryptionHashAnnotation(node *corev1.Node, runtime *config.ControlRuntime) error {
+	existingAnn, err := getEncryptionHashFile(runtime)
+	if err != nil {
+		return err
+	}
+	node.Annotations[EncryptionHashAnnotation] = existingAnn
+	return nil
+}
+
+func WriteEncryptionHashAnnotation(runtime *config.ControlRuntime, node *corev1.Node, stage string) error {
+	encryptionConfigHash, err := GenEncryptionConfigHash(runtime)
+	if err != nil {
+		return err
+	}
+	if node.Annotations == nil {
+		return fmt.Errorf("node annotations do not exist for %s", node.ObjectMeta.Name)
+	}
+	ann := stage + "-" + encryptionConfigHash
+	node.Annotations[EncryptionHashAnnotation] = ann
+	if _, err = runtime.Core.Core().V1().Node().Update(node); err != nil {
+		return err
+	}
+	logrus.Debugf("encryption hash annotation set successfully on node: %s\n", node.ObjectMeta.Name)
+	return ioutil.WriteFile(runtime.EncryptionHash, []byte(ann), 0600)
+}

pkg/secretsencrypt/controller.go

+package secretsencrypt
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/rancher/k3s/pkg/cluster"
+	"github.com/rancher/k3s/pkg/daemons/config"
+	"github.com/rancher/k3s/pkg/util"
+	coreclient "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
+	"github.com/sirupsen/logrus"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/pager"
+	"k8s.io/client-go/tools/record"
+)
+
+const (
+	controllerAgentName        string = "reencrypt-controller"
+	secretsUpdateStartEvent    string = "SecretsUpdateStart"
+	secretsProgressEvent       string = "SecretsProgress"
+	secretsUpdateCompleteEvent string = "SecretsUpdateComplete"
+	secretsUpdateErrorEvent    string = "SecretsUpdateError"
+)
+
+type handler struct {
+	ctx           context.Context
+	controlConfig *config.Control
+	nodes         coreclient.NodeController
+	secrets       coreclient.SecretController
+	recorder      record.EventRecorder
+}
+
+func Register(
+	ctx context.Context,
+	k8s kubernetes.Interface,
+	controlConfig *config.Control,
+	nodes coreclient.NodeController,
+	secrets coreclient.SecretController,
+) error {
+	h := &handler{
+		ctx:           ctx,
+		controlConfig: controlConfig,
+		nodes:         nodes,
+		secrets:       secrets,
+		recorder:      util.BuildControllerEventRecorder(k8s, controllerAgentName),
+	}
+
+	nodes.OnChange(ctx, "reencrypt-controller", h.onChangeNode)
+	return nil
+}
+
+// onChangeNode handles changes to Nodes. We are looking for a specific annotation change
+func (h *handler) onChangeNode(key string, node *corev1.Node) (*corev1.Node, error) {
+	if node == nil {
+		return nil, nil
+	}
+
+	ann, ok := node.Annotations[EncryptionHashAnnotation]
+	if !ok {
+		return node, nil
+	}
+
+	if valid, err := h.validateReencryptStage(node, ann); err != nil {
+		h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error())
+		return node, err
+	} else if !valid {
+		return node, nil
+	}
+
+	reencryptHash, err := GenReencryptHash(h.controlConfig.Runtime, EncryptionReencryptActive)
+	if err != nil {
+		h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error())
+		return node, err
+	}
+	ann = EncryptionReencryptActive + "-" + reencryptHash
+	node.Annotations[EncryptionHashAnnotation] = ann
+	node, err = h.nodes.Update(node)
+	if err != nil {
+		h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error())
+		return node, err
+	}
+
+	if err := h.updateSecrets(node); err != nil {
+		h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error())
+		return node, err
+	}
+
+	// If skipping, revert back to the previous stage
+	if h.controlConfig.EncryptSkip {
+		BootstrapEncryptionHashAnnotation(node, h.controlConfig.Runtime)
+		if node, err := h.nodes.Update(node); err != nil {
+			return node, err
+		}
+		return node, nil
+	}
+
+	// Remove last key
+	curKeys, err := GetEncryptionKeys(h.controlConfig.Runtime)
+	if err != nil {
+		h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error())
+		return node, err
+	}
+
+	curKeys = curKeys[:len(curKeys)-1]
+	if err = WriteEncryptionConfig(h.controlConfig.Runtime, curKeys, true); err != nil {
+		h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error())
+		return node, err
+	}
+	logrus.Infoln("Removed key: ", curKeys[len(curKeys)-1])
+	if err != nil {
+		h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error())
+		return node, err
+	}
+	if err := WriteEncryptionHashAnnotation(h.controlConfig.Runtime, node, EncryptionReencryptFinished); err != nil {
+		h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error())
+		return node, err
+	}
+	if err := cluster.Save(h.ctx, h.controlConfig, h.controlConfig.Runtime.EtcdConfig, true); err != nil {
+		h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error())
+		return node, err
+	}
+	return node, nil
+}
+
+// validateReencryptStage ensures that the request for reencryption is valid and
+// that there is only one active reencryption at a time
+func (h *handler) validateReencryptStage(node *corev1.Node, annotation string) (bool, error) {
+
+	split := strings.Split(annotation, "-")
+	if len(split) != 2 {
+		err := fmt.Errorf("invalid annotation %s found on node %s", annotation, node.ObjectMeta.Name)
+		return false, err
+	}
+	stage := split[0]
+	hash := split[1]
+
+	// Validate the specific stage and the request via sha256 hash
+	if stage != EncryptionReencryptRequest {
+		return false, nil
+	}
+	if reencryptRequestHash, err := GenReencryptHash(h.controlConfig.Runtime, EncryptionReencryptRequest); err != nil {
+		return false, err
+	} else if reencryptRequestHash != hash {
+		err = fmt.Errorf("invalid hash: %s found on node %s", hash, node.ObjectMeta.Name)
+		return false, err
+	}
+
+	nodes, err := h.nodes.List(metav1.ListOptions{})
+	if err != nil {
+		return false, err
+	}
+	reencryptActiveHash, err := GenReencryptHash(h.controlConfig.Runtime, EncryptionReencryptActive)
+	if err != nil {
+		return false, err
+	}
+	for _, node := range nodes.Items {
+		if ann, ok := node.Annotations[EncryptionHashAnnotation]; ok {
+			split := strings.Split(ann, "-")
+			if len(split) != 2 {
+				return false, fmt.Errorf("invalid annotation %s found on node %s", ann, node.ObjectMeta.Name)
+			}
+			stage := split[0]
+			hash := split[1]
+			if stage == EncryptionReencryptActive && hash == reencryptActiveHash {
+				return false, fmt.Errorf("another reencrypt is already active")
+			}
+		}
+	}
+	return true, nil
+}
+
+func (h *handler) updateSecrets(node *corev1.Node) error {
+	secretPager := pager.New(pager.SimplePageFunc(func(opts metav1.ListOptions) (runtime.Object, error) {
+		return h.secrets.List("", opts)
+	}))
+	i := 0
+	secretPager.EachListItem(h.ctx, metav1.ListOptions{}, func(obj runtime.Object) error {
+		if secret, ok := obj.(*corev1.Secret); ok {
+			if _, err := h.secrets.Update(secret); err != nil {
+				return fmt.Errorf("failed to reencrypted secret: %v", err)
+			}
+			if i != 0 && i%10 == 0 {
+				h.recorder.Eventf(node, corev1.EventTypeNormal, secretsProgressEvent, "reencrypted %d secrets", i)
+			}
+			i++
+		}
+		return nil
+	})
+	h.recorder.Eventf(node, corev1.EventTypeNormal, secretsUpdateCompleteEvent, "completed reencrypt of %d secrets", i)
+	return nil
+}

pkg/server/router.go

@@ -48,6 +48,8 @@ func router(ctx context.Context, config *Config, cfg *cmds.Server) http.Handler
 	authed.Path(prefix + "/server-ca.crt").Handler(fileHandler(serverConfig.Runtime.ServerCA))
 	authed.Path(prefix + "/config").Handler(configHandler(serverConfig, cfg))
 	authed.Path(prefix + "/readyz").Handler(readyzHandler(serverConfig))
+	authed.Path(prefix + "/encrypt/status").Handler(encryptionStatusHandler(serverConfig))
+	authed.Path(prefix + "/encrypt/config").Handler(encryptionConfigHandler(ctx, serverConfig))
 
 	nodeAuthed := mux.NewRouter()
 	nodeAuthed.Use(authMiddleware(serverConfig, "system:nodes"))

pkg/server/server.go

@@ -25,6 +25,7 @@ import (
 	"github.com/rancher/k3s/pkg/node"
 	"github.com/rancher/k3s/pkg/nodepassword"
 	"github.com/rancher/k3s/pkg/rootlessports"
+	"github.com/rancher/k3s/pkg/secretsencrypt"
 	"github.com/rancher/k3s/pkg/servicelb"
 	"github.com/rancher/k3s/pkg/static"
 	"github.com/rancher/k3s/pkg/util"
@@ -169,7 +170,7 @@ func runControllers(ctx context.Context, wg *sync.WaitGroup, config *Config) err
 		}
 	}
 
-	go setControlPlaneRoleLabel(ctx, sc.Core.Core().V1().Node(), config)
+	go setNodeLabelsAndAnnotations(ctx, sc.Core.Core().V1().Node(), config)
 
 	go setClusterDNSConfig(ctx, config, sc.Core.Core().V1().ConfigMap())
 
@@ -232,6 +233,16 @@ func coreControllers(ctx context.Context, sc *Context, config *Config) error {
 		return err
 	}
 
+	if config.ControlConfig.EncryptSecrets {
+		if err := secretsencrypt.Register(ctx,
+			sc.K8s,
+			&config.ControlConfig,
+			sc.Core.Core().V1().Node(),
+			sc.Core.Core().V1().Secret()); err != nil {
+			return err
+		}
+	}
+
 	if config.Rootless {
 		return rootlessports.Register(ctx,
 			sc.Core.Core().V1().Service(),
@@ -490,7 +501,7 @@ func isSymlink(config string) bool {
 	return false
 }
 
-func setControlPlaneRoleLabel(ctx context.Context, nodes v1.NodeClient, config *Config) error {
+func setNodeLabelsAndAnnotations(ctx context.Context, nodes v1.NodeClient, config *Config) error {
 	if config.DisableAgent || config.ControlConfig.DisableAPIServer {
 		return nil
 	}
@@ -515,18 +526,25 @@ func setControlPlaneRoleLabel(ctx context.Context, nodes v1.NodeClient, config *
 				etcdRoleLabelExists = true
 			}
 		}
-		if v, ok := node.Labels[ControlPlaneRoleLabelKey]; ok && v == "true" && !etcdRoleLabelExists {
-			break
-		}
 		if node.Labels == nil {
 			node.Labels = make(map[string]string)
 		}
-		node.Labels[ControlPlaneRoleLabelKey] = "true"
-		node.Labels[MasterRoleLabelKey] = "true"
+		v, ok := node.Labels[ControlPlaneRoleLabelKey]
+		if !ok || v != "true" || etcdRoleLabelExists {
+			node.Labels[ControlPlaneRoleLabelKey] = "true"
+			node.Labels[MasterRoleLabelKey] = "true"
+		}
+
+		if config.ControlConfig.EncryptSecrets {
+			if err = secretsencrypt.BootstrapEncryptionHashAnnotation(node, config.ControlConfig.Runtime); err != nil {
+				logrus.Infof("Unable to set encryption hash annotation %s", err.Error())
+				break
+			}
+		}
 
 		_, err = nodes.Update(node)
 		if err == nil {
-			logrus.Infof("Control-plane role label has been set successfully on node: %s", nodeName)
+			logrus.Infof("Labels and annotations have been set successfully on node: %s", nodeName)
 			break
 		}
 		select {

pkg/util/api.go

@@ -5,15 +5,20 @@ import (
 	"fmt"
 	"net"
 	"net/http"
+	"os"
 	"strconv"
 	"time"
 
 	"github.com/pkg/errors"
 	"github.com/rancher/wrangler/pkg/merr"
+	"github.com/rancher/wrangler/pkg/schemes"
 	"github.com/sirupsen/logrus"
 	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
 	clientset "k8s.io/client-go/kubernetes"
+	coregetter "k8s.io/client-go/kubernetes/typed/core/v1"
+	"k8s.io/client-go/tools/record"
 )
 
 // This sets a default duration to wait for the apiserver to become ready. This is primarily used to
@@ -75,3 +80,12 @@ func WaitForAPIServerReady(ctx context.Context, client clientset.Interface, time
 
 	return nil
 }
+
+func BuildControllerEventRecorder(k8s clientset.Interface, controllerName string) record.EventRecorder {
+	logrus.Infof("Creating %s event broadcaster", controllerName)
+	eventBroadcaster := record.NewBroadcaster()
+	eventBroadcaster.StartLogging(logrus.Infof)
+	eventBroadcaster.StartRecordingToSink(&coregetter.EventSinkImpl{Interface: k8s.CoreV1().Events(metav1.NamespaceSystem)})
+	nodeName := os.Getenv("NODE_NAME")
+	return eventBroadcaster.NewRecorder(schemes.All, v1.EventSource{Component: controllerName, Host: nodeName})
+}

scripts/build

@@ -78,6 +78,7 @@ rm -f \
     bin/containerd-shim-runc-v2 \
     bin/k3s-server \
     bin/k3s-etcd-snapshot \
+    bin/k3s-secrets-encrypt \
     bin/k3s-certificate \
     bin/kubectl \
     bin/crictl \
@@ -108,6 +109,7 @@ CGO_ENABLED=1 "${GO}" build -tags "$TAGS" -ldflags "$VERSIONFLAGS $LDFLAGS $STAT
 ln -s containerd ./bin/k3s-agent
 ln -s containerd ./bin/k3s-server
 ln -s containerd ./bin/k3s-etcd-snapshot
+ln -s containerd ./bin/k3s-secrets-encrypt
 ln -s containerd ./bin/k3s-certificate
 ln -s containerd ./bin/kubectl
 ln -s containerd ./bin/crictl

scripts/package-cli

@@ -7,7 +7,7 @@ cd $(dirname $0)/..
 
 GO=${GO-go}
 
-for i in crictl kubectl k3s-agent k3s-server k3s-etcd-snapshot k3s-certificate k3s ; do
+for i in crictl kubectl k3s-agent k3s-server k3s-etcd-snapshot k3s-secrets-encrypt k3s-certificate k3s ; do
     rm -f bin/$i
     ln -s containerd bin/$i
 done

tests/integration/localstorage_int_test.go → tests/integration/localstorage/localstorage_int_test.go

@@ -35,11 +35,11 @@ var _ = Describe("local storage", func() {
 			}, "90s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
 		})
 		It("creates a new pvc", func() {
-			Expect(testutil.K3sCmd("kubectl", "create", "-f", "../testdata/localstorage_pvc.yaml")).
+			Expect(testutil.K3sCmd("kubectl", "create", "-f", "./testdata/localstorage_pvc.yaml")).
 				To(ContainSubstring("persistentvolumeclaim/local-path-pvc created"))
 		})
 		It("creates a new pod", func() {
-			Expect(testutil.K3sCmd("kubectl", "create", "-f", "../testdata/localstorage_pod.yaml")).
+			Expect(testutil.K3sCmd("kubectl", "create", "-f", "./testdata/localstorage_pod.yaml")).
 				To(ContainSubstring("pod/volume-test created"))
 		})
 		It("shows storage up in kubectl", func() {
@@ -48,7 +48,7 @@ var _ = Describe("local storage", func() {
 			}, "45s", "1s").Should(MatchRegexp(`local-path-pvc.+Bound`))
 			Eventually(func() (string, error) {
 				return testutil.K3sCmd("kubectl", "get", "pv")
-			}, "10s", "1s").Should(MatchRegexp(`pvc.+2Gi.+Bound`))
+			}, "10s", "1s").Should(MatchRegexp(`pvc.+1Gi.+Bound`))
 		})
 		It("has proper folder permissions", func() {
 			var k3sStorage = "/var/lib/rancher/k3s/storage"

tests/testdata/localstorage_pvc.yaml → tests/integration/localstorage/testdata/localstorage_pvc.yaml

@@ -9,4 +9,4 @@ spec:
   storageClassName: local-path
   resources:
     requests:
-      storage: 2Gi
+      storage: 1Gi

tests/integration/secretsencryption/secretsencryption_int_test.go

+package integration
+
+import (
+	"fmt"
+	"os"
+	"regexp"
+	"testing"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	"github.com/onsi/ginkgo/reporters"
+	. "github.com/onsi/gomega"
+	testutil "github.com/rancher/k3s/tests/util"
+)
+
+var secretsEncryptionServer *testutil.K3sServer
+var secretsEncryptionDataDir = "/tmp/k3sse"
+
+var secretsEncryptionServerArgs = []string{"--secrets-encryption", "-d", secretsEncryptionDataDir}
+var _ = BeforeSuite(func() {
+	if !testutil.IsExistingServer() {
+		var err error
+		Expect(os.MkdirAll(secretsEncryptionDataDir, 0777)).To(Succeed())
+		secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...)
+		Expect(err).ToNot(HaveOccurred())
+	}
+})
+
+var _ = Describe("secrets encryption rotation", func() {
+	BeforeEach(func() {
+		if testutil.IsExistingServer() {
+			Skip("Test does not support running on existing k3s servers")
+		}
+	})
+	When("A server starts with secrets encryption", func() {
+		It("starts up with no problems", func() {
+			Eventually(func() (string, error) {
+				return testutil.K3sCmd("kubectl", "get", "pods", "-A")
+			}, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
+		})
+		It("it creates a encryption key", func() {
+			result, err := testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(result).To(ContainSubstring("Encryption Status: Enabled"))
+			Expect(result).To(ContainSubstring("Current Rotation Stage: start"))
+		})
+	})
+	When("A server rotates encryption keys", func() {
+		It("it prepares to rotate", func() {
+			Expect(testutil.K3sCmd("secrets-encrypt", "prepare", "-d", secretsEncryptionDataDir)).
+				To(ContainSubstring("prepare completed successfully"))
+
+			result, err := testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(result).To(ContainSubstring("Current Rotation Stage: prepare"))
+			reg, err := regexp.Compile(`AES-CBC.+aescbckey.*`)
+			Expect(err).ToNot(HaveOccurred())
+			keys := reg.FindAllString(result, -1)
+			Expect(keys).To(HaveLen(2))
+			Expect(keys[0]).To(ContainSubstring("aescbckey"))
+			Expect(keys[1]).To(ContainSubstring("aescbckey-" + fmt.Sprint(time.Now().Year())))
+		})
+		It("restarts the server", func() {
+			var err error
+			Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed())
+			secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...)
+			Expect(err).ToNot(HaveOccurred())
+			Eventually(func() (string, error) {
+				return testutil.K3sCmd("kubectl", "get", "pods", "-A")
+			}, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
+		})
+		It("rotates the keys", func() {
+			Eventually(func() (string, error) {
+				return testutil.K3sCmd("secrets-encrypt", "rotate", "-d", secretsEncryptionDataDir)
+			}, "10s", "2s").Should(ContainSubstring("rotate completed successfully"))
+
+			result, err := testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(result).To(ContainSubstring("Current Rotation Stage: rotate"))
+			reg, err := regexp.Compile(`AES-CBC.+aescbckey.*`)
+			Expect(err).ToNot(HaveOccurred())
+			keys := reg.FindAllString(result, -1)
+			Expect(keys).To(HaveLen(2))
+			Expect(keys[0]).To(ContainSubstring("aescbckey-" + fmt.Sprint(time.Now().Year())))
+			Expect(keys[1]).To(ContainSubstring("aescbckey"))
+		})
+		It("restarts the server", func() {
+			var err error
+			Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed())
+			secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...)
+			Expect(err).ToNot(HaveOccurred())
+			Eventually(func() (string, error) {
+				return testutil.K3sCmd("kubectl", "get", "pods", "-A")
+			}, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
+			time.Sleep(10 * time.Second)
+		})
+		It("reencrypts the keys", func() {
+			Expect(testutil.K3sCmd("secrets-encrypt", "reencrypt", "-d", secretsEncryptionDataDir)).
+				To(ContainSubstring("reencryption started"))
+			Eventually(func() (string, error) {
+				return testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir)
+			}, "30s", "2s").Should(ContainSubstring("Current Rotation Stage: reencrypt_finished"))
+			result, err := testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir)
+			Expect(err).NotTo(HaveOccurred())
+			reg, err := regexp.Compile(`AES-CBC.+aescbckey.*`)
+			Expect(err).ToNot(HaveOccurred())
+			keys := reg.FindAllString(result, -1)
+			Expect(keys).To(HaveLen(1))
+			Expect(keys[0]).To(ContainSubstring("aescbckey-" + fmt.Sprint(time.Now().Year())))
+		})
+	})
+	When("A server disables encryption", func() {
+		It("it triggers the disable", func() {
+			Expect(testutil.K3sCmd("secrets-encrypt", "disable", "-d", secretsEncryptionDataDir)).
+				To(ContainSubstring("secrets-encryption disabled"))
+
+			result, err := testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(result).To(ContainSubstring("Encryption Status: Disabled"))
+		})
+		It("restarts the server", func() {
+			var err error
+			Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed())
+			secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...)
+			Expect(err).ToNot(HaveOccurred())
+			Eventually(func() (string, error) {
+				return testutil.K3sCmd("kubectl", "get", "pods", "-A")
+			}, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
+			time.Sleep(10 * time.Second)
+		})
+		It("reencrypts the keys", func() {
+			result, err := testutil.K3sCmd("secrets-encrypt", "reencrypt", "-f", "--skip", "-d", secretsEncryptionDataDir)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(result).To(ContainSubstring("reencryption started"))
+
+			result, err = testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(result).To(ContainSubstring("Encryption Status: Disabled"))
+		})
+	})
+})
+
+var _ = AfterSuite(func() {
+	if !testutil.IsExistingServer() {
+		Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed())
+		Expect(testutil.K3sRemoveDataDir(secretsEncryptionDataDir)).To(Succeed())
+	}
+})
+
+func Test_IntegrationSecretsEncryption(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecsWithDefaultAndCustomReporters(t, "Secrets Encryption Suite", []Reporter{
+		reporters.NewJUnitReporter("/tmp/results/junit-se.xml"),
+	})
+}

tests/util/cmd.go

@@ -68,6 +68,18 @@ func K3sCmd(cmdName string, cmdArgs ...string) (string, error) {
 	return string(byteOut), err
 }
 
+// K3sRemoveDataDir removes the provided directory as root
+func K3sRemoveDataDir(dataDir string) error {
+	var cmd *exec.Cmd
+	if IsRoot() {
+		cmd = exec.Command("rm", "-rf", dataDir)
+	} else {
+		cmd = exec.Command("sudo", "rm", "-rf", dataDir)
+	}
+	_, err := cmd.CombinedOutput()
+	return err
+}
+
 func contains(source []string, target string) bool {
 	for _, s := range source {
 		if s == target {
@@ -121,13 +133,18 @@ type K3sServer struct {
 // K3sStartServer acquires an exclusive lock on a temporary file, then launches a k3s cluster
 // with the provided arguments. Subsequent/parallel calls to this function will block until
 // the original lock is cleared using K3sKillServer
-func K3sStartServer(cmdArgs ...string) (*K3sServer, error) {
+func K3sStartServer(inputArgs ...string) (*K3sServer, error) {
 	logrus.Info("waiting to get server lock")
 	k3sLock, err := flock.Acquire("/var/lock/k3s-test.lock")
 	if err != nil {
 		return nil, err
 	}
 
+	var cmdArgs []string
+	for _, arg := range inputArgs {
+		cmdArgs = append(cmdArgs, strings.Fields(arg)...)
+	}
+
 	k3sBin := findK3sExecutable()
 	var cmd *exec.Cmd
 	if IsRoot() {