[Engine-1.21] Add cert rotation command (#4634)

cmd/cert/main.go

+package main
+
+import (
+	"context"
+	"errors"
+	"os"
+
+	"github.com/rancher/k3s/pkg/cli/cert"
+	"github.com/rancher/k3s/pkg/cli/cmds"
+	"github.com/rancher/k3s/pkg/configfilearg"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+func main() {
+	app := cmds.NewApp()
+	app.Commands = []cli.Command{
+		cmds.NewCertCommand(
+			cmds.NewCertSubcommands(
+				cert.Run),
+		),
+	}
+
+	if err := app.Run(configfilearg.MustParse(os.Args)); err != nil && !errors.Is(err, context.Canceled) {
+		logrus.Fatal(err)
+	}
+}

cmd/k3s/main.go

@@ -35,6 +35,7 @@ func main() {
 	}
 
 	etcdsnapshotCommand := internalCLIAction(version.Program+"-"+cmds.EtcdSnapshotCommand, dataDir, os.Args)
+	certCommand := internalCLIAction(version.Program+"-"+cmds.CertCommand, dataDir, os.Args)
 
 	// Handle subcommand invocation (k3s server, k3s crictl, etc)
 	app := cmds.NewApp()
@@ -52,6 +53,10 @@ func main() {
 				etcdsnapshotCommand,
 				etcdsnapshotCommand),
 		),
+		cmds.NewCertCommand(
+			cmds.NewCertSubcommands(
+				certCommand),
+		),
 	}
 
 	if err := app.Run(os.Args); err != nil && !errors.Is(err, context.Canceled) {

cmd/server/main.go

@@ -9,6 +9,7 @@ import (
 	"github.com/docker/docker/pkg/reexec"
 	crictl2 "github.com/kubernetes-sigs/cri-tools/cmd/crictl"
 	"github.com/rancher/k3s/pkg/cli/agent"
+	"github.com/rancher/k3s/pkg/cli/cert"
 	"github.com/rancher/k3s/pkg/cli/cmds"
 	"github.com/rancher/k3s/pkg/cli/crictl"
 	"github.com/rancher/k3s/pkg/cli/ctr"
@@ -52,6 +53,10 @@ func main() {
 				etcdsnapshot.Prune,
 				etcdsnapshot.Run),
 		),
+		cmds.NewCertCommand(
+			cmds.NewCertSubcommands(
+				cert.Run),
+		),
 	}
 
 	if err := app.Run(configfilearg.MustParse(os.Args)); err != nil && !errors.Is(err, context.Canceled) {

go.mod

@@ -112,10 +112,11 @@ require (
 	// LOOK TO scripts/download FOR THE VERSION OF runc THAT WE ARE BUILDING/SHIPPING
 	github.com/opencontainers/runc v1.0.2
 	github.com/opencontainers/selinux v1.8.2
+	github.com/otiai10/copy v1.6.0
 	github.com/pierrec/lz4 v2.6.0+incompatible
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.11.0 // indirect
-	github.com/rancher/dynamiclistener v0.2.6
+	github.com/rancher/dynamiclistener v0.2.7
 	github.com/rancher/lasso v0.0.0-20210616224652-fc3ebd901c08
 	github.com/rancher/remotedialer v0.2.0
 	github.com/rancher/wharfie v0.3.4

go.sum

@@ -791,6 +791,13 @@ github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3
 github.com/opencontainers/selinux v1.8.2 h1:c4ca10UMgRcvZ6h0K4HtS15UaVSBEaE+iln2LVpAuGc=
 github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
+github.com/otiai10/copy v1.6.0 h1:IinKAryFFuPONZ7cm6T6E2QX/vcJwSnlaA5lfoaXIiQ=
+github.com/otiai10/copy v1.6.0/go.mod h1:XWfuS3CrI0R6IE0FbgHsEazaXO8G0LpMp9o8tos0x4E=
+github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE=
+github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs=
+github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo=
+github.com/otiai10/mint v1.3.2 h1:VYWnrP5fXmz1MXvjuUvcBrXSjGE6xjON+axB/UrpO3E=
+github.com/otiai10/mint v1.3.2/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/paulmach/orb v0.1.3/go.mod h1:VFlX/8C+IQ1p6FTRRKzKoOPJnvEtA5G0Veuqwbu//Vk=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
@@ -843,8 +850,8 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/qri-io/starlib v0.4.2-0.20200213133954-ff2e8cd5ef8d/go.mod h1:7DPO4domFU579Ga6E61sB9VFNaniPVwJP5C4bBCu3wA=
 github.com/quobyte/api v0.1.8/go.mod h1:jL7lIHrmqQ7yh05OJ+eEEdHr0u/kmT1Ff9iHd+4H6VI=
-github.com/rancher/dynamiclistener v0.2.6 h1:qcNhQQOidheum5K6mtLxDTJQMTWKm1zm/PjZ5x0xV7A=
-github.com/rancher/dynamiclistener v0.2.6/go.mod h1:iXFvJLvLjmTzEJBrLFZl9UaMfDLOhv6fHp9fHQRlHGg=
+github.com/rancher/dynamiclistener v0.2.7 h1:4FTlQtmHO6cY/g4XtGNAZTTxJYEdwn7VgtunX06NFjQ=
+github.com/rancher/dynamiclistener v0.2.7/go.mod h1:iXFvJLvLjmTzEJBrLFZl9UaMfDLOhv6fHp9fHQRlHGg=
 github.com/rancher/lasso v0.0.0-20210616224652-fc3ebd901c08 h1:NxR8Fh0eE7/5/5Zvlog9B5NVjWKqBSb1WYMUF7/IE5c=
 github.com/rancher/lasso v0.0.0-20210616224652-fc3ebd901c08/go.mod h1:9qZd/S8DqWzfKtjKGgSoHqGEByYmUE3qRaBaaAHwfEM=
 github.com/rancher/remotedialer v0.2.0 h1:xD7t3K6JYwTdAsxmGtTHQMkEkFgKouQ1foLxVW424Dc=

main.go

@@ -12,6 +12,7 @@ import (
 	"os"
 
 	"github.com/rancher/k3s/pkg/cli/agent"
+	"github.com/rancher/k3s/pkg/cli/cert"
 	"github.com/rancher/k3s/pkg/cli/cmds"
 	"github.com/rancher/k3s/pkg/cli/crictl"
 	"github.com/rancher/k3s/pkg/cli/etcdsnapshot"
@@ -36,6 +37,10 @@ func main() {
 				etcdsnapshot.Prune,
 				etcdsnapshot.Run),
 		),
+		cmds.NewCertCommand(
+			cmds.NewCertSubcommands(
+				cert.Run),
+		),
 	}
 
 	if err := app.Run(configfilearg.MustParse(os.Args)); err != nil && !errors.Is(err, context.Canceled) {

pkg/cli/cert/cert.go

+package cert
+
+import (
+	"errors"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strconv"
+	"time"
+
+	"github.com/erikdubbelboer/gspt"
+	"github.com/otiai10/copy"
+	"github.com/rancher/k3s/pkg/cli/cmds"
+	"github.com/rancher/k3s/pkg/daemons/config"
+	"github.com/rancher/k3s/pkg/daemons/control/deps"
+	"github.com/rancher/k3s/pkg/datadir"
+	"github.com/rancher/k3s/pkg/server"
+	"github.com/rancher/k3s/pkg/version"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+const (
+	adminService             = "admin"
+	apiServerService         = "api-server"
+	controllerManagerService = "controller-manager"
+	schedulerService         = "scheduler"
+	etcdService              = "etcd"
+	programControllerService = "-controller"
+	authProxyService         = "auth-proxy"
+	cloudControllerService   = "cloud-controller"
+	kubeletService           = "kubelet"
+	kubeProxyService         = "kube-proxy"
+	k3sServerService         = "-server"
+)
+
+func commandSetup(app *cli.Context, cfg *cmds.Server, sc *server.Config) (string, string, error) {
+	gspt.SetProcTitle(os.Args[0])
+
+	sc.ControlConfig.DataDir = cfg.DataDir
+	sc.ControlConfig.Runtime = &config.ControlRuntime{}
+	dataDir, err := datadir.Resolve(cfg.DataDir)
+	if err != nil {
+		return "", "", err
+	}
+	return filepath.Join(dataDir, "server"), filepath.Join(dataDir, "agent"), err
+}
+
+func Run(app *cli.Context) error {
+	if err := cmds.InitLogging(); err != nil {
+		return err
+	}
+	return rotate(app, &cmds.ServerConfig)
+}
+
+func rotate(app *cli.Context, cfg *cmds.Server) error {
+	var serverConfig server.Config
+
+	serverDataDir, agentDataDir, err := commandSetup(app, cfg, &serverConfig)
+	if err != nil {
+		return err
+	}
+
+	serverConfig.ControlConfig.DataDir = serverDataDir
+	serverConfig.ControlConfig.Runtime = &config.ControlRuntime{}
+	deps.CreateRuntimeCertFiles(&serverConfig.ControlConfig, serverConfig.ControlConfig.Runtime)
+
+	tlsBackupDir, err := backupCertificates(serverDataDir, agentDataDir)
+	if err != nil {
+		return err
+	}
+
+	if len(cmds.ServicesList) == 0 {
+		// detecting if the service is an agent or server
+		_, err := os.Stat(serverDataDir)
+		if err != nil {
+			if !os.IsNotExist(err) {
+				return err
+			}
+			logrus.Infof("Agent detected, rotating agent certificates")
+			cmds.ServicesList = []string{
+				kubeletService,
+				kubeProxyService,
+				version.Program + programControllerService,
+			}
+		} else {
+			logrus.Infof("Server detected, rotating server certificates")
+			cmds.ServicesList = []string{
+				adminService,
+				etcdService,
+				apiServerService,
+				controllerManagerService,
+				cloudControllerService,
+				schedulerService,
+				version.Program + k3sServerService,
+				version.Program + programControllerService,
+				authProxyService,
+				kubeletService,
+				kubeProxyService,
+			}
+		}
+	}
+	fileList := []string{}
+	for _, service := range cmds.ServicesList {
+		logrus.Infof("Rotating certificates for %s service", service)
+		switch service {
+		case adminService:
+			fileList = append(fileList,
+				serverConfig.ControlConfig.Runtime.ClientAdminCert,
+				serverConfig.ControlConfig.Runtime.ClientAdminKey)
+		case apiServerService:
+			fileList = append(fileList,
+				serverConfig.ControlConfig.Runtime.ClientKubeAPICert,
+				serverConfig.ControlConfig.Runtime.ClientKubeAPIKey,
+				serverConfig.ControlConfig.Runtime.ServingKubeAPICert,
+				serverConfig.ControlConfig.Runtime.ServingKubeAPIKey)
+		case controllerManagerService:
+			fileList = append(fileList,
+				serverConfig.ControlConfig.Runtime.ClientControllerCert,
+				serverConfig.ControlConfig.Runtime.ClientControllerKey)
+		case schedulerService:
+			fileList = append(fileList,
+				serverConfig.ControlConfig.Runtime.ClientSchedulerCert,
+				serverConfig.ControlConfig.Runtime.ClientSchedulerKey)
+		case etcdService:
+			fileList = append(fileList,
+				serverConfig.ControlConfig.Runtime.ClientETCDCert,
+				serverConfig.ControlConfig.Runtime.ClientETCDKey,
+				serverConfig.ControlConfig.Runtime.ServerETCDCert,
+				serverConfig.ControlConfig.Runtime.ServerETCDKey,
+				serverConfig.ControlConfig.Runtime.PeerServerClientETCDCert,
+				serverConfig.ControlConfig.Runtime.PeerServerClientETCDKey)
+		case cloudControllerService:
+			fileList = append(fileList,
+				serverConfig.ControlConfig.Runtime.ClientCloudControllerCert,
+				serverConfig.ControlConfig.Runtime.ClientCloudControllerKey)
+		case version.Program + k3sServerService:
+			dynamicListenerRegenFilePath := filepath.Join(serverDataDir, "tls", "dynamic-cert-regenerate")
+			if err := ioutil.WriteFile(dynamicListenerRegenFilePath, []byte{}, 0600); err != nil {
+				return err
+			}
+			logrus.Infof("Rotating dynamic listener certificate")
+		case version.Program + programControllerService:
+			fileList = append(fileList,
+				serverConfig.ControlConfig.Runtime.ClientK3sControllerCert,
+				serverConfig.ControlConfig.Runtime.ClientK3sControllerKey,
+				filepath.Join(agentDataDir, "client-"+version.Program+"-controller.crt"),
+				filepath.Join(agentDataDir, "client-"+version.Program+"-controller.key"))
+		case authProxyService:
+			fileList = append(fileList,
+				serverConfig.ControlConfig.Runtime.ClientAuthProxyCert,
+				serverConfig.ControlConfig.Runtime.ClientAuthProxyKey)
+		case kubeletService:
+			fileList = append(fileList,
+				serverConfig.ControlConfig.Runtime.ClientKubeletKey,
+				serverConfig.ControlConfig.Runtime.ServingKubeletKey,
+				filepath.Join(agentDataDir, "client-kubelet.crt"),
+				filepath.Join(agentDataDir, "client-kubelet.key"),
+				filepath.Join(agentDataDir, "serving-kubelet.crt"),
+				filepath.Join(agentDataDir, "serving-kubelet.key"))
+		case kubeProxyService:
+			fileList = append(fileList,
+				serverConfig.ControlConfig.Runtime.ClientKubeProxyCert,
+				serverConfig.ControlConfig.Runtime.ClientKubeProxyKey,
+				filepath.Join(agentDataDir, "client-kube-proxy.crt"),
+				filepath.Join(agentDataDir, "client-kube-proxy.key"))
+		default:
+			logrus.Fatalf("%s is not a recognized service", service)
+		}
+	}
+
+	for _, file := range fileList {
+		if err := os.Remove(file); err == nil {
+			logrus.Debugf("file %s is deleted", file)
+		}
+	}
+	logrus.Infof("Successfully backed up certificates for all services to path %s, please restart %s server or agent to rotate certificates", tlsBackupDir, version.Program)
+	return nil
+}
+
+func copyFile(src, destDir string) error {
+	_, err := os.Stat(src)
+	if err == nil {
+		input, err := ioutil.ReadFile(src)
+		if err != nil {
+			return err
+		}
+		return ioutil.WriteFile(filepath.Join(destDir, filepath.Base(src)), input, 0644)
+	} else if errors.Is(err, os.ErrNotExist) {
+		return nil
+	}
+	return err
+}
+
+func backupCertificates(serverDataDir, agentDataDir string) (string, error) {
+	serverTLSDir := filepath.Join(serverDataDir, "tls")
+	tlsBackupDir := filepath.Join(serverDataDir, "tls-"+strconv.Itoa(int(time.Now().Unix())))
+
+	if _, err := os.Stat(serverTLSDir); err != nil {
+		return "", err
+	}
+	if err := copy.Copy(serverTLSDir, tlsBackupDir); err != nil {
+		return "", err
+	}
+	agentCerts := []string{
+		filepath.Join(agentDataDir, "client-"+version.Program+"-controller.crt"),
+		filepath.Join(agentDataDir, "client-"+version.Program+"-controller.key"),
+		filepath.Join(agentDataDir, "client-kubelet.crt"),
+		filepath.Join(agentDataDir, "client-kubelet.key"),
+		filepath.Join(agentDataDir, "serving-kubelet.crt"),
+		filepath.Join(agentDataDir, "serving-kubelet.key"),
+		filepath.Join(agentDataDir, "client-kube-proxy.crt"),
+		filepath.Join(agentDataDir, "client-kube-proxy.key"),
+	}
+	for _, cert := range agentCerts {
+		if err := copyFile(cert, tlsBackupDir); err != nil {
+			return "", err
+		}
+	}
+	return tlsBackupDir, nil
+}

pkg/cli/cmds/certs.go

+package cmds
+
+import (
+	"github.com/rancher/k3s/pkg/version"
+	"github.com/urfave/cli"
+)
+
+const CertCommand = "certificate"
+
+var (
+	ServicesList     cli.StringSlice
+	CertCommandFlags = []cli.Flag{
+		DebugFlag,
+		ConfigFlag,
+		LogFile,
+		AlsoLogToStderr,
+		cli.StringFlag{
+			Name:        "data-dir,d",
+			Usage:       "(data) Folder to hold state default /var/lib/rancher/" + version.Program + " or ${HOME}/.rancher/" + version.Program + " if not root",
+			Destination: &ServerConfig.DataDir,
+		},
+		cli.StringSliceFlag{
+			Name:  "service,s",
+			Usage: "List of services to rotate certificates for. Options include (admin, api-server, controller-manager, scheduler, " + version.Program + "-controller, " + version.Program + "-server, cloud-controller, etcd, auth-proxy, kubelet, kube-proxy)",
+			Value: &ServicesList,
+		},
+	}
+)
+
+func NewCertCommand(subcommands []cli.Command) cli.Command {
+	return cli.Command{
+		Name:            CertCommand,
+		Usage:           "Certificates management",
+		SkipFlagParsing: false,
+		SkipArgReorder:  true,
+		Subcommands:     subcommands,
+		Flags:           CertCommandFlags,
+	}
+}
+
+func NewCertSubcommands(rotate func(ctx *cli.Context) error) []cli.Command {
+	return []cli.Command{
+		{
+			Name:            "rotate",
+			Usage:           "Certificate rotation",
+			SkipFlagParsing: false,
+			SkipArgReorder:  true,
+			Action:          rotate,
+			Flags:           CertCommandFlags,
+		},
+	}
+}

pkg/cluster/https.go

@@ -54,6 +54,15 @@ func (c *Cluster) newListener(ctx context.Context) (net.Listener, http.Handler,
 			MinVersion:   c.config.TLSMinVersion,
 			CipherSuites: c.config.TLSCipherSuites,
 		},
+		RegenerateCerts: func() bool {
+			const regenerateDynamicListenerFile = "dynamic-cert-regenerate"
+			dynamicListenerRegenFilePath := filepath.Join(c.config.DataDir, "tls", regenerateDynamicListenerFile)
+			if _, err := os.Stat(dynamicListenerRegenFilePath); err == nil {
+				os.Remove(dynamicListenerRegenFilePath)
+				return true
+			}
+			return false
+		},
 	})
 }
 

scripts/build

@@ -78,6 +78,7 @@ rm -f \
     bin/containerd-shim-runc-v2 \
     bin/k3s-server \
     bin/k3s-etcd-snapshot \
+    bin/k3s-certificate \
     bin/kubectl \
     bin/crictl \
     bin/ctr
@@ -107,6 +108,7 @@ CGO_ENABLED=1 "${GO}" build -tags "$TAGS" -ldflags "$VERSIONFLAGS $LDFLAGS $STAT
 ln -s containerd ./bin/k3s-agent
 ln -s containerd ./bin/k3s-server
 ln -s containerd ./bin/k3s-etcd-snapshot
+ln -s containerd ./bin/k3s-certificate
 ln -s containerd ./bin/kubectl
 ln -s containerd ./bin/crictl
 ln -s containerd ./bin/ctr

scripts/package-cli

@@ -7,7 +7,7 @@ cd $(dirname $0)/..
 
 GO=${GO-go}
 
-for i in crictl kubectl k3s-agent k3s-server k3s-etcd-snapshot k3s; do
+for i in crictl kubectl k3s-agent k3s-server k3s-etcd-snapshot k3s-certificate k3s ; do
     rm -f bin/$i
     ln -s containerd bin/$i
 done

vendor/github.com/otiai10/copy/.gitignore

+test/data.copy
+coverage.txt
+vendor
+.vagrant
+.idea/

vendor/github.com/otiai10/copy/LICENSE

+The MIT License (MIT)
+
+Copyright (c) 2018 otiai10
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

vendor/github.com/otiai10/copy/README.md

+# copy
+
+[![Go Reference](https://pkg.go.dev/badge/github.com/otiai10/copy.svg)](https://pkg.go.dev/github.com/otiai10/copy)
+[![Actions Status](https://github.com/otiai10/copy/workflows/Go/badge.svg)](https://github.com/otiai10/copy/actions)
+[![codecov](https://codecov.io/gh/otiai10/copy/branch/main/graph/badge.svg)](https://codecov.io/gh/otiai10/copy)
+[![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](https://github.com/otiai10/copy/blob/main/LICENSE)
+[![Go Report Card](https://goreportcard.com/badge/github.com/otiai10/copy)](https://goreportcard.com/report/github.com/otiai10/copy)
+[![GitHub tag (latest SemVer)](https://img.shields.io/github/v/tag/otiai10/copy?sort=semver)](https://pkg.go.dev/github.com/otiai10/copy)
+
+`copy` copies directories recursively.
+
+# Example Usage
+
+```go
+err := Copy("your/directory", "your/directory.copy")
+```
+
+# Advanced Usage
+
+```go
+// Options specifies optional actions on copying.
+type Options struct {
+
+	// OnSymlink can specify what to do on symlink
+	OnSymlink func(src string) SymlinkAction
+
+	// OnDirExists can specify what to do when there is a directory already existing in destination.
+	OnDirExists func(src, dest string) DirExistsAction
+
+	// Skip can specify which files should be skipped
+	Skip func(src string) (bool, error)
+
+	// AddPermission to every entities,
+	// NO MORE THAN 0777
+	AddPermission os.FileMode
+
+	// Sync file after copy.
+	// Useful in case when file must be on the disk
+	// (in case crash happens, for example),
+	// at the expense of some performance penalty
+	Sync bool
+
+	// Preserve the atime and the mtime of the entries
+	// On linux we can preserve only up to 1 millisecond accuracy
+	PreserveTimes bool
+
+}
+```
+
+```go
+// For example...
+opt := Options{
+	Skip: func(src string) (bool, error) {
+		return strings.HasSuffix(src, ".git"), nil
+	},
+}
+err := Copy("your/directory", "your/directory.copy", opt)
+```
+
+# Issues
+
+- https://github.com/otiai10/copy/issues

vendor/github.com/otiai10/copy/copy.go

+package copy
+
+import (
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"time"
+)
+
+const (
+	// tmpPermissionForDirectory makes the destination directory writable,
+	// so that stuff can be copied recursively even if any original directory is NOT writable.
+	// See https://github.com/otiai10/copy/pull/9 for more information.
+	tmpPermissionForDirectory = os.FileMode(0755)
+)
+
+type timespec struct {
+	Mtime time.Time
+	Atime time.Time
+	Ctime time.Time
+}
+
+// Copy copies src to dest, doesn't matter if src is a directory or a file.
+func Copy(src, dest string, opt ...Options) error {
+	info, err := os.Lstat(src)
+	if err != nil {
+		return err
+	}
+	return switchboard(src, dest, info, assure(src, dest, opt...))
+}
+
+// switchboard switches proper copy functions regarding file type, etc...
+// If there would be anything else here, add a case to this switchboard.
+func switchboard(src, dest string, info os.FileInfo, opt Options) (err error) {
+	switch {
+	case info.Mode()&os.ModeSymlink != 0:
+		err = onsymlink(src, dest, info, opt)
+	case info.IsDir():
+		err = dcopy(src, dest, info, opt)
+	case info.Mode()&os.ModeNamedPipe != 0:
+		err = pcopy(dest, info)
+	default:
+		err = fcopy(src, dest, info, opt)
+	}
+
+	return err
+}
+
+// copyNextOrSkip decide if this src should be copied or not.
+// Because this "copy" could be called recursively,
+// "info" MUST be given here, NOT nil.
+func copyNextOrSkip(src, dest string, info os.FileInfo, opt Options) error {
+	skip, err := opt.Skip(src)
+	if err != nil {
+		return err
+	}
+	if skip {
+		return nil
+	}
+	return switchboard(src, dest, info, opt)
+}
+
+// fcopy is for just a file,
+// with considering existence of parent directory
+// and file permission.
+func fcopy(src, dest string, info os.FileInfo, opt Options) (err error) {
+
+	if err = os.MkdirAll(filepath.Dir(dest), os.ModePerm); err != nil {
+		return
+	}
+
+	f, err := os.Create(dest)
+	if err != nil {
+		return
+	}
+	defer fclose(f, &err)
+
+	if err = os.Chmod(f.Name(), info.Mode()|opt.AddPermission); err != nil {
+		return
+	}
+
+	s, err := os.Open(src)
+	if err != nil {
+		return
+	}
+	defer fclose(s, &err)
+
+	var buf []byte = nil
+	var w io.Writer = f
+	// var r io.Reader = s
+	if opt.CopyBufferSize != 0 {
+		buf = make([]byte, opt.CopyBufferSize)
+		// Disable using `ReadFrom` by io.CopyBuffer.
+		// See https://github.com/otiai10/copy/pull/60#discussion_r627320811 for more details.
+		w = struct{ io.Writer }{f}
+		// r = struct{ io.Reader }{s}
+	}
+	if _, err = io.CopyBuffer(w, s, buf); err != nil {
+		return err
+	}
+
+	if opt.Sync {
+		err = f.Sync()
+	}
+
+	if opt.PreserveTimes {
+		return preserveTimes(info, dest)
+	}
+
+	return
+}
+
+// dcopy is for a directory,
+// with scanning contents inside the directory
+// and pass everything to "copy" recursively.
+func dcopy(srcdir, destdir string, info os.FileInfo, opt Options) (err error) {
+
+	_, err = os.Stat(destdir)
+	if err == nil && opt.OnDirExists != nil && destdir != opt.intent.dest {
+		switch opt.OnDirExists(srcdir, destdir) {
+		case Replace:
+			if err := os.RemoveAll(destdir); err != nil {
+				return err
+			}
+		case Untouchable:
+			return nil
+		} // case "Merge" is default behaviour. Go through.
+	} else if err != nil && !os.IsNotExist(err) {
+		return err // Unwelcome error type...!
+	}
+
+	originalMode := info.Mode()
+
+	// Make dest dir with 0755 so that everything writable.
+	if err = os.MkdirAll(destdir, tmpPermissionForDirectory); err != nil {
+		return
+	}
+	// Recover dir mode with original one.
+	defer chmod(destdir, originalMode|opt.AddPermission, &err)
+
+	contents, err := ioutil.ReadDir(srcdir)
+	if err != nil {
+		return
+	}
+
+	for _, content := range contents {
+		cs, cd := filepath.Join(srcdir, content.Name()), filepath.Join(destdir, content.Name())
+
+		if err = copyNextOrSkip(cs, cd, content, opt); err != nil {
+			// If any error, exit immediately
+			return
+		}
+	}
+
+	if opt.PreserveTimes {
+		return preserveTimes(info, destdir)
+	}
+
+	return
+}
+
+func onsymlink(src, dest string, info os.FileInfo, opt Options) error {
+	switch opt.OnSymlink(src) {
+	case Shallow:
+		return lcopy(src, dest)
+	case Deep:
+		orig, err := os.Readlink(src)
+		if err != nil {
+			return err
+		}
+		info, err = os.Lstat(orig)
+		if err != nil {
+			return err
+		}
+		return copyNextOrSkip(orig, dest, info, opt)
+	case Skip:
+		fallthrough
+	default:
+		return nil // do nothing
+	}
+}
+
+// lcopy is for a symlink,
+// with just creating a new symlink by replicating src symlink.
+func lcopy(src, dest string) error {
+	src, err := os.Readlink(src)
+	if err != nil {
+		return err
+	}
+	return os.Symlink(src, dest)
+}
+
+// fclose ANYHOW closes file,
+// with asiging error raised during Close,
+// BUT respecting the error already reported.
+func fclose(f *os.File, reported *error) {
+	if err := f.Close(); *reported == nil {
+		*reported = err
+	}
+}
+
+// chmod ANYHOW changes file mode,
+// with asiging error raised during Chmod,
+// BUT respecting the error already reported.
+func chmod(dir string, mode os.FileMode, reported *error) {
+	if err := os.Chmod(dir, mode); *reported == nil {
+		*reported = err
+	}
+}
+
+// assure Options struct, should be called only once.
+// All optional values MUST NOT BE nil/zero after assured.
+func assure(src, dest string, opts ...Options) Options {
+	defopt := getDefaultOptions(src, dest)
+	if len(opts) == 0 {
+		return defopt
+	}
+	if opts[0].OnSymlink == nil {
+		opts[0].OnSymlink = defopt.OnSymlink
+	}
+	if opts[0].Skip == nil {
+		opts[0].Skip = defopt.Skip
+	}
+	opts[0].intent.src = defopt.intent.src
+	opts[0].intent.dest = defopt.intent.dest
+	return opts[0]
+}

vendor/github.com/otiai10/copy/copy_namedpipes.go

+// +build !windows
+
+package copy
+
+import (
+	"os"
+	"path/filepath"
+	"syscall"
+)
+
+// pcopy is for just named pipes
+func pcopy(dest string, info os.FileInfo) error {
+	if err := os.MkdirAll(filepath.Dir(dest), os.ModePerm); err != nil {
+		return err
+	}
+	return syscall.Mkfifo(dest, uint32(info.Mode()))
+}

vendor/github.com/otiai10/copy/copy_namedpipes_windows.go

+// +build windows
+
+package copy
+
+import (
+	"os"
+)
+
+// pcopy is for just named pipes. Windows doesn't support them
+func pcopy(dest string, info os.FileInfo) error {
+	return nil
+}

vendor/github.com/otiai10/copy/go.mod

+module github.com/otiai10/copy
+
+go 1.14
+
+require github.com/otiai10/mint v1.3.2

vendor/github.com/otiai10/copy/go.sum

+github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE=
+github.com/otiai10/curr v1.0.0 h1:TJIWdbX0B+kpNagQrjgq8bCMrbhiuX73M2XwgtDMoOI=
+github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs=
+github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo=
+github.com/otiai10/mint v1.3.2 h1:VYWnrP5fXmz1MXvjuUvcBrXSjGE6xjON+axB/UrpO3E=
+github.com/otiai10/mint v1.3.2/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc=

vendor/github.com/otiai10/copy/options.go

+package copy
+
+import "os"
+
+// Options specifies optional actions on copying.
+type Options struct {
+
+	// OnSymlink can specify what to do on symlink
+	OnSymlink func(src string) SymlinkAction
+
+	// OnDirExists can specify what to do when there is a directory already existing in destination.
+	OnDirExists func(src, dest string) DirExistsAction
+
+	// Skip can specify which files should be skipped
+	Skip func(src string) (bool, error)
+
+	// AddPermission to every entities,
+	// NO MORE THAN 0777
+	AddPermission os.FileMode
+
+	// Sync file after copy.
+	// Useful in case when file must be on the disk
+	// (in case crash happens, for example),
+	// at the expense of some performance penalty
+	Sync bool
+
+	// Preserve the atime and the mtime of the entries.
+	// On linux we can preserve only up to 1 millisecond accuracy.
+	PreserveTimes bool
+
+	// The byte size of the buffer to use for copying files.
+	// If zero, the internal default buffer of 32KB is used.
+	// See https://golang.org/pkg/io/#CopyBuffer for more information.
+	CopyBufferSize uint
+
+	intent struct {
+		src  string
+		dest string
+	}
+}
+
+// SymlinkAction represents what to do on symlink.
+type SymlinkAction int
+
+const (
+	// Deep creates hard-copy of contents.
+	Deep SymlinkAction = iota
+	// Shallow creates new symlink to the dest of symlink.
+	Shallow
+	// Skip does nothing with symlink.
+	Skip
+)
+
+// DirExistsAction represents what to do on dest dir.
+type DirExistsAction int
+
+const (
+	// Merge preserves or overwrites existing files under the dir (default behavior).
+	Merge DirExistsAction = iota
+	// Replace deletes all contents under the dir and copy src files.
+	Replace
+	// Untouchable does nothing for the dir, and leaves it as it is.
+	Untouchable
+)
+
+// getDefaultOptions provides default options,
+// which would be modified by usage-side.
+func getDefaultOptions(src, dest string) Options {
+	return Options{
+		OnSymlink: func(string) SymlinkAction {
+			return Shallow // Do shallow copy
+		},
+		OnDirExists: nil, // Default behavior is "Merge".
+		Skip: func(string) (bool, error) {
+			return false, nil // Don't skip
+		},
+		AddPermission:  0,     // Add nothing
+		Sync:           false, // Do not sync
+		PreserveTimes:  false, // Do not preserve the modification time
+		CopyBufferSize: 0,     // Do not specify, use default bufsize (32*1024)
+		intent: struct {
+			src  string
+			dest string
+		}{src, dest},
+	}
+}

vendor/github.com/otiai10/copy/preserve_times.go

+package copy
+
+import "os"
+
+func preserveTimes(srcinfo os.FileInfo, dest string) error {
+	spec := getTimeSpec(srcinfo)
+	if err := os.Chtimes(dest, spec.Atime, spec.Mtime); err != nil {
+		return err
+	}
+	return nil
+}

vendor/github.com/otiai10/copy/stat_times.go

+// +build !windows,!darwin,!freebsd
+
+// TODO: add more runtimes
+
+package copy
+
+import (
+	"os"
+	"syscall"
+	"time"
+)
+
+func getTimeSpec(info os.FileInfo) timespec {
+	stat := info.Sys().(*syscall.Stat_t)
+	times := timespec{
+		Mtime: info.ModTime(),
+		Atime: time.Unix(int64(stat.Atim.Sec), int64(stat.Atim.Nsec)),
+		Ctime: time.Unix(int64(stat.Ctim.Sec), int64(stat.Ctim.Nsec)),
+	}
+	return times
+}

vendor/github.com/otiai10/copy/stat_times_darwin.go

+// +build darwin
+
+package copy
+
+import (
+	"os"
+	"syscall"
+	"time"
+)
+
+func getTimeSpec(info os.FileInfo) timespec {
+	stat := info.Sys().(*syscall.Stat_t)
+	times := timespec{
+		Mtime: info.ModTime(),
+		Atime: time.Unix(stat.Atimespec.Sec, stat.Atimespec.Nsec),
+		Ctime: time.Unix(stat.Ctimespec.Sec, stat.Ctimespec.Nsec),
+	}
+	return times
+}

vendor/github.com/otiai10/copy/stat_times_freebsd.go

+// +build freebsd
+
+package copy
+
+import (
+	"os"
+	"syscall"
+	"time"
+)
+
+func getTimeSpec(info os.FileInfo) timespec {
+	stat := info.Sys().(*syscall.Stat_t)
+	times := timespec{
+		Mtime: info.ModTime(),
+		Atime: time.Unix(int64(stat.Atimespec.Sec), int64(stat.Atimespec.Nsec)),
+		Ctime: time.Unix(int64(stat.Ctimespec.Sec), int64(stat.Ctimespec.Nsec)),
+	}
+	return times
+}

vendor/github.com/otiai10/copy/stat_times_windows.go

+// +build windows
+
+package copy
+
+import (
+	"os"
+	"syscall"
+	"time"
+)
+
+func getTimeSpec(info os.FileInfo) timespec {
+	stat := info.Sys().(*syscall.Win32FileAttributeData)
+	return timespec{
+		Mtime: time.Unix(0, stat.LastWriteTime.Nanoseconds()),
+		Atime: time.Unix(0, stat.LastAccessTime.Nanoseconds()),
+		Ctime: time.Unix(0, stat.CreationTime.Nanoseconds()),
+	}
+}

vendor/github.com/otiai10/copy/test_setup.go

+// +build !windows
+
+package copy
+
+import (
+	"os"
+	"syscall"
+	"testing"
+)
+
+func setup(m *testing.M) {
+	os.MkdirAll("test/data.copy", os.ModePerm)
+	os.Symlink("test/data/case01", "test/data/case03/case01")
+	os.Chmod("test/data/case07/dir_0555", 0555)
+	os.Chmod("test/data/case07/file_0444", 0444)
+	syscall.Mkfifo("test/data/case11/foo/bar", 0555)
+}

vendor/github.com/otiai10/copy/test_setup_windows.go

+// +build windows
+
+package copy
+
+import (
+	"os"
+	"testing"
+)
+
+func setup(m *testing.M) {
+	os.MkdirAll("test/data.copy", os.ModePerm)
+	os.Symlink("test/data/case01", "test/data/case03/case01")
+	os.Chmod("test/data/case07/dir_0555", 0555)
+	os.Chmod("test/data/case07/file_0444", 0444)
+}

vendor/github.com/rancher/dynamiclistener/factory/gen.go

@@ -119,6 +119,12 @@ func (t *TLS) AddCN(secret *v1.Secret, cn ...string) (*v1.Secret, bool, error) {
 	return t.generateCert(secret, cn...)
 }
 
+func (t *TLS) Regenerate(secret *v1.Secret) (*v1.Secret, error) {
+	cns := cns(secret)
+	secret, _, err := t.generateCert(nil, cns...)
+	return secret, err
+}
+
 func (t *TLS) generateCert(secret *v1.Secret, cn ...string) (*v1.Secret, bool, error) {
 	secret = secret.DeepCopy()
 	if secret == nil {

vendor/github.com/rancher/dynamiclistener/listener.go

@@ -27,6 +27,7 @@ type TLSFactory interface {
 	AddCN(secret *v1.Secret, cn ...string) (*v1.Secret, bool, error)
 	Merge(target *v1.Secret, additional *v1.Secret) (*v1.Secret, bool, error)
 	Filter(cn ...string) []string
+	Regenerate(secret *v1.Secret) (*v1.Secret, error)
 }
 
 type SetFactory interface {
@@ -74,11 +75,18 @@ func NewListener(l net.Listener, storage TLSStorage, caCert *x509.Certificate, c
 		setter.SetFactory(dynamicListener.factory)
 	}
 
+	if config.RegenerateCerts != nil && config.RegenerateCerts() {
+		if err := dynamicListener.regenerateCerts(); err != nil {
+			return nil, nil, err
+		}
+	}
+
 	if config.ExpirationDaysCheck == 0 {
 		config.ExpirationDaysCheck = 30
 	}
 
 	tlsListener := tls.NewListener(dynamicListener.WrapExpiration(config.ExpirationDaysCheck), dynamicListener.tlsConfig)
+
 	return tlsListener, dynamicListener.cacheHandler(), nil
 }
 
@@ -129,6 +137,7 @@ type Config struct {
 	MaxSANs               int
 	ExpirationDaysCheck   int
 	CloseConnOnCertChange bool
+	RegenerateCerts       func() bool
 	FilterCN              func(...string) []string
 }
 
@@ -180,6 +189,30 @@ func (l *listener) WrapExpiration(days int) net.Listener {
 	}
 }
 
+// regenerateCerts regenerates the used certificates and
+// updates the secret.
+func (l *listener) regenerateCerts() error {
+	l.Lock()
+	defer l.Unlock()
+
+	secret, err := l.storage.Get()
+	if err != nil {
+		return err
+	}
+
+	newSecret, err := l.factory.Regenerate(secret)
+	if err != nil {
+		return err
+	}
+	if err := l.storage.Update(newSecret); err != nil {
+		return err
+	}
+	// clear version to force cert reload
+	l.version = ""
+
+	return nil
+}
+
 func (l *listener) checkExpiration(days int) error {
 	l.Lock()
 	defer l.Unlock()

vendor/modules.txt

@@ -910,6 +910,9 @@ github.com/opencontainers/runtime-spec/specs-go
 github.com/opencontainers/selinux/go-selinux
 github.com/opencontainers/selinux/go-selinux/label
 github.com/opencontainers/selinux/pkg/pwalk
+# github.com/otiai10/copy v1.6.0
+## explicit
+github.com/otiai10/copy
 # github.com/peterbourgon/diskv v2.0.1+incompatible
 github.com/peterbourgon/diskv
 # github.com/pierrec/lz4 v2.6.0+incompatible
@@ -941,7 +944,7 @@ github.com/prometheus/common/model
 github.com/prometheus/procfs
 github.com/prometheus/procfs/internal/fs
 github.com/prometheus/procfs/internal/util
-# github.com/rancher/dynamiclistener v0.2.6
+# github.com/rancher/dynamiclistener v0.2.7
 ## explicit
 github.com/rancher/dynamiclistener
 github.com/rancher/dynamiclistener/cert