initial

Dockerfile.dapper090006740

+FROM k3s:issue-3240
+COPY . /go/src/github.com/rancher/k3s/
\ No newline at end of file

Dockerfile.dapper837489417

+ARG GOLANG=golang:1.16.2-alpine3.12
+FROM ${GOLANG}
+
+ARG http_proxy=$http_proxy
+ARG https_proxy=$https_proxy
+ARG no_proxy=$no_proxy
+ENV http_proxy=$http_proxy
+ENV https_proxy=$https_proxy
+ENV no_proxy=$no_proxy
+
+RUN apk -U --no-cache add bash git gcc musl-dev docker vim less file curl wget ca-certificates jq linux-headers \
+    zlib-dev tar zip squashfs-tools npm coreutils python2 openssl-dev libffi-dev libseccomp libseccomp-dev make \
+    libuv-static sqlite-dev sqlite-static libselinux libselinux-dev zlib-dev zlib-static zstd gzip alpine-sdk binutils-gold
+RUN if [ "$(go env GOARCH)" = "arm64" ]; then                                                               \
+    wget https://github.com/aquasecurity/trivy/releases/download/v0.16.0/trivy_0.16.0_Linux-ARM64.tar.gz && \
+    tar -zxvf trivy_0.16.0_Linux-ARM64.tar.gz                                                            && \
+    mv trivy /usr/local/bin;                                                                                \
+    elif [ "$(go env GOARCH)" = "arm" ]; then                                                               \
+    wget https://github.com/aquasecurity/trivy/releases/download/v0.16.0/trivy_0.16.0_Linux-ARM.tar.gz   && \
+    tar -zxvf trivy_0.16.0_Linux-ARM.tar.gz                                                              && \
+    mv trivy /usr/local/bin;                                                                                \
+    else                                                                                                    \
+    wget https://github.com/aquasecurity/trivy/releases/download/v0.16.0/trivy_0.16.0_Linux-64bit.tar.gz && \
+    tar -zxvf trivy_0.16.0_Linux-64bit.tar.gz                                                            && \
+    mv trivy /usr/local/bin;                                                                                \
+    fi
+# this works for both go 1.15 and 1.16
+RUN GO111MODULE=on GOPROXY=direct go get golang.org/x/tools/cmd/goimports@gopls/v0.6.9
+RUN rm -rf /go/src /go/pkg
+
+RUN if [ "$(go env GOARCH)" = "amd64" ]; then \
+    curl -sL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s v1.38.0; \
+    fi
+
+ENV YQ_URL=https://github.com/mikefarah/yq/releases/download/v4.6.2/yq_linux
+RUN wget -O - ${YQ_URL}_$(go env GOARCH) > /usr/bin/yq && chmod +x /usr/bin/yq
+
+ARG SELINUX=true
+ENV SELINUX $SELINUX
+
+ENV GO111MODULE off
+ENV DAPPER_RUN_ARGS --privileged -v k3s-cache:/go/src/github.com/rancher/k3s/.cache -v trivy-cache:/root/.cache/trivy
+ENV DAPPER_ENV REPO TAG DRONE_TAG IMAGE_NAME SKIP_VALIDATE GCLOUD_AUTH GITHUB_TOKEN GOLANG
+ENV DAPPER_SOURCE /go/src/github.com/rancher/k3s/
+ENV DAPPER_OUTPUT ./bin ./dist ./build/out
+ENV DAPPER_DOCKER_SOCKET true
+ENV HOME ${DAPPER_SOURCE}
+ENV CROSS true
+ENV STATIC_BUILD true
+WORKDIR ${DAPPER_SOURCE}
+
+ENTRYPOINT ["./scripts/entry.sh"]
+CMD ["ci"]

cmd/etcdsnapshot/main.go

@@ -13,7 +13,7 @@ import (
 func main() {
 	app := cmds.NewApp()
 	app.Commands = []cli.Command{
-		cmds.NewEtcdSnapshotCommand(etcdsnapshot.Run),
+		cmds.NewEtcdSnapshotCommand(etcdsnapshot.Run, etcdsnapshot.Delete),
 	}
 
 	if err := app.Run(configfilearg.MustParse(os.Args)); err != nil {

cmd/k3s/main.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/rancher/k3s/pkg/cli/cmds"
+	"github.com/rancher/k3s/pkg/cli/etcdsnapshot"
 	"github.com/rancher/k3s/pkg/configfilearg"
 	"github.com/rancher/k3s/pkg/data"
 	"github.com/rancher/k3s/pkg/datadir"
@@ -42,7 +43,7 @@ func main() {
 		cmds.NewCRICTL(externalCLIAction("crictl", dataDir)),
 		cmds.NewCtrCommand(externalCLIAction("ctr", dataDir)),
 		cmds.NewCheckConfigCommand(externalCLIAction("check-config", dataDir)),
-		cmds.NewEtcdSnapshotCommand(internalCLIAction(version.Program+"-"+cmds.EtcdSnapshotCommand, dataDir, os.Args)),
+		cmds.NewEtcdSnapshotCommand(internalCLIAction(version.Program+"-"+cmds.EtcdSnapshotCommand, dataDir, os.Args), etcdsnapshot.Delete),
 	}
 
 	if err := app.Run(os.Args); err != nil {

cmd/server/main.go

@@ -43,7 +43,7 @@ func main() {
 		cmds.NewKubectlCommand(kubectl.Run),
 		cmds.NewCRICTL(crictl.Run),
 		cmds.NewCtrCommand(ctr.Run),
-		cmds.NewEtcdSnapshotCommand(etcdsnapshot.Run),
+		cmds.NewEtcdSnapshotCommand(etcdsnapshot.Run, nil),
 	}
 
 	err := app.Run(configfilearg.MustParse(os.Args))

main.go

@@ -27,7 +27,7 @@ func main() {
 		cmds.NewAgentCommand(agent.Run),
 		cmds.NewKubectlCommand(kubectl.Run),
 		cmds.NewCRICTL(crictl.Run),
-		cmds.NewEtcdSnapshotCommand(etcdsnapshot.Run),
+		cmds.NewEtcdSnapshotCommand(etcdsnapshot.Run, nil),
 	}
 
 	if err := app.Run(configfilearg.MustParse(os.Args)); err != nil {

pkg/cli/cmds/etcd_snapshot.go

@@ -7,14 +7,7 @@ import (
 
 const EtcdSnapshotCommand = "etcd-snapshot"
 
-func NewEtcdSnapshotCommand(action func(*cli.Context) error) cli.Command {
-	return cli.Command{
-		Name:            EtcdSnapshotCommand,
-		Usage:           "Trigger an immediate etcd snapshot",
-		SkipFlagParsing: false,
-		SkipArgReorder:  true,
-		Action:          action,
-		Flags: []cli.Flag{
+var etcdSnapshotFlags = []cli.Flag{
 	DebugFlag,
 	LogFile,
 	AlsoLogToStderr,
@@ -35,11 +28,6 @@ func NewEtcdSnapshotCommand(action func(*cli.Context) error) cli.Command {
 		Destination: &ServerConfig.EtcdSnapshotName,
 		Value:       "on-demand",
 	},
-			&cli.StringFlag{
-				Name:        "dir",
-				Usage:       "(db) Directory to save etcd on-demand snapshot. (default: ${data-dir}/db/snapshots)",
-				Destination: &ServerConfig.EtcdSnapshotDir,
-			},
 	&cli.BoolFlag{
 		Name:        "s3",
 		Usage:       "(db) Enable backup to S3",
@@ -89,6 +77,29 @@ func NewEtcdSnapshotCommand(action func(*cli.Context) error) cli.Command {
 		Usage:       "(db) S3 folder",
 		Destination: &ServerConfig.EtcdS3Folder,
 	},
+}
+
+func NewEtcdSnapshotCommand(action func(*cli.Context) error, subcommandAction func(*cli.Context) error) cli.Command {
+	return cli.Command{
+		Name:            EtcdSnapshotCommand,
+		Usage:           "Trigger an immediate etcd snapshot",
+		SkipFlagParsing: false,
+		SkipArgReorder:  true,
+		Action:          action,
+		Subcommands: []cli.Command{
+			{
+				Name:            "delete",
+				Usage:           "Delete an etcd snapshot",
+				SkipFlagParsing: false,
+				SkipArgReorder:  true,
+				Action:          subcommandAction,
+				Flags:           etcdSnapshotFlags,
+			},
 		},
+		Flags: append(etcdSnapshotFlags, &cli.StringFlag{
+			Name:        "dir",
+			Usage:       "(db) Directory to save etcd on-demand snapshot. (default: ${data-dir}/db/snapshots)",
+			Destination: &ServerConfig.EtcdSnapshotDir,
+		}),
 	}
 }

pkg/cli/etcdsnapshot/etcd_snapshot.go

@@ -64,8 +64,10 @@ func run(app *cli.Context, cfg *cmds.Server) error {
 	serverConfig.ControlConfig.Runtime.KubeConfigAdmin = filepath.Join(dataDir, "cred", "admin.kubeconfig")
 
 	ctx := signals.SetupSignalHandler(context.Background())
+	e := etcd.NewETCD()
+	e.SetControlConfig(&serverConfig.ControlConfig)
 
-	initialized, err := etcd.NewETCD().IsInitialized(ctx, &serverConfig.ControlConfig)
+	initialized, err := e.IsInitialized(ctx, &serverConfig.ControlConfig)
 	if err != nil {
 		return err
 	}
@@ -87,3 +89,77 @@ func run(app *cli.Context, cfg *cmds.Server) error {
 
 	return cluster.Snapshot(ctx, &serverConfig.ControlConfig)
 }
+
+func Delete(app *cli.Context) error {
+	if err := cmds.InitLogging(); err != nil {
+		return err
+	}
+	return delete(app, &cmds.ServerConfig)
+}
+
+func delete(app *cli.Context, cfg *cmds.Server) error {
+	//gspt.SetProcTitle(os.Args[0])
+
+	dataDir, err := server.ResolveDataDir(cfg.DataDir)
+	if err != nil {
+		return err
+	}
+
+	nodeName := app.String("node-name")
+	if nodeName == "" {
+		h, err := os.Hostname()
+		if err != nil {
+			return err
+		}
+		nodeName = h
+	}
+
+	os.Setenv("NODE_NAME", nodeName)
+
+	var serverConfig server.Config
+	serverConfig.DisableAgent = true
+	serverConfig.ControlConfig.DataDir = dataDir
+	serverConfig.ControlConfig.EtcdSnapshotName = cfg.EtcdSnapshotName
+	serverConfig.ControlConfig.EtcdSnapshotDir = cfg.EtcdSnapshotDir
+	serverConfig.ControlConfig.EtcdSnapshotRetention = 0 // disable retention check
+	serverConfig.ControlConfig.EtcdS3 = cfg.EtcdS3
+	serverConfig.ControlConfig.EtcdS3Endpoint = cfg.EtcdS3Endpoint
+	serverConfig.ControlConfig.EtcdS3EndpointCA = cfg.EtcdS3EndpointCA
+	serverConfig.ControlConfig.EtcdS3SkipSSLVerify = cfg.EtcdS3SkipSSLVerify
+	serverConfig.ControlConfig.EtcdS3AccessKey = cfg.EtcdS3AccessKey
+	serverConfig.ControlConfig.EtcdS3SecretKey = cfg.EtcdS3SecretKey
+	serverConfig.ControlConfig.EtcdS3BucketName = cfg.EtcdS3BucketName
+	serverConfig.ControlConfig.EtcdS3Region = cfg.EtcdS3Region
+	serverConfig.ControlConfig.EtcdS3Folder = cfg.EtcdS3Folder
+	serverConfig.ControlConfig.Runtime = &config.ControlRuntime{}
+	serverConfig.ControlConfig.Runtime.ETCDServerCA = filepath.Join(dataDir, "tls", "etcd", "server-ca.crt")
+	serverConfig.ControlConfig.Runtime.ClientETCDCert = filepath.Join(dataDir, "tls", "etcd", "client.crt")
+	serverConfig.ControlConfig.Runtime.ClientETCDKey = filepath.Join(dataDir, "tls", "etcd", "client.key")
+	serverConfig.ControlConfig.Runtime.KubeConfigAdmin = filepath.Join(dataDir, "cred", "admin.kubeconfig")
+
+	ctx := signals.SetupSignalHandler(context.Background())
+	e := etcd.NewETCD()
+	e.SetControlConfig(&serverConfig.ControlConfig)
+
+	initialized, err := e.IsInitialized(ctx, &serverConfig.ControlConfig)
+	if err != nil {
+		return err
+	}
+	if !initialized {
+		return errors.New("managed etcd database has not been initialized")
+	}
+
+	cluster := cluster.New(&serverConfig.ControlConfig)
+
+	if err := cluster.Bootstrap(ctx); err != nil {
+		return err
+	}
+
+	sc, err := server.NewContext(ctx, serverConfig.ControlConfig.Runtime.KubeConfigAdmin)
+	if err != nil {
+		return err
+	}
+	serverConfig.ControlConfig.Runtime.Core = sc.Core
+
+	return cluster.DeleteSnapshots(ctx, app.Args())
+}

pkg/cluster/bootstrap.go

@@ -156,3 +156,9 @@ func (c *Cluster) Snapshot(ctx context.Context, config *config.Control) error {
 	}
 	return c.managedDB.Snapshot(ctx, config)
 }
+
+// DeleteSnapshots is a proxy method to call the DeleteSnapshot method on
+// the manageddb interface for etcd clusters.
+func (c *Cluster) DeleteSnapshots(ctx context.Context, snapshots []string) error {
+	return c.managedDB.DeleteSnapshots(ctx, snapshots)
+}

pkg/cluster/managed/drivers.go

@@ -22,6 +22,7 @@ type Driver interface {
 	Restore(ctx context.Context) error
 	EndpointName() string
 	Snapshot(ctx context.Context, config *config.Control) error
+	DeleteSnapshots(ctx context.Context, snapshots []string) error
 	StoreSnapshotData(ctx context.Context) error
 	GetMembersClientURLs(ctx context.Context) ([]string, error)
 	RemoveSelf(ctx context.Context) error

pkg/etcd/etcd.go

@@ -102,6 +102,11 @@ func (e *ETCD) EndpointName() string {
 	return "etcd"
 }
 
+// SetControlConfig sets the given config on the etcd struct.
+func (e *ETCD) SetControlConfig(config *config.Control) {
+	e.config = config
+}
+
 // Test ensures that the local node is a voting member of the target cluster.
 // If it is still a learner or not a part of the cluster, an error is raised.
 func (e *ETCD) Test(ctx context.Context) error {
@@ -203,13 +208,9 @@ func (e *ETCD) Reset(ctx context.Context, rebootstrap func() error) error {
 	// If asked to restore from a snapshot, do so
 	if e.config.ClusterResetRestorePath != "" {
 		if e.config.EtcdS3 {
-			if e.s3 == nil {
-				s3, err := newS3(ctx, e.config)
-				if err != nil {
+			if err := e.initS3IfNil(ctx); err != nil {
 				return err
 			}
-				e.s3 = s3
-			}
 			logrus.Infof("Retrieving etcd snapshot %s from S3", e.config.ClusterResetRestorePath)
 			if err := e.s3.download(ctx); err != nil {
 				return err
@@ -838,13 +839,9 @@ func (e *ETCD) Snapshot(ctx context.Context, config *config.Control) error {
 
 	if e.config.EtcdS3 {
 		logrus.Infof("Saving etcd snapshot %s to S3", snapshotName)
-		if e.s3 == nil {
-			s3, err := newS3(ctx, config)
-			if err != nil {
+		if err := e.initS3IfNil(ctx); err != nil {
 			return err
 		}
-			e.s3 = s3
-		}
 		if err := e.s3.upload(ctx, snapshotPath); err != nil {
 			return err
 		}
@@ -899,13 +896,9 @@ func (e *ETCD) listSnapshots(ctx context.Context, snapshotDir string) ([]snapsho
 		ctx, cancel := context.WithCancel(ctx)
 		defer cancel()
 
-		if e.s3 == nil {
-			s3, err := newS3(ctx, e.config)
-			if err != nil {
+		if err := e.initS3IfNil(ctx); err != nil {
 			return nil, err
 		}
-			e.s3 = s3
-		}
 
 		objects := e.s3.client.ListObjects(ctx, e.config.EtcdS3BucketName, minio.ListObjectsOptions{})
 
@@ -963,6 +956,78 @@ func (e *ETCD) listSnapshots(ctx context.Context, snapshotDir string) ([]snapsho
 	return snapshots, nil
 }
 
+// initS3IfNil initializes the S3 client
+// if it hasn't yet been initialized.
+func (e *ETCD) initS3IfNil(ctx context.Context) error {
+	if e.s3 == nil {
+		s3, err := newS3(ctx, e.config)
+		if err != nil {
+			return err
+		}
+		e.s3 = s3
+	}
+
+	return nil
+}
+
+// deleteSnapshots removes the given snapshots from
+// either local storage or S3.
+func (e *ETCD) DeleteSnapshots(ctx context.Context, snapshots []string) error {
+	snapshotDir, err := snapshotDir(e.config)
+	if err != nil {
+		return errors.Wrap(err, "failed to get the snapshot dir")
+	}
+
+	if e.config.EtcdS3 {
+		if e.initS3IfNil(ctx); err != nil {
+			return err
+		}
+
+		if len(snapshots) > 1 {
+			objectsCh := make(chan minio.ObjectInfo)
+
+			go func() {
+				for obj := range e.s3.client.ListObjects(ctx, e.config.EtcdS3BucketName, minio.ListObjectsOptions{}) {
+					if obj.Err != nil {
+						logrus.Error(obj.Err)
+					}
+					objectsCh <- obj
+				}
+			}()
+
+			opts := minio.RemoveObjectsOptions{
+				GovernanceBypass: true,
+			}
+			for roErr := range e.s3.client.RemoveObjects(context.Background(), e.config.EtcdS3BucketName, objectsCh, opts) {
+				logrus.Errorf("Error detected during deletion: %v", roErr)
+			}
+
+			return e.StoreSnapshotData(ctx)
+		}
+
+		opts := minio.RemoveObjectOptions{
+			GovernanceBypass: true,
+		}
+		if err = e.s3.client.RemoveObject(context.Background(), e.config.EtcdS3BucketName, snapshots[0], opts); err != nil {
+			return errors.Wrap(err, "error detected during deletion")
+		}
+
+		return e.StoreSnapshotData(ctx)
+	}
+
+	for _, s := range snapshots {
+		sf := filepath.Join(snapshotDir, s)
+		if _, err := os.Stat(sf); os.IsNotExist(err) {
+			continue
+		}
+		if err := os.Remove(sf); err != nil {
+			return err
+		}
+	}
+
+	return e.StoreSnapshotData(ctx)
+}
+
 // updateSnapshotData populates the given map with the contents of the given slice.
 func updateSnapshotData(data map[string]string, snapshotFiles []snapshotFile) error {
 	for _, v := range snapshotFiles {