fix delete logic, update subcommand functionality

Dockerfile.dapper090006740

-FROM k3s:issue-3240
-COPY . /go/src/github.com/rancher/k3s/
\ No newline at end of file

Dockerfile.dapper837489417

-ARG GOLANG=golang:1.16.2-alpine3.12
-FROM ${GOLANG}
-
-ARG http_proxy=$http_proxy
-ARG https_proxy=$https_proxy
-ARG no_proxy=$no_proxy
-ENV http_proxy=$http_proxy
-ENV https_proxy=$https_proxy
-ENV no_proxy=$no_proxy
-
-RUN apk -U --no-cache add bash git gcc musl-dev docker vim less file curl wget ca-certificates jq linux-headers \
-    zlib-dev tar zip squashfs-tools npm coreutils python2 openssl-dev libffi-dev libseccomp libseccomp-dev make \
-    libuv-static sqlite-dev sqlite-static libselinux libselinux-dev zlib-dev zlib-static zstd gzip alpine-sdk binutils-gold
-RUN if [ "$(go env GOARCH)" = "arm64" ]; then                                                               \
-    wget https://github.com/aquasecurity/trivy/releases/download/v0.16.0/trivy_0.16.0_Linux-ARM64.tar.gz && \
-    tar -zxvf trivy_0.16.0_Linux-ARM64.tar.gz                                                            && \
-    mv trivy /usr/local/bin;                                                                                \
-    elif [ "$(go env GOARCH)" = "arm" ]; then                                                               \
-    wget https://github.com/aquasecurity/trivy/releases/download/v0.16.0/trivy_0.16.0_Linux-ARM.tar.gz   && \
-    tar -zxvf trivy_0.16.0_Linux-ARM.tar.gz                                                              && \
-    mv trivy /usr/local/bin;                                                                                \
-    else                                                                                                    \
-    wget https://github.com/aquasecurity/trivy/releases/download/v0.16.0/trivy_0.16.0_Linux-64bit.tar.gz && \
-    tar -zxvf trivy_0.16.0_Linux-64bit.tar.gz                                                            && \
-    mv trivy /usr/local/bin;                                                                                \
-    fi
-# this works for both go 1.15 and 1.16
-RUN GO111MODULE=on GOPROXY=direct go get golang.org/x/tools/cmd/goimports@gopls/v0.6.9
-RUN rm -rf /go/src /go/pkg
-
-RUN if [ "$(go env GOARCH)" = "amd64" ]; then \
-    curl -sL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s v1.38.0; \
-    fi
-
-ENV YQ_URL=https://github.com/mikefarah/yq/releases/download/v4.6.2/yq_linux
-RUN wget -O - ${YQ_URL}_$(go env GOARCH) > /usr/bin/yq && chmod +x /usr/bin/yq
-
-ARG SELINUX=true
-ENV SELINUX $SELINUX
-
-ENV GO111MODULE off
-ENV DAPPER_RUN_ARGS --privileged -v k3s-cache:/go/src/github.com/rancher/k3s/.cache -v trivy-cache:/root/.cache/trivy
-ENV DAPPER_ENV REPO TAG DRONE_TAG IMAGE_NAME SKIP_VALIDATE GCLOUD_AUTH GITHUB_TOKEN GOLANG
-ENV DAPPER_SOURCE /go/src/github.com/rancher/k3s/
-ENV DAPPER_OUTPUT ./bin ./dist ./build/out
-ENV DAPPER_DOCKER_SOCKET true
-ENV HOME ${DAPPER_SOURCE}
-ENV CROSS true
-ENV STATIC_BUILD true
-WORKDIR ${DAPPER_SOURCE}
-
-ENTRYPOINT ["./scripts/entry.sh"]
-CMD ["ci"]

cmd/etcdsnapshot/main.go

@@ -13,7 +13,7 @@ import (
 func main() {
 	app := cmds.NewApp()
 	app.Commands = []cli.Command{
-		cmds.NewEtcdSnapshotCommand(etcdsnapshot.Run, etcdsnapshot.Delete),
+		cmds.NewEtcdSnapshotCommand(etcdsnapshot.Run, cmds.NewEtcdSnapshotSubcommands(etcdsnapshot.Delete)),
 	}
 
 	if err := app.Run(configfilearg.MustParse(os.Args)); err != nil {

cmd/k3s/main.go

@@ -10,7 +10,6 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/rancher/k3s/pkg/cli/cmds"
-	"github.com/rancher/k3s/pkg/cli/etcdsnapshot"
 	"github.com/rancher/k3s/pkg/configfilearg"
 	"github.com/rancher/k3s/pkg/data"
 	"github.com/rancher/k3s/pkg/datadir"
@@ -34,6 +33,8 @@ func main() {
 		return
 	}
 
+	etcdsnapshotCommand := internalCLIAction(version.Program+"-"+cmds.EtcdSnapshotCommand, dataDir, os.Args)
+
 	// Handle subcommand invocation (k3s server, k3s crictl, etc)
 	app := cmds.NewApp()
 	app.Commands = []cli.Command{
@@ -43,7 +44,7 @@ func main() {
 		cmds.NewCRICTL(externalCLIAction("crictl", dataDir)),
 		cmds.NewCtrCommand(externalCLIAction("ctr", dataDir)),
 		cmds.NewCheckConfigCommand(externalCLIAction("check-config", dataDir)),
-		cmds.NewEtcdSnapshotCommand(internalCLIAction(version.Program+"-"+cmds.EtcdSnapshotCommand, dataDir, os.Args), etcdsnapshot.Delete),
+		cmds.NewEtcdSnapshotCommand(etcdsnapshotCommand, cmds.NewEtcdSnapshotSubcommands(etcdsnapshotCommand)),
 	}
 
 	if err := app.Run(os.Args); err != nil {

cmd/server/main.go

@@ -43,7 +43,7 @@ func main() {
 		cmds.NewKubectlCommand(kubectl.Run),
 		cmds.NewCRICTL(crictl.Run),
 		cmds.NewCtrCommand(ctr.Run),
-		cmds.NewEtcdSnapshotCommand(etcdsnapshot.Run, nil),
+		cmds.NewEtcdSnapshotCommand(etcdsnapshot.Run, cmds.NewEtcdSnapshotSubcommands(etcdsnapshot.Delete)),
 	}
 
 	err := app.Run(configfilearg.MustParse(os.Args))

main.go

@@ -27,7 +27,7 @@ func main() {
 		cmds.NewAgentCommand(agent.Run),
 		cmds.NewKubectlCommand(kubectl.Run),
 		cmds.NewCRICTL(crictl.Run),
-		cmds.NewEtcdSnapshotCommand(etcdsnapshot.Run, nil),
+		cmds.NewEtcdSnapshotCommand(etcdsnapshot.Run, cmds.NewEtcdSnapshotSubcommands(etcdsnapshot.Delete)),
 	}
 
 	if err := app.Run(configfilearg.MustParse(os.Args)); err != nil {

pkg/cli/cmds/etcd_snapshot.go

@@ -7,7 +7,7 @@ import (
 
 const EtcdSnapshotCommand = "etcd-snapshot"
 
-var etcdSnapshotFlags = []cli.Flag{
+var EtcdSnapshotFlags = []cli.Flag{
 	DebugFlag,
 	LogFile,
 	AlsoLogToStderr,
@@ -79,27 +79,31 @@ var etcdSnapshotFlags = []cli.Flag{
 	},
 }
 
-func NewEtcdSnapshotCommand(action func(*cli.Context) error, subcommandAction func(*cli.Context) error) cli.Command {
+func NewEtcdSnapshotCommand(action func(*cli.Context) error, subcommands []cli.Command) cli.Command {
 	return cli.Command{
 		Name:            EtcdSnapshotCommand,
 		Usage:           "Trigger an immediate etcd snapshot",
 		SkipFlagParsing: false,
 		SkipArgReorder:  true,
 		Action:          action,
-		Subcommands: []cli.Command{
-			{
-				Name:            "delete",
-				Usage:           "Delete an etcd snapshot",
-				SkipFlagParsing: false,
-				SkipArgReorder:  true,
-				Action:          subcommandAction,
-				Flags:           etcdSnapshotFlags,
-			},
-		},
-		Flags: append(etcdSnapshotFlags, &cli.StringFlag{
+		Subcommands:     subcommands,
+		Flags: append(EtcdSnapshotFlags, &cli.StringFlag{
 			Name:        "dir",
 			Usage:       "(db) Directory to save etcd on-demand snapshot. (default: ${data-dir}/db/snapshots)",
 			Destination: &ServerConfig.EtcdSnapshotDir,
 		}),
 	}
 }
+
+func NewEtcdSnapshotSubcommands(delete func(ctx *cli.Context) error) []cli.Command {
+	return []cli.Command{
+		{
+			Name:            "delete",
+			Usage:           "Delete given snapshot(s)",
+			SkipFlagParsing: false,
+			SkipArgReorder:  true,
+			Action:          delete,
+			Flags:           EtcdSnapshotFlags,
+		},
+	}
+}

pkg/cli/etcdsnapshot/etcd_snapshot.go

@@ -16,6 +16,24 @@ import (
 	"github.com/urfave/cli"
 )
 
+// commandSetup
+func commandSetup(app *cli.Context) error {
+	gspt.SetProcTitle(os.Args[0])
+
+	nodeName := app.String("node-name")
+	if nodeName == "" {
+		h, err := os.Hostname()
+		if err != nil {
+			return err
+		}
+		nodeName = h
+	}
+
+	os.Setenv("NODE_NAME", nodeName)
+
+	return nil
+}
+
 func Run(app *cli.Context) error {
 	if err := cmds.InitLogging(); err != nil {
 		return err
@@ -24,23 +42,18 @@ func Run(app *cli.Context) error {
 }
 
 func run(app *cli.Context, cfg *cmds.Server) error {
-	gspt.SetProcTitle(os.Args[0])
-
-	dataDir, err := server.ResolveDataDir(cfg.DataDir)
-	if err != nil {
+	if err := commandSetup(app); err != nil {
 		return err
 	}
 
-	nodeName := app.String("node-name")
-	if nodeName == "" {
-		h, err := os.Hostname()
-		if err != nil {
-			return err
-		}
-		nodeName = h
+	if len(app.Args()) > 0 {
+		return errors.New("this command doesn't take arguments")
 	}
 
-	os.Setenv("NODE_NAME", nodeName)
+	dataDir, err := server.ResolveDataDir(cfg.DataDir)
+	if err != nil {
+		return err
+	}
 
 	var serverConfig server.Config
 	serverConfig.DisableAgent = true
@@ -98,30 +111,25 @@ func Delete(app *cli.Context) error {
 }
 
 func delete(app *cli.Context, cfg *cmds.Server) error {
-	//gspt.SetProcTitle(os.Args[0])
-
-	dataDir, err := server.ResolveDataDir(cfg.DataDir)
-	if err != nil {
+	if err := commandSetup(app); err != nil {
 		return err
 	}
 
-	nodeName := app.String("node-name")
-	if nodeName == "" {
-		h, err := os.Hostname()
-		if err != nil {
-			return err
-		}
-		nodeName = h
+	snapshots := app.Args()
+	if len(snapshots) == 0 {
+		return errors.New("no snapshots given for removal")
 	}
 
-	os.Setenv("NODE_NAME", nodeName)
+	dataDir, err := server.ResolveDataDir(cfg.DataDir)
+	if err != nil {
+		return err
+	}
 
 	var serverConfig server.Config
 	serverConfig.DisableAgent = true
 	serverConfig.ControlConfig.DataDir = dataDir
 	serverConfig.ControlConfig.EtcdSnapshotName = cfg.EtcdSnapshotName
 	serverConfig.ControlConfig.EtcdSnapshotDir = cfg.EtcdSnapshotDir
-	serverConfig.ControlConfig.EtcdSnapshotRetention = 0 // disable retention check
 	serverConfig.ControlConfig.EtcdS3 = cfg.EtcdS3
 	serverConfig.ControlConfig.EtcdS3Endpoint = cfg.EtcdS3Endpoint
 	serverConfig.ControlConfig.EtcdS3EndpointCA = cfg.EtcdS3EndpointCA
@@ -132,34 +140,17 @@ func delete(app *cli.Context, cfg *cmds.Server) error {
 	serverConfig.ControlConfig.EtcdS3Region = cfg.EtcdS3Region
 	serverConfig.ControlConfig.EtcdS3Folder = cfg.EtcdS3Folder
 	serverConfig.ControlConfig.Runtime = &config.ControlRuntime{}
-	serverConfig.ControlConfig.Runtime.ETCDServerCA = filepath.Join(dataDir, "tls", "etcd", "server-ca.crt")
-	serverConfig.ControlConfig.Runtime.ClientETCDCert = filepath.Join(dataDir, "tls", "etcd", "client.crt")
-	serverConfig.ControlConfig.Runtime.ClientETCDKey = filepath.Join(dataDir, "tls", "etcd", "client.key")
 	serverConfig.ControlConfig.Runtime.KubeConfigAdmin = filepath.Join(dataDir, "cred", "admin.kubeconfig")
 
 	ctx := signals.SetupSignalHandler(context.Background())
 	e := etcd.NewETCD()
 	e.SetControlConfig(&serverConfig.ControlConfig)
 
-	initialized, err := e.IsInitialized(ctx, &serverConfig.ControlConfig)
-	if err != nil {
-		return err
-	}
-	if !initialized {
-		return errors.New("managed etcd database has not been initialized")
-	}
-
-	cluster := cluster.New(&serverConfig.ControlConfig)
-
-	if err := cluster.Bootstrap(ctx); err != nil {
-		return err
-	}
-
 	sc, err := server.NewContext(ctx, serverConfig.ControlConfig.Runtime.KubeConfigAdmin)
 	if err != nil {
 		return err
 	}
 	serverConfig.ControlConfig.Runtime.Core = sc.Core
 
-	return cluster.DeleteSnapshots(ctx, app.Args())
+	return e.DeleteSnapshots(ctx, app.Args())
 }

pkg/etcd/etcd.go

@@ -979,43 +979,58 @@ func (e *ETCD) DeleteSnapshots(ctx context.Context, snapshots []string) error {
 	}
 
 	if e.config.EtcdS3 {
+		logrus.Info("Removing the given etcd snapshot from S3")
+		logrus.Debugf("Removing the given etcd snapshot from S3: %v", snapshots)
+
 		if e.initS3IfNil(ctx); err != nil {
 			return err
 		}
 
 		if len(snapshots) > 1 {
+			logrus.Info("Removing the given etcd snapshot(s) from S3")
+
 			objectsCh := make(chan minio.ObjectInfo)
 
 			go func() {
 				for obj := range e.s3.client.ListObjects(ctx, e.config.EtcdS3BucketName, minio.ListObjectsOptions{}) {
 					if obj.Err != nil {
 						logrus.Error(obj.Err)
+						continue
+					}
+
+					// iterate through the given snapshots and only
+					// add them to the channel for remove if they're
+					// actually found from the bucket listing.
+					for _, snapshot := range snapshots {
+						if snapshot == obj.Key {
+							objectsCh <- obj
+						}
 					}
-					objectsCh <- obj
 				}
+
+				close(objectsCh)
 			}()
 
-			opts := minio.RemoveObjectsOptions{
-				GovernanceBypass: true,
-			}
-			for roErr := range e.s3.client.RemoveObjects(context.Background(), e.config.EtcdS3BucketName, objectsCh, opts) {
+			for roErr := range e.s3.client.RemoveObjects(ctx, e.config.EtcdS3BucketName, objectsCh, minio.RemoveObjectsOptions{}) {
 				logrus.Errorf("Error detected during deletion: %v", roErr)
 			}
 
 			return e.StoreSnapshotData(ctx)
 		}
 
-		opts := minio.RemoveObjectOptions{
-			GovernanceBypass: true,
-		}
-		if err = e.s3.client.RemoveObject(context.Background(), e.config.EtcdS3BucketName, snapshots[0], opts); err != nil {
+		if err = e.s3.client.RemoveObject(ctx, e.config.EtcdS3BucketName, snapshots[0], minio.RemoveObjectOptions{}); err != nil {
 			return errors.Wrap(err, "error detected during deletion")
 		}
 
 		return e.StoreSnapshotData(ctx)
 	}
 
+	logrus.Info("Removing the given locally stored etcd snapshot(s)")
+	logrus.Debugf("Removing the given locally stored etcd snapshot(s): %v", snapshots)
+
 	for _, s := range snapshots {
+		// check if the given snapshot exists. If it does,
+		// remove it, otherwise continue.
 		sf := filepath.Join(snapshotDir, s)
 		if _, err := os.Stat(sf); os.IsNotExist(err) {
 			continue