initial

Dockerfile.dapper090006740

+FROM k3s:issue-3240
+COPY . /go/src/github.com/rancher/k3s/
\ No newline at end of file

Dockerfile.dapper837489417

+ARG GOLANG=golang:1.16.2-alpine3.12
+FROM ${GOLANG}
+
+ARG http_proxy=$http_proxy
+ARG https_proxy=$https_proxy
+ARG no_proxy=$no_proxy
+ENV http_proxy=$http_proxy
+ENV https_proxy=$https_proxy
+ENV no_proxy=$no_proxy
+
+RUN apk -U --no-cache add bash git gcc musl-dev docker vim less file curl wget ca-certificates jq linux-headers \
+    zlib-dev tar zip squashfs-tools npm coreutils python2 openssl-dev libffi-dev libseccomp libseccomp-dev make \
+    libuv-static sqlite-dev sqlite-static libselinux libselinux-dev zlib-dev zlib-static zstd gzip alpine-sdk binutils-gold
+RUN if [ "$(go env GOARCH)" = "arm64" ]; then                                                               \
+    wget https://github.com/aquasecurity/trivy/releases/download/v0.16.0/trivy_0.16.0_Linux-ARM64.tar.gz && \
+    tar -zxvf trivy_0.16.0_Linux-ARM64.tar.gz                                                            && \
+    mv trivy /usr/local/bin;                                                                                \
+    elif [ "$(go env GOARCH)" = "arm" ]; then                                                               \
+    wget https://github.com/aquasecurity/trivy/releases/download/v0.16.0/trivy_0.16.0_Linux-ARM.tar.gz   && \
+    tar -zxvf trivy_0.16.0_Linux-ARM.tar.gz                                                              && \
+    mv trivy /usr/local/bin;                                                                                \
+    else                                                                                                    \
+    wget https://github.com/aquasecurity/trivy/releases/download/v0.16.0/trivy_0.16.0_Linux-64bit.tar.gz && \
+    tar -zxvf trivy_0.16.0_Linux-64bit.tar.gz                                                            && \
+    mv trivy /usr/local/bin;                                                                                \
+    fi
+# this works for both go 1.15 and 1.16
+RUN GO111MODULE=on GOPROXY=direct go get golang.org/x/tools/cmd/goimports@gopls/v0.6.9
+RUN rm -rf /go/src /go/pkg
+
+RUN if [ "$(go env GOARCH)" = "amd64" ]; then \
+    curl -sL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s v1.38.0; \
+    fi
+
+ENV YQ_URL=https://github.com/mikefarah/yq/releases/download/v4.6.2/yq_linux
+RUN wget -O - ${YQ_URL}_$(go env GOARCH) > /usr/bin/yq && chmod +x /usr/bin/yq
+
+ARG SELINUX=true
+ENV SELINUX $SELINUX
+
+ENV GO111MODULE off
+ENV DAPPER_RUN_ARGS --privileged -v k3s-cache:/go/src/github.com/rancher/k3s/.cache -v trivy-cache:/root/.cache/trivy
+ENV DAPPER_ENV REPO TAG DRONE_TAG IMAGE_NAME SKIP_VALIDATE GCLOUD_AUTH GITHUB_TOKEN GOLANG
+ENV DAPPER_SOURCE /go/src/github.com/rancher/k3s/
+ENV DAPPER_OUTPUT ./bin ./dist ./build/out
+ENV DAPPER_DOCKER_SOCKET true
+ENV HOME ${DAPPER_SOURCE}
+ENV CROSS true
+ENV STATIC_BUILD true
+WORKDIR ${DAPPER_SOURCE}
+
+ENTRYPOINT ["./scripts/entry.sh"]
+CMD ["ci"]

cmd/etcdsnapshot/main.go

@@ -13,7 +13,7 @@ import (
 func main() {
 	app := cmds.NewApp()
 	app.Commands = []cli.Command{
-		cmds.NewEtcdSnapshotCommand(etcdsnapshot.Run),
+		cmds.NewEtcdSnapshotCommand(etcdsnapshot.Run, etcdsnapshot.Delete),
 	}
 
 	if err := app.Run(configfilearg.MustParse(os.Args)); err != nil {

cmd/k3s/main.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/rancher/k3s/pkg/cli/cmds"
+	"github.com/rancher/k3s/pkg/cli/etcdsnapshot"
 	"github.com/rancher/k3s/pkg/configfilearg"
 	"github.com/rancher/k3s/pkg/data"
 	"github.com/rancher/k3s/pkg/datadir"
@@ -42,7 +43,7 @@ func main() {
 		cmds.NewCRICTL(externalCLIAction("crictl", dataDir)),
 		cmds.NewCtrCommand(externalCLIAction("ctr", dataDir)),
 		cmds.NewCheckConfigCommand(externalCLIAction("check-config", dataDir)),
-		cmds.NewEtcdSnapshotCommand(internalCLIAction(version.Program+"-"+cmds.EtcdSnapshotCommand, dataDir, os.Args)),
+		cmds.NewEtcdSnapshotCommand(internalCLIAction(version.Program+"-"+cmds.EtcdSnapshotCommand, dataDir, os.Args), etcdsnapshot.Delete),
 	}
 
 	if err := app.Run(os.Args); err != nil {

cmd/server/main.go

@@ -43,7 +43,7 @@ func main() {
 		cmds.NewKubectlCommand(kubectl.Run),
 		cmds.NewCRICTL(crictl.Run),
 		cmds.NewCtrCommand(ctr.Run),
-		cmds.NewEtcdSnapshotCommand(etcdsnapshot.Run),
+		cmds.NewEtcdSnapshotCommand(etcdsnapshot.Run, nil),
 	}
 
 	err := app.Run(configfilearg.MustParse(os.Args))

main.go

@@ -27,7 +27,7 @@ func main() {
 		cmds.NewAgentCommand(agent.Run),
 		cmds.NewKubectlCommand(kubectl.Run),
 		cmds.NewCRICTL(crictl.Run),
-		cmds.NewEtcdSnapshotCommand(etcdsnapshot.Run),
+		cmds.NewEtcdSnapshotCommand(etcdsnapshot.Run, nil),
 	}
 
 	if err := app.Run(configfilearg.MustParse(os.Args)); err != nil {

pkg/cli/cmds/etcd_snapshot.go

@@ -7,88 +7,99 @@ import (
 
 const EtcdSnapshotCommand = "etcd-snapshot"
 
-func NewEtcdSnapshotCommand(action func(*cli.Context) error) cli.Command {
+var etcdSnapshotFlags = []cli.Flag{
+	DebugFlag,
+	LogFile,
+	AlsoLogToStderr,
+	cli.StringFlag{
+		Name:        "node-name",
+		Usage:       "(agent/node) Node name",
+		EnvVar:      version.ProgramUpper + "_NODE_NAME",
+		Destination: &AgentConfig.NodeName,
+	},
+	cli.StringFlag{
+		Name:        "data-dir,d",
+		Usage:       "(data) Folder to hold state default /var/lib/rancher/" + version.Program + " or ${HOME}/.rancher/" + version.Program + " if not root",
+		Destination: &ServerConfig.DataDir,
+	},
+	&cli.StringFlag{
+		Name:        "name",
+		Usage:       "(db) Set the base name of the etcd on-demand snapshot (appended with UNIX timestamp).",
+		Destination: &ServerConfig.EtcdSnapshotName,
+		Value:       "on-demand",
+	},
+	&cli.BoolFlag{
+		Name:        "s3",
+		Usage:       "(db) Enable backup to S3",
+		Destination: &ServerConfig.EtcdS3,
+	},
+	&cli.StringFlag{
+		Name:        "s3-endpoint",
+		Usage:       "(db) S3 endpoint url",
+		Destination: &ServerConfig.EtcdS3Endpoint,
+		Value:       "s3.amazonaws.com",
+	},
+	&cli.StringFlag{
+		Name:        "s3-endpoint-ca",
+		Usage:       "(db) S3 custom CA cert to connect to S3 endpoint",
+		Destination: &ServerConfig.EtcdS3EndpointCA,
+	},
+	&cli.BoolFlag{
+		Name:        "s3-skip-ssl-verify",
+		Usage:       "(db) Disables S3 SSL certificate validation",
+		Destination: &ServerConfig.EtcdS3SkipSSLVerify,
+	},
+	&cli.StringFlag{
+		Name:        "s3-access-key",
+		Usage:       "(db) S3 access key",
+		EnvVar:      "AWS_ACCESS_KEY_ID",
+		Destination: &ServerConfig.EtcdS3AccessKey,
+	},
+	&cli.StringFlag{
+		Name:        "s3-secret-key",
+		Usage:       "(db) S3 secret key",
+		EnvVar:      "AWS_SECRET_ACCESS_KEY",
+		Destination: &ServerConfig.EtcdS3SecretKey,
+	},
+	&cli.StringFlag{
+		Name:        "s3-bucket",
+		Usage:       "(db) S3 bucket name",
+		Destination: &ServerConfig.EtcdS3BucketName,
+	},
+	&cli.StringFlag{
+		Name:        "s3-region",
+		Usage:       "(db) S3 region / bucket location (optional)",
+		Destination: &ServerConfig.EtcdS3Region,
+		Value:       "us-east-1",
+	},
+	&cli.StringFlag{
+		Name:        "s3-folder",
+		Usage:       "(db) S3 folder",
+		Destination: &ServerConfig.EtcdS3Folder,
+	},
+}
+
+func NewEtcdSnapshotCommand(action func(*cli.Context) error, subcommandAction func(*cli.Context) error) cli.Command {
 	return cli.Command{
 		Name:            EtcdSnapshotCommand,
 		Usage:           "Trigger an immediate etcd snapshot",
 		SkipFlagParsing: false,
 		SkipArgReorder:  true,
 		Action:          action,
-		Flags: []cli.Flag{
-			DebugFlag,
-			LogFile,
-			AlsoLogToStderr,
-			cli.StringFlag{
-				Name:        "node-name",
-				Usage:       "(agent/node) Node name",
-				EnvVar:      version.ProgramUpper + "_NODE_NAME",
-				Destination: &AgentConfig.NodeName,
-			},
-			cli.StringFlag{
-				Name:        "data-dir,d",
-				Usage:       "(data) Folder to hold state default /var/lib/rancher/" + version.Program + " or ${HOME}/.rancher/" + version.Program + " if not root",
-				Destination: &ServerConfig.DataDir,
-			},
-			&cli.StringFlag{
-				Name:        "name",
-				Usage:       "(db) Set the base name of the etcd on-demand snapshot (appended with UNIX timestamp).",
-				Destination: &ServerConfig.EtcdSnapshotName,
-				Value:       "on-demand",
-			},
-			&cli.StringFlag{
-				Name:        "dir",
-				Usage:       "(db) Directory to save etcd on-demand snapshot. (default: ${data-dir}/db/snapshots)",
-				Destination: &ServerConfig.EtcdSnapshotDir,
-			},
-			&cli.BoolFlag{
-				Name:        "s3",
-				Usage:       "(db) Enable backup to S3",
-				Destination: &ServerConfig.EtcdS3,
-			},
-			&cli.StringFlag{
-				Name:        "s3-endpoint",
-				Usage:       "(db) S3 endpoint url",
-				Destination: &ServerConfig.EtcdS3Endpoint,
-				Value:       "s3.amazonaws.com",
-			},
-			&cli.StringFlag{
-				Name:        "s3-endpoint-ca",
-				Usage:       "(db) S3 custom CA cert to connect to S3 endpoint",
-				Destination: &ServerConfig.EtcdS3EndpointCA,
-			},
-			&cli.BoolFlag{
-				Name:        "s3-skip-ssl-verify",
-				Usage:       "(db) Disables S3 SSL certificate validation",
-				Destination: &ServerConfig.EtcdS3SkipSSLVerify,
-			},
-			&cli.StringFlag{
-				Name:        "s3-access-key",
-				Usage:       "(db) S3 access key",
-				EnvVar:      "AWS_ACCESS_KEY_ID",
-				Destination: &ServerConfig.EtcdS3AccessKey,
-			},
-			&cli.StringFlag{
-				Name:        "s3-secret-key",
-				Usage:       "(db) S3 secret key",
-				EnvVar:      "AWS_SECRET_ACCESS_KEY",
-				Destination: &ServerConfig.EtcdS3SecretKey,
-			},
-			&cli.StringFlag{
-				Name:        "s3-bucket",
-				Usage:       "(db) S3 bucket name",
-				Destination: &ServerConfig.EtcdS3BucketName,
-			},
-			&cli.StringFlag{
-				Name:        "s3-region",
-				Usage:       "(db) S3 region / bucket location (optional)",
-				Destination: &ServerConfig.EtcdS3Region,
-				Value:       "us-east-1",
-			},
-			&cli.StringFlag{
-				Name:        "s3-folder",
-				Usage:       "(db) S3 folder",
-				Destination: &ServerConfig.EtcdS3Folder,
+		Subcommands: []cli.Command{
+			{
+				Name:            "delete",
+				Usage:           "Delete an etcd snapshot",
+				SkipFlagParsing: false,
+				SkipArgReorder:  true,
+				Action:          subcommandAction,
+				Flags:           etcdSnapshotFlags,
 			},
 		},
+		Flags: append(etcdSnapshotFlags, &cli.StringFlag{
+			Name:        "dir",
+			Usage:       "(db) Directory to save etcd on-demand snapshot. (default: ${data-dir}/db/snapshots)",
+			Destination: &ServerConfig.EtcdSnapshotDir,
+		}),
 	}
 }

pkg/cli/etcdsnapshot/etcd_snapshot.go

@@ -64,8 +64,10 @@ func run(app *cli.Context, cfg *cmds.Server) error {
 	serverConfig.ControlConfig.Runtime.KubeConfigAdmin = filepath.Join(dataDir, "cred", "admin.kubeconfig")
 
 	ctx := signals.SetupSignalHandler(context.Background())
+	e := etcd.NewETCD()
+	e.SetControlConfig(&serverConfig.ControlConfig)
 
-	initialized, err := etcd.NewETCD().IsInitialized(ctx, &serverConfig.ControlConfig)
+	initialized, err := e.IsInitialized(ctx, &serverConfig.ControlConfig)
 	if err != nil {
 		return err
 	}
@@ -87,3 +89,77 @@ func run(app *cli.Context, cfg *cmds.Server) error {
 
 	return cluster.Snapshot(ctx, &serverConfig.ControlConfig)
 }
+
+func Delete(app *cli.Context) error {
+	if err := cmds.InitLogging(); err != nil {
+		return err
+	}
+	return delete(app, &cmds.ServerConfig)
+}
+
+func delete(app *cli.Context, cfg *cmds.Server) error {
+	//gspt.SetProcTitle(os.Args[0])
+
+	dataDir, err := server.ResolveDataDir(cfg.DataDir)
+	if err != nil {
+		return err
+	}
+
+	nodeName := app.String("node-name")
+	if nodeName == "" {
+		h, err := os.Hostname()
+		if err != nil {
+			return err
+		}
+		nodeName = h
+	}
+
+	os.Setenv("NODE_NAME", nodeName)
+
+	var serverConfig server.Config
+	serverConfig.DisableAgent = true
+	serverConfig.ControlConfig.DataDir = dataDir
+	serverConfig.ControlConfig.EtcdSnapshotName = cfg.EtcdSnapshotName
+	serverConfig.ControlConfig.EtcdSnapshotDir = cfg.EtcdSnapshotDir
+	serverConfig.ControlConfig.EtcdSnapshotRetention = 0 // disable retention check
+	serverConfig.ControlConfig.EtcdS3 = cfg.EtcdS3
+	serverConfig.ControlConfig.EtcdS3Endpoint = cfg.EtcdS3Endpoint
+	serverConfig.ControlConfig.EtcdS3EndpointCA = cfg.EtcdS3EndpointCA
+	serverConfig.ControlConfig.EtcdS3SkipSSLVerify = cfg.EtcdS3SkipSSLVerify
+	serverConfig.ControlConfig.EtcdS3AccessKey = cfg.EtcdS3AccessKey
+	serverConfig.ControlConfig.EtcdS3SecretKey = cfg.EtcdS3SecretKey
+	serverConfig.ControlConfig.EtcdS3BucketName = cfg.EtcdS3BucketName
+	serverConfig.ControlConfig.EtcdS3Region = cfg.EtcdS3Region
+	serverConfig.ControlConfig.EtcdS3Folder = cfg.EtcdS3Folder
+	serverConfig.ControlConfig.Runtime = &config.ControlRuntime{}
+	serverConfig.ControlConfig.Runtime.ETCDServerCA = filepath.Join(dataDir, "tls", "etcd", "server-ca.crt")
+	serverConfig.ControlConfig.Runtime.ClientETCDCert = filepath.Join(dataDir, "tls", "etcd", "client.crt")
+	serverConfig.ControlConfig.Runtime.ClientETCDKey = filepath.Join(dataDir, "tls", "etcd", "client.key")
+	serverConfig.ControlConfig.Runtime.KubeConfigAdmin = filepath.Join(dataDir, "cred", "admin.kubeconfig")
+
+	ctx := signals.SetupSignalHandler(context.Background())
+	e := etcd.NewETCD()
+	e.SetControlConfig(&serverConfig.ControlConfig)
+
+	initialized, err := e.IsInitialized(ctx, &serverConfig.ControlConfig)
+	if err != nil {
+		return err
+	}
+	if !initialized {
+		return errors.New("managed etcd database has not been initialized")
+	}
+
+	cluster := cluster.New(&serverConfig.ControlConfig)
+
+	if err := cluster.Bootstrap(ctx); err != nil {
+		return err
+	}
+
+	sc, err := server.NewContext(ctx, serverConfig.ControlConfig.Runtime.KubeConfigAdmin)
+	if err != nil {
+		return err
+	}
+	serverConfig.ControlConfig.Runtime.Core = sc.Core
+
+	return cluster.DeleteSnapshots(ctx, app.Args())
+}

pkg/cluster/bootstrap.go

@@ -156,3 +156,9 @@ func (c *Cluster) Snapshot(ctx context.Context, config *config.Control) error {
 	}
 	return c.managedDB.Snapshot(ctx, config)
 }
+
+// DeleteSnapshots is a proxy method to call the DeleteSnapshot method on
+// the manageddb interface for etcd clusters.
+func (c *Cluster) DeleteSnapshots(ctx context.Context, snapshots []string) error {
+	return c.managedDB.DeleteSnapshots(ctx, snapshots)
+}

pkg/cluster/managed/drivers.go

@@ -22,6 +22,7 @@ type Driver interface {
 	Restore(ctx context.Context) error
 	EndpointName() string
 	Snapshot(ctx context.Context, config *config.Control) error
+	DeleteSnapshots(ctx context.Context, snapshots []string) error
 	StoreSnapshotData(ctx context.Context) error
 	GetMembersClientURLs(ctx context.Context) ([]string, error)
 	RemoveSelf(ctx context.Context) error

pkg/etcd/etcd.go

@@ -102,6 +102,11 @@ func (e *ETCD) EndpointName() string {
 	return "etcd"
 }
 
+// SetControlConfig sets the given config on the etcd struct.
+func (e *ETCD) SetControlConfig(config *config.Control) {
+	e.config = config
+}
+
 // Test ensures that the local node is a voting member of the target cluster.
 // If it is still a learner or not a part of the cluster, an error is raised.
 func (e *ETCD) Test(ctx context.Context) error {
@@ -203,12 +208,8 @@ func (e *ETCD) Reset(ctx context.Context, rebootstrap func() error) error {
 	// If asked to restore from a snapshot, do so
 	if e.config.ClusterResetRestorePath != "" {
 		if e.config.EtcdS3 {
-			if e.s3 == nil {
-				s3, err := newS3(ctx, e.config)
-				if err != nil {
-					return err
-				}
-				e.s3 = s3
+			if err := e.initS3IfNil(ctx); err != nil {
+				return err
 			}
 			logrus.Infof("Retrieving etcd snapshot %s from S3", e.config.ClusterResetRestorePath)
 			if err := e.s3.download(ctx); err != nil {
@@ -838,12 +839,8 @@ func (e *ETCD) Snapshot(ctx context.Context, config *config.Control) error {
 
 	if e.config.EtcdS3 {
 		logrus.Infof("Saving etcd snapshot %s to S3", snapshotName)
-		if e.s3 == nil {
-			s3, err := newS3(ctx, config)
-			if err != nil {
-				return err
-			}
-			e.s3 = s3
+		if err := e.initS3IfNil(ctx); err != nil {
+			return err
 		}
 		if err := e.s3.upload(ctx, snapshotPath); err != nil {
 			return err
@@ -899,12 +896,8 @@ func (e *ETCD) listSnapshots(ctx context.Context, snapshotDir string) ([]snapsho
 		ctx, cancel := context.WithCancel(ctx)
 		defer cancel()
 
-		if e.s3 == nil {
-			s3, err := newS3(ctx, e.config)
-			if err != nil {
-				return nil, err
-			}
-			e.s3 = s3
+		if err := e.initS3IfNil(ctx); err != nil {
+			return nil, err
 		}
 
 		objects := e.s3.client.ListObjects(ctx, e.config.EtcdS3BucketName, minio.ListObjectsOptions{})
@@ -963,6 +956,78 @@ func (e *ETCD) listSnapshots(ctx context.Context, snapshotDir string) ([]snapsho
 	return snapshots, nil
 }
 
+// initS3IfNil initializes the S3 client
+// if it hasn't yet been initialized.
+func (e *ETCD) initS3IfNil(ctx context.Context) error {
+	if e.s3 == nil {
+		s3, err := newS3(ctx, e.config)
+		if err != nil {
+			return err
+		}
+		e.s3 = s3
+	}
+
+	return nil
+}
+
+// deleteSnapshots removes the given snapshots from
+// either local storage or S3.
+func (e *ETCD) DeleteSnapshots(ctx context.Context, snapshots []string) error {
+	snapshotDir, err := snapshotDir(e.config)
+	if err != nil {
+		return errors.Wrap(err, "failed to get the snapshot dir")
+	}
+
+	if e.config.EtcdS3 {
+		if e.initS3IfNil(ctx); err != nil {
+			return err
+		}
+
+		if len(snapshots) > 1 {
+			objectsCh := make(chan minio.ObjectInfo)
+
+			go func() {
+				for obj := range e.s3.client.ListObjects(ctx, e.config.EtcdS3BucketName, minio.ListObjectsOptions{}) {
+					if obj.Err != nil {
+						logrus.Error(obj.Err)
+					}
+					objectsCh <- obj
+				}
+			}()
+
+			opts := minio.RemoveObjectsOptions{
+				GovernanceBypass: true,
+			}
+			for roErr := range e.s3.client.RemoveObjects(context.Background(), e.config.EtcdS3BucketName, objectsCh, opts) {
+				logrus.Errorf("Error detected during deletion: %v", roErr)
+			}
+
+			return e.StoreSnapshotData(ctx)
+		}
+
+		opts := minio.RemoveObjectOptions{
+			GovernanceBypass: true,
+		}
+		if err = e.s3.client.RemoveObject(context.Background(), e.config.EtcdS3BucketName, snapshots[0], opts); err != nil {
+			return errors.Wrap(err, "error detected during deletion")
+		}
+
+		return e.StoreSnapshotData(ctx)
+	}
+
+	for _, s := range snapshots {
+		sf := filepath.Join(snapshotDir, s)
+		if _, err := os.Stat(sf); os.IsNotExist(err) {
+			continue
+		}
+		if err := os.Remove(sf); err != nil {
+			return err
+		}
+	}
+
+	return e.StoreSnapshotData(ctx)
+}
+
 // updateSnapshotData populates the given map with the contents of the given slice.
 func updateSnapshotData(data map[string]string, snapshotFiles []snapshotFile) error {
 	for _, v := range snapshotFiles {