[Release-1.20] - Add etcd s3 timeout (#4207) (#4227)

pkg/cli/cmds/etcd_snapshot.go

 package cmds
 
 import (
+	"time"
+
 	"github.com/rancher/k3s/pkg/version"
 	"github.com/urfave/cli"
 )
@@ -77,6 +79,17 @@ var EtcdSnapshotFlags = []cli.Flag{
 		Usage:       "(db) S3 folder",
 		Destination: &ServerConfig.EtcdS3Folder,
 	},
+	&cli.BoolFlag{
+		Name:        "s3-insecure,etcd-s3-insecure",
+		Usage:       "(db) Disables S3 over HTTPS",
+		Destination: &ServerConfig.EtcdS3Insecure,
+	},
+	&cli.DurationFlag{
+		Name:        "s3-timeout,etcd-s3-timeout",
+		Usage:       "(db) S3 timeout",
+		Destination: &ServerConfig.EtcdS3Timeout,
+		Value:       30 * time.Second,
+	},
 }
 
 func NewEtcdSnapshotCommand(action func(*cli.Context) error, subcommands []cli.Command) cli.Command {

pkg/cli/cmds/server.go

@@ -2,6 +2,7 @@ package cmds
 
 import (
 	"context"
+	"time"
 
 	"github.com/rancher/k3s/pkg/version"
 	"github.com/urfave/cli"
@@ -77,6 +78,8 @@ type Server struct {
 	EtcdS3BucketName         string
 	EtcdS3Region             string
 	EtcdS3Folder             string
+	EtcdS3Timeout            time.Duration
+	EtcdS3Insecure           bool
 }
 
 var ServerConfig Server
@@ -312,6 +315,17 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command {
 				Usage:       "(db) S3 folder",
 				Destination: &ServerConfig.EtcdS3Folder,
 			},
+			&cli.BoolFlag{
+				Name:        "etcd-s3-insecure",
+				Usage:       "(db) Disables S3 over HTTPS",
+				Destination: &ServerConfig.EtcdS3Insecure,
+			},
+			&cli.DurationFlag{
+				Name:        "etcd-s3-timeout",
+				Usage:       "(db) S3 timeout",
+				Destination: &ServerConfig.EtcdS3Timeout,
+				Value:       30 * time.Second,
+			},
 			cli.StringFlag{
 				Name:        "default-local-storage-path",
 				Usage:       "(storage) Default local storage path for local provisioner storage class",

pkg/cli/etcdsnapshot/etcd_snapshot.go

@@ -48,6 +48,8 @@ func commandSetup(app *cli.Context, cfg *cmds.Server, sc *server.Config) (string
 	sc.ControlConfig.EtcdS3BucketName = cfg.EtcdS3BucketName
 	sc.ControlConfig.EtcdS3Region = cfg.EtcdS3Region
 	sc.ControlConfig.EtcdS3Folder = cfg.EtcdS3Folder
+	sc.ControlConfig.EtcdS3Insecure = cfg.EtcdS3Insecure
+	sc.ControlConfig.EtcdS3Timeout = cfg.EtcdS3Timeout
 	sc.ControlConfig.Runtime = &config.ControlRuntime{}
 
 	return server.ResolveDataDir(cfg.DataDir)

pkg/cli/server/server.go

@@ -150,6 +150,8 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
 		serverConfig.ControlConfig.EtcdS3BucketName = cfg.EtcdS3BucketName
 		serverConfig.ControlConfig.EtcdS3Region = cfg.EtcdS3Region
 		serverConfig.ControlConfig.EtcdS3Folder = cfg.EtcdS3Folder
+		serverConfig.ControlConfig.EtcdS3Insecure = cfg.EtcdS3Insecure
+		serverConfig.ControlConfig.EtcdS3Timeout = cfg.EtcdS3Timeout
 	} else {
 		logrus.Info("ETCD snapshots are disabled")
 	}

pkg/daemons/config/types.go

@@ -8,6 +8,7 @@ import (
 	"net/http"
 	"sort"
 	"strings"
+	"time"
 
 	"github.com/k3s-io/kine/pkg/endpoint"
 	"github.com/rancher/wrangler-api/pkg/generated/controllers/core"
@@ -153,6 +154,9 @@ type Control struct {
 	EtcdS3BucketName         string
 	EtcdS3Region             string
 	EtcdS3Folder             string
+	EtcdS3Timeout            time.Duration
+	EtcdS3Insecure           bool
+	ServerNodeName           string
 
 	BindAddress string
 	SANs        []string

pkg/etcd/etcd.go

@@ -71,7 +71,7 @@ type ETCD struct {
 	runtime *config.ControlRuntime
 	address string
 	cron    *cron.Cron
-	s3      *s3
+	s3      *S3
 }
 
 type learnerProgress struct {
@@ -1032,7 +1032,7 @@ func (e *ETCD) DeleteSnapshots(ctx context.Context, snapshots []string) error {
 
 		objectsCh := make(chan minio.ObjectInfo)
 
-		ctx, cancel := context.WithTimeout(ctx, defaultS3OpTimeout)
+		ctx, cancel := context.WithTimeout(ctx, e.config.EtcdS3Timeout)
 		defer cancel()
 
 		go func() {

pkg/etcd/s3.go

@@ -14,7 +14,6 @@ import (
 	"path/filepath"
 	"sort"
 	"strings"
-	"time"
 
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
@@ -23,10 +22,8 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-const defaultS3OpTimeout = time.Second * 30
-
-// s3 maintains state for S3 functionality.
-type s3 struct {
+// S3 maintains state for S3 functionality.
+type S3 struct {
 	config *config.Control
 	client *minio.Client
 }
@@ -34,7 +31,7 @@ type s3 struct {
 // newS3 creates a new value of type s3 pointer with a
 // copy of the config.Control pointer and initializes
 // a new Minio client.
-func newS3(ctx context.Context, config *config.Control) (*s3, error) {
+func newS3(ctx context.Context, config *config.Control) (*S3, error) {
 	tr := http.DefaultTransport
 
 	switch {
@@ -71,7 +68,7 @@ func newS3(ctx context.Context, config *config.Control) (*s3, error) {
 
 	logrus.Infof("Checking if S3 bucket %s exists", config.EtcdS3BucketName)
 
-	ctx, cancel := context.WithTimeout(ctx, defaultS3OpTimeout)
+	ctx, cancel := context.WithTimeout(ctx, config.EtcdS3Timeout)
 	defer cancel()
 
 	exists, err := c.BucketExists(ctx, config.EtcdS3BucketName)
@@ -83,7 +80,7 @@ func newS3(ctx context.Context, config *config.Control) (*s3, error) {
 	}
 	logrus.Infof("S3 bucket %s exists", config.EtcdS3BucketName)
 
-	return &s3{
+	return &S3{
 		config: config,
 		client: c,
 	}, nil
@@ -91,7 +88,7 @@ func newS3(ctx context.Context, config *config.Control) (*s3, error) {
 
 // upload uploads the given snapshot to the configured S3
 // compatible backend.
-func (s *s3) upload(ctx context.Context, snapshot string) error {
+func (s *S3) upload(ctx context.Context, snapshot string) error {
 	basename := filepath.Base(snapshot)
 	var snapshotFileName string
 	if s.config.EtcdS3Folder != "" {
@@ -100,7 +97,7 @@ func (s *s3) upload(ctx context.Context, snapshot string) error {
 		snapshotFileName = basename
 	}
 
-	toCtx, cancel := context.WithTimeout(ctx, defaultS3OpTimeout)
+	toCtx, cancel := context.WithTimeout(ctx, s.config.EtcdS3Timeout)
 	defer cancel()
 	opts := minio.PutObjectOptions{
 		ContentType: "application/zip",
@@ -115,7 +112,7 @@ func (s *s3) upload(ctx context.Context, snapshot string) error {
 
 // download downloads the given snapshot from the configured S3
 // compatible backend.
-func (s *s3) download(ctx context.Context) error {
+func (s *S3) download(ctx context.Context) error {
 	var remotePath string
 	if s.config.EtcdS3Folder != "" {
 		remotePath = filepath.Join(s.config.EtcdS3Folder, s.config.ClusterResetRestorePath)
@@ -124,7 +121,7 @@ func (s *s3) download(ctx context.Context) error {
 	}
 
 	logrus.Debugf("retrieving snapshot: %s", remotePath)
-	toCtx, cancel := context.WithTimeout(ctx, defaultS3OpTimeout)
+	toCtx, cancel := context.WithTimeout(ctx, s.config.EtcdS3Timeout)
 	defer cancel()
 
 	r, err := s.client.GetObject(toCtx, s.config.EtcdS3BucketName, remotePath, minio.GetObjectOptions{})
@@ -161,7 +158,7 @@ func (s *s3) download(ctx context.Context) error {
 
 // snapshotPrefix returns the prefix used in the
 // naming of the snapshots.
-func (s *s3) snapshotPrefix() string {
+func (s *S3) snapshotPrefix() string {
 	nodeName := os.Getenv("NODE_NAME")
 	fullSnapshotPrefix := s.config.EtcdSnapshotName + "-" + nodeName
 	var prefix string
@@ -175,10 +172,10 @@ func (s *s3) snapshotPrefix() string {
 
 // snapshotRetention deletes the given snapshot from the configured S3
 // compatible backend.
-func (s *s3) snapshotRetention(ctx context.Context) error {
+func (s *S3) snapshotRetention(ctx context.Context) error {
 	var snapshotFiles []minio.ObjectInfo
 
-	toCtx, cancel := context.WithTimeout(ctx, defaultS3OpTimeout)
+	toCtx, cancel := context.WithTimeout(ctx, s.config.EtcdS3Timeout)
 	defer cancel()
 
 	loo := minio.ListObjectsOptions{