Galal hussein etcd backup restore (#2154)

Makefile

@@ -10,13 +10,10 @@ TARGETS := $(shell ls scripts | grep -v \\.sh)
 $(TARGETS): .dapper
 	./.dapper $@
 
-trash: .dapper
-	./.dapper -m bind trash
-
-trash-keep: .dapper
-	./.dapper -m bind trash -k
-
-deps: trash
+.PHONY: deps
+deps:
+	go mod vendor
+	go mod tidy
 
 release:
 	./scripts/release.sh
@@ -24,3 +21,11 @@ release:
 .DEFAULT_GOAL := ci
 
 .PHONY: $(TARGETS)
+
+.PHONY: generate
+generate: build/data 
+	./scripts/download
+	go generate
+
+build/data:
+	mkdir -p $@

go.mod

@@ -103,6 +103,7 @@ require (
 	github.com/rancher/spur v0.0.0-20200617165101-8702c8e4ce7a
 	github.com/rancher/wrangler v0.6.1
 	github.com/rancher/wrangler-api v0.6.0
+	github.com/robfig/cron/v3 v3.0.1
 	github.com/rootless-containers/rootlesskit v0.10.0
 	github.com/sirupsen/logrus v1.6.0
 	github.com/spf13/pflag v1.0.5

go.sum

@@ -712,6 +712,8 @@ github.com/rancher/wrangler-api v0.6.0/go.mod h1:RbuDkPNHhxcXuwAbLVvEAhH+UPAh+MI
 github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M=
 github.com/robfig/cron v1.1.0 h1:jk4/Hud3TTdcrJgUOBgsqrZBarcxl6ADIjSC2iniwLY=
 github.com/robfig/cron v1.1.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k=
+github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
+github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=

pkg/agent/netpol/netpol_windows.go

@@ -2,6 +2,7 @@ package netpol
 
 import (
 	"context"
+
 	daemonconfig "github.com/rancher/k3s/pkg/daemons/config"
 )
 

pkg/cli/cmds/server.go

@@ -11,6 +11,9 @@ import (
 
 const (
 	DisableItems = "coredns, servicelb, traefik, local-storage, metrics-server"
+
+	defaultSnapshotRentention    = 5
+	defaultSnapshotIntervalHours = 12
 )
 
 type Server struct {
@@ -56,8 +59,13 @@ type Server struct {
 	DisableKubeProxy         bool
 	ClusterInit              bool
 	ClusterReset             bool
+	ClusterResetRestorePath  string
 	EncryptSecrets           bool
 	StartupHooks             []func(context.Context, config.Control) error
+	EtcdDisableSnapshots     bool
+	EtcdSnapshotDir          string
+	EtcdSnapshotCron         string
+	EtcdSnapshotRetention    int
 }
 
 var ServerConfig Server
@@ -210,6 +218,28 @@ func NewServerCommand(action func(*cli.Context) error) *cli.Command {
 				Destination: &ServerConfig.DatastoreKeyFile,
 				EnvVars:     []string{version.ProgramUpper + "_DATASTORE_KEYFILE"},
 			},
+			&cli.BoolFlag{
+				Name:        "etcd-disable-snapshots",
+				Usage:       "(db) Disable automatic etcd snapshots",
+				Destination: &ServerConfig.EtcdDisableSnapshots,
+			},
+			&cli.StringFlag{
+				Name:        "etcd-snapshot-schedule-cron",
+				Usage:       "(db) Snapshot interval time in cron spec. eg. every 5 hours '* */5 * * *'",
+				Destination: &ServerConfig.EtcdSnapshotCron,
+				Value:       "0 */12 * * *",
+			},
+			&cli.IntFlag{
+				Name:        "etcd-snapshot-retention",
+				Usage:       "(db) Number of snapshots to retain",
+				Destination: &ServerConfig.EtcdSnapshotRetention,
+				Value:       defaultSnapshotRentention,
+			},
+			&cli.StringFlag{
+				Name:        "etcd-snapshot-dir",
+				Usage:       "(db) Directory to save db snapshots. (Default location: ${data-dir}/db/snapshots)",
+				Destination: &ServerConfig.EtcdSnapshotDir,
+			},
 			&cli.StringFlag{
 				Name:        "default-local-storage-path",
 				Usage:       "(storage) Default local storage path for local provisioner storage class",
@@ -294,6 +324,11 @@ func NewServerCommand(action func(*cli.Context) error) *cli.Command {
 				EnvVars:     []string{version.ProgramUpper + "_CLUSTER_RESET"},
 				Destination: &ServerConfig.ClusterReset,
 			},
+			&cli.StringFlag{
+				Name:        "cluster-reset-restore-path",
+				Usage:       "(db) Path to snapshot file to be restored",
+				Destination: &ServerConfig.ClusterResetRestorePath,
+			},
 			&cli.BoolFlag{
 				Name:        "secrets-encryption",
 				Usage:       "(experimental) Enable Secret encryption at rest",

pkg/cli/server/server.go

@@ -107,8 +107,18 @@ func run(app *cli.Context, cfg *cmds.Server) error {
 	serverConfig.ControlConfig.DisableNPC = cfg.DisableNPC
 	serverConfig.ControlConfig.DisableKubeProxy = cfg.DisableKubeProxy
 	serverConfig.ControlConfig.ClusterInit = cfg.ClusterInit
-	serverConfig.ControlConfig.ClusterReset = cfg.ClusterReset
 	serverConfig.ControlConfig.EncryptSecrets = cfg.EncryptSecrets
+	serverConfig.ControlConfig.EtcdSnapshotCron = cfg.EtcdSnapshotCron
+	serverConfig.ControlConfig.EtcdSnapshotDir = cfg.EtcdSnapshotDir
+	serverConfig.ControlConfig.EtcdSnapshotRetention = cfg.EtcdSnapshotRetention
+	serverConfig.ControlConfig.EtcdDisableSnapshots = cfg.EtcdDisableSnapshots
+
+	if cfg.ClusterResetRestorePath != "" && !cfg.ClusterReset {
+		return errors.New("Invalid flag use. --cluster-reset required with --cluster-reset-restore-path")
+	}
+
+	serverConfig.ControlConfig.ClusterReset = cfg.ClusterReset
+	serverConfig.ControlConfig.ClusterResetRestorePath = cfg.ClusterResetRestorePath
 
 	if serverConfig.ControlConfig.SupervisorPort == 0 {
 		serverConfig.ControlConfig.SupervisorPort = serverConfig.ControlConfig.HTTPSPort

pkg/cluster/etcd.go

@@ -8,5 +8,5 @@ import (
 )
 
 func init() {
-	managed.RegisterDriver(&etcd.ETCD{})
+	managed.RegisterDriver(etcd.NewETCD())
 }

pkg/cluster/managed/drivers.go

@@ -20,6 +20,7 @@ type Driver interface {
 	Reset(ctx context.Context, clientAccessInfo *clientaccess.Info) error
 	Start(ctx context.Context, clientAccessInfo *clientaccess.Info) error
 	Test(ctx context.Context, clientAccessInfo *clientaccess.Info) error
+	Restore(ctx context.Context) error
 	EndpointName() string
 }
 

pkg/daemons/config/types.go

@@ -126,9 +126,14 @@ type Control struct {
 	DisableKubeProxy         bool
 	ClusterInit              bool
 	ClusterReset             bool
+	ClusterResetRestorePath  string
 	EncryptSecrets           bool
 	TLSMinVersion            uint16
 	TLSCipherSuites          []uint16
+	EtcdDisableSnapshots     bool
+	EtcdSnapshotDir          string
+	EtcdSnapshotCron         string
+	EtcdSnapshotRetention    int
 
 	BindAddress string
 	SANs        []string

pkg/etcd/etcd.go

@@ -11,6 +11,8 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
+	"sort"
+	"strconv"
 	"strings"
 	"time"
 
@@ -21,8 +23,10 @@ import (
 	"github.com/rancher/k3s/pkg/clientaccess"
 	"github.com/rancher/k3s/pkg/daemons/config"
 	"github.com/rancher/k3s/pkg/daemons/executor"
+	"github.com/robfig/cron/v3"
 	"github.com/sirupsen/logrus"
 	etcd "go.etcd.io/etcd/clientv3"
+	"go.etcd.io/etcd/clientv3/snapshot"
 	"go.etcd.io/etcd/etcdserver/etcdserverpb"
 	utilnet "k8s.io/apimachinery/pkg/util/net"
 )
@@ -33,20 +37,39 @@ type ETCD struct {
 	name    string
 	runtime *config.ControlRuntime
 	address string
+	cron    *cron.Cron
 }
 
+// NewETCD creates a new value of type
+// ETCD with an initialized cron value.
+func NewETCD() *ETCD {
+	return &ETCD{
+		cron: cron.New(),
+	}
+}
+
+const (
+	snapshotPrefix = "etcd-snapshot-"
+	endpoint       = "https://127.0.0.1:2379"
+
+	testTimeout = time.Second * 10
+)
+
+// Members contains a slice that holds all
+// members of the cluster.
 type Members struct {
 	Members []*etcdserverpb.Member `json:"members"`
 }
 
+// EndpointName returns the name of the endpoint.
 func (e *ETCD) EndpointName() string {
 	return "etcd"
 }
 
 func (e *ETCD) Test(ctx context.Context, clientAccessInfo *clientaccess.Info) error {
-	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+	ctx, cancel := context.WithTimeout(ctx, testTimeout)
 	defer cancel()
-	status, err := e.client.Status(ctx, "https://127.0.0.1:2379")
+	status, err := e.client.Status(ctx, endpoint)
 	if err != nil {
 		return err
 	}
@@ -61,7 +84,7 @@ func (e *ETCD) Test(ctx context.Context, clientAccessInfo *clientaccess.Info) er
 		return err
 	}
 
-	var cluster []string
+	var memberNameUrls []string
 	for _, member := range members.Members {
 		for _, peerURL := range member.PeerURLs {
 			if peerURL == e.peerURL() && e.name == member.Name {
@@ -69,10 +92,10 @@ func (e *ETCD) Test(ctx context.Context, clientAccessInfo *clientaccess.Info) er
 			}
 		}
 		if len(member.PeerURLs) > 0 {
-			cluster = append(cluster, fmt.Sprintf("%s=%s", member.Name, member.PeerURLs[0]))
+			memberNameUrls = append(memberNameUrls, member.Name+"="+member.PeerURLs[0])
 		}
 	}
-	msg := fmt.Sprintf("This server is a not a member of the etcd cluster "+"found %v and expecting to contain %s=%s", cluster, e.name, e.address)
+	msg := fmt.Sprintf("This server is a not a member of the etcd cluster. Found %v, expect: %s=%s", memberNameUrls, e.name, e.address)
 	logrus.Error(msg)
 	return fmt.Errorf(msg)
 }
@@ -117,6 +140,16 @@ func (e *ETCD) Reset(ctx context.Context, clientAccessInfo *clientaccess.Info) e
 			}
 		}
 	}()
+	if e.config.ClusterResetRestorePath != "" {
+		info, err := os.Stat(e.config.ClusterResetRestorePath)
+		if os.IsNotExist(err) {
+			return fmt.Errorf("etcd: snapshot path does not exist: %s", e.config.ClusterResetRestorePath)
+		}
+		if info.IsDir() {
+			return fmt.Errorf("etcd: snapshot path is directory: %s", e.config.ClusterResetRestorePath)
+		}
+		return e.Restore(ctx)
+	}
 	return e.newCluster(ctx, true)
 }
 
@@ -131,6 +164,11 @@ func (e *ETCD) Start(ctx context.Context, clientAccessInfo *clientaccess.Info) e
 		return nil
 	}
 
+	if !e.config.EtcdDisableSnapshots {
+		e.setSnapshotFunction(ctx)
+		e.cron.Start()
+	}
+
 	if existingCluster {
 		opt, err := executor.CurrentETCDOptions()
 		if err != nil {
@@ -229,7 +267,7 @@ func (e *ETCD) Register(ctx context.Context, config *config.Control, l net.Liste
 	}
 	e.address = address
 
-	e.config.Datastore.Endpoint = "https://127.0.0.1:2379"
+	e.config.Datastore.Endpoint = endpoint
 	e.config.Datastore.Config.CAFile = e.runtime.ETCDServerCA
 	e.config.Datastore.Config.CertFile = e.runtime.ClientETCDCert
 	e.config.Datastore.Config.KeyFile = e.runtime.ClientETCDKey
@@ -317,7 +355,7 @@ func newClient(ctx context.Context, runtime *config.ControlRuntime) (*etcd.Clien
 
 	cfg := etcd.Config{
 		Context:   ctx,
-		Endpoints: []string{"https://127.0.0.1:2379"},
+		Endpoints: []string{endpoint},
 		TLS:       tlsConfig,
 	}
 
@@ -376,7 +414,7 @@ func (e *ETCD) cluster(ctx context.Context, forceNew bool, options executor.Init
 		InitialOptions:      options,
 		ForceNewCluster:     forceNew,
 		ListenClientURLs:    fmt.Sprintf(e.clientURL() + ",https://127.0.0.1:2379"),
-		ListenMetricsURLs:   fmt.Sprintf("http://127.0.0.1:2381"),
+		ListenMetricsURLs:   "http://127.0.0.1:2381",
 		ListenPeerURLs:      e.peerURL(),
 		AdvertiseClientURLs: e.clientURL(),
 		DataDir:             dataDir(e.config),
@@ -480,3 +518,130 @@ func (e *ETCD) clientURLs(ctx context.Context, clientAccessInfo *clientaccess.In
 	}
 	return clientURLs, memberList, nil
 }
+
+func snapshotDir(config *config.Control) (string, error) {
+	if config.EtcdSnapshotDir == "" {
+		// we have to create the snapshot dir if we are using
+		// the default snapshot dir if it doesn't exist
+		defaultSnapshotDir := filepath.Join(config.DataDir, "db", "snapshots")
+		s, err := os.Stat(defaultSnapshotDir)
+		if err != nil {
+			if os.IsNotExist(err) {
+				if err := os.MkdirAll(defaultSnapshotDir, 0700); err != nil {
+					return "", err
+				}
+				return defaultSnapshotDir, nil
+			}
+			return "", err
+		}
+		if s.IsDir() {
+			return defaultSnapshotDir, nil
+		}
+	}
+	return config.EtcdSnapshotDir, nil
+}
+
+func (e *ETCD) snapshot(ctx context.Context) {
+	snapshotTime := time.Now()
+	logrus.Infof("Snapshot retention check")
+	snapshotDir, err := snapshotDir(e.config)
+	if err != nil {
+		logrus.Errorf("failed to get the snapshot dir: %v", err)
+		return
+	}
+	logrus.Infof("Taking etcd snapshot at %s", snapshotTime.String())
+	sManager := snapshot.NewV3(nil)
+	tlsConfig, err := toTLSConfig(e.runtime)
+	if err != nil {
+		logrus.Errorf("failed to get tls config for etcd: %v", err)
+		return
+	}
+	etcdConfig := etcd.Config{
+		Context:   ctx,
+		Endpoints: []string{endpoint},
+		TLS:       tlsConfig,
+	}
+	snapshotPath := filepath.Join(snapshotDir, snapshotPrefix+strconv.Itoa(int(snapshotTime.Unix())))
+
+	if err := sManager.Save(ctx, etcdConfig, snapshotPath); err != nil {
+		logrus.Errorf("failed to save snapshot %s: %v", snapshotPath, err)
+		return
+	}
+	if err := snapshotRetention(e.config.EtcdSnapshotRetention, snapshotDir); err != nil {
+		logrus.Errorf("failed to apply snapshot retention: %v", err)
+		return
+	}
+}
+
+// snapshot performs an ETCD snapshot at the given interval and
+// saves the file to either the default snapshot directory or
+// the user provided directory.
+func (e *ETCD) setSnapshotFunction(ctx context.Context) {
+	e.cron.AddFunc(e.config.EtcdSnapshotCron, func() { e.snapshot(ctx) })
+}
+
+// Restore performs a restore of the ETCD datastore from
+// the given snapshot path. This operation exists upon
+// completion.
+func (e *ETCD) Restore(ctx context.Context) error {
+	// check the old etcd data dir
+	oldDataDir := dataDir(e.config) + "-old"
+	if s, err := os.Stat(oldDataDir); err == nil && s.IsDir() {
+		logrus.Infof("etcd already restored from a snapshot. Restart without --snapshot-restore-path flag. Backup and delete ${datadir}/server/db on each peer etcd server and rejoin the nodes")
+		os.Exit(0)
+	} else if os.IsNotExist(err) {
+		if e.config.ClusterResetRestorePath == "" {
+			return errors.New("no etcd restore path was specified")
+		}
+		// make sure snapshot exists before restoration
+		if _, err := os.Stat(e.config.ClusterResetRestorePath); err != nil {
+			return err
+		}
+		// move the data directory to a temp path
+		if err := os.Rename(dataDir(e.config), oldDataDir); err != nil {
+			return err
+		}
+		sManager := snapshot.NewV3(nil)
+		if err := sManager.Restore(snapshot.RestoreConfig{
+			SnapshotPath:   e.config.ClusterResetRestorePath,
+			Name:           e.name,
+			OutputDataDir:  dataDir(e.config),
+			OutputWALDir:   walDir(e.config),
+			PeerURLs:       []string{e.peerURL()},
+			InitialCluster: e.name + "=" + e.peerURL(),
+		}); err != nil {
+			return err
+		}
+	} else {
+		return err
+	}
+	if err := e.setName(); err != nil {
+		return err
+	}
+
+	return e.newCluster(ctx, true)
+}
+
+// snapshotRetention iterates through the snapshots and removes the oldest
+// leaving the desired number of snapshots.
+func snapshotRetention(retention int, snapshotDir string) error {
+	var snapshotFiles []os.FileInfo
+	if err := filepath.Walk(snapshotDir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if strings.HasPrefix(info.Name(), snapshotPrefix) {
+			snapshotFiles = append(snapshotFiles, info)
+		}
+		return nil
+	}); err != nil {
+		return err
+	}
+	if len(snapshotFiles) <= retention {
+		return nil
+	}
+	sort.Slice(snapshotFiles, func(i, j int) bool {
+		return snapshotFiles[i].Name() < snapshotFiles[j].Name()
+	})
+	return os.Remove(filepath.Join(snapshotDir, snapshotFiles[0].Name()))
+}

vendor/github.com/robfig/cron/v3/.gitignore

+# Compiled Object files, Static and Dynamic libs (Shared Objects)
+*.o
+*.a
+*.so
+
+# Folders
+_obj
+_test
+
+# Architecture specific extensions/prefixes
+*.[568vq]
+[568vq].out
+
+*.cgo1.go
+*.cgo2.c
+_cgo_defun.c
+_cgo_gotypes.go
+_cgo_export.*
+
+_testmain.go
+
+*.exe

vendor/github.com/robfig/cron/v3/.travis.yml

+language: go

vendor/github.com/robfig/cron/v3/LICENSE

+Copyright (C) 2012 Rob Figueiredo
+All Rights Reserved.
+
+MIT LICENSE
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

vendor/github.com/robfig/cron/v3/README.md

+[![GoDoc](http://godoc.org/github.com/robfig/cron?status.png)](http://godoc.org/github.com/robfig/cron)
+[![Build Status](https://travis-ci.org/robfig/cron.svg?branch=master)](https://travis-ci.org/robfig/cron)
+
+# cron
+
+Cron V3 has been released!
+
+To download the specific tagged release, run:
+
+	go get github.com/robfig/cron/v3@v3.0.0
+
+Import it in your program as:
+
+	import "github.com/robfig/cron/v3"
+
+It requires Go 1.11 or later due to usage of Go Modules.
+
+Refer to the documentation here:
+http://godoc.org/github.com/robfig/cron
+
+The rest of this document describes the the advances in v3 and a list of
+breaking changes for users that wish to upgrade from an earlier version.
+
+## Upgrading to v3 (June 2019)
+
+cron v3 is a major upgrade to the library that addresses all outstanding bugs,
+feature requests, and rough edges. It is based on a merge of master which
+contains various fixes to issues found over the years and the v2 branch which
+contains some backwards-incompatible features like the ability to remove cron
+jobs. In addition, v3 adds support for Go Modules, cleans up rough edges like
+the timezone support, and fixes a number of bugs.
+
+New features:
+
+- Support for Go modules. Callers must now import this library as
+  `github.com/robfig/cron/v3`, instead of `gopkg.in/...`
+
+- Fixed bugs:
+  - 0f01e6b parser: fix combining of Dow and Dom (#70)
+  - dbf3220 adjust times when rolling the clock forward to handle non-existent midnight (#157)
+  - eeecf15 spec_test.go: ensure an error is returned on 0 increment (#144)
+  - 70971dc cron.Entries(): update request for snapshot to include a reply channel (#97)
+  - 1cba5e6 cron: fix: removing a job causes the next scheduled job to run too late (#206)
+
+- Standard cron spec parsing by default (first field is "minute"), with an easy
+  way to opt into the seconds field (quartz-compatible). Although, note that the
+  year field (optional in Quartz) is not supported.
+
+- Extensible, key/value logging via an interface that complies with
+  the https://github.com/go-logr/logr project.
+
+- The new Chain & JobWrapper types allow you to install "interceptors" to add
+  cross-cutting behavior like the following:
+  - Recover any panics from jobs
+  - Delay a job's execution if the previous run hasn't completed yet
+  - Skip a job's execution if the previous run hasn't completed yet
+  - Log each job's invocations
+  - Notification when jobs are completed
+
+It is backwards incompatible with both v1 and v2. These updates are required:
+
+- The v1 branch accepted an optional seconds field at the beginning of the cron
+  spec. This is non-standard and has led to a lot of confusion. The new default
+  parser conforms to the standard as described by [the Cron wikipedia page].
+
+  UPDATING: To retain the old behavior, construct your Cron with a custom
+  parser:
+
+      // Seconds field, required
+      cron.New(cron.WithSeconds())
+
+      // Seconds field, optional
+      cron.New(
+          cron.WithParser(
+              cron.SecondOptional | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.Descriptor))
+
+- The Cron type now accepts functional options on construction rather than the
+  previous ad-hoc behavior modification mechanisms (setting a field, calling a setter).
+
+  UPDATING: Code that sets Cron.ErrorLogger or calls Cron.SetLocation must be
+  updated to provide those values on construction.
+
+- CRON_TZ is now the recommended way to specify the timezone of a single
+  schedule, which is sanctioned by the specification. The legacy "TZ=" prefix
+  will continue to be supported since it is unambiguous and easy to do so.
+
+  UPDATING: No update is required.
+
+- By default, cron will no longer recover panics in jobs that it runs.
+  Recovering can be surprising (see issue #192) and seems to be at odds with
+  typical behavior of libraries. Relatedly, the `cron.WithPanicLogger` option
+  has been removed to accommodate the more general JobWrapper type.
+
+  UPDATING: To opt into panic recovery and configure the panic logger:
+
+      cron.New(cron.WithChain(
+          cron.Recover(logger),  // or use cron.DefaultLogger
+      ))
+
+- In adding support for https://github.com/go-logr/logr, `cron.WithVerboseLogger` was
+  removed, since it is duplicative with the leveled logging.
+
+  UPDATING: Callers should use `WithLogger` and specify a logger that does not
+  discard `Info` logs. For convenience, one is provided that wraps `*log.Logger`:
+
+      cron.New(
+          cron.WithLogger(cron.VerbosePrintfLogger(logger)))
+
+
+### Background - Cron spec format
+
+There are two cron spec formats in common usage:
+
+- The "standard" cron format, described on [the Cron wikipedia page] and used by
+  the cron Linux system utility.
+
+- The cron format used by [the Quartz Scheduler], commonly used for scheduled
+  jobs in Java software
+
+[the Cron wikipedia page]: https://en.wikipedia.org/wiki/Cron
+[the Quartz Scheduler]: http://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/tutorial-lesson-06.html
+
+The original version of this package included an optional "seconds" field, which
+made it incompatible with both of these formats. Now, the "standard" format is
+the default format accepted, and the Quartz format is opt-in.

vendor/github.com/robfig/cron/v3/chain.go

+package cron
+
+import (
+	"fmt"
+	"runtime"
+	"sync"
+	"time"
+)
+
+// JobWrapper decorates the given Job with some behavior.
+type JobWrapper func(Job) Job
+
+// Chain is a sequence of JobWrappers that decorates submitted jobs with
+// cross-cutting behaviors like logging or synchronization.
+type Chain struct {
+	wrappers []JobWrapper
+}
+
+// NewChain returns a Chain consisting of the given JobWrappers.
+func NewChain(c ...JobWrapper) Chain {
+	return Chain{c}
+}
+
+// Then decorates the given job with all JobWrappers in the chain.
+//
+// This:
+//     NewChain(m1, m2, m3).Then(job)
+// is equivalent to:
+//     m1(m2(m3(job)))
+func (c Chain) Then(j Job) Job {
+	for i := range c.wrappers {
+		j = c.wrappers[len(c.wrappers)-i-1](j)
+	}
+	return j
+}
+
+// Recover panics in wrapped jobs and log them with the provided logger.
+func Recover(logger Logger) JobWrapper {
+	return func(j Job) Job {
+		return FuncJob(func() {
+			defer func() {
+				if r := recover(); r != nil {
+					const size = 64 << 10
+					buf := make([]byte, size)
+					buf = buf[:runtime.Stack(buf, false)]
+					err, ok := r.(error)
+					if !ok {
+						err = fmt.Errorf("%v", r)
+					}
+					logger.Error(err, "panic", "stack", "...\n"+string(buf))
+				}
+			}()
+			j.Run()
+		})
+	}
+}
+
+// DelayIfStillRunning serializes jobs, delaying subsequent runs until the
+// previous one is complete. Jobs running after a delay of more than a minute
+// have the delay logged at Info.
+func DelayIfStillRunning(logger Logger) JobWrapper {
+	return func(j Job) Job {
+		var mu sync.Mutex
+		return FuncJob(func() {
+			start := time.Now()
+			mu.Lock()
+			defer mu.Unlock()
+			if dur := time.Since(start); dur > time.Minute {
+				logger.Info("delay", "duration", dur)
+			}
+			j.Run()
+		})
+	}
+}
+
+// SkipIfStillRunning skips an invocation of the Job if a previous invocation is
+// still running. It logs skips to the given logger at Info level.
+func SkipIfStillRunning(logger Logger) JobWrapper {
+	return func(j Job) Job {
+		var ch = make(chan struct{}, 1)
+		ch <- struct{}{}
+		return FuncJob(func() {
+			select {
+			case v := <-ch:
+				j.Run()
+				ch <- v
+			default:
+				logger.Info("skip")
+			}
+		})
+	}
+}

vendor/github.com/robfig/cron/v3/constantdelay.go

+package cron
+
+import "time"
+
+// ConstantDelaySchedule represents a simple recurring duty cycle, e.g. "Every 5 minutes".
+// It does not support jobs more frequent than once a second.
+type ConstantDelaySchedule struct {
+	Delay time.Duration
+}
+
+// Every returns a crontab Schedule that activates once every duration.
+// Delays of less than a second are not supported (will round up to 1 second).
+// Any fields less than a Second are truncated.
+func Every(duration time.Duration) ConstantDelaySchedule {
+	if duration < time.Second {
+		duration = time.Second
+	}
+	return ConstantDelaySchedule{
+		Delay: duration - time.Duration(duration.Nanoseconds())%time.Second,
+	}
+}
+
+// Next returns the next time this should be run.
+// This rounds so that the next activation time will be on the second.
+func (schedule ConstantDelaySchedule) Next(t time.Time) time.Time {
+	return t.Add(schedule.Delay - time.Duration(t.Nanosecond())*time.Nanosecond)
+}

vendor/github.com/robfig/cron/v3/cron.go

+package cron
+
+import (
+	"context"
+	"sort"
+	"sync"
+	"time"
+)
+
+// Cron keeps track of any number of entries, invoking the associated func as
+// specified by the schedule. It may be started, stopped, and the entries may
+// be inspected while running.
+type Cron struct {
+	entries   []*Entry
+	chain     Chain
+	stop      chan struct{}
+	add       chan *Entry
+	remove    chan EntryID
+	snapshot  chan chan []Entry
+	running   bool
+	logger    Logger
+	runningMu sync.Mutex
+	location  *time.Location
+	parser    ScheduleParser
+	nextID    EntryID
+	jobWaiter sync.WaitGroup
+}
+
+// ScheduleParser is an interface for schedule spec parsers that return a Schedule
+type ScheduleParser interface {
+	Parse(spec string) (Schedule, error)
+}
+
+// Job is an interface for submitted cron jobs.
+type Job interface {
+	Run()
+}
+
+// Schedule describes a job's duty cycle.
+type Schedule interface {
+	// Next returns the next activation time, later than the given time.
+	// Next is invoked initially, and then each time the job is run.
+	Next(time.Time) time.Time
+}
+
+// EntryID identifies an entry within a Cron instance
+type EntryID int
+
+// Entry consists of a schedule and the func to execute on that schedule.
+type Entry struct {
+	// ID is the cron-assigned ID of this entry, which may be used to look up a
+	// snapshot or remove it.
+	ID EntryID
+
+	// Schedule on which this job should be run.
+	Schedule Schedule
+
+	// Next time the job will run, or the zero time if Cron has not been
+	// started or this entry's schedule is unsatisfiable
+	Next time.Time
+
+	// Prev is the last time this job was run, or the zero time if never.
+	Prev time.Time
+
+	// WrappedJob is the thing to run when the Schedule is activated.
+	WrappedJob Job
+
+	// Job is the thing that was submitted to cron.
+	// It is kept around so that user code that needs to get at the job later,
+	// e.g. via Entries() can do so.
+	Job Job
+}
+
+// Valid returns true if this is not the zero entry.
+func (e Entry) Valid() bool { return e.ID != 0 }
+
+// byTime is a wrapper for sorting the entry array by time
+// (with zero time at the end).
+type byTime []*Entry
+
+func (s byTime) Len() int      { return len(s) }
+func (s byTime) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
+func (s byTime) Less(i, j int) bool {
+	// Two zero times should return false.
+	// Otherwise, zero is "greater" than any other time.
+	// (To sort it at the end of the list.)
+	if s[i].Next.IsZero() {
+		return false
+	}
+	if s[j].Next.IsZero() {
+		return true
+	}
+	return s[i].Next.Before(s[j].Next)
+}
+
+// New returns a new Cron job runner, modified by the given options.
+//
+// Available Settings
+//
+//   Time Zone
+//     Description: The time zone in which schedules are interpreted
+//     Default:     time.Local
+//
+//   Parser
+//     Description: Parser converts cron spec strings into cron.Schedules.
+//     Default:     Accepts this spec: https://en.wikipedia.org/wiki/Cron
+//
+//   Chain
+//     Description: Wrap submitted jobs to customize behavior.
+//     Default:     A chain that recovers panics and logs them to stderr.
+//
+// See "cron.With*" to modify the default behavior.
+func New(opts ...Option) *Cron {
+	c := &Cron{
+		entries:   nil,
+		chain:     NewChain(),
+		add:       make(chan *Entry),
+		stop:      make(chan struct{}),
+		snapshot:  make(chan chan []Entry),
+		remove:    make(chan EntryID),
+		running:   false,
+		runningMu: sync.Mutex{},
+		logger:    DefaultLogger,
+		location:  time.Local,
+		parser:    standardParser,
+	}
+	for _, opt := range opts {
+		opt(c)
+	}
+	return c
+}
+
+// FuncJob is a wrapper that turns a func() into a cron.Job
+type FuncJob func()
+
+func (f FuncJob) Run() { f() }
+
+// AddFunc adds a func to the Cron to be run on the given schedule.
+// The spec is parsed using the time zone of this Cron instance as the default.
+// An opaque ID is returned that can be used to later remove it.
+func (c *Cron) AddFunc(spec string, cmd func()) (EntryID, error) {
+	return c.AddJob(spec, FuncJob(cmd))
+}
+
+// AddJob adds a Job to the Cron to be run on the given schedule.
+// The spec is parsed using the time zone of this Cron instance as the default.
+// An opaque ID is returned that can be used to later remove it.
+func (c *Cron) AddJob(spec string, cmd Job) (EntryID, error) {
+	schedule, err := c.parser.Parse(spec)
+	if err != nil {
+		return 0, err
+	}
+	return c.Schedule(schedule, cmd), nil
+}
+
+// Schedule adds a Job to the Cron to be run on the given schedule.
+// The job is wrapped with the configured Chain.
+func (c *Cron) Schedule(schedule Schedule, cmd Job) EntryID {
+	c.runningMu.Lock()
+	defer c.runningMu.Unlock()
+	c.nextID++
+	entry := &Entry{
+		ID:         c.nextID,
+		Schedule:   schedule,
+		WrappedJob: c.chain.Then(cmd),
+		Job:        cmd,
+	}
+	if !c.running {
+		c.entries = append(c.entries, entry)
+	} else {
+		c.add <- entry
+	}
+	return entry.ID
+}
+
+// Entries returns a snapshot of the cron entries.
+func (c *Cron) Entries() []Entry {
+	c.runningMu.Lock()
+	defer c.runningMu.Unlock()
+	if c.running {
+		replyChan := make(chan []Entry, 1)
+		c.snapshot <- replyChan
+		return <-replyChan
+	}
+	return c.entrySnapshot()
+}
+
+// Location gets the time zone location
+func (c *Cron) Location() *time.Location {
+	return c.location
+}
+
+// Entry returns a snapshot of the given entry, or nil if it couldn't be found.
+func (c *Cron) Entry(id EntryID) Entry {
+	for _, entry := range c.Entries() {
+		if id == entry.ID {
+			return entry
+		}
+	}
+	return Entry{}
+}
+
+// Remove an entry from being run in the future.
+func (c *Cron) Remove(id EntryID) {
+	c.runningMu.Lock()
+	defer c.runningMu.Unlock()
+	if c.running {
+		c.remove <- id
+	} else {
+		c.removeEntry(id)
+	}
+}
+
+// Start the cron scheduler in its own goroutine, or no-op if already started.
+func (c *Cron) Start() {
+	c.runningMu.Lock()
+	defer c.runningMu.Unlock()
+	if c.running {
+		return
+	}
+	c.running = true
+	go c.run()
+}
+
+// Run the cron scheduler, or no-op if already running.
+func (c *Cron) Run() {
+	c.runningMu.Lock()
+	if c.running {
+		c.runningMu.Unlock()
+		return
+	}
+	c.running = true
+	c.runningMu.Unlock()
+	c.run()
+}
+
+// run the scheduler.. this is private just due to the need to synchronize
+// access to the 'running' state variable.
+func (c *Cron) run() {
+	c.logger.Info("start")
+
+	// Figure out the next activation times for each entry.
+	now := c.now()
+	for _, entry := range c.entries {
+		entry.Next = entry.Schedule.Next(now)
+		c.logger.Info("schedule", "now", now, "entry", entry.ID, "next", entry.Next)
+	}
+
+	for {
+		// Determine the next entry to run.
+		sort.Sort(byTime(c.entries))
+
+		var timer *time.Timer
+		if len(c.entries) == 0 || c.entries[0].Next.IsZero() {
+			// If there are no entries yet, just sleep - it still handles new entries
+			// and stop requests.
+			timer = time.NewTimer(100000 * time.Hour)
+		} else {
+			timer = time.NewTimer(c.entries[0].Next.Sub(now))
+		}
+
+		for {
+			select {
+			case now = <-timer.C:
+				now = now.In(c.location)
+				c.logger.Info("wake", "now", now)
+
+				// Run every entry whose next time was less than now
+				for _, e := range c.entries {
+					if e.Next.After(now) || e.Next.IsZero() {
+						break
+					}
+					c.startJob(e.WrappedJob)
+					e.Prev = e.Next
+					e.Next = e.Schedule.Next(now)
+					c.logger.Info("run", "now", now, "entry", e.ID, "next", e.Next)
+				}
+
+			case newEntry := <-c.add:
+				timer.Stop()
+				now = c.now()
+				newEntry.Next = newEntry.Schedule.Next(now)
+				c.entries = append(c.entries, newEntry)
+				c.logger.Info("added", "now", now, "entry", newEntry.ID, "next", newEntry.Next)
+
+			case replyChan := <-c.snapshot:
+				replyChan <- c.entrySnapshot()
+				continue
+
+			case <-c.stop:
+				timer.Stop()
+				c.logger.Info("stop")
+				return
+
+			case id := <-c.remove:
+				timer.Stop()
+				now = c.now()
+				c.removeEntry(id)
+				c.logger.Info("removed", "entry", id)
+			}
+
+			break
+		}
+	}
+}
+
+// startJob runs the given job in a new goroutine.
+func (c *Cron) startJob(j Job) {
+	c.jobWaiter.Add(1)
+	go func() {
+		defer c.jobWaiter.Done()
+		j.Run()
+	}()
+}
+
+// now returns current time in c location
+func (c *Cron) now() time.Time {
+	return time.Now().In(c.location)
+}
+
+// Stop stops the cron scheduler if it is running; otherwise it does nothing.
+// A context is returned so the caller can wait for running jobs to complete.
+func (c *Cron) Stop() context.Context {
+	c.runningMu.Lock()
+	defer c.runningMu.Unlock()
+	if c.running {
+		c.stop <- struct{}{}
+		c.running = false
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	go func() {
+		c.jobWaiter.Wait()
+		cancel()
+	}()
+	return ctx
+}
+
+// entrySnapshot returns a copy of the current cron entry list.
+func (c *Cron) entrySnapshot() []Entry {
+	var entries = make([]Entry, len(c.entries))
+	for i, e := range c.entries {
+		entries[i] = *e
+	}
+	return entries
+}
+
+func (c *Cron) removeEntry(id EntryID) {
+	var entries []*Entry
+	for _, e := range c.entries {
+		if e.ID != id {
+			entries = append(entries, e)
+		}
+	}
+	c.entries = entries
+}

vendor/github.com/robfig/cron/v3/doc.go

+/*
+Package cron implements a cron spec parser and job runner.
+
+Installation
+
+To download the specific tagged release, run:
+
+	go get github.com/robfig/cron/v3@v3.0.0
+
+Import it in your program as:
+
+	import "github.com/robfig/cron/v3"
+
+It requires Go 1.11 or later due to usage of Go Modules.
+
+Usage
+
+Callers may register Funcs to be invoked on a given schedule.  Cron will run
+them in their own goroutines.
+
+	c := cron.New()
+	c.AddFunc("30 * * * *", func() { fmt.Println("Every hour on the half hour") })
+	c.AddFunc("30 3-6,20-23 * * *", func() { fmt.Println(".. in the range 3-6am, 8-11pm") })
+	c.AddFunc("CRON_TZ=Asia/Tokyo 30 04 * * *", func() { fmt.Println("Runs at 04:30 Tokyo time every day") })
+	c.AddFunc("@hourly",      func() { fmt.Println("Every hour, starting an hour from now") })
+	c.AddFunc("@every 1h30m", func() { fmt.Println("Every hour thirty, starting an hour thirty from now") })
+	c.Start()
+	..
+	// Funcs are invoked in their own goroutine, asynchronously.
+	...
+	// Funcs may also be added to a running Cron
+	c.AddFunc("@daily", func() { fmt.Println("Every day") })
+	..
+	// Inspect the cron job entries' next and previous run times.
+	inspect(c.Entries())
+	..
+	c.Stop()  // Stop the scheduler (does not stop any jobs already running).
+
+CRON Expression Format
+
+A cron expression represents a set of times, using 5 space-separated fields.
+
+	Field name   | Mandatory? | Allowed values  | Allowed special characters
+	----------   | ---------- | --------------  | --------------------------
+	Minutes      | Yes        | 0-59            | * / , -
+	Hours        | Yes        | 0-23            | * / , -
+	Day of month | Yes        | 1-31            | * / , - ?
+	Month        | Yes        | 1-12 or JAN-DEC | * / , -
+	Day of week  | Yes        | 0-6 or SUN-SAT  | * / , - ?
+
+Month and Day-of-week field values are case insensitive.  "SUN", "Sun", and
+"sun" are equally accepted.
+
+The specific interpretation of the format is based on the Cron Wikipedia page:
+https://en.wikipedia.org/wiki/Cron
+
+Alternative Formats
+
+Alternative Cron expression formats support other fields like seconds. You can
+implement that by creating a custom Parser as follows.
+
+	cron.New(
+		cron.WithParser(
+			cron.NewParser(
+				cron.SecondOptional | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.Descriptor)))
+
+Since adding Seconds is the most common modification to the standard cron spec,
+cron provides a builtin function to do that, which is equivalent to the custom
+parser you saw earlier, except that its seconds field is REQUIRED:
+
+	cron.New(cron.WithSeconds())
+
+That emulates Quartz, the most popular alternative Cron schedule format:
+http://www.quartz-scheduler.org/documentation/quartz-2.x/tutorials/crontrigger.html
+
+Special Characters
+
+Asterisk ( * )
+
+The asterisk indicates that the cron expression will match for all values of the
+field; e.g., using an asterisk in the 5th field (month) would indicate every
+month.
+
+Slash ( / )
+
+Slashes are used to describe increments of ranges. For example 3-59/15 in the
+1st field (minutes) would indicate the 3rd minute of the hour and every 15
+minutes thereafter. The form "*\/..." is equivalent to the form "first-last/...",
+that is, an increment over the largest possible range of the field.  The form
+"N/..." is accepted as meaning "N-MAX/...", that is, starting at N, use the
+increment until the end of that specific range.  It does not wrap around.
+
+Comma ( , )
+
+Commas are used to separate items of a list. For example, using "MON,WED,FRI" in
+the 5th field (day of week) would mean Mondays, Wednesdays and Fridays.
+
+Hyphen ( - )
+
+Hyphens are used to define ranges. For example, 9-17 would indicate every
+hour between 9am and 5pm inclusive.
+
+Question mark ( ? )
+
+Question mark may be used instead of '*' for leaving either day-of-month or
+day-of-week blank.
+
+Predefined schedules
+
+You may use one of several pre-defined schedules in place of a cron expression.
+
+	Entry                  | Description                                | Equivalent To
+	-----                  | -----------                                | -------------
+	@yearly (or @annually) | Run once a year, midnight, Jan. 1st        | 0 0 1 1 *
+	@monthly               | Run once a month, midnight, first of month | 0 0 1 * *
+	@weekly                | Run once a week, midnight between Sat/Sun  | 0 0 * * 0
+	@daily (or @midnight)  | Run once a day, midnight                   | 0 0 * * *
+	@hourly                | Run once an hour, beginning of hour        | 0 * * * *
+
+Intervals
+
+You may also schedule a job to execute at fixed intervals, starting at the time it's added
+or cron is run. This is supported by formatting the cron spec like this:
+
+    @every <duration>
+
+where "duration" is a string accepted by time.ParseDuration
+(http://golang.org/pkg/time/#ParseDuration).
+
+For example, "@every 1h30m10s" would indicate a schedule that activates after
+1 hour, 30 minutes, 10 seconds, and then every interval after that.
+
+Note: The interval does not take the job runtime into account.  For example,
+if a job takes 3 minutes to run, and it is scheduled to run every 5 minutes,
+it will have only 2 minutes of idle time between each run.
+
+Time zones
+
+By default, all interpretation and scheduling is done in the machine's local
+time zone (time.Local). You can specify a different time zone on construction:
+
+      cron.New(
+          cron.WithLocation(time.UTC))
+
+Individual cron schedules may also override the time zone they are to be
+interpreted in by providing an additional space-separated field at the beginning
+of the cron spec, of the form "CRON_TZ=Asia/Tokyo".
+
+For example:
+
+	# Runs at 6am in time.Local
+	cron.New().AddFunc("0 6 * * ?", ...)
+
+	# Runs at 6am in America/New_York
+	nyc, _ := time.LoadLocation("America/New_York")
+	c := cron.New(cron.WithLocation(nyc))
+	c.AddFunc("0 6 * * ?", ...)
+
+	# Runs at 6am in Asia/Tokyo
+	cron.New().AddFunc("CRON_TZ=Asia/Tokyo 0 6 * * ?", ...)
+
+	# Runs at 6am in Asia/Tokyo
+	c := cron.New(cron.WithLocation(nyc))
+	c.SetLocation("America/New_York")
+	c.AddFunc("CRON_TZ=Asia/Tokyo 0 6 * * ?", ...)
+
+The prefix "TZ=(TIME ZONE)" is also supported for legacy compatibility.
+
+Be aware that jobs scheduled during daylight-savings leap-ahead transitions will
+not be run!
+
+Job Wrappers
+
+A Cron runner may be configured with a chain of job wrappers to add
+cross-cutting functionality to all submitted jobs. For example, they may be used
+to achieve the following effects:
+
+  - Recover any panics from jobs (activated by default)
+  - Delay a job's execution if the previous run hasn't completed yet
+  - Skip a job's execution if the previous run hasn't completed yet
+  - Log each job's invocations
+
+Install wrappers for all jobs added to a cron using the `cron.WithChain` option:
+
+	cron.New(cron.WithChain(
+		cron.SkipIfStillRunning(logger),
+	))
+
+Install wrappers for individual jobs by explicitly wrapping them:
+
+	job = cron.NewChain(
+		cron.SkipIfStillRunning(logger),
+	).Then(job)
+
+Thread safety
+
+Since the Cron service runs concurrently with the calling code, some amount of
+care must be taken to ensure proper synchronization.
+
+All cron methods are designed to be correctly synchronized as long as the caller
+ensures that invocations have a clear happens-before ordering between them.
+
+Logging
+
+Cron defines a Logger interface that is a subset of the one defined in
+github.com/go-logr/logr. It has two logging levels (Info and Error), and
+parameters are key/value pairs. This makes it possible for cron logging to plug
+into structured logging systems. An adapter, [Verbose]PrintfLogger, is provided
+to wrap the standard library *log.Logger.
+
+For additional insight into Cron operations, verbose logging may be activated
+which will record job runs, scheduling decisions, and added or removed jobs.
+Activate it with a one-off logger as follows:
+
+	cron.New(
+		cron.WithLogger(
+			cron.VerbosePrintfLogger(log.New(os.Stdout, "cron: ", log.LstdFlags))))
+
+
+Implementation
+
+Cron entries are stored in an array, sorted by their next activation time.  Cron
+sleeps until the next job is due to be run.
+
+Upon waking:
+ - it runs each entry that is active on that second
+ - it calculates the next run times for the jobs that were run
+ - it re-sorts the array of entries by next activation time.
+ - it goes to sleep until the soonest job.
+*/
+package cron

vendor/github.com/robfig/cron/v3/go.mod

+module github.com/robfig/cron/v3
+
+go 1.12

vendor/github.com/robfig/cron/v3/logger.go

+package cron
+
+import (
+	"io/ioutil"
+	"log"
+	"os"
+	"strings"
+	"time"
+)
+
+// DefaultLogger is used by Cron if none is specified.
+var DefaultLogger Logger = PrintfLogger(log.New(os.Stdout, "cron: ", log.LstdFlags))
+
+// DiscardLogger can be used by callers to discard all log messages.
+var DiscardLogger Logger = PrintfLogger(log.New(ioutil.Discard, "", 0))
+
+// Logger is the interface used in this package for logging, so that any backend
+// can be plugged in. It is a subset of the github.com/go-logr/logr interface.
+type Logger interface {
+	// Info logs routine messages about cron's operation.
+	Info(msg string, keysAndValues ...interface{})
+	// Error logs an error condition.
+	Error(err error, msg string, keysAndValues ...interface{})
+}
+
+// PrintfLogger wraps a Printf-based logger (such as the standard library "log")
+// into an implementation of the Logger interface which logs errors only.
+func PrintfLogger(l interface{ Printf(string, ...interface{}) }) Logger {
+	return printfLogger{l, false}
+}
+
+// VerbosePrintfLogger wraps a Printf-based logger (such as the standard library
+// "log") into an implementation of the Logger interface which logs everything.
+func VerbosePrintfLogger(l interface{ Printf(string, ...interface{}) }) Logger {
+	return printfLogger{l, true}
+}
+
+type printfLogger struct {
+	logger  interface{ Printf(string, ...interface{}) }
+	logInfo bool
+}
+
+func (pl printfLogger) Info(msg string, keysAndValues ...interface{}) {
+	if pl.logInfo {
+		keysAndValues = formatTimes(keysAndValues)
+		pl.logger.Printf(
+			formatString(len(keysAndValues)),
+			append([]interface{}{msg}, keysAndValues...)...)
+	}
+}
+
+func (pl printfLogger) Error(err error, msg string, keysAndValues ...interface{}) {
+	keysAndValues = formatTimes(keysAndValues)
+	pl.logger.Printf(
+		formatString(len(keysAndValues)+2),
+		append([]interface{}{msg, "error", err}, keysAndValues...)...)
+}
+
+// formatString returns a logfmt-like format string for the number of
+// key/values.
+func formatString(numKeysAndValues int) string {
+	var sb strings.Builder
+	sb.WriteString("%s")
+	if numKeysAndValues > 0 {
+		sb.WriteString(", ")
+	}
+	for i := 0; i < numKeysAndValues/2; i++ {
+		if i > 0 {
+			sb.WriteString(", ")
+		}
+		sb.WriteString("%v=%v")
+	}
+	return sb.String()
+}
+
+// formatTimes formats any time.Time values as RFC3339.
+func formatTimes(keysAndValues []interface{}) []interface{} {
+	var formattedArgs []interface{}
+	for _, arg := range keysAndValues {
+		if t, ok := arg.(time.Time); ok {
+			arg = t.Format(time.RFC3339)
+		}
+		formattedArgs = append(formattedArgs, arg)
+	}
+	return formattedArgs
+}

vendor/github.com/robfig/cron/v3/option.go

+package cron
+
+import (
+	"time"
+)
+
+// Option represents a modification to the default behavior of a Cron.
+type Option func(*Cron)
+
+// WithLocation overrides the timezone of the cron instance.
+func WithLocation(loc *time.Location) Option {
+	return func(c *Cron) {
+		c.location = loc
+	}
+}
+
+// WithSeconds overrides the parser used for interpreting job schedules to
+// include a seconds field as the first one.
+func WithSeconds() Option {
+	return WithParser(NewParser(
+		Second | Minute | Hour | Dom | Month | Dow | Descriptor,
+	))
+}
+
+// WithParser overrides the parser used for interpreting job schedules.
+func WithParser(p ScheduleParser) Option {
+	return func(c *Cron) {
+		c.parser = p
+	}
+}
+
+// WithChain specifies Job wrappers to apply to all jobs added to this cron.
+// Refer to the Chain* functions in this package for provided wrappers.
+func WithChain(wrappers ...JobWrapper) Option {
+	return func(c *Cron) {
+		c.chain = NewChain(wrappers...)
+	}
+}
+
+// WithLogger uses the provided logger.
+func WithLogger(logger Logger) Option {
+	return func(c *Cron) {
+		c.logger = logger
+	}
+}

vendor/github.com/robfig/cron/v3/spec.go

+package cron
+
+import "time"
+
+// SpecSchedule specifies a duty cycle (to the second granularity), based on a
+// traditional crontab specification. It is computed initially and stored as bit sets.
+type SpecSchedule struct {
+	Second, Minute, Hour, Dom, Month, Dow uint64
+
+	// Override location for this schedule.
+	Location *time.Location
+}
+
+// bounds provides a range of acceptable values (plus a map of name to value).
+type bounds struct {
+	min, max uint
+	names    map[string]uint
+}
+
+// The bounds for each field.
+var (
+	seconds = bounds{0, 59, nil}
+	minutes = bounds{0, 59, nil}
+	hours   = bounds{0, 23, nil}
+	dom     = bounds{1, 31, nil}
+	months  = bounds{1, 12, map[string]uint{
+		"jan": 1,
+		"feb": 2,
+		"mar": 3,
+		"apr": 4,
+		"may": 5,
+		"jun": 6,
+		"jul": 7,
+		"aug": 8,
+		"sep": 9,
+		"oct": 10,
+		"nov": 11,
+		"dec": 12,
+	}}
+	dow = bounds{0, 6, map[string]uint{
+		"sun": 0,
+		"mon": 1,
+		"tue": 2,
+		"wed": 3,
+		"thu": 4,
+		"fri": 5,
+		"sat": 6,
+	}}
+)
+
+const (
+	// Set the top bit if a star was included in the expression.
+	starBit = 1 << 63
+)
+
+// Next returns the next time this schedule is activated, greater than the given
+// time.  If no time can be found to satisfy the schedule, return the zero time.
+func (s *SpecSchedule) Next(t time.Time) time.Time {
+	// General approach
+	//
+	// For Month, Day, Hour, Minute, Second:
+	// Check if the time value matches.  If yes, continue to the next field.
+	// If the field doesn't match the schedule, then increment the field until it matches.
+	// While incrementing the field, a wrap-around brings it back to the beginning
+	// of the field list (since it is necessary to re-verify previous field
+	// values)
+
+	// Convert the given time into the schedule's timezone, if one is specified.
+	// Save the original timezone so we can convert back after we find a time.
+	// Note that schedules without a time zone specified (time.Local) are treated
+	// as local to the time provided.
+	origLocation := t.Location()
+	loc := s.Location
+	if loc == time.Local {
+		loc = t.Location()
+	}
+	if s.Location != time.Local {
+		t = t.In(s.Location)
+	}
+
+	// Start at the earliest possible time (the upcoming second).
+	t = t.Add(1*time.Second - time.Duration(t.Nanosecond())*time.Nanosecond)
+
+	// This flag indicates whether a field has been incremented.
+	added := false
+
+	// If no time is found within five years, return zero.
+	yearLimit := t.Year() + 5
+
+WRAP:
+	if t.Year() > yearLimit {
+		return time.Time{}
+	}
+
+	// Find the first applicable month.
+	// If it's this month, then do nothing.
+	for 1<<uint(t.Month())&s.Month == 0 {
+		// If we have to add a month, reset the other parts to 0.
+		if !added {
+			added = true
+			// Otherwise, set the date at the beginning (since the current time is irrelevant).
+			t = time.Date(t.Year(), t.Month(), 1, 0, 0, 0, 0, loc)
+		}
+		t = t.AddDate(0, 1, 0)
+
+		// Wrapped around.
+		if t.Month() == time.January {
+			goto WRAP
+		}
+	}
+
+	// Now get a day in that month.
+	//
+	// NOTE: This causes issues for daylight savings regimes where midnight does
+	// not exist.  For example: Sao Paulo has DST that transforms midnight on
+	// 11/3 into 1am. Handle that by noticing when the Hour ends up != 0.
+	for !dayMatches(s, t) {
+		if !added {
+			added = true
+			t = time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, loc)
+		}
+		t = t.AddDate(0, 0, 1)
+		// Notice if the hour is no longer midnight due to DST.
+		// Add an hour if it's 23, subtract an hour if it's 1.
+		if t.Hour() != 0 {
+			if t.Hour() > 12 {
+				t = t.Add(time.Duration(24-t.Hour()) * time.Hour)
+			} else {
+				t = t.Add(time.Duration(-t.Hour()) * time.Hour)
+			}
+		}
+
+		if t.Day() == 1 {
+			goto WRAP
+		}
+	}
+
+	for 1<<uint(t.Hour())&s.Hour == 0 {
+		if !added {
+			added = true
+			t = time.Date(t.Year(), t.Month(), t.Day(), t.Hour(), 0, 0, 0, loc)
+		}
+		t = t.Add(1 * time.Hour)
+
+		if t.Hour() == 0 {
+			goto WRAP
+		}
+	}
+
+	for 1<<uint(t.Minute())&s.Minute == 0 {
+		if !added {
+			added = true
+			t = t.Truncate(time.Minute)
+		}
+		t = t.Add(1 * time.Minute)
+
+		if t.Minute() == 0 {
+			goto WRAP
+		}
+	}
+
+	for 1<<uint(t.Second())&s.Second == 0 {
+		if !added {
+			added = true
+			t = t.Truncate(time.Second)
+		}
+		t = t.Add(1 * time.Second)
+
+		if t.Second() == 0 {
+			goto WRAP
+		}
+	}
+
+	return t.In(origLocation)
+}
+
+// dayMatches returns true if the schedule's day-of-week and day-of-month
+// restrictions are satisfied by the given time.
+func dayMatches(s *SpecSchedule, t time.Time) bool {
+	var (
+		domMatch bool = 1<<uint(t.Day())&s.Dom > 0
+		dowMatch bool = 1<<uint(t.Weekday())&s.Dow > 0
+	)
+	if s.Dom&starBit > 0 || s.Dow&starBit > 0 {
+		return domMatch && dowMatch
+	}
+	return domMatch || dowMatch
+}

vendor/go.etcd.io/etcd/clientv3/snapshot/doc.go

+// Copyright 2018 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package snapshot implements utilities around etcd snapshot.
+package snapshot

vendor/go.etcd.io/etcd/clientv3/snapshot/util.go

+// Copyright 2018 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package snapshot
+
+import "encoding/binary"
+
+type revision struct {
+	main int64
+	sub  int64
+}
+
+func bytesToRev(bytes []byte) revision {
+	return revision{
+		main: int64(binary.BigEndian.Uint64(bytes[0:8])),
+		sub:  int64(binary.BigEndian.Uint64(bytes[9:])),
+	}
+}
+
+// initIndex implements ConsistentIndexGetter so the snapshot won't block
+// the new raft instance by waiting for a future raft index.
+type initIndex int
+
+func (i *initIndex) ConsistentIndex() uint64 { return uint64(*i) }

vendor/modules.txt

@@ -812,6 +812,8 @@ github.com/rancher/wrangler-api/pkg/generated/controllers/rbac
 github.com/rancher/wrangler-api/pkg/generated/controllers/rbac/v1
 # github.com/robfig/cron v1.1.0
 github.com/robfig/cron
+# github.com/robfig/cron/v3 v3.0.1
+github.com/robfig/cron/v3
 # github.com/rootless-containers/rootlesskit v0.10.0
 github.com/rootless-containers/rootlesskit/pkg/api
 github.com/rootless-containers/rootlesskit/pkg/api/client
@@ -919,6 +921,7 @@ go.etcd.io/etcd/clientv3/balancer/picker
 go.etcd.io/etcd/clientv3/balancer/resolver/endpoint
 go.etcd.io/etcd/clientv3/concurrency
 go.etcd.io/etcd/clientv3/credentials
+go.etcd.io/etcd/clientv3/snapshot
 go.etcd.io/etcd/embed
 go.etcd.io/etcd/etcdserver
 go.etcd.io/etcd/etcdserver/api