Skip to content

K
k3s
Commit ff8193b9 authored by Brad Davidson's avatar Brad Davidson Committed by Brad Davidson
Browse files
Options

Add server token hash to CR and S3

This required pulling the token hash stuff out of the cluster package, into util. Signed-off-by: 's avatarBrad Davidson <brad.davidson@rancher.com> (cherry picked from commit d8851629) Signed-off-by: 's avatarBrad Davidson <brad.davidson@rancher.com>
parent e3ea7b74
Hide whitespace changes
Inline Side-by-side
Showing with 112 additions and 58 deletions
docs/adrs/etcd-snapshot-cr.md
View file @ ff8193b9
......@@ -45,10 +45,13 @@ it into a neutral project for use by both projects.
3. The new Custom Resource will be cluster-scoped, as etcd and its snapshots are a cluster-level resource.
4. Snapshot metadata will also be written alongside snapshot files created on disk and/or uploaded to S3. The metadata
files will have the same basename as their corresponding snapshot file.
5. Downstream consumers of etcd snapshot lists will migrate to watching Custom Resource types, instead of the ConfigMap.
6. K3s will observe a three minor version transition period, where both the new Custom Resources, and the existing
5. A hash of the server token will be stored as an annotation on the Custom Resource, and stored as metadata on snapshots uploaded to S3.
This hash should be compared to a current etcd snapshot's token hash to determine if the server token must be rolled back as part of the
snapshot restore process.
6. Downstream consumers of etcd snapshot lists will migrate to watching Custom Resource types, instead of the ConfigMap.
7. K3s will observe a three minor version transition period, where both the new Custom Resources, and the existing
ConfigMap, will both be used.
7. During the transition period, older snapshot metadata may be removed from the ConfigMap while those snapshots still
8. During the transition period, older snapshot metadata may be removed from the ConfigMap while those snapshots still
exist and are referenced by new Custom Resources, if the ConfigMap exceeds a preset size or key count limit.
## Consequences
......
pkg/cluster/bootstrap.go
View file @ ff8193b9
......@@ -19,6 +19,7 @@ import (
"github.com/k3s-io/k3s/pkg/clientaccess"
"github.com/k3s-io/k3s/pkg/daemons/config"
"github.com/k3s-io/k3s/pkg/etcd"
"github.com/k3s-io/k3s/pkg/util"
"github.com/k3s-io/k3s/pkg/version"
"github.com/k3s-io/kine/pkg/client"
"github.com/k3s-io/kine/pkg/endpoint"
......@@ -248,7 +249,7 @@ func (c *Cluster) ReconcileBootstrapData(ctx context.Context, buf io.ReadSeeker,
if c.managedDB != nil && !isHTTP {
token := c.config.Token
if token == "" {
tokenFromFile, err := readTokenFromFile(c.config.Runtime.ServerToken, c.config.Runtime.ServerCA, c.config.DataDir)
tokenFromFile, err := util.ReadTokenFromFile(c.config.Runtime.ServerToken, c.config.Runtime.ServerCA, c.config.DataDir)
if err != nil {
return err
}
......@@ -260,7 +261,7 @@ func (c *Cluster) ReconcileBootstrapData(ctx context.Context, buf io.ReadSeeker,
token = tokenFromFile
}
normalizedToken, err := normalizeToken(token)
normalizedToken, err := util.NormalizeToken(token)
if err != nil {
return err
}
......
pkg/cluster/encrypt.go
View file @ ff8193b9
......@@ -5,9 +5,7 @@ import (
"crypto/cipher"
"crypto/rand"
"crypto/sha1"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"fmt"
"io"
"strings"
......@@ -19,14 +17,7 @@ import (
// storageKey returns the etcd key for storing bootstrap data for a given passphrase.
// The key is derived from the sha256 hash of the passphrase.
func storageKey(passphrase string) string {
return "/bootstrap/" + keyHash(passphrase)
}
// keyHash returns the first 12 characters of the sha256 sum of the passphrase.
func keyHash(passphrase string) string {
d := sha256.New()
d.Write([]byte(passphrase))
return hex.EncodeToString(d.Sum(nil)[:])[:12]
return "/bootstrap/" + util.ShortHash(passphrase, 12)
}
// encrypt encrypts a byte slice using aes+gcm with a pbkdf2 key derived from the passphrase and a random salt.
......
pkg/cluster/storage.go
View file @ ff8193b9
......@@ -4,13 +4,11 @@ import (
"bytes"
"context"
"errors"
"os"
"path/filepath"
"time"
"github.com/k3s-io/k3s/pkg/bootstrap"
"github.com/k3s-io/k3s/pkg/clientaccess"
"github.com/k3s-io/k3s/pkg/daemons/config"
"github.com/k3s-io/k3s/pkg/util"
"github.com/k3s-io/kine/pkg/client"
"github.com/sirupsen/logrus"
"go.etcd.io/etcd/api/v3/v3rpc/rpctypes"
......@@ -23,12 +21,12 @@ const maxBootstrapWaitAttempts = 5
func RotateBootstrapToken(ctx context.Context, config *config.Control, oldToken string) error {
token, err := readTokenFromFile(config.Runtime.ServerToken, config.Runtime.ServerCA, config.DataDir)
token, err := util.ReadTokenFromFile(config.Runtime.ServerToken, config.Runtime.ServerCA, config.DataDir)
if err != nil {
return err
}
normalizedToken, err := normalizeToken(token)
normalizedToken, err := util.NormalizeToken(token)
if err != nil {
return err
}
......@@ -52,7 +50,7 @@ func RotateBootstrapToken(ctx context.Context, config *config.Control, oldToken
return err
}
normalizedOldToken, err := normalizeToken(oldToken)
normalizedOldToken, err := util.NormalizeToken(oldToken)
if err != nil {
return err
}
......@@ -76,13 +74,13 @@ func Save(ctx context.Context, config *config.Control, override bool) error {
}
token := config.Token
if token == "" {
tokenFromFile, err := readTokenFromFile(config.Runtime.ServerToken, config.Runtime.ServerCA, config.DataDir)
tokenFromFile, err := util.ReadTokenFromFile(config.Runtime.ServerToken, config.Runtime.ServerCA, config.DataDir)
if err != nil {
return err
}
token = tokenFromFile
}
normalizedToken, err := normalizeToken(token)
normalizedToken, err := util.NormalizeToken(token)
if err != nil {
return err
}
......@@ -165,7 +163,7 @@ func (c *Cluster) storageBootstrap(ctx context.Context) error {
token := c.config.Token
if token == "" {
tokenFromFile, err := readTokenFromFile(c.config.Runtime.ServerToken, c.config.Runtime.ServerCA, c.config.DataDir)
tokenFromFile, err := util.ReadTokenFromFile(c.config.Runtime.ServerToken, c.config.Runtime.ServerCA, c.config.DataDir)
if err != nil {
return err
}
......@@ -181,7 +179,7 @@ func (c *Cluster) storageBootstrap(ctx context.Context) error {
}
token = tokenFromFile
}
normalizedToken, err := normalizeToken(token)
normalizedToken, err := util.NormalizeToken(token)
if err != nil {
return err
}
......@@ -288,39 +286,6 @@ func getBootstrapKeyFromStorage(ctx context.Context, storageClient client.Client
return nil, false, errors.New("bootstrap data already found and encrypted with different token")
}
// readTokenFromFile will attempt to get the token from <data-dir>/token if it the file not found
// in case of fresh installation it will try to use the runtime serverToken saved in memory
// after stripping it from any additional information like the username or cahash, if the file
// found then it will still strip the token from any additional info
func readTokenFromFile(serverToken, certs, dataDir string) (string, error) {
tokenFile := filepath.Join(dataDir, "token")
b, err := os.ReadFile(tokenFile)
if err != nil {
if os.IsNotExist(err) {
token, err := clientaccess.FormatToken(serverToken, certs)
if err != nil {
return token, err
}
return token, nil
}
return "", err
}
// strip the token from any new line if its read from file
return string(bytes.TrimRight(b, "\n")), nil
}
// normalizeToken will normalize the token read from file or passed as a cli flag
func normalizeToken(token string) (string, error) {
_, password, ok := clientaccess.ParseUsernamePassword(token)
if !ok {
return password, errors.New("failed to normalize server token; must be in format K10<CA-HASH>::<USERNAME>:<PASSWORD> or <PASSWORD>")
}
return password, nil
}
// migrateTokens will list all keys that has prefix /bootstrap and will check for key that is
// hashed with empty string and keys that is hashed with old token format before normalizing
// then migrate those and resave only with the normalized token
......
pkg/etcd/s3.go
View file @ ff8193b9
......@@ -20,6 +20,7 @@ import (
"time"
"github.com/k3s-io/k3s/pkg/daemons/config"
"github.com/k3s-io/k3s/pkg/util"
"github.com/k3s-io/k3s/pkg/version"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
......@@ -31,6 +32,7 @@ import (
var (
clusterIDKey = textproto.CanonicalMIMEHeaderKey(version.Program + "-cluster-id")
tokenHashKey = textproto.CanonicalMIMEHeaderKey(version.Program + "-token-hash")
nodeNameKey = textproto.CanonicalMIMEHeaderKey(version.Program + "-node-name")
)
......@@ -39,6 +41,7 @@ type S3 struct {
config *config.Control
client *minio.Client
clusterID string
tokenHash string
nodeName string
}
......@@ -109,10 +112,16 @@ func NewS3(ctx context.Context, config *config.Control) (*S3, error) {
clusterID = string(ns.UID)
}
tokenHash, err := util.GetTokenHash(config)
if err != nil {
return nil, errors.Wrap(err, "failed to get server token hash for etcd snapshot")
}
return &S3{
config: config,
client: c,
clusterID: clusterID,
tokenHash: tokenHash,
nodeName: os.Getenv("NODE_NAME"),
}, nil
}
......@@ -154,6 +163,7 @@ func (s *S3) upload(ctx context.Context, snapshot string, extraMetadata *v1.Conf
} else {
sf.Status = successfulSnapshotStatus
sf.Size = uploadInfo.Size
sf.tokenHash = s.tokenHash
}
if _, err := s.uploadSnapshotMetadata(ctx, metadataKey, metadata); err != nil {
logrus.Warnf("Failed to upload snapshot metadata to S3: %v", err)
......@@ -170,6 +180,7 @@ func (s *S3) uploadSnapshot(ctx context.Context, key, path string) (info minio.U
UserMetadata: map[string]string{
clusterIDKey: s.clusterID,
nodeNameKey: s.nodeName,
tokenHashKey: s.tokenHash,
},
}
if strings.HasSuffix(key, compressedExtension) {
......@@ -392,6 +403,7 @@ func (s *S3) listSnapshots(ctx context.Context) (map[string]snapshotFile, error)
Status: successfulSnapshotStatus,
Compressed: compressed,
nodeSource: obj.UserMetadata[nodeNameKey],
tokenHash: obj.UserMetadata[tokenHashKey],
}
sfKey := generateSnapshotConfigMapKey(sf)
snapshots[sfKey] = sf
......
pkg/etcd/snapshot.go
View file @ ff8193b9
......@@ -57,6 +57,7 @@ var (
labelStorageNode = "etcd." + version.Program + ".cattle.io/snapshot-storage-node"
annotationLocalReconciled = "etcd." + version.Program + ".cattle.io/local-snapshots-timestamp"
annotationS3Reconciled = "etcd." + version.Program + ".cattle.io/s3-snapshots-timestamp"
annotationTokenHash = "etcd." + version.Program + ".cattle.io/snapshot-token-hash"
// snapshotDataBackoff will retry at increasing steps for up to ~30 seconds.
// If the ConfigMap update fails, the list won't be reconciled again until next time
......@@ -252,6 +253,11 @@ func (e *ETCD) Snapshot(ctx context.Context) error {
return errors.Wrap(err, "failed to get config for etcd snapshot")
}
tokenHash, err := util.GetTokenHash(e.config)
if err != nil {
return errors.Wrap(err, "failed to get server token hash for etcd snapshot")
}
nodeName := os.Getenv("NODE_NAME")
now := time.Now().Round(time.Second)
snapshotName := fmt.Sprintf("%s-%s-%d", e.config.EtcdSnapshotName, nodeName, now.Unix())
......@@ -314,6 +320,7 @@ func (e *ETCD) Snapshot(ctx context.Context) error {
Size: f.Size(),
Compressed: e.config.EtcdSnapshotCompress,
metadataSource: extraMetadata,
tokenHash: tokenHash,
}
if err := saveSnapshotMetadata(snapshotPath, extraMetadata); err != nil {
......@@ -412,6 +419,7 @@ type snapshotFile struct {
// to populate other fields before serialization to the legacy configmap.
metadataSource *v1.ConfigMap `json:"-"`
nodeSource string `json:"-"`
tokenHash string `json:"-"`
}
// listLocalSnapshots provides a list of the currently stored
......@@ -1016,6 +1024,10 @@ func (sf *snapshotFile) fromETCDSnapshotFile(esf *apisv1.ETCDSnapshotFile) {
}
}
if tokenHash := esf.Annotations[annotationTokenHash]; tokenHash != "" {
sf.tokenHash = tokenHash
}
if esf.Spec.S3 == nil {
sf.NodeName = esf.Spec.NodeName
} else {
......@@ -1080,6 +1092,14 @@ func (sf *snapshotFile) toETCDSnapshotFile(esf *apisv1.ETCDSnapshotFile) {
esf.ObjectMeta.Labels = map[string]string{}
}
if esf.ObjectMeta.Annotations == nil {
esf.ObjectMeta.Annotations = map[string]string{}
}
if sf.tokenHash != "" {
esf.ObjectMeta.Annotations[annotationTokenHash] = sf.tokenHash
}
if sf.S3 == nil {
esf.ObjectMeta.Labels[labelStorageNode] = esf.Spec.NodeName
} else {
......
pkg/util/token.go
View file @ ff8193b9
package util
import (
"bytes"
cryptorand "crypto/rand"
"crypto/sha256"
"encoding/hex"
"os"
"path/filepath"
"github.com/k3s-io/k3s/pkg/clientaccess"
"github.com/k3s-io/k3s/pkg/daemons/config"
"github.com/pkg/errors"
)
func Random(size int) (string, error) {
......@@ -13,3 +21,57 @@ func Random(size int) (string, error) {
}
return hex.EncodeToString(token), err
}
// ReadTokenFromFile will attempt to get the token from <data-dir>/token if it the file not found
// in case of fresh installation it will try to use the runtime serverToken saved in memory
// after stripping it from any additional information like the username or cahash, if the file
// found then it will still strip the token from any additional info
func ReadTokenFromFile(serverToken, certs, dataDir string) (string, error) {
tokenFile := filepath.Join(dataDir, "token")
b, err := os.ReadFile(tokenFile)
if err != nil {
if os.IsNotExist(err) {
token, err := clientaccess.FormatToken(serverToken, certs)
if err != nil {
return token, err
}
return token, nil
}
return "", err
}
// strip the token from any new line if its read from file
return string(bytes.TrimRight(b, "\n")), nil
}
// NormalizeToken will normalize the token read from file or passed as a cli flag
func NormalizeToken(token string) (string, error) {
_, password, ok := clientaccess.ParseUsernamePassword(token)
if !ok {
return password, errors.New("failed to normalize server token; must be in format K10<CA-HASH>::<USERNAME>:<PASSWORD> or <PASSWORD>")
}
return password, nil
}
func GetTokenHash(config *config.Control) (string, error) {
token := config.Token
if token == "" {
tokenFromFile, err := ReadTokenFromFile(config.Runtime.ServerToken, config.Runtime.ServerCA, config.DataDir)
if err != nil {
return "", err
}
token = tokenFromFile
}
normalizedToken, err := NormalizeToken(token)
if err != nil {
return "", err
}
return ShortHash(normalizedToken, 12), nil
}
func ShortHash(s string, i int) string {
digest := sha256.Sum256([]byte(s))
return hex.EncodeToString(digest[:])[:i]
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment