kubeadm: Initial refactor of the Bootstrap Tokens. Add the new API objects,…

cmd/kubeadm/app/apis/kubeadm/bootstraptokenhelpers.go

+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kubeadm
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+	"time"
+
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	bootstrapapi "k8s.io/client-go/tools/bootstrap/token/api"
+	bootstraputil "k8s.io/client-go/tools/bootstrap/token/util"
+)
+
+// ToSecret converts the given BootstrapToken object to its Secret representation that
+// may be submitted to the API Server in order to be stored.
+func (bt *BootstrapToken) ToSecret() *v1.Secret {
+	return &v1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      bootstraputil.BootstrapTokenSecretName(bt.Token.ID),
+			Namespace: metav1.NamespaceSystem,
+		},
+		Type: v1.SecretType(bootstrapapi.SecretTypeBootstrapToken),
+		Data: encodeTokenSecretData(bt, time.Now()),
+	}
+}
+
+// encodeTokenSecretData takes the token discovery object and an optional duration and returns the .Data for the Secret
+// now is passed in order to be able to used in unit testing
+func encodeTokenSecretData(token *BootstrapToken, now time.Time) map[string][]byte {
+	data := map[string][]byte{
+		bootstrapapi.BootstrapTokenIDKey:     []byte(token.Token.ID),
+		bootstrapapi.BootstrapTokenSecretKey: []byte(token.Token.Secret),
+	}
+
+	if len(token.Description) > 0 {
+		data[bootstrapapi.BootstrapTokenDescriptionKey] = []byte(token.Description)
+	}
+
+	// If for some strange reason both token.TTL and token.Expires would be set
+	// (they are mutually exlusive in validation so this shouldn't be the case),
+	// token.Expires has higher priority, as can be seen in the logic here.
+	if token.Expires != nil {
+		// Format the expiration date accordingly
+		// TODO: This maybe should be a helper function in bootstraputil?
+		expirationString := token.Expires.Time.Format(time.RFC3339)
+		data[bootstrapapi.BootstrapTokenExpirationKey] = []byte(expirationString)
+
+	} else if token.TTL != nil && token.TTL.Duration > 0 {
+		// Only if .Expires is unset, TTL might have an effect
+		// Get the current time, add the specified duration, and format it accordingly
+		expirationString := now.Add(token.TTL.Duration).Format(time.RFC3339)
+		data[bootstrapapi.BootstrapTokenExpirationKey] = []byte(expirationString)
+	}
+
+	for _, usage := range token.Usages {
+		data[bootstrapapi.BootstrapTokenUsagePrefix+usage] = []byte("true")
+	}
+
+	if len(token.Groups) > 0 {
+		data[bootstrapapi.BootstrapTokenExtraGroupsKey] = []byte(strings.Join(token.Groups, ","))
+	}
+	return data
+}
+
+// BootstrapTokenFromSecret returns a BootstrapToken object from the given Secret
+func BootstrapTokenFromSecret(secret *v1.Secret) (*BootstrapToken, error) {
+	// Get the Token ID field from the Secret data
+	tokenID := getSecretString(secret, bootstrapapi.BootstrapTokenIDKey)
+	if len(tokenID) == 0 {
+		return nil, fmt.Errorf("Bootstrap Token Secret has no token-id data: %s\n", secret.Name)
+	}
+
+	// Enforce the right naming convention
+	if secret.Name != bootstraputil.BootstrapTokenSecretName(tokenID) {
+		return nil, fmt.Errorf("bootstrap token name is not of the form '%s(token-id)'. Actual: %q. Expected: %q\n",
+			bootstrapapi.BootstrapTokenSecretPrefix, secret.Name, bootstraputil.BootstrapTokenSecretName(tokenID))
+	}
+
+	tokenSecret := getSecretString(secret, bootstrapapi.BootstrapTokenSecretKey)
+	if len(tokenSecret) == 0 {
+		return nil, fmt.Errorf("Bootstrap Token Secret has no token-secret data: %s\n", secret.Name)
+	}
+
+	// Create the BootstrapTokenString object based on the ID and Secret
+	bts, err := NewBootstrapTokenStringFromIDAndSecret(tokenID, tokenSecret)
+	if err != nil {
+		return nil, fmt.Errorf("Bootstrap Token Secret is invalid and couldn't be parsed: %v\n", err)
+	}
+
+	// Get the description (if any) from the Secret
+	description := getSecretString(secret, bootstrapapi.BootstrapTokenDescriptionKey)
+
+	// Expiration time is optional, if not specified this implies the token
+	// never expires.
+	secretExpiration := getSecretString(secret, bootstrapapi.BootstrapTokenExpirationKey)
+	var expires *metav1.Time
+	if len(secretExpiration) > 0 {
+		expTime, err := time.Parse(time.RFC3339, secretExpiration)
+		if err != nil {
+			return nil, fmt.Errorf("can't parse expiration time of bootstrap token %q: %v", secret.Name, err)
+		}
+		expires = &metav1.Time{expTime}
+	}
+
+	// Build an usages string slice from the Secret data
+	var usages []string
+	for k, v := range secret.Data {
+		// Skip all fields that don't include this prefix
+		if !strings.HasPrefix(k, bootstrapapi.BootstrapTokenUsagePrefix) {
+			continue
+		}
+		// Skip those that don't have this usage set to true
+		if string(v) != "true" {
+			continue
+		}
+		usages = append(usages, strings.TrimPrefix(k, bootstrapapi.BootstrapTokenUsagePrefix))
+	}
+	// Only sort the slice if defined
+	if usages != nil {
+		sort.Strings(usages)
+	}
+
+	// Get the extra groups information from the Secret
+	// It's done this way to make .Groups be nil in case there is no items, rather than an
+	// empty slice or an empty slice with a "" string only
+	var groups []string
+	groupsString := getSecretString(secret, bootstrapapi.BootstrapTokenExtraGroupsKey)
+	g := strings.Split(groupsString, ",")
+	if len(g) > 0 && len(g[0]) > 0 {
+		groups = g
+	}
+
+	return &BootstrapToken{
+		Token:       bts,
+		Description: description,
+		Expires:     expires,
+		Usages:      usages,
+		Groups:      groups,
+	}, nil
+}
+
+// getSecretString returns the string value for the given key in the specified Secret
+func getSecretString(secret *v1.Secret, key string) string {
+	if secret.Data == nil {
+		return ""
+	}
+	if val, ok := secret.Data[key]; ok {
+		return string(val)
+	}
+	return ""
+}

cmd/kubeadm/app/apis/kubeadm/bootstraptokenstring.go

+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Note: This file should be kept in sync with the similar one for the external API
+// TODO: The BootstrapTokenString object should move out to either k8s.io/client-go or k8s.io/api in the future
+// (probably as part of Bootstrap Tokens going GA). It should not be staged under the kubeadm API as it is now.
+package kubeadm
+
+import (
+	"fmt"
+	"strings"
+
+	bootstrapapi "k8s.io/client-go/tools/bootstrap/token/api"
+	bootstraputil "k8s.io/client-go/tools/bootstrap/token/util"
+)
+
+// BootstrapTokenString is a token of the format abcdef.abcdef0123456789 that is used
+// for both validation of the authenticy of the API server from a joining node's point
+// of view and as an authentication method for the node in the bootstrap phase of
+// "kubeadm join". This token is and should be short-lived
+type BootstrapTokenString struct {
+	ID     string
+	Secret string
+}
+
+// MarshalJSON implements the json.Marshaler interface.
+func (bts BootstrapTokenString) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`"%s"`, bts.String())), nil
+}
+
+// UnmarshalJSON implements the json.Unmarshaller interface.
+func (bts *BootstrapTokenString) UnmarshalJSON(b []byte) error {
+	// If the token is represented as "", just return quickly without an error
+	if len(b) == 0 {
+		return nil
+	}
+
+	// Remove unnecessary " characters coming from the JSON parser
+	token := strings.Replace(string(b), `"`, ``, -1)
+	// Convert the string Token to a BootstrapTokenString object
+	newbts, err := NewBootstrapTokenString(token)
+	if err != nil {
+		return err
+	}
+	bts.ID = newbts.ID
+	bts.Secret = newbts.Secret
+	return nil
+}
+
+// String returns the string representation of the BootstrapTokenString
+func (bts BootstrapTokenString) String() string {
+	if len(bts.ID) > 0 && len(bts.Secret) > 0 {
+		return bootstraputil.TokenFromIDAndSecret(bts.ID, bts.Secret)
+	}
+	return ""
+}
+
+// NewBootstrapTokenString converts the given Bootstrap Token as a string
+// to the BootstrapTokenString object used for serialization/deserialization
+// and internal usage. It also automatically validates that the given token
+// is of the right format
+func NewBootstrapTokenString(token string) (*BootstrapTokenString, error) {
+	substrs := bootstraputil.BootstrapTokenRegexp.FindStringSubmatch(token)
+	// TODO: Add a constant for the 3 value here, and explain better why it's needed (other than because how the regexp parsin works)
+	if len(substrs) != 3 {
+		return nil, fmt.Errorf("the bootstrap token %q was not of the form %q", token, bootstrapapi.BootstrapTokenPattern)
+	}
+
+	return &BootstrapTokenString{ID: substrs[1], Secret: substrs[2]}, nil
+}
+
+// NewBootstrapTokenStringFromIDAndSecret is a wrapper around NewBootstrapTokenString
+// that allows the caller to specify the ID and Secret separately
+func NewBootstrapTokenStringFromIDAndSecret(id, secret string) (*BootstrapTokenString, error) {
+	return NewBootstrapTokenString(bootstraputil.TokenFromIDAndSecret(id, secret))
+}

cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go

@@ -43,10 +43,17 @@ func Funcs(codecs runtimeserializer.CodecFactory) []interface{} {
 			obj.CertificatesDir = "foo"
 			obj.APIServerCertSANs = []string{"foo"}
 
-			obj.Token = "foo"
-			obj.TokenTTL = &metav1.Duration{Duration: 1 * time.Hour}
-			obj.TokenUsages = []string{"foo"}
-			obj.TokenGroups = []string{"foo"}
+			obj.BootstrapTokens = []kubeadm.BootstrapToken{
+				{
+					Token: &kubeadm.BootstrapTokenString{
+						ID:     "abcdef",
+						Secret: "abcdef0123456789",
+					},
+					TTL:    &metav1.Duration{Duration: 1 * time.Hour},
+					Usages: []string{"foo"},
+					Groups: []string{"foo"},
+				},
+			}
 			obj.ImageRepository = "foo"
 			obj.CIImageRepository = ""
 			obj.UnifiedControlPlaneImage = "foo"

cmd/kubeadm/app/apis/kubeadm/types.go

@@ -48,15 +48,9 @@ type MasterConfiguration struct {
 	// NodeRegistration holds fields that relate to registering the new master node to the cluster
 	NodeRegistration NodeRegistrationOptions
 
-	// Token is used for establishing bidirectional trust between nodes and masters.
-	// Used for joining nodes in the cluster.
-	Token string
-	// TokenTTL defines the ttl for Token. Defaults to 24h.
-	TokenTTL *metav1.Duration
-	// TokenUsages describes the ways in which this token can be used.
-	TokenUsages []string
-	// Extra groups that this token will authenticate as when used for authentication
-	TokenGroups []string
+	// BootstrapTokens is respected at `kubeadm init` time and describes a set of Bootstrap Tokens to create.
+	// This information IS NOT uploaded to the kubeadm cluster configmap, due to its sensitive nature
+	BootstrapTokens []BootstrapToken
 
 	// APIServerExtraArgs is a set of extra flags to pass to the API Server or override
 	// default ones in form of <flagname>=<value>.
@@ -153,18 +147,6 @@ type NodeRegistrationOptions struct {
 	ExtraArgs map[string]string
 }
 
-// TokenDiscovery contains elements needed for token discovery.
-type TokenDiscovery struct {
-	// ID is the first part of a bootstrap token. Considered public information.
-	// It is used when referring to a token without leaking the secret part.
-	ID string
-	// Secret is the second part of a bootstrap token. Should only be shared
-	// with trusted parties.
-	Secret string
-	// TODO: Seems unused. Remove?
-	// Addresses []string
-}
-
 // Networking contains elements describing cluster's networking configuration.
 type Networking struct {
 	// ServiceSubnet is the subnet used by k8s services. Defaults to "10.96.0.0/12".
@@ -175,6 +157,30 @@ type Networking struct {
 	DNSDomain string
 }
 
+// BootstrapToken describes one bootstrap token, stored as a Secret in the cluster
+// TODO: The BootstrapToken object should move out to either k8s.io/client-go or k8s.io/api in the future
+// (probably as part of Bootstrap Tokens going GA). It should not be staged under the kubeadm API as it is now.
+type BootstrapToken struct {
+	// Token is used for establishing bidirectional trust between nodes and masters.
+	// Used for joining nodes in the cluster.
+	Token *BootstrapTokenString
+	// Description sets a human-friendly message why this token exists and what it's used
+	// for, so other administrators can know its purpose.
+	Description string
+	// TTL defines the time to live for this token. Defaults to 24h.
+	// Expires and TTL are mutually exclusive.
+	TTL *metav1.Duration
+	// Expires specifies the timestamp when this token expires. Defaults to being set
+	// dynamically at runtime based on the TTL. Expires and TTL are mutually exclusive.
+	Expires *metav1.Time
+	// Usages describes the ways in which this token can be used. Can by default be used
+	// for establishing bidirectional trust, but that can be changed here.
+	Usages []string
+	// Groups specifies the extra groups that this token will authenticate as when/if
+	// used for authentication
+	Groups []string
+}
+
 // Etcd contains elements describing Etcd configuration.
 type Etcd struct {
 

cmd/kubeadm/app/apis/kubeadm/v1alpha1/conversion.go

@@ -17,6 +17,7 @@ limitations under the License.
 package v1alpha1
 
 import (
+	"fmt"
 	"reflect"
 	"strings"
 
@@ -54,6 +55,9 @@ func Convert_v1alpha1_MasterConfiguration_To_kubeadm_MasterConfiguration(in *Mas
 	UpgradeCloudProvider(in, out)
 	UpgradeAuthorizationModes(in, out)
 	UpgradeNodeRegistrationOptionsForMaster(in, out)
+	if err := UpgradeBootstrapTokens(in, out); err != nil {
+		return err
+	}
 	// We don't support migrating information from the .PrivilegedPods field which was removed in v1alpha2
 	// We don't support migrating information from the .ImagePullPolicy field which was removed in v1alpha2
 
@@ -100,25 +104,6 @@ func Convert_v1alpha1_Etcd_To_kubeadm_Etcd(in *Etcd, out *kubeadm.Etcd, s conver
 	return nil
 }
 
-// no-op, as we don't support converting from newer API to old alpha API
-func Convert_kubeadm_Etcd_To_v1alpha1_Etcd(in *kubeadm.Etcd, out *Etcd, s conversion.Scope) error {
-
-	if in.External != nil {
-		out.Endpoints = in.External.Endpoints
-		out.CAFile = in.External.CAFile
-		out.CertFile = in.External.CertFile
-		out.KeyFile = in.External.KeyFile
-	} else {
-		out.Image = in.Local.Image
-		out.DataDir = in.Local.DataDir
-		out.ExtraArgs = in.Local.ExtraArgs
-		out.ServerCertSANs = in.Local.ServerCertSANs
-		out.PeerCertSANs = in.Local.PeerCertSANs
-	}
-
-	return nil
-}
-
 // UpgradeCloudProvider handles the removal of .CloudProvider as smoothly as possible
 func UpgradeCloudProvider(in *MasterConfiguration, out *kubeadm.MasterConfiguration) {
 	if len(in.CloudProvider) != 0 {
@@ -160,8 +145,30 @@ func UpgradeNodeRegistrationOptionsForMaster(in *MasterConfiguration, out *kubea
 	}
 }
 
+func UpgradeBootstrapTokens(in *MasterConfiguration, out *kubeadm.MasterConfiguration) error {
+	if len(in.Token) == 0 {
+		return nil
+	}
+
+	bts, err := kubeadm.NewBootstrapTokenString(in.Token)
+	if err != nil {
+		return fmt.Errorf("can't parse .Token, and hence can't convert v1alpha1 API to a newer version: %v", err)
+	}
+
+	out.BootstrapTokens = []kubeadm.BootstrapToken{
+		{
+			Token:  bts,
+			TTL:    in.TokenTTL,
+			Usages: in.TokenUsages,
+			Groups: in.TokenGroups,
+		},
+	}
+	return nil
+}
+
 // Downgrades below
 
+// This downgrade path IS NOT SUPPORTED. This is just here for roundtripping purposes at the moment.
 func Convert_kubeadm_MasterConfiguration_To_v1alpha1_MasterConfiguration(in *kubeadm.MasterConfiguration, out *MasterConfiguration, s conversion.Scope) error {
 	if err := autoConvert_kubeadm_MasterConfiguration_To_v1alpha1_MasterConfiguration(in, out, s); err != nil {
 		return err
@@ -172,9 +179,17 @@ func Convert_kubeadm_MasterConfiguration_To_v1alpha1_MasterConfiguration(in *kub
 	out.CRISocket = in.NodeRegistration.CRISocket
 	out.NoTaintMaster = in.NodeRegistration.Taints != nil && len(in.NodeRegistration.Taints) == 0
 
+	if len(in.BootstrapTokens) > 0 {
+		out.Token = in.BootstrapTokens[0].Token.String()
+		out.TokenTTL = in.BootstrapTokens[0].TTL
+		out.TokenUsages = in.BootstrapTokens[0].Usages
+		out.TokenGroups = in.BootstrapTokens[0].Groups
+	}
+
 	return nil
 }
 
+// This downgrade path IS NOT SUPPORTED. This is just here for roundtripping purposes at the moment.
 func Convert_kubeadm_NodeConfiguration_To_v1alpha1_NodeConfiguration(in *kubeadm.NodeConfiguration, out *NodeConfiguration, s conversion.Scope) error {
 	if err := autoConvert_kubeadm_NodeConfiguration_To_v1alpha1_NodeConfiguration(in, out, s); err != nil {
 		return err
@@ -183,6 +198,27 @@ func Convert_kubeadm_NodeConfiguration_To_v1alpha1_NodeConfiguration(in *kubeadm
 	// Converting from newer API version to an older API version isn't supported. This is here only for the roundtrip tests meanwhile.
 	out.NodeName = in.NodeRegistration.Name
 	out.CRISocket = in.NodeRegistration.CRISocket
+	return nil
+}
+
+// This downgrade path IS NOT SUPPORTED. This is just here for roundtripping purposes at the moment.
+func Convert_kubeadm_Etcd_To_v1alpha1_Etcd(in *kubeadm.Etcd, out *Etcd, s conversion.Scope) error {
+	if err := autoConvert_kubeadm_Etcd_To_v1alpha1_Etcd(in, out, s); err != nil {
+		return err
+	}
+
+	if in.External != nil {
+		out.Endpoints = in.External.Endpoints
+		out.CAFile = in.External.CAFile
+		out.CertFile = in.External.CertFile
+		out.KeyFile = in.External.KeyFile
+	} else {
+		out.Image = in.Local.Image
+		out.DataDir = in.Local.DataDir
+		out.ExtraArgs = in.Local.ExtraArgs
+		out.ServerCertSANs = in.Local.ServerCertSANs
+		out.PeerCertSANs = in.Local.PeerCertSANs
+	}
 
 	return nil
 }

cmd/kubeadm/app/apis/kubeadm/v1alpha2/bootstraptokenstring.go

+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Note: This file should be kept in sync with the similar one for the internal API
+// TODO: The BootstrapTokenString object should move out to either k8s.io/client-go or k8s.io/api in the future
+// (probably as part of Bootstrap Tokens going GA). It should not be staged under the kubeadm API as it is now.
+package v1alpha2
+
+import (
+	"fmt"
+	"strings"
+
+	bootstrapapi "k8s.io/client-go/tools/bootstrap/token/api"
+	bootstraputil "k8s.io/client-go/tools/bootstrap/token/util"
+)
+
+// BootstrapTokenString is a token of the format abcdef.abcdef0123456789 that is used
+// for both validation of the authenticy of the API server from a joining node's point
+// of view and as an authentication method for the node in the bootstrap phase of
+// "kubeadm join". This token is and should be short-lived
+type BootstrapTokenString struct {
+	ID     string
+	Secret string
+}
+
+// MarshalJSON implements the json.Marshaler interface.
+func (bts BootstrapTokenString) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`"%s"`, bts.String())), nil
+}
+
+// UnmarshalJSON implements the json.Unmarshaller interface.
+func (bts *BootstrapTokenString) UnmarshalJSON(b []byte) error {
+	// If the token is represented as "", just return quickly without an error
+	if len(b) == 0 {
+		return nil
+	}
+
+	// Remove unnecessary " characters coming from the JSON parser
+	token := strings.Replace(string(b), `"`, ``, -1)
+	// Convert the string Token to a BootstrapTokenString object
+	newbts, err := NewBootstrapTokenString(token)
+	if err != nil {
+		return err
+	}
+	bts.ID = newbts.ID
+	bts.Secret = newbts.Secret
+	return nil
+}
+
+// String returns the string representation of the BootstrapTokenString
+func (bts BootstrapTokenString) String() string {
+	if len(bts.ID) > 0 && len(bts.Secret) > 0 {
+		return bootstraputil.TokenFromIDAndSecret(bts.ID, bts.Secret)
+	}
+	return ""
+}
+
+// NewBootstrapTokenString converts the given Bootstrap Token as a string
+// to the BootstrapTokenString object used for serialization/deserialization
+// and internal usage. It also automatically validates that the given token
+// is of the right format
+func NewBootstrapTokenString(token string) (*BootstrapTokenString, error) {
+	substrs := bootstraputil.BootstrapTokenRegexp.FindStringSubmatch(token)
+	// TODO: Add a constant for the 3 value here, and explain better why it's needed (other than because how the regexp parsin works)
+	if len(substrs) != 3 {
+		return nil, fmt.Errorf("the bootstrap token %q was not of the form %q", token, bootstrapapi.BootstrapTokenPattern)
+	}
+
+	return &BootstrapTokenString{ID: substrs[1], Secret: substrs[2]}, nil
+}
+
+// NewBootstrapTokenStringFromIDAndSecret is a wrapper around NewBootstrapTokenString
+// that allows the caller to specify the ID and Secret separately
+func NewBootstrapTokenStringFromIDAndSecret(id, secret string) (*BootstrapTokenString, error) {
+	return NewBootstrapTokenString(bootstraputil.TokenFromIDAndSecret(id, secret))
+}

cmd/kubeadm/app/apis/kubeadm/v1alpha2/defaults.go

@@ -97,20 +97,6 @@ func SetDefaults_MasterConfiguration(obj *MasterConfiguration) {
 		obj.CertificatesDir = DefaultCertificatesDir
 	}
 
-	if obj.TokenTTL == nil {
-		obj.TokenTTL = &metav1.Duration{
-			Duration: constants.DefaultTokenDuration,
-		}
-	}
-
-	if len(obj.TokenUsages) == 0 {
-		obj.TokenUsages = constants.DefaultTokenUsages
-	}
-
-	if len(obj.TokenGroups) == 0 {
-		obj.TokenGroups = constants.DefaultTokenGroups
-	}
-
 	if obj.ImageRepository == "" {
 		obj.ImageRepository = DefaultImageRepository
 	}
@@ -120,6 +106,7 @@ func SetDefaults_MasterConfiguration(obj *MasterConfiguration) {
 	}
 
 	SetDefaults_NodeRegistrationOptions(&obj.NodeRegistration)
+	SetDefaults_BootstrapTokens(obj)
 	SetDefaults_KubeletConfiguration(obj)
 	SetDefaults_Etcd(obj)
 	SetDefaults_ProxyConfiguration(obj)
@@ -248,3 +235,35 @@ func SetDefaults_AuditPolicyConfiguration(obj *MasterConfiguration) {
 		obj.AuditPolicyConfiguration.LogMaxAge = &DefaultAuditPolicyLogMaxAge
 	}
 }
+
+// SetDefaults_BootstrapTokens sets the defaults for the .BootstrapTokens field
+// If the slice is empty, it's defaulted with one token. Otherwise it just loops
+// through the slice and sets the defaults for the omitempty fields that are TTL,
+// Usages and Groups. Token is NOT defaulted with a random one in the API defaulting
+// layer, but set to a random value later at runtime if not set before.
+func SetDefaults_BootstrapTokens(obj *MasterConfiguration) {
+
+	if obj.BootstrapTokens == nil || len(obj.BootstrapTokens) == 0 {
+		obj.BootstrapTokens = []BootstrapToken{{}}
+	}
+
+	for _, bt := range obj.BootstrapTokens {
+		SetDefaults_BootstrapToken(&bt)
+	}
+}
+
+// SetDefaults_BootstrapToken sets the defaults for an individual Bootstrap Token
+func SetDefaults_BootstrapToken(bt *BootstrapToken) {
+	if bt.TTL == nil {
+		bt.TTL = &metav1.Duration{
+			Duration: constants.DefaultTokenDuration,
+		}
+	}
+	if len(bt.Usages) == 0 {
+		bt.Usages = constants.DefaultTokenUsages
+	}
+
+	if len(bt.Groups) == 0 {
+		bt.Groups = constants.DefaultTokenGroups
+	}
+}

cmd/kubeadm/app/apis/kubeadm/v1alpha2/types.go

@@ -47,15 +47,9 @@ type MasterConfiguration struct {
 	// KubernetesVersion is the target version of the control plane.
 	KubernetesVersion string `json:"kubernetesVersion"`
 
-	// Token is used for establishing bidirectional trust between nodes and masters.
-	// Used for joining nodes in the cluster.
-	Token string `json:"token"`
-	// TokenTTL defines the ttl for Token. Defaults to 24h.
-	TokenTTL *metav1.Duration `json:"tokenTTL,omitempty"`
-	// TokenUsages describes the ways in which this token can be used.
-	TokenUsages []string `json:"tokenUsages,omitempty"`
-	// Extra groups that this token will authenticate as when used for authentication
-	TokenGroups []string `json:"tokenGroups,omitempty"`
+	// BootstrapTokens is respected at `kubeadm init` time and describes a set of Bootstrap Tokens to create.
+	// This information IS NOT uploaded to the kubeadm cluster configmap, due to its sensitive nature
+	BootstrapTokens []BootstrapToken `json:"bootstrapTokens,omitempty"`
 
 	// APIServerExtraArgs is a set of extra flags to pass to the API Server or override
 	// default ones in form of <flagname>=<value>.
@@ -145,18 +139,6 @@ type NodeRegistrationOptions struct {
 	ExtraArgs map[string]string `json:"kubeletExtraArgs,omitempty"`
 }
 
-// TokenDiscovery contains elements needed for token discovery.
-type TokenDiscovery struct {
-	// ID is the first part of a bootstrap token. Considered public information.
-	// It is used when referring to a token without leaking the secret part.
-	ID string `json:"id"`
-	// Secret is the second part of a bootstrap token. Should only be shared
-	// with trusted parties.
-	Secret string `json:"secret"`
-	// TODO: Seems unused. Remove?
-	// Addresses []string `json:"addresses"`
-}
-
 // Networking contains elements describing cluster's networking configuration
 type Networking struct {
 	// ServiceSubnet is the subnet used by k8s services. Defaults to "10.96.0.0/12".
@@ -167,6 +149,28 @@ type Networking struct {
 	DNSDomain string `json:"dnsDomain"`
 }
 
+// BootstrapToken describes one bootstrap token, stored as a Secret in the cluster
+type BootstrapToken struct {
+	// Token is used for establishing bidirectional trust between nodes and masters.
+	// Used for joining nodes in the cluster.
+	Token *BootstrapTokenString `json:"token"`
+	// Description sets a human-friendly message why this token exists and what it's used
+	// for, so other administrators can know its purpose.
+	Description string `json:"description,omitempty"`
+	// TTL defines the time to live for this token. Defaults to 24h.
+	// Expires and TTL are mutually exclusive.
+	TTL *metav1.Duration `json:"ttl,omitempty"`
+	// Expires specifies the timestamp when this token expires. Defaults to being set
+	// dynamically at runtime based on the TTL. Expires and TTL are mutually exclusive.
+	Expires *metav1.Time `json:"expires,omitempty"`
+	// Usages describes the ways in which this token can be used. Can by default be used
+	// for establishing bidirectional trust, but that can be changed here.
+	Usages []string `json:"usages,omitempty"`
+	// Groups specifies the extra groups that this token will authenticate as when/if
+	// used for authentication
+	Groups []string `json:"groups,omitempty"`
+}
+
 // Etcd contains elements describing Etcd configuration.
 type Etcd struct {
 

cmd/kubeadm/app/apis/kubeadm/validation/validation.go

@@ -35,7 +35,6 @@ import (
 	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/kubernetes/cmd/kubeadm/app/features"
 	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
-	tokenutil "k8s.io/kubernetes/cmd/kubeadm/app/util/token"
 	apivalidation "k8s.io/kubernetes/pkg/apis/core/validation"
 	"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
 	kubeletscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme"
@@ -55,9 +54,7 @@ func ValidateMasterConfiguration(c *kubeadm.MasterConfiguration) field.ErrorList
 	allErrs = append(allErrs, ValidateCertSANs(c.APIServerCertSANs, field.NewPath("apiServerCertSANs"))...)
 	allErrs = append(allErrs, ValidateAbsolutePath(c.CertificatesDir, field.NewPath("certificatesDir"))...)
 	allErrs = append(allErrs, ValidateNodeRegistrationOptions(&c.NodeRegistration, field.NewPath("nodeRegistration"))...)
-	allErrs = append(allErrs, ValidateToken(c.Token, field.NewPath("token"))...)
-	allErrs = append(allErrs, ValidateTokenUsages(c.TokenUsages, field.NewPath("tokenUsages"))...)
-	allErrs = append(allErrs, ValidateTokenGroups(c.TokenUsages, c.TokenGroups, field.NewPath("tokenGroups"))...)
+	allErrs = append(allErrs, ValidateBootstrapTokens(c.BootstrapTokens, field.NewPath("bootstrapTokens"))...)
 	allErrs = append(allErrs, ValidateFeatureGates(c.FeatureGates, field.NewPath("featureGates"))...)
 	allErrs = append(allErrs, ValidateAPIEndpoint(&c.API, field.NewPath("api"))...)
 	allErrs = append(allErrs, ValidateProxy(c.KubeProxy.Config, field.NewPath("kubeProxy").Child("config"))...)
@@ -181,18 +178,29 @@ func ValidateDiscoveryFile(discoveryFile string, fldPath *field.Path) field.Erro
 	return allErrs
 }
 
-// ValidateToken validates token
-func ValidateToken(t string, fldPath *field.Path) field.ErrorList {
+func ValidateBootstrapTokens(bts []kubeadm.BootstrapToken, fldPath *field.Path) field.ErrorList {
 	allErrs := field.ErrorList{}
-
-	id, secret, err := tokenutil.ParseToken(t)
-	if err != nil {
-		allErrs = append(allErrs, field.Invalid(fldPath, t, err.Error()))
+	for i, bt := range bts {
+		btPath := fldPath.Child(fmt.Sprintf("%d", i))
+		allErrs = append(allErrs, ValidateToken(bt.Token.String(), btPath.Child("token"))...)
+		allErrs = append(allErrs, ValidateTokenUsages(bt.Usages, btPath.Child("usages"))...)
+		allErrs = append(allErrs, ValidateTokenGroups(bt.Usages, bt.Groups, btPath.Child("groups"))...)
+
+		if bt.Expires != nil && bt.TTL != nil {
+			allErrs = append(allErrs, field.Invalid(btPath, "", "the BootstrapToken .TTL and .Expires fields are mutually exclusive"))
+		}
 	}
+	return allErrs
+}
+
+// ValidateToken validates a Bootstrap Token
+func ValidateToken(token string, fldPath *field.Path) field.ErrorList {
+	allErrs := field.ErrorList{}
 
-	if len(id) == 0 || len(secret) == 0 {
-		allErrs = append(allErrs, field.Invalid(fldPath, t, "token must be of form '[a-z0-9]{6}.[a-z0-9]{16}'"))
+	if !bootstraputil.IsValidBootstrapToken(token) {
+		allErrs = append(allErrs, field.Invalid(fldPath, token, "the bootstrap token is invalid"))
 	}
+
 	return allErrs
 }
 

cmd/kubeadm/app/cmd/config.go

@@ -48,10 +48,18 @@ const (
 	// TODO: Figure out how to get these constants from the API machinery
 	masterConfig = "MasterConfiguration"
 	nodeConfig   = "NodeConfiguration"
-	sillyToken   = "abcdef.0123456789abcdef"
 )
 
-var availableAPIObjects = []string{masterConfig, nodeConfig}
+var (
+	availableAPIObjects = []string{masterConfig, nodeConfig}
+	// sillyToken is only set statically to make kubeadm not randomize the token on every run
+	sillyToken = kubeadmapiv1alpha2.BootstrapToken{
+		Token: &kubeadmapiv1alpha2.BootstrapTokenString{
+			ID:     "abcdef",
+			Secret: "0123456789abcdef",
+		},
+	}
+)
 
 // NewCmdConfig returns cobra.Command for "kubeadm config" command
 func NewCmdConfig(out io.Writer) *cobra.Command {
@@ -123,7 +131,7 @@ func getDefaultAPIObjectBytes(apiObject string) ([]byte, error) {
 	if apiObject == masterConfig {
 
 		internalcfg, err := configutil.ConfigFileAndDefaultsToInternalConfig("", &kubeadmapiv1alpha2.MasterConfiguration{
-			Token: sillyToken,
+			BootstrapTokens: []kubeadmapiv1alpha2.BootstrapToken{sillyToken},
 		})
 		kubeadmutil.CheckErr(err)
 
@@ -131,7 +139,7 @@ func getDefaultAPIObjectBytes(apiObject string) ([]byte, error) {
 	}
 	if apiObject == nodeConfig {
 		internalcfg, err := configutil.NodeConfigFileAndDefaultsToInternalConfig("", &kubeadmapiv1alpha2.NodeConfiguration{
-			Token: sillyToken,
+			Token: sillyToken.Token.String(),
 			DiscoveryTokenAPIServers:               []string{"kube-apiserver:6443"},
 			DiscoveryTokenUnsafeSkipCAVerification: true,
 		})

cmd/kubeadm/app/cmd/init.go

@@ -37,6 +37,7 @@ import (
 	kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
 	kubeadmapiv1alpha2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha2"
 	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation"
+	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
 	cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/kubernetes/cmd/kubeadm/app/features"
@@ -123,6 +124,9 @@ func NewCmdInit(out io.Writer) *cobra.Command {
 	var dryRun bool
 	var featureGatesString string
 	var ignorePreflightErrors []string
+	// Create the options object for the bootstrap token-related flags, and override the default value for .Description
+	bto := options.NewBootstrapTokenOptions()
+	bto.Description = "The default bootstrap token generated by 'kubeadm init'."
 
 	cmd := &cobra.Command{
 		Use:   "init",
@@ -142,6 +146,9 @@ func NewCmdInit(out io.Writer) *cobra.Command {
 			err = validation.ValidateMixedArguments(cmd.Flags())
 			kubeadmutil.CheckErr(err)
 
+			err = bto.ApplyTo(externalcfg)
+			kubeadmutil.CheckErr(err)
+
 			i, err := NewInit(cfgPath, externalcfg, ignorePreflightErrorsSet, skipTokenPrint, dryRun)
 			kubeadmutil.CheckErr(err)
 			kubeadmutil.CheckErr(i.Run(out))
@@ -150,6 +157,8 @@ func NewCmdInit(out io.Writer) *cobra.Command {
 
 	AddInitConfigFlags(cmd.PersistentFlags(), externalcfg, &featureGatesString)
 	AddInitOtherFlags(cmd.PersistentFlags(), &cfgPath, &skipPreFlight, &skipTokenPrint, &dryRun, &ignorePreflightErrors)
+	bto.AddTokenFlag(cmd.PersistentFlags())
+	bto.AddTTLFlag(cmd.PersistentFlags())
 
 	return cmd
 }
@@ -193,20 +202,11 @@ func AddInitConfigFlags(flagSet *flag.FlagSet, cfg *kubeadmapiv1alpha2.MasterCon
 		`Specify the node name.`,
 	)
 	flagSet.StringVar(
-		&cfg.Token, "token", cfg.Token,
-		"The token to use for establishing bidirectional trust between nodes and masters. The format is [a-z0-9]{6}\\.[a-z0-9]{16} - e.g. abcdef.0123456789abcdef",
-	)
-	flagSet.DurationVar(
-		&cfg.TokenTTL.Duration, "token-ttl", cfg.TokenTTL.Duration,
-		"The duration before the bootstrap token is automatically deleted. If set to '0', the token will never expire.",
-	)
-	flagSet.StringVar(
 		&cfg.NodeRegistration.CRISocket, "cri-socket", cfg.NodeRegistration.CRISocket,
 		`Specify the CRI socket to connect to.`,
 	)
 	flagSet.StringVar(featureGatesString, "feature-gates", *featureGatesString, "A set of key=value pairs that describe feature gates for various features. "+
 		"Options are:\n"+strings.Join(features.KnownFeatures(&features.InitFeatureGates), "\n"))
-
 }
 
 // AddInitOtherFlags adds init flags that are not bound to a configuration file to the given flagset
@@ -427,14 +427,21 @@ func (i *Init) Run(out io.Writer) error {
 	}
 
 	// PHASE 5: Set up the node bootstrap tokens
+	tokens := []string{}
+	for _, bt := range i.cfg.BootstrapTokens {
+		tokens = append(tokens, bt.Token.String())
+	}
 	if !i.skipTokenPrint {
-		glog.Infof("[bootstraptoken] using token: %s\n", i.cfg.Token)
+		if len(tokens) == 1 {
+			glog.Infof("[bootstraptoken] using token: %s\n", tokens[0])
+		} else if len(tokens) > 1 {
+			glog.Infof("[bootstraptoken] using tokens: %v\n", tokens)
+		}
 	}
 
 	// Create the default node bootstrap token
 	glog.V(1).Infof("[init] creating RBAC rules to generate default bootstrap token")
-	tokenDescription := "The default bootstrap token generated by 'kubeadm init'."
-	if err := nodebootstraptokenphase.UpdateOrCreateToken(client, i.cfg.Token, false, i.cfg.TokenTTL.Duration, i.cfg.TokenUsages, i.cfg.TokenGroups, tokenDescription); err != nil {
+	if err := nodebootstraptokenphase.UpdateOrCreateTokens(client, false, i.cfg.BootstrapTokens); err != nil {
 		return fmt.Errorf("error updating or creating token: %v", err)
 	}
 	// Create RBAC rules that makes the bootstrap tokens able to post CSRs
@@ -491,10 +498,19 @@ func (i *Init) Run(out io.Writer) error {
 		return nil
 	}
 
-	// Gets the join command
-	joinCommand, err := cmdutil.GetJoinCommand(kubeadmconstants.GetAdminKubeConfigPath(), i.cfg.Token, i.skipTokenPrint)
+	// Prints the join command, multiple times in case the user has multiple tokens
+	for _, token := range tokens {
+		if err := printJoinCommand(out, adminKubeConfigPath, token, i.skipTokenPrint); err != nil {
+			return fmt.Errorf("failed to print join command: %v", err)
+		}
+	}
+	return nil
+}
+
+func printJoinCommand(out io.Writer, adminKubeConfigPath, token string, skipTokenPrint bool) error {
+	joinCommand, err := cmdutil.GetJoinCommand(adminKubeConfigPath, token, skipTokenPrint)
 	if err != nil {
-		return fmt.Errorf("failed to get join command: %v", err)
+		return err
 	}
 
 	ctx := map[string]string{

cmd/kubeadm/app/cmd/options/token.go

+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/spf13/pflag"
+
+	bootstrapapi "k8s.io/client-go/tools/bootstrap/token/api"
+	kubeadmapiv1alpha2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha2"
+	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
+)
+
+func NewBootstrapTokenOptions() *BootstrapTokenOptions {
+	bto := &BootstrapTokenOptions{&kubeadmapiv1alpha2.BootstrapToken{}, ""}
+	kubeadmapiv1alpha2.SetDefaults_BootstrapToken(bto.BootstrapToken)
+	return bto
+}
+
+// BootstrapTokenOptions is a wrapper struct for adding bootstrap token-related flags to a FlagSet
+// and applying the parsed flags to a MasterConfiguration object later at runtime
+// TODO: In the future, we might want to group the flags in a better way than adding them all individually like this
+type BootstrapTokenOptions struct {
+	*kubeadmapiv1alpha2.BootstrapToken
+	TokenStr string
+}
+
+func (bto *BootstrapTokenOptions) AddTokenFlag(fs *pflag.FlagSet) {
+	fs.StringVar(
+		&bto.TokenStr, "token", "",
+		"The token to use for establishing bidirectional trust between nodes and masters. The format is [a-z0-9]{6}\\.[a-z0-9]{16} - e.g. abcdef.0123456789abcdef",
+	)
+}
+
+func (bto *BootstrapTokenOptions) AddTTLFlag(fs *pflag.FlagSet) {
+	fs.DurationVar(
+		&bto.TTL.Duration, "ttl", bto.TTL.Duration,
+		"The duration before the token is automatically deleted (e.g. 1s, 2m, 3h). If set to '0', the token will never expire",
+	)
+}
+
+func (bto *BootstrapTokenOptions) AddUsagesFlag(fs *pflag.FlagSet) {
+	fs.StringSliceVar(
+		&bto.Usages, "usages", bto.Usages,
+		fmt.Sprintf("Describes the ways in which this token can be used. You can pass --usages multiple times or provide a comma separated list of options. Valid options: [%s]", strings.Join(kubeadmconstants.DefaultTokenUsages, ",")),
+	)
+}
+
+func (bto *BootstrapTokenOptions) AddGroupsFlag(fs *pflag.FlagSet) {
+	fs.StringSliceVar(
+		&bto.Groups, "groups", bto.Groups,
+		fmt.Sprintf("Extra groups that this token will authenticate as when used for authentication. Must match %q", bootstrapapi.BootstrapGroupPattern),
+	)
+}
+
+func (bto *BootstrapTokenOptions) AddDescriptionFlag(fs *pflag.FlagSet) {
+	fs.StringVar(
+		&bto.Description, "description", bto.Description,
+		"A human friendly description of how this token is used.",
+	)
+}
+
+func (bto *BootstrapTokenOptions) ApplyTo(cfg *kubeadmapiv1alpha2.MasterConfiguration) error {
+	if len(bto.TokenStr) > 0 {
+		var err error
+		bto.Token, err = kubeadmapiv1alpha2.NewBootstrapTokenString(bto.TokenStr)
+		if err != nil {
+			return err
+		}
+	}
+
+	// Set the token specified by the flags as the first and only token to create in case --config is not specified
+	cfg.BootstrapTokens = []kubeadmapiv1alpha2.BootstrapToken{*bto.BootstrapToken}
+	return nil
+}

cmd/kubeadm/app/cmd/phases/bootstraptoken.go

@@ -18,7 +18,6 @@ package phases
 
 import (
 	"fmt"
-	"strings"
 
 	"github.com/golang/glog"
 	"github.com/spf13/cobra"
@@ -30,8 +29,8 @@ import (
 	kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
 	kubeadmapiv1alpha2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha2"
 	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation"
+	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
 	cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
-	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo"
 	"k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/node"
 	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
@@ -111,14 +110,15 @@ func NewSubCmdBootstrapTokenAll(kubeConfigFile *string) *cobra.Command {
 	cfg := &kubeadmapiv1alpha2.MasterConfiguration{
 		// KubernetesVersion is not used by bootstrap-token, but we set this explicitly to avoid
 		// the lookup of the version from the internet when executing ConfigFileAndDefaultsToInternalConfig
-		KubernetesVersion: "v1.9.0",
+		KubernetesVersion: "v1.10.0",
 	}
 
 	// Default values for the cobra help text
 	kubeadmscheme.Scheme.Default(cfg)
 
-	var cfgPath, description string
+	var cfgPath string
 	var skipTokenPrint bool
+	bto := options.NewBootstrapTokenOptions()
 
 	cmd := &cobra.Command{
 		Use:     "all",
@@ -129,11 +129,14 @@ func NewSubCmdBootstrapTokenAll(kubeConfigFile *string) *cobra.Command {
 			err := validation.ValidateMixedArguments(cmd.Flags())
 			kubeadmutil.CheckErr(err)
 
+			err = bto.ApplyTo(cfg)
+			kubeadmutil.CheckErr(err)
+
 			client, err := kubeconfigutil.ClientSetFromFile(*kubeConfigFile)
 			kubeadmutil.CheckErr(err)
 
 			// Creates the bootstap token
-			err = createBootstrapToken(*kubeConfigFile, client, cfgPath, cfg, description, skipTokenPrint)
+			err = createBootstrapToken(*kubeConfigFile, client, cfgPath, cfg, skipTokenPrint)
 			kubeadmutil.CheckErr(err)
 
 			// Create the cluster-info ConfigMap or update if it already exists
@@ -159,7 +162,12 @@ func NewSubCmdBootstrapTokenAll(kubeConfigFile *string) *cobra.Command {
 	}
 
 	// Adds flags to the command
-	addBootstrapTokenFlags(cmd.Flags(), cfg, &cfgPath, &description, &skipTokenPrint)
+	addGenericFlags(cmd.Flags(), &cfgPath, &skipTokenPrint)
+	bto.AddTokenFlag(cmd.Flags())
+	bto.AddTTLFlag(cmd.Flags())
+	bto.AddUsagesFlag(cmd.Flags())
+	bto.AddGroupsFlag(cmd.Flags())
+	bto.AddDescriptionFlag(cmd.Flags())
 
 	return cmd
 }
@@ -169,14 +177,15 @@ func NewSubCmdBootstrapToken(kubeConfigFile *string) *cobra.Command {
 	cfg := &kubeadmapiv1alpha2.MasterConfiguration{
 		// KubernetesVersion is not used by bootstrap-token, but we set this explicitly to avoid
 		// the lookup of the version from the internet when executing ConfigFileAndDefaultsToInternalConfig
-		KubernetesVersion: "v1.9.0",
+		KubernetesVersion: "v1.10.0",
 	}
 
 	// Default values for the cobra help text
 	kubeadmscheme.Scheme.Default(cfg)
 
-	var cfgPath, description string
+	var cfgPath string
 	var skipTokenPrint bool
+	bto := options.NewBootstrapTokenOptions()
 
 	cmd := &cobra.Command{
 		Use:   "create",
@@ -186,16 +195,24 @@ func NewSubCmdBootstrapToken(kubeConfigFile *string) *cobra.Command {
 			err := validation.ValidateMixedArguments(cmd.Flags())
 			kubeadmutil.CheckErr(err)
 
+			err = bto.ApplyTo(cfg)
+			kubeadmutil.CheckErr(err)
+
 			client, err := kubeconfigutil.ClientSetFromFile(*kubeConfigFile)
 			kubeadmutil.CheckErr(err)
 
-			err = createBootstrapToken(*kubeConfigFile, client, cfgPath, cfg, description, skipTokenPrint)
+			err = createBootstrapToken(*kubeConfigFile, client, cfgPath, cfg, skipTokenPrint)
 			kubeadmutil.CheckErr(err)
 		},
 	}
 
 	// Adds flags to the command
-	addBootstrapTokenFlags(cmd.Flags(), cfg, &cfgPath, &description, &skipTokenPrint)
+	addGenericFlags(cmd.Flags(), &cfgPath, &skipTokenPrint)
+	bto.AddTokenFlag(cmd.Flags())
+	bto.AddTTLFlag(cmd.Flags())
+	bto.AddUsagesFlag(cmd.Flags())
+	bto.AddGroupsFlag(cmd.Flags())
+	bto.AddDescriptionFlag(cmd.Flags())
 
 	return cmd
 }
@@ -278,38 +295,18 @@ func NewSubCmdNodeBootstrapTokenAutoApprove(kubeConfigFile *string) *cobra.Comma
 	return cmd
 }
 
-func addBootstrapTokenFlags(flagSet *pflag.FlagSet, cfg *kubeadmapiv1alpha2.MasterConfiguration, cfgPath, description *string, skipTokenPrint *bool) {
+func addGenericFlags(flagSet *pflag.FlagSet, cfgPath *string, skipTokenPrint *bool) {
 	flagSet.StringVar(
 		cfgPath, "config", *cfgPath,
 		"Path to kubeadm config file. WARNING: Usage of a configuration file is experimental",
 	)
-	flagSet.StringVar(
-		&cfg.Token, "token", cfg.Token,
-		"The token to use for establishing bidirectional trust between nodes and masters",
-	)
-	flagSet.DurationVar(
-		&cfg.TokenTTL.Duration, "ttl", cfg.TokenTTL.Duration,
-		"The duration before the token is automatically deleted (e.g. 1s, 2m, 3h). If set to '0', the token will never expire",
-	)
-	flagSet.StringSliceVar(
-		&cfg.TokenUsages, "usages", cfg.TokenUsages,
-		fmt.Sprintf("Describes the ways in which this token can be used. You can pass --usages multiple times or provide a comma separated list of options. Valid options: [%s]", strings.Join(kubeadmconstants.DefaultTokenUsages, ",")),
-	)
-	flagSet.StringSliceVar(
-		&cfg.TokenGroups, "groups", cfg.TokenGroups,
-		fmt.Sprintf("Extra groups that this token will authenticate as when used for authentication. Must match %q", bootstrapapi.BootstrapGroupPattern),
-	)
-	flagSet.StringVar(
-		description, "description", "The default bootstrap token generated by 'kubeadm init'.",
-		"A human friendly description of how this token is used.",
-	)
 	flagSet.BoolVar(
 		skipTokenPrint, "skip-token-print", *skipTokenPrint,
 		"Skip printing of the bootstrap token",
 	)
 }
 
-func createBootstrapToken(kubeConfigFile string, client clientset.Interface, cfgPath string, cfg *kubeadmapiv1alpha2.MasterConfiguration, description string, skipTokenPrint bool) error {
+func createBootstrapToken(kubeConfigFile string, client clientset.Interface, cfgPath string, cfg *kubeadmapiv1alpha2.MasterConfiguration, skipTokenPrint bool) error {
 
 	// This call returns the ready-to-use configuration based on the configuration file that might or might not exist and the default cfg populated by flags
 	internalcfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(cfgPath, cfg)
@@ -319,18 +316,19 @@ func createBootstrapToken(kubeConfigFile string, client clientset.Interface, cfg
 
 	glog.V(1).Infoln("[bootstraptoken] creating/updating token")
 	// Creates or updates the token
-	if err := node.UpdateOrCreateToken(client, internalcfg.Token, false, internalcfg.TokenTTL.Duration, internalcfg.TokenUsages, internalcfg.TokenGroups, description); err != nil {
+	if err := node.UpdateOrCreateTokens(client, false, internalcfg.BootstrapTokens); err != nil {
 		return err
 	}
 
 	glog.Infoln("[bootstraptoken] bootstrap token created")
 	glog.Infoln("[bootstraptoken] you can now join any number of machines by running:")
 
-	joinCommand, err := cmdutil.GetJoinCommand(kubeConfigFile, internalcfg.Token, skipTokenPrint)
-	if err != nil {
-		return fmt.Errorf("failed to get join command: %v", err)
+	if len(internalcfg.BootstrapTokens) > 0 {
+		joinCommand, err := cmdutil.GetJoinCommand(kubeConfigFile, internalcfg.BootstrapTokens[0].Token.String(), skipTokenPrint)
+		if err != nil {
+			return fmt.Errorf("failed to get join command: %v", err)
+		}
+		glog.Infoln(joinCommand)
 	}
-	glog.Infoln(joinCommand)
-
 	return nil
 }

cmd/kubeadm/app/cmd/phases/markmaster.go

@@ -50,7 +50,7 @@ func NewCmdMarkMaster() *cobra.Command {
 	cfg := &kubeadmapiv1alpha2.MasterConfiguration{
 		// KubernetesVersion is not used by mark master, but we set this explicitly to avoid
 		// the lookup of the version from the internet when executing ConfigFileAndDefaultsToInternalConfig
-		KubernetesVersion: "v1.9.0",
+		KubernetesVersion: "v1.10.0",
 	}
 
 	// Default values for the cobra help text

cmd/kubeadm/app/discovery/token/token.go

@@ -34,7 +34,6 @@ import (
 	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
 	"k8s.io/kubernetes/cmd/kubeadm/app/util/pubkeypin"
-	tokenutil "k8s.io/kubernetes/cmd/kubeadm/app/util/token"
 	"k8s.io/kubernetes/pkg/controller/bootstrap"
 )
 
@@ -45,7 +44,7 @@ const BootstrapUser = "token-bootstrap-client"
 // It then makes sure it can trust the API Server by looking at the JWS-signed tokens and (if cfg.DiscoveryTokenCACertHashes is not empty)
 // validating the cluster CA against a set of pinned public keys
 func RetrieveValidatedClusterInfo(cfg *kubeadmapi.NodeConfiguration) (*clientcmdapi.Cluster, error) {
-	tokenID, tokenSecret, err := tokenutil.ParseToken(cfg.DiscoveryToken)
+	token, err := kubeadmapi.NewBootstrapTokenString(cfg.DiscoveryToken)
 	if err != nil {
 		return nil, err
 	}
@@ -88,11 +87,11 @@ func RetrieveValidatedClusterInfo(cfg *kubeadmapi.NodeConfiguration) (*clientcmd
 		if !ok || len(insecureKubeconfigString) == 0 {
 			return nil, fmt.Errorf("there is no %s key in the %s ConfigMap. This API Server isn't set up for token bootstrapping, can't connect", bootstrapapi.KubeConfigKey, bootstrapapi.ConfigMapClusterInfo)
 		}
-		detachedJWSToken, ok := insecureClusterInfo.Data[bootstrapapi.JWSSignatureKeyPrefix+tokenID]
+		detachedJWSToken, ok := insecureClusterInfo.Data[bootstrapapi.JWSSignatureKeyPrefix+token.ID]
 		if !ok || len(detachedJWSToken) == 0 {
-			return nil, fmt.Errorf("token id %q is invalid for this cluster or it has expired. Use \"kubeadm token create\" on the master node to creating a new valid token", tokenID)
+			return nil, fmt.Errorf("token id %q is invalid for this cluster or it has expired. Use \"kubeadm token create\" on the master node to creating a new valid token", token.ID)
 		}
-		if !bootstrap.DetachedTokenIsValid(detachedJWSToken, insecureKubeconfigString, tokenID, tokenSecret) {
+		if !bootstrap.DetachedTokenIsValid(detachedJWSToken, insecureKubeconfigString, token.ID, token.Secret) {
 			return nil, fmt.Errorf("failed to verify JWS signature of received cluster info object, can't trust this API Server")
 		}
 		insecureKubeconfigBytes := []byte(insecureKubeconfigString)

cmd/kubeadm/app/phases/bootstraptoken/node/token.go

@@ -18,109 +18,43 @@ package node
 
 import (
 	"fmt"
-	"strings"
-	"time"
 
-	"k8s.io/api/core/v1"
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	clientset "k8s.io/client-go/kubernetes"
-	bootstrapapi "k8s.io/client-go/tools/bootstrap/token/api"
 	bootstraputil "k8s.io/client-go/tools/bootstrap/token/util"
-	tokenutil "k8s.io/kubernetes/cmd/kubeadm/app/util/token"
+	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
+	"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
 )
 
-const tokenCreateRetries = 5
+// TODO(mattmoyer): Move CreateNewTokens, UpdateOrCreateTokens out of this package to client-go for a generic abstraction and client for a Bootstrap Token
 
-// TODO(mattmoyer): Move CreateNewToken, UpdateOrCreateToken and encodeTokenSecretData out of this package to client-go for a generic abstraction and client for a Bootstrap Token
-
-// CreateNewToken tries to create a token and fails if one with the same ID already exists
-func CreateNewToken(client clientset.Interface, token string, tokenDuration time.Duration, usages []string, extraGroups []string, description string) error {
-	return UpdateOrCreateToken(client, token, true, tokenDuration, usages, extraGroups, description)
+// CreateNewTokens tries to create a token and fails if one with the same ID already exists
+func CreateNewTokens(client clientset.Interface, tokens []kubeadmapi.BootstrapToken) error {
+	return UpdateOrCreateTokens(client, true, tokens)
 }
 
-// UpdateOrCreateToken attempts to update a token with the given ID, or create if it does not already exist.
-func UpdateOrCreateToken(client clientset.Interface, token string, failIfExists bool, tokenDuration time.Duration, usages []string, extraGroups []string, description string) error {
-	tokenID, tokenSecret, err := tokenutil.ParseToken(token)
-	if err != nil {
-		return err
-	}
-	secretName := fmt.Sprintf("%s%s", bootstrapapi.BootstrapTokenSecretPrefix, tokenID)
-	var lastErr error
-	for i := 0; i < tokenCreateRetries; i++ {
+// UpdateOrCreateTokens attempts to update a token with the given ID, or create if it does not already exist.
+func UpdateOrCreateTokens(client clientset.Interface, failIfExists bool, tokens []kubeadmapi.BootstrapToken) error {
+
+	for _, token := range tokens {
+
+		secretName := bootstraputil.BootstrapTokenSecretName(token.Token.ID)
 		secret, err := client.CoreV1().Secrets(metav1.NamespaceSystem).Get(secretName, metav1.GetOptions{})
-		if err == nil {
-			if failIfExists {
-				return fmt.Errorf("a token with id %q already exists", tokenID)
-			}
-			// Secret with this ID already exists, update it:
-			tokenSecretData, err := encodeTokenSecretData(tokenID, tokenSecret, tokenDuration, usages, extraGroups, description)
-			if err != nil {
-				return err
-			}
-			secret.Data = tokenSecretData
-			if _, err := client.CoreV1().Secrets(metav1.NamespaceSystem).Update(secret); err == nil {
-				return nil
-			}
-			lastErr = err
-			continue
+		if secret != nil && err == nil && failIfExists {
+			return fmt.Errorf("a token with id %q already exists", token.Token.ID)
 		}
 
-		// Secret does not already exist:
-		if apierrors.IsNotFound(err) {
-			tokenSecretData, err := encodeTokenSecretData(tokenID, tokenSecret, tokenDuration, usages, extraGroups, description)
-			if err != nil {
-				return err
-			}
-
-			secret = &v1.Secret{
-				ObjectMeta: metav1.ObjectMeta{
-					Name: secretName,
-				},
-				Type: v1.SecretType(bootstrapapi.SecretTypeBootstrapToken),
-				Data: tokenSecretData,
+		updatedOrNewSecret := token.ToSecret()
+		// Try to create or update the token with an exponential backoff
+		err = apiclient.TryRunCommand(func() error {
+			if err := apiclient.CreateOrUpdateSecret(client, updatedOrNewSecret); err != nil {
+				return fmt.Errorf("failed to create or update bootstrap token with name %s: %v", secretName, err)
 			}
-			if _, err := client.CoreV1().Secrets(metav1.NamespaceSystem).Create(secret); err == nil {
-				return nil
-			}
-			lastErr = err
-			continue
+			return nil
+		}, 5)
+		if err != nil {
+			return err
 		}
-
-	}
-	return fmt.Errorf(
-		"unable to create bootstrap token after %d attempts [%v]",
-		tokenCreateRetries,
-		lastErr,
-	)
-}
-
-// encodeTokenSecretData takes the token discovery object and an optional duration and returns the .Data for the Secret
-func encodeTokenSecretData(tokenID, tokenSecret string, duration time.Duration, usages []string, extraGroups []string, description string) (map[string][]byte, error) {
-	data := map[string][]byte{
-		bootstrapapi.BootstrapTokenIDKey:     []byte(tokenID),
-		bootstrapapi.BootstrapTokenSecretKey: []byte(tokenSecret),
-	}
-
-	if len(extraGroups) > 0 {
-		data[bootstrapapi.BootstrapTokenExtraGroupsKey] = []byte(strings.Join(extraGroups, ","))
-	}
-
-	if duration > 0 {
-		// Get the current time, add the specified duration, and format it accordingly
-		durationString := time.Now().Add(duration).Format(time.RFC3339)
-		data[bootstrapapi.BootstrapTokenExpirationKey] = []byte(durationString)
-	}
-	if len(description) > 0 {
-		data[bootstrapapi.BootstrapTokenDescriptionKey] = []byte(description)
-	}
-
-	// validate usages
-	if err := bootstraputil.ValidateUsages(usages); err != nil {
-		return nil, err
-	}
-	for _, usage := range usages {
-		data[bootstrapapi.BootstrapTokenUsagePrefix+usage] = []byte("true")
 	}
-	return data, nil
+	return nil
 }

cmd/kubeadm/app/phases/uploadconfig/uploadconfig.go

@@ -41,7 +41,7 @@ func UploadConfiguration(cfg *kubeadmapi.MasterConfiguration, client clientset.I
 	kubeadmscheme.Scheme.Convert(cfg, externalcfg, nil)
 
 	// Removes sensitive info from the data that will be stored in the config map
-	externalcfg.Token = ""
+	externalcfg.BootstrapTokens = nil
 
 	cfgYaml, err := util.MarshalToYamlForCodecs(externalcfg, kubeadmapiv1alpha2.SchemeGroupVersion, scheme.Codecs)
 	if err != nil {

cmd/kubeadm/app/util/config/masterconfig.go

@@ -26,6 +26,7 @@ import (
 	"k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	netutil "k8s.io/apimachinery/pkg/util/net"
+	bootstraputil "k8s.io/client-go/tools/bootstrap/token/util"
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
 	kubeadmapiv1alpha1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1"
@@ -33,7 +34,6 @@ import (
 	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
-	tokenutil "k8s.io/kubernetes/cmd/kubeadm/app/util/token"
 	"k8s.io/kubernetes/pkg/util/node"
 	"k8s.io/kubernetes/pkg/util/version"
 )
@@ -60,17 +60,29 @@ func SetInitDynamicDefaults(cfg *kubeadmapi.MasterConfiguration) error {
 		cfg.KubeProxy.Config.BindAddress = kubeadmapiv1alpha2.DefaultProxyBindAddressv6
 	}
 	// Resolve possible version labels and validate version string
-	err = NormalizeKubernetesVersion(cfg)
-	if err != nil {
+	if err := NormalizeKubernetesVersion(cfg); err != nil {
 		return err
 	}
 
-	if cfg.Token == "" {
-		var err error
-		cfg.Token, err = tokenutil.GenerateToken()
+	// Populate the .Token field with a random value if unset
+	// We do this at this layer, and not the API defaulting layer
+	// because of possible security concerns, and more practially
+	// because we can't return errors in the API object defaulting
+	// process but here we can.
+	for i, bt := range cfg.BootstrapTokens {
+		if bt.Token != nil && len(bt.Token.String()) > 0 {
+			continue
+		}
+
+		tokenStr, err := bootstraputil.GenerateBootstrapToken()
 		if err != nil {
 			return fmt.Errorf("couldn't generate random token: %v", err)
 		}
+		token, err := kubeadmapi.NewBootstrapTokenString(tokenStr)
+		if err != nil {
+			return err
+		}
+		cfg.BootstrapTokens[i].Token = token
 	}
 
 	cfg.NodeRegistration.Name = node.GetHostname(cfg.NodeRegistration.Name)

cmd/kubeadm/app/util/token/tokens.go

-/*
-Copyright 2017 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package token
-
-import (
-	"bufio"
-	"crypto/rand"
-	"fmt"
-	"regexp"
-
-	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
-)
-
-const (
-	// TokenIDBytes defines a number of bytes used for a token id
-	TokenIDBytes = 6
-	// TokenSecretBytes defines a number of bytes used for a secret
-	TokenSecretBytes = 16
-)
-
-var (
-	// TokenIDRegexpString defines token's id regular expression pattern
-	TokenIDRegexpString = "^([a-z0-9]{6})$"
-	// TokenIDRegexp is a compiled regular expression of TokenIDRegexpString
-	TokenIDRegexp = regexp.MustCompile(TokenIDRegexpString)
-	// TokenRegexpString defines id.secret regular expression pattern
-	TokenRegexpString = "^([a-z0-9]{6})\\.([a-z0-9]{16})$"
-	// TokenRegexp is a compiled regular expression of TokenRegexpString
-	TokenRegexp = regexp.MustCompile(TokenRegexpString)
-)
-
-const validBootstrapTokenChars = "0123456789abcdefghijklmnopqrstuvwxyz"
-
-func randBytes(length int) (string, error) {
-	// len("0123456789abcdefghijklmnopqrstuvwxyz") = 36 which doesn't evenly divide
-	// the possible values of a byte: 256 mod 36 = 4. Discard any random bytes we
-	// read that are >= 252 so the bytes we evenly divide the character set.
-	const maxByteValue = 252
-
-	var (
-		b     byte
-		err   error
-		token = make([]byte, length)
-	)
-
-	reader := bufio.NewReaderSize(rand.Reader, length*2)
-	for i := range token {
-		for {
-			if b, err = reader.ReadByte(); err != nil {
-				return "", err
-			}
-			if b < maxByteValue {
-				break
-			}
-		}
-
-		token[i] = validBootstrapTokenChars[int(b)%len(validBootstrapTokenChars)]
-	}
-
-	return string(token), nil
-}
-
-// GenerateToken generates a new token with a token ID that is valid as a
-// Kubernetes DNS label.
-// For more info, see kubernetes/pkg/util/validation/validation.go.
-func GenerateToken() (string, error) {
-	tokenID, err := randBytes(TokenIDBytes)
-	if err != nil {
-		return "", err
-	}
-
-	tokenSecret, err := randBytes(TokenSecretBytes)
-	if err != nil {
-		return "", err
-	}
-
-	return fmt.Sprintf("%s.%s", tokenID, tokenSecret), nil
-}
-
-// ParseTokenID tries and parse a valid token ID from a string.
-// An error is returned in case of failure.
-func ParseTokenID(s string) error {
-	if !TokenIDRegexp.MatchString(s) {
-		return fmt.Errorf("token ID [%q] was not of form [%q]", s, TokenIDRegexpString)
-	}
-	return nil
-}
-
-// ParseToken tries and parse a valid token from a string.
-// A token ID and token secret are returned in case of success, an error otherwise.
-func ParseToken(s string) (string, string, error) {
-	split := TokenRegexp.FindStringSubmatch(s)
-	if len(split) != 3 {
-		return "", "", fmt.Errorf("token [%q] was not of form [%q]", s, TokenRegexpString)
-	}
-	return split[1], split[2], nil
-}
-
-// BearerToken returns a string representation of the passed token.
-func BearerToken(d *kubeadmapi.TokenDiscovery) string {
-	return fmt.Sprintf("%s.%s", d.ID, d.Secret)
-}
-
-// ValidateToken validates whether a token is well-formed.
-// In case it's not, the corresponding error is returned as well.
-func ValidateToken(d *kubeadmapi.TokenDiscovery) (bool, error) {
-	if _, _, err := ParseToken(d.ID + "." + d.Secret); err != nil {
-		return false, err
-	}
-	return true, nil
-}