kubeadm: Enable the Node Authorizer in v1.7 and fix some small bugs related to the enablement

cmd/kubeadm/app/cmd/BUILD

@@ -41,6 +41,7 @@ go_library(
         "//cmd/kubeadm/app/util/token:go_default_library",
         "//pkg/api:go_default_library",
         "//pkg/bootstrap/api:go_default_library",
+        "//pkg/kubeapiserver/authorizer/modes:go_default_library",
         "//pkg/kubectl/cmd/util:go_default_library",
         "//pkg/printers:go_default_library",
         "//pkg/util/i18n:go_default_library",
@@ -54,6 +55,7 @@ go_library(
         "//vendor/k8s.io/apimachinery/pkg/fields:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/version:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/util/flag:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes:go_default_library",
@@ -65,6 +67,7 @@ go_library(
 go_test(
     name = "go_default_test",
     srcs = [
+        "defaults_test.go",
         "reset_test.go",
         "token_test.go",
     ],
@@ -73,6 +76,7 @@ go_test(
     deps = [
         "//cmd/kubeadm/app/constants:go_default_library",
         "//cmd/kubeadm/app/preflight:go_default_library",
+        "//pkg/util/version:go_default_library",
     ],
 )
 

cmd/kubeadm/app/cmd/defaults.go

@@ -21,10 +21,12 @@ import (
 	"net"
 
 	netutil "k8s.io/apimachinery/pkg/util/net"
+	"k8s.io/apimachinery/pkg/util/sets"
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
 	tokenutil "k8s.io/kubernetes/cmd/kubeadm/app/util/token"
+	authzmodes "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes"
 	"k8s.io/kubernetes/pkg/util/version"
 )
 
@@ -54,8 +56,12 @@ func setInitDynamicDefaults(cfg *kubeadmapi.MasterConfiguration) error {
 		return fmt.Errorf("this version of kubeadm only supports deploying clusters with the control plane version >= %s. Current version: %s", kubeadmconstants.MinimumControlPlaneVersion.String(), cfg.KubernetesVersion)
 	}
 
+	// Defaulting is made here because it's dependent on the version currently, which is determined above
+	// TODO(luxas): Cleanup this once we have dropped v1.6 support and move this code into the API group defaulting
+	cfg.AuthorizationModes = defaultAuthorizationModes(cfg.AuthorizationModes, k8sVersion)
+
 	fmt.Printf("[init] Using Kubernetes version: %s\n", cfg.KubernetesVersion)
-	fmt.Printf("[init] Using Authorization mode: %v\n", cfg.AuthorizationModes)
+	fmt.Printf("[init] Using Authorization modes: %v\n", cfg.AuthorizationModes)
 
 	// Warn about the limitations with the current cloudprovider solution.
 	if cfg.CloudProvider != "" {
@@ -73,3 +79,13 @@ func setInitDynamicDefaults(cfg *kubeadmapi.MasterConfiguration) error {
 
 	return nil
 }
+
+func defaultAuthorizationModes(authzModes []string, k8sVersion *version.Version) []string {
+	if k8sVersion.AtLeast(kubeadmconstants.MinimumNodeAuthorizerVersion) {
+		strset := sets.NewString(authzModes...)
+		if !strset.Has(authzmodes.ModeNode) {
+			return append([]string{authzmodes.ModeNode}, authzModes...)
+		}
+	}
+	return authzModes
+}

cmd/kubeadm/app/cmd/defaults_test.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cmd
+
+import (
+	"strings"
+	"testing"
+
+	"k8s.io/kubernetes/pkg/util/version"
+)
+
+func TestDefaultAuthorizationModes(t *testing.T) {
+	var tests = []struct {
+		authzModes []string
+		version    string
+		expected   []string
+	}{
+		{[]string{"RBAC"}, "v1.6.0", []string{"RBAC"}},
+		{[]string{"RBAC", "ABAC"}, "v1.6.4", []string{"RBAC", "ABAC"}},
+		{[]string{"RBAC", "ABAC"}, "v1.7.0-beta.0", []string{"RBAC", "ABAC"}},
+		{[]string{"RBAC"}, "v1.7.0", []string{"Node", "RBAC"}},
+		{[]string{"RBAC", "Webhook"}, "v1.7.0-beta.1", []string{"Node", "RBAC", "Webhook"}},
+		{[]string{"RBAC", "Webhook", "Node"}, "v1.7.0", []string{"RBAC", "Webhook", "Node"}},
+		{[]string{"Node", "RBAC", "Webhook"}, "v1.7.0", []string{"Node", "RBAC", "Webhook"}},
+	}
+	for _, rt := range tests {
+		k8sVersion, err := version.ParseSemantic(rt.version)
+		if err != nil {
+			t.Fatalf("couldn't parse given version")
+		}
+		actual := defaultAuthorizationModes(rt.authzModes, k8sVersion)
+		if strings.Join(actual, ",") != strings.Join(rt.expected, ",") {
+			t.Errorf(
+				"failed TestDefaultAuthorizationModes:\n\texpected: %s\n\t  actual: %s",
+				strings.Join(rt.expected, ","),
+				strings.Join(actual, ","),
+			)
+		}
+	}
+}

cmd/kubeadm/app/cmd/init.go

@@ -41,6 +41,7 @@ import (
 	"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
 	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
 	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/util/version"
 )
 
 var (
@@ -253,13 +254,18 @@ func (i *Init) Run(out io.Writer) error {
 
 	// PHASE 5: Install and deploy all addons, and configure things as necessary
 
+	k8sVersion, err := version.ParseSemantic(i.cfg.KubernetesVersion)
+	if err != nil {
+		return fmt.Errorf("couldn't parse kubernetes version %q: %v", i.cfg.KubernetesVersion, err)
+	}
+
 	// Create the necessary ServiceAccounts
 	err = apiconfigphase.CreateServiceAccounts(client)
 	if err != nil {
 		return err
 	}
 
-	err = apiconfigphase.CreateRBACRules(client)
+	err = apiconfigphase.CreateRBACRules(client, k8sVersion)
 	if err != nil {
 		return err
 	}

cmd/kubeadm/app/constants/constants.go

@@ -61,10 +61,11 @@ const (
 
 	// Some well-known users and groups in the core Kubernetes authorization system
 
-	ControllerManagerUser = "system:kube-controller-manager"
-	SchedulerUser         = "system:kube-scheduler"
-	MastersGroup          = "system:masters"
-	NodesGroup            = "system:nodes"
+	ControllerManagerUser   = "system:kube-controller-manager"
+	SchedulerUser           = "system:kube-scheduler"
+	MastersGroup            = "system:masters"
+	NodesGroup              = "system:nodes"
+	NodesClusterRoleBinding = "system:node"
 
 	// Constants for what we name our ServiceAccounts with limited access to the cluster in case of RBAC
 	KubeDNSServiceAccountName   = "kube-dns"
@@ -89,9 +90,6 @@ const (
 
 	// MinExternalEtcdVersion indicates minimum external etcd version which kubeadm supports
 	MinExternalEtcdVersion = "3.0.14"
-
-	// DefaultAdmissionControl specifies the default admission control options that will be used
-	DefaultAdmissionControl = "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds"
 )
 
 var (
@@ -116,4 +114,7 @@ var (
 
 	// MinimumAPIAggregationVersion specifies the minimum kubernetes version that can be used enabling the API aggregation in the apiserver and the front proxy flags
 	MinimumAPIAggregationVersion = version.MustParseSemantic("v1.7.0-alpha.1")
+
+	// MinimumNodeAuthorizerVersion specifies the minimum kubernetes version that can be used for enabling the node authorizer
+	MinimumNodeAuthorizerVersion = version.MustParseSemantic("v1.7.0-beta.1")
 )

cmd/kubeadm/app/master/BUILD

@@ -30,6 +30,7 @@ go_library(
         "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes:go_default_library",
         "//vendor/k8s.io/client-go/pkg/api/v1:go_default_library",

cmd/kubeadm/app/master/manifests.go

@@ -28,6 +28,7 @@ import (
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/apimachinery/pkg/util/sets"
 	api "k8s.io/client-go/pkg/api/v1"
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1"
@@ -43,6 +44,9 @@ import (
 const (
 	DefaultCloudConfigPath = "/etc/kubernetes/cloud-config"
 
+	defaultv16AdmissionControl = "NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,ResourceQuota"
+	defaultv17AdmissionControl = "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota"
+
 	etcd                  = "etcd"
 	apiServer             = "apiserver"
 	controllerManager     = "controller-manager"
@@ -326,7 +330,7 @@ func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration, selfHosted bool, k
 
 	defaultArguments := map[string]string{
 		"insecure-port":                     "0",
-		"admission-control":                 kubeadmconstants.DefaultAdmissionControl,
+		"admission-control":                 defaultv16AdmissionControl,
 		"service-cluster-ip-range":          cfg.Networking.ServiceSubnet,
 		"service-account-key-file":          filepath.Join(cfg.CertificatesDir, kubeadmconstants.ServiceAccountPublicKeyName),
 		"client-ca-file":                    filepath.Join(cfg.CertificatesDir, kubeadmconstants.CACertName),
@@ -351,6 +355,10 @@ func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration, selfHosted bool, k
 		defaultArguments["proxy-client-cert-file"] = filepath.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyClientCertName)
 		defaultArguments["proxy-client-key-file"] = filepath.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyClientKeyName)
 	}
+	if k8sVersion.AtLeast(kubeadmconstants.MinimumNodeAuthorizerVersion) {
+		// enable the NodeRestriction admission plugin
+		defaultArguments["admission-control"] = defaultv17AdmissionControl
+	}
 
 	command = getComponentBaseCommand(apiServer)
 	command = append(command, getExtraParameters(cfg.APIServerExtraArgs, defaultArguments)...)
@@ -505,21 +513,25 @@ func getSelfHostedAPIServerEnv() []api.EnvVar {
 
 func getAuthzParameters(modes []string) []string {
 	command := []string{}
-	// RBAC is always on. If the user specified
-	authzModes := []string{authzmodes.ModeRBAC}
-	for _, authzMode := range modes {
-		if len(authzMode) != 0 && authzMode != authzmodes.ModeRBAC {
-			authzModes = append(authzModes, authzMode)
-		}
-		switch authzMode {
-		case authzmodes.ModeABAC:
-			command = append(command, "--authorization-policy-file="+kubeadmconstants.AuthorizationPolicyPath)
-		case authzmodes.ModeWebhook:
-			command = append(command, "--authorization-webhook-config-file="+kubeadmconstants.AuthorizationWebhookConfigPath)
-		}
+	strset := sets.NewString(modes...)
+
+	// RBAC must always be set, prepend that if not present in the list
+	// TODO(luxas): In v1.8, require the Node and RBAC authorizers to be present
+	// In v1.8 we can utilize the API defaulting/validation to do enforce this,
+	// currently this logic has to be split up into two places, which is sub-optimal.
+	// Cleanup work is possible in and should be done in v1.8
+	if !strset.Has(authzmodes.ModeRBAC) {
+		modes = append([]string{authzmodes.ModeRBAC}, modes...)
+	}
+
+	if strset.Has(authzmodes.ModeABAC) {
+		command = append(command, "--authorization-policy-file="+kubeadmconstants.AuthorizationPolicyPath)
+	}
+	if strset.Has(authzmodes.ModeWebhook) {
+		command = append(command, "--authorization-webhook-config-file="+kubeadmconstants.AuthorizationWebhookConfigPath)
 	}
 
-	command = append(command, "--authorization-mode="+strings.Join(authzModes, ","))
+	command = append(command, "--authorization-mode="+strings.Join(modes, ","))
 	return command
 }
 

cmd/kubeadm/app/master/manifests_test.go

@@ -522,7 +522,7 @@ func TestGetAPIServerCommand(t *testing.T) {
 			expected: []string{
 				"kube-apiserver",
 				"--insecure-port=0",
-				"--admission-control=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds",
+				"--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,ResourceQuota",
 				"--service-cluster-ip-range=bar",
 				"--service-account-key-file=" + testCertsDir + "/sa.pub",
 				"--client-ca-file=" + testCertsDir + "/ca.crt",
@@ -554,7 +554,7 @@ func TestGetAPIServerCommand(t *testing.T) {
 			expected: []string{
 				"kube-apiserver",
 				"--insecure-port=0",
-				"--admission-control=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds",
+				"--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,ResourceQuota",
 				"--service-cluster-ip-range=bar",
 				"--service-account-key-file=" + testCertsDir + "/sa.pub",
 				"--client-ca-file=" + testCertsDir + "/ca.crt",
@@ -587,7 +587,7 @@ func TestGetAPIServerCommand(t *testing.T) {
 			expected: []string{
 				"kube-apiserver",
 				"--insecure-port=0",
-				"--admission-control=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds",
+				"--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,ResourceQuota",
 				"--service-cluster-ip-range=bar",
 				"--service-account-key-file=" + testCertsDir + "/sa.pub",
 				"--client-ca-file=" + testCertsDir + "/ca.crt",
@@ -622,7 +622,7 @@ func TestGetAPIServerCommand(t *testing.T) {
 			expected: []string{
 				"kube-apiserver",
 				"--insecure-port=0",
-				"--admission-control=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds",
+				"--admission-control=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota",
 				"--service-cluster-ip-range=bar",
 				"--service-account-key-file=" + testCertsDir + "/sa.pub",
 				"--client-ca-file=" + testCertsDir + "/ca.crt",
@@ -884,7 +884,15 @@ func TestGetAuthzParameters(t *testing.T) {
 		{
 			authMode: []string{"ABAC", "RBAC", "Webhook"},
 			expected: []string{
-				"--authorization-mode=RBAC,ABAC,Webhook",
+				"--authorization-mode=ABAC,RBAC,Webhook",
+				"--authorization-policy-file=/etc/kubernetes/abac_policy.json",
+				"--authorization-webhook-config-file=/etc/kubernetes/webhook_authz.conf",
+			},
+		},
+		{
+			authMode: []string{"Node", "RBAC", "Webhook", "ABAC"},
+			expected: []string{
+				"--authorization-mode=Node,RBAC,Webhook,ABAC",
 				"--authorization-policy-file=/etc/kubernetes/abac_policy.json",
 				"--authorization-webhook-config-file=/etc/kubernetes/webhook_authz.conf",
 			},

cmd/kubeadm/app/phases/apiconfig/BUILD

@@ -20,6 +20,7 @@ go_library(
         "//pkg/bootstrap/api:go_default_library",
         "//pkg/kubelet/apis:go_default_library",
         "//pkg/util/node:go_default_library",
+        "//pkg/util/version:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",

cmd/kubeadm/app/phases/apiconfig/clusterroles.go

@@ -26,6 +26,7 @@ import (
 	rbac "k8s.io/client-go/pkg/apis/rbac/v1beta1"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	bootstrapapi "k8s.io/kubernetes/pkg/bootstrap/api"
+	"k8s.io/kubernetes/pkg/util/version"
 )
 
 const (
@@ -74,7 +75,7 @@ func CreateServiceAccounts(clientset clientset.Interface) error {
 }
 
 // CreateRBACRules creates the essential RBAC rules for a minimally set-up cluster
-func CreateRBACRules(clientset *clientset.Clientset) error {
+func CreateRBACRules(clientset *clientset.Clientset, k8sVersion *version.Version) error {
 	if err := createRoles(clientset); err != nil {
 		return err
 	}
@@ -87,6 +88,9 @@ func CreateRBACRules(clientset *clientset.Clientset) error {
 	if err := createClusterRoleBindings(clientset); err != nil {
 		return err
 	}
+	if err := deletePermissiveNodesBindingWhenUsingNodeAuthorization(clientset, k8sVersion); err != nil {
+		return fmt.Errorf("failed to remove the permissive 'system:nodes' Group Subject in the 'system:node' ClusterRoleBinding: %v", err)
+	}
 
 	fmt.Println("[apiconfig] Created RBAC rules")
 	return nil
@@ -221,3 +225,37 @@ func createClusterRoleBindings(clientset *clientset.Clientset) error {
 	}
 	return nil
 }
+
+func deletePermissiveNodesBindingWhenUsingNodeAuthorization(clientset *clientset.Clientset, k8sVersion *version.Version) error {
+
+	// If the server version is higher than the Node Authorizer's minimum, try to delete the Group=system:nodes->ClusterRole=system:node binding
+	// which is much more permissive than the Node Authorizer
+	if k8sVersion.AtLeast(kubeadmconstants.MinimumNodeAuthorizerVersion) {
+
+		nodesRoleBinding, err := clientset.RbacV1beta1().ClusterRoleBindings().Get(kubeadmconstants.NodesClusterRoleBinding, metav1.GetOptions{})
+		if err != nil {
+			if apierrors.IsNotFound(err) {
+				// Nothing to do; the RoleBinding doesn't exist
+				return nil
+			}
+			return err
+		}
+
+		newSubjects := []rbac.Subject{}
+		for _, subject := range nodesRoleBinding.Subjects {
+			// Skip the subject that binds to the system:nodes group
+			if subject.Name == kubeadmconstants.NodesGroup && subject.Kind == "Group" {
+				continue
+			}
+			newSubjects = append(newSubjects, subject)
+		}
+
+		nodesRoleBinding.Subjects = newSubjects
+
+		if _, err := clientset.RbacV1beta1().ClusterRoleBindings().Update(nodesRoleBinding); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

cmd/kubeadm/app/phases/kubeconfig/BUILD

@@ -18,6 +18,7 @@ go_library(
         "//cmd/kubeadm/app/constants:go_default_library",
         "//cmd/kubeadm/app/phases/certs/pkiutil:go_default_library",
         "//cmd/kubeadm/app/util/kubeconfig:go_default_library",
+        "//pkg/util/node:go_default_library",
         "//vendor/k8s.io/client-go/tools/clientcmd:go_default_library",
         "//vendor/k8s.io/client-go/tools/clientcmd/api:go_default_library",
         "//vendor/k8s.io/client-go/util/cert:go_default_library",

cmd/kubeadm/app/phases/kubeconfig/kubeconfig.go

@@ -29,6 +29,7 @@ import (
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/pkiutil"
 	kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
+	"k8s.io/kubernetes/pkg/util/node"
 )
 
 // BuildConfigProperties holds some simple information about how this phase should build the KubeConfig object
@@ -54,9 +55,9 @@ type BuildConfigProperties struct {
 // CreateInitKubeConfigFiles is called from the main init and does the work for the default phase behaviour
 func CreateInitKubeConfigFiles(masterEndpoint, pkiDir, outDir string) error {
 
-	hostname, err := os.Hostname()
-	if err != nil {
-		return err
+	nodeName := node.GetHostname("")
+	if len(nodeName) == 0 {
+		return fmt.Errorf("unable to get hostname for master node")
 	}
 
 	// Create a lightweight specification for what the files should look like
@@ -69,7 +70,7 @@ func CreateInitKubeConfigFiles(masterEndpoint, pkiDir, outDir string) error {
 			MakeClientCerts: true,
 		},
 		kubeadmconstants.KubeletKubeConfigFileName: {
-			ClientName:      fmt.Sprintf("system:node:%s", hostname),
+			ClientName:      fmt.Sprintf("system:node:%s", nodeName),
 			APIServer:       masterEndpoint,
 			CertDir:         pkiDir,
 			Organization:    []string{kubeadmconstants.NodesGroup},