Merge pull request #31647 from mikedanese/register-tainted

cluster/gce/gci/configure-helper.sh

@@ -507,6 +507,7 @@ function start-kubelet {
     if [[ ! -z "${KUBELET_APISERVER:-}" && ! -z "${KUBELET_CERT:-}" && ! -z "${KUBELET_KEY:-}" ]]; then
       flags+=" --api-servers=https://${KUBELET_APISERVER}"
       flags+=" --register-schedulable=false"
+      flags+=" --register-with-taints=node.alpha.kubernetes.io/ismaster=:NoSchedule"
     else
       # Standalone mode (not widely used?)
       flags+=" --pod-cidr=${MASTER_IP_RANGE}"

cluster/saltbase/salt/kubelet/default

@@ -33,7 +33,7 @@
     # running on the master.
     {% if grains.kubelet_api_servers is defined -%}
       {% set api_servers_with_port = "--api-servers=https://" + grains.kubelet_api_servers -%}
-      {% set master_kubelet_args = master_kubelet_args + "--register-schedulable=false" -%}
+      {% set master_kubelet_args = master_kubelet_args + "--register-schedulable=false --register-with-taints=node.alpha.kubernetes.io/ismaster=:NoSchedule" -%}
     {% else -%}
       {% set api_servers_with_port = "" -%}
     {% endif -%}

cmd/kubelet/app/options/BUILD

@@ -20,6 +20,7 @@ go_library(
         "//pkg/apis/componentconfig/v1alpha1:go_default_library",
         "//pkg/util/config:go_default_library",
         "//pkg/util/flag:go_default_library",
+        "//pkg/util/taints:go_default_library",
         "//vendor:github.com/spf13/pflag",
     ],
 )

cmd/kubelet/app/options/options.go

@@ -26,6 +26,7 @@ import (
 	"k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1"
 	utilconfig "k8s.io/kubernetes/pkg/util/config"
 	"k8s.io/kubernetes/pkg/util/flag"
+	utiltaints "k8s.io/kubernetes/pkg/util/taints"
 
 	"github.com/spf13/pflag"
 )
@@ -232,6 +233,8 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) {
 	fs.Var(&s.SystemReserved, "system-reserved", "A set of ResourceName=ResourceQuantity (e.g. cpu=200m,memory=150G) pairs that describe resources reserved for non-kubernetes components. Currently only cpu and memory are supported. See http://kubernetes.io/docs/user-guide/compute-resources for more detail. [default=none]")
 	fs.Var(&s.KubeReserved, "kube-reserved", "A set of ResourceName=ResourceQuantity (e.g. cpu=200m,memory=150G) pairs that describe resources reserved for kubernetes system components. Currently only cpu and memory are supported. See http://kubernetes.io/docs/user-guide/compute-resources for more detail. [default=none]")
 	fs.BoolVar(&s.RegisterSchedulable, "register-schedulable", s.RegisterSchedulable, "Register the node as schedulable. Won't have any effect if register-node is false. [default=true]")
+	fs.MarkDeprecated("register-schedulable", "will be removed in a future version")
+	fs.Var(utiltaints.NewTaintsVar(&s.RegisterWithTaints), "register-with-taints", "Register the node with the given list of taints (comma seperated \"<key>=<value>:<effect>\"). No-op if register-node is false.")
 	fs.StringVar(&s.ContentType, "kube-api-content-type", s.ContentType, "Content type of requests sent to apiserver.")
 	fs.Int32Var(&s.KubeAPIQPS, "kube-api-qps", s.KubeAPIQPS, "QPS to use while talking with kubernetes apiserver")
 	fs.Int32Var(&s.KubeAPIBurst, "kube-api-burst", s.KubeAPIBurst, "Burst to use while talking with kubernetes apiserver")

hack/verify-flags/known-flags.txt

@@ -485,6 +485,7 @@ reconcile-interval
 register-node
 register-retry-count
 register-schedulable
+register-with-taints
 registry-burst
 registry-qps
 reject-methods

pkg/apis/componentconfig/BUILD

@@ -22,6 +22,7 @@ go_library(
     ],
     tags = ["automanaged"],
     deps = [
+        "//pkg/api:go_default_library",
         "//pkg/apis/meta/v1:go_default_library",
         "//pkg/conversion:go_default_library",
         "//pkg/runtime:go_default_library",

pkg/apis/componentconfig/types.go

@@ -17,6 +17,7 @@ limitations under the License.
 package componentconfig
 
 import (
+	"k8s.io/kubernetes/pkg/api"
 	metav1 "k8s.io/kubernetes/pkg/apis/meta/v1"
 	utilconfig "k8s.io/kubernetes/pkg/util/config"
 )
@@ -379,7 +380,12 @@ type KubeletConfiguration struct {
 	ReconcileCIDR bool `json:"reconcileCIDR"`
 	// registerSchedulable tells the kubelet to register the node as
 	// schedulable. Won't have any effect if register-node is false.
+	// DEPRECATED: use registerWithTaints instead
 	RegisterSchedulable bool `json:"registerSchedulable"`
+	// registerWithTaints are an array of taints to add to a node object when
+	// the kubelet registers itself. This only takes effect when registerNode
+	// is true and upon the initial registration of the node.
+	RegisterWithTaints []api.Taint `json:"registerWithTaints"`
 	// contentType is contentType of requests sent to apiserver.
 	ContentType string `json:"contentType"`
 	// kubeAPIQPS is the QPS to use while talking with kubernetes apiserver

pkg/apis/componentconfig/v1alpha1/BUILD

@@ -24,6 +24,7 @@ go_library(
     tags = ["automanaged"],
     deps = [
         "//pkg/api:go_default_library",
+        "//pkg/api/v1:go_default_library",
         "//pkg/apis/componentconfig:go_default_library",
         "//pkg/apis/meta/v1:go_default_library",
         "//pkg/conversion:go_default_library",

pkg/apis/componentconfig/v1alpha1/types.go

@@ -17,6 +17,7 @@ limitations under the License.
 package v1alpha1
 
 import (
+	"k8s.io/kubernetes/pkg/api/v1"
 	metav1 "k8s.io/kubernetes/pkg/apis/meta/v1"
 )
 
@@ -429,7 +430,12 @@ type KubeletConfiguration struct {
 	ReconcileCIDR *bool `json:"reconcileCIDR"`
 	// registerSchedulable tells the kubelet to register the node as
 	// schedulable. Won't have any effect if register-node is false.
+	// DEPRECATED: use registerWithTaints instead
 	RegisterSchedulable *bool `json:"registerSchedulable"`
+	// registerWithTaints are an array of taints to add to a node object when
+	// the kubelet registers itself. This only takes effect when registerNode
+	// is true and upon the initial registration of the node.
+	RegisterWithTaints []v1.Taint `json:"registerWithTaints"`
 	// contentType is contentType of requests sent to apiserver.
 	ContentType string `json:"contentType"`
 	// kubeAPIQPS is the QPS to use while talking with kubernetes apiserver

pkg/apis/componentconfig/v1alpha1/zz_generated.conversion.go

@@ -22,6 +22,7 @@ package v1alpha1
 
 import (
 	api "k8s.io/kubernetes/pkg/api"
+	v1 "k8s.io/kubernetes/pkg/api/v1"
 	componentconfig "k8s.io/kubernetes/pkg/apis/componentconfig"
 	conversion "k8s.io/kubernetes/pkg/conversion"
 	runtime "k8s.io/kubernetes/pkg/runtime"
@@ -368,6 +369,7 @@ func autoConvert_v1alpha1_KubeletConfiguration_To_componentconfig_KubeletConfigu
 	if err := api.Convert_Pointer_bool_To_bool(&in.RegisterSchedulable, &out.RegisterSchedulable, s); err != nil {
 		return err
 	}
+	out.RegisterWithTaints = *(*[]api.Taint)(unsafe.Pointer(&in.RegisterWithTaints))
 	out.ContentType = in.ContentType
 	if err := api.Convert_Pointer_int32_To_int32(&in.KubeAPIQPS, &out.KubeAPIQPS, s); err != nil {
 		return err
@@ -537,6 +539,7 @@ func autoConvert_componentconfig_KubeletConfiguration_To_v1alpha1_KubeletConfigu
 	if err := api.Convert_bool_To_Pointer_bool(&in.RegisterSchedulable, &out.RegisterSchedulable, s); err != nil {
 		return err
 	}
+	out.RegisterWithTaints = *(*[]v1.Taint)(unsafe.Pointer(&in.RegisterWithTaints))
 	out.ContentType = in.ContentType
 	if err := api.Convert_int32_To_Pointer_int32(&in.KubeAPIQPS, &out.KubeAPIQPS, s); err != nil {
 		return err

pkg/apis/componentconfig/v1alpha1/zz_generated.deepcopy.go

@@ -21,6 +21,7 @@ limitations under the License.
 package v1alpha1
 
 import (
+	v1 "k8s.io/kubernetes/pkg/api/v1"
 	conversion "k8s.io/kubernetes/pkg/conversion"
 	runtime "k8s.io/kubernetes/pkg/runtime"
 	reflect "reflect"
@@ -363,6 +364,15 @@ func DeepCopy_v1alpha1_KubeletConfiguration(in interface{}, out interface{}, c *
 		} else {
 			out.RegisterSchedulable = nil
 		}
+		if in.RegisterWithTaints != nil {
+			in, out := &in.RegisterWithTaints, &out.RegisterWithTaints
+			*out = make([]v1.Taint, len(*in))
+			for i := range *in {
+				(*out)[i] = (*in)[i]
+			}
+		} else {
+			out.RegisterWithTaints = nil
+		}
 		out.ContentType = in.ContentType
 		if in.KubeAPIQPS != nil {
 			in, out := &in.KubeAPIQPS, &out.KubeAPIQPS

pkg/apis/componentconfig/zz_generated.deepcopy.go

@@ -21,6 +21,7 @@ limitations under the License.
 package componentconfig
 
 import (
+	api "k8s.io/kubernetes/pkg/api"
 	conversion "k8s.io/kubernetes/pkg/conversion"
 	runtime "k8s.io/kubernetes/pkg/runtime"
 	config "k8s.io/kubernetes/pkg/util/config"
@@ -336,6 +337,15 @@ func DeepCopy_componentconfig_KubeletConfiguration(in interface{}, out interface
 		out.MaxOpenFiles = in.MaxOpenFiles
 		out.ReconcileCIDR = in.ReconcileCIDR
 		out.RegisterSchedulable = in.RegisterSchedulable
+		if in.RegisterWithTaints != nil {
+			in, out := &in.RegisterWithTaints, &out.RegisterWithTaints
+			*out = make([]api.Taint, len(*in))
+			for i := range *in {
+				(*out)[i] = (*in)[i]
+			}
+		} else {
+			out.RegisterWithTaints = nil
+		}
 		out.ContentType = in.ContentType
 		out.KubeAPIQPS = in.KubeAPIQPS
 		out.KubeAPIBurst = in.KubeAPIBurst

pkg/kubectl/cmd/BUILD

@@ -106,6 +106,7 @@ go_library(
         "//pkg/util/intstr:go_default_library",
         "//pkg/util/sets:go_default_library",
         "//pkg/util/strategicpatch:go_default_library",
+        "//pkg/util/taints:go_default_library",
         "//pkg/util/term:go_default_library",
         "//pkg/util/validation:go_default_library",
         "//pkg/util/validation/field:go_default_library",

pkg/kubectl/cmd/taint.go

@@ -35,6 +35,7 @@ import (
 	utilerrors "k8s.io/kubernetes/pkg/util/errors"
 	"k8s.io/kubernetes/pkg/util/sets"
 	"k8s.io/kubernetes/pkg/util/strategicpatch"
+	utiltaints "k8s.io/kubernetes/pkg/util/taints"
 	"k8s.io/kubernetes/pkg/util/validation"
 )
 
@@ -171,28 +172,10 @@ func parseTaints(spec []string) ([]api.Taint, []api.Taint, error) {
 
 	for _, taintSpec := range spec {
 		if strings.Index(taintSpec, "=") != -1 && strings.Index(taintSpec, ":") != -1 {
-			parts := strings.Split(taintSpec, "=")
-			if len(parts) != 2 || len(parts[1]) == 0 || len(validation.IsQualifiedName(parts[0])) > 0 {
-				return nil, nil, fmt.Errorf("invalid taint spec: %v", taintSpec)
+			newTaint, err := utiltaints.ParseTaint(taintSpec)
+			if err != nil {
+				return nil, nil, err
 			}
-
-			parts2 := strings.Split(parts[1], ":")
-			errs := validation.IsValidLabelValue(parts2[0])
-			if len(parts2) != 2 || len(errs) != 0 {
-				return nil, nil, fmt.Errorf("invalid taint spec: %v, %s", taintSpec, strings.Join(errs, "; "))
-			}
-
-			if parts2[1] != string(api.TaintEffectNoSchedule) && parts2[1] != string(api.TaintEffectPreferNoSchedule) {
-				return nil, nil, fmt.Errorf("invalid taint spec: %v, unsupported taint effect", taintSpec)
-			}
-
-			effect := api.TaintEffect(parts2[1])
-			newTaint := api.Taint{
-				Key:    parts[0],
-				Value:  parts2[0],
-				Effect: effect,
-			}
-
 			// validate if taint is unique by <key, effect>
 			if len(uniqueTaints[newTaint.Effect]) > 0 && uniqueTaints[newTaint.Effect].Has(newTaint.Key) {
 				return nil, nil, fmt.Errorf("duplicated taints with the same key and effect: %v", newTaint)

pkg/kubelet/kubelet_node_status.go

@@ -17,6 +17,7 @@ limitations under the License.
 package kubelet
 
 import (
+	"encoding/json"
 	"fmt"
 	"math"
 	"net"
@@ -185,6 +186,16 @@ func (kl *Kubelet) initialNode() (*v1.Node, error) {
 			Unschedulable: !kl.registerSchedulable,
 		},
 	}
+	if len(kl.kubeletConfiguration.RegisterWithTaints) > 0 {
+		annotations := make(map[string]string)
+		b, err := json.Marshal(kl.kubeletConfiguration.RegisterWithTaints)
+		if err != nil {
+			return nil, err
+		}
+		annotations[v1.TaintsAnnotationKey] = string(b)
+		node.ObjectMeta.Annotations = annotations
+
+	}
 	// Initially, set NodeNetworkUnavailable to true.
 	if kl.providerRequiresNetworkingConfiguration() {
 		node.Status.Conditions = append(node.Status.Conditions, v1.NodeCondition{

pkg/util/taints/BUILD

+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_binary",
+    "go_library",
+    "go_test",
+    "cgo_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["taints.go"],
+    tags = ["automanaged"],
+    deps = [
+        "//pkg/api:go_default_library",
+        "//pkg/util/validation:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["taints_test.go"],
+    library = "go_default_library",
+    tags = ["automanaged"],
+    deps = [
+        "//pkg/api:go_default_library",
+        "//vendor:github.com/spf13/pflag",
+    ],
+)

pkg/util/taints/taints.go

+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// package taints implements uitilites for working with taints
+package taints
+
+import (
+	"fmt"
+	"strings"
+
+	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/util/validation"
+)
+
+// ParseTaint parses a taint from a string. Taint must be off the format '<key>=<value>:<effect>'.
+func ParseTaint(st string) (api.Taint, error) {
+	var taint api.Taint
+	parts := strings.Split(st, "=")
+	if len(parts) != 2 || len(parts[1]) == 0 || len(validation.IsQualifiedName(parts[0])) > 0 {
+		return taint, fmt.Errorf("invalid taint spec: %v", st)
+	}
+
+	parts2 := strings.Split(parts[1], ":")
+
+	effect := api.TaintEffect(parts2[1])
+
+	errs := validation.IsValidLabelValue(parts2[0])
+	if len(parts2) != 2 || len(errs) != 0 {
+		return taint, fmt.Errorf("invalid taint spec: %v, %s", st, strings.Join(errs, "; "))
+	}
+
+	if effect != api.TaintEffectNoSchedule && effect != api.TaintEffectPreferNoSchedule {
+		return taint, fmt.Errorf("invalid taint spec: %v, unsupported taint effect", st)
+	}
+
+	taint.Key = parts[0]
+	taint.Value = parts2[0]
+	taint.Effect = effect
+
+	return taint, nil
+}
+
+// NewTaintsVar wraps []api.Taint in a struct that implements flag.Value to allow taints to be
+// bound to command line flags.
+func NewTaintsVar(ptr *[]api.Taint) taintsVar {
+	return taintsVar{
+		ptr: ptr,
+	}
+}
+
+type taintsVar struct {
+	ptr *[]api.Taint
+}
+
+func (t taintsVar) Set(s string) error {
+	sts := strings.Split(s, ",")
+	var taints []api.Taint
+	for _, st := range sts {
+		taint, err := ParseTaint(st)
+		if err != nil {
+			return err
+		}
+		taints = append(taints, taint)
+	}
+	*t.ptr = taints
+	return nil
+}
+
+func (t taintsVar) String() string {
+	if len(*t.ptr) == 0 {
+		return "<nil>"
+	}
+	var taints []string
+	for _, taint := range *t.ptr {
+		taints = append(taints, fmt.Sprintf("%s=%s:%s", taint.Key, taint.Value, taint.Effect))
+	}
+	return strings.Join(taints, ",")
+}
+
+func (t taintsVar) Type() string {
+	return "[]api.Taint"
+}

pkg/util/taints/taints_test.go

+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package taints
+
+import (
+	"reflect"
+	"strings"
+	"testing"
+
+	"k8s.io/kubernetes/pkg/api"
+
+	"github.com/spf13/pflag"
+)
+
+func TestTaintsVar(t *testing.T) {
+	cases := []struct {
+		f   string
+		err bool
+		t   []api.Taint
+	}{
+		{
+			f: "",
+			t: []api.Taint(nil),
+		},
+		{
+			f: "--t=foo=bar:NoSchedule",
+			t: []api.Taint{{Key: "foo", Value: "bar", Effect: "NoSchedule"}},
+		},
+		{
+			f: "--t=foo=bar:NoSchedule,bing=bang:PreferNoSchedule",
+			t: []api.Taint{
+				{Key: "foo", Value: "bar", Effect: api.TaintEffectNoSchedule},
+				{Key: "bing", Value: "bang", Effect: api.TaintEffectPreferNoSchedule},
+			},
+		},
+	}
+
+	for i, c := range cases {
+		args := append([]string{"test"}, strings.Fields(c.f)...)
+		cli := pflag.NewFlagSet("test", pflag.ContinueOnError)
+		var taints []api.Taint
+		cli.Var(NewTaintsVar(&taints), "t", "bar")
+
+		err := cli.Parse(args)
+		if err == nil && c.err {
+			t.Errorf("[%v] expected error", i)
+			continue
+		}
+		if err != nil && !c.err {
+			t.Errorf("[%v] unexpected error: %v", i, err)
+			continue
+		}
+		if !reflect.DeepEqual(c.t, taints) {
+			t.Errorf("[%v] unexpected taints:\n\texpected:\n\t\t%#v\n\tgot:\n\t\t%#v", i, c.t, taints)
+		}
+	}
+
+}