Commit 131b4195 authored by Michael Taufen's avatar Michael Taufen

Make feature gates loadable from a map[string]bool

Command line flag API remains the same. This allows ComponentConfig structures (e.g. KubeletConfiguration) to express the map structure behind feature gates in a natural way when written as JSON or YAML. For example: KubeletConfiguration Before: ``` apiVersion: kubeletconfig/v1alpha1 kind: KubeletConfiguration featureGates: "DynamicKubeletConfig=true,Accelerators=true" ``` KubeletConfiguration After: ``` apiVersion: kubeletconfig/v1alpha1 kind: KubeletConfiguration featureGates: DynamicKubeletConfig: true Accelerators: true ```
parent 73d1b386
......@@ -312,9 +312,8 @@ func AddKubeletConfigFlags(fs *pflag.FlagSet, c *kubeletconfig.KubeletConfigurat
fs.Int32Var(&c.ImageGCLowThresholdPercent, "image-gc-low-threshold", c.ImageGCLowThresholdPercent, "The percent of disk usage before which image garbage collection is never run. Lowest disk usage to garbage collect to.")
fs.DurationVar(&c.VolumeStatsAggPeriod.Duration, "volume-stats-agg-period", c.VolumeStatsAggPeriod.Duration, "Specifies interval for kubelet to calculate and cache the volume disk usage for all pods and volumes. To disable volume calculations, set to 0.")
fs.StringVar(&c.VolumePluginDir, "volume-plugin-dir", c.VolumePluginDir, "<Warning: Alpha feature> The full path of the directory in which to search for additional third party volume plugins")
fs.StringVar(&c.FeatureGates, "feature-gates", c.FeatureGates, "A set of key=value pairs that describe feature gates for alpha/experimental features. "+
fs.Var(flag.MapStringBool(c.FeatureGates), "feature-gates", "A set of key=value pairs that describe feature gates for alpha/experimental features. "+
"Options are:\n"+strings.Join(utilfeature.DefaultFeatureGate.KnownFeatures(), "\n"))
fs.StringVar(&c.KubeletCgroups, "kubelet-cgroups", c.KubeletCgroups, "Optional absolute name of cgroups to create and run the Kubelet in.")
fs.StringVar(&c.SystemCgroups, "system-cgroups", c.SystemCgroups, "Optional absolute name of cgroups in which to place all non-kernel processes that are not already inside a cgroup under `/`. Empty for no container. Rolling back the flag requires a reboot.")
......
......@@ -238,7 +238,7 @@ func makeEventRecorder(kubeDeps *kubelet.Dependencies, nodeName types.NodeName)
func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies) (err error) {
// Set global feature gates based on the value on the initial KubeletServer
err = utilfeature.DefaultFeatureGate.Set(s.KubeletConfiguration.FeatureGates)
err = utilfeature.DefaultFeatureGate.SetFromMap(s.KubeletConfiguration.FeatureGates)
if err != nil {
return err
}
......
......@@ -65,7 +65,7 @@ func main() {
// TODO(mtaufen): won't need this this once dynamic config is GA
// set feature gates so we can check if dynamic config is enabled
if err := utilfeature.DefaultFeatureGate.Set(defaultConfig.FeatureGates); err != nil {
if err := utilfeature.DefaultFeatureGate.SetFromMap(defaultConfig.FeatureGates); err != nil {
die(err)
}
// validate the initial KubeletFlags, to make sure the dynamic-config-related flags aren't used unless the feature gate is on
......
......@@ -331,9 +331,8 @@ type KubeletConfiguration struct {
// Whitelist of unsafe sysctls or sysctl patterns (ending in *).
// +optional
AllowedUnsafeSysctls []string
// featureGates is a string of comma-separated key=value pairs that describe feature
// gates for alpha/experimental features.
FeatureGates string
// featureGates is a map of feature names to bools that enable or disable alpha/experimental features.
FeatureGates map[string]bool
// Tells the Kubelet to fail to start if swap is enabled on the node.
FailSwapOn bool
// This flag, if set, enables a check prior to mount operations to verify that the required components
......
......@@ -275,6 +275,9 @@ func SetDefaults_KubeletConfiguration(obj *KubeletConfiguration) {
obj.RemoteRuntimeEndpoint = "tcp://localhost:3735"
}
}
if obj.FeatureGates == nil {
obj.FeatureGates = make(map[string]bool)
}
}
func boolVar(b bool) *bool {
......
......@@ -317,9 +317,8 @@ type KubeletConfiguration struct {
// Resource isolation might be lacking and pod might influence each other on the same node.
// +optional
AllowedUnsafeSysctls []string `json:"allowedUnsafeSysctls,omitempty"`
// featureGates is a string of comma-separated key=value pairs that describe feature
// gates for alpha/experimental features.
FeatureGates string `json:"featureGates,omitempty"`
// featureGates is a map of feature names to bools that enable or disable alpha/experimental features.
FeatureGates map[string]bool `json:"featureGates,omitempty"`
// Tells the Kubelet to fail to start if swap is enabled on the node.
FailSwapOn bool `json:"failSwapOn,omitempty"`
// This flag, if set, enables a check prior to mount operations to verify that the required components
......
......@@ -288,7 +288,7 @@ func autoConvert_v1alpha1_KubeletConfiguration_To_kubeletconfig_KubeletConfigura
return err
}
out.AllowedUnsafeSysctls = *(*[]string)(unsafe.Pointer(&in.AllowedUnsafeSysctls))
out.FeatureGates = in.FeatureGates
out.FeatureGates = *(*map[string]bool)(unsafe.Pointer(&in.FeatureGates))
out.FailSwapOn = in.FailSwapOn
out.ExperimentalCheckNodeCapabilitiesBeforeMount = in.ExperimentalCheckNodeCapabilitiesBeforeMount
out.KeepTerminatedPodVolumes = in.KeepTerminatedPodVolumes
......@@ -453,7 +453,7 @@ func autoConvert_kubeletconfig_KubeletConfiguration_To_v1alpha1_KubeletConfigura
return err
}
out.AllowedUnsafeSysctls = *(*[]string)(unsafe.Pointer(&in.AllowedUnsafeSysctls))
out.FeatureGates = in.FeatureGates
out.FeatureGates = *(*map[string]bool)(unsafe.Pointer(&in.FeatureGates))
out.FailSwapOn = in.FailSwapOn
out.ExperimentalCheckNodeCapabilitiesBeforeMount = in.ExperimentalCheckNodeCapabilitiesBeforeMount
out.KeepTerminatedPodVolumes = in.KeepTerminatedPodVolumes
......
......@@ -436,6 +436,13 @@ func (in *KubeletConfiguration) DeepCopyInto(out *KubeletConfiguration) {
*out = make([]string, len(*in))
copy(*out, *in)
}
if in.FeatureGates != nil {
in, out := &in.FeatureGates, &out.FeatureGates
*out = make(map[string]bool, len(*in))
for key, val := range *in {
(*out)[key] = val
}
}
if in.SystemReserved != nil {
in, out := &in.SystemReserved, &out.SystemReserved
*out = make(map[string]string, len(*in))
......
......@@ -193,6 +193,13 @@ func (in *KubeletConfiguration) DeepCopyInto(out *KubeletConfiguration) {
*out = make([]string, len(*in))
copy(*out, *in)
}
if in.FeatureGates != nil {
in, out := &in.FeatureGates, &out.FeatureGates
*out = make(map[string]bool, len(*in))
for key, val := range *in {
(*out)[key] = val
}
}
if in.SystemReserved != nil {
in, out := &in.SystemReserved, &out.SystemReserved
*out = make(ConfigurationMap, len(*in))
......
......@@ -77,6 +77,8 @@ type FeatureGate interface {
// Set parses and stores flag gates for known features
// from a string like feature1=true,feature2=false,...
Set(value string) error
// SetFromMap stores flag gates for known features from a map[string]bool or returns an error
SetFromMap(m map[string]bool) error
// Enabled returns true if the key is enabled.
Enabled(key Feature) bool
// Add adds features to the featureGate.
......@@ -133,7 +135,7 @@ func NewFeatureGate() *featureGate {
return f
}
// Set Parses a string of the form "key1=value1,key2=value2,..." into a
// Set parses a string of the form "key1=value1,key2=value2,..." into a
// map[string]bool of known keys or returns an error.
func (f *featureGate) Set(value string) error {
f.lock.Lock()
......@@ -183,6 +185,42 @@ func (f *featureGate) Set(value string) error {
return nil
}
// SetFromMap stores flag gates for known features from a map[string]bool or returns an error
func (f *featureGate) SetFromMap(m map[string]bool) error {
f.lock.Lock()
defer f.lock.Unlock()
// Copy existing state
known := map[Feature]FeatureSpec{}
for k, v := range f.known.Load().(map[Feature]FeatureSpec) {
known[k] = v
}
enabled := map[Feature]bool{}
for k, v := range f.enabled.Load().(map[Feature]bool) {
enabled[k] = v
}
for k, v := range m {
k := Feature(k)
_, ok := known[k]
if !ok {
return fmt.Errorf("unrecognized key: %s", k)
}
enabled[k] = v
// Handle "special" features like "all alpha gates"
if fn, found := f.special[k]; found {
fn(known, enabled, v)
}
}
// Persist changes
f.known.Store(known)
f.enabled.Store(enabled)
glog.Infof("feature gates: %v", f.enabled)
return nil
}
// String returns a string containing all enabled feature gates, formatted as "key1=value1,key2=value2,...".
func (f *featureGate) String() string {
pairs := []string{}
......
......@@ -8,9 +8,16 @@ load(
go_test(
name = "go_default_test",
srcs = ["namedcertkey_flag_test.go"],
srcs = [
"map_string_bool_test.go",
"namedcertkey_flag_test.go",
],
library = ":go_default_library",
deps = ["//vendor/github.com/spf13/pflag:go_default_library"],
deps = [
"//vendor/github.com/spf13/pflag:go_default_library",
"//vendor/github.com/stretchr/testify/assert:go_default_library",
"//vendor/github.com/stretchr/testify/require:go_default_library",
],
)
go_library(
......@@ -18,6 +25,7 @@ go_library(
srcs = [
"configuration_map.go",
"flags.go",
"map_string_bool.go",
"namedcertkey_flag.go",
"string_flag.go",
"tristate.go",
......
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package flag
import (
"fmt"
"sort"
"strconv"
"strings"
)
type MapStringBool map[string]bool
// String implements github.com/spf13/pflag.Value
func (m MapStringBool) String() string {
pairs := []string{}
for k, v := range m {
pairs = append(pairs, fmt.Sprintf("%s=%t", k, v))
}
sort.Strings(pairs)
return strings.Join(pairs, ",")
}
// Set implements github.com/spf13/pflag.Value
func (m MapStringBool) Set(value string) error {
for _, s := range strings.Split(value, ",") {
if len(s) == 0 {
continue
}
arr := strings.SplitN(s, "=", 2)
if len(arr) != 2 {
return fmt.Errorf("malformed pair, expect string=bool")
}
k := strings.TrimSpace(arr[0])
v := strings.TrimSpace(arr[1])
boolValue, err := strconv.ParseBool(v)
if err != nil {
return fmt.Errorf("invalid value of %s: %s, err: %v", k, v, err)
}
m[k] = boolValue
}
return nil
}
// Type implements github.com/spf13/pflag.Value
func (MapStringBool) Type() string {
return "mapStringBool"
}
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package flag
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestStringMapStringBool(t *testing.T) {
cases := []struct {
desc string
m MapStringBool
expect string
}{
{"empty", MapStringBool{}, ""},
{"one key", MapStringBool{"one": true}, "one=true"},
{"two keys", MapStringBool{"one": true, "two": false}, "one=true,two=false"},
}
for _, c := range cases {
t.Run(c.desc, func(t *testing.T) {
str := c.m.String()
assert.Equal(t, c.expect, str)
})
}
}
func TestSetMapStringBool(t *testing.T) {
cases := []struct {
desc string
val string
expect MapStringBool
err string
}{
{"empty", "", MapStringBool{}, ""},
{"one key", "one=true", MapStringBool{"one": true}, ""},
{"two keys", "one=true,two=false", MapStringBool{"one": true, "two": false}, ""},
{"two keys with space", "one=true, two=false", MapStringBool{"one": true, "two": false}, ""},
{"empty key", "=true", MapStringBool{"": true}, ""},
{"missing value", "one", MapStringBool{}, "malformed pair, expect string=bool"},
{"non-boolean value", "one=foo", MapStringBool{}, `invalid value of one: foo, err: strconv.ParseBool: parsing "foo": invalid syntax`},
}
for _, c := range cases {
t.Run(c.desc, func(t *testing.T) {
m := MapStringBool{}
err := m.Set(c.val)
if c.err != "" {
require.EqualError(t, err, c.err)
} else {
require.NoError(t, err)
}
assert.Equal(t, c.expect, m)
})
}
}
......@@ -27,6 +27,7 @@ go_library(
}),
deps = [
"//pkg/api/v1/pod:go_default_library",
"//pkg/features:go_default_library",
"//pkg/kubelet/apis/cri:go_default_library",
"//pkg/kubelet/apis/cri/v1alpha1/runtime:go_default_library",
"//pkg/kubelet/apis/kubeletconfig:go_default_library",
......@@ -113,6 +114,7 @@ go_test(
deps = [
"//pkg/api:go_default_library",
"//pkg/api/v1/node:go_default_library",
"//pkg/features:go_default_library",
"//pkg/kubelet:go_default_library",
"//pkg/kubelet/apis/kubeletconfig:go_default_library",
"//pkg/kubelet/apis/stats/v1alpha1:go_default_library",
......
......@@ -26,6 +26,7 @@ import (
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager"
"k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
......@@ -35,10 +36,6 @@ import (
. "github.com/onsi/gomega"
)
const (
cpuManagerFeatureGate = "CPUManager=true"
)
// Helper for makeCPUManagerPod().
type ctnAttribute struct {
ctnName string
......@@ -148,11 +145,7 @@ func enableCPUManagerInKubelet(f *framework.Framework) (oldCfg *kubeletconfig.Ku
newCfg := oldCfg.DeepCopy()
// Enable CPU Manager using feature gate.
if newCfg.FeatureGates != "" {
newCfg.FeatureGates = fmt.Sprintf("%s,%s", cpuManagerFeatureGate, newCfg.FeatureGates)
} else {
newCfg.FeatureGates = cpuManagerFeatureGate
}
newCfg.FeatureGates[string(features.CPUManager)] = true
// Set the CPU Manager policy to static.
newCfg.CPUManagerPolicy = string(cpumanager.PolicyStatic)
......
......@@ -23,6 +23,7 @@ import (
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kubeapi "k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
kubelettypes "k8s.io/kubernetes/pkg/kubelet/types"
"k8s.io/kubernetes/test/e2e/framework"
......@@ -43,7 +44,7 @@ var _ = framework.KubeDescribe("CriticalPod [Serial] [Disruptive]", func() {
Context("when we need to admit a critical pod", func() {
tempSetCurrentKubeletConfig(f, func(initialConfig *kubeletconfig.KubeletConfiguration) {
initialConfig.FeatureGates += ", ExperimentalCriticalPodAnnotation=true"
initialConfig.FeatureGates[string(features.ExperimentalCriticalPodAnnotation)] = true
})
It("should be able to create and delete a critical pod", func() {
......
......@@ -25,6 +25,7 @@ import (
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
nodeutil "k8s.io/kubernetes/pkg/api/v1/node"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
stats "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
"k8s.io/kubernetes/pkg/kubelet/cm"
......@@ -133,10 +134,7 @@ var _ = framework.KubeDescribe("LocalStorageAllocatableEviction [Slow] [Serial]
initialConfig.EnforceNodeAllocatable = []string{cm.NodeAllocatableEnforcementKey}
initialConfig.CgroupsPerQOS = true
initialConfig.ExperimentalNodeAllocatableIgnoreEvictionThreshold = false
if initialConfig.FeatureGates != "" {
initialConfig.FeatureGates += ","
}
initialConfig.FeatureGates += "LocalStorageCapacityIsolation=true"
initialConfig.FeatureGates[string(features.LocalStorageCapacityIsolation)] = true
// set evictionHard to be very small, so that only the allocatable eviction threshold triggers
initialConfig.EvictionHard = "nodefs.available<1"
initialConfig.EvictionMinimumReclaim = ""
......@@ -221,10 +219,7 @@ var _ = framework.KubeDescribe("LocalStorageCapacityIsolationEviction [Slow] [Se
evictionTestTimeout := 10 * time.Minute
Context(fmt.Sprintf(testContextFmt, "evictions due to pod local storage violations"), func() {
tempSetCurrentKubeletConfig(f, func(initialConfig *kubeletconfig.KubeletConfiguration) {
if initialConfig.FeatureGates != "" {
initialConfig.FeatureGates += ","
}
initialConfig.FeatureGates += "LocalStorageCapacityIsolation=true"
initialConfig.FeatureGates[string(features.LocalStorageCapacityIsolation)] = true
initialConfig.EvictionHard = ""
})
sizeLimit := resource.MustParse("100Mi")
......
......@@ -26,6 +26,7 @@ import (
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/uuid"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
"k8s.io/kubernetes/test/e2e/framework"
......@@ -45,7 +46,7 @@ var _ = framework.KubeDescribe("NVIDIA GPU Device Plugin [Feature:GPUDevicePlugi
Context("DevicePlugin", func() {
By("Enabling support for Device Plugin")
tempSetCurrentKubeletConfig(f, func(initialConfig *kubeletconfig.KubeletConfiguration) {
initialConfig.FeatureGates += "," + devicePluginFeatureGate
initialConfig.FeatureGates[string(features.DevicePlugins)] = true
})
BeforeEach(func() {
......
......@@ -24,6 +24,7 @@ import (
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
"k8s.io/kubernetes/test/e2e/framework"
......@@ -31,8 +32,6 @@ import (
. "github.com/onsi/gomega"
)
const acceleratorsFeatureGate = "Accelerators=true"
func getGPUsAvailable(f *framework.Framework) int64 {
nodeList, err := f.ClientSet.Core().Nodes().List(metav1.ListOptions{})
framework.ExpectNoError(err, "getting node list")
......@@ -91,14 +90,11 @@ var _ = framework.KubeDescribe("GPU [Serial]", func() {
}
}()
// Enable Accelerators
oldCfg, err = getCurrentKubeletConfig()
framework.ExpectNoError(err)
newCfg := oldCfg.DeepCopy()
if newCfg.FeatureGates != "" {
newCfg.FeatureGates = fmt.Sprintf("%s,%s", acceleratorsFeatureGate, newCfg.FeatureGates)
} else {
newCfg.FeatureGates = acceleratorsFeatureGate
}
newCfg.FeatureGates[string(features.Accelerators)] = true
framework.ExpectNoError(setKubeletConfiguration(f, newCfg))
By("Waiting for GPUs to become available on the local node")
......
......@@ -34,6 +34,7 @@ import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
kubeletscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme"
kubeletconfigv1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1"
......@@ -123,7 +124,11 @@ func isKubeletConfigEnabled(f *framework.Framework) (bool, error) {
if err != nil {
return false, fmt.Errorf("could not determine whether 'DynamicKubeletConfig' feature is enabled, err: %v", err)
}
return strings.Contains(cfgz.FeatureGates, "DynamicKubeletConfig=true"), nil
v, ok := cfgz.FeatureGates[string(features.DynamicKubeletConfig)]
if !ok {
return false, nil
}
return v, nil
}
// Creates or updates the configmap for KubeletConfiguration, waits for the Kubelet to restart
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment