Skip to content

K
k3s
Unverified Commit 45db5e72 authored by Kubernetes Submit Queue's avatar Kubernetes Submit Queue Committed by GitHub
Browse files
Options

Merge pull request #56280 from xiangpengzhao/kubeadm-validate-kc

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>. Add validation of kubelet and kube-proxy configuration in kubeadm. **What this PR does / why we need it**: kubeadm has implemented the support for Kubelet Dynamic Configuration. This PR adds validation on the kubelet configuration. kube-proxy validation also added in this PR. **Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*: ref: https://github.com/kubernetes/kubeadm/issues/28 **Special notes for your reviewer**: /cc @luxas @kubernetes/sig-cluster-lifecycle-pr-reviews **Release note**: ```release-note NONE ```
parents 64ccd766 cf97d6f6
Hide whitespace changes
Inline Side-by-side
Showing with 178 additions and 34 deletions
cmd/kubeadm/app/apis/kubeadm/v1alpha1/BUILD
View file @ 45db5e72
package(default_visibility = ["//visibility:public"]) load("@io_bazel_rules_go//go:def.bzl", "go_library")
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library( go_library(
name = "go_default_library", name = "go_default_library",
...@@ -23,9 +18,12 @@ go_library( ...@@ -23,9 +18,12 @@ go_library(
"//conditions:default": [], "//conditions:default": [],
}), }),
importpath = "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1", importpath = "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1",
visibility = ["//visibility:public"],
deps = [ deps = [
"//cmd/kubeadm/app/apis/kubeadm:go_default_library", "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
"//cmd/kubeadm/app/constants:go_default_library", "//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/features:go_default_library",
"//pkg/kubelet/apis/kubeletconfig/scheme:go_default_library",
"//pkg/kubelet/apis/kubeletconfig/v1alpha1:go_default_library", "//pkg/kubelet/apis/kubeletconfig/v1alpha1:go_default_library",
"//pkg/proxy/apis/kubeproxyconfig/scheme:go_default_library", "//pkg/proxy/apis/kubeproxyconfig/scheme:go_default_library",
"//pkg/proxy/apis/kubeproxyconfig/v1alpha1:go_default_library", "//pkg/proxy/apis/kubeproxyconfig/v1alpha1:go_default_library",
...@@ -48,4 +46,5 @@ filegroup( ...@@ -48,4 +46,5 @@ filegroup(
name = "all-srcs", name = "all-srcs",
srcs = [":package-srcs"], srcs = [":package-srcs"],
tags = ["automanaged"], tags = ["automanaged"],
visibility = ["//visibility:public"],
) )
cmd/kubeadm/app/apis/kubeadm/v1alpha1/defaults.go
View file @ 45db5e72
...@@ -23,6 +23,8 @@ import ( ...@@ -23,6 +23,8 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime"
"k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/features"
kubeletscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme"
kubeletconfigv1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1" kubeletconfigv1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1"
kubeproxyscheme "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/scheme" kubeproxyscheme "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/scheme"
kubeproxyconfigv1alpha1 "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/v1alpha1" kubeproxyconfigv1alpha1 "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/v1alpha1"
...@@ -110,7 +112,9 @@ func SetDefaults_MasterConfiguration(obj *MasterConfiguration) { ...@@ -110,7 +112,9 @@ func SetDefaults_MasterConfiguration(obj *MasterConfiguration) {
} }
SetDefaultsEtcdSelfHosted(obj) SetDefaultsEtcdSelfHosted(obj)
SetDefaults_KubeletConfiguration(obj) if features.Enabled(obj.FeatureGates, features.DynamicKubeletConfig) {
SetDefaults_KubeletConfiguration(obj)
}
SetDefaults_ProxyConfiguration(obj) SetDefaults_ProxyConfiguration(obj)
} }
...@@ -200,4 +204,9 @@ func SetDefaults_KubeletConfiguration(obj *MasterConfiguration) { ...@@ -200,4 +204,9 @@ func SetDefaults_KubeletConfiguration(obj *MasterConfiguration) {
if obj.KubeletConfiguration.BaseConfig.CAdvisorPort == nil { if obj.KubeletConfiguration.BaseConfig.CAdvisorPort == nil {
obj.KubeletConfiguration.BaseConfig.CAdvisorPort = utilpointer.Int32Ptr(0) obj.KubeletConfiguration.BaseConfig.CAdvisorPort = utilpointer.Int32Ptr(0)
} }
scheme, _, _ := kubeletscheme.NewSchemeAndCodecs()
if scheme != nil {
scheme.Default(obj.KubeletConfiguration.BaseConfig)
}
} }
cmd/kubeadm/app/apis/kubeadm/validation/BUILD
View file @ 45db5e72
package(default_visibility = ["//visibility:public"]) load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
"go_test",
)
go_test(
name = "go_default_test",
srcs = ["validation_test.go"],
importpath = "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation",
library = ":go_default_library",
deps = [
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
"//pkg/proxy/apis/kubeproxyconfig/v1alpha1:go_default_library",
"//pkg/util/pointer:go_default_library",
"//vendor/github.com/spf13/pflag:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/validation/field:go_default_library",
],
)
go_library( go_library(
name = "go_default_library", name = "go_default_library",
srcs = ["validation.go"], srcs = ["validation.go"],
importpath = "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation", importpath = "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation",
visibility = ["//visibility:public"],
deps = [ deps = [
"//cmd/kubeadm/app/apis/kubeadm:go_default_library", "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
"//cmd/kubeadm/app/constants:go_default_library", "//cmd/kubeadm/app/constants:go_default_library",
...@@ -33,6 +13,9 @@ go_library( ...@@ -33,6 +13,9 @@ go_library(
"//cmd/kubeadm/app/util/token:go_default_library", "//cmd/kubeadm/app/util/token:go_default_library",
"//pkg/apis/core/validation:go_default_library", "//pkg/apis/core/validation:go_default_library",
"//pkg/kubeapiserver/authorizer/modes:go_default_library", "//pkg/kubeapiserver/authorizer/modes:go_default_library",
"//pkg/kubelet/apis/kubeletconfig:go_default_library",
"//pkg/kubelet/apis/kubeletconfig/scheme:go_default_library",
"//pkg/kubelet/apis/kubeletconfig/validation:go_default_library",
"//pkg/proxy/apis/kubeproxyconfig:go_default_library", "//pkg/proxy/apis/kubeproxyconfig:go_default_library",
"//pkg/proxy/apis/kubeproxyconfig/scheme:go_default_library", "//pkg/proxy/apis/kubeproxyconfig/scheme:go_default_library",
"//pkg/proxy/apis/kubeproxyconfig/validation:go_default_library", "//pkg/proxy/apis/kubeproxyconfig/validation:go_default_library",
...@@ -45,6 +28,22 @@ go_library( ...@@ -45,6 +28,22 @@ go_library(
], ],
) )
go_test(
name = "go_default_test",
srcs = ["validation_test.go"],
importpath = "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation",
library = ":go_default_library",
deps = [
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
"//pkg/kubelet/apis/kubeletconfig/v1alpha1:go_default_library",
"//pkg/proxy/apis/kubeproxyconfig/v1alpha1:go_default_library",
"//pkg/util/pointer:go_default_library",
"//vendor/github.com/spf13/pflag:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/validation/field:go_default_library",
],
)
filegroup( filegroup(
name = "package-srcs", name = "package-srcs",
srcs = glob(["**"]), srcs = glob(["**"]),
...@@ -56,4 +55,5 @@ filegroup( ...@@ -56,4 +55,5 @@ filegroup(
name = "all-srcs", name = "all-srcs",
srcs = [":package-srcs"], srcs = [":package-srcs"],
tags = ["automanaged"], tags = ["automanaged"],
visibility = ["//visibility:public"],
) )
cmd/kubeadm/app/apis/kubeadm/validation/validation.go
View file @ 45db5e72
...@@ -36,6 +36,9 @@ import ( ...@@ -36,6 +36,9 @@ import (
tokenutil "k8s.io/kubernetes/cmd/kubeadm/app/util/token" tokenutil "k8s.io/kubernetes/cmd/kubeadm/app/util/token"
apivalidation "k8s.io/kubernetes/pkg/apis/core/validation" apivalidation "k8s.io/kubernetes/pkg/apis/core/validation"
authzmodes "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes" authzmodes "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes"
"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
kubeletscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme"
kubeletvalidation "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/validation"
"k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig" "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig"
kubeproxyscheme "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/scheme" kubeproxyscheme "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/scheme"
proxyvalidation "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/validation" proxyvalidation "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/validation"
...@@ -74,7 +77,10 @@ func ValidateMasterConfiguration(c *kubeadm.MasterConfiguration) field.ErrorList ...@@ -74,7 +77,10 @@ func ValidateMasterConfiguration(c *kubeadm.MasterConfiguration) field.ErrorList
allErrs = append(allErrs, ValidateToken(c.Token, field.NewPath("token"))...) allErrs = append(allErrs, ValidateToken(c.Token, field.NewPath("token"))...)
allErrs = append(allErrs, ValidateFeatureGates(c.FeatureGates, field.NewPath("feature-gates"))...) allErrs = append(allErrs, ValidateFeatureGates(c.FeatureGates, field.NewPath("feature-gates"))...)
allErrs = append(allErrs, ValidateAPIEndpoint(c, field.NewPath("api-endpoint"))...) allErrs = append(allErrs, ValidateAPIEndpoint(c, field.NewPath("api-endpoint"))...)
//allErrs = append(allErrs, ValidateProxy(c, field.NewPath("kube-proxy"))...) allErrs = append(allErrs, ValidateProxy(c, field.NewPath("kube-proxy"))...)
if features.Enabled(c.FeatureGates, features.DynamicKubeletConfig) {
allErrs = append(allErrs, ValidateKubeletConfiguration(&c.KubeletConfiguration, field.NewPath("kubeletConfiguration"))...)
}
return allErrs return allErrs
} }
...@@ -369,3 +375,29 @@ func ValidateIgnorePreflightErrors(ignorePreflightErrors []string, skipPreflight ...@@ -369,3 +375,29 @@ func ValidateIgnorePreflightErrors(ignorePreflightErrors []string, skipPreflight
return ignoreErrors, allErrs.ToAggregate() return ignoreErrors, allErrs.ToAggregate()
} }
// ValidateKubeletConfiguration validates kubelet configuration and collects all encountered errors
func ValidateKubeletConfiguration(c *kubeadm.KubeletConfiguration, fldPath *field.Path) field.ErrorList {
allErrs := field.ErrorList{}
scheme, _, err := kubeletscheme.NewSchemeAndCodecs()
if err != nil {
allErrs = append(allErrs, field.Invalid(fldPath, "kubeletConfiguration", err.Error()))
return allErrs
}
// Convert versioned config to internal config
internalcfg := &kubeletconfig.KubeletConfiguration{}
err = scheme.Convert(c.BaseConfig, internalcfg, nil)
if err != nil {
allErrs = append(allErrs, field.Invalid(fldPath, "kubeletConfiguration", err.Error()))
return allErrs
}
err = kubeletvalidation.ValidateKubeletConfiguration(internalcfg)
if err != nil {
allErrs = append(allErrs, field.Invalid(fldPath, "kubeletConfiguration", err.Error()))
}
return allErrs
}
cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go
View file @ 45db5e72
...@@ -25,8 +25,9 @@ import ( ...@@ -25,8 +25,9 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/apimachinery/pkg/util/validation/field"
"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeletconfigv1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1"
kubeproxyconfigv1alpha1 "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/v1alpha1" kubeproxyconfigv1alpha1 "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/v1alpha1"
"k8s.io/kubernetes/pkg/util/pointer" utilpointer "k8s.io/kubernetes/pkg/util/pointer"
) )
func TestValidateTokenDiscovery(t *testing.T) { func TestValidateTokenDiscovery(t *testing.T) {
...@@ -337,10 +338,25 @@ func TestValidateMasterConfiguration(t *testing.T) { ...@@ -337,10 +338,25 @@ func TestValidateMasterConfiguration(t *testing.T) {
}, },
KubeProxy: kubeadm.KubeProxy{ KubeProxy: kubeadm.KubeProxy{
Config: &kubeproxyconfigv1alpha1.KubeProxyConfiguration{ Config: &kubeproxyconfigv1alpha1.KubeProxyConfiguration{
BindAddress: "192.168.59.103",
HealthzBindAddress: "0.0.0.0:10256",
MetricsBindAddress: "127.0.0.1:10249",
ClusterCIDR: "192.168.59.0/24",
UDPIdleTimeout: metav1.Duration{Duration: 1 * time.Second},
ConfigSyncPeriod: metav1.Duration{Duration: 1 * time.Second},
IPTables: kubeproxyconfigv1alpha1.KubeProxyIPTablesConfiguration{
MasqueradeAll: true,
SyncPeriod: metav1.Duration{Duration: 5 * time.Second},
MinSyncPeriod: metav1.Duration{Duration: 2 * time.Second},
},
IPVS: kubeproxyconfigv1alpha1.KubeProxyIPVSConfiguration{
SyncPeriod: metav1.Duration{Duration: 10 * time.Second},
MinSyncPeriod: metav1.Duration{Duration: 5 * time.Second},
},
Conntrack: kubeproxyconfigv1alpha1.KubeProxyConntrackConfiguration{ Conntrack: kubeproxyconfigv1alpha1.KubeProxyConntrackConfiguration{
Max: pointer.Int32Ptr(2), Max: utilpointer.Int32Ptr(2),
MaxPerCore: pointer.Int32Ptr(1), MaxPerCore: utilpointer.Int32Ptr(1),
Min: pointer.Int32Ptr(1), Min: utilpointer.Int32Ptr(1),
TCPEstablishedTimeout: &metav1.Duration{Duration: 5 * time.Second}, TCPEstablishedTimeout: &metav1.Duration{Duration: 5 * time.Second},
TCPCloseWaitTimeout: &metav1.Duration{Duration: 5 * time.Second}, TCPCloseWaitTimeout: &metav1.Duration{Duration: 5 * time.Second},
}, },
...@@ -361,6 +377,32 @@ func TestValidateMasterConfiguration(t *testing.T) { ...@@ -361,6 +377,32 @@ func TestValidateMasterConfiguration(t *testing.T) {
AdvertiseAddress: "1:2:3::4", AdvertiseAddress: "1:2:3::4",
BindPort: 3446, BindPort: 3446,
}, },
KubeProxy: kubeadm.KubeProxy{
Config: &kubeproxyconfigv1alpha1.KubeProxyConfiguration{
BindAddress: "192.168.59.103",
HealthzBindAddress: "0.0.0.0:10256",
MetricsBindAddress: "127.0.0.1:10249",
ClusterCIDR: "192.168.59.0/24",
UDPIdleTimeout: metav1.Duration{Duration: 1 * time.Second},
ConfigSyncPeriod: metav1.Duration{Duration: 1 * time.Second},
IPTables: kubeproxyconfigv1alpha1.KubeProxyIPTablesConfiguration{
MasqueradeAll: true,
SyncPeriod: metav1.Duration{Duration: 5 * time.Second},
MinSyncPeriod: metav1.Duration{Duration: 2 * time.Second},
},
IPVS: kubeproxyconfigv1alpha1.KubeProxyIPVSConfiguration{
SyncPeriod: metav1.Duration{Duration: 10 * time.Second},
MinSyncPeriod: metav1.Duration{Duration: 5 * time.Second},
},
Conntrack: kubeproxyconfigv1alpha1.KubeProxyConntrackConfiguration{
Max: utilpointer.Int32Ptr(2),
MaxPerCore: utilpointer.Int32Ptr(1),
Min: utilpointer.Int32Ptr(1),
TCPEstablishedTimeout: &metav1.Duration{Duration: 5 * time.Second},
TCPCloseWaitTimeout: &metav1.Duration{Duration: 5 * time.Second},
},
},
},
AuthorizationModes: []string{"Node", "RBAC"}, AuthorizationModes: []string{"Node", "RBAC"},
Networking: kubeadm.Networking{ Networking: kubeadm.Networking{
ServiceSubnet: "2001:db8::1/98", ServiceSubnet: "2001:db8::1/98",
...@@ -502,3 +544,65 @@ func TestValidateIgnorePreflightErrors(t *testing.T) { ...@@ -502,3 +544,65 @@ func TestValidateIgnorePreflightErrors(t *testing.T) {
} }
} }
} }
func TestValidateKubeletConfiguration(t *testing.T) {
successCase := &kubeadm.KubeletConfiguration{
BaseConfig: &kubeletconfigv1alpha1.KubeletConfiguration{
CgroupsPerQOS: utilpointer.BoolPtr(true),
EnforceNodeAllocatable: []string{"pods", "system-reserved", "kube-reserved"},
SystemCgroups: "",
CgroupRoot: "",
CAdvisorPort: utilpointer.Int32Ptr(0),
EventBurst: 10,
EventRecordQPS: utilpointer.Int32Ptr(5),
HealthzPort: utilpointer.Int32Ptr(10248),
ImageGCHighThresholdPercent: utilpointer.Int32Ptr(85),
ImageGCLowThresholdPercent: utilpointer.Int32Ptr(80),
IPTablesDropBit: utilpointer.Int32Ptr(15),
IPTablesMasqueradeBit: utilpointer.Int32Ptr(14),
KubeAPIBurst: 10,
KubeAPIQPS: utilpointer.Int32Ptr(5),
MaxOpenFiles: 1000000,
MaxPods: 110,
OOMScoreAdj: utilpointer.Int32Ptr(-999),
PodsPerCore: 100,
Port: 65535,
ReadOnlyPort: utilpointer.Int32Ptr(0),
RegistryBurst: 10,
RegistryPullQPS: utilpointer.Int32Ptr(5),
},
}
if allErrors := ValidateKubeletConfiguration(successCase, nil); len(allErrors) != 0 {
t.Errorf("failed ValidateKubeletConfiguration: expect no errors but got %v", allErrors)
}
errorCase := &kubeadm.KubeletConfiguration{
BaseConfig: &kubeletconfigv1alpha1.KubeletConfiguration{
CgroupsPerQOS: utilpointer.BoolPtr(false),
EnforceNodeAllocatable: []string{"pods", "system-reserved", "kube-reserved", "illegal-key"},
SystemCgroups: "/",
CgroupRoot: "",
CAdvisorPort: utilpointer.Int32Ptr(-10),
EventBurst: -10,
EventRecordQPS: utilpointer.Int32Ptr(-10),
HealthzPort: utilpointer.Int32Ptr(-10),
ImageGCHighThresholdPercent: utilpointer.Int32Ptr(101),
ImageGCLowThresholdPercent: utilpointer.Int32Ptr(101),
IPTablesDropBit: utilpointer.Int32Ptr(-10),
IPTablesMasqueradeBit: utilpointer.Int32Ptr(-10),
KubeAPIBurst: -10,
KubeAPIQPS: utilpointer.Int32Ptr(-10),
MaxOpenFiles: -10,
MaxPods: -10,
OOMScoreAdj: utilpointer.Int32Ptr(-1001),
PodsPerCore: -10,
Port: 0,
ReadOnlyPort: utilpointer.Int32Ptr(-10),
RegistryBurst: -10,
RegistryPullQPS: utilpointer.Int32Ptr(-10),
},
}
if allErrors := ValidateKubeletConfiguration(errorCase, nil); len(allErrors) == 0 {
t.Errorf("failed ValidateKubeletConfiguration: expect errors but got no error")
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment