Unverified Commit 5540edc1 authored by Kubernetes Submit Queue's avatar Kubernetes Submit Queue Committed by GitHub

Merge pull request #67944 from fabriziopandini/kubeadm-config-configMap

Automatic merge from submit-queue (batch tested with PRs 63011, 68089, 67944, 68132). If you want to cherry-pick this change to another branch, please follow the instructions here: https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md. Kubeadm upload and fetch of kubeam config v1alpha3 **What this PR does / why we need it**: This PR implements upload and fetch of kubeam config v1alpha3 from cluster. More in detail: In upload, `kubeadm-config` gets - `ClusterConfiguration` (without components config which are already stored in separated ConfigMaps) - `ClusterStatus`(initialised or updated with the API endpoint of the current node) During fetch `InitConfiguration` is composed with: - `ClusterConfiguration` from `kubeadm-config` - The `APIEndpoint` of the current node from `ClusterStatus` in `kubeadm-config` - Component configs from corresponding ConfigMaps **Which issue(s) this PR fixes** : refs https://github.com/kubernetes/kubeadm/issues/911, refs https://github.com/kubernetes/kubeadm/issues/963 **Special notes for your reviewer**: In order to implement this it was necessary to extend current component config management with a new GetFromConfigMap operation. This is implemented in a separated commit " implement component configs GetFromConfigMap". The real change build on this (commi "upload and fetch kubeadm v1alpha3") **Release note**: ```release-note NONE ``` /cc @kubernetes/sig-cluster-lifecycle-pr-reviews /sig cluster-lifecycle /area kubeadm /kind enhancement /assign @luxas /assign @timothysc /cc @chuckha @rosti @neolit123 @liztio
parents 47434899 d9b4b1fc
......@@ -64,7 +64,7 @@ func enforceRequirements(flags *applyPlanFlags, dryRun bool, newK8sVersion strin
// Fetch the configuration from a file or ConfigMap and validate it
fmt.Println("[upgrade/config] Making sure the configuration is correct:")
cfg, err := configutil.FetchConfigFromFileOrCluster(client, os.Stdout, "upgrade/config", flags.cfgPath)
cfg, err := configutil.FetchConfigFromFileOrCluster(client, os.Stdout, "upgrade/config", flags.cfgPath, false)
if err != nil {
if apierrors.IsNotFound(err) {
fmt.Printf("[upgrade/config] In order to upgrade, a ConfigMap called %q in the %s namespace must exist.\n", constants.InitConfigurationConfigMap, metav1.NamespaceSystem)
......
......@@ -149,10 +149,6 @@ func NewCmdUpgradeControlPlane() *cobra.Command {
options.AddKubeConfigFlag(cmd.Flags(), &flags.kubeConfigPath)
cmd.Flags().BoolVar(&flags.dryRun, "dry-run", flags.dryRun, "Do not change any state, just output the actions that would be performed.")
//TODO: following values should retrieved form the kubeadm-config config map; remove as soon as the new config wil be in place
cmd.Flags().StringVar(&flags.advertiseAddress, "apiserver-advertise-address", flags.advertiseAddress, "If the node is joining as a master, the IP address the API Server will advertise it's listening on.")
cmd.Flags().StringVar(&flags.nodeName, "node-name", flags.nodeName, "Specify the node name.")
return cmd
}
......@@ -230,16 +226,11 @@ func RunUpgradeControlPlane(flags *controlplaneUpgradeFlags) error {
waiter := apiclient.NewKubeWaiter(client, upgrade.UpgradeManifestTimeout, os.Stdout)
// Fetches the cluster configuration
cfg, err := configutil.FetchConfigFromFileOrCluster(client, os.Stdout, "upgrade", "")
cfg, err := configutil.FetchConfigFromFileOrCluster(client, os.Stdout, "upgrade", "", false)
if err != nil {
return fmt.Errorf("Unable to fetch the kubeadm-config ConfigMap: %v", err)
}
//TODO: as soon as the new config wil be in place check if the node is a known control plane instance
// and retrive corresponding infos (now are temporary managed as flag)
cfg.NodeRegistration.Name = flags.nodeName
cfg.APIEndpoint.AdvertiseAddress = flags.advertiseAddress
// Rotate API server certificate if needed
if err := upgrade.BackupAPIServerCertIfNeeded(cfg, flags.dryRun); err != nil {
return fmt.Errorf("Unable to rotate API server certificate: %v", err)
......
......@@ -3,6 +3,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
go_library(
name = "go_default_library",
srcs = [
"config.go",
"defaults.go",
"registrations.go",
"scheme.go",
......@@ -19,12 +20,14 @@ go_library(
"//pkg/kubelet/apis/config/validation:go_default_library",
"//pkg/proxy/apis/config:go_default_library",
"//pkg/proxy/apis/config/validation:go_default_library",
"//pkg/util/version:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/validation/field:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//staging/src/k8s.io/kube-proxy/config/v1alpha1:go_default_library",
"//staging/src/k8s.io/kubelet/config/v1beta1:go_default_library",
"//vendor/k8s.io/utils/pointer:go_default_library",
......@@ -47,14 +50,22 @@ filegroup(
go_test(
name = "go_default_test",
srcs = ["validation_test.go"],
srcs = [
"config_test.go",
"validation_test.go",
],
embed = [":go_default_library"],
deps = [
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/util/apiclient:go_default_library",
"//pkg/kubelet/apis/config:go_default_library",
"//pkg/proxy/apis/config:go_default_library",
"//pkg/util/version:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
"//vendor/k8s.io/utils/pointer:go_default_library",
],
)
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package componentconfigs
import (
"fmt"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
clientset "k8s.io/client-go/kubernetes"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
kubeproxyconfig "k8s.io/kubernetes/pkg/proxy/apis/config"
"k8s.io/kubernetes/pkg/util/version"
)
// GetFromKubeletConfigMap returns the pointer to the ComponentConfig API object read from the kubelet-config-version
// ConfigMap map stored in the cluster
func GetFromKubeletConfigMap(client clientset.Interface, version *version.Version) (runtime.Object, error) {
// Read the ConfigMap from the cluster based on what version the kubelet is
configMapName := kubeadmconstants.GetKubeletConfigMapName(version)
kubeletCfg, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(configMapName, metav1.GetOptions{})
if err != nil {
return nil, err
}
kubeletConfigData, ok := kubeletCfg.Data[kubeadmconstants.KubeletBaseConfigurationConfigMapKey]
if !ok {
return nil, fmt.Errorf("unexpected error when reading %s ConfigMap: %s key value pair missing", configMapName, kubeadmconstants.KubeletBaseConfigurationConfigMapKey)
}
// Decodes the kubeletConfigData into the internal component config
obj := &kubeletconfig.KubeletConfiguration{}
err = unmarshalObject(obj, []byte(kubeletConfigData))
if err != nil {
return nil, err
}
return obj, nil
}
// GetFromKubeProxyConfigMap returns the pointer to the ComponentConfig API object read from the kube-proxy
// ConfigMap map stored in the cluster
func GetFromKubeProxyConfigMap(client clientset.Interface, version *version.Version) (runtime.Object, error) {
// Read the ConfigMap from the cluster
kubeproxyCfg, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(kubeadmconstants.KubeProxyConfigMap, metav1.GetOptions{})
if err != nil {
return nil, err
}
kubeproxyConfigData, ok := kubeproxyCfg.Data[kubeadmconstants.KubeProxyConfigMapKey]
if !ok {
return nil, fmt.Errorf("unexpected error when reading %s ConfigMap: %s key value pair missing", kubeadmconstants.KubeProxyConfigMap, kubeadmconstants.KubeProxyConfigMapKey)
}
// Decodes the Config map dat into the internal component config
obj := &kubeproxyconfig.KubeProxyConfiguration{}
err = unmarshalObject(obj, []byte(kubeproxyConfigData))
if err != nil {
return nil, err
}
return obj, nil
}
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package componentconfigs
import (
"testing"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
clientset "k8s.io/client-go/kubernetes"
clientsetfake "k8s.io/client-go/kubernetes/fake"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
"k8s.io/kubernetes/pkg/util/version"
)
var cfgFiles = map[string][]byte{
"Kube-proxy_componentconfig": []byte(`
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
`),
"Kubelet_componentconfig": []byte(`
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
`),
}
func TestGetFromConfigMap(t *testing.T) {
k8sVersion := version.MustParseGeneric("v1.12.0")
var tests = []struct {
name string
component RegistrationKind
configMap *fakeConfigMap
expectedError bool
}{
{
name: "valid kube-proxy",
component: KubeProxyConfigurationKind,
configMap: &fakeConfigMap{
name: kubeadmconstants.KubeProxyConfigMap,
data: map[string]string{
kubeadmconstants.KubeProxyConfigMapKey: string(cfgFiles["Kube-proxy_componentconfig"]),
},
},
},
{
name: "invalid kube-proxy - missing ConfigMap",
component: KubeProxyConfigurationKind,
configMap: nil,
expectedError: true,
},
{
name: "invalid kube-proxy - missing key",
component: KubeProxyConfigurationKind,
configMap: &fakeConfigMap{
name: kubeadmconstants.KubeProxyConfigMap,
data: map[string]string{},
},
expectedError: true,
},
{
name: "valid kubelet",
component: KubeletConfigurationKind,
configMap: &fakeConfigMap{
name: kubeadmconstants.GetKubeletConfigMapName(k8sVersion),
data: map[string]string{
kubeadmconstants.KubeletBaseConfigurationConfigMapKey: string(cfgFiles["Kubelet_componentconfig"]),
},
},
},
{
name: "invalid kubelet - missing ConfigMap",
component: KubeletConfigurationKind,
configMap: nil,
expectedError: true,
},
{
name: "invalid kubelet - missing key",
component: KubeletConfigurationKind,
configMap: &fakeConfigMap{
name: kubeadmconstants.GetKubeletConfigMapName(k8sVersion),
data: map[string]string{},
},
expectedError: true,
},
}
for _, rt := range tests {
t.Run(rt.name, func(t2 *testing.T) {
client := clientsetfake.NewSimpleClientset()
if rt.configMap != nil {
err := rt.configMap.create(client)
if err != nil {
t.Errorf("unexpected create ConfigMap %s", rt.configMap.name)
return
}
}
registration := Known[rt.component]
obj, err := registration.GetFromConfigMap(client, k8sVersion)
if rt.expectedError != (err != nil) {
t.Errorf("unexpected return err from GetFromConfigMap: %v", err)
return
}
if rt.expectedError {
return
}
if obj == nil {
t.Error("unexpected nil return value")
}
})
}
}
type fakeConfigMap struct {
name string
data map[string]string
}
func (c *fakeConfigMap) create(client clientset.Interface) error {
return apiclient.CreateOrUpdateConfigMap(client, &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: c.name,
Namespace: metav1.NamespaceSystem,
},
Data: c.data,
})
}
......@@ -20,12 +20,14 @@ import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/validation/field"
clientset "k8s.io/client-go/kubernetes"
kubeproxyconfigv1alpha1 "k8s.io/kube-proxy/config/v1alpha1"
kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
kubeproxyconfig "k8s.io/kubernetes/pkg/proxy/apis/config"
"k8s.io/kubernetes/pkg/util/version"
)
// AddToSchemeFunc is a function that adds known types and API GroupVersions to a scheme
......@@ -47,6 +49,8 @@ type Registration struct {
GetFromInternalConfig func(*kubeadmapi.ClusterConfiguration) (runtime.Object, bool)
// SetToInternalConfig sets the pointer to a ComponentConfig API object embedded in the internal kubeadm config struct
SetToInternalConfig func(runtime.Object, *kubeadmapi.ClusterConfiguration) bool
// GetFromConfigMap returns the pointer to the ComponentConfig API object read from the config map stored in the cluster
GetFromConfigMap func(clientset.Interface, *version.Version) (runtime.Object, error)
}
// Marshal marshals obj to bytes for the current Registration
......@@ -60,13 +64,20 @@ func (r Registration) Unmarshal(fileContent []byte) (runtime.Object, error) {
obj := r.EmptyValue.DeepCopyObject()
// Decode the file content into obj which is a pointer to an empty struct of the internal ComponentConfig
// object, using the componentconfig Codecs that knows about all APIs
if err := runtime.DecodeInto(Codecs.UniversalDecoder(), fileContent, obj); err != nil {
if err := unmarshalObject(obj, fileContent); err != nil {
return nil, err
}
return obj, nil
}
func unmarshalObject(obj runtime.Object, fileContent []byte) error {
// Decode the file content using the componentconfig Codecs that knows about all APIs
if err := runtime.DecodeInto(Codecs.UniversalDecoder(), fileContent, obj); err != nil {
return err
}
return nil
}
const (
// KubeletConfigurationKind is the kind for the kubelet ComponentConfig
KubeletConfigurationKind RegistrationKind = "KubeletConfiguration"
......@@ -99,6 +110,7 @@ var Known Registrations = map[RegistrationKind]Registration{
}
return ok
},
GetFromConfigMap: GetFromKubeProxyConfigMap,
},
KubeletConfigurationKind: {
MarshalGroupVersion: kubeletconfigv1beta1.SchemeGroupVersion,
......@@ -116,6 +128,7 @@ var Known Registrations = map[RegistrationKind]Registration{
}
return ok
},
GetFromConfigMap: GetFromKubeletConfigMap,
},
}
......
......@@ -154,6 +154,8 @@ const (
MastersGroup = "system:masters"
// NodesGroup defines the well-known group for all nodes.
NodesGroup = "system:nodes"
// NodesUserPrefix defines the user name prefix as requested by the Node authorizer.
NodesUserPrefix = "system:node:"
// NodesClusterRoleBinding defines the well-known ClusterRoleBinding which binds the too permissive system:node
// ClusterRole to the system:nodes group. Since kubeadm is using the Node Authorizer, this ClusterRoleBinding's
// system:nodes group subject is removed if present.
......@@ -187,13 +189,24 @@ const (
AnnotationKubeadmCRISocket = "kubeadm.alpha.kubernetes.io/cri-socket"
// InitConfigurationConfigMap specifies in what ConfigMap in the kube-system namespace the `kubeadm init` configuration should be stored
// TODO: Rename this to ClusterConfigurationConfigMap
// TODO: Rename this to KubeadmConfigConfigMap
InitConfigurationConfigMap = "kubeadm-config"
// InitConfigurationConfigMapKey specifies in what ConfigMap key the master configuration should be stored
// TODO: Rename this to ClusterConfigurationConfigMapKey, and migrate the value to ClusterConfiguration,
// but still support the old MasterConfiguration naming in earlier versions
InitConfigurationConfigMapKey = "InitConfiguration"
// TODO: This was used in v1.11 with vi1alpha2 config and older. Remove in v1.13
InitConfigurationConfigMapKey = "MasterConfiguration"
// ClusterConfigurationConfigMapKey specifies in what ConfigMap key the cluster configuration should be stored
ClusterConfigurationConfigMapKey = "ClusterConfiguration"
// ClusterStatusConfigMapKey specifies in what ConfigMap key the cluster status should be stored
ClusterStatusConfigMapKey = "ClusterStatus"
// KubeProxyConfigMap specifies in what ConfigMap in the kube-system namespace the kube-proxy configuration should be stored
KubeProxyConfigMap = "kube-proxy"
// KubeProxyConfigMapKey specifies in what ConfigMap key the component config of kube-proxy should be stored
KubeProxyConfigMapKey = "config.conf"
// KubeletBaseConfigurationConfigMapPrefix specifies in what ConfigMap in the kube-system namespace the initial remote configuration of kubelet should be stored
KubeletBaseConfigurationConfigMapPrefix = "kubelet-config-"
......@@ -463,3 +476,8 @@ func GetDNSVersion(dnsType string) string {
return KubeDNSVersion
}
}
// GetKubeletConfigMapName returns the right ConfigMap name for the right branch of k8s
func GetKubeletConfigMapName(k8sVersion *version.Version) string {
return fmt.Sprintf("%s%d.%d", KubeletBaseConfigurationConfigMapPrefix, k8sVersion.Major(), k8sVersion.Minor())
}
......@@ -39,6 +39,7 @@ go_library(
"//cmd/kubeadm/app/images:go_default_library",
"//cmd/kubeadm/app/util:go_default_library",
"//cmd/kubeadm/app/util/apiclient:go_default_library",
"//pkg/apis/rbac/v1:go_default_library",
"//staging/src/k8s.io/api/apps/v1:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/api/rbac/v1:go_default_library",
......
......@@ -22,7 +22,7 @@ const (
kind: ConfigMap
apiVersion: v1
metadata:
name: kube-proxy
name: {{ .ProxyConfigMap }}
namespace: kube-system
labels:
app: kube-proxy
......@@ -46,7 +46,7 @@ data:
- name: default
user:
tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
config.conf: |-
{{ .ProxyConfigMapKey }}: |-
{{ .ProxyConfig}}
`
......@@ -79,7 +79,7 @@ spec:
imagePullPolicy: IfNotPresent
command:
- /usr/local/bin/kube-proxy
- --config=/var/lib/kube-proxy/config.conf
- --config=/var/lib/kube-proxy/{{ .ProxyConfigMapKey }}
securityContext:
privileged: true
volumeMounts:
......@@ -96,7 +96,7 @@ spec:
volumes:
- name: kube-proxy
configMap:
name: kube-proxy
name: {{ .ProxyConfigMap }}
- name: xtables-lock
hostPath:
path: /run/xtables.lock
......
......@@ -33,6 +33,7 @@ import (
"k8s.io/kubernetes/cmd/kubeadm/app/images"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
rbachelper "k8s.io/kubernetes/pkg/apis/rbac/v1"
)
const (
......@@ -65,17 +66,23 @@ func EnsureProxyAddon(cfg *kubeadmapi.InitConfiguration, client clientset.Interf
var proxyConfigMapBytes, proxyDaemonSetBytes []byte
proxyConfigMapBytes, err = kubeadmutil.ParseTemplate(KubeProxyConfigMap19,
struct {
MasterEndpoint string
ProxyConfig string
MasterEndpoint string
ProxyConfig string
ProxyConfigMap string
ProxyConfigMapKey string
}{
MasterEndpoint: masterEndpoint,
ProxyConfig: prefixBytes.String(),
MasterEndpoint: masterEndpoint,
ProxyConfig: prefixBytes.String(),
ProxyConfigMap: constants.KubeProxyConfigMap,
ProxyConfigMapKey: constants.KubeProxyConfigMapKey,
})
if err != nil {
return fmt.Errorf("error when parsing kube-proxy configmap template: %v", err)
}
proxyDaemonSetBytes, err = kubeadmutil.ParseTemplate(KubeProxyDaemonSet19, struct{ Image string }{
Image: images.GetKubeControlPlaneImage(constants.KubeProxy, &cfg.ClusterConfiguration),
proxyDaemonSetBytes, err = kubeadmutil.ParseTemplate(KubeProxyDaemonSet19, struct{ Image, ProxyConfigMap, ProxyConfigMapKey string }{
Image: images.GetKubeControlPlaneImage(constants.KubeProxy, &cfg.ClusterConfiguration),
ProxyConfigMap: constants.KubeProxyConfigMap,
ProxyConfigMapKey: constants.KubeProxyConfigMapKey,
})
if err != nil {
return fmt.Errorf("error when parsing kube-proxy daemonset template: %v", err)
......@@ -128,7 +135,7 @@ func createKubeProxyAddon(configMapBytes, daemonSetbytes []byte, client clientse
}
func createClusterRoleBindings(client clientset.Interface) error {
return apiclient.CreateOrUpdateClusterRoleBinding(client, &rbac.ClusterRoleBinding{
if err := apiclient.CreateOrUpdateClusterRoleBinding(client, &rbac.ClusterRoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: "kubeadm:node-proxier",
},
......@@ -144,5 +151,39 @@ func createClusterRoleBindings(client clientset.Interface) error {
Namespace: metav1.NamespaceSystem,
},
},
}); err != nil {
return err
}
// Create a role for granting read only access to the kube-proxy component config ConfigMap
if err := apiclient.CreateOrUpdateRole(client, &rbac.Role{
ObjectMeta: metav1.ObjectMeta{
Name: constants.KubeProxyConfigMap,
Namespace: metav1.NamespaceSystem,
},
Rules: []rbac.PolicyRule{
rbachelper.NewRule("get").Groups("").Resources("configmaps").Names(constants.KubeProxyConfigMap).RuleOrDie(),
},
}); err != nil {
return err
}
// Bind the role to bootstrap tokens for allowing fetchConfiguration during join
return apiclient.CreateOrUpdateRoleBinding(client, &rbac.RoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: constants.KubeProxyConfigMap,
Namespace: metav1.NamespaceSystem,
},
RoleRef: rbac.RoleRef{
APIGroup: rbac.GroupName,
Kind: "Role",
Name: constants.KubeProxyConfigMap,
},
Subjects: []rbac.Subject{
{
Kind: rbac.GroupKind,
Name: constants.NodeBootstrapTokenAuthGroup,
},
},
})
}
......@@ -99,17 +99,21 @@ func TestCompileManifests(t *testing.T) {
{
manifest: KubeProxyConfigMap19,
data: struct {
MasterEndpoint, ProxyConfig string
MasterEndpoint, ProxyConfig, ProxyConfigMap, ProxyConfigMapKey string
}{
MasterEndpoint: "foo",
ProxyConfig: " bindAddress: 0.0.0.0\n clusterCIDR: 192.168.1.1\n enableProfiling: false",
MasterEndpoint: "foo",
ProxyConfig: " bindAddress: 0.0.0.0\n clusterCIDR: 192.168.1.1\n enableProfiling: false",
ProxyConfigMap: "bar",
ProxyConfigMapKey: "baz",
},
expected: true,
},
{
manifest: KubeProxyDaemonSet19,
data: struct{ Image string }{
Image: "foo",
data: struct{ Image, ProxyConfigMap, ProxyConfigMapKey string }{
Image: "foo",
ProxyConfigMap: "bar",
ProxyConfigMapKey: "baz",
},
expected: true,
},
......
......@@ -174,7 +174,7 @@ func getKubeConfigSpecs(cfg *kubeadmapi.InitConfiguration) (map[string]*kubeConf
kubeadmconstants.KubeletKubeConfigFileName: {
CACert: caCert,
APIServer: masterEndpoint,
ClientName: fmt.Sprintf("system:node:%s", cfg.NodeRegistration.Name),
ClientName: fmt.Sprintf("%s%s", kubeadmconstants.NodesUserPrefix, cfg.NodeRegistration.Name),
ClientCertAuth: &clientCertAuth{
CAKey: caKey,
Organizations: []string{kubeadmconstants.NodesGroup},
......
......@@ -119,7 +119,7 @@ func TestGetKubeConfigSpecs(t *testing.T) {
},
{
kubeConfigFile: kubeadmconstants.KubeletKubeConfigFileName,
clientName: fmt.Sprintf("system:node:%s", cfg.NodeRegistration.Name),
clientName: fmt.Sprintf("%s%s", kubeadmconstants.NodesUserPrefix, cfg.NodeRegistration.Name),
organizations: []string{kubeadmconstants.NodesGroup},
},
{
......
......@@ -56,7 +56,7 @@ func CreateConfigMap(cfg *kubeadmapi.InitConfiguration, client clientset.Interfa
return err
}
configMapName := configMapName(k8sVersion)
configMapName := kubeadmconstants.GetKubeletConfigMapName(k8sVersion)
fmt.Printf("[kubelet] Creating a ConfigMap %q in namespace %s with the configuration for the kubelets in the cluster\n", configMapName, metav1.NamespaceSystem)
kubeletBytes, err := getConfigBytes(cfg.ComponentConfigs.Kubelet)
......@@ -90,7 +90,7 @@ func createConfigMapRBACRules(client clientset.Interface, k8sVersion *version.Ve
Namespace: metav1.NamespaceSystem,
},
Rules: []rbac.PolicyRule{
rbachelper.NewRule("get").Groups("").Resources("configmaps").Names(configMapName(k8sVersion)).RuleOrDie(),
rbachelper.NewRule("get").Groups("").Resources("configmaps").Names(kubeadmconstants.GetKubeletConfigMapName(k8sVersion)).RuleOrDie(),
},
}); err != nil {
return err
......@@ -124,7 +124,7 @@ func createConfigMapRBACRules(client clientset.Interface, k8sVersion *version.Ve
func DownloadConfig(client clientset.Interface, kubeletVersion *version.Version, kubeletDir string) error {
// Download the ConfigMap from the cluster based on what version the kubelet is
configMapName := configMapName(kubeletVersion)
configMapName := kubeadmconstants.GetKubeletConfigMapName(kubeletVersion)
fmt.Printf("[kubelet] Downloading configuration for the kubelet from the %q ConfigMap in the %s namespace\n",
configMapName, metav1.NamespaceSystem)
......@@ -142,11 +142,6 @@ func DownloadConfig(client clientset.Interface, kubeletVersion *version.Version,
return writeConfigBytesToDisk([]byte(kubeletCfg.Data[kubeadmconstants.KubeletBaseConfigurationConfigMapKey]), kubeletDir)
}
// configMapName returns the right ConfigMap name for the right branch of k8s
func configMapName(k8sVersion *version.Version) string {
return fmt.Sprintf("%s%d.%d", kubeadmconstants.KubeletBaseConfigurationConfigMapPrefix, k8sVersion.Major(), k8sVersion.Minor())
}
// configMapRBACName returns the name for the Role/RoleBinding for the kubelet config configmap for the right branch of k8s
func configMapRBACName(k8sVersion *version.Version) string {
return fmt.Sprintf("%s%d.%d", kubeadmconstants.KubeletBaseConfigMapRolePrefix, k8sVersion.Major(), k8sVersion.Minor())
......
......@@ -32,7 +32,7 @@ import (
// This func is ONLY run if the user enables the `DynamicKubeletConfig` feature gate, which is by default off
func EnableDynamicConfigForNode(client clientset.Interface, nodeName string, kubeletVersion *version.Version) error {
configMapName := configMapName(kubeletVersion)
configMapName := kubeadmconstants.GetKubeletConfigMapName(kubeletVersion)
fmt.Printf("[kubelet] Enabling Dynamic Kubelet Config for Node %q; config sourced from ConfigMap %q in namespace %s\n",
nodeName, configMapName, metav1.NamespaceSystem)
fmt.Println("[kubelet] WARNING: The Dynamic Kubelet Config feature is beta, but off by default. It hasn't been well-tested yet at this stage, use with caution.")
......
......@@ -12,13 +12,16 @@ go_library(
importpath = "k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadconfig",
deps = [
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
"//cmd/kubeadm/app/apis/kubeadm/scheme:go_default_library",
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/util/apiclient:go_default_library",
"//cmd/kubeadm/app/util/config:go_default_library",
"//pkg/apis/rbac/v1:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/api/rbac/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
],
)
......@@ -45,10 +48,13 @@ go_test(
"//cmd/kubeadm/app/apis/kubeadm/scheme:go_default_library",
"//cmd/kubeadm/app/apis/kubeadm/v1alpha3:go_default_library",
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/util/apiclient:go_default_library",
"//cmd/kubeadm/app/util/config:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
"//staging/src/k8s.io/client-go/testing:go_default_library",
],
......
......@@ -21,9 +21,12 @@ import (
"k8s.io/api/core/v1"
rbac "k8s.io/api/rbac/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
clientset "k8s.io/client-go/kubernetes"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
......@@ -39,20 +42,36 @@ const (
// UploadConfiguration saves the InitConfiguration used for later reference (when upgrading for instance)
func UploadConfiguration(cfg *kubeadmapi.InitConfiguration, client clientset.Interface) error {
fmt.Printf("[uploadconfig] storing the configuration used in ConfigMap %q in the %q Namespace\n", kubeadmconstants.InitConfigurationConfigMap, metav1.NamespaceSystem)
// Prepare the ClusterConfiguration for upload
// The components store their config in their own ConfigMaps, then reset the .ComponentConfig struct;
// We don't want to mutate the cfg itself, so create a copy of it using .DeepCopy of it first
clusterConfigToUpload := cfg.ClusterConfiguration.DeepCopy()
// TODO: Reset the .ComponentConfig struct like this:
// cfgToUpload.ComponentConfigs = kubeadmapi.ComponentConfigs{}
// in order to not upload any other components' config to the kubeadm-config
// ConfigMap. The components store their config in their own ConfigMaps.
// Before this line can be uncommented util/config.loadConfigurationBytes()
// needs to support reading the different components' ConfigMaps first.
// Marshal the object into YAML
cfgYaml, err := configutil.MarshalKubeadmConfigObject(clusterConfigToUpload)
clusterConfigurationToUpload := cfg.ClusterConfiguration.DeepCopy()
clusterConfigurationToUpload.ComponentConfigs = kubeadmapi.ComponentConfigs{}
// Marshal the ClusterConfiguration into YAML
clusterConfigurationYaml, err := configutil.MarshalKubeadmConfigObject(clusterConfigurationToUpload)
if err != nil {
return err
}
// Prepare the ClusterStatus for upload
// Gets the current cluster status
// TODO: use configmap locks on this object on the get before the update.
clusterStatus, err := getClusterStatus(client)
if err != nil {
return err
}
// Updates the ClusterStatus with the current control plane instance
if clusterStatus.APIEndpoints == nil {
clusterStatus.APIEndpoints = map[string]kubeadmapi.APIEndpoint{}
}
clusterStatus.APIEndpoints[cfg.NodeRegistration.Name] = cfg.APIEndpoint
// Marshal the ClusterStatus back into into YAML
clusterStatusYaml, err := configutil.MarshalKubeadmConfigObject(clusterStatus)
if err != nil {
return err
}
......@@ -63,7 +82,8 @@ func UploadConfiguration(cfg *kubeadmapi.InitConfiguration, client clientset.Int
Namespace: metav1.NamespaceSystem,
},
Data: map[string]string{
kubeadmconstants.InitConfigurationConfigMapKey: string(cfgYaml),
kubeadmconstants.ClusterConfigurationConfigMapKey: string(clusterConfigurationYaml),
kubeadmconstants.ClusterStatusConfigMapKey: string(clusterStatusYaml),
},
})
if err != nil {
......@@ -109,3 +129,22 @@ func UploadConfiguration(cfg *kubeadmapi.InitConfiguration, client clientset.Int
},
})
}
func getClusterStatus(client clientset.Interface) (*kubeadmapi.ClusterStatus, error) {
obj := &kubeadmapi.ClusterStatus{}
// Read the ConfigMap from the cluster
configMap, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(kubeadmconstants.InitConfigurationConfigMap, metav1.GetOptions{})
if apierrors.IsNotFound(err) {
return obj, nil
}
if err != nil {
return nil, err
}
// Decode the file content using the componentconfig Codecs that knows about all APIs
if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), []byte(configMap.Data[kubeadmconstants.ClusterStatusConfigMapKey]), obj); err != nil {
return nil, err
}
return obj, nil
}
......@@ -20,15 +20,18 @@ import (
"reflect"
"testing"
"k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
clientset "k8s.io/client-go/kubernetes"
clientsetfake "k8s.io/client-go/kubernetes/fake"
core "k8s.io/client-go/testing"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
kubeadmapiv1alpha3 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha3"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
)
......@@ -89,6 +92,12 @@ func TestUploadConfiguration(t *testing.T) {
t2.Fatalf("UploadConfiguration() error = %v", err)
}
status := &kubeadmapi.ClusterStatus{
APIEndpoints: map[string]kubeadmapi.APIEndpoint{
"node-foo": cfg.APIEndpoint,
},
}
client := clientsetfake.NewSimpleClientset()
if tt.errOnCreate != nil {
client.PrependReactor("create", "configmaps", func(action core.Action) (bool, runtime.Object, error) {
......@@ -114,9 +123,9 @@ func TestUploadConfiguration(t *testing.T) {
if err != nil {
t2.Fatalf("Fail to query ConfigMap error = %v", err)
}
configData := masterCfg.Data[kubeadmconstants.InitConfigurationConfigMapKey]
configData := masterCfg.Data[kubeadmconstants.ClusterConfigurationConfigMapKey]
if configData == "" {
t2.Fatalf("Fail to find ConfigMap key")
t2.Fatal("Fail to find ClusterConfigurationConfigMapKey key")
}
decodedCfg := &kubeadmapi.ClusterConfiguration{}
......@@ -127,7 +136,80 @@ func TestUploadConfiguration(t *testing.T) {
if !reflect.DeepEqual(decodedCfg, &cfg.ClusterConfiguration) {
t2.Errorf("the initial and decoded ClusterConfiguration didn't match")
}
statusData := masterCfg.Data[kubeadmconstants.ClusterStatusConfigMapKey]
if statusData == "" {
t2.Fatal("failed to find ClusterStatusConfigMapKey key")
}
decodedStatus := &kubeadmapi.ClusterStatus{}
if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), []byte(statusData), decodedStatus); err != nil {
t2.Fatalf("unable to decode status from bytes: %v", err)
}
if !reflect.DeepEqual(decodedStatus, status) {
t2.Error("the initial and decoded ClusterStatus didn't match")
}
}
})
}
}
func TestGetClusterStatus(t *testing.T) {
var tests = []struct {
name string
clusterStatus *kubeadmapi.ClusterStatus
expectedClusterEndpoints int
}{
{
name: "return empty ClusterStatus if cluster kubeadm-config doesn't exist (e.g init)",
expectedClusterEndpoints: 0,
},
{
name: "return ClusterStatus if cluster kubeadm-config exist (e.g upgrade)",
clusterStatus: &kubeadmapi.ClusterStatus{
APIEndpoints: map[string]kubeadmapi.APIEndpoint{
"dummy": {AdvertiseAddress: "1.2.3.4", BindPort: 1234},
},
},
expectedClusterEndpoints: 1,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
client := clientsetfake.NewSimpleClientset()
if tt.clusterStatus != nil {
createConfigMapWithStatus(tt.clusterStatus, client)
}
actual, err := getClusterStatus(client)
if err != nil {
t.Error("GetClusterStatus returned unexpected error")
return
}
if tt.expectedClusterEndpoints != len(actual.APIEndpoints) {
t.Error("actual ClusterStatus doesn't return expected endpoints")
}
})
}
}
// createConfigMapWithStatus create a ConfigMap with ClusterStatus for TestGetClusterStatus
func createConfigMapWithStatus(statusToCreate *kubeadmapi.ClusterStatus, client clientset.Interface) error {
statusYaml, err := configutil.MarshalKubeadmConfigObject(statusToCreate)
if err != nil {
return err
}
return apiclient.CreateOrUpdateConfigMap(client, &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: kubeadmconstants.InitConfigurationConfigMap,
Namespace: metav1.NamespaceSystem,
},
Data: map[string]string{
kubeadmconstants.ClusterStatusConfigMapKey: string(statusYaml),
},
})
}
......@@ -26,13 +26,14 @@ go_library(
"//pkg/util/node:go_default_library",
"//pkg/util/version:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//staging/src/k8s.io/client-go/tools/bootstrap/token/util:go_default_library",
"//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library",
"//staging/src/k8s.io/client-go/util/cert:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
],
)
......@@ -54,6 +55,7 @@ go_test(
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/util:go_default_library",
"//cmd/kubeadm/app/util/apiclient:go_default_library",
"//pkg/util/version:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
......
......@@ -23,6 +23,7 @@ import (
"net"
"reflect"
"sort"
"strconv"
"github.com/golang/glog"
......@@ -51,7 +52,7 @@ func SetInitDynamicDefaults(cfg *kubeadmapi.InitConfiguration) error {
if err := SetAPIEndpointDynamicDefaults(&cfg.APIEndpoint); err != nil {
return err
}
if err := SetClusterDynamicDefaults(&cfg.ClusterConfiguration, cfg.APIEndpoint.AdvertiseAddress); err != nil {
if err := SetClusterDynamicDefaults(&cfg.ClusterConfiguration, cfg.APIEndpoint.AdvertiseAddress, cfg.APIEndpoint.BindPort); err != nil {
return err
}
return nil
......@@ -118,7 +119,7 @@ func SetAPIEndpointDynamicDefaults(cfg *kubeadmapi.APIEndpoint) error {
}
// SetClusterDynamicDefaults checks and sets configuration values for the InitConfiguration object
func SetClusterDynamicDefaults(cfg *kubeadmapi.ClusterConfiguration, advertiseAddress string) error {
func SetClusterDynamicDefaults(cfg *kubeadmapi.ClusterConfiguration, advertiseAddress string, bindPort int32) error {
// Default all the embedded ComponentConfig structs
componentconfigs.Known.Default(cfg)
......@@ -134,6 +135,19 @@ func SetClusterDynamicDefaults(cfg *kubeadmapi.ClusterConfiguration, advertiseAd
return err
}
// If ControlPlaneEndpoint is specified without a port number defaults it to
// the bindPort number of the APIEndpoint.
// This will allow join of additional control plane instances with different bindPort number
if cfg.ControlPlaneEndpoint != "" {
host, port, err := kubeadmutil.ParseHostPort(cfg.ControlPlaneEndpoint)
if err != nil {
return err
}
if port == "" {
cfg.ControlPlaneEndpoint = net.JoinHostPort(host, strconv.FormatInt(int64(bindPort), 10))
}
}
// Downcase SANs. Some domain names (like ELBs) have capitals in them.
LowercaseSANs(cfg.APIServerCertSANs)
return nil
......
......@@ -60,7 +60,9 @@ func GetMasterEndpoint(cfg *kubeadmapi.InitConfiguration) (string, error) {
// if a port is provided within the controlPlaneAddress warn the users we are using it, else use the bindport
if port != "" {
fmt.Println("[endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address")
if port != bindPortString {
fmt.Println("[endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address")
}
} else {
port = bindPortString
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment