Skip to content

K
k3s
Commit 4c22e6bc authored by Jacob Simpson's avatar Jacob Simpson
Browse files
Options

Certificate rotation for kubelet server certs.

Replaces the current kubelet server side self signed certs with certs signed by the Certificate Request Signing API on the API server. Also renews expiring kubelet server certs as expiration approaches.
parent 9801a7da
Hide whitespace changes
Inline Side-by-side
Showing with 145 additions and 10 deletions
cmd/kubelet/app/server.go
View file @ 4c22e6bc
......@@ -288,7 +288,6 @@ func initKubeletConfigSync(s *options.KubeletServer) (*componentconfig.KubeletCo
func Run(s *options.KubeletServer, kubeDeps *kubelet.KubeletDeps) error {
if err := run(s, kubeDeps); err != nil {
return fmt.Errorf("failed to run Kubelet: %v", err)
}
return nil
}
......@@ -623,7 +622,7 @@ func getNodeName(cloud cloudprovider.Interface, hostname string) (types.NodeName
// InitializeTLS checks for a configured TLSCertFile and TLSPrivateKeyFile: if unspecified a new self-signed
// certificate and key file are generated. Returns a configured server.TLSOptions object.
func InitializeTLS(kf *options.KubeletFlags, kc *componentconfig.KubeletConfiguration) (*server.TLSOptions, error) {
if kc.TLSCertFile == "" && kc.TLSPrivateKeyFile == "" {
if !utilfeature.DefaultFeatureGate.Enabled(features.RotateKubeletServerCertificate) && kc.TLSCertFile == "" && kc.TLSPrivateKeyFile == "" {
kc.TLSCertFile = path.Join(kc.CertDirectory, "kubelet.crt")
kc.TLSPrivateKeyFile = path.Join(kc.CertDirectory, "kubelet.key")
......
pkg/features/kube_features.go
View file @ 4c22e6bc
......@@ -89,6 +89,14 @@ const (
// to take advantage of NoExecute Taints and Tolerations.
TaintBasedEvictions utilfeature.Feature = "TaintBasedEvictions"
// owner: @jcbsmpsn
// alpha: v1.7
//
// Gets a server certificate for the kubelet from the Certificate Signing
// Request API instead of generating one self signed and auto rotates the
// certificate as expiration approaches.
RotateKubeletServerCertificate utilfeature.Feature = "RotateKubeletServerCertificate"
// owner: @msau
// alpha: v1.7
//
......@@ -113,6 +121,7 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS
AffinityInAnnotations: {Default: false, PreRelease: utilfeature.Alpha},
Accelerators: {Default: false, PreRelease: utilfeature.Alpha},
TaintBasedEvictions: {Default: false, PreRelease: utilfeature.Alpha},
RotateKubeletServerCertificate: {Default: false, PreRelease: utilfeature.Alpha},
PersistentLocalVolumes: {Default: false, PreRelease: utilfeature.Alpha},
// inherited features from generic apiserver, relisted here to get a conflict if it is changed
......
pkg/kubelet/BUILD
View file @ 4c22e6bc
......@@ -42,16 +42,19 @@ go_library(
"//pkg/api/v1/pod:go_default_library",
"//pkg/api/v1/resource:go_default_library",
"//pkg/api/v1/validation:go_default_library",
"//pkg/apis/certificates/v1beta1:go_default_library",
"//pkg/apis/componentconfig:go_default_library",
"//pkg/apis/componentconfig/v1alpha1:go_default_library",
"//pkg/capabilities:go_default_library",
"//pkg/client/clientset_generated/clientset:go_default_library",
"//pkg/client/clientset_generated/clientset/typed/certificates/v1beta1:go_default_library",
"//pkg/client/listers/core/v1:go_default_library",
"//pkg/cloudprovider:go_default_library",
"//pkg/features:go_default_library",
"//pkg/fieldpath:go_default_library",
"//pkg/kubelet/apis/cri:go_default_library",
"//pkg/kubelet/cadvisor:go_default_library",
"//pkg/kubelet/certificate:go_default_library",
"//pkg/kubelet/cm:go_default_library",
"//pkg/kubelet/config:go_default_library",
"//pkg/kubelet/container:go_default_library",
......
pkg/kubelet/certificate/certificate_store.go
View file @ 4c22e6bc
......@@ -195,6 +195,9 @@ func (s *fileStore) Update(certData, keyData []byte) (*tls.Certificate, error) {
ts := time.Now().Format("2006-01-02-15-04-05")
pemFilename := s.filename(ts)
if err := os.MkdirAll(s.certDirectory, 0755); err != nil {
return nil, fmt.Errorf("could not create directory %q to store certificates: %v", s.certDirectory, err)
}
certPath := filepath.Join(s.certDirectory, pemFilename)
f, err := os.OpenFile(certPath, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0600)
......
pkg/kubelet/kubelet.go
View file @ 4c22e6bc
......@@ -17,6 +17,9 @@ limitations under the License.
package kubelet
import (
"crypto/tls"
"crypto/x509"
"crypto/x509/pkix"
"fmt"
"net"
"net/http"
......@@ -51,14 +54,17 @@ import (
"k8s.io/client-go/util/integer"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/api/v1"
certificates "k8s.io/kubernetes/pkg/apis/certificates/v1beta1"
"k8s.io/kubernetes/pkg/apis/componentconfig"
componentconfigv1alpha1 "k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1"
"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
clientcertificates "k8s.io/kubernetes/pkg/client/clientset_generated/clientset/typed/certificates/v1beta1"
corelisters "k8s.io/kubernetes/pkg/client/listers/core/v1"
"k8s.io/kubernetes/pkg/cloudprovider"
"k8s.io/kubernetes/pkg/features"
internalapi "k8s.io/kubernetes/pkg/kubelet/apis/cri"
"k8s.io/kubernetes/pkg/kubelet/cadvisor"
"k8s.io/kubernetes/pkg/kubelet/certificate"
"k8s.io/kubernetes/pkg/kubelet/cm"
"k8s.io/kubernetes/pkg/kubelet/config"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
......@@ -304,6 +310,8 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Kub
hostname := nodeutil.GetHostname(hostnameOverride)
// Query the cloud provider for our node name, default to hostname
nodeName := types.NodeName(hostname)
cloudIPs := []net.IP{}
cloudNames := []string{}
if kubeDeps.Cloud != nil {
var err error
instances, ok := kubeDeps.Cloud.Instances()
......@@ -317,6 +325,25 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Kub
}
glog.V(2).Infof("cloud provider determined current node name to be %s", nodeName)
if utilfeature.DefaultFeatureGate.Enabled(features.RotateKubeletServerCertificate) {
nodeAddresses, err := instances.NodeAddresses(nodeName)
if err != nil {
return nil, fmt.Errorf("failed to get the addresses of the current instance from the cloud provider: %v", err)
}
for _, nodeAddress := range nodeAddresses {
switch nodeAddress.Type {
case v1.NodeExternalIP, v1.NodeInternalIP:
ip := net.ParseIP(nodeAddress.Address)
if ip != nil && !ip.IsLoopback() {
cloudIPs = append(cloudIPs, ip)
}
case v1.NodeExternalDNS, v1.NodeInternalDNS, v1.NodeHostName:
cloudNames = append(cloudNames, nodeAddress.Address)
}
}
}
}
if kubeDeps.PodConfig == nil {
......@@ -654,6 +681,33 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Kub
klet.statusManager = status.NewManager(klet.kubeClient, klet.podManager, klet)
if utilfeature.DefaultFeatureGate.Enabled(features.RotateKubeletServerCertificate) && kubeDeps.TLSOptions != nil {
var ips []net.IP
cfgAddress := net.ParseIP(kubeCfg.Address)
if cfgAddress == nil || cfgAddress.IsUnspecified() {
if localIPs, err := allLocalIPsWithoutLoopback(); err != nil {
return nil, err
} else {
ips = localIPs
}
} else {
ips = []net.IP{cfgAddress}
}
ips = append(ips, cloudIPs...)
names := append([]string{klet.GetHostname(), hostnameOverride}, cloudNames...)
klet.serverCertificateManager, err = initializeServerCertificateManager(klet.kubeClient, kubeCfg, klet.nodeName, ips, names)
if err != nil {
return nil, fmt.Errorf("failed to initialize certificate manager: %v", err)
}
kubeDeps.TLSOptions.Config.GetCertificate = func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
cert := klet.serverCertificateManager.Current()
if cert == nil {
return nil, fmt.Errorf("no certificate available")
}
return cert, nil
}
}
klet.probeManager = prober.NewManager(
klet.statusManager,
klet.livenessManager,
......@@ -865,6 +919,9 @@ type Kubelet struct {
// Cached MachineInfo returned by cadvisor.
machineInfo *cadvisorapi.MachineInfo
// Handles certificate rotations.
serverCertificateManager certificate.Manager
// Syncs pods statuses with apiserver; also used as a cache of statuses.
statusManager status.Manager
......@@ -1037,6 +1094,62 @@ type Kubelet struct {
dockerLegacyService dockershim.DockerLegacyService
}
func initializeServerCertificateManager(kubeClient clientset.Interface, kubeCfg *componentconfig.KubeletConfiguration, nodeName types.NodeName, ips []net.IP, hostnames []string) (certificate.Manager, error) {
var certSigningRequestClient clientcertificates.CertificateSigningRequestInterface
if kubeClient != nil && kubeClient.Certificates() != nil {
certSigningRequestClient = kubeClient.Certificates().CertificateSigningRequests()
}
certificateStore, err := certificate.NewFileStore(
"kubelet-server",
kubeCfg.CertDirectory,
kubeCfg.CertDirectory,
kubeCfg.TLSCertFile,
kubeCfg.TLSPrivateKeyFile)
if err != nil {
return nil, fmt.Errorf("failed to initialize certificate store: %v", err)
}
return certificate.NewManager(&certificate.Config{
CertificateSigningRequestClient: certSigningRequestClient,
Template: &x509.CertificateRequest{
Subject: pkix.Name{
CommonName: string(nodeName),
Organization: []string{"system:nodes"},
},
DNSNames: hostnames,
IPAddresses: ips,
},
Usages: []certificates.KeyUsage{
certificates.UsageKeyEncipherment,
certificates.UsageServerAuth,
certificates.UsageSigning,
},
CertificateStore: certificateStore,
})
}
func allLocalIPsWithoutLoopback() ([]net.IP, error) {
interfaces, err := net.Interfaces()
if err != nil {
return nil, fmt.Errorf("could not list network interfaces: %v", err)
}
var ips []net.IP
for _, i := range interfaces {
addresses, err := i.Addrs()
if err != nil {
return nil, fmt.Errorf("could not list the addresses for network interface %v: %v\n", i, err)
}
for _, address := range addresses {
switch v := address.(type) {
case *net.IPNet:
if !v.IP.IsLoopback() {
ips = append(ips, v.IP)
}
}
}
}
return ips, nil
}
// setupDataDirs creates:
// 1. the root directory
// 2. the pods directory
......@@ -1100,25 +1213,30 @@ func (kl *Kubelet) StartGarbageCollection() {
// initializeModules will initialize internal modules that do not require the container runtime to be up.
// Note that the modules here must not depend on modules that are not initialized here.
func (kl *Kubelet) initializeModules() error {
// Step 1: Prometheus metrics.
// Prometheus metrics.
metrics.Register(kl.runtimeCache)
// Step 2: Setup filesystem directories.
// Setup filesystem directories.
if err := kl.setupDataDirs(); err != nil {
return err
}
// Step 3: If the container logs directory does not exist, create it.
// If the container logs directory does not exist, create it.
if _, err := os.Stat(ContainerLogsDir); err != nil {
if err := kl.os.MkdirAll(ContainerLogsDir, 0755); err != nil {
glog.Errorf("Failed to create directory %q: %v", ContainerLogsDir, err)
}
}
// Step 4: Start the image manager.
// Start the image manager.
kl.imageManager.Start()
// Step 5: Start container manager.
// Start the certificate manager.
if utilfeature.DefaultFeatureGate.Enabled(features.RotateKubeletServerCertificate) {
kl.serverCertificateManager.Start()
}
// Start container manager.
node, err := kl.getNodeAnyWay()
if err != nil {
return fmt.Errorf("Kubelet failed to get node info: %v", err)
......@@ -1128,17 +1246,17 @@ func (kl *Kubelet) initializeModules() error {
return fmt.Errorf("Failed to start ContainerManager %v", err)
}
// Step 6: Start out of memory watcher.
// Start out of memory watcher.
if err := kl.oomWatcher.Start(kl.nodeRef); err != nil {
return fmt.Errorf("Failed to start OOM watcher %v", err)
}
// Step 7: Initialize GPUs
// Initialize GPUs
if err := kl.gpuManager.Start(); err != nil {
glog.Errorf("Failed to start gpuManager %v", err)
}
// Step 8: Start resource analyzer
// Start resource analyzer
kl.resourceAnalyzer.Start()
return nil
......
pkg/kubelet/server/server.go
View file @ 4c22e6bc
......@@ -134,6 +134,9 @@ func ListenAndServeKubeletServer(
}
if tlsOptions != nil {
s.TLSConfig = tlsOptions.Config
// Passing empty strings as the cert and key files means no
// cert/keys are specified and GetCertificate in the TLSConfig
// should be called instead.
glog.Fatal(s.ListenAndServeTLS(tlsOptions.CertFile, tlsOptions.KeyFile))
} else {
glog.Fatal(s.ListenAndServe())
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment