Skip to content

K
k3s
Commit 070a7b58 authored by Walter Fender's avatar Walter Fender Committed by Walter Fender
Browse files
Options

Seperate loop and plugin control

Seperate loop and plugin control in the kube-controller-manager. Adding an "--external-plugin" flag to specify a plugin to load when cloud-provider is set to "external". Flag has no effect currently when the cloud-provider is not set to external. The expectation is that the cloud provider and external plugin flags would go away once all cloud providers are on stage 2 cloud-controller-manager solutions. Managing the control loops more directly based on start up flags. Addressing issue brought up by @wlan0 Switched to using the main node controller in CCM. Changes to enable full NodeController to start in CCM. Fix related tests. Unifying some common code between KCM and CCM. Fix related tests and comments. Folded in feedback from @jhorwit2 and @wlan0
parent eddb00e7
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 300 additions and 85 deletions
cmd/BUILD
View file @ 070a7b58
......@@ -13,6 +13,7 @@ filegroup(
":package-srcs",
"//cmd/clicheck:all-srcs",
"//cmd/cloud-controller-manager:all-srcs",
"//cmd/controller-manager/app/options:all-srcs",
"//cmd/gendocs:all-srcs",
"//cmd/genkubedocs:all-srcs",
"//cmd/genman:all-srcs",
......
cmd/cloud-controller-manager/app/controllermanager.go
View file @ 070a7b58
......@@ -147,7 +147,7 @@ func Run(s *options.CloudControllerManagerServer) error {
clientBuilder = rootClientBuilder
}
if err := StartControllers(s, kubeconfig, clientBuilder, stop, recorder, cloud); err != nil {
if err := StartControllers(s, kubeconfig, rootClientBuilder, clientBuilder, stop, recorder, cloud); err != nil {
glog.Fatalf("error running controllers: %v", err)
}
}
......@@ -193,7 +193,7 @@ func Run(s *options.CloudControllerManagerServer) error {
}
// StartControllers starts the cloud specific controller loops.
func StartControllers(s *options.CloudControllerManagerServer, kubeconfig *restclient.Config, clientBuilder controller.ControllerClientBuilder, stop <-chan struct{}, recorder record.EventRecorder, cloud cloudprovider.Interface) error {
func StartControllers(s *options.CloudControllerManagerServer, kubeconfig *restclient.Config, rootClientBuilder, clientBuilder controller.ControllerClientBuilder, stop <-chan struct{}, recorder record.EventRecorder, cloud cloudprovider.Interface) error {
// Function to build the kube client object
client := func(serviceAccountName string) clientset.Interface {
return clientBuilder.ClientOrDie(serviceAccountName)
......@@ -204,7 +204,7 @@ func StartControllers(s *options.CloudControllerManagerServer, kubeconfig *restc
cloud.Initialize(clientBuilder)
}
versionedClient := client("shared-informers")
versionedClient := rootClientBuilder.ClientOrDie("shared-informers")
sharedInformers := informers.NewSharedInformerFactory(versionedClient, resyncPeriod(s)())
// Start the CloudNodeController
......
cmd/cloud-controller-manager/app/options/BUILD
View file @ 070a7b58
......@@ -11,7 +11,7 @@ go_library(
srcs = ["options.go"],
importpath = "k8s.io/kubernetes/cmd/cloud-controller-manager/app/options",
deps = [
"//pkg/apis/componentconfig:go_default_library",
"//cmd/controller-manager/app/options:go_default_library",
"//pkg/client/leaderelectionconfig:go_default_library",
"//pkg/features:go_default_library",
"//pkg/master/ports:go_default_library",
......@@ -40,6 +40,7 @@ go_test(
importpath = "k8s.io/kubernetes/cmd/cloud-controller-manager/app/options",
library = ":go_default_library",
deps = [
"//cmd/controller-manager/app/options:go_default_library",
"//pkg/apis/componentconfig:go_default_library",
"//vendor/github.com/spf13/pflag:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
......
cmd/cloud-controller-manager/app/options/options.go
View file @ 070a7b58
......@@ -21,7 +21,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/kubernetes/pkg/apis/componentconfig"
cmoptions "k8s.io/kubernetes/cmd/controller-manager/app/options"
"k8s.io/kubernetes/pkg/client/leaderelectionconfig"
"k8s.io/kubernetes/pkg/master/ports"
......@@ -33,10 +33,7 @@ import (
// CloudControllerManagerServer is the main context object for the controller manager.
type CloudControllerManagerServer struct {
componentconfig.KubeControllerManagerConfiguration
Master string
Kubeconfig string
cmoptions.ControllerManagerServer
// NodeStatusUpdateFrequency is the frequency at which the controller updates nodes' status
NodeStatusUpdateFrequency metav1.Duration
......@@ -45,22 +42,10 @@ type CloudControllerManagerServer struct {
// NewCloudControllerManagerServer creates a new ExternalCMServer with a default config.
func NewCloudControllerManagerServer() *CloudControllerManagerServer {
s := CloudControllerManagerServer{
// Part of these default values also present in 'cmd/kube-controller-manager/app/options/options.go'.
// Please keep them in sync when doing update.
KubeControllerManagerConfiguration: componentconfig.KubeControllerManagerConfiguration{
Port: ports.CloudControllerManagerPort,
Address: "0.0.0.0",
ConcurrentServiceSyncs: 1,
MinResyncPeriod: metav1.Duration{Duration: 12 * time.Hour},
NodeMonitorPeriod: metav1.Duration{Duration: 5 * time.Second},
ClusterName: "kubernetes",
ConfigureCloudRoutes: true,
ContentType: "application/vnd.kubernetes.protobuf",
KubeAPIQPS: 20.0,
KubeAPIBurst: 30,
LeaderElection: leaderelectionconfig.DefaultLeaderElectionConfiguration(),
ControllerStartInterval: metav1.Duration{Duration: 0 * time.Second},
RouteReconciliationPeriod: metav1.Duration{Duration: 10 * time.Second},
// The common/default are kept in 'cmd/kube-controller-manager/app/options/util.go'.
// Please make common changes there and put anything cloud specific here.
ControllerManagerServer: cmoptions.ControllerManagerServer{
KubeControllerManagerConfiguration: cmoptions.GetDefaultControllerOptions(ports.CloudControllerManagerPort),
},
NodeStatusUpdateFrequency: metav1.Duration{Duration: 5 * time.Minute},
}
......@@ -70,34 +55,14 @@ func NewCloudControllerManagerServer() *CloudControllerManagerServer {
// AddFlags adds flags for a specific ExternalCMServer to the specified FlagSet
func (s *CloudControllerManagerServer) AddFlags(fs *pflag.FlagSet) {
fs.Int32Var(&s.Port, "port", s.Port, "The port that the cloud-controller-manager's http service runs on.")
fs.Var(componentconfig.IPVar{Val: &s.Address}, "address", "The IP address to serve on (set to 0.0.0.0 for all interfaces).")
cmoptions.AddDefaultControllerFlags(&s.ControllerManagerServer, fs)
fs.StringVar(&s.CloudProvider, "cloud-provider", s.CloudProvider, "The provider of cloud services. Cannot be empty.")
fs.StringVar(&s.CloudConfigFile, "cloud-config", s.CloudConfigFile, "The path to the cloud provider configuration file. Empty string for no configuration file.")
fs.BoolVar(&s.AllowUntaggedCloud, "allow-untagged-cloud", false, "Allow the cluster to run without the cluster-id on cloud instances. This is a legacy mode of operation and a cluster-id will be required in the future.")
fs.MarkDeprecated("allow-untagged-cloud", "This flag is deprecated and will be removed in a future release. A cluster-id will be required on cloud instances.")
fs.DurationVar(&s.MinResyncPeriod.Duration, "min-resync-period", s.MinResyncPeriod.Duration, "The resync period in reflectors will be random between MinResyncPeriod and 2*MinResyncPeriod.")
fs.DurationVar(&s.NodeMonitorPeriod.Duration, "node-monitor-period", s.NodeMonitorPeriod.Duration,
"The period for syncing NodeStatus in NodeController.")
fs.DurationVar(&s.NodeStatusUpdateFrequency.Duration, "node-status-update-frequency", s.NodeStatusUpdateFrequency.Duration, "Specifies how often the controller updates nodes' status.")
// TODO: remove --service-account-private-key-file 6 months after 1.8 is released (~1.10)
fs.StringVar(&s.ServiceAccountKeyFile, "service-account-private-key-file", s.ServiceAccountKeyFile, "Filename containing a PEM-encoded private RSA or ECDSA key used to sign service account tokens.")
fs.MarkDeprecated("service-account-private-key-file", "This flag is currently no-op and will be deleted.")
fs.BoolVar(&s.UseServiceAccountCredentials, "use-service-account-credentials", s.UseServiceAccountCredentials, "If true, use individual service account credentials for each controller.")
fs.DurationVar(&s.RouteReconciliationPeriod.Duration, "route-reconciliation-period", s.RouteReconciliationPeriod.Duration, "The period for reconciling routes created for Nodes by cloud provider.")
fs.BoolVar(&s.ConfigureCloudRoutes, "configure-cloud-routes", true, "Should CIDRs allocated by allocate-node-cidrs be configured on the cloud provider.")
fs.BoolVar(&s.EnableProfiling, "profiling", true, "Enable profiling via web interface host:port/debug/pprof/.")
fs.BoolVar(&s.EnableContentionProfiling, "contention-profiling", false, "Enable lock contention profiling, if profiling is enabled.")
fs.StringVar(&s.ClusterCIDR, "cluster-cidr", s.ClusterCIDR, "CIDR Range for Pods in cluster.")
fs.StringVar(&s.ClusterName, "cluster-name", s.ClusterName, "The instance prefix for the cluster.")
fs.BoolVar(&s.AllocateNodeCIDRs, "allocate-node-cidrs", false, "Should CIDRs for Pods be allocated and set on the cloud provider.")
fs.StringVar(&s.Master, "master", s.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig).")
fs.StringVar(&s.Kubeconfig, "kubeconfig", s.Kubeconfig, "Path to kubeconfig file with authorization and master location information.")
fs.StringVar(&s.ContentType, "kube-api-content-type", s.ContentType, "Content type of requests sent to apiserver.")
fs.Float32Var(&s.KubeAPIQPS, "kube-api-qps", s.KubeAPIQPS, "QPS to use while talking with kubernetes apiserver.")
fs.Int32Var(&s.KubeAPIBurst, "kube-api-burst", s.KubeAPIBurst, "Burst to use while talking with kubernetes apiserver.")
fs.DurationVar(&s.ControllerStartInterval.Duration, "controller-start-interval", s.ControllerStartInterval.Duration, "Interval between starting controller managers.")
fs.Int32Var(&s.ConcurrentServiceSyncs, "concurrent-service-syncs", s.ConcurrentServiceSyncs, "The number of services that are allowed to sync concurrently. Larger number = more responsive service management, but more CPU (and network) load")
leaderelectionconfig.BindFlags(&s.LeaderElection, fs)
utilfeature.DefaultFeatureGate.AddFlag(fs)
......
cmd/cloud-controller-manager/app/options/options_test.go
View file @ 070a7b58
......@@ -25,6 +25,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/diff"
cmoptions "k8s.io/kubernetes/cmd/controller-manager/app/options"
"k8s.io/kubernetes/pkg/apis/componentconfig"
)
......@@ -65,34 +66,82 @@ func TestAddFlags(t *testing.T) {
f.Parse(args)
expected := &CloudControllerManagerServer{
KubeControllerManagerConfiguration: componentconfig.KubeControllerManagerConfiguration{
CloudProvider: "gce",
CloudConfigFile: "/cloud-config",
Port: 10000,
Address: "192.168.4.10",
ConcurrentServiceSyncs: 1,
MinResyncPeriod: metav1.Duration{Duration: 100 * time.Minute},
NodeMonitorPeriod: metav1.Duration{Duration: 5 * time.Second},
ClusterName: "k8s",
ConfigureCloudRoutes: false,
AllocateNodeCIDRs: true,
ContentType: "application/vnd.kubernetes.protobuf",
EnableContentionProfiling: true,
KubeAPIQPS: 50.0,
KubeAPIBurst: 100,
LeaderElection: componentconfig.LeaderElectionConfiguration{
ResourceLock: "configmap",
LeaderElect: false,
LeaseDuration: metav1.Duration{Duration: 30 * time.Second},
RenewDeadline: metav1.Duration{Duration: 15 * time.Second},
RetryPeriod: metav1.Duration{Duration: 5 * time.Second},
ControllerManagerServer: cmoptions.ControllerManagerServer{
KubeControllerManagerConfiguration: componentconfig.KubeControllerManagerConfiguration{
CloudProvider: "gce",
CloudConfigFile: "/cloud-config",
Port: 10000,
Address: "192.168.4.10",
ConcurrentEndpointSyncs: 5,
ConcurrentRSSyncs: 5,
ConcurrentResourceQuotaSyncs: 5,
ConcurrentDeploymentSyncs: 5,
ConcurrentDaemonSetSyncs: 2,
ConcurrentJobSyncs: 5,
ConcurrentNamespaceSyncs: 10,
ConcurrentSATokenSyncs: 5,
ConcurrentServiceSyncs: 1,
ConcurrentGCSyncs: 20,
ConcurrentRCSyncs: 5,
MinResyncPeriod: metav1.Duration{Duration: 100 * time.Minute},
NodeMonitorPeriod: metav1.Duration{Duration: 5 * time.Second},
ServiceSyncPeriod: metav1.Duration{Duration: 5 * time.Minute},
ResourceQuotaSyncPeriod: metav1.Duration{Duration: 5 * time.Minute},
NamespaceSyncPeriod: metav1.Duration{Duration: 5 * time.Minute},
PVClaimBinderSyncPeriod: metav1.Duration{Duration: 15 * time.Second},
HorizontalPodAutoscalerSyncPeriod: metav1.Duration{Duration: 30 * time.Second},
HorizontalPodAutoscalerUpscaleForbiddenWindow: metav1.Duration{Duration: 3 * time.Minute},
HorizontalPodAutoscalerDownscaleForbiddenWindow: metav1.Duration{Duration: 5 * time.Minute},
HorizontalPodAutoscalerTolerance: 0.1,
DeploymentControllerSyncPeriod: metav1.Duration{Duration: 30 * time.Second},
PodEvictionTimeout: metav1.Duration{Duration: 5 * time.Minute},
NodeMonitorGracePeriod: metav1.Duration{Duration: 40 * time.Second},
NodeStartupGracePeriod: metav1.Duration{Duration: 1 * time.Minute},
ClusterSigningDuration: metav1.Duration{Duration: 8760 * time.Hour},
ReconcilerSyncLoopPeriod: metav1.Duration{Duration: 1 * time.Minute},
TerminatedPodGCThreshold: 12500,
RegisterRetryCount: 10,
ClusterName: "k8s",
ConfigureCloudRoutes: false,
AllocateNodeCIDRs: true,
EnableGarbageCollector: true,
EnableTaintManager: true,
HorizontalPodAutoscalerUseRESTClients: true,
VolumeConfiguration: componentconfig.VolumeConfiguration{
EnableDynamicProvisioning: true,
EnableHostPathProvisioning: false,
FlexVolumePluginDir: "/usr/libexec/kubernetes/kubelet-plugins/volume/exec/",
PersistentVolumeRecyclerConfiguration: componentconfig.PersistentVolumeRecyclerConfiguration{
MaximumRetry: 3,
MinimumTimeoutNFS: 300,
IncrementTimeoutNFS: 30,
MinimumTimeoutHostPath: 60,
IncrementTimeoutHostPath: 30,
},
},
ContentType: "application/vnd.kubernetes.protobuf",
ClusterSigningCertFile: "/etc/kubernetes/ca/ca.pem",
ClusterSigningKeyFile: "/etc/kubernetes/ca/ca.key",
EnableContentionProfiling: true,
KubeAPIQPS: 50.0,
KubeAPIBurst: 100,
LeaderElection: componentconfig.LeaderElectionConfiguration{
ResourceLock: "configmap",
LeaderElect: false,
LeaseDuration: metav1.Duration{Duration: 30 * time.Second},
RenewDeadline: metav1.Duration{Duration: 15 * time.Second},
RetryPeriod: metav1.Duration{Duration: 5 * time.Second},
},
ControllerStartInterval: metav1.Duration{Duration: 2 * time.Minute},
RouteReconciliationPeriod: metav1.Duration{Duration: 30 * time.Second},
ClusterCIDR: "1.2.3.4/24",
NodeCIDRMaskSize: 24,
CIDRAllocatorType: "RangeAllocator",
Controllers: []string{"*"},
},
ControllerStartInterval: metav1.Duration{Duration: 2 * time.Minute},
RouteReconciliationPeriod: metav1.Duration{Duration: 30 * time.Second},
ClusterCIDR: "1.2.3.4/24",
Kubeconfig: "/kubeconfig",
Master: "192.168.4.20",
},
Kubeconfig: "/kubeconfig",
Master: "192.168.4.20",
NodeStatusUpdateFrequency: metav1.Duration{Duration: 10 * time.Minute},
}
if !reflect.DeepEqual(expected, s) {
......
cmd/controller-manager/app/options/BUILD 0 → 100644
View file @ 070a7b58
load("@io_bazel_rules_go//go:def.bzl", "go_library")
go_library(
name = "go_default_library",
srcs = ["utils.go"],
importpath = "k8s.io/kubernetes/cmd/controller-manager/app/options",
visibility = ["//visibility:public"],
deps = [
"//pkg/apis/componentconfig:go_default_library",
"//pkg/client/leaderelectionconfig:go_default_library",
"//vendor/github.com/cloudflare/cfssl/helpers:go_default_library",
"//vendor/github.com/spf13/pflag:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)
cmd/controller-manager/app/options/utils.go 0 → 100644
View file @ 070a7b58
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package options
import (
"github.com/cloudflare/cfssl/helpers"
"time"
"github.com/spf13/pflag"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/kubernetes/pkg/apis/componentconfig"
"k8s.io/kubernetes/pkg/client/leaderelectionconfig"
)
// ControllerManagerServer is the common structure for a controller manager. It works with GetDefaultControllerOptions
// and AddDefaultControllerFlags to create the common components of kube-controller-manager and cloud-controller-manager.
type ControllerManagerServer struct {
componentconfig.KubeControllerManagerConfiguration
Master string
Kubeconfig string
}
const (
// These defaults are deprecated and exported so that we can warn if
// they are being used.
// DefaultClusterSigningCertFile is deprecated. Do not use.
DefaultClusterSigningCertFile = "/etc/kubernetes/ca/ca.pem"
// DefaultClusterSigningKeyFile is deprecated. Do not use.
DefaultClusterSigningKeyFile = "/etc/kubernetes/ca/ca.key"
)
// GetDefaultControllerOptions returns common/default configuration values for both
// the kube-controller-manager and the cloud-contoller-manager. Any common changes should
// be made here. Any individual changes should be made in that controller.
func GetDefaultControllerOptions(port int32) componentconfig.KubeControllerManagerConfiguration {
return componentconfig.KubeControllerManagerConfiguration{
Controllers: []string{"*"},
Port: port,
Address: "0.0.0.0",
ConcurrentEndpointSyncs: 5,
ConcurrentServiceSyncs: 1,
ConcurrentRCSyncs: 5,
ConcurrentRSSyncs: 5,
ConcurrentDaemonSetSyncs: 2,
ConcurrentJobSyncs: 5,
ConcurrentResourceQuotaSyncs: 5,
ConcurrentDeploymentSyncs: 5,
ConcurrentNamespaceSyncs: 10,
ConcurrentSATokenSyncs: 5,
ServiceSyncPeriod: metav1.Duration{Duration: 5 * time.Minute},
RouteReconciliationPeriod: metav1.Duration{Duration: 10 * time.Second},
ResourceQuotaSyncPeriod: metav1.Duration{Duration: 5 * time.Minute},
NamespaceSyncPeriod: metav1.Duration{Duration: 5 * time.Minute},
PVClaimBinderSyncPeriod: metav1.Duration{Duration: 15 * time.Second},
HorizontalPodAutoscalerSyncPeriod: metav1.Duration{Duration: 30 * time.Second},
HorizontalPodAutoscalerUpscaleForbiddenWindow: metav1.Duration{Duration: 3 * time.Minute},
HorizontalPodAutoscalerDownscaleForbiddenWindow: metav1.Duration{Duration: 5 * time.Minute},
HorizontalPodAutoscalerTolerance: 0.1,
DeploymentControllerSyncPeriod: metav1.Duration{Duration: 30 * time.Second},
MinResyncPeriod: metav1.Duration{Duration: 12 * time.Hour},
RegisterRetryCount: 10,
PodEvictionTimeout: metav1.Duration{Duration: 5 * time.Minute},
NodeMonitorGracePeriod: metav1.Duration{Duration: 40 * time.Second},
NodeStartupGracePeriod: metav1.Duration{Duration: 60 * time.Second},
NodeMonitorPeriod: metav1.Duration{Duration: 5 * time.Second},
ClusterName: "kubernetes",
NodeCIDRMaskSize: 24,
ConfigureCloudRoutes: true,
TerminatedPodGCThreshold: 12500,
VolumeConfiguration: componentconfig.VolumeConfiguration{
EnableHostPathProvisioning: false,
EnableDynamicProvisioning: true,
PersistentVolumeRecyclerConfiguration: componentconfig.PersistentVolumeRecyclerConfiguration{
MaximumRetry: 3,
MinimumTimeoutNFS: 300,
IncrementTimeoutNFS: 30,
MinimumTimeoutHostPath: 60,
IncrementTimeoutHostPath: 30,
},
FlexVolumePluginDir: "/usr/libexec/kubernetes/kubelet-plugins/volume/exec/",
},
ContentType: "application/vnd.kubernetes.protobuf",
KubeAPIQPS: 20.0,
KubeAPIBurst: 30,
LeaderElection: leaderelectionconfig.DefaultLeaderElectionConfiguration(),
ControllerStartInterval: metav1.Duration{Duration: 0 * time.Second},
EnableGarbageCollector: true,
ConcurrentGCSyncs: 20,
ClusterSigningCertFile: DefaultClusterSigningCertFile,
ClusterSigningKeyFile: DefaultClusterSigningKeyFile,
ClusterSigningDuration: metav1.Duration{Duration: helpers.OneYear},
ReconcilerSyncLoopPeriod: metav1.Duration{Duration: 60 * time.Second},
EnableTaintManager: true,
HorizontalPodAutoscalerUseRESTClients: true,
}
}
// AddDefaultControllerFlags adds common/default flags for both the kube and cloud Controller Manager Server to the
// specified FlagSet. Any common changes should be made here. Any individual changes should be made in that controller.
func AddDefaultControllerFlags(s *ControllerManagerServer, fs *pflag.FlagSet) {
fs.Int32Var(&s.Port, "port", s.Port, "The port that the controller-manager's http service runs on.")
fs.Var(componentconfig.IPVar{Val: &s.Address}, "address", "The IP address to serve on (set to 0.0.0.0 for all interfaces).")
fs.BoolVar(&s.UseServiceAccountCredentials, "use-service-account-credentials", s.UseServiceAccountCredentials, "If true, use individual service account credentials for each controller.")
fs.StringVar(&s.CloudConfigFile, "cloud-config", s.CloudConfigFile, "The path to the cloud provider configuration file. Empty string for no configuration file.")
fs.BoolVar(&s.AllowUntaggedCloud, "allow-untagged-cloud", false, "Allow the cluster to run without the cluster-id on cloud instances. This is a legacy mode of operation and a cluster-id will be required in the future.")
fs.MarkDeprecated("allow-untagged-cloud", "This flag is deprecated and will be removed in a future release. A cluster-id will be required on cloud instances.")
fs.DurationVar(&s.RouteReconciliationPeriod.Duration, "route-reconciliation-period", s.RouteReconciliationPeriod.Duration, "The period for reconciling routes created for Nodes by cloud provider.")
fs.DurationVar(&s.MinResyncPeriod.Duration, "min-resync-period", s.MinResyncPeriod.Duration, "The resync period in reflectors will be random between MinResyncPeriod and 2*MinResyncPeriod.")
fs.DurationVar(&s.NodeMonitorPeriod.Duration, "node-monitor-period", s.NodeMonitorPeriod.Duration,
"The period for syncing NodeStatus in NodeController.")
fs.BoolVar(&s.EnableProfiling, "profiling", true, "Enable profiling via web interface host:port/debug/pprof/")
fs.BoolVar(&s.EnableContentionProfiling, "contention-profiling", false, "Enable lock contention profiling, if profiling is enabled.")
fs.StringVar(&s.ClusterName, "cluster-name", s.ClusterName, "The instance prefix for the cluster.")
fs.StringVar(&s.ClusterCIDR, "cluster-cidr", s.ClusterCIDR, "CIDR Range for Pods in cluster. Requires --allocate-node-cidrs to be true")
fs.BoolVar(&s.AllocateNodeCIDRs, "allocate-node-cidrs", false, "Should CIDRs for Pods be allocated and set on the cloud provider.")
fs.StringVar(&s.CIDRAllocatorType, "cidr-allocator-type", "RangeAllocator", "Type of CIDR allocator to use")
fs.BoolVar(&s.ConfigureCloudRoutes, "configure-cloud-routes", true, "Should CIDRs allocated by allocate-node-cidrs be configured on the cloud provider.")
fs.StringVar(&s.Master, "master", s.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig).")
fs.StringVar(&s.Kubeconfig, "kubeconfig", s.Kubeconfig, "Path to kubeconfig file with authorization and master location information.")
fs.StringVar(&s.ContentType, "kube-api-content-type", s.ContentType, "Content type of requests sent to apiserver.")
fs.Float32Var(&s.KubeAPIQPS, "kube-api-qps", s.KubeAPIQPS, "QPS to use while talking with kubernetes apiserver.")
fs.Int32Var(&s.KubeAPIBurst, "kube-api-burst", s.KubeAPIBurst, "Burst to use while talking with kubernetes apiserver.")
fs.DurationVar(&s.ControllerStartInterval.Duration, "controller-start-interval", s.ControllerStartInterval.Duration, "Interval between starting controller managers.")
}
cmd/kube-controller-manager/app/BUILD
View file @ 070a7b58
......@@ -24,6 +24,7 @@ go_library(
],
importpath = "k8s.io/kubernetes/cmd/kube-controller-manager/app",
deps = [
"//cmd/controller-manager/app/options:go_default_library",
"//cmd/kube-controller-manager/app/options:go_default_library",
"//pkg/api/legacyscheme:go_default_library",
"//pkg/apis/apps/install:go_default_library",
......
cmd/kube-controller-manager/app/certificates.go
View file @ 070a7b58
......@@ -27,7 +27,7 @@ import (
"github.com/golang/glog"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/kubernetes/cmd/kube-controller-manager/app/options"
cmoptions "k8s.io/kubernetes/cmd/controller-manager/app/options"
"k8s.io/kubernetes/pkg/controller/certificates/approver"
"k8s.io/kubernetes/pkg/controller/certificates/cleaner"
"k8s.io/kubernetes/pkg/controller/certificates/signer"
......@@ -55,12 +55,12 @@ func startCSRSigningController(ctx ControllerContext) (bool, error) {
_, err := os.Stat(ctx.Options.ClusterSigningCertFile)
certFileExists = !os.IsNotExist(err)
certUsesDefault = (ctx.Options.ClusterSigningCertFile == options.DefaultClusterSigningCertFile)
certUsesDefault = (ctx.Options.ClusterSigningCertFile == cmoptions.DefaultClusterSigningCertFile)
_, err = os.Stat(ctx.Options.ClusterSigningKeyFile)
keyFileExists = !os.IsNotExist(err)
keyUsesDefault = (ctx.Options.ClusterSigningKeyFile == options.DefaultClusterSigningKeyFile)
keyUsesDefault = (ctx.Options.ClusterSigningKeyFile == cmoptions.DefaultClusterSigningKeyFile)
switch {
case (keyFileExists && keyUsesDefault) || (certFileExists && certUsesDefault):
......
cmd/kube-controller-manager/app/controllermanager.go
View file @ 070a7b58
......@@ -71,6 +71,13 @@ const (
ControllerStartJitter = 1.0
)
type ControllerLoopMode int
const (
IncludeCloudLoops ControllerLoopMode = iota
ExternalLoops
)
// NewControllerManagerCommand creates a *cobra.Command object with default parameters
func NewControllerManagerCommand() *cobra.Command {
s := options.NewCMServer()
......@@ -151,7 +158,7 @@ func Run(s *options.CMServer) error {
}
saTokenControllerInitFunc := serviceAccountTokenControllerStarter{rootClientBuilder: rootClientBuilder}.startServiceAccountTokenController
if err := StartControllers(ctx, saTokenControllerInitFunc, NewControllerInitializers()); err != nil {
if err := StartControllers(ctx, saTokenControllerInitFunc, NewControllerInitializers(ctx.LoopMode)); err != nil {
glog.Fatalf("error starting controllers: %v", err)
}
......@@ -262,6 +269,11 @@ type ControllerContext struct {
// It must be initialized and ready to use.
Cloud cloudprovider.Interface
// Control for which control loops to be run
// IncludeCloudLoops is for a kube-controller-manager running all loops
// ExternalLoops is for a kube-controller-manager running with a cloud-controller-manager
LoopMode ControllerLoopMode
// Stop is the stop channel
Stop <-chan struct{}
......@@ -305,7 +317,7 @@ func IsControllerEnabled(name string, disabledByDefaultControllers sets.String,
type InitFunc func(ctx ControllerContext) (bool, error)
func KnownControllers() []string {
ret := sets.StringKeySet(NewControllerInitializers())
ret := sets.StringKeySet(NewControllerInitializers(IncludeCloudLoops))
// add "special" controllers that aren't initialized normally. These controllers cannot be initialized
// using a normal function. The only known special case is the SA token controller which *must* be started
......@@ -329,7 +341,7 @@ const (
// NewControllerInitializers is a public map of named controller groups (you can start more than one in an init func)
// paired to their InitFunc. This allows for structured downstream composition and subdivision.
func NewControllerInitializers() map[string]InitFunc {
func NewControllerInitializers(loopMode ControllerLoopMode) map[string]InitFunc {
controllers := map[string]InitFunc{}
controllers["endpoint"] = startEndpointController
controllers["replicationcontroller"] = startReplicationController
......@@ -352,9 +364,12 @@ func NewControllerInitializers() map[string]InitFunc {
controllers["ttl"] = startTTLController
controllers["bootstrapsigner"] = startBootstrapSignerController
controllers["tokencleaner"] = startTokenCleanerController
controllers["service"] = startServiceController
if loopMode == IncludeCloudLoops {
controllers["service"] = startServiceController
controllers["route"] = startRouteController
// TODO: Move node controller and volume controller into the IncludeCloudLoops only set.
}
controllers["node"] = startNodeController
controllers["route"] = startRouteController
controllers["persistentvolume-binder"] = startPersistentVolumeBinderController
controllers["attachdetach"] = startAttachDetachController
controllers["persistentvolume-expander"] = startVolumeExpandController
......@@ -430,7 +445,17 @@ func CreateControllerContext(s *options.CMServer, rootClientBuilder, clientBuild
return ControllerContext{}, err
}
cloud, err := cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile)
var cloud cloudprovider.Interface
var loopMode ControllerLoopMode
if cloudprovider.IsExternal(s.CloudProvider) {
loopMode = ExternalLoops
if s.ExternalCloudVolumePlugin != "" {
cloud, err = cloudprovider.InitCloudProvider(s.ExternalCloudVolumePlugin, s.CloudConfigFile)
}
} else {
loopMode = IncludeCloudLoops
cloud, err = cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile)
}
if err != nil {
return ControllerContext{}, fmt.Errorf("cloud provider could not be initialized: %v", err)
}
......@@ -453,6 +478,7 @@ func CreateControllerContext(s *options.CMServer, rootClientBuilder, clientBuild
Options: *s,
AvailableResources: availableResources,
Cloud: cloud,
LoopMode: loopMode,
Stop: stop,
InformersStarted: make(chan struct{}),
}
......
cmd/kube-controller-manager/app/options/BUILD
View file @ 070a7b58
......@@ -11,14 +11,13 @@ go_library(
srcs = ["options.go"],
importpath = "k8s.io/kubernetes/cmd/kube-controller-manager/app/options",
deps = [
"//cmd/controller-manager/app/options:go_default_library",
"//pkg/apis/componentconfig:go_default_library",
"//pkg/client/leaderelectionconfig:go_default_library",
"//pkg/controller/garbagecollector:go_default_library",
"//pkg/features:go_default_library",
"//pkg/master/ports:go_default_library",
"//vendor/github.com/cloudflare/cfssl/helpers:go_default_library",
"//vendor/github.com/spf13/pflag:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
......@@ -45,6 +44,7 @@ go_test(
library = ":go_default_library",
tags = ["automanaged"],
deps = [
"//cmd/controller-manager/app/options:go_default_library",
"//pkg/apis/componentconfig:go_default_library",
"//vendor/github.com/spf13/pflag:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
......
pkg/apis/componentconfig/types.go
View file @ 070a7b58
......@@ -181,6 +181,9 @@ type KubeControllerManagerConfiguration struct {
CloudProvider string
// cloudConfigFile is the path to the cloud provider configuration file.
CloudConfigFile string
// externalCloudVolumePlugin specifies the plugin to use when cloudProvider is "external".
// It is currently used by the in repo cloud providers to handle node and volume control in the KCM.
ExternalCloudVolumePlugin string
// run with untagged cloud instances
AllowUntaggedCloud bool
// concurrentEndpointSyncs is the number of endpoint syncing operations
......
pkg/controller/node/node_controller.go
View file @ 070a7b58
......@@ -259,7 +259,7 @@ func NewNodeController(
}
mask := clusterCIDR.Mask
if maskSize, _ := mask.Size(); maskSize > nodeCIDRMaskSize {
glog.Fatal("Controller: Invalid clusterCIDR, mask size of clusterCIDR must be less than nodeCIDRMaskSize.")
glog.Fatalf("Controller: Invalid clusterCIDR, mask size of clusterCIDR(%d) must be less than nodeCIDRMaskSize(%d).", maskSize, nodeCIDRMaskSize)
}
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment