scheduler: align plumbing with controller-manager and apiservers

cmd/controller-manager/app/insecure_serving.go

@@ -30,6 +30,8 @@ import (
 type InsecureServingInfo struct {
 	// Listener is the secure server network listener.
 	Listener net.Listener
+	// optional server name for log messages
+	Name string
 }
 
 // Serve starts an insecure http server with the given handler. It fails only if
@@ -41,6 +43,10 @@ func (s *InsecureServingInfo) Serve(handler http.Handler, shutdownTimeout time.D
 		MaxHeaderBytes: 1 << 20,
 	}
 
+	if len(s.Name) > 0 {
+		glog.Infof("Serving %s insecurely on %s", s.Name, s.Listener.Addr())
+	} else {
 		glog.Infof("Serving insecurely on %s", s.Listener.Addr())
+	}
 	return server.RunServer(insecureServer, s.Listener, shutdownTimeout, stopCh)
 }

cmd/controller-manager/app/options/insecure_serving.go

@@ -38,6 +38,10 @@ type InsecureServingOptions struct {
 	// either Listener or BindAddress/BindPort/BindNetwork is set,
 	// if Listener is set, use it and omit BindAddress/BindPort/BindNetwork.
 	Listener net.Listener
+
+	// ListenFunc can be overridden to create a custom listener, e.g. for mocking in tests.
+	// It defaults to options.CreateListener.
+	ListenFunc func(network, addr string) (net.Listener, int, error)
 }
 
 // Validate ensures that the insecure port values within the range of the port.
@@ -81,8 +85,12 @@ func (s *InsecureServingOptions) ApplyTo(c **genericcontrollermanager.InsecureSe
 
 	if s.Listener == nil {
 		var err error
+		listen := options.CreateListener
+		if s.ListenFunc != nil {
+			listen = s.ListenFunc
+		}
 		addr := net.JoinHostPort(s.BindAddress.String(), fmt.Sprintf("%d", s.BindPort))
-		s.Listener, s.BindPort, err = options.CreateListener(s.BindNetwork, addr)
+		s.Listener, s.BindPort, err = listen(s.BindNetwork, addr)
 		if err != nil {
 			return fmt.Errorf("failed to create listener: %v", err)
 		}

cmd/kube-scheduler/app/config/config.go

+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package config
+
+import (
+	"k8s.io/client-go/informers"
+	coreinformers "k8s.io/client-go/informers/core/v1"
+	clientset "k8s.io/client-go/kubernetes"
+	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
+	"k8s.io/client-go/tools/leaderelection"
+	"k8s.io/client-go/tools/record"
+	"k8s.io/kubernetes/cmd/controller-manager/app"
+	"k8s.io/kubernetes/pkg/apis/componentconfig"
+)
+
+// Config has all the context to run a Scheduler
+type Config struct {
+	// config is the scheduler server's configuration object.
+	ComponentConfig componentconfig.KubeSchedulerConfiguration
+
+	InsecureServing        *app.InsecureServingInfo // nil will disable serving on an insecure port
+	InsecureMetricsServing *app.InsecureServingInfo // non-nil if metrics should be served independently
+
+	Client          clientset.Interface
+	InformerFactory informers.SharedInformerFactory
+	PodInformer     coreinformers.PodInformer
+	EventClient     v1core.EventsGetter
+	Recorder        record.EventRecorder
+	Broadcaster     record.EventBroadcaster
+
+	// LeaderElection is optional.
+	LeaderElection *leaderelection.LeaderElectionConfig
+}
+
+type completedConfig struct {
+	*Config
+}
+
+// CompletedConfig same as Config, just to swap private object.
+type CompletedConfig struct {
+	// Embed a private pointer that cannot be instantiated outside of this package.
+	*completedConfig
+}
+
+// Complete fills in any fields not set that are required to have valid data. It's mutating the receiver.
+func (c *Config) Complete() CompletedConfig {
+	cc := completedConfig{c}
+
+	if c.InsecureServing != nil {
+		c.InsecureServing.Name = "healthz"
+	}
+	if c.InsecureMetricsServing != nil {
+		c.InsecureMetricsServing.Name = "metrics"
+	}
+
+	return CompletedConfig{&cc}
+}

cmd/kube-scheduler/app/options/configfile.go

+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+	"k8s.io/apimachinery/pkg/runtime/serializer/json"
+	"k8s.io/kubernetes/pkg/apis/componentconfig"
+	componentconfigv1alpha1 "k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1"
+)
+
+var (
+	configScheme = runtime.NewScheme()
+	configCodecs = serializer.NewCodecFactory(configScheme)
+)
+
+func init() {
+	componentconfig.AddToScheme(configScheme)
+	componentconfigv1alpha1.AddToScheme(configScheme)
+}
+
+func loadConfigFromFile(file string) (*componentconfig.KubeSchedulerConfiguration, error) {
+	data, err := ioutil.ReadFile(file)
+	if err != nil {
+		return nil, err
+	}
+
+	return loadConfig(data)
+}
+
+func loadConfig(data []byte) (*componentconfig.KubeSchedulerConfiguration, error) {
+	configObj, gvk, err := configCodecs.UniversalDecoder().Decode(data, nil, nil)
+	if err != nil {
+		return nil, err
+	}
+	config, ok := configObj.(*componentconfig.KubeSchedulerConfiguration)
+	if !ok {
+		return nil, fmt.Errorf("got unexpected config type: %v", gvk)
+	}
+	return config, nil
+}
+
+// WriteConfigFile writes the config into the given file name as YAML.
+func WriteConfigFile(fileName string, cfg *componentconfig.KubeSchedulerConfiguration) error {
+	var encoder runtime.Encoder
+	mediaTypes := configCodecs.SupportedMediaTypes()
+	for _, info := range mediaTypes {
+		if info.MediaType == "application/yaml" {
+			encoder = info.Serializer
+			break
+		}
+	}
+	if encoder == nil {
+		return errors.New("unable to locate yaml encoder")
+	}
+	encoder = json.NewYAMLSerializer(json.DefaultMetaFactory, configScheme, configScheme)
+	encoder = configCodecs.EncoderForVersion(encoder, componentconfigv1alpha1.SchemeGroupVersion)
+
+	configFile, err := os.Create(fileName)
+	if err != nil {
+		return err
+	}
+	defer configFile.Close()
+	if err := encoder.Encode(cfg, configFile); err != nil {
+		return err
+	}
+
+	return nil
+}

cmd/kube-scheduler/app/options/deprecated.go

+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"fmt"
+
+	"github.com/spf13/pflag"
+
+	"k8s.io/kubernetes/pkg/apis/componentconfig"
+	"k8s.io/kubernetes/pkg/scheduler/factory"
+)
+
+// DeprecatedOptions contains deprecated options and their flags.
+// TODO remove these fields once the deprecated flags are removed.
+type DeprecatedOptions struct {
+	// The fields below here are placeholders for flags that can't be directly
+	// mapped into componentconfig.KubeSchedulerConfiguration.
+	PolicyConfigFile         string
+	PolicyConfigMapName      string
+	PolicyConfigMapNamespace string
+	UseLegacyPolicyConfig    bool
+	AlgorithmProvider        string
+}
+
+// AddFlags adds flags for the deprecated options.
+func (o *DeprecatedOptions) AddFlags(fs *pflag.FlagSet, cfg *componentconfig.KubeSchedulerConfiguration) {
+	if o == nil {
+		return
+	}
+
+	// TODO: unify deprecation mechanism, string prefix or MarkDeprecated (the latter hides the flag. We also don't want that).
+
+	fs.StringVar(&o.AlgorithmProvider, "algorithm-provider", o.AlgorithmProvider, "DEPRECATED: the scheduling algorithm provider to use, one of: "+factory.ListAlgorithmProviders())
+	fs.StringVar(&o.PolicyConfigFile, "policy-config-file", o.PolicyConfigFile, "DEPRECATED: file with scheduler policy configuration. This file is used if policy ConfigMap is not provided or --use-legacy-policy-config=true")
+	usage := fmt.Sprintf("DEPRECATED: name of the ConfigMap object that contains scheduler's policy configuration. It must exist in the system namespace before scheduler initialization if --use-legacy-policy-config=false. The config must be provided as the value of an element in 'Data' map with the key='%v'", componentconfig.SchedulerPolicyConfigMapKey)
+	fs.StringVar(&o.PolicyConfigMapName, "policy-configmap", o.PolicyConfigMapName, usage)
+	fs.StringVar(&o.PolicyConfigMapNamespace, "policy-configmap-namespace", o.PolicyConfigMapNamespace, "DEPRECATED: the namespace where policy ConfigMap is located. The kube-system namespace will be used if this is not provided or is empty.")
+	fs.BoolVar(&o.UseLegacyPolicyConfig, "use-legacy-policy-config", o.UseLegacyPolicyConfig, "DEPRECATED: when set to true, scheduler will ignore policy ConfigMap and uses policy config file")
+
+	fs.BoolVar(&cfg.EnableProfiling, "profiling", cfg.EnableProfiling, "DEPRECATED: enable profiling via web interface host:port/debug/pprof/")
+	fs.BoolVar(&cfg.EnableContentionProfiling, "contention-profiling", cfg.EnableContentionProfiling, "DEPRECATED: enable lock contention profiling, if profiling is enabled")
+	fs.StringVar(&cfg.ClientConnection.KubeConfigFile, "kubeconfig", cfg.ClientConnection.KubeConfigFile, "DEPRECATED: path to kubeconfig file with authorization and master location information.")
+	fs.StringVar(&cfg.ClientConnection.ContentType, "kube-api-content-type", cfg.ClientConnection.ContentType, "DEPRECATED: content type of requests sent to apiserver.")
+	fs.Float32Var(&cfg.ClientConnection.QPS, "kube-api-qps", cfg.ClientConnection.QPS, "DEPRECATED: QPS to use while talking with kubernetes apiserver")
+	fs.Int32Var(&cfg.ClientConnection.Burst, "kube-api-burst", cfg.ClientConnection.Burst, "DEPRECATED: burst to use while talking with kubernetes apiserver")
+	fs.StringVar(&cfg.SchedulerName, "scheduler-name", cfg.SchedulerName, "DEPRECATED: name of the scheduler, used to select which pods will be processed by this scheduler, based on pod's \"spec.schedulerName\".")
+	fs.StringVar(&cfg.LeaderElection.LockObjectNamespace, "lock-object-namespace", cfg.LeaderElection.LockObjectNamespace, "DEPRECATED: define the namespace of the lock object.")
+	fs.StringVar(&cfg.LeaderElection.LockObjectName, "lock-object-name", cfg.LeaderElection.LockObjectName, "DEPRECATED: define the name of the lock object.")
+
+	fs.Int32Var(&cfg.HardPodAffinitySymmetricWeight, "hard-pod-affinity-symmetric-weight", cfg.HardPodAffinitySymmetricWeight,
+		"RequiredDuringScheduling affinity is not symmetric, but there is an implicit PreferredDuringScheduling affinity rule corresponding "+
+			"to every RequiredDuringScheduling affinity rule. --hard-pod-affinity-symmetric-weight represents the weight of implicit PreferredDuringScheduling affinity rule.")
+	fs.MarkDeprecated("hard-pod-affinity-symmetric-weight", "This option was moved to the policy configuration file")
+	fs.StringVar(&cfg.FailureDomains, "failure-domains", cfg.FailureDomains, "Indicate the \"all topologies\" set for an empty topologyKey when it's used for PreferredDuringScheduling pod anti-affinity.")
+	fs.MarkDeprecated("failure-domains", "Doesn't have any effect. Will be removed in future version.")
+}
+
+// Validate validates the deprecated scheduler options.
+func (o *DeprecatedOptions) Validate() []error {
+	if o != nil {
+		return nil
+	}
+	return nil
+}
+
+// ApplyTo sets cfg.AlgorithmSource from flags passed on the command line in the following precedence order:
+//
+// 1. --use-legacy-policy-config to use a policy file.
+// 2. --policy-configmap to use a policy config map value.
+// 3. --algorithm-provider to use a named algorithm provider.
+func (o *DeprecatedOptions) ApplyTo(cfg *componentconfig.KubeSchedulerConfiguration) error {
+	if o == nil {
+		return nil
+	}
+
+	switch {
+	case o.UseLegacyPolicyConfig || (len(o.PolicyConfigFile) > 0 && o.PolicyConfigMapName == ""):
+		cfg.AlgorithmSource = componentconfig.SchedulerAlgorithmSource{
+			Policy: &componentconfig.SchedulerPolicySource{
+				File: &componentconfig.SchedulerPolicyFileSource{
+					Path: o.PolicyConfigFile,
+				},
+			},
+		}
+	case len(o.PolicyConfigMapName) > 0:
+		cfg.AlgorithmSource = componentconfig.SchedulerAlgorithmSource{
+			Policy: &componentconfig.SchedulerPolicySource{
+				ConfigMap: &componentconfig.SchedulerPolicyConfigMapSource{
+					Name:      o.PolicyConfigMapName,
+					Namespace: o.PolicyConfigMapNamespace,
+				},
+			},
+		}
+	case len(o.AlgorithmProvider) > 0:
+		cfg.AlgorithmSource = componentconfig.SchedulerAlgorithmSource{
+			Provider: &o.AlgorithmProvider,
+		}
+	}
+
+	return nil
+}

cmd/kube-scheduler/app/options/insecure_serving.go

+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"fmt"
+	"net"
+	"strconv"
+
+	"github.com/spf13/pflag"
+
+	controlleroptions "k8s.io/kubernetes/cmd/controller-manager/app/options"
+	schedulerappconfig "k8s.io/kubernetes/cmd/kube-scheduler/app/config"
+	"k8s.io/kubernetes/pkg/apis/componentconfig"
+)
+
+// CombinedInsecureServingOptions sets up up to two insecure listeners for healthz and metrics. The flags
+// override the ComponentConfig and InsecureServingOptions values for both.
+type CombinedInsecureServingOptions struct {
+	Healthz *controlleroptions.InsecureServingOptions
+	Metrics *controlleroptions.InsecureServingOptions
+
+	BindPort    int    // overrides the structs above on ApplyTo, ignored on ApplyToFromLoadedConfig
+	BindAddress string // overrides the structs above on ApplyTo, ignored on ApplyToFromLoadedConfig
+}
+
+// AddFlags adds flags for the insecure serving options.
+func (o *CombinedInsecureServingOptions) AddFlags(fs *pflag.FlagSet) {
+	if o == nil {
+		return
+	}
+
+	fs.StringVar(&o.BindAddress, "address", o.BindAddress, "DEPRECATED: the IP address on which to listen for the --port port (set to 0.0.0.0 for all IPv4 interfaces and :: for all IPv6 interfaces). See --bind-address instead.")
+	// MarkDeprecated hides the flag from the help. We don't want that:
+	// fs.MarkDeprecated("address", "see --bind-address instead.")
+	fs.IntVar(&o.BindPort, "port", o.BindPort, "DEPRECATED: the port on which to serve HTTP insecurely without authentication and authorization. If 0, don't serve HTTPS at all. See --secure-port instead.")
+	// MarkDeprecated hides the flag from the help. We don't want that:
+	// fs.MarkDeprecated("port", "see --secure-port instead.")
+}
+
+func (o *CombinedInsecureServingOptions) applyTo(c *schedulerappconfig.Config, componentConfig *componentconfig.KubeSchedulerConfiguration) error {
+	if err := updateAddressFromInsecureServingOptions(&componentConfig.HealthzBindAddress, o.Healthz); err != nil {
+		return err
+	}
+	if err := updateAddressFromInsecureServingOptions(&componentConfig.MetricsBindAddress, o.Metrics); err != nil {
+		return err
+	}
+
+	if err := o.Healthz.ApplyTo(&c.InsecureServing); err != nil {
+		return err
+	}
+	if o.Metrics != nil && (c.ComponentConfig.MetricsBindAddress != c.ComponentConfig.HealthzBindAddress || o.Healthz == nil) {
+		if err := o.Metrics.ApplyTo(&c.InsecureMetricsServing); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// ApplyTo applies the insecure serving options to the given scheduler app configuration, and updates the componentConfig.
+func (o *CombinedInsecureServingOptions) ApplyTo(c *schedulerappconfig.Config, componentConfig *componentconfig.KubeSchedulerConfiguration) error {
+	if o == nil {
+		componentConfig.HealthzBindAddress = ""
+		componentConfig.MetricsBindAddress = ""
+		return nil
+	}
+
+	if o.Healthz != nil {
+		o.Healthz.BindPort = o.BindPort
+		o.Healthz.BindAddress = net.ParseIP(o.BindAddress)
+	}
+	if o.Metrics != nil {
+		o.Metrics.BindPort = o.BindPort
+		o.Metrics.BindAddress = net.ParseIP(o.BindAddress)
+	}
+
+	return o.applyTo(c, componentConfig)
+}
+
+// ApplyToFromLoadedConfig updates the insecure serving options from the component config and then appies it to the given scheduler app configuration.
+func (o *CombinedInsecureServingOptions) ApplyToFromLoadedConfig(c *schedulerappconfig.Config, componentConfig *componentconfig.KubeSchedulerConfiguration) error {
+	if o == nil {
+		return nil
+	}
+
+	if err := updateInsecureServingOptionsFromAddress(o.Healthz, componentConfig.HealthzBindAddress); err != nil {
+		return fmt.Errorf("invalid healthz address: %v", err)
+	}
+	if err := updateInsecureServingOptionsFromAddress(o.Metrics, componentConfig.MetricsBindAddress); err != nil {
+		return fmt.Errorf("invalid metrics address: %v", err)
+	}
+
+	return o.applyTo(c, componentConfig)
+}
+
+func updateAddressFromInsecureServingOptions(addr *string, is *controlleroptions.InsecureServingOptions) error {
+	if is == nil {
+		*addr = ""
+	} else {
+		if is.Listener != nil {
+			*addr = is.Listener.Addr().String()
+		} else if is.BindPort == 0 {
+			*addr = ""
+		} else {
+			*addr = net.JoinHostPort(is.BindAddress.String(), strconv.Itoa(is.BindPort))
+		}
+	}
+
+	return nil
+}
+
+func updateInsecureServingOptionsFromAddress(is *controlleroptions.InsecureServingOptions, addr string) error {
+	if is == nil {
+		return nil
+	}
+	if len(addr) == 0 {
+		is.BindPort = 0
+		return nil
+	}
+
+	host, portInt, err := splitHostIntPort(addr)
+	if err != nil {
+		return fmt.Errorf("invalid address %q", addr)
+	}
+
+	is.BindAddress = net.ParseIP(host)
+	is.BindPort = portInt
+
+	return nil
+}
+
+// Validate validates the insecure serving options.
+func (o *CombinedInsecureServingOptions) Validate() []error {
+	if o == nil {
+		return nil
+	}
+
+	errors := []error{}
+
+	if o.BindPort <= 0 || o.BindPort > 32767 {
+		errors = append(errors, fmt.Errorf("--insecure-port %v must be between 0 and 32767, inclusive. 0 for turning off insecure (HTTP) port", o.BindPort))
+	}
+
+	if len(o.BindAddress) > 0 && net.ParseIP(o.BindAddress) == nil {
+		errors = append(errors, fmt.Errorf("--address has no valid IP address"))
+	}
+
+	return errors
+}

cmd/kube-scheduler/app/options/insecure_serving_test.go

+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"fmt"
+	"net"
+	"strconv"
+	"testing"
+
+	"k8s.io/apimachinery/pkg/util/rand"
+	"k8s.io/kubernetes/cmd/controller-manager/app/options"
+	schedulerappconfig "k8s.io/kubernetes/cmd/kube-scheduler/app/config"
+	"k8s.io/kubernetes/pkg/apis/componentconfig"
+)
+
+func TestOptions_ApplyTo(t *testing.T) {
+	tests := []struct {
+		name                                                              string
+		options                                                           Options
+		configLoaded                                                      bool
+		expectHealthzBindAddress, expectMetricsBindAddress                string
+		expectInsecureServingAddress, expectInsecureMetricsServingAddress string
+		expectInsecureServingPort, expectInsecureMetricsServingPort       int
+		wantErr                                                           bool
+	}{
+		{
+			name: "no config, zero port",
+			options: Options{
+				ComponentConfig: componentconfig.KubeSchedulerConfiguration{
+					HealthzBindAddress: "1.2.3.4:1234",
+					MetricsBindAddress: "1.2.3.4:1234",
+				},
+				CombinedInsecureServing: &CombinedInsecureServingOptions{
+					Healthz:  &options.InsecureServingOptions{},
+					Metrics:  &options.InsecureServingOptions{},
+					BindPort: 0,
+				},
+			},
+			configLoaded: false,
+		},
+		{
+			name: "config loaded, non-nil healthz",
+			options: Options{
+				ComponentConfig: componentconfig.KubeSchedulerConfiguration{
+					HealthzBindAddress: "1.2.3.4:1234",
+					MetricsBindAddress: "1.2.3.4:1234",
+				},
+				CombinedInsecureServing: &CombinedInsecureServingOptions{
+					Healthz:  &options.InsecureServingOptions{},
+					BindPort: 0,
+				},
+			},
+			configLoaded: true,
+
+			expectHealthzBindAddress:     "1.2.3.4:1234",
+			expectInsecureServingPort:    1234,
+			expectInsecureServingAddress: "1.2.3.4",
+		},
+		{
+			name: "config loaded, non-nil metrics",
+			options: Options{
+				ComponentConfig: componentconfig.KubeSchedulerConfiguration{
+					HealthzBindAddress: "1.2.3.4:1234",
+					MetricsBindAddress: "1.2.3.4:1234",
+				},
+				CombinedInsecureServing: &CombinedInsecureServingOptions{
+					Metrics:  &options.InsecureServingOptions{},
+					BindPort: 0,
+				},
+			},
+			configLoaded: true,
+
+			expectMetricsBindAddress:            "1.2.3.4:1234",
+			expectInsecureMetricsServingPort:    1234,
+			expectInsecureMetricsServingAddress: "1.2.3.4",
+		},
+		{
+			name: "config loaded, all set, zero BindPort",
+			options: Options{
+				ComponentConfig: componentconfig.KubeSchedulerConfiguration{
+					HealthzBindAddress: "1.2.3.4:1234",
+					MetricsBindAddress: "1.2.3.4:1234",
+				},
+				CombinedInsecureServing: &CombinedInsecureServingOptions{
+					Healthz:  &options.InsecureServingOptions{},
+					Metrics:  &options.InsecureServingOptions{},
+					BindPort: 0,
+				},
+			},
+			configLoaded: true,
+
+			expectHealthzBindAddress:     "1.2.3.4:1234",
+			expectInsecureServingPort:    1234,
+			expectInsecureServingAddress: "1.2.3.4",
+
+			expectMetricsBindAddress: "1.2.3.4:1234",
+		},
+		{
+			name: "config loaded, all set, different addresses",
+			options: Options{
+				ComponentConfig: componentconfig.KubeSchedulerConfiguration{
+					HealthzBindAddress: "1.2.3.4:1234",
+					MetricsBindAddress: "1.2.3.4:1235",
+				},
+				CombinedInsecureServing: &CombinedInsecureServingOptions{
+					Healthz:  &options.InsecureServingOptions{},
+					Metrics:  &options.InsecureServingOptions{},
+					BindPort: 0,
+				},
+			},
+			configLoaded: true,
+
+			expectHealthzBindAddress:     "1.2.3.4:1234",
+			expectInsecureServingPort:    1234,
+			expectInsecureServingAddress: "1.2.3.4",
+
+			expectMetricsBindAddress:            "1.2.3.4:1235",
+			expectInsecureMetricsServingPort:    1235,
+			expectInsecureMetricsServingAddress: "1.2.3.4",
+		},
+		{
+			name: "no config, all set, port passed",
+			options: Options{
+				ComponentConfig: componentconfig.KubeSchedulerConfiguration{
+					HealthzBindAddress: "1.2.3.4:1234",
+					MetricsBindAddress: "1.2.3.4:1234",
+				},
+				CombinedInsecureServing: &CombinedInsecureServingOptions{
+					Healthz:     &options.InsecureServingOptions{},
+					Metrics:     &options.InsecureServingOptions{},
+					BindPort:    1236,
+					BindAddress: "1.2.3.4",
+				},
+			},
+			configLoaded: false,
+
+			expectHealthzBindAddress:     "1.2.3.4:1236",
+			expectInsecureServingPort:    1236,
+			expectInsecureServingAddress: "1.2.3.4",
+
+			expectMetricsBindAddress: "1.2.3.4:1236",
+		},
+		{
+			name: "no config, all set, address passed",
+			options: Options{
+				ComponentConfig: componentconfig.KubeSchedulerConfiguration{
+					HealthzBindAddress: "1.2.3.4:1234",
+					MetricsBindAddress: "1.2.3.4:1234",
+				},
+				CombinedInsecureServing: &CombinedInsecureServingOptions{
+					Healthz:     &options.InsecureServingOptions{},
+					Metrics:     &options.InsecureServingOptions{},
+					BindAddress: "2.3.4.5",
+					BindPort:    1234,
+				},
+			},
+			configLoaded: false,
+
+			expectHealthzBindAddress:     "2.3.4.5:1234",
+			expectInsecureServingPort:    1234,
+			expectInsecureServingAddress: "2.3.4.5",
+
+			expectMetricsBindAddress: "2.3.4.5:1234",
+		},
+		{
+			name: "no config, all set, zero port passed",
+			options: Options{
+				ComponentConfig: componentconfig.KubeSchedulerConfiguration{
+					HealthzBindAddress: "1.2.3.4:1234",
+					MetricsBindAddress: "1.2.3.4:1234",
+				},
+				CombinedInsecureServing: &CombinedInsecureServingOptions{
+					Healthz:     &options.InsecureServingOptions{},
+					Metrics:     &options.InsecureServingOptions{},
+					BindAddress: "2.3.4.5",
+					BindPort:    0,
+				},
+			},
+			configLoaded: false,
+		},
+	}
+	for i, tt := range tests {
+		t.Run(fmt.Sprintf("%d-%s", i, tt.name), func(t *testing.T) {
+			c := schedulerappconfig.Config{
+				ComponentConfig: tt.options.ComponentConfig,
+			}
+
+			if tt.options.CombinedInsecureServing != nil {
+				if tt.options.CombinedInsecureServing.Healthz != nil {
+					tt.options.CombinedInsecureServing.Healthz.ListenFunc = createMockListener
+				}
+				if tt.options.CombinedInsecureServing.Metrics != nil {
+					tt.options.CombinedInsecureServing.Metrics.ListenFunc = createMockListener
+				}
+			}
+
+			if tt.configLoaded {
+				if err := tt.options.CombinedInsecureServing.ApplyToFromLoadedConfig(&c, &c.ComponentConfig); (err != nil) != tt.wantErr {
+					t.Fatalf("%d - Options.ApplyTo() error = %v, wantErr %v", i, err, tt.wantErr)
+				}
+			} else {
+				if err := tt.options.CombinedInsecureServing.ApplyTo(&c, &c.ComponentConfig); (err != nil) != tt.wantErr {
+					t.Fatalf("%d - Options.ApplyTo() error = %v, wantErr %v", i, err, tt.wantErr)
+				}
+			}
+			if got, expect := c.ComponentConfig.HealthzBindAddress, tt.expectHealthzBindAddress; got != expect {
+				t.Errorf("%d - expected HealthzBindAddress %q, got %q", i, expect, got)
+			}
+			if got, expect := c.ComponentConfig.MetricsBindAddress, tt.expectMetricsBindAddress; got != expect {
+				t.Errorf("%d - expected MetricsBindAddress %q, got %q", i, expect, got)
+			}
+			if got, expect := c.InsecureServing != nil, tt.expectInsecureServingPort != 0; got != expect {
+				t.Errorf("%d - expected InsecureServing != nil to be %v, got %v", i, expect, got)
+			} else if c.InsecureServing != nil {
+				if got, expect := c.InsecureServing.Listener.(*mockListener).address, tt.expectInsecureServingAddress; got != expect {
+					t.Errorf("%d - expected healthz address %q, got %q", i, expect, got)
+				}
+				if got, expect := c.InsecureServing.Listener.(*mockListener).port, tt.expectInsecureServingPort; got != expect {
+					t.Errorf("%d - expected healthz port %v, got %v", i, expect, got)
+				}
+			}
+			if got, expect := c.InsecureMetricsServing != nil, tt.expectInsecureMetricsServingPort != 0; got != expect {
+				t.Errorf("%d - expected Metrics != nil to be %v, got %v", i, expect, got)
+			} else if c.InsecureMetricsServing != nil {
+				if got, expect := c.InsecureMetricsServing.Listener.(*mockListener).address, tt.expectInsecureMetricsServingAddress; got != expect {
+					t.Errorf("%d - expected metrics address %q, got %q", i, expect, got)
+				}
+				if got, expect := c.InsecureMetricsServing.Listener.(*mockListener).port, tt.expectInsecureMetricsServingPort; got != expect {
+					t.Errorf("%d - expected metrics port %v, got %v", i, expect, got)
+				}
+			}
+		})
+	}
+}
+
+type mockListener struct {
+	address string
+	port    int
+}
+
+func createMockListener(network, addr string) (net.Listener, int, error) {
+	host, portInt, err := splitHostIntPort(addr)
+	if err != nil {
+		return nil, 0, err
+	}
+	if portInt == 0 {
+		portInt = rand.IntnRange(1, 32767)
+	}
+	return &mockListener{host, portInt}, portInt, nil
+}
+
+func (l *mockListener) Accept() (net.Conn, error) { return nil, nil }
+func (l *mockListener) Close() error              { return nil }
+func (l *mockListener) Addr() net.Addr {
+	return mockAddr(net.JoinHostPort(l.address, strconv.Itoa(l.port)))
+}
+
+type mockAddr string
+
+func (a mockAddr) Network() string { return "tcp" }
+func (a mockAddr) String() string  { return string(a) }

cmd/kube-scheduler/app/options/options.go

+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"fmt"
+	"net"
+	"os"
+	"strconv"
+
+	"github.com/golang/glog"
+	"github.com/spf13/pflag"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/uuid"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	"k8s.io/client-go/informers"
+	clientset "k8s.io/client-go/kubernetes"
+	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
+	restclient "k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
+	"k8s.io/client-go/tools/leaderelection"
+	"k8s.io/client-go/tools/leaderelection/resourcelock"
+	"k8s.io/client-go/tools/record"
+	controlleroptions "k8s.io/kubernetes/cmd/controller-manager/app/options"
+	schedulerappconfig "k8s.io/kubernetes/cmd/kube-scheduler/app/config"
+	"k8s.io/kubernetes/pkg/api/legacyscheme"
+	"k8s.io/kubernetes/pkg/apis/componentconfig"
+	componentconfigv1alpha1 "k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1"
+	"k8s.io/kubernetes/pkg/client/leaderelectionconfig"
+	"k8s.io/kubernetes/pkg/scheduler/factory"
+)
+
+// Options has all the params needed to run a Scheduler
+type Options struct {
+	// The default values. These are overridden if ConfigFile is set or by values in InsecureServing.
+	ComponentConfig componentconfig.KubeSchedulerConfiguration
+
+	CombinedInsecureServing *CombinedInsecureServingOptions
+	Deprecated              *DeprecatedOptions
+
+	// ConfigFile is the location of the scheduler server's configuration file.
+	ConfigFile string
+
+	// WriteConfigTo is the path where the default configuration will be written.
+	WriteConfigTo string
+
+	Master string
+}
+
+// NewOptions returns default scheduler app options.
+func NewOptions() (*Options, error) {
+	cfg, err := newDefaultComponentConfig()
+	if err != nil {
+		return nil, err
+	}
+
+	hhost, hport, err := splitHostIntPort(cfg.HealthzBindAddress)
+	if err != nil {
+		return nil, err
+	}
+
+	o := &Options{
+		ComponentConfig: *cfg,
+		CombinedInsecureServing: &CombinedInsecureServingOptions{
+			Healthz: &controlleroptions.InsecureServingOptions{
+				BindNetwork: "tcp",
+			},
+			Metrics: &controlleroptions.InsecureServingOptions{
+				BindNetwork: "tcp",
+			},
+			BindPort:    hport,
+			BindAddress: hhost,
+		},
+		Deprecated: &DeprecatedOptions{
+			UseLegacyPolicyConfig:    false,
+			PolicyConfigMapNamespace: metav1.NamespaceSystem,
+		},
+	}
+
+	return o, nil
+}
+
+func splitHostIntPort(s string) (string, int, error) {
+	host, port, err := net.SplitHostPort(s)
+	if err != nil {
+		return "", 0, err
+	}
+	portInt, err := strconv.Atoi(port)
+	if err != nil {
+		return "", 0, err
+	}
+	return host, portInt, err
+}
+
+func newDefaultComponentConfig() (*componentconfig.KubeSchedulerConfiguration, error) {
+	cfgv1alpha1 := componentconfigv1alpha1.KubeSchedulerConfiguration{}
+	configScheme.Default(&cfgv1alpha1)
+	cfg := componentconfig.KubeSchedulerConfiguration{}
+	if err := configScheme.Convert(&cfgv1alpha1, &cfg, nil); err != nil {
+		return nil, err
+	}
+	return &cfg, nil
+}
+
+// AddFlags adds flags for the scheduler options.
+func (o *Options) AddFlags(fs *pflag.FlagSet) {
+	fs.StringVar(&o.ConfigFile, "config", o.ConfigFile, "The path to the configuration file. Flags override values in this file.")
+	fs.StringVar(&o.WriteConfigTo, "write-config-to", o.WriteConfigTo, "If set, write the configuration values to this file and exit.")
+	fs.StringVar(&o.Master, "master", o.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig)")
+
+	o.CombinedInsecureServing.AddFlags(fs)
+	o.Deprecated.AddFlags(fs, &o.ComponentConfig)
+
+	leaderelectionconfig.BindFlags(&o.ComponentConfig.LeaderElection.LeaderElectionConfiguration, fs)
+	utilfeature.DefaultFeatureGate.AddFlag(fs)
+}
+
+// ApplyTo applies the scheduler options to the given scheduler app configuration.
+func (o *Options) ApplyTo(c *schedulerappconfig.Config) error {
+	if len(o.ConfigFile) == 0 && len(o.WriteConfigTo) == 0 {
+		glog.Warning("WARNING: all flags other than --config, --write-config-to, and --cleanup are deprecated. Please begin using a config file ASAP.")
+	}
+	if len(o.ConfigFile) == 0 {
+		c.ComponentConfig = o.ComponentConfig
+
+		// only apply deprecated flags if no config file is loaded (this is the old behaviour).
+		if err := o.Deprecated.ApplyTo(&c.ComponentConfig); err != nil {
+			return err
+		}
+		if err := o.CombinedInsecureServing.ApplyTo(c, &c.ComponentConfig); err != nil {
+			return err
+		}
+	} else {
+		cfg, err := loadConfigFromFile(o.ConfigFile)
+		if err != nil {
+			return err
+		}
+
+		// use the loaded config file only, with the exception of --address and --port. This means that
+		// none of the deprectated flags in o.Deprecated are taken into consideration. This is the old
+		// behaviour of the flags we have to keep.
+		c.ComponentConfig = *cfg
+
+		if err := o.CombinedInsecureServing.ApplyToFromLoadedConfig(c, &c.ComponentConfig); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// Validate validates all the required options.
+func (o *Options) Validate() []error {
+	var errs []error
+
+	errs = append(errs, o.CombinedInsecureServing.Validate()...)
+	errs = append(errs, o.Deprecated.Validate()...)
+
+	return errs
+}
+
+// Config return a scheduler config object
+func (o *Options) Config() (*schedulerappconfig.Config, error) {
+	// prepare kube clients.
+	client, leaderElectionClient, eventClient, err := createClients(o.ComponentConfig.ClientConnection, o.Master)
+	if err != nil {
+		return nil, err
+	}
+
+	// Prepare event clients.
+	eventBroadcaster := record.NewBroadcaster()
+	recorder := eventBroadcaster.NewRecorder(legacyscheme.Scheme, corev1.EventSource{Component: o.ComponentConfig.SchedulerName})
+
+	// Set up leader election if enabled.
+	var leaderElectionConfig *leaderelection.LeaderElectionConfig
+	if o.ComponentConfig.LeaderElection.LeaderElect {
+		leaderElectionConfig, err = makeLeaderElectionConfig(o.ComponentConfig.LeaderElection, leaderElectionClient, recorder)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	c := &schedulerappconfig.Config{
+		Client:          client,
+		InformerFactory: informers.NewSharedInformerFactory(client, 0),
+		PodInformer:     factory.NewPodInformer(client, 0),
+		EventClient:     eventClient,
+		Recorder:        recorder,
+		Broadcaster:     eventBroadcaster,
+		LeaderElection:  leaderElectionConfig,
+	}
+	if err := o.ApplyTo(c); err != nil {
+		return nil, err
+	}
+
+	return c, nil
+}
+
+// makeLeaderElectionConfig builds a leader election configuration. It will
+// create a new resource lock associated with the configuration.
+func makeLeaderElectionConfig(config componentconfig.KubeSchedulerLeaderElectionConfiguration, client clientset.Interface, recorder record.EventRecorder) (*leaderelection.LeaderElectionConfig, error) {
+	hostname, err := os.Hostname()
+	if err != nil {
+		return nil, fmt.Errorf("unable to get hostname: %v", err)
+	}
+	// add a uniquifier so that two processes on the same host don't accidentally both become active
+	id := hostname + "_" + string(uuid.NewUUID())
+
+	rl, err := resourcelock.New(config.ResourceLock,
+		config.LockObjectNamespace,
+		config.LockObjectName,
+		client.CoreV1(),
+		resourcelock.ResourceLockConfig{
+			Identity:      id,
+			EventRecorder: recorder,
+		})
+	if err != nil {
+		return nil, fmt.Errorf("couldn't create resource lock: %v", err)
+	}
+
+	return &leaderelection.LeaderElectionConfig{
+		Lock:          rl,
+		LeaseDuration: config.LeaseDuration.Duration,
+		RenewDeadline: config.RenewDeadline.Duration,
+		RetryPeriod:   config.RetryPeriod.Duration,
+	}, nil
+}
+
+// createClients creates a kube client and an event client from the given config and masterOverride.
+// TODO remove masterOverride when CLI flags are removed.
+func createClients(config componentconfig.ClientConnectionConfiguration, masterOverride string) (clientset.Interface, clientset.Interface, v1core.EventsGetter, error) {
+	if len(config.KubeConfigFile) == 0 && len(masterOverride) == 0 {
+		glog.Warningf("Neither --kubeconfig nor --master was specified. Using default API client. This might not work.")
+	}
+
+	// This creates a client, first loading any specified kubeconfig
+	// file, and then overriding the Master flag, if non-empty.
+	kubeConfig, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
+		&clientcmd.ClientConfigLoadingRules{ExplicitPath: config.KubeConfigFile},
+		&clientcmd.ConfigOverrides{ClusterInfo: clientcmdapi.Cluster{Server: masterOverride}}).ClientConfig()
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	kubeConfig.AcceptContentTypes = config.AcceptContentTypes
+	kubeConfig.ContentType = config.ContentType
+	kubeConfig.QPS = config.QPS
+	//TODO make config struct use int instead of int32?
+	kubeConfig.Burst = int(config.Burst)
+
+	client, err := clientset.NewForConfig(restclient.AddUserAgent(kubeConfig, "scheduler"))
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	leaderElectionClient, err := clientset.NewForConfig(restclient.AddUserAgent(kubeConfig, "leader-election"))
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	eventClient, err := clientset.NewForConfig(kubeConfig)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	return client, leaderElectionClient, eventClient.CoreV1(), nil
+}

cmd/kubeadm/app/preflight/checks.go

@@ -46,7 +46,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/sets"
 	apiservoptions "k8s.io/kubernetes/cmd/kube-apiserver/app/options"
 	cmoptions "k8s.io/kubernetes/cmd/kube-controller-manager/app/options"
-	schedulerapp "k8s.io/kubernetes/cmd/kube-scheduler/app"
+	scheduleroptions "k8s.io/kubernetes/cmd/kube-scheduler/app/options"
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	kubeadmdefaults "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
@@ -548,7 +548,7 @@ func (eac ExtraArgsCheck) Check() (warnings, errors []error) {
 		warnings = append(warnings, argsCheck("kube-controller-manager", eac.ControllerManagerExtraArgs, flags)...)
 	}
 	if len(eac.SchedulerExtraArgs) > 0 {
-		opts, err := schedulerapp.NewOptions()
+		opts, err := scheduleroptions.NewOptions()
 		if err != nil {
 			warnings = append(warnings, err)
 		}

test/integration/scheduler/scheduler_test.go

@@ -42,6 +42,7 @@ import (
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/client-go/tools/record"
 	schedulerapp "k8s.io/kubernetes/cmd/kube-scheduler/app"
+	schedulerappconfig "k8s.io/kubernetes/cmd/kube-scheduler/app/config"
 	"k8s.io/kubernetes/pkg/api/legacyscheme"
 	"k8s.io/kubernetes/pkg/api/testapi"
 	"k8s.io/kubernetes/pkg/apis/componentconfig"
@@ -181,7 +182,9 @@ func TestSchedulerCreationFromConfigMap(t *testing.T) {
 		eventBroadcaster := record.NewBroadcaster()
 		eventBroadcaster.StartRecordingToSink(&clientv1core.EventSinkImpl{Interface: clientSet.CoreV1().Events("")})
 
-		ss := &schedulerapp.SchedulerServer{
+		ss := &schedulerappconfig.Config{
+			ComponentConfig: componentconfig.KubeSchedulerConfiguration{
+				HardPodAffinitySymmetricWeight: v1.DefaultHardPodAffinitySymmetricWeight,
 				SchedulerName:                  v1.DefaultSchedulerName,
 				AlgorithmSource: componentconfig.SchedulerAlgorithmSource{
 					Policy: &componentconfig.SchedulerPolicySource{
@@ -191,7 +194,7 @@ func TestSchedulerCreationFromConfigMap(t *testing.T) {
 						},
 					},
 				},
-			HardPodAffinitySymmetricWeight: v1.DefaultHardPodAffinitySymmetricWeight,
+			},
 			Client:          clientSet,
 			InformerFactory: informerFactory,
 			PodInformer:     factory.NewPodInformer(clientSet, 0),
@@ -200,7 +203,7 @@ func TestSchedulerCreationFromConfigMap(t *testing.T) {
 			Broadcaster:     eventBroadcaster,
 		}
 
-		config, err := ss.SchedulerConfig()
+		config, err := schedulerapp.NewSchedulerConfig(ss.Complete())
 		if err != nil {
 			t.Fatalf("couldn't make scheduler config: %v", err)
 		}
@@ -240,7 +243,8 @@ func TestSchedulerCreationFromNonExistentConfigMap(t *testing.T) {
 	eventBroadcaster := record.NewBroadcaster()
 	eventBroadcaster.StartRecordingToSink(&clientv1core.EventSinkImpl{Interface: clientSet.CoreV1().Events("")})
 
-	ss := &schedulerapp.SchedulerServer{
+	ss := &schedulerappconfig.Config{
+		ComponentConfig: componentconfig.KubeSchedulerConfiguration{
 			SchedulerName: v1.DefaultSchedulerName,
 			AlgorithmSource: componentconfig.SchedulerAlgorithmSource{
 				Policy: &componentconfig.SchedulerPolicySource{
@@ -251,6 +255,7 @@ func TestSchedulerCreationFromNonExistentConfigMap(t *testing.T) {
 				},
 			},
 			HardPodAffinitySymmetricWeight: v1.DefaultHardPodAffinitySymmetricWeight,
+		},
 		Client:          clientSet,
 		InformerFactory: informerFactory,
 		PodInformer:     factory.NewPodInformer(clientSet, 0),
@@ -259,7 +264,7 @@ func TestSchedulerCreationFromNonExistentConfigMap(t *testing.T) {
 		Broadcaster:     eventBroadcaster,
 	}
 
-	_, err := ss.SchedulerConfig()
+	_, err := schedulerapp.NewSchedulerConfig(ss.Complete())
 	if err == nil {
 		t.Fatalf("Creation of scheduler didn't fail while the policy ConfigMap didn't exist.")
 	}