Add option to disable federation ingress controller

federation/cmd/federation-controller-manager/app/BUILD

@@ -33,9 +33,13 @@ go_library(
         "//federation/pkg/federation-controller/secret:go_default_library",
         "//federation/pkg/federation-controller/service:go_default_library",
         "//federation/pkg/federation-controller/util:go_default_library",
+        "//pkg/apis/meta/v1:go_default_library",
         "//pkg/client/restclient:go_default_library",
+        "//pkg/client/typed/discovery:go_default_library",
         "//pkg/client/unversioned/clientcmd:go_default_library",
         "//pkg/healthz:go_default_library",
+        "//pkg/runtime/schema:go_default_library",
+        "//pkg/util/config:go_default_library",
         "//pkg/util/configz:go_default_library",
         "//pkg/util/wait:go_default_library",
         "//pkg/version:go_default_library",
@@ -45,3 +49,16 @@ go_library(
         "//vendor:github.com/spf13/pflag",
     ],
 )
+
+go_test(
+    name = "go_default_test",
+    srcs = ["controllermanager_test.go"],
+    library = "go_default_library",
+    tags = ["automanaged"],
+    deps = [
+        "//federation/pkg/federation-controller/ingress:go_default_library",
+        "//pkg/apis/meta/v1:go_default_library",
+        "//pkg/runtime/schema:go_default_library",
+        "//pkg/util/config:go_default_library",
+    ],
+)

federation/cmd/federation-controller-manager/app/controllermanager.go

@@ -39,6 +39,7 @@ import (
 	secretcontroller "k8s.io/kubernetes/federation/pkg/federation-controller/secret"
 	servicecontroller "k8s.io/kubernetes/federation/pkg/federation-controller/service"
 	"k8s.io/kubernetes/federation/pkg/federation-controller/util"
+	metav1 "k8s.io/kubernetes/pkg/apis/meta/v1"
 	"k8s.io/kubernetes/pkg/client/restclient"
 	"k8s.io/kubernetes/pkg/client/unversioned/clientcmd"
 	"k8s.io/kubernetes/pkg/healthz"
@@ -50,6 +51,9 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
+	"k8s.io/kubernetes/pkg/client/typed/discovery"
+	"k8s.io/kubernetes/pkg/runtime/schema"
+	"k8s.io/kubernetes/pkg/util/config"
 )
 
 const (
@@ -155,6 +159,12 @@ func StartControllers(s *options.CMServer, restClientCfg *restclient.Config) err
 		glog.Fatalf("Cloud provider could not be initialized: %v", err)
 	}
 
+	discoveryClient := discovery.NewDiscoveryClientForConfigOrDie(restClientCfg)
+	serverResources, err := discoveryClient.ServerResources()
+	if err != nil {
+		glog.Fatalf("Could not find resources from API Server: %v", err)
+	}
+
 	glog.Infof("Loading client config for namespace controller %q", "namespace-controller")
 	nsClientset := federationclientset.NewForConfigOrDie(restclient.AddUserAgent(restClientCfg, "namespace-controller"))
 	namespaceController := namespacecontroller.NewNamespaceController(nsClientset)
@@ -182,11 +192,13 @@ func StartControllers(s *options.CMServer, restClientCfg *restclient.Config) err
 	// TODO: rename s.ConcurentReplicaSetSyncs
 	go deploymentController.Run(s.ConcurrentReplicaSetSyncs, wait.NeverStop)
 
-	glog.Infof("Loading client config for ingress controller %q", "ingress-controller")
-	ingClientset := federationclientset.NewForConfigOrDie(restclient.AddUserAgent(restClientCfg, "ingress-controller"))
-	ingressController := ingresscontroller.NewIngressController(ingClientset)
-	glog.Infof("Running ingress controller")
-	ingressController.Run(wait.NeverStop)
+	if controllerEnabled(s.Controllers, serverResources, ingresscontroller.ControllerName, ingresscontroller.RequiredResources, true) {
+		glog.Infof("Loading client config for ingress controller %q", "ingress-controller")
+		ingClientset := federationclientset.NewForConfigOrDie(restclient.AddUserAgent(restClientCfg, "ingress-controller"))
+		ingressController := ingresscontroller.NewIngressController(ingClientset)
+		glog.Infof("Running ingress controller")
+		ingressController.Run(wait.NeverStop)
+	}
 
 	glog.Infof("Loading client config for service controller %q", servicecontroller.UserAgentName)
 	scClientset := federationclientset.NewForConfigOrDie(restclient.AddUserAgent(restClientCfg, servicecontroller.UserAgentName))
@@ -209,3 +221,46 @@ func restClientConfigFromSecret(master string) (*restclient.Config, error) {
 	}
 	return restClientCfg, nil
 }
+
+func controllerEnabled(controllers config.ConfigurationMap, serverResources []*metav1.APIResourceList, controller string, requiredResources []schema.GroupVersionResource, defaultValue bool) bool {
+	controllerConfig, ok := controllers[controller]
+	if ok {
+		if controllerConfig == "false" {
+			glog.Infof("%s controller disabled by config", controller)
+			return false
+		}
+		if controllerConfig == "true" {
+			if !hasRequiredResources(serverResources, requiredResources) {
+				glog.Fatalf("%s controller enabled explicitly but API Server does not have required resources", controller)
+				panic("unreachable")
+			}
+			return true
+		}
+	} else if defaultValue {
+		if !hasRequiredResources(serverResources, requiredResources) {
+			glog.Warningf("%s controller disabled because API Server does not have required resources", controller)
+			return false
+		}
+	}
+	return defaultValue
+}
+
+func hasRequiredResources(serverResources []*metav1.APIResourceList, requiredResources []schema.GroupVersionResource) bool {
+	for _, resource := range requiredResources {
+		found := false
+		for _, serverResource := range serverResources {
+			if serverResource.GroupVersion == resource.GroupVersion().String() {
+				for _, apiResource := range serverResource.APIResources {
+					if apiResource.Name == resource.Resource {
+						found = true
+						break
+					}
+				}
+			}
+		}
+		if !found {
+			return false
+		}
+	}
+	return true
+}

federation/cmd/federation-controller-manager/app/controllermanager_test.go

+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package app
+
+import (
+	ingresscontroller "k8s.io/kubernetes/federation/pkg/federation-controller/ingress"
+	metav1 "k8s.io/kubernetes/pkg/apis/meta/v1"
+	"k8s.io/kubernetes/pkg/runtime/schema"
+	"k8s.io/kubernetes/pkg/util/config"
+	"testing"
+)
+
+func TestControllerEnabled(t *testing.T) {
+
+	testCases := []struct {
+		controllersConfig config.ConfigurationMap
+		serverResources   []*metav1.APIResourceList
+		controller        string
+		requiredResources []schema.GroupVersionResource
+		defaultValue      bool
+		expectedResult    bool
+	}{
+		// no override, API server has Ingress enabled
+		{
+			controllersConfig: config.ConfigurationMap{},
+			serverResources: []*metav1.APIResourceList{
+				{
+					GroupVersion: "extensions/v1beta1",
+					APIResources: []metav1.APIResource{
+						{Name: "ingresses", Namespaced: true, Kind: "Ingress"},
+					},
+				},
+			},
+			controller:        ingresscontroller.ControllerName,
+			requiredResources: ingresscontroller.RequiredResources,
+			defaultValue:      true,
+			expectedResult:    true,
+		},
+		// no override, API server has Ingress disabled
+		{
+			controllersConfig: config.ConfigurationMap{},
+			serverResources:   []*metav1.APIResourceList{},
+			controller:        ingresscontroller.ControllerName,
+			requiredResources: ingresscontroller.RequiredResources,
+			defaultValue:      true,
+			expectedResult:    false,
+		},
+		// API server has Ingress enabled, override config to disable Ingress controller
+		{
+			controllersConfig: config.ConfigurationMap{
+				ingresscontroller.ControllerName: "false",
+			},
+			serverResources: []*metav1.APIResourceList{
+				{
+					GroupVersion: "extensions/v1beta1",
+					APIResources: []metav1.APIResource{
+						{Name: "ingresses", Namespaced: true, Kind: "Ingress"},
+					},
+				},
+			},
+			controller:        ingresscontroller.ControllerName,
+			requiredResources: ingresscontroller.RequiredResources,
+			defaultValue:      true,
+			expectedResult:    false,
+		},
+	}
+
+	for _, test := range testCases {
+		actualEnabled := controllerEnabled(test.controllersConfig, test.serverResources, test.controller, test.requiredResources, test.defaultValue)
+		if actualEnabled != test.expectedResult {
+			t.Errorf("%s controller: expected %v, got %v", test.controller, test.expectedResult, actualEnabled)
+		}
+	}
+}

federation/cmd/federation-controller-manager/app/options/BUILD

@@ -19,6 +19,7 @@ go_library(
         "//pkg/apis/componentconfig:go_default_library",
         "//pkg/apis/meta/v1:go_default_library",
         "//pkg/client/leaderelection:go_default_library",
+        "//pkg/util/config:go_default_library",
         "//vendor:github.com/spf13/pflag",
     ],
 )

federation/cmd/federation-controller-manager/app/options/options.go

@@ -28,6 +28,7 @@ import (
 	"k8s.io/kubernetes/pkg/apis/componentconfig"
 	metav1 "k8s.io/kubernetes/pkg/apis/meta/v1"
 	"k8s.io/kubernetes/pkg/client/leaderelection"
+	"k8s.io/kubernetes/pkg/util/config"
 )
 
 type ControllerManagerConfiguration struct {
@@ -67,6 +68,8 @@ type ControllerManagerConfiguration struct {
 	LeaderElection componentconfig.LeaderElectionConfiguration `json:"leaderElection"`
 	// contentType is contentType of requests sent to apiserver.
 	ContentType string `json:"contentType"`
+	// ConfigurationMap determining which controllers should be enabled or disabled
+	Controllers config.ConfigurationMap `json:"controllers"`
 }
 
 // CMServer is the main context object for the controller manager.
@@ -94,6 +97,7 @@ func NewCMServer() *CMServer {
 			APIServerQPS:              20.0,
 			APIServerBurst:            30,
 			LeaderElection:            leaderelection.DefaultLeaderElectionConfiguration(),
+			Controllers:               make(config.ConfigurationMap),
 		},
 	}
 	return &s
@@ -118,5 +122,9 @@ func (s *CMServer) AddFlags(fs *pflag.FlagSet) {
 	fs.IntVar(&s.APIServerBurst, "federated-api-burst", s.APIServerBurst, "Burst to use while talking with federation apiserver")
 	fs.StringVar(&s.DnsProvider, "dns-provider", s.DnsProvider, "DNS provider. Valid values are: "+fmt.Sprintf("%q", dnsprovider.RegisteredDnsProviders()))
 	fs.StringVar(&s.DnsConfigFile, "dns-provider-config", s.DnsConfigFile, "Path to config file for configuring DNS provider.")
+	fs.Var(&s.Controllers, "controllers", ""+
+		"A set of key=value pairs that describe controller configuration that may be passed "+
+		"to controller manager to enable/disable specific controllers. Valid options are: \n"+
+		"ingress=true|false (default=true)")
 	leaderelection.BindFlags(&s.LeaderElection, fs)
 }

federation/pkg/federation-controller/ingress/BUILD

@@ -30,6 +30,7 @@ go_library(
         "//pkg/controller:go_default_library",
         "//pkg/conversion:go_default_library",
         "//pkg/runtime:go_default_library",
+        "//pkg/runtime/schema:go_default_library",
         "//pkg/types:go_default_library",
         "//pkg/util/flowcontrol:go_default_library",
         "//pkg/watch:go_default_library",

federation/pkg/federation-controller/ingress/ingress_controller.go

@@ -36,6 +36,7 @@ import (
 	"k8s.io/kubernetes/pkg/controller"
 	"k8s.io/kubernetes/pkg/conversion"
 	pkgruntime "k8s.io/kubernetes/pkg/runtime"
+	"k8s.io/kubernetes/pkg/runtime/schema"
 	"k8s.io/kubernetes/pkg/types"
 	"k8s.io/kubernetes/pkg/util/flowcontrol"
 	"k8s.io/kubernetes/pkg/watch"
@@ -58,6 +59,11 @@ const (
 	// We wait for ingress to be created in this cluster before creating it any
 	// other cluster.
 	firstClusterAnnotation = "ingress.federation.kubernetes.io/first-cluster"
+	ControllerName         = "ingress"
+)
+
+var (
+	RequiredResources = []schema.GroupVersionResource{extensionsv1beta1.SchemeGroupVersion.WithResource("ingresses")}
 )
 
 type IngressController struct {
@@ -494,6 +500,12 @@ func (ic *IngressController) reconcileConfigMapForCluster(clusterName string) {
 		return
 	}
 
+	ingressList := ic.ingressInformerStore.List()
+	if len(ingressList) <= 0 {
+		glog.V(4).Infof("No federated ingresses, ignore reconcile config map.")
+		return
+	}
+
 	if clusterName == allClustersKey {
 		clusters, err := ic.configMapFederatedInformer.GetReadyClusters()
 		if err != nil {

federation/pkg/federation-controller/ingress/ingress_controller_test.go

@@ -129,15 +129,15 @@ func TestIngressController(t *testing.T) {
 	t.Log("Adding Ingress UID ConfigMap to cluster 1")
 	cluster1ConfigMapWatch.Add(cfg1)
 
-	t.Log("Checking that UID annotation on Cluster 1 annotation was correctly updated")
-	cluster := GetClusterFromChan(fedClusterUpdateChan)
-	assert.NotNil(t, cluster)
-	assert.Equal(t, cluster.ObjectMeta.Annotations[uidAnnotationKey], cfg1.Data[uidKey])
-
 	// Test add federated ingress.
 	t.Log("Adding Federated Ingress")
 	fedIngressWatch.Add(&fedIngress)
 
+	t.Log("Checking that UID annotation on Cluster 1 annotation was correctly updated after adding Federated Ingress")
+	cluster := GetClusterFromChan(fedClusterUpdateChan)
+	assert.NotNil(t, cluster)
+	assert.Equal(t, cluster.ObjectMeta.Annotations[uidAnnotationKey], cfg1.Data[uidKey])
+
 	t.Logf("Checking that approproate finalizers are added")
 	// There should be 2 updates to add both the finalizers.
 	updatedIngress := GetIngressFromChan(t, fedIngressUpdateChan)

test/test_owners.csv

@@ -519,6 +519,7 @@ k8s.io/kubernetes/cmd/mungedocs,mwielgus,1
 k8s.io/kubernetes/examples,Random-Liu,0
 k8s.io/kubernetes/federation/apis/federation/install,nikhiljindal,0
 k8s.io/kubernetes/federation/apis/federation/validation,nikhiljindal,0
+k8s.io/kubernetes/federation/cmd/federation-controller-manager/app,kzwang,0
 k8s.io/kubernetes/federation/pkg/dnsprovider,sttts,1
 k8s.io/kubernetes/federation/pkg/dnsprovider/providers/aws/route53,cjcullen,1
 k8s.io/kubernetes/federation/pkg/dnsprovider/providers/google/clouddns,jsafrane,1