Add stubDomains to kube-dns configmap for private nameserver

federation/pkg/kubefed/init/init.go

@@ -42,6 +42,7 @@ import (
 	triple "k8s.io/client-go/util/cert/triple"
 	kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
 	"k8s.io/kubernetes/federation/apis/federation"
+	"k8s.io/kubernetes/federation/pkg/dnsprovider/providers/coredns"
 	"k8s.io/kubernetes/federation/pkg/kubefed/util"
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/api/v1"
@@ -55,6 +56,7 @@ import (
 	"github.com/golang/glog"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
+	"gopkg.in/gcfg.v1"
 )
 
 const (
@@ -379,7 +381,7 @@ func (i *initFederation) Run(cmdOut io.Writer, config util.AdminConfig) error {
 
 	glog.V(4).Info("Creating federation controller manager deployment")
 
-	_, err = createControllerManager(hostClientset, i.commonOptions.FederationSystemNamespace, i.commonOptions.Name, svc.Name, cmName, i.options.image, cmKubeconfigName, i.options.dnsZoneName, i.options.dnsProvider, sa.Name, dnsProviderSecret, i.options.controllerManagerOverrides, i.options.dryRun)
+	_, err = createControllerManager(hostClientset, i.commonOptions.FederationSystemNamespace, i.commonOptions.Name, svc.Name, cmName, i.options.image, cmKubeconfigName, i.options.dnsZoneName, i.options.dnsProvider, i.options.dnsProviderConfig, sa.Name, dnsProviderSecret, i.options.controllerManagerOverrides, i.options.dryRun)
 	if err != nil {
 		return err
 	}
@@ -847,7 +849,7 @@ func createRoleBindings(clientset client.Interface, namespace, saName, federatio
 	return newRole, newRolebinding, err
 }
 
-func createControllerManager(clientset client.Interface, namespace, name, svcName, cmName, image, kubeconfigName, dnsZoneName, dnsProvider, saName string, dnsProviderSecret *api.Secret, argOverrides map[string]string, dryRun bool) (*extensions.Deployment, error) {
+func createControllerManager(clientset client.Interface, namespace, name, svcName, cmName, image, kubeconfigName, dnsZoneName, dnsProvider, dnsProviderConfig, saName string, dnsProviderSecret *api.Secret, argOverrides map[string]string, dryRun bool) (*extensions.Deployment, error) {
 	command := []string{
 		"/hyperkube",
 		"federation-controller-manager",
@@ -935,12 +937,19 @@ func createControllerManager(clientset client.Interface, namespace, name, svcNam
 		dep.Spec.Template.Spec.ServiceAccountName = saName
 	}
 
-	if dryRun {
-		return dep, nil
-	}
-
 	if dnsProviderSecret != nil {
 		dep = addDNSProviderConfig(dep, dnsProviderSecret.Name)
+		if dnsProvider == util.FedDNSProviderCoreDNS {
+			var err error
+			dep, err = addCoreDNSServerAnnotation(dep, dnsZoneName, dnsProviderConfig)
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	if dryRun {
+		return dep, nil
 	}
 
 	return clientset.Extensions().Deployments(namespace).Create(dep)
@@ -1154,3 +1163,15 @@ func addDNSProviderConfig(dep *extensions.Deployment, secretName string) *extens
 func authFileContents(username, authSecret string) []byte {
 	return []byte(fmt.Sprintf("%s,%s,%s\n", authSecret, username, uuid.NewUUID()))
 }
+
+func addCoreDNSServerAnnotation(deployment *extensions.Deployment, dnsZoneName, dnsProviderConfig string) (*extensions.Deployment, error) {
+	var cfg coredns.Config
+	if err := gcfg.ReadFileInto(&cfg, dnsProviderConfig); err != nil {
+		return nil, err
+	}
+
+	deployment.Annotations[util.FedDNSZoneName] = dnsZoneName
+	deployment.Annotations[util.FedNameServer] = cfg.Global.CoreDNSEndpoints
+	deployment.Annotations[util.FedDNSProvider] = util.FedDNSProviderCoreDNS
+	return deployment, nil
+}

federation/pkg/kubefed/init/init_test.go

@@ -44,6 +44,7 @@ import (
 	"k8s.io/client-go/rest/fake"
 	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/kubernetes/federation/apis/federation"
+	"k8s.io/kubernetes/federation/pkg/dnsprovider/providers/coredns"
 	kubefedtesting "k8s.io/kubernetes/federation/pkg/kubefed/testing"
 	"k8s.io/kubernetes/federation/pkg/kubefed/util"
 	"k8s.io/kubernetes/pkg/api"
@@ -55,6 +56,8 @@ import (
 	rbacv1beta1 "k8s.io/kubernetes/pkg/apis/rbac/v1beta1"
 	cmdtesting "k8s.io/kubernetes/pkg/kubectl/cmd/testing"
 	cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
+
+	"gopkg.in/gcfg.v1"
 )
 
 const (
@@ -74,7 +77,6 @@ const (
 
 func TestInitFederation(t *testing.T) {
 	cmdErrMsg := ""
-	dnsProvider := "google-clouddns"
 	cmdutil.BehaviorOnFatal(func(str string, code int) {
 		cmdErrMsg = str
 	})
@@ -97,6 +99,7 @@ func TestInitFederation(t *testing.T) {
 		etcdPVCapacity               string
 		etcdPersistence              string
 		expectedErr                  string
+		dnsProvider                  string
 		dnsProviderConfig            string
 		dryRun                       string
 		apiserverArgOverrides        string
@@ -116,6 +119,7 @@ func TestInitFederation(t *testing.T) {
 			etcdPVCapacity:        "5Gi",
 			etcdPersistence:       "true",
 			expectedErr:           "",
+			dnsProvider:           util.FedDNSProviderCoreDNS,
 			dnsProviderConfig:     "dns-provider.conf",
 			dryRun:                "",
 			apiserverArgOverrides: "--client-ca-file=override,--log-dir=override",
@@ -210,6 +214,9 @@ func TestInitFederation(t *testing.T) {
 		tmpDirPath := ""
 		buf := bytes.NewBuffer([]byte{})
 
+		if tc.dnsProvider == "" {
+			tc.dnsProvider = "google-clouddns"
+		}
 		if tc.dnsProviderConfig != "" {
 			tmpfile, err := ioutil.TempFile("", tc.dnsProviderConfig)
 			if err != nil {
@@ -227,7 +234,7 @@ func TestInitFederation(t *testing.T) {
 		}
 		defer os.Remove(tmpDirPath)
 
-		hostFactory, err := fakeInitHostFactory(tc.apiserverServiceType, tc.federation, util.DefaultFederationSystemNamespace, tc.advertiseAddress, tc.lbIP, tc.dnsZoneName, tc.image, dnsProvider, tc.dnsProviderConfig, tc.etcdPersistence, tc.etcdPVCapacity, tc.apiserverArgOverrides, tc.cmArgOverrides, tmpDirPath, tc.apiserverEnableHTTPBasicAuth, tc.apiserverEnableTokenAuth, tc.isRBACAPIAvailable)
+		hostFactory, err := fakeInitHostFactory(tc.apiserverServiceType, tc.federation, util.DefaultFederationSystemNamespace, tc.advertiseAddress, tc.lbIP, tc.dnsZoneName, tc.image, tc.dnsProvider, tc.dnsProviderConfig, tc.etcdPersistence, tc.etcdPVCapacity, tc.apiserverArgOverrides, tc.cmArgOverrides, tmpDirPath, tc.apiserverEnableHTTPBasicAuth, tc.apiserverEnableTokenAuth, tc.isRBACAPIAvailable)
 		if err != nil {
 			t.Fatalf("[%d] unexpected error: %v", i, err)
 		}
@@ -243,7 +250,7 @@ func TestInitFederation(t *testing.T) {
 		cmd.Flags().Set("host-cluster-context", "substrate")
 		cmd.Flags().Set("dns-zone-name", tc.dnsZoneName)
 		cmd.Flags().Set("image", tc.image)
-		cmd.Flags().Set("dns-provider", dnsProvider)
+		cmd.Flags().Set("dns-provider", tc.dnsProvider)
 		cmd.Flags().Set("apiserver-arg-overrides", tc.apiserverArgOverrides)
 		cmd.Flags().Set("controllermanager-arg-overrides", tc.cmArgOverrides)
 
@@ -1043,6 +1050,12 @@ func fakeInitHostFactory(apiserverServiceType v1.ServiceType, federationName, na
 	}
 	if dnsProviderConfig != "" {
 		cm = addDNSProviderConfigTest(cm, cmDNSProviderSecret.Name)
+		if dnsProvider == util.FedDNSProviderCoreDNS {
+			cm, err = addCoreDNSServerAnnotationTest(cm, dnsZoneName, dnsProviderConfig)
+			if err != nil {
+				return nil, err
+			}
+		}
 	}
 
 	podList := v1.PodList{}
@@ -1540,3 +1553,16 @@ func addDNSProviderConfigTest(dep *v1beta1.Deployment, secretName string) *v1bet
 
 	return dep
 }
+
+// TODO: Reuse the function addCoreDNSServerAnnotation once that function is converted to use versioned objects.
+func addCoreDNSServerAnnotationTest(deployment *v1beta1.Deployment, dnsZoneName, dnsProviderConfig string) (*v1beta1.Deployment, error) {
+	var cfg coredns.Config
+	if err := gcfg.ReadFileInto(&cfg, dnsProviderConfig); err != nil {
+		return nil, err
+	}
+
+	deployment.Annotations[util.FedDNSZoneName] = dnsZoneName
+	deployment.Annotations[util.FedNameServer] = cfg.Global.CoreDNSEndpoints
+	deployment.Annotations[util.FedDNSProvider] = util.FedDNSProviderCoreDNS
+	return deployment, nil
+}

federation/pkg/kubefed/join.go

@@ -278,6 +278,7 @@ func createConfigMap(hostClientSet internalclientset.Interface, config util.Admi
 				util.FedDomainMapKey: domainMap,
 			},
 		}
+		newConfigMap = populateStubDomainsIfRequired(newConfigMap, cmDep.Annotations)
 
 		if dryRun {
 			return newConfigMap, nil
@@ -397,3 +398,15 @@ func getFederationName(hostClientSet internalclientset.Interface, fedNamespace s
 
 	return name, nil
 }
+
+func populateStubDomainsIfRequired(configMap *api.ConfigMap, annotations map[string]string) *api.ConfigMap {
+	dnsProvider := annotations[util.FedDNSProvider]
+	dnsZoneName := annotations[util.FedDNSZoneName]
+	nameServer := annotations[util.FedNameServer]
+
+	if dnsProvider != util.FedDNSProviderCoreDNS || dnsZoneName == "" || nameServer == "" {
+		return configMap
+	}
+	configMap.Data[util.KubeDnsStubDomains] = fmt.Sprintf(`{"%s":["%s"]}`, dnsZoneName, nameServer)
+	return configMap
+}

federation/pkg/kubefed/join_test.go

@@ -42,10 +42,15 @@ import (
 	cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
 )
 
-// testFederationName is a name to use for the federation in tests. Since the federation
-// name is recovered from the federation itself, this constant is an appropriate
-// functional replica.
-const testFederationName = "test-federation"
+const (
+	// testFederationName is a name to use for the federation in tests. Since the federation
+	// name is recovered from the federation itself, this constant is an appropriate
+	// functional replica.
+	testFederationName = "test-federation"
+
+	zoneName      = "test-dns-zone"
+	coreDNSServer = "11.22.33.44:53"
+)
 
 func TestJoinFederation(t *testing.T) {
 	cmdErrMsg := ""
@@ -69,6 +74,7 @@ func TestJoinFederation(t *testing.T) {
 		kubeconfigExplicit string
 		expectedServer     string
 		expectedErr        string
+		dnsProvider        string
 	}{
 		{
 			cluster:            "syndicate",
@@ -79,6 +85,7 @@ func TestJoinFederation(t *testing.T) {
 			kubeconfigExplicit: "",
 			expectedServer:     "https://10.20.30.40",
 			expectedErr:        "",
+			dnsProvider:        util.FedDNSProviderCoreDNS,
 		},
 		{
 			cluster:            "ally",
@@ -138,12 +145,12 @@ func TestJoinFederation(t *testing.T) {
 		f := testJoinFederationFactory(tc.cluster, tc.secret, tc.expectedServer)
 		buf := bytes.NewBuffer([]byte{})
 
-		hostFactory, err := fakeJoinHostFactory(tc.cluster, tc.clusterCtx, tc.secret, tc.server, tc.token)
+		hostFactory, err := fakeJoinHostFactory(tc.cluster, tc.clusterCtx, tc.secret, tc.server, tc.token, tc.dnsProvider)
 		if err != nil {
 			t.Fatalf("[%d] unexpected error: %v", i, err)
 		}
 
-		targetClusterFactory, err := fakeJoinTargetClusterFactory(tc.cluster, tc.clusterCtx)
+		targetClusterFactory, err := fakeJoinTargetClusterFactory(tc.cluster, tc.clusterCtx, tc.dnsProvider)
 		if err != nil {
 			t.Fatalf("[%d] unexpected error: %v", i, err)
 		}
@@ -229,7 +236,7 @@ func testJoinFederationFactory(clusterName, secretName, server string) cmdutil.F
 	return f
 }
 
-func fakeJoinHostFactory(clusterName, clusterCtx, secretName, server, token string) (cmdutil.Factory, error) {
+func fakeJoinHostFactory(clusterName, clusterCtx, secretName, server, token, dnsProvider string) (cmdutil.Factory, error) {
 	if clusterCtx == "" {
 		clusterCtx = clusterName
 	}
@@ -281,28 +288,26 @@ func fakeJoinHostFactory(clusterName, clusterCtx, secretName, server, token stri
 	}
 
 	cmName := "controller-manager"
-	deploymentList := v1beta1.DeploymentList{
+	deployment := v1beta1.Deployment{
 		TypeMeta: metav1.TypeMeta{
-			Kind:       "DeploymentList",
+			Kind:       "Deployment",
 			APIVersion: testapi.Extensions.GroupVersion().String(),
 		},
-		Items: []v1beta1.Deployment{
-			{
-				TypeMeta: metav1.TypeMeta{
-					Kind:       "Deployment",
-					APIVersion: testapi.Extensions.GroupVersion().String(),
-				},
-				ObjectMeta: metav1.ObjectMeta{
-					Name:      cmName,
-					Namespace: util.DefaultFederationSystemNamespace,
-					Annotations: map[string]string{
-						util.FedDomainMapKey:                fmt.Sprintf("%s=%s", clusterCtx, "test-dns-zone"),
-						federation.FederationNameAnnotation: testFederationName,
-					},
-				},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      cmName,
+			Namespace: util.DefaultFederationSystemNamespace,
+			Annotations: map[string]string{
+				util.FedDomainMapKey:                fmt.Sprintf("%s=%s", clusterCtx, zoneName),
+				federation.FederationNameAnnotation: testFederationName,
 			},
 		},
 	}
+	if dnsProvider == util.FedDNSProviderCoreDNS {
+		deployment.Annotations[util.FedDNSZoneName] = zoneName
+		deployment.Annotations[util.FedNameServer] = coreDNSServer
+		deployment.Annotations[util.FedDNSProvider] = util.FedDNSProviderCoreDNS
+	}
+	deploymentList := v1beta1.DeploymentList{Items: []v1beta1.Deployment{deployment}}
 
 	f, tf, codec, _ := cmdtesting.NewAPIFactory()
 	extensionCodec := testapi.Extensions.Codec()
@@ -346,12 +351,12 @@ func fakeJoinHostFactory(clusterName, clusterCtx, secretName, server, token stri
 	return f, nil
 }
 
-func fakeJoinTargetClusterFactory(clusterName, clusterCtx string) (cmdutil.Factory, error) {
+func fakeJoinTargetClusterFactory(clusterName, clusterCtx, dnsProvider string) (cmdutil.Factory, error) {
 	if clusterCtx == "" {
 		clusterCtx = clusterName
 	}
 
-	configmapObject := v1.ConfigMap{
+	configmapObject := &v1.ConfigMap{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      util.KubeDnsConfigmapName,
 			Namespace: metav1.NamespaceSystem,
@@ -361,9 +366,17 @@ func fakeJoinTargetClusterFactory(clusterName, clusterCtx string) (cmdutil.Facto
 			},
 		},
 		Data: map[string]string{
-			util.FedDomainMapKey: fmt.Sprintf("%s=%s", clusterCtx, "test-dns-zone"),
+			util.FedDomainMapKey: fmt.Sprintf("%s=%s", clusterCtx, zoneName),
 		},
 	}
+	if dnsProvider == util.FedDNSProviderCoreDNS {
+		annotations := map[string]string{
+			util.FedDNSProvider: util.FedDNSProviderCoreDNS,
+			util.FedDNSZoneName: zoneName,
+			util.FedNameServer:  coreDNSServer,
+		}
+		configmapObject = populateStubDomainsIfRequiredTest(configmapObject, annotations)
+	}
 
 	f, tf, codec, _ := cmdtesting.NewAPIFactory()
 	ns := dynamic.ContentConfig().NegotiatedSerializer
@@ -383,10 +396,10 @@ func fakeJoinTargetClusterFactory(clusterName, clusterCtx string) (cmdutil.Facto
 				if err != nil {
 					return nil, err
 				}
-				if !apiequality.Semantic.DeepEqual(got, configmapObject) {
-					return nil, fmt.Errorf("Unexpected configmap object\n\tDiff: %s", diff.ObjectGoPrintDiff(got, configmapObject))
+				if !apiequality.Semantic.DeepEqual(&got, configmapObject) {
+					return nil, fmt.Errorf("Unexpected configmap object\n\tDiff: %s", diff.ObjectGoPrintDiff(&got, configmapObject))
 				}
-				return &http.Response{StatusCode: http.StatusCreated, Header: kubefedtesting.DefaultHeader(), Body: kubefedtesting.ObjBody(codec, &configmapObject)}, nil
+				return &http.Response{StatusCode: http.StatusCreated, Header: kubefedtesting.DefaultHeader(), Body: kubefedtesting.ObjBody(codec, configmapObject)}, nil
 			default:
 				return nil, fmt.Errorf("unexpected request: %#v\n%#v", req.URL, req)
 			}
@@ -413,3 +426,16 @@ func fakeCluster(clusterName, secretName, server string) federationapi.Cluster {
 		},
 	}
 }
+
+// TODO: Reuse the function populateStubDomainsIfRequired once that function is converted to use versioned objects.
+func populateStubDomainsIfRequiredTest(configMap *v1.ConfigMap, annotations map[string]string) *v1.ConfigMap {
+	dnsProvider := annotations[util.FedDNSProvider]
+	dnsZoneName := annotations[util.FedDNSZoneName]
+	nameServer := annotations[util.FedNameServer]
+
+	if dnsProvider != util.FedDNSProviderCoreDNS || dnsZoneName == "" || nameServer == "" {
+		return configMap
+	}
+	configMap.Data[util.KubeDnsStubDomains] = fmt.Sprintf(`{"%s":["%s"]}`, dnsZoneName, nameServer)
+	return configMap
+}

federation/pkg/kubefed/unjoin.go

@@ -111,10 +111,10 @@ func (u *unjoinFederation) Run(f cmdutil.Factory, cmdOut, cmdErr io.Writer, conf
 			// We anyways continue to try and delete the config map but with above warning
 		}
 
-		// We need to ensure deleting the config map created in the deregistered cluster
-		// This configmap was created when the cluster joined this federation to aid
+		// We need to ensure updating the config map created in the deregistered cluster
+		// This configmap was created/updated when the cluster joined this federation to aid
 		// the kube-dns of that cluster to aid service discovery.
-		err = deleteConfigMapFromCluster(hostClientset, secret, cluster, u.commonOptions.FederationSystemNamespace)
+		err = updateConfigMapFromCluster(hostClientset, secret, cluster, u.commonOptions.FederationSystemNamespace)
 		if err != nil {
 			fmt.Fprintf(cmdErr, "WARNING: Encountered error in deleting kube-dns configmap, %v", err)
 			// We anyways continue to print success message but with above warning
@@ -162,7 +162,7 @@ func popCluster(f cmdutil.Factory, name string) (*federationapi.Cluster, error)
 	return cluster, rh.Delete("", name)
 }
 
-func deleteConfigMapFromCluster(hostClientset internalclientset.Interface, secret *api.Secret, cluster *federationapi.Cluster, fedSystemNamespace string) error {
+func updateConfigMapFromCluster(hostClientset internalclientset.Interface, secret *api.Secret, cluster *federationapi.Cluster, fedSystemNamespace string) error {
 	clientset, err := getClientsetFromCluster(secret, cluster)
 	if err != nil {
 		return err
@@ -182,12 +182,20 @@ func deleteConfigMapFromCluster(hostClientset internalclientset.Interface, secre
 		return err
 	}
 
-	if _, ok := configMap.Data[util.FedDomainMapKey]; !ok {
-		return nil
+	needUpdate := false
+	if _, ok := configMap.Data[util.FedDomainMapKey]; ok {
+		configMap.Data[util.FedDomainMapKey] = removeConfigMapString(configMap.Data[util.FedDomainMapKey], domainMap)
+		needUpdate = true
+	}
+
+	if _, ok := configMap.Data[util.KubeDnsStubDomains]; ok {
+		delete(configMap.Data, util.KubeDnsStubDomains)
+		needUpdate = true
 	}
-	configMap.Data[util.FedDomainMapKey] = removeConfigMapString(configMap.Data[util.FedDomainMapKey], domainMap)
 
-	_, err = clientset.Core().ConfigMaps(metav1.NamespaceSystem).Update(configMap)
+	if needUpdate {
+		_, err = clientset.Core().ConfigMaps(metav1.NamespaceSystem).Update(configMap)
+	}
 	return err
 }
 

federation/pkg/kubefed/util/util.go

@@ -44,8 +44,13 @@ const (
 	KubeconfigSecretDataKey = "kubeconfig"
 
 	// Used in and to create the kube-dns configmap storing the zone info
-	FedDomainMapKey      = "federations"
-	KubeDnsConfigmapName = "kube-dns"
+	FedDomainMapKey       = "federations"
+	KubeDnsConfigmapName  = "kube-dns"
+	FedDNSZoneName        = "dns-zone-name"
+	FedNameServer         = "nameserver"
+	FedDNSProvider        = "dns-provider"
+	FedDNSProviderCoreDNS = "coredns"
+	KubeDnsStubDomains    = "stubDomains"
 
 	// DefaultFederationSystemNamespace is the namespace in which
 	// federation system components are hosted.