Merge pull request #41258 from shashidharatd/federation-service-controller-1

federation/apis/federation/BUILD

@@ -19,6 +19,7 @@ go_library(
     tags = ["automanaged"],
     deps = [
         "//pkg/api:go_default_library",
+        "//pkg/api/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/conversion:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",

federation/apis/federation/types.go

@@ -19,6 +19,7 @@ package federation
 import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/api/v1"
 )
 
 // ServerAddressByClientCIDR helps the client to determine the server address that they should use, depending on the clientCIDR that they match.
@@ -153,3 +154,18 @@ type ClusterReplicaSetPreferences struct {
 	// A number expressing the preference to put an additional replica to this LocalReplicaSet. 0 by default.
 	Weight int64
 }
+
+// Annotation for a federated service to keep record of service loadbalancer ingresses in federated cluster
+type FederatedServiceIngress struct {
+	// List of loadbalancer ingress of a service in all federated clusters
+	// +optional
+	Items []ClusterServiceIngress `json:"items,omitempty"`
+}
+
+// Loadbalancer ingresses of a service within a federated cluster
+type ClusterServiceIngress struct {
+	// Cluster is the name of the federated cluster
+	Cluster string `json:"cluster"`
+	// List of loadbalancer ingresses of a federated service within a federated cluster
+	Items []v1.LoadBalancerIngress `json:"items"`
+}

federation/apis/federation/zz_generated.deepcopy.go

@@ -25,6 +25,7 @@ import (
 	conversion "k8s.io/apimachinery/pkg/conversion"
 	runtime "k8s.io/apimachinery/pkg/runtime"
 	api "k8s.io/kubernetes/pkg/api"
+	api_v1 "k8s.io/kubernetes/pkg/api/v1"
 	reflect "reflect"
 )
 
@@ -40,9 +41,11 @@ func RegisterDeepCopies(scheme *runtime.Scheme) error {
 		conversion.GeneratedDeepCopyFunc{Fn: DeepCopy_federation_ClusterCondition, InType: reflect.TypeOf(&ClusterCondition{})},
 		conversion.GeneratedDeepCopyFunc{Fn: DeepCopy_federation_ClusterList, InType: reflect.TypeOf(&ClusterList{})},
 		conversion.GeneratedDeepCopyFunc{Fn: DeepCopy_federation_ClusterReplicaSetPreferences, InType: reflect.TypeOf(&ClusterReplicaSetPreferences{})},
+		conversion.GeneratedDeepCopyFunc{Fn: DeepCopy_federation_ClusterServiceIngress, InType: reflect.TypeOf(&ClusterServiceIngress{})},
 		conversion.GeneratedDeepCopyFunc{Fn: DeepCopy_federation_ClusterSpec, InType: reflect.TypeOf(&ClusterSpec{})},
 		conversion.GeneratedDeepCopyFunc{Fn: DeepCopy_federation_ClusterStatus, InType: reflect.TypeOf(&ClusterStatus{})},
 		conversion.GeneratedDeepCopyFunc{Fn: DeepCopy_federation_FederatedReplicaSetPreferences, InType: reflect.TypeOf(&FederatedReplicaSetPreferences{})},
+		conversion.GeneratedDeepCopyFunc{Fn: DeepCopy_federation_FederatedServiceIngress, InType: reflect.TypeOf(&FederatedServiceIngress{})},
 		conversion.GeneratedDeepCopyFunc{Fn: DeepCopy_federation_ServerAddressByClientCIDR, InType: reflect.TypeOf(&ServerAddressByClientCIDR{})},
 	)
 }
@@ -110,6 +113,20 @@ func DeepCopy_federation_ClusterReplicaSetPreferences(in interface{}, out interf
 	}
 }
 
+func DeepCopy_federation_ClusterServiceIngress(in interface{}, out interface{}, c *conversion.Cloner) error {
+	{
+		in := in.(*ClusterServiceIngress)
+		out := out.(*ClusterServiceIngress)
+		*out = *in
+		if in.Items != nil {
+			in, out := &in.Items, &out.Items
+			*out = make([]api_v1.LoadBalancerIngress, len(*in))
+			copy(*out, *in)
+		}
+		return nil
+	}
+}
+
 func DeepCopy_federation_ClusterSpec(in interface{}, out interface{}, c *conversion.Cloner) error {
 	{
 		in := in.(*ClusterSpec)
@@ -172,6 +189,24 @@ func DeepCopy_federation_FederatedReplicaSetPreferences(in interface{}, out inte
 	}
 }
 
+func DeepCopy_federation_FederatedServiceIngress(in interface{}, out interface{}, c *conversion.Cloner) error {
+	{
+		in := in.(*FederatedServiceIngress)
+		out := out.(*FederatedServiceIngress)
+		*out = *in
+		if in.Items != nil {
+			in, out := &in.Items, &out.Items
+			*out = make([]ClusterServiceIngress, len(*in))
+			for i := range *in {
+				if err := DeepCopy_federation_ClusterServiceIngress(&(*in)[i], &(*out)[i], c); err != nil {
+					return err
+				}
+			}
+		}
+		return nil
+	}
+}
+
 func DeepCopy_federation_ServerAddressByClientCIDR(in interface{}, out interface{}, c *conversion.Cloner) error {
 	{
 		in := in.(*ServerAddressByClientCIDR)

federation/pkg/federation-controller/service/BUILD

@@ -15,11 +15,13 @@ go_library(
         "dns.go",
         "doc.go",
         "endpoint_helper.go",
+        "ingress.go",
         "service_helper.go",
         "servicecontroller.go",
     ],
     tags = ["automanaged"],
     deps = [
+        "//federation/apis/federation:go_default_library",
         "//federation/apis/federation/v1beta1:go_default_library",
         "//federation/client/cache:go_default_library",
         "//federation/client/clientset_generated/federation_clientset:go_default_library",
@@ -36,8 +38,10 @@ go_library(
         "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/conversion:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
@@ -46,6 +50,7 @@ go_library(
         "//vendor/k8s.io/client-go/rest:go_default_library",
         "//vendor/k8s.io/client-go/tools/cache:go_default_library",
         "//vendor/k8s.io/client-go/tools/record:go_default_library",
+        "//vendor/k8s.io/client-go/util/flowcontrol:go_default_library",
         "//vendor/k8s.io/client-go/util/workqueue:go_default_library",
     ],
 )
@@ -62,10 +67,22 @@ go_test(
     tags = ["automanaged"],
     deps = [
         "//federation/apis/federation/v1beta1:go_default_library",
+        "//federation/client/clientset_generated/federation_clientset/fake:go_default_library",
         "//federation/pkg/dnsprovider/providers/google/clouddns:go_default_library",
+        "//federation/pkg/federation-controller/util:go_default_library",
+        "//federation/pkg/federation-controller/util/test:go_default_library",
         "//pkg/api/v1:go_default_library",
+        "//pkg/client/clientset_generated/clientset:go_default_library",
+        "//pkg/client/clientset_generated/clientset/fake:go_default_library",
+        "//pkg/client/listers/core/v1:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/github.com/stretchr/testify/require:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//vendor/k8s.io/client-go/tools/cache:go_default_library",
     ],
 )
 

federation/pkg/federation-controller/service/dns.go

@@ -24,8 +24,10 @@ import (
 
 	"strings"
 
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/kubernetes/federation/pkg/dnsprovider"
 	"k8s.io/kubernetes/federation/pkg/dnsprovider/rrstype"
+	"k8s.io/kubernetes/pkg/api/v1"
 )
 
 const (
@@ -34,7 +36,7 @@ const (
 )
 
 // getHealthyEndpoints returns the hostnames and/or IP addresses of healthy endpoints for the service, at a zone, region and global level (or an error)
-func (s *ServiceController) getHealthyEndpoints(clusterName string, cachedService *cachedService) (zoneEndpoints, regionEndpoints, globalEndpoints []string, err error) {
+func (s *ServiceController) getHealthyEndpoints(clusterName string, service *v1.Service) (zoneEndpoints, regionEndpoints, globalEndpoints []string, err error) {
 	var (
 		zoneNames  []string
 		regionName string
@@ -42,16 +44,24 @@ func (s *ServiceController) getHealthyEndpoints(clusterName string, cachedServic
 	if zoneNames, regionName, err = s.getClusterZoneNames(clusterName); err != nil {
 		return nil, nil, nil, err
 	}
-	for lbClusterName, lbStatus := range cachedService.serviceStatusMap {
+
+	// If federated service is deleted, return empty endpoints, so that DNS records are removed
+	if service.DeletionTimestamp != nil {
+		return zoneEndpoints, regionEndpoints, globalEndpoints, nil
+	}
+
+	serviceIngress, err := ParseFederatedServiceIngress(service)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	for _, lbClusterIngress := range serviceIngress.Items {
+		lbClusterName := lbClusterIngress.Cluster
 		lbZoneNames, lbRegionName, err := s.getClusterZoneNames(lbClusterName)
 		if err != nil {
 			return nil, nil, nil, err
 		}
-		for _, ingress := range lbStatus.Ingress {
-			readyEndpoints, ok := cachedService.endpointMap[lbClusterName]
-			if !ok || readyEndpoints == 0 {
-				continue
-			}
+		for _, ingress := range lbClusterIngress.Items {
 			var address string
 			// We should get either an IP address or a hostname - use whichever one we get
 			if ingress.IP != "" {
@@ -61,7 +71,7 @@ func (s *ServiceController) getHealthyEndpoints(clusterName string, cachedServic
 			}
 			if len(address) <= 0 {
 				return nil, nil, nil, fmt.Errorf("Service %s/%s in cluster %s has neither LoadBalancerStatus.ingress.ip nor LoadBalancerStatus.ingress.hostname. Cannot use it as endpoint for federated service.",
-					cachedService.lastState.Name, cachedService.lastState.Namespace, clusterName)
+					service.Name, service.Namespace, clusterName)
 			}
 			for _, lbZoneName := range lbZoneNames {
 				for _, zoneName := range zoneNames {
@@ -80,15 +90,12 @@ func (s *ServiceController) getHealthyEndpoints(clusterName string, cachedServic
 }
 
 // getClusterZoneNames returns the name of the zones (and the region) where the specified cluster exists (e.g. zones "us-east1-c" on GCE, or "us-east-1b" on AWS)
-func (s *ServiceController) getClusterZoneNames(clusterName string) (zones []string, region string, err error) {
-	client, ok := s.clusterCache.clientMap[clusterName]
-	if !ok {
-		return nil, "", fmt.Errorf("Cluster cache does not contain entry for cluster %s", clusterName)
-	}
-	if client.cluster == nil {
-		return nil, "", fmt.Errorf("Cluster cache entry for cluster %s is nil", clusterName)
+func (s *ServiceController) getClusterZoneNames(clusterName string) ([]string, string, error) {
+	cluster, err := s.federationClient.Federation().Clusters().Get(clusterName, metav1.GetOptions{})
+	if err != nil {
+		return nil, "", err
 	}
-	return client.cluster.Status.Zones, client.cluster.Status.Region, nil
+	return cluster.Status.Zones, cluster.Status.Region, nil
 }
 
 // getServiceDnsSuffix returns the DNS suffix to use when creating federated-service DNS records
@@ -284,7 +291,7 @@ given the current state of that service in that cluster.  This should be called
 (or vice versa).  Only shards of the service which have both a loadbalancer ingress IP address or hostname AND at least one healthy backend endpoint
 are included in DNS records for that service (at all of zone, region and global levels). All other addresses are removed.  Also, if no shards exist
 in the zone or region of the cluster, a CNAME reference to the next higher level is ensured to exist. */
-func (s *ServiceController) ensureDnsRecords(clusterName string, cachedService *cachedService) error {
+func (s *ServiceController) ensureDnsRecords(clusterName string, service *v1.Service) error {
 	// Quinton: Pseudocode....
 	// See https://github.com/kubernetes/kubernetes/pull/25107#issuecomment-218026648
 	// For each service we need the following DNS names:
@@ -298,21 +305,21 @@ func (s *ServiceController) ensureDnsRecords(clusterName string, cachedService *
 	//         - a set of A records to IP addresses of all healthy shards in all regions, if one or more of these exist.
 	//         - no record (NXRECORD response) if no healthy shards exist in any regions
 	//
-	// For each cached service, cachedService.lastState tracks the current known state of the service, while cachedService.appliedState contains
-	// the state of the service when we last successfully synced its DNS records.
-	// So this time around we only need to patch that (add new records, remove deleted records, and update changed records).
+	// Each service has the current known state of loadbalancer ingress for the federated cluster stored in annotations.
+	// So generate the DNS records based on the current state and ensure those desired DNS records match the
+	// actual DNS records (add new records, remove deleted records, and update changed records).
 	//
 	if s == nil {
-		return fmt.Errorf("nil ServiceController passed to ServiceController.ensureDnsRecords(clusterName: %s, cachedService: %v)", clusterName, cachedService)
+		return fmt.Errorf("nil ServiceController passed to ServiceController.ensureDnsRecords(clusterName: %s, service: %v)", clusterName, service)
 	}
 	if s.dns == nil {
 		return nil
 	}
-	if cachedService == nil {
-		return fmt.Errorf("nil cachedService passed to ServiceController.ensureDnsRecords(clusterName: %s, cachedService: %v)", clusterName, cachedService)
+	if service == nil {
+		return fmt.Errorf("nil service passed to ServiceController.ensureDnsRecords(clusterName: %s, service: %v)", clusterName, service)
 	}
-	serviceName := cachedService.lastState.Name
-	namespaceName := cachedService.lastState.Namespace
+	serviceName := service.Name
+	namespaceName := service.Namespace
 	zoneNames, regionName, err := s.getClusterZoneNames(clusterName)
 	if err != nil {
 		return err
@@ -324,7 +331,7 @@ func (s *ServiceController) ensureDnsRecords(clusterName string, cachedService *
 	if err != nil {
 		return err
 	}
-	zoneEndpoints, regionEndpoints, globalEndpoints, err := s.getHealthyEndpoints(clusterName, cachedService)
+	zoneEndpoints, regionEndpoints, globalEndpoints, err := s.getHealthyEndpoints(clusterName, service)
 	if err != nil {
 		return err
 	}

federation/pkg/federation-controller/service/endpoint_helper.go

@@ -122,7 +122,7 @@ func (cc *clusterClientCache) processEndpointDeletion(cachedService *cachedServi
 		glog.V(4).Infof("Cached endpoint was found for %s/%s, cluster %s, removing", cachedService.lastState.Namespace, cachedService.lastState.Name, clusterName)
 		delete(cachedService.endpointMap, clusterName)
 		for i := 0; i < clientRetryCount; i++ {
-			err := serviceController.ensureDnsRecords(clusterName, cachedService)
+			err := serviceController.ensureDnsRecords(clusterName, cachedService.lastState)
 			if err == nil {
 				return nil
 			}
@@ -154,7 +154,7 @@ func (cc *clusterClientCache) processEndpointUpdate(cachedService *cachedService
 			glog.V(4).Infof("Reachable endpoint was found for %s/%s, cluster %s, building endpointMap", endpoint.Namespace, endpoint.Name, clusterName)
 			cachedService.endpointMap[clusterName] = 1
 			for i := 0; i < clientRetryCount; i++ {
-				err := serviceController.ensureDnsRecords(clusterName, cachedService)
+				err := serviceController.ensureDnsRecords(clusterName, cachedService.lastState)
 				if err == nil {
 					return nil
 				}
@@ -175,7 +175,7 @@ func (cc *clusterClientCache) processEndpointUpdate(cachedService *cachedService
 			glog.V(4).Infof("Reachable endpoint was lost for %s/%s, cluster %s, deleting endpointMap", endpoint.Namespace, endpoint.Name, clusterName)
 			delete(cachedService.endpointMap, clusterName)
 			for i := 0; i < clientRetryCount; i++ {
-				err := serviceController.ensureDnsRecords(clusterName, cachedService)
+				err := serviceController.ensureDnsRecords(clusterName, cachedService.lastState)
 				if err == nil {
 					return nil
 				}

federation/pkg/federation-controller/service/endpoint_helper_test.go

@@ -21,14 +21,18 @@ import (
 
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/kubernetes/federation/apis/federation/v1beta1"
+	fakefedclientset "k8s.io/kubernetes/federation/client/clientset_generated/federation_clientset/fake"
 	"k8s.io/kubernetes/federation/pkg/dnsprovider/providers/google/clouddns" // Only for unit testing purposes.
+	. "k8s.io/kubernetes/federation/pkg/federation-controller/util/test"
 	v1 "k8s.io/kubernetes/pkg/api/v1"
 )
 
 var fakeDns, _ = clouddns.NewFakeInterface() // No need to check for unsupported interfaces, as the fake interface supports everything that's required.
 var fakeDnsZones, _ = fakeDns.Zones()
+var fakeClient = &fakefedclientset.Clientset{}
 
 var fakeServiceController = ServiceController{
+	federationClient: fakeClient,
 	dns:              fakeDns,
 	dnsZones:         fakeDnsZones,
 	federationName:   "fed1",
@@ -68,6 +72,7 @@ func TestProcessEndpointUpdate(t *testing.T) {
 			},
 		},
 	}
+	RegisterFakeClusterGet(&fakeClient.Fake, &v1beta1.ClusterList{Items: []v1beta1.Cluster{*NewCluster(clusterName, v1.ConditionTrue)}})
 	tests := []struct {
 		name          string
 		cachedService *cachedService
@@ -121,6 +126,7 @@ func TestProcessEndpointDeletion(t *testing.T) {
 			},
 		},
 	}
+	RegisterFakeClusterGet(&fakeClient.Fake, &v1beta1.ClusterList{Items: []v1beta1.Cluster{*NewCluster(clusterName, v1.ConditionTrue)}})
 	tests := []struct {
 		name          string
 		cachedService *cachedService

federation/pkg/federation-controller/service/ingress.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package service
+
+import (
+	"encoding/json"
+	"sort"
+	"strings"
+
+	fedapi "k8s.io/kubernetes/federation/apis/federation"
+	"k8s.io/kubernetes/pkg/api/v1"
+)
+
+// Compile time check for interface adherence
+var _ sort.Interface = &FederatedServiceIngress{}
+
+const (
+	FederatedServiceIngressAnnotation = "federation.kubernetes.io/service-ingresses"
+)
+
+// FederatedServiceIngress implements sort.Interface.
+type FederatedServiceIngress struct {
+	fedapi.FederatedServiceIngress
+}
+
+func NewFederatedServiceIngress() *FederatedServiceIngress {
+	return &FederatedServiceIngress{}
+}
+
+func (ingress *FederatedServiceIngress) String() string {
+	annotationBytes, _ := json.Marshal(ingress)
+	return string(annotationBytes[:])
+}
+
+// Len is to satisfy of sort.Interface.
+func (ingress *FederatedServiceIngress) Len() int {
+	return len(ingress.Items)
+}
+
+// Less is to satisfy of sort.Interface.
+func (ingress *FederatedServiceIngress) Less(i, j int) bool {
+	return (strings.Compare(ingress.Items[i].Cluster, ingress.Items[j].Cluster) < 0)
+}
+
+// Swap is to satisfy of sort.Interface.
+func (ingress *FederatedServiceIngress) Swap(i, j int) {
+	ingress.Items[i].Cluster, ingress.Items[j].Cluster = ingress.Items[j].Cluster, ingress.Items[i].Cluster
+	ingress.Items[i].Items, ingress.Items[j].Items = ingress.Items[j].Items, ingress.Items[i].Items
+}
+
+// GetClusterLoadBalancerIngresses returns loadbalancer ingresses for given cluster if exist otherwise returns an empty slice
+func (ingress *FederatedServiceIngress) GetClusterLoadBalancerIngresses(cluster string) []v1.LoadBalancerIngress {
+	for _, clusterIngress := range ingress.Items {
+		if cluster == clusterIngress.Cluster {
+			return clusterIngress.Items
+		}
+	}
+	return []v1.LoadBalancerIngress{}
+}
+
+// AddClusterLoadBalancerIngresses adds the ladbalancer ingresses for a given cluster to federated service ingress
+func (ingress *FederatedServiceIngress) AddClusterLoadBalancerIngresses(cluster string, loadbalancerIngresses []v1.LoadBalancerIngress) {
+	for i, clusterIngress := range ingress.Items {
+		if cluster == clusterIngress.Cluster {
+			ingress.Items[i].Items = append(ingress.Items[i].Items, loadbalancerIngresses...)
+			return
+		}
+	}
+	clusterNewIngress := fedapi.ClusterServiceIngress{Cluster: cluster, Items: loadbalancerIngresses}
+	ingress.Items = append(ingress.Items, clusterNewIngress)
+	sort.Sort(ingress)
+}
+
+// AddEndpoints add one or more endpoints to federated service ingress.
+// endpoints are federated cluster's loadbalancer ip/hostname for the service
+func (ingress *FederatedServiceIngress) AddEndpoints(cluster string, endpoints []string) *FederatedServiceIngress {
+	lbIngress := []v1.LoadBalancerIngress{}
+	for _, endpoint := range endpoints {
+		lbIngress = append(lbIngress, v1.LoadBalancerIngress{IP: endpoint})
+	}
+	ingress.AddClusterLoadBalancerIngresses(cluster, lbIngress)
+	return ingress
+}
+
+// RemoveEndpoint removes a single endpoint (ip/hostname) from the federated service ingress
+func (ingress *FederatedServiceIngress) RemoveEndpoint(cluster string, endpoint string) *FederatedServiceIngress {
+	for i, clusterIngress := range ingress.Items {
+		if cluster == clusterIngress.Cluster {
+			for j, lbIngress := range clusterIngress.Items {
+				if lbIngress.IP == endpoint {
+					ingress.Items[i].Items = append(ingress.Items[i].Items[:j], ingress.Items[i].Items[j+1:]...)
+				}
+			}
+		}
+	}
+	return ingress
+}
+
+// ParseFederatedServiceIngress extracts federated service ingresses from a federated service
+func ParseFederatedServiceIngress(service *v1.Service) (*FederatedServiceIngress, error) {
+	ingress := FederatedServiceIngress{}
+	if service.Annotations == nil {
+		return &ingress, nil
+	}
+	federatedServiceIngressString, found := service.Annotations[FederatedServiceIngressAnnotation]
+	if !found {
+		return &ingress, nil
+	}
+	if err := json.Unmarshal([]byte(federatedServiceIngressString), &ingress); err != nil {
+		return &ingress, err
+	}
+	return &ingress, nil
+}
+
+// UpdateIngressAnnotation updates the federated service with service ingress annotation
+func UpdateIngressAnnotation(service *v1.Service, ingress *FederatedServiceIngress) *v1.Service {
+	if service.Annotations == nil {
+		service.Annotations = make(map[string]string)
+	}
+	service.Annotations[FederatedServiceIngressAnnotation] = ingress.String()
+	return service
+}

federation/pkg/federation-controller/service/service_helper.go

@@ -108,7 +108,7 @@ func (cc *clusterClientCache) syncService(key, clusterName string, clusterCache
 
 	if needUpdate {
 		for i := 0; i < clientRetryCount; i++ {
-			err := sc.ensureDnsRecords(clusterName, cachedService)
+			err := sc.ensureDnsRecords(clusterName, service)
 			if err == nil {
 				break
 			}

federation/pkg/federation-controller/util/handlers.go

@@ -77,3 +77,35 @@ func NewTriggerOnMetaAndSpecChanges(triggerFunc func(pkgruntime.Object)) *cache.
 		},
 	}
 }
+
+// Returns cache.ResourceEventHandlerFuncs that trigger the given function
+// on object add/delete or ObjectMeta or given field is updated.
+func NewTriggerOnMetaAndFieldChanges(field string, triggerFunc func(pkgruntime.Object)) *cache.ResourceEventHandlerFuncs {
+	getFieldOrPanic := func(obj interface{}, fieldName string) interface{} {
+		val := reflect.ValueOf(obj).Elem().FieldByName(fieldName)
+		if val.IsValid() {
+			return val.Interface()
+		} else {
+			panic(fmt.Errorf("field not found: %s", fieldName))
+		}
+	}
+	return &cache.ResourceEventHandlerFuncs{
+		DeleteFunc: func(old interface{}) {
+			oldObj := old.(pkgruntime.Object)
+			triggerFunc(oldObj)
+		},
+		AddFunc: func(cur interface{}) {
+			curObj := cur.(pkgruntime.Object)
+			triggerFunc(curObj)
+		},
+		UpdateFunc: func(old, cur interface{}) {
+			curObj := cur.(pkgruntime.Object)
+			oldMeta := getFieldOrPanic(old, "ObjectMeta").(metav1.ObjectMeta)
+			curMeta := getFieldOrPanic(cur, "ObjectMeta").(metav1.ObjectMeta)
+			if !ObjectMetaEquivalent(oldMeta, curMeta) ||
+				!reflect.DeepEqual(getFieldOrPanic(old, field), getFieldOrPanic(cur, field)) {
+				triggerFunc(curObj)
+			}
+		},
+	}
+}

federation/pkg/federation-controller/util/test/test_helper.go

@@ -181,6 +181,38 @@ func RegisterFakeList(resource string, client *core.Fake, obj runtime.Object) {
 	})
 }
 
+// RegisterFakeClusterGet registers a get response for the cluster resource inside the given fake client.
+func RegisterFakeClusterGet(client *core.Fake, obj runtime.Object) {
+	clusterList, ok := obj.(*federationapi.ClusterList)
+	client.AddReactor("get", "clusters", func(action core.Action) (bool, runtime.Object, error) {
+		name := action.(core.GetAction).GetName()
+		if ok {
+			for _, cluster := range clusterList.Items {
+				if cluster.Name == name {
+					return true, &cluster, nil
+				}
+			}
+		}
+		return false, nil, fmt.Errorf("could not find the requested cluster: %s", name)
+	})
+}
+
+// RegisterFakeOnCreate registers a reactor in the given fake client that passes
+// all created objects to the given watcher.
+func RegisterFakeOnCreate(resource string, client *core.Fake, watcher *WatcherDispatcher) {
+	client.AddReactor("create", resource, func(action core.Action) (bool, runtime.Object, error) {
+		createAction := action.(core.CreateAction)
+		originalObj := createAction.GetObject()
+		// Create a copy of the object here to prevent data races while reading the object in go routine.
+		obj := copy(originalObj)
+		watcher.orderExecution <- func() {
+			glog.V(4).Infof("Object created: %v", obj)
+			watcher.Add(obj)
+		}
+		return true, originalObj, nil
+	})
+}
+
 // RegisterFakeCopyOnCreate registers a reactor in the given fake client that passes
 // all created objects to the given watcher and also copies them to a channel for
 // in-test inspection.
@@ -201,6 +233,32 @@ func RegisterFakeCopyOnCreate(resource string, client *core.Fake, watcher *Watch
 	return objChan
 }
 
+// RegisterFakeOnUpdate registers a reactor in the given fake client that passes
+// all updated objects to the given watcher.
+func RegisterFakeOnUpdate(resource string, client *core.Fake, watcher *WatcherDispatcher) {
+	client.AddReactor("update", resource, func(action core.Action) (bool, runtime.Object, error) {
+		updateAction := action.(core.UpdateAction)
+		originalObj := updateAction.GetObject()
+		glog.V(7).Infof("Updating %s: %v", resource, updateAction.GetObject())
+
+		// Create a copy of the object here to prevent data races while reading the object in go routine.
+		obj := copy(originalObj)
+		operation := func() {
+			glog.V(4).Infof("Object updated %v", obj)
+			watcher.Modify(obj)
+		}
+		select {
+		case watcher.orderExecution <- operation:
+			break
+		case <-time.After(pushTimeout):
+			glog.Errorf("Fake client execution channel blocked")
+			glog.Errorf("Tried to push %v", updateAction)
+		}
+		return true, originalObj, nil
+	})
+	return
+}
+
 // RegisterFakeCopyOnUpdate registers a reactor in the given fake client that passes
 // all updated objects to the given watcher and also copies them to a channel for
 // in-test inspection.
@@ -313,6 +371,8 @@ func NewCluster(name string, readyStatus apiv1.ConditionStatus) *federationapi.C
 			Conditions: []federationapi.ClusterCondition{
 				{Type: federationapi.ClusterReady, Status: readyStatus},
 			},
+			Zones:  []string{"foozone"},
+			Region: "fooregion",
 		},
 	}
 }