Merge pull request #45524 from MrHohn/l4-lb-healthcheck

pkg/cloudprovider/providers/gce/BUILD

@@ -41,7 +41,9 @@ go_library(
         "//pkg/client/clientset_generated/clientset:go_default_library",
         "//pkg/cloudprovider:go_default_library",
         "//pkg/controller:go_default_library",
+        "//pkg/master/ports:go_default_library",
         "//pkg/util/net/sets:go_default_library",
+        "//pkg/util/version:go_default_library",
         "//pkg/volume:go_default_library",
         "//vendor/cloud.google.com/go/compute/metadata:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
@@ -70,11 +72,13 @@ go_test(
     name = "go_default_test",
     srcs = [
         "gce_disks_test.go",
+        "gce_healthchecks_test.go",
         "gce_test.go",
     ],
     library = ":go_default_library",
     tags = ["automanaged"],
     deps = [
+        "//pkg/api/v1:go_default_library",
         "//pkg/cloudprovider:go_default_library",
         "//vendor/google.golang.org/api/compute/v1:go_default_library",
         "//vendor/google.golang.org/api/googleapi:go_default_library",

pkg/cloudprovider/providers/gce/gce.go

@@ -22,6 +22,7 @@ import (
 	"net/http"
 	"regexp"
 	"strings"
+	"sync"
 	"time"
 
 	"cloud.google.com/go/compute/metadata"
@@ -80,7 +81,7 @@ type GCECloud struct {
 	serviceBeta              *computebeta.Service
 	containerService         *container.Service
 	clientBuilder            controller.ControllerClientBuilder
-	ClusterId                ClusterId
+	ClusterID                ClusterID
 	projectID                string
 	region                   string
 	localZone                string   // The zone in which we are running
@@ -92,6 +93,11 @@ type GCECloud struct {
 	useMetadataServer        bool
 	operationPollRateLimiter flowcontrol.RateLimiter
 	manager                  ServiceManager
+	// sharedResourceLock is used to serialize GCE operations that may mutate shared state to
+	// prevent inconsistencies. For example, load balancers manipulation methods will take the
+	// lock to prevent shared resources from being prematurely deleted while the operation is
+	// in progress.
+	sharedResourceLock sync.Mutex
 }
 
 type ServiceManager interface {
@@ -270,10 +276,10 @@ func CreateGCECloud(projectID, region, zone string, managedZones []string, netwo
 }
 
 // Initialize takes in a clientBuilder and spawns a goroutine for watching the clusterid configmap.
-// This must be called before utilizing the funcs of gce.ClusterId
+// This must be called before utilizing the funcs of gce.ClusterID
 func (gce *GCECloud) Initialize(clientBuilder controller.ControllerClientBuilder) {
 	gce.clientBuilder = clientBuilder
-	go gce.watchClusterId()
+	go gce.watchClusterID()
 }
 
 // LoadBalancer returns an implementation of LoadBalancer for Google Compute Engine.

pkg/cloudprovider/providers/gce/gce_clusterid.go

 /*
-Copyright 2014 The Kubernetes Authors.
+Copyright 2017 The Kubernetes Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -49,18 +49,18 @@ const (
 	updateFuncFrequency = 10 * time.Minute
 )
 
-type ClusterId struct {
+type ClusterID struct {
 	idLock     sync.RWMutex
 	client     clientset.Interface
 	cfgMapKey  string
 	store      cache.Store
-	providerId *string
-	clusterId  *string
+	providerID *string
+	clusterID  *string
 }
 
-// Continually watches for changes to the cluser id config map
-func (gce *GCECloud) watchClusterId() {
-	gce.ClusterId = ClusterId{
+// Continually watches for changes to the cluster id config map
+func (gce *GCECloud) watchClusterID() {
+	gce.ClusterID = ClusterID{
 		cfgMapKey: fmt.Sprintf("%v/%v", UIDNamespace, UIDConfigMapName),
 		client:    gce.clientBuilder.ClientOrDie("cloud-provider"),
 	}
@@ -77,8 +77,8 @@ func (gce *GCECloud) watchClusterId() {
 				return
 			}
 
-			glog.V(4).Infof("Observed new configmap for clusterid: %v, %v; setting local values", m.Name, m.Data)
-			gce.ClusterId.setIds(m)
+			glog.V(4).Infof("Observed new configmap for clusteriD: %v, %v; setting local values", m.Name, m.Data)
+			gce.ClusterID.update(m)
 		},
 		UpdateFunc: func(old, cur interface{}) {
 			m, ok := cur.(*v1.ConfigMap)
@@ -96,71 +96,71 @@ func (gce *GCECloud) watchClusterId() {
 				return
 			}
 
-			glog.V(4).Infof("Observed updated configmap for clusterid %v, %v; setting local values", m.Name, m.Data)
-			gce.ClusterId.setIds(m)
+			glog.V(4).Infof("Observed updated configmap for clusteriD %v, %v; setting local values", m.Name, m.Data)
+			gce.ClusterID.update(m)
 		},
 	}
 
-	listerWatcher := cache.NewListWatchFromClient(gce.ClusterId.client.Core().RESTClient(), "configmaps", UIDNamespace, fields.Everything())
+	listerWatcher := cache.NewListWatchFromClient(gce.ClusterID.client.Core().RESTClient(), "configmaps", UIDNamespace, fields.Everything())
 	var controller cache.Controller
-	gce.ClusterId.store, controller = cache.NewInformer(newSingleObjectListerWatcher(listerWatcher, UIDConfigMapName), &v1.ConfigMap{}, updateFuncFrequency, mapEventHandler)
+	gce.ClusterID.store, controller = cache.NewInformer(newSingleObjectListerWatcher(listerWatcher, UIDConfigMapName), &v1.ConfigMap{}, updateFuncFrequency, mapEventHandler)
 
 	controller.Run(nil)
 }
 
-// GetId returns the id which is unique to this cluster
+// GetID returns the id which is unique to this cluster
 // if federated, return the provider id (unique to the cluster)
 // if not federated, return the cluster id
-func (ci *ClusterId) GetId() (string, error) {
+func (ci *ClusterID) GetID() (string, error) {
 	if err := ci.getOrInitialize(); err != nil {
 		return "", err
 	}
 
 	ci.idLock.RLock()
 	defer ci.idLock.RUnlock()
-	if ci.clusterId == nil {
+	if ci.clusterID == nil {
 		return "", errors.New("Could not retrieve cluster id")
 	}
 
 	// If provider ID is set, (Federation is enabled) use this field
-	if ci.providerId != nil && *ci.providerId != *ci.clusterId {
-		return *ci.providerId, nil
+	if ci.providerID != nil {
+		return *ci.providerID, nil
 	}
 
-	// providerId is not set, use the cluster id
-	return *ci.clusterId, nil
+	// providerID is not set, use the cluster id
+	return *ci.clusterID, nil
 }
 
 // GetFederationId returns the id which could represent the entire Federation
 // or just the cluster if not federated.
-func (ci *ClusterId) GetFederationId() (string, bool, error) {
+func (ci *ClusterID) GetFederationId() (string, bool, error) {
 	if err := ci.getOrInitialize(); err != nil {
 		return "", false, err
 	}
 
 	ci.idLock.RLock()
 	defer ci.idLock.RUnlock()
-	if ci.clusterId == nil {
+	if ci.clusterID == nil {
 		return "", false, errors.New("Could not retrieve cluster id")
 	}
 
 	// If provider ID is not set, return false
-	if ci.providerId == nil || *ci.clusterId == *ci.providerId {
+	if ci.providerID == nil || *ci.clusterID == *ci.providerID {
 		return "", false, nil
 	}
 
-	return *ci.clusterId, true, nil
+	return *ci.clusterID, true, nil
 }
 
 // getOrInitialize either grabs the configmaps current value or defines the value
-// and sets the configmap. This is for the case of the user calling GetClusterId()
+// and sets the configmap. This is for the case of the user calling GetClusterID()
 // before the watch has begun.
-func (ci *ClusterId) getOrInitialize() error {
+func (ci *ClusterID) getOrInitialize() error {
 	if ci.store == nil {
-		return errors.New("GCECloud.ClusterId is not ready. Call Initialize() before using.")
+		return errors.New("GCECloud.ClusterID is not ready. Call Initialize() before using.")
 	}
 
-	if ci.clusterId != nil {
+	if ci.clusterID != nil {
 		return nil
 	}
 
@@ -177,7 +177,7 @@ func (ci *ClusterId) getOrInitialize() error {
 		return err
 	}
 
-	glog.V(4).Infof("Creating clusterid: %v", newId)
+	glog.V(4).Infof("Creating clusteriD: %v", newId)
 	cfg := &v1.ConfigMap{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      UIDConfigMapName,
@@ -194,12 +194,12 @@ func (ci *ClusterId) getOrInitialize() error {
 		return err
 	}
 
-	glog.V(2).Infof("Created a config map containing clusterid: %v", newId)
-	ci.setIds(cfg)
+	glog.V(2).Infof("Created a config map containing clusteriD: %v", newId)
+	ci.update(cfg)
 	return nil
 }
 
-func (ci *ClusterId) getConfigMap() (bool, error) {
+func (ci *ClusterID) getConfigMap() (bool, error) {
 	item, exists, err := ci.store.GetByKey(ci.cfgMapKey)
 	if err != nil {
 		return false, err
@@ -214,18 +214,18 @@ func (ci *ClusterId) getConfigMap() (bool, error) {
 		glog.Error(err)
 		return false, err
 	}
-	ci.setIds(m)
+	ci.update(m)
 	return true, nil
 }
 
-func (ci *ClusterId) setIds(m *v1.ConfigMap) {
+func (ci *ClusterID) update(m *v1.ConfigMap) {
 	ci.idLock.Lock()
 	defer ci.idLock.Unlock()
-	if clusterId, exists := m.Data[UIDCluster]; exists {
-		ci.clusterId = &clusterId
+	if clusterID, exists := m.Data[UIDCluster]; exists {
+		ci.clusterID = &clusterID
 	}
 	if provId, exists := m.Data[UIDProvider]; exists {
-		ci.providerId = &provId
+		ci.providerID = &provId
 	}
 }
 

pkg/cloudprovider/providers/gce/gce_healthchecks.go

@@ -17,11 +17,23 @@ limitations under the License.
 package gce
 
 import (
+	"fmt"
 	"time"
 
+	"k8s.io/kubernetes/pkg/api/v1"
+	"k8s.io/kubernetes/pkg/master/ports"
+	utilversion "k8s.io/kubernetes/pkg/util/version"
+
+	"github.com/golang/glog"
 	compute "google.golang.org/api/compute/v1"
 )
 
+const (
+	minNodesHealthCheckVersion = "1.7.0"
+	nodesHealthCheckPath       = "/healthz"
+	lbNodesHealthCheckPort     = ports.ProxyHealthzPort
+)
+
 func newHealthcheckMetricContext(request string) *metricContext {
 	return &metricContext{
 		start:      time.Now(),
@@ -178,3 +190,59 @@ func (gce *GCECloud) ListHealthChecks() (*compute.HealthCheckList, error) {
 	v, err := gce.service.HealthChecks.List(gce.projectID).Do()
 	return v, mc.Observe(err)
 }
+
+// GetNodesHealthCheckPort returns the health check port used by the GCE load
+// balancers (l4) for performing health checks on nodes.
+func GetNodesHealthCheckPort() int32 {
+	return lbNodesHealthCheckPort
+}
+
+// getNodesHealthCheckPath returns the health check path used by the GCE load
+// balancers (l4) for performing health checks on nodes.
+func getNodesHealthCheckPath() string {
+	return nodesHealthCheckPath
+}
+
+// makeNodesHealthCheckName returns name of the health check resource used by
+// the GCE load balancers (l4) for performing health checks on nodes.
+func makeNodesHealthCheckName(clusterID string) string {
+	return fmt.Sprintf("k8s-%v-node", clusterID)
+}
+
+// MakeHealthCheckFirewallName returns the firewall name used by the GCE load
+// balancers (l4) for performing health checks.
+func MakeHealthCheckFirewallName(clusterID, hcName string, isNodesHealthCheck bool) string {
+	// TODO: Change below fwName to match the proposed schema: k8s-{clusteriD}-{namespace}-{name}-{shortid}-hc.
+	fwName := "k8s-" + hcName + "-http-hc"
+	if isNodesHealthCheck {
+		fwName = makeNodesHealthCheckName(clusterID) + "-http-hc"
+	}
+	return fwName
+}
+
+// isAtLeastMinNodesHealthCheckVersion checks if a version is higher than
+// `minNodesHealthCheckVersion`.
+func isAtLeastMinNodesHealthCheckVersion(vstring string) bool {
+	minVersion, err := utilversion.ParseGeneric(minNodesHealthCheckVersion)
+	if err != nil {
+		glog.Errorf("MinNodesHealthCheckVersion (%s) is not a valid version string: %v", minNodesHealthCheckVersion, err)
+		return false
+	}
+	version, err := utilversion.ParseGeneric(vstring)
+	if err != nil {
+		glog.Errorf("vstring (%s) is not a valid version string: %v", vstring, err)
+		return false
+	}
+	return version.AtLeast(minVersion)
+}
+
+// supportsNodesHealthCheck returns false if anyone of the nodes has version
+// lower than `minNodesHealthCheckVersion`.
+func supportsNodesHealthCheck(nodes []*v1.Node) bool {
+	for _, node := range nodes {
+		if !isAtLeastMinNodesHealthCheckVersion(node.Status.NodeInfo.KubeProxyVersion) {
+			return false
+		}
+	}
+	return true
+}

pkg/cloudprovider/providers/gce/gce_healthchecks_test.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gce
+
+import (
+	"testing"
+
+	"k8s.io/kubernetes/pkg/api/v1"
+)
+
+func TestIsAtLeastMinNodesHealthCheckVersion(t *testing.T) {
+	testCases := []struct {
+		version string
+		expect  bool
+	}{
+		{"v1.7.1", true},
+		{"v1.7.0-alpha.2.597+276d289b90d322", true},
+		{"v1.6.0-beta.3.472+831q821c907t31a", false},
+		{"v1.5.2", false},
+	}
+
+	for _, tc := range testCases {
+		if res := isAtLeastMinNodesHealthCheckVersion(tc.version); res != tc.expect {
+			t.Errorf("%v: want %v, got %v", tc.version, tc.expect, res)
+		}
+	}
+}
+
+func TestSupportsNodesHealthCheck(t *testing.T) {
+	testCases := []struct {
+		desc   string
+		nodes  []*v1.Node
+		expect bool
+	}{
+		{
+			"All nodes support nodes health check",
+			[]*v1.Node{
+				{
+					Status: v1.NodeStatus{
+						NodeInfo: v1.NodeSystemInfo{
+							KubeProxyVersion: "v1.7.1",
+						},
+					},
+				},
+				{
+					Status: v1.NodeStatus{
+						NodeInfo: v1.NodeSystemInfo{
+							KubeProxyVersion: "v1.7.0-alpha.2.597+276d289b90d322",
+						},
+					},
+				},
+			},
+			true,
+		},
+		{
+			"All nodes don't support nodes health check",
+			[]*v1.Node{
+				{
+					Status: v1.NodeStatus{
+						NodeInfo: v1.NodeSystemInfo{
+							KubeProxyVersion: "v1.6.0-beta.3.472+831q821c907t31a",
+						},
+					},
+				},
+				{
+					Status: v1.NodeStatus{
+						NodeInfo: v1.NodeSystemInfo{
+							KubeProxyVersion: "v1.5.2",
+						},
+					},
+				},
+			},
+			false,
+		},
+		{
+			"One node doesn't support nodes health check",
+			[]*v1.Node{
+				{
+					Status: v1.NodeStatus{
+						NodeInfo: v1.NodeSystemInfo{
+							KubeProxyVersion: "v1.7.1",
+						},
+					},
+				},
+				{
+					Status: v1.NodeStatus{
+						NodeInfo: v1.NodeSystemInfo{
+							KubeProxyVersion: "v1.7.0-alpha.2.597+276d289b90d322",
+						},
+					},
+				},
+				{
+					Status: v1.NodeStatus{
+						NodeInfo: v1.NodeSystemInfo{
+							KubeProxyVersion: "v1.5.2",
+						},
+					},
+				},
+			},
+			false,
+		},
+	}
+
+	for _, tc := range testCases {
+		if res := supportsNodesHealthCheck(tc.nodes); res != tc.expect {
+			t.Errorf("%v: want %v, got %v", tc.desc, tc.expect, res)
+		}
+	}
+}

pkg/cloudprovider/providers/gce/gce_util.go

@@ -19,10 +19,12 @@ package gce
 import (
 	"errors"
 	"fmt"
+	"net/http"
 	"regexp"
 	"strings"
 
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/sets"
 
 	"cloud.google.com/go/compute/metadata"
 	compute "google.golang.org/api/compute/v1"
@@ -105,6 +107,14 @@ func isHTTPErrorCode(err error, code int) bool {
 	return ok && apiErr.Code == code
 }
 
+func isInUsedByError(err error) bool {
+	apiErr, ok := err.(*googleapi.Error)
+	if !ok || apiErr.Code != http.StatusBadRequest {
+		return false
+	}
+	return strings.Contains(apiErr.Message, "being used by")
+}
+
 // splitProviderID splits a provider's id into core components.
 // A providerID is build out of '${ProviderName}://${project-id}/${zone}/${instance-name}'
 // See cloudprovider.GetInstanceProviderID.
@@ -115,3 +125,12 @@ func splitProviderID(providerID string) (project, zone, instance string, err err
 	}
 	return matches[1], matches[2], matches[3], nil
 }
+
+func equalStringSets(x, y []string) bool {
+	if len(x) != len(y) {
+		return false
+	}
+	xString := sets.NewString(x...)
+	yString := sets.NewString(y...)
+	return xString.Equal(yString)
+}

test/e2e/firewall.go

@@ -22,6 +22,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/kubernetes/pkg/api/v1"
 	"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
+	"k8s.io/kubernetes/pkg/cloudprovider"
 	gcecloud "k8s.io/kubernetes/pkg/cloudprovider/providers/gce"
 	"k8s.io/kubernetes/pkg/master/ports"
 	"k8s.io/kubernetes/test/e2e/framework"
@@ -45,13 +46,18 @@ var _ = framework.KubeDescribe("Firewall rule", func() {
 		gceCloud = cloudConfig.Provider.(*gcecloud.GCECloud)
 	})
 
-	// This test takes around 4 minutes to run
+	// This test takes around 6 minutes to run
 	It("[Slow] [Serial] should create valid firewall rules for LoadBalancer type service", func() {
 		ns := f.Namespace.Name
 		// This source ranges is just used to examine we have exact same things on LB firewall rules
 		firewallTestSourceRanges := []string{"0.0.0.0/1", "128.0.0.0/1"}
 		serviceName := "firewall-test-loadbalancer"
 
+		By("Getting cluster ID")
+		clusterID, err := framework.GetClusterID(cs)
+		Expect(err).NotTo(HaveOccurred())
+		framework.Logf("Got cluster ID: %v", clusterID)
+
 		jig := framework.NewServiceTestJig(cs, serviceName)
 		nodesNames := jig.GetNodesNames(framework.MaxNodesForEndpointsTests)
 		if len(nodesNames) <= 0 {
@@ -59,28 +65,52 @@ var _ = framework.KubeDescribe("Firewall rule", func() {
 		}
 		nodesSet := sets.NewString(nodesNames...)
 
-		// OnlyLocal service is needed to examine which exact nodes the requests are being forwarded to by the Load Balancer on GCE
-		By("Creating a LoadBalancer type service with ExternalTrafficPolicy=Local")
-		svc := jig.CreateOnlyLocalLoadBalancerService(ns, serviceName,
-			framework.LoadBalancerCreateTimeoutDefault, false, func(svc *v1.Service) {
-				svc.Spec.Ports = []v1.ServicePort{{Protocol: "TCP", Port: framework.FirewallTestHttpPort}}
-				svc.Spec.LoadBalancerSourceRanges = firewallTestSourceRanges
-			})
+		By("Creating a LoadBalancer type service with ExternalTrafficPolicy=Global")
+		svc := jig.CreateLoadBalancerService(ns, serviceName, framework.LoadBalancerCreateTimeoutDefault, func(svc *v1.Service) {
+			svc.Spec.Ports = []v1.ServicePort{{Protocol: "TCP", Port: framework.FirewallTestHttpPort}}
+			svc.Spec.LoadBalancerSourceRanges = firewallTestSourceRanges
+		})
 		defer func() {
 			jig.UpdateServiceOrFail(svc.Namespace, svc.Name, func(svc *v1.Service) {
 				svc.Spec.Type = v1.ServiceTypeNodePort
 				svc.Spec.LoadBalancerSourceRanges = nil
 			})
 			Expect(cs.Core().Services(svc.Namespace).Delete(svc.Name, nil)).NotTo(HaveOccurred())
+			By("Waiting for the local traffic health check firewall rule to be deleted")
+			localHCFwName := framework.MakeHealthCheckFirewallNameForLBService(clusterID, cloudprovider.GetLoadBalancerName(svc), false)
+			_, err := framework.WaitForFirewallRule(gceCloud, localHCFwName, false, framework.LoadBalancerCleanupTimeout)
+			Expect(err).NotTo(HaveOccurred())
 		}()
 		svcExternalIP := svc.Status.LoadBalancer.Ingress[0].IP
 
-		By("Checking if service's firewall rules are correct")
+		By("Checking if service's firewall rule is correct")
 		nodeTags := framework.GetInstanceTags(cloudConfig, nodesNames[0])
-		expFw := framework.ConstructFirewallForLBService(svc, nodeTags.Items)
-		fw, err := gceCloud.GetFirewall(expFw.Name)
+		lbFw := framework.ConstructFirewallForLBService(svc, nodeTags.Items)
+		fw, err := gceCloud.GetFirewall(lbFw.Name)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(framework.VerifyFirewallRule(fw, lbFw, cloudConfig.Network, false)).NotTo(HaveOccurred())
+
+		By("Checking if service's nodes health check firewall rule is correct")
+		nodesHCFw := framework.ConstructHealthCheckFirewallForLBService(clusterID, svc, nodeTags.Items, true)
+		fw, err = gceCloud.GetFirewall(nodesHCFw.Name)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(framework.VerifyFirewallRule(fw, nodesHCFw, cloudConfig.Network, false)).NotTo(HaveOccurred())
+
+		// OnlyLocal service is needed to examine which exact nodes the requests are being forwarded to by the Load Balancer on GCE
+		By("Updating LoadBalancer service to ExternalTrafficPolicy=Local")
+		svc = jig.UpdateServiceOrFail(svc.Namespace, svc.Name, func(svc *v1.Service) {
+			svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyTypeLocal
+		})
+
+		By("Waiting for the nodes health check firewall rule to be deleted")
+		_, err = framework.WaitForFirewallRule(gceCloud, nodesHCFw.Name, false, framework.LoadBalancerCleanupTimeout)
+		Expect(err).NotTo(HaveOccurred())
+
+		By("Waiting for the correct local traffic health check firewall rule to be created")
+		localHCFw := framework.ConstructHealthCheckFirewallForLBService(clusterID, svc, nodeTags.Items, false)
+		fw, err = framework.WaitForFirewallRule(gceCloud, localHCFw.Name, true, framework.LoadBalancerCreateTimeoutDefault)
 		Expect(err).NotTo(HaveOccurred())
-		Expect(framework.VerifyFirewallRule(fw, expFw, cloudConfig.Network, false)).NotTo(HaveOccurred())
+		Expect(framework.VerifyFirewallRule(fw, localHCFw, cloudConfig.Network, false)).NotTo(HaveOccurred())
 
 		By(fmt.Sprintf("Creating netexec pods on at most %v nodes", framework.MaxNodesForEndpointsTests))
 		for i, nodeName := range nodesNames {
@@ -100,7 +130,7 @@ var _ = framework.KubeDescribe("Firewall rule", func() {
 		// by removing the tag on one vm and make sure it doesn't get any traffic. This is an imperfect
 		// simulation, we really want to check that traffic doesn't reach a vm outside the GKE cluster, but
 		// that's much harder to do in the current e2e framework.
-		By("Removing tags from one of the nodes")
+		By(fmt.Sprintf("Removing tags from one of the nodes: %v", nodesNames[0]))
 		nodesSet.Delete(nodesNames[0])
 		removedTags := framework.SetInstanceTags(cloudConfig, nodesNames[0], []string{})
 		defer func() {

test/e2e/framework/BUILD

@@ -45,6 +45,7 @@ go_library(
         "//pkg/api/v1/helper:go_default_library",
         "//pkg/api/v1/node:go_default_library",
         "//pkg/api/v1/pod:go_default_library",
+        "//pkg/api/v1/service:go_default_library",
         "//pkg/apis/apps/v1beta1:go_default_library",
         "//pkg/apis/authorization/v1beta1:go_default_library",
         "//pkg/apis/batch:go_default_library",

test/e2e/framework/firewall_util.go

@@ -18,12 +18,16 @@ package framework
 
 import (
 	"fmt"
+	"net/http"
 	"strconv"
 	"strings"
 	"time"
 
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/kubernetes/pkg/api/v1"
+	apiservice "k8s.io/kubernetes/pkg/api/v1/service"
 	"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
 	"k8s.io/kubernetes/pkg/cloudprovider"
 	gcecloud "k8s.io/kubernetes/pkg/cloudprovider/providers/gce"
@@ -41,7 +45,7 @@ const (
 )
 
 // MakeFirewallNameForLBService return the expected firewall name for a LB service.
-// This should match the formatting of makeFirewallName() in pkg/cloudprovider/providers/gce/gce.go
+// This should match the formatting of makeFirewallName() in pkg/cloudprovider/providers/gce/gce_loadbalancer.go
 func MakeFirewallNameForLBService(name string) string {
 	return fmt.Sprintf("k8s-fw-%s", name)
 }
@@ -68,6 +72,32 @@ func ConstructFirewallForLBService(svc *v1.Service, nodesTags []string) *compute
 	return &fw
 }
 
+func MakeHealthCheckFirewallNameForLBService(clusterID, name string, isNodesHealthCheck bool) string {
+	return gcecloud.MakeHealthCheckFirewallName(clusterID, name, isNodesHealthCheck)
+}
+
+// ConstructHealthCheckFirewallForLBService returns the expected GCE firewall rule for a loadbalancer type service
+func ConstructHealthCheckFirewallForLBService(clusterID string, svc *v1.Service, nodesTags []string, isNodesHealthCheck bool) *compute.Firewall {
+	if svc.Spec.Type != v1.ServiceTypeLoadBalancer {
+		Failf("can not construct firewall rule for non-loadbalancer type service")
+	}
+	fw := compute.Firewall{}
+	fw.Name = MakeHealthCheckFirewallNameForLBService(clusterID, cloudprovider.GetLoadBalancerName(svc), isNodesHealthCheck)
+	fw.TargetTags = nodesTags
+	fw.SourceRanges = gcecloud.LoadBalancerSrcRanges()
+	healthCheckPort := gcecloud.GetNodesHealthCheckPort()
+	if !isNodesHealthCheck {
+		healthCheckPort = apiservice.GetServiceHealthCheckNodePort(svc)
+	}
+	fw.Allowed = []*compute.FirewallAllowed{
+		{
+			IPProtocol: "tcp",
+			Ports:      []string{fmt.Sprintf("%d", healthCheckPort)},
+		},
+	}
+	return &fw
+}
+
 // GetNodeTags gets tags from one of the Kubernetes nodes
 func GetNodeTags(c clientset.Interface, cloudConfig CloudConfig) *compute.Tags {
 	nodes := GetReadySchedulableNodesOrDie(c)
@@ -303,6 +333,9 @@ func SameStringArray(result, expected []string, include bool) error {
 // VerifyFirewallRule verifies whether the result firewall is consistent with the expected firewall.
 // When `portsSubset` is false, match given ports exactly. Otherwise, only check ports are included.
 func VerifyFirewallRule(res, exp *compute.Firewall, network string, portsSubset bool) error {
+	if res == nil || exp == nil {
+		return fmt.Errorf("res and exp must not be nil")
+	}
 	if res.Name != exp.Name {
 		return fmt.Errorf("incorrect name: %v, expected %v", res.Name, exp.Name)
 	}
@@ -325,3 +358,40 @@ func VerifyFirewallRule(res, exp *compute.Firewall, network string, portsSubset
 	}
 	return nil
 }
+
+func WaitForFirewallRule(gceCloud *gcecloud.GCECloud, fwName string, exist bool, timeout time.Duration) (*compute.Firewall, error) {
+	Logf("Waiting up to %v for firewall %v exist=%v", timeout, fwName, exist)
+	var fw *compute.Firewall
+	var err error
+
+	condition := func() (bool, error) {
+		fw, err = gceCloud.GetFirewall(fwName)
+		if err != nil && exist ||
+			err == nil && !exist ||
+			err != nil && !exist && !IsGoogleAPIHTTPErrorCode(err, http.StatusNotFound) {
+			return false, nil
+		}
+		return true, nil
+	}
+
+	if err := wait.PollImmediate(5*time.Second, timeout, condition); err != nil {
+		return nil, fmt.Errorf("error waiting for firewall %v exist=%v", fwName, exist)
+	}
+	return fw, nil
+}
+
+func GetClusterID(c clientset.Interface) (string, error) {
+	cm, err := c.Core().ConfigMaps(metav1.NamespaceSystem).Get(gcecloud.UIDConfigMapName, metav1.GetOptions{})
+	if err != nil || cm == nil {
+		return "", fmt.Errorf("error getting cluster ID: %v", err)
+	}
+	clusterID, clusterIDExists := cm.Data[gcecloud.UIDCluster]
+	providerID, providerIDExists := cm.Data[gcecloud.UIDProvider]
+	if !clusterIDExists {
+		return "", fmt.Errorf("cluster ID not set")
+	}
+	if providerIDExists {
+		return providerID, nil
+	}
+	return clusterID, nil
+}

test/e2e/framework/service_util.go

@@ -235,6 +235,25 @@ func (j *ServiceTestJig) CreateOnlyLocalLoadBalancerService(namespace, serviceNa
 	return svc
 }
 
+// CreateLoadBalancerService creates a loadbalancer service and waits
+// for it to acquire an ingress IP.
+func (j *ServiceTestJig) CreateLoadBalancerService(namespace, serviceName string, timeout time.Duration, tweak func(svc *v1.Service)) *v1.Service {
+	By("creating a service " + namespace + "/" + serviceName + " with type=LoadBalancer")
+	svc := j.CreateTCPServiceOrFail(namespace, func(svc *v1.Service) {
+		svc.Spec.Type = v1.ServiceTypeLoadBalancer
+		// We need to turn affinity off for our LB distribution tests
+		svc.Spec.SessionAffinity = v1.ServiceAffinityNone
+		if tweak != nil {
+			tweak(svc)
+		}
+	})
+
+	By("waiting for loadbalancer for service " + namespace + "/" + serviceName)
+	svc = j.WaitForLoadBalancerOrFail(namespace, serviceName, timeout)
+	j.SanityCheckService(svc, v1.ServiceTypeLoadBalancer)
+	return svc
+}
+
 func GetNodeAddresses(node *v1.Node, addressType v1.NodeAddressType) (ips []string) {
 	for j := range node.Status.Addresses {
 		nodeAddress := &node.Status.Addresses[j]

test/e2e/framework/util.go

@@ -5181,12 +5181,16 @@ func CleanupGCEResources(loadBalancerName string) (retErr error) {
 	if err := DeleteGCEStaticIP(loadBalancerName); err != nil {
 		Logf("%v", err)
 	}
+	var hcNames []string
 	hc, getErr := gceCloud.GetHttpHealthCheck(loadBalancerName)
 	if getErr != nil && !IsGoogleAPIHTTPErrorCode(getErr, http.StatusNotFound) {
 		retErr = fmt.Errorf("%v\n%v", retErr, getErr)
 		return
 	}
-	if err := gceCloud.DeleteTargetPool(loadBalancerName, hc); err != nil &&
+	if hc != nil {
+		hcNames = append(hcNames, hc.Name)
+	}
+	if err := gceCloud.DeleteTargetPool(loadBalancerName, hcNames...); err != nil &&
 		!IsGoogleAPIHTTPErrorCode(err, http.StatusNotFound) {
 		retErr = fmt.Errorf("%v\n%v", retErr, err)
 	}