e2e: network tiers should retry on 404 errors

test/e2e/framework/networking_utils.go

@@ -650,11 +650,20 @@ func TestReachableHTTP(ip string, port int, request string, expect string) (bool
 	return TestReachableHTTPWithContent(ip, port, request, expect, nil)
 }
 
+func TestReachableHTTPWithRetriableErrorCodes(ip string, port int, request string, expect string, retriableErrCodes []int) (bool, error) {
+	return TestReachableHTTPWithContentTimeoutWithRetriableErrorCodes(ip, port, request, expect, nil, retriableErrCodes, time.Second*5)
+}
+
 func TestReachableHTTPWithContent(ip string, port int, request string, expect string, content *bytes.Buffer) (bool, error) {
 	return TestReachableHTTPWithContentTimeout(ip, port, request, expect, content, 5*time.Second)
 }
 
 func TestReachableHTTPWithContentTimeout(ip string, port int, request string, expect string, content *bytes.Buffer, timeout time.Duration) (bool, error) {
+	return TestReachableHTTPWithContentTimeoutWithRetriableErrorCodes(ip, port, request, expect, content, []int{}, timeout)
+}
+
+func TestReachableHTTPWithContentTimeoutWithRetriableErrorCodes(ip string, port int, request string, expect string, content *bytes.Buffer, retriableErrCodes []int, timeout time.Duration) (bool, error) {
+
 	url := fmt.Sprintf("http://%s:%d%s", ip, port, request)
 	if ip == "" {
 		Failf("Got empty IP for reachability check (%s)", url)
@@ -679,6 +688,11 @@ func TestReachableHTTPWithContentTimeout(ip string, port int, request string, ex
 		return false, nil
 	}
 	if resp.StatusCode != 200 {
+		for _, code := range retriableErrCodes {
+			if resp.StatusCode == code {
+				return false, nil
+			}
+		}
 		return false, fmt.Errorf("received non-success return status %q trying to access %s; got body: %s",
 			resp.Status, url, string(body))
 	}

test/e2e/framework/service_util.go

@@ -832,7 +832,13 @@ func (j *ServiceTestJig) LaunchEchoserverPodOnNode(f *Framework, nodeName, podNa
 }
 
 func (j *ServiceTestJig) TestReachableHTTP(host string, port int, timeout time.Duration) {
-	if err := wait.PollImmediate(Poll, timeout, func() (bool, error) { return TestReachableHTTP(host, port, "/echo?msg=hello", "hello") }); err != nil {
+	j.TestReachableHTTPWithRetriableErrorCodes(host, port, []int{}, timeout)
+}
+
+func (j *ServiceTestJig) TestReachableHTTPWithRetriableErrorCodes(host string, port int, retriableErrCodes []int, timeout time.Duration) {
+	if err := wait.PollImmediate(Poll, timeout, func() (bool, error) {
+		return TestReachableHTTPWithRetriableErrorCodes(host, port, "/echo?msg=hello", "hello", retriableErrCodes)
+	}); err != nil {
 		Failf("Could not reach HTTP service through %v:%v after %v: %v", host, port, timeout, err)
 	}
 }

test/e2e/network/network_tiers.go

@@ -18,6 +18,7 @@ package network
 
 import (
 	"fmt"
+	"net/http"
 	"time"
 
 	computealpha "google.golang.org/api/compute/v0.alpha"
@@ -153,7 +154,10 @@ func waitAndVerifyLBWithTier(jig *framework.ServiceTestJig, ns, svcName, existin
 		Expect(ingressIP).To(Equal(svc.Spec.LoadBalancerIP))
 	}
 	jig.SanityCheckService(svc, v1.ServiceTypeLoadBalancer)
-	jig.TestReachableHTTP(ingressIP, svcPort, checkTimeout)
+	// If the IP has been used by previous test, sometimes we get the lingering
+	// 404 errors even after the LB is long gone. Tolerate and retry until the
+	// the new LB is fully established since this feature is still Alpha in GCP.
+	jig.TestReachableHTTPWithRetriableErrorCodes(ingressIP, svcPort, []int{http.StatusNotFound}, checkTimeout)
 
 	// Verify the network tier matches the desired.
 	svcNetTier, err := gcecloud.GetServiceNetworkTier(svc)