Improve the upgrade test for ingress.

test/e2e/framework/ingress_utils.go

@@ -372,6 +372,19 @@ func CleanupGCEIngressController(gceController *GCEIngressController) {
 	}
 }
 
+func (cont *GCEIngressController) ListGlobalForwardingRules() []*compute.ForwardingRule {
+	gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud)
+	fwdList := []*compute.ForwardingRule{}
+	l, err := gceCloud.ListGlobalForwardingRules()
+	Expect(err).NotTo(HaveOccurred())
+	for _, fwd := range l {
+		if cont.isOwned(fwd.Name) {
+			fwdList = append(fwdList, fwd)
+		}
+	}
+	return fwdList
+}
+
 func (cont *GCEIngressController) deleteForwardingRule(del bool) string {
 	msg := ""
 	fwList := []compute.ForwardingRule{}
@@ -394,6 +407,13 @@ func (cont *GCEIngressController) deleteForwardingRule(del bool) string {
 	return msg
 }
 
+func (cont *GCEIngressController) GetGlobalAddress(ipName string) *compute.Address {
+	gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud)
+	ip, err := gceCloud.GetGlobalAddress(ipName)
+	Expect(err).NotTo(HaveOccurred())
+	return ip
+}
+
 func (cont *GCEIngressController) deleteAddresses(del bool) string {
 	msg := ""
 	ipList := []compute.Address{}
@@ -414,6 +434,32 @@ func (cont *GCEIngressController) deleteAddresses(del bool) string {
 	return msg
 }
 
+func (cont *GCEIngressController) ListTargetHttpProxies() []*compute.TargetHttpProxy {
+	gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud)
+	tpList := []*compute.TargetHttpProxy{}
+	l, err := gceCloud.ListTargetHttpProxies()
+	Expect(err).NotTo(HaveOccurred())
+	for _, tp := range l {
+		if cont.isOwned(tp.Name) {
+			tpList = append(tpList, tp)
+		}
+	}
+	return tpList
+}
+
+func (cont *GCEIngressController) ListTargetHttpsProxies() []*compute.TargetHttpsProxy {
+	gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud)
+	tpsList := []*compute.TargetHttpsProxy{}
+	l, err := gceCloud.ListTargetHttpsProxies()
+	Expect(err).NotTo(HaveOccurred())
+	for _, tps := range l {
+		if cont.isOwned(tps.Name) {
+			tpsList = append(tpsList, tps)
+		}
+	}
+	return tpsList
+}
+
 func (cont *GCEIngressController) deleteTargetProxy(del bool) string {
 	msg := ""
 	tpList := []compute.TargetHttpProxy{}
@@ -449,6 +495,19 @@ func (cont *GCEIngressController) deleteTargetProxy(del bool) string {
 	return msg
 }
 
+func (cont *GCEIngressController) ListUrlMaps() []*compute.UrlMap {
+	gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud)
+	umList := []*compute.UrlMap{}
+	l, err := gceCloud.ListUrlMaps()
+	Expect(err).NotTo(HaveOccurred())
+	for _, um := range l {
+		if cont.isOwned(um.Name) {
+			umList = append(umList, um)
+		}
+	}
+	return umList
+}
+
 func (cont *GCEIngressController) deleteURLMap(del bool) (msg string) {
 	gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud)
 	umList, err := gceCloud.ListUrlMaps()
@@ -478,6 +537,19 @@ func (cont *GCEIngressController) deleteURLMap(del bool) (msg string) {
 	return msg
 }
 
+func (cont *GCEIngressController) ListGlobalBackendServices() []*compute.BackendService {
+	gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud)
+	beList := []*compute.BackendService{}
+	l, err := gceCloud.ListGlobalBackendServices()
+	Expect(err).NotTo(HaveOccurred())
+	for _, be := range l {
+		if cont.isOwned(be.Name) {
+			beList = append(beList, be)
+		}
+	}
+	return beList
+}
+
 func (cont *GCEIngressController) deleteBackendService(del bool) (msg string) {
 	gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud)
 	beList, err := gceCloud.ListGlobalBackendServices()
@@ -537,6 +609,19 @@ func (cont *GCEIngressController) deleteHTTPHealthCheck(del bool) (msg string) {
 	return msg
 }
 
+func (cont *GCEIngressController) ListSslCertificates() []*compute.SslCertificate {
+	gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud)
+	sslList := []*compute.SslCertificate{}
+	l, err := gceCloud.ListSslCertificates()
+	Expect(err).NotTo(HaveOccurred())
+	for _, ssl := range l {
+		if cont.isOwned(ssl.Name) {
+			sslList = append(sslList, ssl)
+		}
+	}
+	return sslList
+}
+
 func (cont *GCEIngressController) deleteSSLCertificate(del bool) (msg string) {
 	gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud)
 	sslList, err := gceCloud.ListSslCertificates()
@@ -565,6 +650,19 @@ func (cont *GCEIngressController) deleteSSLCertificate(del bool) (msg string) {
 	return msg
 }
 
+func (cont *GCEIngressController) ListInstanceGroups() []*compute.InstanceGroup {
+	gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud)
+	igList := []*compute.InstanceGroup{}
+	l, err := gceCloud.ListInstanceGroups(cont.Cloud.Zone)
+	Expect(err).NotTo(HaveOccurred())
+	for _, ig := range l {
+		if cont.isOwned(ig.Name) {
+			igList = append(igList, ig)
+		}
+	}
+	return igList
+}
+
 func (cont *GCEIngressController) deleteInstanceGroup(del bool) (msg string) {
 	gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud)
 	// TODO: E2E cloudprovider has only 1 zone, but the cluster can have many.
@@ -658,6 +756,12 @@ func (cont *GCEIngressController) canDelete(resourceName, creationTimestamp stri
 	return canDeleteWithTimestamp(resourceName, creationTimestamp)
 }
 
+// isOwned returns true if the resourceName ends in a suffix matching this
+// controller UID.
+func (cont *GCEIngressController) isOwned(resourceName string) bool {
+	return cont.canDelete(resourceName, "", false)
+}
+
 // canDeleteNEG returns true if either the name contains this controller's UID,
 // or the creationTimestamp exceeds the maxAge and del is set to true.
 func (cont *GCEIngressController) canDeleteNEG(resourceName, creationTimestamp string, delOldResources bool) bool {

test/e2e/framework/nodes_util.go

@@ -40,6 +40,15 @@ func EtcdUpgrade(target_storage, target_version string) error {
 	}
 }
 
+func IngressUpgrade() error {
+	switch TestContext.Provider {
+	case "gce":
+		return ingressUpgradeGCE()
+	default:
+		return fmt.Errorf("IngressUpgrade() is not implemented for provider %s", TestContext.Provider)
+	}
+}
+
 func MasterUpgrade(v string) error {
 	switch TestContext.Provider {
 	case "gce":
@@ -64,6 +73,15 @@ func etcdUpgradeGCE(target_storage, target_version string) error {
 	return err
 }
 
+func ingressUpgradeGCE() error {
+	// Flip glbc image from latest release image to HEAD to simulate an upgrade.
+	// Kubelet should restart glbc automatically.
+	sshResult, err := NodeExec(GetMasterHost(), "sudo sed -i -re 's/(image:)(.*)/\\1 gcr.io\\/e2e-ingress-gce\\/ingress-gce-e2e-glbc-amd64:latest/' /etc/kubernetes/manifests/glbc.manifest")
+	// TODO(rramkumar): Ensure glbc pod is in "Running" state before proceeding.
+	LogSSHResult(sshResult)
+	return err
+}
+
 // TODO(mrhohn): Remove this function when kube-proxy is run as a DaemonSet by default.
 func MasterUpgradeGCEWithKubeProxyDaemonSet(v string, enableKubeProxyDaemonSet bool) error {
 	return masterUpgradeGCE(v, enableKubeProxyDaemonSet)

test/e2e/lifecycle/cluster_upgrade.go

@@ -72,6 +72,11 @@ var kubeProxyDowngradeTests = []upgrades.Test{
 	&upgrades.IngressUpgradeTest{},
 }
 
+// Upgrade ingress with custom image.
+var ingressUpgradeTests = []upgrades.Test{
+	&upgrades.IngressUpgradeTest{},
+}
+
 var _ = SIGDescribe("Upgrade [Feature:Upgrade]", func() {
 	f := framework.NewDefaultFramework("cluster-upgrade")
 
@@ -201,6 +206,31 @@ var _ = SIGDescribe("etcd Upgrade [Feature:EtcdUpgrade]", func() {
 	})
 })
 
+var _ = SIGDescribe("ingress Upgrade [Feature:IngressUpgrade]", func() {
+	f := framework.NewDefaultFramework("ingress-upgrade")
+
+	// Create the frameworks here because we can only create them
+	// in a "Describe".
+	testFrameworks := createUpgradeFrameworks(ingressUpgradeTests)
+	Describe("ingress upgrade", func() {
+		It("should maintain a functioning ingress", func() {
+			upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), "")
+			framework.ExpectNoError(err)
+
+			testSuite := &junit.TestSuite{Name: "ingress upgrade"}
+			ingressTest := &junit.TestCase{Name: "[sig-networking] ingress-upgrade", Classname: "upgrade_tests"}
+			testSuite.TestCases = append(testSuite.TestCases, ingressTest)
+
+			upgradeFunc := func() {
+				start := time.Now()
+				defer finalizeUpgradeTest(start, ingressTest)
+				framework.ExpectNoError(framework.IngressUpgrade())
+			}
+			runUpgradeSuite(f, ingressUpgradeTests, testFrameworks, testSuite, upgCtx, upgrades.IngressUpgrade, upgradeFunc)
+		})
+	})
+})
+
 var _ = Describe("[sig-apps] stateful Upgrade [Feature:StatefulUpgrade]", func() {
 	f := framework.NewDefaultFramework("stateful-upgrade")
 

test/e2e/testing-manifests/ingress/static-ip-2/ing.yaml

+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: static-ip
+  # This annotation is added by the test upon allocating a staticip.
+  # annotations:
+  #  kubernetes.io/ingress.global-static-ip-name: "staticip"
+spec:
+  backend:
+    serviceName: echoheaders-https
+    servicePort: 80

test/e2e/testing-manifests/ingress/static-ip-2/rc.yaml

+apiVersion: v1
+kind: ReplicationController
+metadata:
+  name: echoheaders-https
+spec:
+  replicas: 2
+  template:
+    metadata:
+      labels:
+        app: echoheaders-https
+    spec:
+      containers:
+      - name: echoheaders-https
+        image: gcr.io/google_containers/echoserver:1.6
+        ports:
+        - containerPort: 8080

test/e2e/testing-manifests/ingress/static-ip-2/svc.yaml

+apiVersion: v1
+kind: Service
+metadata:
+  name: echoheaders-https
+  labels:
+    app: echoheaders-https
+spec:
+  type: NodePort
+  ports:
+  - port: 80
+    targetPort: 8080
+    protocol: TCP
+    name: http
+  selector:
+    app: echoheaders-https

test/e2e/testing-manifests/ingress/static-ip/ing.yaml

@@ -13,4 +13,3 @@ spec:
   backend:
     serviceName: echoheaders-https
     servicePort: 80
-

test/e2e/upgrades/BUILD

@@ -29,9 +29,11 @@ go_library(
         "//test/e2e/common:go_default_library",
         "//test/e2e/framework:go_default_library",
         "//test/utils/image:go_default_library",
+        "//vendor/github.com/davecgh/go-spew/spew:go_default_library",
         "//vendor/github.com/onsi/ginkgo:go_default_library",
         "//vendor/github.com/onsi/gomega:go_default_library",
         "//vendor/github.com/onsi/gomega/gstruct:go_default_library",
+        "//vendor/google.golang.org/api/compute/v1:go_default_library",
         "//vendor/k8s.io/api/autoscaling/v1:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
         "//vendor/k8s.io/api/extensions/v1beta1:go_default_library",

test/e2e/upgrades/ingress.go

@@ -17,12 +17,17 @@ limitations under the License.
 package upgrades
 
 import (
+	"encoding/json"
 	"fmt"
 	"net/http"
 	"path/filepath"
+	"reflect"
 
+	"github.com/davecgh/go-spew/spew"
 	. "github.com/onsi/ginkgo"
 
+	compute "google.golang.org/api/compute/v1"
+	extensions "k8s.io/api/extensions/v1beta1"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/kubernetes/test/e2e/framework"
 )
@@ -30,16 +35,34 @@ import (
 // IngressUpgradeTest adapts the Ingress e2e for upgrade testing
 type IngressUpgradeTest struct {
 	gceController *framework.GCEIngressController
+	// holds GCP resources pre-upgrade
+	resourceStore *GCPResourceStore
 	jig           *framework.IngressTestJig
 	httpClient    *http.Client
 	ip            string
 	ipName        string
 }
 
+// GCPResourceStore keeps track of the GCP resources spun up by an ingress.
+// Note: Fields are exported so that we can utilize reflection.
+type GCPResourceStore struct {
+	Fw      *compute.Firewall
+	FwdList []*compute.ForwardingRule
+	UmList  []*compute.UrlMap
+	TpList  []*compute.TargetHttpProxy
+	TpsList []*compute.TargetHttpsProxy
+	SslList []*compute.SslCertificate
+	BeList  []*compute.BackendService
+	Ip      *compute.Address
+	IgList  []*compute.InstanceGroup
+}
+
 func (IngressUpgradeTest) Name() string { return "ingress-upgrade" }
 
 // Setup creates a GLBC, allocates an ip, and an ingress resource,
 // then waits for a successful connectivity check to the ip.
+// Also keeps track of all load balancer resources for cross-checking
+// during an IngressUpgrade.
 func (t *IngressUpgradeTest) Setup(f *framework.Framework) {
 	framework.SkipUnlessProviderIs("gce", "gke")
 
@@ -66,13 +89,18 @@ func (t *IngressUpgradeTest) Setup(f *framework.Framework) {
 
 	// Create a working basic Ingress
 	By(fmt.Sprintf("allocated static ip %v: %v through the GCE cloud provider", t.ipName, t.ip))
-	jig.CreateIngress(filepath.Join(framework.IngressManifestPath, "static-ip"), ns.Name, map[string]string{
+	jig.CreateIngress(filepath.Join(framework.IngressManifestPath, "static-ip-2"), ns.Name, map[string]string{
 		"kubernetes.io/ingress.global-static-ip-name": t.ipName,
 		"kubernetes.io/ingress.allow-http":            "false",
 	}, map[string]string{})
+	jig.AddHTTPS("tls-secret", "ingress.test.com")
 
 	By("waiting for Ingress to come up with ip: " + t.ip)
 	framework.ExpectNoError(framework.PollURL(fmt.Sprintf("https://%v/", t.ip), "", framework.LoadBalancerPollTimeout, jig.PollInterval, t.httpClient, false))
+
+	By("keeping track of GCP resources created by Ingress")
+	t.resourceStore = &GCPResourceStore{}
+	t.populateGCPResourceStore(t.resourceStore)
 }
 
 // Test waits for the upgrade to complete, and then verifies
@@ -85,6 +113,8 @@ func (t *IngressUpgradeTest) Test(f *framework.Framework, done <-chan struct{},
 		// while it's down will leak cloud resources, because the ingress
 		// controller doesn't checkpoint to disk.
 		t.verify(f, done, true)
+	case IngressUpgrade:
+		t.verify(f, done, true)
 	default:
 		// Currently ingress gets disrupted across node upgrade, because endpoints
 		// get killed and we don't have any guarantees that 2 nodes don't overlap
@@ -99,6 +129,7 @@ func (t *IngressUpgradeTest) Teardown(f *framework.Framework) {
 	if CurrentGinkgoTestDescription().Failed {
 		framework.DescribeIng(t.gceController.Ns)
 	}
+
 	if t.jig.Ingress != nil {
 		By("Deleting ingress")
 		t.jig.TryDeleteIngress()
@@ -122,4 +153,80 @@ func (t *IngressUpgradeTest) verify(f *framework.Framework, done <-chan struct{}
 	}
 	By("hitting the Ingress IP " + t.ip)
 	framework.ExpectNoError(framework.PollURL(fmt.Sprintf("https://%v/", t.ip), "", framework.LoadBalancerPollTimeout, t.jig.PollInterval, t.httpClient, false))
+
+	// We want to manually trigger a sync because then we can easily verify
+	// a correct sync completed after update.
+	By("updating ingress spec to manually trigger a sync")
+	t.jig.Update(func(ing *extensions.Ingress) {
+		ing.Spec.TLS[0].Hosts = append(ing.Spec.TLS[0].Hosts, "ingress.test.com")
+		ing.Spec.Rules = append(
+			ing.Spec.Rules,
+			extensions.IngressRule{
+				Host: "ingress.test.com",
+				IngressRuleValue: extensions.IngressRuleValue{
+					HTTP: &extensions.HTTPIngressRuleValue{
+						Paths: []extensions.HTTPIngressPath{
+							{
+								Path: "/test",
+								// Note: Dependant on using "static-ip-2" manifest.
+								Backend: *(ing.Spec.Backend),
+							},
+						},
+					},
+				},
+			})
+	})
+	// WaitForIngress() tests that all paths are pinged, which is how we know
+	// everything is synced with the cloud.
+	t.jig.WaitForIngress(false)
+	By("comparing GCP resources post-upgrade")
+	postUpgradeResourceStore := &GCPResourceStore{}
+	t.populateGCPResourceStore(postUpgradeResourceStore)
+	framework.ExpectNoError(compareGCPResourceStores(t.resourceStore, postUpgradeResourceStore, func(v1 reflect.Value, v2 reflect.Value) error {
+		i1 := v1.Interface()
+		i2 := v2.Interface()
+		// Skip verifying the UrlMap since we did that via WaitForIngress()
+		if !reflect.DeepEqual(i1, i2) && (v1.Type() != reflect.TypeOf([]*compute.UrlMap{})) {
+			return spew.Errorf("resources after ingress upgrade were different:\n Pre-Upgrade: %#v\n Post-Upgrade: %#v", i1, i2)
+		}
+		return nil
+	}))
+}
+
+func (t *IngressUpgradeTest) populateGCPResourceStore(resourceStore *GCPResourceStore) {
+	cont := t.gceController
+	resourceStore.Fw = cont.GetFirewallRule()
+	resourceStore.FwdList = cont.ListGlobalForwardingRules()
+	resourceStore.UmList = cont.ListUrlMaps()
+	resourceStore.TpList = cont.ListTargetHttpProxies()
+	resourceStore.TpsList = cont.ListTargetHttpsProxies()
+	resourceStore.SslList = cont.ListSslCertificates()
+	resourceStore.BeList = cont.ListGlobalBackendServices()
+	resourceStore.Ip = cont.GetGlobalAddress(t.ipName)
+	resourceStore.IgList = cont.ListInstanceGroups()
+}
+
+func compareGCPResourceStores(rs1 *GCPResourceStore, rs2 *GCPResourceStore, compare func(v1 reflect.Value, v2 reflect.Value) error) error {
+	// Before we do a comparison, remove the ServerResponse field from the
+	// Compute API structs. This is needed because two objects could be the same
+	// but their ServerResponse will be different if they were populated through
+	// separate API calls.
+	rs1Json, _ := json.Marshal(rs1)
+	rs2Json, _ := json.Marshal(rs2)
+	rs1New := &GCPResourceStore{}
+	rs2New := &GCPResourceStore{}
+	json.Unmarshal(rs1Json, rs1New)
+	json.Unmarshal(rs2Json, rs2New)
+
+	// Iterate through struct fields and perform equality checks on the fields.
+	// We do this rather than performing a deep equal on the struct itself because
+	// it is easier to log which field, if any, is not the same.
+	rs1V := reflect.ValueOf(*rs1New)
+	rs2V := reflect.ValueOf(*rs2New)
+	for i := 0; i < rs1V.NumField(); i++ {
+		if err := compare(rs1V.Field(i), rs2V.Field(i)); err != nil {
+			return err
+		}
+	}
+	return nil
 }

test/e2e/upgrades/upgrade.go

@@ -40,6 +40,9 @@ const (
 	// EtcdUpgrade indicates that only etcd is being upgraded (or migrated
 	// between storage versions).
 	EtcdUpgrade
+
+	// IngressUpgrade indicates that only ingress is being upgraded.
+	IngressUpgrade
 )
 
 // Test is an interface for upgrade tests.