Merge pull request #59313 from MrHohn/e2e-ingress-scale-new

test/e2e/network/BUILD

@@ -16,6 +16,7 @@ go_library(
         "firewall.go",
         "framework.go",
         "ingress.go",
+        "ingress_scale.go",
         "kube_proxy.go",
         "network_policy.go",
         "network_tiers.go",
@@ -40,6 +41,7 @@ go_library(
         "//pkg/master/ports:go_default_library",
         "//test/e2e/framework:go_default_library",
         "//test/e2e/manifest:go_default_library",
+        "//test/e2e/network/scale:go_default_library",
         "//test/images/net/nat:go_default_library",
         "//test/utils:go_default_library",
         "//test/utils/image:go_default_library",
@@ -80,6 +82,9 @@ filegroup(
 
 filegroup(
     name = "all-srcs",
-    srcs = [":package-srcs"],
+    srcs = [
+        ":package-srcs",
+        "//test/e2e/network/scale:all-srcs",
+    ],
     tags = ["automanaged"],
 )

test/e2e/network/ingress.go

@@ -90,7 +90,8 @@ var _ = SIGDescribe("Loadbalancing: L7", func() {
 				Client: jig.Client,
 				Cloud:  framework.TestContext.CloudConfig,
 			}
-			gceController.Init()
+			err := gceController.Init()
+			Expect(err).NotTo(HaveOccurred())
 		})
 
 		// Platform specific cleanup
@@ -106,7 +107,7 @@ var _ = SIGDescribe("Loadbalancing: L7", func() {
 			jig.TryDeleteIngress()
 
 			By("Cleaning up cloud resources")
-			framework.CleanupGCEIngressController(gceController)
+			Expect(gceController.CleanupGCEIngressController()).NotTo(HaveOccurred())
 		})
 
 		It("should conform to Ingress spec", func() {
@@ -356,7 +357,8 @@ var _ = SIGDescribe("Loadbalancing: L7", func() {
 			}, map[string]string{})
 
 			By("Test that ingress works with the pre-shared certificate")
-			jig.WaitForIngressWithCert(true, []string{testHostname}, cert)
+			err = jig.WaitForIngressWithCert(true, []string{testHostname}, cert)
+			Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Unexpected error while waiting for ingress: %v", err))
 		})
 
 		It("multicluster ingress should get instance group annotation", func() {
@@ -398,7 +400,8 @@ var _ = SIGDescribe("Loadbalancing: L7", func() {
 				Client: jig.Client,
 				Cloud:  framework.TestContext.CloudConfig,
 			}
-			gceController.Init()
+			err := gceController.Init()
+			Expect(err).NotTo(HaveOccurred())
 		})
 
 		// Platform specific cleanup
@@ -414,7 +417,7 @@ var _ = SIGDescribe("Loadbalancing: L7", func() {
 			jig.TryDeleteIngress()
 
 			By("Cleaning up cloud resources")
-			framework.CleanupGCEIngressController(gceController)
+			Expect(gceController.CleanupGCEIngressController()).NotTo(HaveOccurred())
 		})
 
 		It("should conform to Ingress spec", func() {

test/e2e/network/ingress_scale.go

+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package network
+
+import (
+	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/network/scale"
+
+	. "github.com/onsi/ginkgo"
+)
+
+var _ = SIGDescribe("Loadbalancing: L7 Scalability", func() {
+	defer GinkgoRecover()
+	var (
+		ns string
+	)
+	f := framework.NewDefaultFramework("ingress-scale")
+
+	BeforeEach(func() {
+		ns = f.Namespace.Name
+	})
+
+	Describe("GCE [Slow] [Serial] [Feature:IngressScale]", func() {
+		var (
+			scaleFramework *scale.IngressScaleFramework
+		)
+
+		BeforeEach(func() {
+			framework.SkipUnlessProviderIs("gce", "gke")
+
+			scaleFramework = scale.NewIngressScaleFramework(f.ClientSet, ns, framework.TestContext.CloudConfig)
+			if err := scaleFramework.PrepareScaleTest(); err != nil {
+				framework.Failf("Unexpected error while preraring ingress scale test: %v", err)
+			}
+		})
+
+		AfterEach(func() {
+			if errs := scaleFramework.CleanupScaleTest(); len(errs) != 0 {
+				framework.Failf("Unexpected error while cleaning up ingress scale test: %v", errs)
+			}
+		})
+
+		It("Creating and updating ingresses should happen promptly with small/medium/large amount of ingresses", func() {
+			if errs := scaleFramework.RunScaleTest(); len(errs) != 0 {
+				framework.Failf("Unexpected error while running ingress scale test: %v", errs)
+			}
+
+		})
+	})
+})

test/e2e/network/scale/BUILD

+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["ingress.go"],
+    importpath = "k8s.io/kubernetes/test/e2e/network/scale",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//test/e2e/framework:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/api/extensions/v1beta1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//test/e2e/network/scale/localrun:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

test/e2e/network/scale/localrun/BUILD

+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["ingress_scale.go"],
+    importpath = "k8s.io/kubernetes/test/e2e/network/scale/localrun",
+    visibility = ["//visibility:private"],
+    deps = [
+        "//pkg/cloudprovider/providers/gce:go_default_library",
+        "//test/e2e/framework:go_default_library",
+        "//test/e2e/network/scale:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes:go_default_library",
+        "//vendor/k8s.io/client-go/tools/clientcmd:go_default_library",
+    ],
+)
+
+go_binary(
+    name = "localrun",
+    embed = [":go_default_library"],
+    importpath = "k8s.io/kubernetes/test/e2e/network/scale/localrun",
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

test/e2e/network/scale/localrun/ingress_scale.go

+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strconv"
+
+	"github.com/golang/glog"
+
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+	gcecloud "k8s.io/kubernetes/pkg/cloudprovider/providers/gce"
+
+	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/network/scale"
+)
+
+var (
+	kubeconfig       string
+	enableTLS        bool
+	numIngressesTest numIngressesSlice
+	testNamespace    string
+	cloudConfig      framework.CloudConfig
+	outputFile       string
+	cleanup          bool
+)
+
+type numIngressesSlice []int
+
+func (i *numIngressesSlice) String() string {
+	return fmt.Sprintf("%d", *i)
+}
+
+func (i *numIngressesSlice) Set(value string) error {
+	v, err := strconv.Atoi(value)
+	if err != nil {
+		return err
+	}
+	*i = append(*i, v)
+	sort.Ints(*i)
+	return nil
+}
+
+func registerFlags() {
+	if home := os.Getenv("HOME"); home != "" {
+		flag.StringVar(&kubeconfig, "kubeconfig", filepath.Join(home, ".kube", "config"), "(optional) Absolute path to the kubeconfig file")
+	} else {
+		flag.StringVar(&kubeconfig, "kubeconfig", "", "Absolute path to the kubeconfig file")
+	}
+	flag.StringVar(&cloudConfig.ProjectID, "project", "", "GCE project being used")
+	flag.StringVar(&cloudConfig.Zone, "zone", "", "GCE zone being used")
+	flag.StringVar(&cloudConfig.Region, "region", "", "GCE region being used")
+	flag.Var(&numIngressesTest, "num-ingresses", "The number of ingresses to test, specify multiple times for step testing (e.g. 5 ingresses -> 20 ingresses -> 100 ingresses)")
+	flag.BoolVar(&enableTLS, "enable-tls", true, "Whether to enable TLS on ingress")
+	flag.StringVar(&testNamespace, "namespace", "ingress-test-scale", "Namespace for testing")
+	flag.StringVar(&outputFile, "output", "", "If specify, dump latencies to the specified file")
+	flag.BoolVar(&cleanup, "cleanup", true, "Whether to cleanup resources after test")
+}
+
+func verifyFlags() error {
+	if cloudConfig.ProjectID == "" || cloudConfig.Zone == "" || cloudConfig.Region == "" {
+		return fmt.Errorf("must set all of --project, --zone and --region")
+	}
+	return nil
+}
+
+func main() {
+	registerFlags()
+	flag.Parse()
+	if err := verifyFlags(); err != nil {
+		glog.Errorf("Failed to verify flags: %v", err)
+		os.Exit(1)
+	}
+
+	// Initializing a k8s client.
+	config, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
+	if err != nil {
+		glog.Errorf("Failed to build kubeconfig: %v", err)
+		os.Exit(1)
+	}
+	cs, err := clientset.NewForConfig(config)
+	if err != nil {
+		glog.Errorf("Failed to create kubeclient: %v", err)
+		os.Exit(1)
+	}
+
+	// Initializing a GCE client.
+	gceAlphaFeatureGate, err := gcecloud.NewAlphaFeatureGate([]string{})
+	if err != nil {
+		glog.Errorf("Encountered error for creating alpha feature gate: %v", err)
+		os.Exit(1)
+	}
+	gceCloud, err := gcecloud.CreateGCECloud(&gcecloud.CloudConfig{
+		ProjectID:        cloudConfig.ProjectID,
+		Region:           cloudConfig.Region,
+		Zone:             cloudConfig.Zone,
+		AlphaFeatureGate: gceAlphaFeatureGate,
+	})
+	if err != nil {
+		glog.Errorf("Error building GCE provider: %v", err)
+		os.Exit(1)
+	}
+	cloudConfig.Provider = gceCloud
+
+	testSuccessFlag := true
+	defer func() {
+		if !testSuccessFlag {
+			glog.Errorf("Ingress scale test failed.")
+			os.Exit(1)
+		}
+	}()
+
+	ns := &v1.Namespace{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: testNamespace,
+		},
+	}
+	glog.Infof("Creating namespace %s...", ns.Name)
+	if _, err := cs.CoreV1().Namespaces().Create(ns); err != nil {
+		glog.Errorf("Failed to create namespace %s: %v", ns.Name, err)
+		testSuccessFlag = false
+		return
+	}
+	if cleanup {
+		defer func() {
+			glog.Infof("Deleting namespace %s...", ns.Name)
+			if err := cs.CoreV1().Namespaces().Delete(ns.Name, nil); err != nil {
+				glog.Errorf("Failed to delete namespace %s: %v", ns.Name, err)
+				testSuccessFlag = false
+			}
+		}()
+	}
+
+	// Setting up a localized scale test framework.
+	f := scale.NewIngressScaleFramework(cs, ns.Name, cloudConfig)
+	f.Logger = &framework.GLogger{}
+	// Customizing scale test.
+	f.EnableTLS = enableTLS
+	f.OutputFile = outputFile
+	if len(numIngressesTest) != 0 {
+		f.NumIngressesTest = numIngressesTest
+	}
+
+	// Real test begins.
+	if cleanup {
+		defer func() {
+			if errs := f.CleanupScaleTest(); len(errs) != 0 {
+				glog.Errorf("Failed to cleanup scale test: %v", errs)
+				testSuccessFlag = false
+			}
+		}()
+	}
+	err = f.PrepareScaleTest()
+	if err != nil {
+		glog.Errorf("Failed to prepare scale test: %v", err)
+		testSuccessFlag = false
+		return
+	}
+
+	if errs := f.RunScaleTest(); len(errs) != 0 {
+		glog.Errorf("Failed while running scale test: %v", errs)
+		testSuccessFlag = false
+	}
+}

test/e2e/upgrades/ingress.go

@@ -81,7 +81,7 @@ func (t *IngressUpgradeTest) Setup(f *framework.Framework) {
 		Client: jig.Client,
 		Cloud:  framework.TestContext.CloudConfig,
 	}
-	gceController.Init()
+	framework.ExpectNoError(gceController.Init())
 
 	t.gceController = gceController
 	t.jig = jig
@@ -142,7 +142,7 @@ func (t *IngressUpgradeTest) Teardown(f *framework.Framework) {
 	}
 
 	By("Cleaning up cloud resources")
-	framework.CleanupGCEIngressController(t.gceController)
+	framework.ExpectNoError(t.gceController.CleanupGCEIngressController())
 }
 
 func (t *IngressUpgradeTest) verify(f *framework.Framework, done <-chan struct{}, testDuringDisruption bool) {