Merge pull request #30116 from Random-Liu/start-e2e-services-in-separate-process

hack/verify-flags/known-flags.txt

@@ -289,6 +289,7 @@ log-flush-frequency
 long-running-request-regexp
 low-diskspace-threshold-mb
 make-symlinks
+manifest-path
 manifest-url
 manifest-url-header
 masquerade-all
@@ -421,6 +422,7 @@ rkt-stage1-image
 root-ca-file
 root-dir
 run-proxy
+run-services-mode
 runtime-cgroups
 runtime-config
 runtime-request-timeout

test/e2e/framework/test_context.go

@@ -67,14 +67,24 @@ type TestContextType struct {
 	CreateTestingNS CreateTestingNSFn
 	// If set to true test will dump data about the namespace in which test was running.
 	DumpLogsOnFailure bool
+	// If the garbage collector is enabled in the kube-apiserver and kube-controller-manager.
+	GarbageCollectorEnabled bool
+	// Node e2e specific test context
+	NodeTestContextType
+}
+
+// NodeTestContextType is part of TestContextType, it is shared by all node e2e test.
+type NodeTestContextType struct {
 	// Name of the node to run tests on (node e2e suite only).
 	NodeName string
+	// DisableKubenet disables kubenet when starting kubelet.
+	DisableKubenet bool
 	// Whether to enable the QoS Cgroup Hierarchy or not
 	CgroupsPerQOS bool
 	// The hard eviction thresholds
 	EvictionHard string
-	// If the garbage collector is enabled in the kube-apiserver and kube-controller-manager.
-	GarbageCollectorEnabled bool
+	// ManifestPath is the static pod manifest path.
+	ManifestPath string
 }
 
 type CloudConfig struct {
@@ -112,7 +122,6 @@ func RegisterCommonFlags() {
 	flag.StringVar(&TestContext.Host, "host", "http://127.0.0.1:8080", "The host, or apiserver, to connect to")
 	flag.StringVar(&TestContext.ReportPrefix, "report-prefix", "", "Optional prefix for JUnit XML reports. Default is empty, which doesn't prepend anything to the default name.")
 	flag.StringVar(&TestContext.ReportDir, "report-dir", "", "Path to the directory where the JUnit XML reports should be saved. Default is empty, which doesn't generate these reports.")
-	flag.BoolVar(&TestContext.GarbageCollectorEnabled, "garbage-collector-enabled", false, "Set to true if the garbage collector is enabled in the kube-apiserver and kube-controller-manager, then some tests will rely on the garbage collector to delete dependent resources.")
 }
 
 // Register flags specific to the cluster e2e test suite.
@@ -149,11 +158,16 @@ func RegisterClusterFlags() {
 	flag.StringVar(&TestContext.UpgradeTarget, "upgrade-target", "ci/latest", "Version to upgrade to (e.g. 'release/stable', 'release/latest', 'ci/latest', '0.19.1', '0.19.1-669-gabac8c8') if doing an upgrade test.")
 	flag.StringVar(&TestContext.PrometheusPushGateway, "prom-push-gateway", "", "The URL to prometheus gateway, so that metrics can be pushed during e2es and scraped by prometheus. Typically something like 127.0.0.1:9091.")
 	flag.BoolVar(&TestContext.CleanStart, "clean-start", false, "If true, purge all namespaces except default and system before running tests. This serves to Cleanup test namespaces from failed/interrupted e2e runs in a long-lived cluster.")
+	flag.BoolVar(&TestContext.GarbageCollectorEnabled, "garbage-collector-enabled", false, "Set to true if the garbage collector is enabled in the kube-apiserver and kube-controller-manager, then some tests will rely on the garbage collector to delete dependent resources.")
 }
 
 // Register flags specific to the node e2e test suite.
 func RegisterNodeFlags() {
 	flag.StringVar(&TestContext.NodeName, "node-name", "", "Name of the node to run tests on (node e2e suite only).")
+	// TODO(random-liu): Remove kubelet related flags when we move the kubelet start logic out of the test.
+	// TODO(random-liu): Find someway to get kubelet configuration, and automatic config and filter test based on the configuration.
+	flag.BoolVar(&TestContext.DisableKubenet, "disable-kubenet", false, "If true, start kubelet without kubenet. (default false)")
 	flag.BoolVar(&TestContext.CgroupsPerQOS, "cgroups-per-qos", false, "Enable creation of QoS cgroup hierarchy, if true top level QoS and pod cgroups are created.")
 	flag.StringVar(&TestContext.EvictionHard, "eviction-hard", "memory.available<250Mi", "The hard eviction thresholds. If set, pods get evicted when the specified resources drop below the thresholds.")
+	flag.StringVar(&TestContext.ManifestPath, "manifest-path", "", "The path to the static pod manifest file.")
 }

test/e2e_node/e2e_node_suite_test.go

@@ -45,41 +45,30 @@ import (
 	"github.com/onsi/ginkgo/config"
 	more_reporters "github.com/onsi/ginkgo/reporters"
 	. "github.com/onsi/gomega"
+	"github.com/spf13/pflag"
 )
 
-var e2es *e2eService
-
-// context is the test context shared by all parallel nodes.
-// Originally we setup the test environment and initialize global variables
-// in BeforeSuite, and then used the global variables in the test.
-// However, after we make the test parallel, ginkgo will run all tests
-// in several parallel test nodes. And for each test node, the BeforeSuite
-// and AfterSuite will be run.
-// We don't want to start services (kubelet, apiserver and etcd) for all
-// parallel nodes, but we do want to set some globally shared variable which
-// could be used in test.
-// We have to use SynchronizedBeforeSuite to achieve that. The first
-// function of SynchronizedBeforeSuite is only called once, and the second
-// function is called in each parallel test node. The result returned by
-// the first function will be the parameter of the second function.
-// So we'll start all services and initialize the shared context in the first
-// function, and propagate the context to all parallel test nodes in the
-// second function.
-// Notice no lock is needed for shared context, because context should only be
-// initialized in the first function in SynchronizedBeforeSuite. After that
-// it should never be modified.
-var context SharedContext
+var e2es *E2EServices
 
 var prePullImages = flag.Bool("prepull-images", true, "If true, prepull images so image pull failures do not cause test failures.")
+var runServicesMode = flag.Bool("run-services-mode", false, "If true, only run services (etcd, apiserver) in current process, and not run test.")
 
 func init() {
 	framework.RegisterCommonFlags()
 	framework.RegisterNodeFlags()
+	pflag.CommandLine.AddGoFlagSet(flag.CommandLine)
+	// Mark the run-services-mode flag as hidden to prevent user from using it.
+	pflag.CommandLine.MarkHidden("run-services-mode")
 }
 
 func TestE2eNode(t *testing.T) {
-	flag.Parse()
-
+	pflag.Parse()
+	if *runServicesMode {
+		// If run-services-mode is specified, only run services in current process.
+		RunE2EServices()
+		return
+	}
+	// If run-services-mode is not specified, run test.
 	rand.Seed(time.Now().UTC().UnixNano())
 	RegisterFailHandler(Fail)
 	reporters := []Reporter{}
@@ -103,6 +92,8 @@ var _ = SynchronizedBeforeSuite(func() []byte {
 	if *buildServices {
 		Expect(buildGo()).To(Succeed())
 	}
+
+	// Initialize node name here, so that the following code can get right node name.
 	if framework.TestContext.NodeName == "" {
 		hostname, err := os.Hostname()
 		if err != nil {
@@ -110,7 +101,6 @@ var _ = SynchronizedBeforeSuite(func() []byte {
 		}
 		framework.TestContext.NodeName = hostname
 	}
-
 	// Pre-pull the images tests depend on so we can fail immediately if there is an image pull issue
 	// This helps with debugging test flakes since it is hard to tell when a test failure is due to image pulling.
 	if *prePullImages {
@@ -124,11 +114,10 @@ var _ = SynchronizedBeforeSuite(func() []byte {
 	// We should mask locksmithd when provisioning the machine.
 	maskLocksmithdOnCoreos()
 
-	shared := &SharedContext{}
 	if *startServices {
-		e2es = newE2eService(framework.TestContext.NodeName, framework.TestContext.CgroupsPerQOS, framework.TestContext.EvictionHard, shared)
-		if err := e2es.start(); err != nil {
-			Fail(fmt.Sprintf("Unable to start node services.\n%v", err))
+		e2es = NewE2EServices()
+		if err := e2es.Start(); err != nil {
+			glog.Fatalf("Unable to start node services: %v", err)
 		}
 		glog.Infof("Node services started.  Running tests...")
 	} else {
@@ -136,33 +125,31 @@ var _ = SynchronizedBeforeSuite(func() []byte {
 	}
 
 	glog.Infof("Starting namespace controller")
+	// TODO(random-liu): Move namespace controller into namespace services.
 	startNamespaceController()
 
 	// Reference common test to make the import valid.
 	commontest.CurrentSuite = commontest.NodeE2E
 
-	// Share the node name with the other test nodes.
-	shared.NodeName = framework.TestContext.NodeName
-	data, err := json.Marshal(shared)
-	Expect(err).NotTo(HaveOccurred())
-
+	data, err := json.Marshal(&framework.TestContext.NodeTestContextType)
+	if err != nil {
+		glog.Fatalf("Failed to serialize node test context: %v", err)
+	}
 	return data
 }, func(data []byte) {
-	// Set the shared context got from the synchronized initialize function
-	shared := &SharedContext{}
-	Expect(json.Unmarshal(data, shared)).To(Succeed())
-	context = *shared
-
-	framework.TestContext.NodeName = shared.NodeName
+	// The node test context is updated in the first function, update it on every test node.
+	err := json.Unmarshal(data, &framework.TestContext.NodeTestContextType)
+	if err != nil {
+		glog.Fatalf("Failed to deserialize node test context: %v", err)
+	}
 })
 
 // Tear down the kubelet on the node
 var _ = SynchronizedAfterSuite(func() {}, func() {
 	if e2es != nil {
-		e2es.getLogFiles()
 		if *startServices && *stopServices {
 			glog.Infof("Stopping node services...")
-			e2es.stop()
+			e2es.Stop()
 		}
 	}
 

test/e2e_node/mirror_pod_test.go

@@ -44,7 +44,7 @@ var _ = framework.KubeDescribe("MirrorPod", func() {
 			mirrorPodName = staticPodName + "-" + framework.TestContext.NodeName
 
 			By("create the static pod")
-			err := createStaticPod(context.PodConfigPath, staticPodName, ns, ImageRegistry[nginxImage], api.RestartPolicyAlways)
+			err := createStaticPod(framework.TestContext.ManifestPath, staticPodName, ns, ImageRegistry[nginxImage], api.RestartPolicyAlways)
 			Expect(err).ShouldNot(HaveOccurred())
 
 			By("wait for the mirror pod to be running")
@@ -60,7 +60,7 @@ var _ = framework.KubeDescribe("MirrorPod", func() {
 
 			By("update the static pod container image")
 			image := ImageRegistry[pauseImage]
-			err = createStaticPod(context.PodConfigPath, staticPodName, ns, image, api.RestartPolicyAlways)
+			err = createStaticPod(framework.TestContext.ManifestPath, staticPodName, ns, image, api.RestartPolicyAlways)
 			Expect(err).ShouldNot(HaveOccurred())
 
 			By("wait for the mirror pod to be updated")
@@ -106,7 +106,7 @@ var _ = framework.KubeDescribe("MirrorPod", func() {
 		})
 		AfterEach(func() {
 			By("delete the static pod")
-			err := deleteStaticPod(context.PodConfigPath, staticPodName, ns)
+			err := deleteStaticPod(framework.TestContext.ManifestPath, staticPodName, ns)
 			Expect(err).ShouldNot(HaveOccurred())
 
 			By("wait for the mirror pod to disappear")

test/e2e_node/util.go

@@ -22,12 +22,6 @@ import (
 
 var kubeletAddress = flag.String("kubelet-address", "http://127.0.0.1:10255", "Host and port of the kubelet")
 
-var disableKubenet = flag.Bool("disable-kubenet", false, "If true, start kubelet without kubenet")
 var buildServices = flag.Bool("build-services", true, "If true, build local executables")
 var startServices = flag.Bool("start-services", true, "If true, start local node services")
 var stopServices = flag.Bool("stop-services", true, "If true, stop local node services after running tests")
-
-type SharedContext struct {
-	NodeName      string
-	PodConfigPath string
-}