Merge pull request #45255 from crassirostris/sd-logging-e2e-performance-refactoring

Godeps/Godeps.json

@@ -2821,6 +2821,10 @@
 			"Rev": "e3824ed33c72bf7e81da0286772c34b987520914"
 		},
 		{
+			"ImportPath": "google.golang.org/api/pubsub/v1",
+			"Rev": "e3824ed33c72bf7e81da0286772c34b987520914"
+		},
+		{
 			"ImportPath": "google.golang.org/grpc",
 			"Comment": "v1.0.4",
 			"Rev": "777daa17ff9b5daef1cfdf915088a2ada3332bf0"

Godeps/LICENSES

@@ -84431,6 +84431,41 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 ================================================================================
+= vendor/google.golang.org/api/pubsub/v1 licensed under: =
+
+Copyright (c) 2011 Google Inc. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+= vendor/google.golang.org/api/LICENSE a651bb3d8b1c412632e28823bb432b40  -
+================================================================================
+
+
+================================================================================
 = vendor/google.golang.org/grpc licensed under: =
 
 Copyright 2014, Google Inc.

test/e2e/cluster-logging/BUILD

@@ -27,10 +27,12 @@ go_library(
         "//vendor/golang.org/x/net/context:go_default_library",
         "//vendor/golang.org/x/oauth2/google:go_default_library",
         "//vendor/google.golang.org/api/logging/v2beta1:go_default_library",
+        "//vendor/google.golang.org/api/pubsub/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/fields:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
         "//vendor/k8s.io/client-go/util/integer:go_default_library",
     ],
 )

test/e2e/cluster-logging/es.go

@@ -39,10 +39,11 @@ var _ = framework.KubeDescribe("Cluster level logging using Elasticsearch [Featu
 	It("should check that logs from containers are ingested into Elasticsearch", func() {
 		podName := "synthlogger"
 		esLogsProvider, err := newEsLogsProvider(f)
-		framework.ExpectNoError(err, "Failed to create GCL logs provider")
+		framework.ExpectNoError(err, "Failed to create Elasticsearch logs provider")
 
-		err = esLogsProvider.EnsureWorking()
-		framework.ExpectNoError(err, "Elasticsearch is not working")
+		err = esLogsProvider.Init()
+		defer esLogsProvider.Cleanup()
+		framework.ExpectNoError(err, "Failed to init Elasticsearch logs provider")
 
 		err = ensureSingleFluentdOnEachNode(f, esLogsProvider.FluentdApplicationName())
 		framework.ExpectNoError(err, "Fluentd deployed incorrectly")

test/e2e/cluster-logging/es_utils.go

@@ -46,12 +46,8 @@ func newEsLogsProvider(f *framework.Framework) (*esLogsProvider, error) {
 	return &esLogsProvider{Framework: f}, nil
 }
 
-func (logsProvider *esLogsProvider) FluentdApplicationName() string {
-	return "fluentd-es"
-}
-
 // Ensures that elasticsearch is running and ready to serve requests
-func (logsProvider *esLogsProvider) EnsureWorking() error {
+func (logsProvider *esLogsProvider) Init() error {
 	f := logsProvider.Framework
 	// Check for the existence of the Elasticsearch service.
 	By("Checking the Elasticsearch service exists.")
@@ -157,7 +153,11 @@ func (logsProvider *esLogsProvider) EnsureWorking() error {
 	return nil
 }
 
-func (logsProvider *esLogsProvider) ReadEntries(pod *loggingPod) []*logEntry {
+func (logsProvider *esLogsProvider) Cleanup() {
+	// Nothing to do
+}
+
+func (logsProvider *esLogsProvider) ReadEntries(pod *loggingPod) []logEntry {
 	f := logsProvider.Framework
 
 	proxyRequest, errProxy := framework.GetServicesProxyRequest(f.ClientSet, f.ClientSet.Core().RESTClient().Get())
@@ -202,7 +202,7 @@ func (logsProvider *esLogsProvider) ReadEntries(pod *loggingPod) []*logEntry {
 		return nil
 	}
 
-	entries := []*logEntry{}
+	entries := []logEntry{}
 	// Iterate over the hits and populate the observed array.
 	for _, e := range h {
 		l, ok := e.(map[string]interface{})
@@ -223,22 +223,12 @@ func (logsProvider *esLogsProvider) ReadEntries(pod *loggingPod) []*logEntry {
 			continue
 		}
 
-		timestampString, ok := source["@timestamp"].(string)
-		if !ok {
-			framework.Logf("Timestamp not of the expected type: %T", source["@timestamp"])
-			continue
-		}
-		timestamp, err := time.Parse(time.RFC3339, timestampString)
-		if err != nil {
-			framework.Logf("Timestamp was not in correct format: %s", timestampString)
-			continue
-		}
-
-		entries = append(entries, &logEntry{
-			Payload:   msg,
-			Timestamp: timestamp,
-		})
+		entries = append(entries, logEntry{Payload: msg})
 	}
 
 	return entries
 }
+
+func (logsProvider *esLogsProvider) FluentdApplicationName() string {
+	return "fluentd-es"
+}

test/e2e/cluster-logging/sd.go

@@ -39,8 +39,9 @@ var _ = framework.KubeDescribe("Cluster level logging using GCL", func() {
 		gclLogsProvider, err := newGclLogsProvider(f)
 		framework.ExpectNoError(err, "Failed to create GCL logs provider")
 
-		err = gclLogsProvider.EnsureWorking()
-		framework.ExpectNoError(err, "GCL is not working")
+		err = gclLogsProvider.Init()
+		defer gclLogsProvider.Cleanup()
+		framework.ExpectNoError(err, "Failed to init GCL logs provider")
 
 		err = ensureSingleFluentdOnEachNode(f, gclLogsProvider.FluentdApplicationName())
 		framework.ExpectNoError(err, "Fluentd deployed incorrectly")

test/e2e/cluster-logging/sd_load.go

@@ -38,13 +38,17 @@ var _ = framework.KubeDescribe("Cluster level logging using GCL [Feature:Stackdr
 		gclLogsProvider, err := newGclLogsProvider(f)
 		framework.ExpectNoError(err, "Failed to create GCL logs provider")
 
+		err = gclLogsProvider.Init()
+		defer gclLogsProvider.Cleanup()
+		framework.ExpectNoError(err, "Failed to init GCL logs provider")
+
 		nodes := framework.GetReadySchedulableNodesOrDie(f.ClientSet).Items
 		nodeCount := len(nodes)
 		podCount := 30 * nodeCount
 		loggingDuration := 10 * time.Minute
 		linesPerSecond := 1000 * nodeCount
 		linesPerPod := linesPerSecond * int(loggingDuration.Seconds()) / podCount
-		ingestionTimeout := 60 * time.Minute
+		ingestionTimeout := 20 * time.Minute
 
 		By("Running logs generator pods")
 		pods := []*loggingPod{}
@@ -56,9 +60,6 @@ var _ = framework.KubeDescribe("Cluster level logging using GCL [Feature:Stackdr
 			defer f.PodClient().Delete(podName, &meta_v1.DeleteOptions{})
 		}
 
-		By("Waiting for pods to succeed")
-		time.Sleep(loggingDuration)
-
 		By("Waiting for all log lines to be ingested")
 		config := &loggingTestConfig{
 			LogsProvider:              gclLogsProvider,
@@ -79,12 +80,16 @@ var _ = framework.KubeDescribe("Cluster level logging using GCL [Feature:Stackdr
 		gclLogsProvider, err := newGclLogsProvider(f)
 		framework.ExpectNoError(err, "Failed to create GCL logs provider")
 
+		err = gclLogsProvider.Init()
+		defer gclLogsProvider.Cleanup()
+		framework.ExpectNoError(err, "Failed to init GCL logs provider")
+
 		nodes := framework.GetReadySchedulableNodesOrDie(f.ClientSet).Items
 		maxPodCount := 10
 		jobDuration := 1 * time.Minute
 		linesPerPodPerSecond := 100
 		testDuration := 10 * time.Minute
-		ingestionTimeout := 60 * time.Minute
+		ingestionTimeout := 20 * time.Minute
 
 		podRunDelay := time.Duration(int64(jobDuration) / int64(maxPodCount))
 		podRunCount := int(testDuration.Seconds())/int(podRunDelay.Seconds()) - 1
@@ -102,9 +107,6 @@ var _ = framework.KubeDescribe("Cluster level logging using GCL [Feature:Stackdr
 			time.Sleep(podRunDelay)
 		}
 
-		By("Waiting for the last pods to finish")
-		time.Sleep(jobDuration)
-
 		By("Waiting for all log lines to be ingested")
 		config := &loggingTestConfig{
 			LogsProvider:              gclLogsProvider,

test/e2e/cluster-logging/utils.go

@@ -33,13 +33,16 @@ import (
 
 const (
 	// Duration of delay between any two attempts to check if all logs are ingested
-	ingestionRetryDelay = 100 * time.Second
+	ingestionRetryDelay = 30 * time.Second
 
 	// Amount of requested cores for logging container in millicores
 	loggingContainerCpuRequest = 10
 
 	// Amount of requested memory for logging container in bytes
 	loggingContainerMemoryRequest = 10 * 1024 * 1024
+
+	// Name of the container used for logging tests
+	loggingContainerName = "logging-container"
 )
 
 var (
@@ -51,26 +54,21 @@ var (
 type loggingPod struct {
 	// Name of the pod
 	Name string
-	// If we didn't read some log entries, their
-	// timestamps should be no less than this timestamp.
-	// Effectively, timestamp of the last ingested entry
-	// for which there's no missing entry before it
-	LastTimestamp time.Time
 	// Cache of ingested and read entries
-	Occurrences map[int]*logEntry
+	Occurrences map[int]logEntry
 	// Number of lines expected to be ingested from this pod
 	ExpectedLinesNumber int
 }
 
 type logEntry struct {
-	Payload   string
-	Timestamp time.Time
+	Payload string
 }
 
 type logsProvider interface {
+	Init() error
+	Cleanup()
+	ReadEntries(*loggingPod) []logEntry
 	FluentdApplicationName() string
-	EnsureWorking() error
-	ReadEntries(*loggingPod) []*logEntry
 }
 
 type loggingTestConfig struct {
@@ -81,7 +79,7 @@ type loggingTestConfig struct {
 	MaxAllowedFluentdRestarts int
 }
 
-func (entry *logEntry) getLogEntryNumber() (int, bool) {
+func (entry logEntry) getLogEntryNumber() (int, bool) {
 	submatch := logEntryMessageRegex.FindStringSubmatch(entry.Payload)
 	if submatch == nil || len(submatch) < 2 {
 		return 0, false
@@ -96,10 +94,8 @@ func createLoggingPod(f *framework.Framework, podName string, nodeName string, t
 	createLogsGeneratorPod(f, podName, nodeName, totalLines, loggingDuration)
 
 	return &loggingPod{
-		Name: podName,
-		// It's used to avoid querying logs from before the pod was started
-		LastTimestamp:       time.Now(),
-		Occurrences:         make(map[int]*logEntry),
+		Name:                podName,
+		Occurrences:         make(map[int]logEntry),
 		ExpectedLinesNumber: totalLines,
 	}
 }
@@ -113,7 +109,7 @@ func createLogsGeneratorPod(f *framework.Framework, podName string, nodeName str
 			RestartPolicy: api_v1.RestartPolicyNever,
 			Containers: []api_v1.Container{
 				{
-					Name:  podName,
+					Name:  loggingContainerName,
 					Image: "gcr.io/google_containers/logs-generator:v0.1.0",
 					Env: []api_v1.EnvVar{
 						{
@@ -146,7 +142,7 @@ func waitForSomeLogs(f *framework.Framework, config *loggingTestConfig) error {
 	podHasIngestedLogs := make([]bool, len(config.Pods))
 	podWithIngestedLogsCount := 0
 
-	for start := time.Now(); podWithIngestedLogsCount < len(config.Pods) && time.Since(start) < config.IngestionTimeout; time.Sleep(ingestionRetryDelay) {
+	for start := time.Now(); time.Since(start) < config.IngestionTimeout; time.Sleep(ingestionRetryDelay) {
 		for podIdx, pod := range config.Pods {
 			if podHasIngestedLogs[podIdx] {
 				continue
@@ -167,6 +163,10 @@ func waitForSomeLogs(f *framework.Framework, config *loggingTestConfig) error {
 				}
 			}
 		}
+
+		if podWithIngestedLogsCount == len(config.Pods) {
+			break
+		}
 	}
 
 	if podWithIngestedLogsCount < len(config.Pods) {
@@ -189,7 +189,7 @@ func waitForFullLogsIngestion(f *framework.Framework, config *loggingTestConfig)
 		missingByPod[podIdx] = pod.ExpectedLinesNumber
 	}
 
-	for start := time.Now(); totalMissing > 0 && time.Since(start) < config.IngestionTimeout; time.Sleep(ingestionRetryDelay) {
+	for start := time.Now(); time.Since(start) < config.IngestionTimeout; time.Sleep(ingestionRetryDelay) {
 		missing := 0
 		for podIdx, pod := range config.Pods {
 			if missingByPod[podIdx] == 0 {
@@ -203,6 +203,8 @@ func waitForFullLogsIngestion(f *framework.Framework, config *loggingTestConfig)
 		totalMissing = missing
 		if totalMissing > 0 {
 			framework.Logf("Still missing %d lines in total", totalMissing)
+		} else {
+			break
 		}
 	}
 
@@ -245,19 +247,14 @@ func pullMissingLogsCount(logsProvider logsProvider, pod *loggingPod) int {
 	if err != nil {
 		framework.Logf("Failed to get missing lines count from pod %s due to %v", pod.Name, err)
 		return pod.ExpectedLinesNumber
-	} else if missingOnPod > 0 {
-		framework.Logf("Pod %s is missing %d lines", pod.Name, missingOnPod)
-	} else {
-		framework.Logf("All logs from pod %s are ingested", pod.Name)
 	}
+
 	return missingOnPod
 }
 
 func getMissingLinesCount(logsProvider logsProvider, pod *loggingPod) (int, error) {
 	entries := logsProvider.ReadEntries(pod)
 
-	framework.Logf("Got %d entries from provider", len(entries))
-
 	for _, entry := range entries {
 		lineNumber, ok := entry.getLogEntryNumber()
 		if !ok {
@@ -271,17 +268,6 @@ func getMissingLinesCount(logsProvider logsProvider, pod *loggingPod) (int, erro
 		}
 	}
 
-	for i := 0; i < pod.ExpectedLinesNumber; i++ {
-		entry, ok := pod.Occurrences[i]
-		if !ok {
-			break
-		}
-
-		if entry.Timestamp.After(pod.LastTimestamp) {
-			pod.LastTimestamp = entry.Timestamp
-		}
-	}
-
 	return pod.ExpectedLinesNumber - len(pod.Occurrences), nil
 }
 

vendor/BUILD

@@ -362,6 +362,7 @@ filegroup(
         "//vendor/google.golang.org/api/googleapi:all-srcs",
         "//vendor/google.golang.org/api/logging/v2beta1:all-srcs",
         "//vendor/google.golang.org/api/monitoring/v3:all-srcs",
+        "//vendor/google.golang.org/api/pubsub/v1:all-srcs",
         "//vendor/google.golang.org/grpc:all-srcs",
         "//vendor/gopkg.in/gcfg.v1:all-srcs",
         "//vendor/gopkg.in/inf.v0:all-srcs",

vendor/google.golang.org/api/pubsub/v1/BUILD

+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["pubsub-gen.go"],
+    tags = ["automanaged"],
+    deps = [
+        "//vendor/golang.org/x/net/context:go_default_library",
+        "//vendor/golang.org/x/net/context/ctxhttp:go_default_library",
+        "//vendor/google.golang.org/api/gensupport:go_default_library",
+        "//vendor/google.golang.org/api/googleapi:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)