Implement Stackdrvier Logging e2e tests using PubSub

test/e2e/cluster-logging/BUILD

@@ -27,10 +27,12 @@ go_library(
         "//vendor/golang.org/x/net/context:go_default_library",
         "//vendor/golang.org/x/oauth2/google:go_default_library",
         "//vendor/google.golang.org/api/logging/v2beta1:go_default_library",
+        "//vendor/google.golang.org/api/pubsub/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/fields:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
         "//vendor/k8s.io/client-go/util/integer:go_default_library",
     ],
 )

test/e2e/cluster-logging/es.go

@@ -39,10 +39,11 @@ var _ = framework.KubeDescribe("Cluster level logging using Elasticsearch [Featu
 	It("should check that logs from containers are ingested into Elasticsearch", func() {
 		podName := "synthlogger"
 		esLogsProvider, err := newEsLogsProvider(f)
-		framework.ExpectNoError(err, "Failed to create GCL logs provider")
+		framework.ExpectNoError(err, "Failed to create Elasticsearch logs provider")
 
-		err = esLogsProvider.EnsureWorking()
-		framework.ExpectNoError(err, "Elasticsearch is not working")
+		err = esLogsProvider.Init()
+		defer esLogsProvider.Cleanup()
+		framework.ExpectNoError(err, "Failed to init Elasticsearch logs provider")
 
 		err = ensureSingleFluentdOnEachNode(f, esLogsProvider.FluentdApplicationName())
 		framework.ExpectNoError(err, "Fluentd deployed incorrectly")

test/e2e/cluster-logging/es_utils.go

@@ -46,12 +46,8 @@ func newEsLogsProvider(f *framework.Framework) (*esLogsProvider, error) {
 	return &esLogsProvider{Framework: f}, nil
 }
 
-func (logsProvider *esLogsProvider) FluentdApplicationName() string {
-	return "fluentd-es"
-}
-
 // Ensures that elasticsearch is running and ready to serve requests
-func (logsProvider *esLogsProvider) EnsureWorking() error {
+func (logsProvider *esLogsProvider) Init() error {
 	f := logsProvider.Framework
 	// Check for the existence of the Elasticsearch service.
 	By("Checking the Elasticsearch service exists.")
@@ -157,7 +153,11 @@ func (logsProvider *esLogsProvider) EnsureWorking() error {
 	return nil
 }
 
-func (logsProvider *esLogsProvider) ReadEntries(pod *loggingPod) []*logEntry {
+func (logsProvider *esLogsProvider) Cleanup() {
+	// Nothing to do
+}
+
+func (logsProvider *esLogsProvider) ReadEntries(pod *loggingPod) []logEntry {
 	f := logsProvider.Framework
 
 	proxyRequest, errProxy := framework.GetServicesProxyRequest(f.ClientSet, f.ClientSet.Core().RESTClient().Get())
@@ -202,7 +202,7 @@ func (logsProvider *esLogsProvider) ReadEntries(pod *loggingPod) []*logEntry {
 		return nil
 	}
 
-	entries := []*logEntry{}
+	entries := []logEntry{}
 	// Iterate over the hits and populate the observed array.
 	for _, e := range h {
 		l, ok := e.(map[string]interface{})
@@ -223,22 +223,12 @@ func (logsProvider *esLogsProvider) ReadEntries(pod *loggingPod) []*logEntry {
 			continue
 		}
 
-		timestampString, ok := source["@timestamp"].(string)
-		if !ok {
-			framework.Logf("Timestamp not of the expected type: %T", source["@timestamp"])
-			continue
-		}
-		timestamp, err := time.Parse(time.RFC3339, timestampString)
-		if err != nil {
-			framework.Logf("Timestamp was not in correct format: %s", timestampString)
-			continue
-		}
-
-		entries = append(entries, &logEntry{
-			Payload:   msg,
-			Timestamp: timestamp,
-		})
+		entries = append(entries, logEntry{Payload: msg})
 	}
 
 	return entries
 }
+
+func (logsProvider *esLogsProvider) FluentdApplicationName() string {
+	return "fluentd-es"
+}

test/e2e/cluster-logging/sd.go

@@ -39,8 +39,9 @@ var _ = framework.KubeDescribe("Cluster level logging using GCL", func() {
 		gclLogsProvider, err := newGclLogsProvider(f)
 		framework.ExpectNoError(err, "Failed to create GCL logs provider")
 
-		err = gclLogsProvider.EnsureWorking()
-		framework.ExpectNoError(err, "GCL is not working")
+		err = gclLogsProvider.Init()
+		defer gclLogsProvider.Cleanup()
+		framework.ExpectNoError(err, "Failed to init GCL logs provider")
 
 		err = ensureSingleFluentdOnEachNode(f, gclLogsProvider.FluentdApplicationName())
 		framework.ExpectNoError(err, "Fluentd deployed incorrectly")

test/e2e/cluster-logging/sd_load.go

@@ -38,13 +38,17 @@ var _ = framework.KubeDescribe("Cluster level logging using GCL [Feature:Stackdr
 		gclLogsProvider, err := newGclLogsProvider(f)
 		framework.ExpectNoError(err, "Failed to create GCL logs provider")
 
+		err = gclLogsProvider.Init()
+		defer gclLogsProvider.Cleanup()
+		framework.ExpectNoError(err, "Failed to init GCL logs provider")
+
 		nodes := framework.GetReadySchedulableNodesOrDie(f.ClientSet).Items
 		nodeCount := len(nodes)
 		podCount := 30 * nodeCount
 		loggingDuration := 10 * time.Minute
 		linesPerSecond := 1000 * nodeCount
 		linesPerPod := linesPerSecond * int(loggingDuration.Seconds()) / podCount
-		ingestionTimeout := 60 * time.Minute
+		ingestionTimeout := 20 * time.Minute
 
 		By("Running logs generator pods")
 		pods := []*loggingPod{}
@@ -56,9 +60,6 @@ var _ = framework.KubeDescribe("Cluster level logging using GCL [Feature:Stackdr
 			defer f.PodClient().Delete(podName, &meta_v1.DeleteOptions{})
 		}
 
-		By("Waiting for pods to succeed")
-		time.Sleep(loggingDuration)
-
 		By("Waiting for all log lines to be ingested")
 		config := &loggingTestConfig{
 			LogsProvider:              gclLogsProvider,
@@ -79,12 +80,16 @@ var _ = framework.KubeDescribe("Cluster level logging using GCL [Feature:Stackdr
 		gclLogsProvider, err := newGclLogsProvider(f)
 		framework.ExpectNoError(err, "Failed to create GCL logs provider")
 
+		err = gclLogsProvider.Init()
+		defer gclLogsProvider.Cleanup()
+		framework.ExpectNoError(err, "Failed to init GCL logs provider")
+
 		nodes := framework.GetReadySchedulableNodesOrDie(f.ClientSet).Items
 		maxPodCount := 10
 		jobDuration := 1 * time.Minute
 		linesPerPodPerSecond := 100
 		testDuration := 10 * time.Minute
-		ingestionTimeout := 60 * time.Minute
+		ingestionTimeout := 20 * time.Minute
 
 		podRunDelay := time.Duration(int64(jobDuration) / int64(maxPodCount))
 		podRunCount := int(testDuration.Seconds())/int(podRunDelay.Seconds()) - 1
@@ -102,9 +107,6 @@ var _ = framework.KubeDescribe("Cluster level logging using GCL [Feature:Stackdr
 			time.Sleep(podRunDelay)
 		}
 
-		By("Waiting for the last pods to finish")
-		time.Sleep(jobDuration)
-
 		By("Waiting for all log lines to be ingested")
 		config := &loggingTestConfig{
 			LogsProvider:              gclLogsProvider,

test/e2e/cluster-logging/utils.go

@@ -33,13 +33,16 @@ import (
 
 const (
 	// Duration of delay between any two attempts to check if all logs are ingested
-	ingestionRetryDelay = 100 * time.Second
+	ingestionRetryDelay = 30 * time.Second
 
 	// Amount of requested cores for logging container in millicores
 	loggingContainerCpuRequest = 10
 
 	// Amount of requested memory for logging container in bytes
 	loggingContainerMemoryRequest = 10 * 1024 * 1024
+
+	// Name of the container used for logging tests
+	loggingContainerName = "logging-container"
 )
 
 var (
@@ -51,26 +54,21 @@ var (
 type loggingPod struct {
 	// Name of the pod
 	Name string
-	// If we didn't read some log entries, their
-	// timestamps should be no less than this timestamp.
-	// Effectively, timestamp of the last ingested entry
-	// for which there's no missing entry before it
-	LastTimestamp time.Time
 	// Cache of ingested and read entries
-	Occurrences map[int]*logEntry
+	Occurrences map[int]logEntry
 	// Number of lines expected to be ingested from this pod
 	ExpectedLinesNumber int
 }
 
 type logEntry struct {
-	Payload   string
-	Timestamp time.Time
+	Payload string
 }
 
 type logsProvider interface {
+	Init() error
+	Cleanup()
+	ReadEntries(*loggingPod) []logEntry
 	FluentdApplicationName() string
-	EnsureWorking() error
-	ReadEntries(*loggingPod) []*logEntry
 }
 
 type loggingTestConfig struct {
@@ -81,7 +79,7 @@ type loggingTestConfig struct {
 	MaxAllowedFluentdRestarts int
 }
 
-func (entry *logEntry) getLogEntryNumber() (int, bool) {
+func (entry logEntry) getLogEntryNumber() (int, bool) {
 	submatch := logEntryMessageRegex.FindStringSubmatch(entry.Payload)
 	if submatch == nil || len(submatch) < 2 {
 		return 0, false
@@ -96,10 +94,8 @@ func createLoggingPod(f *framework.Framework, podName string, nodeName string, t
 	createLogsGeneratorPod(f, podName, nodeName, totalLines, loggingDuration)
 
 	return &loggingPod{
-		Name: podName,
-		// It's used to avoid querying logs from before the pod was started
-		LastTimestamp:       time.Now(),
-		Occurrences:         make(map[int]*logEntry),
+		Name:                podName,
+		Occurrences:         make(map[int]logEntry),
 		ExpectedLinesNumber: totalLines,
 	}
 }
@@ -113,7 +109,7 @@ func createLogsGeneratorPod(f *framework.Framework, podName string, nodeName str
 			RestartPolicy: api_v1.RestartPolicyNever,
 			Containers: []api_v1.Container{
 				{
-					Name:  podName,
+					Name:  loggingContainerName,
 					Image: "gcr.io/google_containers/logs-generator:v0.1.0",
 					Env: []api_v1.EnvVar{
 						{
@@ -146,7 +142,7 @@ func waitForSomeLogs(f *framework.Framework, config *loggingTestConfig) error {
 	podHasIngestedLogs := make([]bool, len(config.Pods))
 	podWithIngestedLogsCount := 0
 
-	for start := time.Now(); podWithIngestedLogsCount < len(config.Pods) && time.Since(start) < config.IngestionTimeout; time.Sleep(ingestionRetryDelay) {
+	for start := time.Now(); time.Since(start) < config.IngestionTimeout; time.Sleep(ingestionRetryDelay) {
 		for podIdx, pod := range config.Pods {
 			if podHasIngestedLogs[podIdx] {
 				continue
@@ -167,6 +163,10 @@ func waitForSomeLogs(f *framework.Framework, config *loggingTestConfig) error {
 				}
 			}
 		}
+
+		if podWithIngestedLogsCount == len(config.Pods) {
+			break
+		}
 	}
 
 	if podWithIngestedLogsCount < len(config.Pods) {
@@ -189,7 +189,7 @@ func waitForFullLogsIngestion(f *framework.Framework, config *loggingTestConfig)
 		missingByPod[podIdx] = pod.ExpectedLinesNumber
 	}
 
-	for start := time.Now(); totalMissing > 0 && time.Since(start) < config.IngestionTimeout; time.Sleep(ingestionRetryDelay) {
+	for start := time.Now(); time.Since(start) < config.IngestionTimeout; time.Sleep(ingestionRetryDelay) {
 		missing := 0
 		for podIdx, pod := range config.Pods {
 			if missingByPod[podIdx] == 0 {
@@ -203,6 +203,8 @@ func waitForFullLogsIngestion(f *framework.Framework, config *loggingTestConfig)
 		totalMissing = missing
 		if totalMissing > 0 {
 			framework.Logf("Still missing %d lines in total", totalMissing)
+		} else {
+			break
 		}
 	}
 
@@ -245,19 +247,14 @@ func pullMissingLogsCount(logsProvider logsProvider, pod *loggingPod) int {
 	if err != nil {
 		framework.Logf("Failed to get missing lines count from pod %s due to %v", pod.Name, err)
 		return pod.ExpectedLinesNumber
-	} else if missingOnPod > 0 {
-		framework.Logf("Pod %s is missing %d lines", pod.Name, missingOnPod)
-	} else {
-		framework.Logf("All logs from pod %s are ingested", pod.Name)
 	}
+
 	return missingOnPod
 }
 
 func getMissingLinesCount(logsProvider logsProvider, pod *loggingPod) (int, error) {
 	entries := logsProvider.ReadEntries(pod)
 
-	framework.Logf("Got %d entries from provider", len(entries))
-
 	for _, entry := range entries {
 		lineNumber, ok := entry.getLogEntryNumber()
 		if !ok {
@@ -271,17 +268,6 @@ func getMissingLinesCount(logsProvider logsProvider, pod *loggingPod) (int, erro
 		}
 	}
 
-	for i := 0; i < pod.ExpectedLinesNumber; i++ {
-		entry, ok := pod.Occurrences[i]
-		if !ok {
-			break
-		}
-
-		if entry.Timestamp.After(pod.LastTimestamp) {
-			pod.LastTimestamp = entry.Timestamp
-		}
-	}
-
 	return pod.ExpectedLinesNumber - len(pod.Occurrences), nil
 }