Covert Stackdriver Logging load tests to soak tests

test/e2e/cluster-logging/BUILD

@@ -14,7 +14,7 @@ go_library(
         "es_utils.go",
         "sd.go",
         "sd_events.go",
-        "sd_load.go",
+        "sd_soak.go",
         "sd_utils.go",
         "utils.go",
     ],

test/e2e/cluster-logging/sd.go

@@ -26,24 +26,24 @@ import (
 	. "github.com/onsi/ginkgo"
 )
 
-var _ = framework.KubeDescribe("Cluster level logging using GCL", func() {
-	f := framework.NewDefaultFramework("gcl-logging")
+var _ = framework.KubeDescribe("Cluster level logging implemented by Stackdriver", func() {
+	f := framework.NewDefaultFramework("sd-logging")
 
 	BeforeEach(func() {
 		framework.SkipUnlessProviderIs("gce", "gke")
 	})
 
-	It("should check that logs from containers are ingested in GCL", func() {
+	It("should ingest logs from applications", func() {
 		podName := "synthlogger"
 
-		gclLogsProvider, err := newGclLogsProvider(f)
-		framework.ExpectNoError(err, "Failed to create GCL logs provider")
+		sdLogsProvider, err := newSdLogsProvider(f)
+		framework.ExpectNoError(err, "Failed to create Stackdriver logs provider")
 
-		err = gclLogsProvider.Init()
-		defer gclLogsProvider.Cleanup()
-		framework.ExpectNoError(err, "Failed to init GCL logs provider")
+		err = sdLogsProvider.Init()
+		defer sdLogsProvider.Cleanup()
+		framework.ExpectNoError(err, "Failed to init Stackdriver logs provider")
 
-		err = ensureSingleFluentdOnEachNode(f, gclLogsProvider.FluentdApplicationName())
+		err = ensureSingleFluentdOnEachNode(f, sdLogsProvider.FluentdApplicationName())
 		framework.ExpectNoError(err, "Fluentd deployed incorrectly")
 
 		By("Running synthetic logger")
@@ -54,7 +54,7 @@ var _ = framework.KubeDescribe("Cluster level logging using GCL", func() {
 
 		By("Waiting for logs to ingest")
 		config := &loggingTestConfig{
-			LogsProvider:              gclLogsProvider,
+			LogsProvider:              sdLogsProvider,
 			Pods:                      []*loggingPod{pod},
 			IngestionTimeout:          10 * time.Minute,
 			MaxAllowedLostFraction:    0,

test/e2e/cluster-logging/sd_events.go

@@ -40,26 +40,26 @@ const (
 	eventCreationInterval = 10 * time.Second
 )
 
-var _ = framework.KubeDescribe("Cluster level logging using GCL", func() {
-	f := framework.NewDefaultFramework("gcl-logging-events")
+var _ = framework.KubeDescribe("Cluster level logging implemented by Stackdriver", func() {
+	f := framework.NewDefaultFramework("sd-logging-events")
 
 	BeforeEach(func() {
 		framework.SkipUnlessProviderIs("gce", "gke")
 	})
 
 	It("should ingest events", func() {
-		gclLogsProvider, err := newGclLogsProvider(f)
-		framework.ExpectNoError(err, "Failed to create GCL logs provider")
+		sdLogsProvider, err := newSdLogsProvider(f)
+		framework.ExpectNoError(err, "Failed to create Stackdriver logs provider")
 
-		err = gclLogsProvider.Init()
-		defer gclLogsProvider.Cleanup()
-		framework.ExpectNoError(err, "Failed to init GCL logs provider")
+		err = sdLogsProvider.Init()
+		defer sdLogsProvider.Cleanup()
+		framework.ExpectNoError(err, "Failed to init Stackdriver logs provider")
 
 		stopCh := make(chan struct{})
 		successCh := make(chan struct{})
 		go func() {
 			wait.Poll(eventPollingInterval, eventsIngestionTimeout, func() (bool, error) {
-				events := gclLogsProvider.ReadEvents()
+				events := sdLogsProvider.ReadEvents()
 				if len(events) > 0 {
 					framework.Logf("Some events are ingested, sample event: %v", events[0])
 					close(successCh)

test/e2e/cluster-logging/sd_load.go → test/e2e/cluster-logging/sd_soak.go

@@ -18,6 +18,7 @@ package e2e
 
 import (
 	"fmt"
+	"math"
 	"time"
 
 	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -27,69 +28,33 @@ import (
 )
 
 const (
-	loadTestMaxAllowedLostFraction    = 0.01
-	loadTestMaxAllowedFluentdRestarts = 1
+	// maxAllowedLostFraction is the fraction of lost logs considered acceptable.
+	maxAllowedLostFraction = 0.01
+	// maxAllowedRestartsPerHour is the number of fluentd container restarts
+	// considered acceptable. Once per hour is fine for now, as long as it
+	// doesn't loose too much logs.
+	maxAllowedRestartsPerHour = 1.0
 )
 
-var _ = framework.KubeDescribe("Cluster level logging using GCL [Feature:StackdriverLogging]", func() {
-	f := framework.NewDefaultFramework("gcl-logging-load")
+var _ = framework.KubeDescribe("Cluster level logging implemented by Stackdriver [Feature:StackdriverLogging] [Soak]", func() {
+	f := framework.NewDefaultFramework("sd-logging-load")
 
-	It("should create a constant load with long-living pods and ensure logs delivery", func() {
-		gclLogsProvider, err := newGclLogsProvider(f)
-		framework.ExpectNoError(err, "Failed to create GCL logs provider")
+	It("should ingest logs from applications running for a prolonged amount of time", func() {
+		sdLogsProvider, err := newSdLogsProvider(f)
+		framework.ExpectNoError(err, "Failed to create Stackdriver logs provider")
 
-		err = gclLogsProvider.Init()
-		defer gclLogsProvider.Cleanup()
-		framework.ExpectNoError(err, "Failed to init GCL logs provider")
-
-		nodes := framework.GetReadySchedulableNodesOrDie(f.ClientSet).Items
-		nodeCount := len(nodes)
-		podCount := 30 * nodeCount
-		loggingDuration := 10 * time.Minute
-		linesPerSecond := 1000 * nodeCount
-		linesPerPod := linesPerSecond * int(loggingDuration.Seconds()) / podCount
-		ingestionTimeout := 20 * time.Minute
-
-		By("Running logs generator pods")
-		pods := []*loggingPod{}
-		for podIdx := 0; podIdx < podCount; podIdx++ {
-			node := nodes[podIdx%len(nodes)]
-			podName := fmt.Sprintf("logs-generator-%d-%d", linesPerPod, podIdx)
-			pods = append(pods, createLoggingPod(f, podName, node.Name, linesPerPod, loggingDuration))
-
-			defer f.PodClient().Delete(podName, &meta_v1.DeleteOptions{})
-		}
-
-		By("Waiting for all log lines to be ingested")
-		config := &loggingTestConfig{
-			LogsProvider:              gclLogsProvider,
-			Pods:                      pods,
-			IngestionTimeout:          ingestionTimeout,
-			MaxAllowedLostFraction:    loadTestMaxAllowedLostFraction,
-			MaxAllowedFluentdRestarts: loadTestMaxAllowedFluentdRestarts,
-		}
-		err = waitForFullLogsIngestion(f, config)
-		if err != nil {
-			framework.Failf("Failed to ingest logs: %v", err)
-		} else {
-			framework.Logf("Successfully ingested all logs")
-		}
-	})
-
-	It("should create a constant load with short-living pods and ensure logs delivery", func() {
-		gclLogsProvider, err := newGclLogsProvider(f)
-		framework.ExpectNoError(err, "Failed to create GCL logs provider")
-
-		err = gclLogsProvider.Init()
-		defer gclLogsProvider.Cleanup()
-		framework.ExpectNoError(err, "Failed to init GCL logs provider")
+		err = sdLogsProvider.Init()
+		defer sdLogsProvider.Cleanup()
+		framework.ExpectNoError(err, "Failed to init Stackdriver logs provider")
 
 		nodes := framework.GetReadySchedulableNodesOrDie(f.ClientSet).Items
 		maxPodCount := 10
-		jobDuration := 1 * time.Minute
+		jobDuration := 1 * time.Hour
 		linesPerPodPerSecond := 100
-		testDuration := 10 * time.Minute
-		ingestionTimeout := 20 * time.Minute
+		testDuration := 21 * time.Hour
+		ingestionTimeout := testDuration + 30*time.Minute
+		allowedRestarts := int(math.Ceil(float64(testDuration) /
+			float64(time.Hour) * maxAllowedRestartsPerHour))
 
 		podRunDelay := time.Duration(int64(jobDuration) / int64(maxPodCount))
 		podRunCount := int(testDuration.Seconds())/int(podRunDelay.Seconds()) - 1
@@ -109,11 +74,11 @@ var _ = framework.KubeDescribe("Cluster level logging using GCL [Feature:Stackdr
 
 		By("Waiting for all log lines to be ingested")
 		config := &loggingTestConfig{
-			LogsProvider:              gclLogsProvider,
+			LogsProvider:              sdLogsProvider,
 			Pods:                      pods,
 			IngestionTimeout:          ingestionTimeout,
-			MaxAllowedLostFraction:    loadTestMaxAllowedLostFraction,
-			MaxAllowedFluentdRestarts: loadTestMaxAllowedFluentdRestarts,
+			MaxAllowedLostFraction:    maxAllowedLostFraction,
+			MaxAllowedFluentdRestarts: allowedRestarts,
 		}
 		err = waitForFullLogsIngestion(f, config)
 		if err != nil {

test/e2e/cluster-logging/utils.go

@@ -213,6 +213,11 @@ func waitForFullLogsIngestion(f *framework.Framework, config *loggingTestConfig)
 	if totalMissing > 0 {
 		framework.Logf("After %v still missing %d lines, %.2f%% of total number of lines",
 			config.IngestionTimeout, totalMissing, lostFraction*100)
+		for podIdx, missing := range missingByPod {
+			if missing != 0 {
+				framework.Logf("Still missing %d lines for pod %v", missing, config.Pods[podIdx])
+			}
+		}
 	}
 
 	if lostFraction > config.MaxAllowedLostFraction {