Merge pull request #60237 from crassirostris/audit-use-buffered-backend

cluster/gce/gci/configure-helper.sh

@@ -1538,7 +1538,7 @@ function start-kube-apiserver {
         params+=" --audit-webhook-batch-throttle-burst=${ADVANCED_AUDIT_WEBHOOK_THROTTLE_BURST}"
       fi
       if [[ -n "${ADVANCED_AUDIT_WEBHOOK_INITIAL_BACKOFF:-}" ]]; then
-        params+=" --audit-webhook-batch-initial-backoff=${ADVANCED_AUDIT_WEBHOOK_INITIAL_BACKOFF}"
+        params+=" --audit-webhook-initial-backoff=${ADVANCED_AUDIT_WEBHOOK_INITIAL_BACKOFF}"
       fi
       create-master-audit-webhook-config "${audit_webhook_config_file}"
       audit_webhook_config_mount="{\"name\": \"auditwebhookconfigmount\",\"mountPath\": \"${audit_webhook_config_file}\", \"readOnly\": true},"

cmd/kube-apiserver/app/options/BUILD

@@ -46,7 +46,7 @@ go_test(
         "//vendor/k8s.io/apiserver/pkg/server/options:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/util/flag:go_default_library",
-        "//vendor/k8s.io/apiserver/plugin/pkg/audit/webhook:go_default_library",
+        "//vendor/k8s.io/apiserver/plugin/pkg/audit/buffered:go_default_library",
         "//vendor/k8s.io/client-go/rest:go_default_library",
     ],
 )

cmd/kube-apiserver/app/options/options_test.go

@@ -29,7 +29,7 @@ import (
 	genericoptions "k8s.io/apiserver/pkg/server/options"
 	"k8s.io/apiserver/pkg/storage/storagebackend"
 	utilflag "k8s.io/apiserver/pkg/util/flag"
-	auditwebhook "k8s.io/apiserver/plugin/pkg/audit/webhook"
+	auditbuffered "k8s.io/apiserver/plugin/pkg/audit/buffered"
 	restclient "k8s.io/client-go/rest"
 	"k8s.io/kubernetes/pkg/api/legacyscheme"
 	kapi "k8s.io/kubernetes/pkg/apis/core"
@@ -54,15 +54,23 @@ func TestAddFlags(t *testing.T) {
 		"--audit-log-maxbackup=12",
 		"--audit-log-maxsize=13",
 		"--audit-log-path=/var/log",
+		"--audit-log-mode=blocking",
+		"--audit-log-batch-buffer-size=46",
+		"--audit-log-batch-max-size=47",
+		"--audit-log-batch-max-wait=48s",
+		"--audit-log-batch-throttle-enable=true",
+		"--audit-log-batch-throttle-qps=49.5",
+		"--audit-log-batch-throttle-burst=50",
 		"--audit-policy-file=/policy",
 		"--audit-webhook-config-file=/webhook-config",
 		"--audit-webhook-mode=blocking",
 		"--audit-webhook-batch-buffer-size=42",
 		"--audit-webhook-batch-max-size=43",
 		"--audit-webhook-batch-max-wait=1s",
+		"--audit-webhook-batch-throttle-enable=false",
 		"--audit-webhook-batch-throttle-qps=43.5",
 		"--audit-webhook-batch-throttle-burst=44",
-		"--audit-webhook-batch-initial-backoff=2s",
+		"--audit-webhook-initial-backoff=2s",
 		"--authentication-token-webhook-cache-ttl=3m",
 		"--authentication-token-webhook-config-file=/token-webhook-config",
 		"--authorization-mode=AlwaysDeny",
@@ -180,18 +188,32 @@ func TestAddFlags(t *testing.T) {
 				MaxBackups: 12,
 				MaxSize:    13,
 				Format:     "json",
+				BatchOptions: apiserveroptions.AuditBatchOptions{
+					Mode: "blocking",
+					BatchConfig: auditbuffered.BatchConfig{
+						BufferSize:     46,
+						MaxBatchSize:   47,
+						MaxBatchWait:   48 * time.Second,
+						ThrottleEnable: true,
+						ThrottleQPS:    49.5,
+						ThrottleBurst:  50,
+					},
+				},
 			},
 			WebhookOptions: apiserveroptions.AuditWebhookOptions{
-				Mode:       "blocking",
 				ConfigFile: "/webhook-config",
-				BatchConfig: auditwebhook.BatchBackendConfig{
-					BufferSize:     42,
-					MaxBatchSize:   43,
-					MaxBatchWait:   1 * time.Second,
-					ThrottleQPS:    43.5,
-					ThrottleBurst:  44,
-					InitialBackoff: 2 * time.Second,
+				BatchOptions: apiserveroptions.AuditBatchOptions{
+					Mode: "blocking",
+					BatchConfig: auditbuffered.BatchConfig{
+						BufferSize:     42,
+						MaxBatchSize:   43,
+						MaxBatchWait:   1 * time.Second,
+						ThrottleEnable: false,
+						ThrottleQPS:    43.5,
+						ThrottleBurst:  44,
+					},
 				},
+				InitialBackoff: 2 * time.Second,
 			},
 			PolicyFile: "/policy",
 		},

staging/src/k8s.io/apiextensions-apiserver/Godeps/Godeps.json

@@ -1471,6 +1471,10 @@
 			"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
 		},
 		{
+			"ImportPath": "k8s.io/apiserver/plugin/pkg/audit/buffered",
+			"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+		},
+		{
 			"ImportPath": "k8s.io/apiserver/plugin/pkg/audit/log",
 			"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
 		},

staging/src/k8s.io/apiserver/pkg/server/options/BUILD

@@ -55,6 +55,7 @@ go_library(
         "//vendor/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/util/flag:go_default_library",
+        "//vendor/k8s.io/apiserver/plugin/pkg/audit/buffered:go_default_library",
         "//vendor/k8s.io/apiserver/plugin/pkg/audit/log:go_default_library",
         "//vendor/k8s.io/apiserver/plugin/pkg/audit/webhook:go_default_library",
         "//vendor/k8s.io/client-go/informers:go_default_library",

staging/src/k8s.io/apiserver/plugin/pkg/audit/buffered/buffered.go

@@ -153,6 +153,12 @@ func (b *bufferedBackend) Shutdown() {
 	<-b.shutdownCh
 
 	// Wait until all sending routines exit.
+	//
+	// - When b.shutdownCh is closed, we know that the goroutine in Run has terminated.
+	// - This means that processIncomingEvents has terminated.
+	// - Which means that b.buffer is closed and cannot accept any new events anymore.
+	// - Because processEvents is called synchronously from the Run goroutine, the waitgroup has its final value.
+	// Hence wg.Wait will not miss any more outgoing batches.
 	b.wg.Wait()
 
 	b.delegateBackend.Shutdown()
@@ -250,7 +256,7 @@ func (b *bufferedBackend) ProcessEvents(ev ...*auditinternal.Event) {
 	// recover from.
 	defer func() {
 		if err := recover(); err != nil {
-			sendErr = fmt.Errorf("panic when processing events: %v", err)
+			sendErr = fmt.Errorf("audit backend shut down")
 		}
 		if sendErr != nil {
 			audit.HandlePluginError(pluginName, sendErr, ev[evIndex:]...)

staging/src/k8s.io/apiserver/plugin/pkg/audit/log/backend.go

@@ -32,6 +32,9 @@ const (
 	FormatLegacy = "legacy"
 	// FormatJson saves event in structured json format.
 	FormatJson = "json"
+
+	// PluginName is the name of this plugin, to be used in help and logs.
+	PluginName = "log"
 )
 
 // AllowedFormats are the formats known by log backend.
@@ -70,17 +73,17 @@ func (b *backend) logEvent(ev *auditinternal.Event) {
 	case FormatJson:
 		bs, err := runtime.Encode(audit.Codecs.LegacyCodec(b.groupVersion), ev)
 		if err != nil {
-			audit.HandlePluginError("log", err, ev)
+			audit.HandlePluginError(PluginName, err, ev)
 			return
 		}
 		line = string(bs[:])
 	default:
-		audit.HandlePluginError("log", fmt.Errorf("log format %q is not in list of known formats (%s)",
+		audit.HandlePluginError(PluginName, fmt.Errorf("log format %q is not in list of known formats (%s)",
 			b.format, strings.Join(AllowedFormats, ",")), ev)
 		return
 	}
 	if _, err := fmt.Fprint(b.out, line); err != nil {
-		audit.HandlePluginError("log", err, ev)
+		audit.HandlePluginError(PluginName, err, ev)
 	}
 }
 

staging/src/k8s.io/apiserver/plugin/pkg/audit/webhook/BUILD

@@ -8,10 +8,7 @@ load(
 
 go_test(
     name = "go_default_test",
-    srcs = [
-        "webhook_test.go",
-        "webhook_v1alpha1_test.go",
-    ],
+    srcs = ["webhook_test.go"],
     embed = [":go_default_library"],
     deps = [
         "//vendor/github.com/stretchr/testify/assert:go_default_library",
@@ -20,7 +17,6 @@ go_test(
         "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime/serializer/json:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/apis/audit:go_default_library",
-        "//vendor/k8s.io/apiserver/pkg/apis/audit/v1alpha1:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/apis/audit/v1beta1:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/audit:go_default_library",
         "//vendor/k8s.io/client-go/tools/clientcmd/api/v1:go_default_library",
@@ -35,7 +31,6 @@ go_library(
         "//vendor/k8s.io/apimachinery/pkg/apimachinery/announced:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/apis/audit:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/apis/audit/install:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/apis/audit/v1alpha1:go_default_library",
@@ -43,7 +38,6 @@ go_library(
         "//vendor/k8s.io/apiserver/pkg/audit:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/util/webhook:go_default_library",
         "//vendor/k8s.io/client-go/rest:go_default_library",
-        "//vendor/k8s.io/client-go/util/flowcontrol:go_default_library",
     ],
 )
 

staging/src/k8s.io/apiserver/plugin/pkg/audit/webhook/webhook_test.go

@@ -25,10 +25,7 @@ import (
 	"net/http/httptest"
 	"os"
 	"reflect"
-	"sync"
-	"sync/atomic"
 	"testing"
-	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -91,15 +88,7 @@ func (t *testWebhookHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	http.Error(w, err.Error(), http.StatusInternalServerError)
 }
 
-func newTestBlockingWebhook(t *testing.T, endpoint string, groupVersion schema.GroupVersion) *blockingBackend {
-	return newWebhook(t, endpoint, ModeBlocking, groupVersion).(*blockingBackend)
-}
-
-func newTestBatchWebhook(t *testing.T, endpoint string, groupVersion schema.GroupVersion) *batchBackend {
-	return newWebhook(t, endpoint, ModeBatch, groupVersion).(*batchBackend)
-}
-
-func newWebhook(t *testing.T, endpoint string, mode string, groupVersion schema.GroupVersion) audit.Backend {
+func newWebhook(t *testing.T, endpoint string, groupVersion schema.GroupVersion) *backend {
 	config := v1.Config{
 		Clusters: []v1.NamedCluster{
 			{Cluster: v1.Cluster{Server: endpoint, InsecureSkipTLSVerify: true}},
@@ -116,10 +105,10 @@ func newWebhook(t *testing.T, endpoint string, mode string, groupVersion schema.
 	// NOTE(ericchiang): Do we need to use a proper serializer?
 	require.NoError(t, stdjson.NewEncoder(f).Encode(config), "writing kubeconfig")
 
-	backend, err := NewBackend(f.Name(), mode, groupVersion, NewDefaultBatchBackendConfig())
+	b, err := NewBackend(f.Name(), groupVersion)
 	require.NoError(t, err, "initializing backend")
 
-	return backend
+	return b.(*backend)
 }
 
 func TestWebhook(t *testing.T) {
@@ -131,275 +120,9 @@ func TestWebhook(t *testing.T) {
 	}))
 	defer s.Close()
 
-	backend := newTestBlockingWebhook(t, s.URL, auditv1beta1.SchemeGroupVersion)
+	backend := newWebhook(t, s.URL, auditv1beta1.SchemeGroupVersion)
 
 	// Ensure this doesn't return a serialization error.
 	event := &auditinternal.Event{}
 	require.NoError(t, backend.processEvents(event), "failed to send events")
 }
-
-// waitForEmptyBuffer indicates when the sendBatchEvents method has read from the
-// existing buffer. This lets test coordinate closing a timer and stop channel
-// until the for loop has read from the buffer.
-func waitForEmptyBuffer(b *batchBackend) {
-	for len(b.buffer) != 0 {
-		time.Sleep(time.Millisecond)
-	}
-}
-
-func TestBatchWebhookMaxEvents(t *testing.T) {
-	nRest := 10
-	events := make([]*auditinternal.Event, defaultBatchMaxSize+nRest) // greater than max size.
-	for i := range events {
-		events[i] = &auditinternal.Event{}
-	}
-
-	got := make(chan int, 2)
-	s := httptest.NewServer(newWebhookHandler(t, &auditv1beta1.EventList{}, func(events runtime.Object) {
-		got <- len(events.(*auditv1beta1.EventList).Items)
-	}))
-	defer s.Close()
-
-	backend := newTestBatchWebhook(t, s.URL, auditv1beta1.SchemeGroupVersion)
-
-	backend.ProcessEvents(events...)
-
-	stopCh := make(chan struct{})
-	timer := make(chan time.Time, 1)
-
-	backend.sendBatchEvents(backend.collectEvents(stopCh, timer))
-	require.Equal(t, defaultBatchMaxSize, <-got, "did not get batch max size")
-
-	go func() {
-		waitForEmptyBuffer(backend) // wait for the buffer to empty
-		timer <- time.Now()         // Trigger the wait timeout
-	}()
-
-	backend.sendBatchEvents(backend.collectEvents(stopCh, timer))
-	require.Equal(t, nRest, <-got, "failed to get the rest of the events")
-}
-
-func TestBatchWebhookStopCh(t *testing.T) {
-	events := make([]*auditinternal.Event, 1) // less than max size.
-	for i := range events {
-		events[i] = &auditinternal.Event{}
-	}
-
-	expected := len(events)
-	got := make(chan int, 2)
-	s := httptest.NewServer(newWebhookHandler(t, &auditv1beta1.EventList{}, func(events runtime.Object) {
-		got <- len(events.(*auditv1beta1.EventList).Items)
-	}))
-	defer s.Close()
-
-	backend := newTestBatchWebhook(t, s.URL, auditv1beta1.SchemeGroupVersion)
-	backend.ProcessEvents(events...)
-
-	stopCh := make(chan struct{})
-	timer := make(chan time.Time)
-
-	go func() {
-		waitForEmptyBuffer(backend)
-		close(stopCh) // stop channel has stopped
-	}()
-	backend.sendBatchEvents(backend.collectEvents(stopCh, timer))
-	require.Equal(t, expected, <-got, "get queued events after timer expires")
-}
-
-func TestBatchWebhookProcessEventsAfterStop(t *testing.T) {
-	events := make([]*auditinternal.Event, 1) // less than max size.
-	for i := range events {
-		events[i] = &auditinternal.Event{}
-	}
-
-	got := make(chan struct{})
-	s := httptest.NewServer(newWebhookHandler(t, &auditv1beta1.EventList{}, func(events runtime.Object) {
-		close(got)
-	}))
-	defer s.Close()
-
-	backend := newTestBatchWebhook(t, s.URL, auditv1beta1.SchemeGroupVersion)
-	stopCh := make(chan struct{})
-
-	backend.Run(stopCh)
-	close(stopCh)
-	<-backend.shutdownCh
-	backend.ProcessEvents(events...)
-	assert.Equal(t, 0, len(backend.buffer), "processed events after the backed has been stopped")
-}
-
-func TestBatchWebhookShutdown(t *testing.T) {
-	events := make([]*auditinternal.Event, 1)
-	for i := range events {
-		events[i] = &auditinternal.Event{}
-	}
-
-	got := make(chan struct{})
-	contReqCh := make(chan struct{})
-	shutdownCh := make(chan struct{})
-	s := httptest.NewServer(newWebhookHandler(t, &auditv1beta1.EventList{}, func(events runtime.Object) {
-		close(got)
-		<-contReqCh
-	}))
-	defer s.Close()
-
-	backend := newTestBatchWebhook(t, s.URL, auditv1beta1.SchemeGroupVersion)
-	backend.ProcessEvents(events...)
-
-	go func() {
-		// Assume stopCh was closed.
-		close(backend.buffer)
-		backend.sendBatchEvents(backend.collectLastEvents())
-	}()
-
-	<-got
-
-	go func() {
-		close(backend.shutdownCh)
-		backend.Shutdown()
-		close(shutdownCh)
-	}()
-
-	// Wait for some time in case there's a bug that allows for the Shutdown
-	// method to exit before all requests has been completed.
-	time.Sleep(1 * time.Second)
-	select {
-	case <-shutdownCh:
-		t.Fatal("Backend shut down before all requests finished")
-	default:
-		// Continue.
-	}
-
-	close(contReqCh)
-	<-shutdownCh
-}
-
-func TestBatchWebhookEmptyBuffer(t *testing.T) {
-	events := make([]*auditinternal.Event, 1) // less than max size.
-	for i := range events {
-		events[i] = &auditinternal.Event{}
-	}
-
-	expected := len(events)
-	got := make(chan int, 2)
-	s := httptest.NewServer(newWebhookHandler(t, &auditv1beta1.EventList{}, func(events runtime.Object) {
-		got <- len(events.(*auditv1beta1.EventList).Items)
-	}))
-	defer s.Close()
-
-	backend := newTestBatchWebhook(t, s.URL, auditv1beta1.SchemeGroupVersion)
-
-	stopCh := make(chan struct{})
-	timer := make(chan time.Time, 1)
-
-	timer <- time.Now() // Timer is done.
-
-	// Buffer is empty, no events have been queued. This should exit but send no events.
-	backend.sendBatchEvents(backend.collectEvents(stopCh, timer))
-
-	// Send additional events after the sendBatchEvents has been called.
-	backend.ProcessEvents(events...)
-	go func() {
-		waitForEmptyBuffer(backend)
-		timer <- time.Now()
-	}()
-
-	backend.sendBatchEvents(backend.collectEvents(stopCh, timer))
-
-	// Make sure we didn't get a POST with zero events.
-	require.Equal(t, expected, <-got, "expected one event")
-}
-
-func TestBatchBufferFull(t *testing.T) {
-	events := make([]*auditinternal.Event, defaultBatchBufferSize+1) // More than buffered size
-	for i := range events {
-		events[i] = &auditinternal.Event{}
-	}
-	s := httptest.NewServer(newWebhookHandler(t, &auditv1beta1.EventList{}, func(events runtime.Object) {
-		// Do nothing.
-	}))
-	defer s.Close()
-
-	backend := newTestBatchWebhook(t, s.URL, auditv1beta1.SchemeGroupVersion)
-
-	// Make sure this doesn't block.
-	backend.ProcessEvents(events...)
-}
-
-func TestBatchRun(t *testing.T) {
-
-	// Divisable by max batch size so we don't have to wait for a minute for
-	// the test to finish.
-	events := make([]*auditinternal.Event, defaultBatchMaxSize*3)
-	for i := range events {
-		events[i] = &auditinternal.Event{}
-	}
-
-	got := new(int64)
-	want := len(events)
-
-	wg := new(sync.WaitGroup)
-	wg.Add(want)
-	done := make(chan struct{})
-
-	go func() {
-		wg.Wait()
-		// When the expected number of events have been received, close the channel.
-		close(done)
-	}()
-
-	s := httptest.NewServer(newWebhookHandler(t, &auditv1beta1.EventList{}, func(obj runtime.Object) {
-		events := obj.(*auditv1beta1.EventList)
-		atomic.AddInt64(got, int64(len(events.Items)))
-		wg.Add(-len(events.Items))
-	}))
-	defer s.Close()
-
-	stopCh := make(chan struct{})
-	defer close(stopCh)
-
-	backend := newTestBatchWebhook(t, s.URL, auditv1beta1.SchemeGroupVersion)
-
-	// Test the Run codepath. E.g. that the spawned goroutines behave correctly.
-	backend.Run(stopCh)
-
-	backend.ProcessEvents(events...)
-
-	select {
-	case <-done:
-		// Received all the events.
-	case <-time.After(2 * time.Minute):
-		t.Errorf("expected %d events got %d", want, atomic.LoadInt64(got))
-	}
-}
-
-func TestBatchConcurrentRequests(t *testing.T) {
-	events := make([]*auditinternal.Event, defaultBatchBufferSize) // Don't drop events
-	for i := range events {
-		events[i] = &auditinternal.Event{}
-	}
-
-	wg := new(sync.WaitGroup)
-	wg.Add(len(events))
-
-	s := httptest.NewServer(newWebhookHandler(t, &auditv1beta1.EventList{}, func(events runtime.Object) {
-		wg.Add(-len(events.(*auditv1beta1.EventList).Items))
-
-		// Since the webhook makes concurrent requests, blocking on the webhook response
-		// shouldn't block the webhook from sending more events.
-		//
-		// Wait for all responses to be received before sending the response.
-		wg.Wait()
-	}))
-	defer s.Close()
-
-	stopCh := make(chan struct{})
-	defer close(stopCh)
-
-	backend := newTestBatchWebhook(t, s.URL, auditv1beta1.SchemeGroupVersion)
-	backend.Run(stopCh)
-
-	backend.ProcessEvents(events...)
-	// Wait for the webhook to receive all events.
-	wg.Wait()
-}

staging/src/k8s.io/apiserver/plugin/pkg/audit/webhook/webhook_v1alpha1_test.go

-/*
-Copyright 2017 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package webhook
-
-import (
-	"net/http/httptest"
-	"sync"
-	"sync/atomic"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	"k8s.io/apimachinery/pkg/runtime"
-	auditinternal "k8s.io/apiserver/pkg/apis/audit"
-	auditv1alpha1 "k8s.io/apiserver/pkg/apis/audit/v1alpha1"
-)
-
-func TestBatchWebhookMaxEventsV1Alpha1(t *testing.T) {
-	nRest := 10
-	events := make([]*auditinternal.Event, defaultBatchMaxSize+nRest) // greater than max size.
-	for i := range events {
-		events[i] = &auditinternal.Event{}
-	}
-
-	got := make(chan int, 2)
-	s := httptest.NewServer(newWebhookHandler(t, &auditv1alpha1.EventList{}, func(events runtime.Object) {
-		got <- len(events.(*auditv1alpha1.EventList).Items)
-	}))
-	defer s.Close()
-
-	backend := newTestBatchWebhook(t, s.URL, auditv1alpha1.SchemeGroupVersion)
-
-	backend.ProcessEvents(events...)
-
-	stopCh := make(chan struct{})
-	timer := make(chan time.Time, 1)
-
-	backend.sendBatchEvents(backend.collectEvents(stopCh, timer))
-	require.Equal(t, defaultBatchMaxSize, <-got, "did not get batch max size")
-
-	go func() {
-		waitForEmptyBuffer(backend) // wait for the buffer to empty
-		timer <- time.Now()         // Trigger the wait timeout
-	}()
-
-	backend.sendBatchEvents(backend.collectEvents(stopCh, timer))
-	require.Equal(t, nRest, <-got, "failed to get the rest of the events")
-}
-
-func TestBatchWebhookStopChV1Alpha1(t *testing.T) {
-	events := make([]*auditinternal.Event, 1) // less than max size.
-	for i := range events {
-		events[i] = &auditinternal.Event{}
-	}
-
-	expected := len(events)
-	got := make(chan int, 2)
-	s := httptest.NewServer(newWebhookHandler(t, &auditv1alpha1.EventList{}, func(events runtime.Object) {
-		got <- len(events.(*auditv1alpha1.EventList).Items)
-	}))
-	defer s.Close()
-
-	backend := newTestBatchWebhook(t, s.URL, auditv1alpha1.SchemeGroupVersion)
-	backend.ProcessEvents(events...)
-
-	stopCh := make(chan struct{})
-	timer := make(chan time.Time)
-
-	go func() {
-		waitForEmptyBuffer(backend)
-		close(stopCh) // stop channel has stopped
-	}()
-	backend.sendBatchEvents(backend.collectEvents(stopCh, timer))
-	require.Equal(t, expected, <-got, "get queued events after timer expires")
-}
-
-func TestBatchWebhookProcessEventsAfterStopV1Alpha1(t *testing.T) {
-	events := make([]*auditinternal.Event, 1) // less than max size.
-	for i := range events {
-		events[i] = &auditinternal.Event{}
-	}
-
-	got := make(chan struct{})
-	s := httptest.NewServer(newWebhookHandler(t, &auditv1alpha1.EventList{}, func(events runtime.Object) {
-		close(got)
-	}))
-	defer s.Close()
-
-	backend := newTestBatchWebhook(t, s.URL, auditv1alpha1.SchemeGroupVersion)
-	stopCh := make(chan struct{})
-
-	backend.Run(stopCh)
-	close(stopCh)
-	<-backend.shutdownCh
-	backend.ProcessEvents(events...)
-	assert.Equal(t, 0, len(backend.buffer), "processed events after the backed has been stopped")
-}
-
-func TestBatchWebhookShutdownV1Alpha1(t *testing.T) {
-	events := make([]*auditinternal.Event, 1)
-	for i := range events {
-		events[i] = &auditinternal.Event{}
-	}
-
-	got := make(chan struct{})
-	contReqCh := make(chan struct{})
-	shutdownCh := make(chan struct{})
-	s := httptest.NewServer(newWebhookHandler(t, &auditv1alpha1.EventList{}, func(events runtime.Object) {
-		close(got)
-		<-contReqCh
-	}))
-	defer s.Close()
-
-	backend := newTestBatchWebhook(t, s.URL, auditv1alpha1.SchemeGroupVersion)
-	backend.ProcessEvents(events...)
-
-	go func() {
-		// Assume stopCh was closed.
-		close(backend.buffer)
-		backend.sendBatchEvents(backend.collectLastEvents())
-	}()
-
-	<-got
-
-	go func() {
-		close(backend.shutdownCh)
-		backend.Shutdown()
-		close(shutdownCh)
-	}()
-
-	// Wait for some time in case there's a bug that allows for the Shutdown
-	// method to exit before all requests has been completed.
-	time.Sleep(1 * time.Second)
-	select {
-	case <-shutdownCh:
-		t.Fatal("Backend shut down before all requests finished")
-	default:
-		// Continue.
-	}
-
-	close(contReqCh)
-	<-shutdownCh
-}
-
-func TestBatchWebhookEmptyBufferV1Alpha1(t *testing.T) {
-	events := make([]*auditinternal.Event, 1) // less than max size.
-	for i := range events {
-		events[i] = &auditinternal.Event{}
-	}
-
-	expected := len(events)
-	got := make(chan int, 2)
-	s := httptest.NewServer(newWebhookHandler(t, &auditv1alpha1.EventList{}, func(events runtime.Object) {
-		got <- len(events.(*auditv1alpha1.EventList).Items)
-	}))
-	defer s.Close()
-
-	backend := newTestBatchWebhook(t, s.URL, auditv1alpha1.SchemeGroupVersion)
-
-	stopCh := make(chan struct{})
-	timer := make(chan time.Time, 1)
-
-	timer <- time.Now() // Timer is done.
-
-	// Buffer is empty, no events have been queued. This should exit but send no events.
-	backend.sendBatchEvents(backend.collectEvents(stopCh, timer))
-
-	// Send additional events after the sendBatchEvents has been called.
-	backend.ProcessEvents(events...)
-	go func() {
-		waitForEmptyBuffer(backend)
-		timer <- time.Now()
-	}()
-
-	backend.sendBatchEvents(backend.collectEvents(stopCh, timer))
-
-	// Make sure we didn't get a POST with zero events.
-	require.Equal(t, expected, <-got, "expected one event")
-}
-
-func TestBatchBufferFullV1Alpha1(t *testing.T) {
-	events := make([]*auditinternal.Event, defaultBatchBufferSize+1) // More than buffered size
-	for i := range events {
-		events[i] = &auditinternal.Event{}
-	}
-	s := httptest.NewServer(newWebhookHandler(t, &auditv1alpha1.EventList{}, func(events runtime.Object) {
-		// Do nothing.
-	}))
-	defer s.Close()
-
-	backend := newTestBatchWebhook(t, s.URL, auditv1alpha1.SchemeGroupVersion)
-
-	// Make sure this doesn't block.
-	backend.ProcessEvents(events...)
-}
-
-func TestBatchRunV1Alpha1(t *testing.T) {
-
-	// Divisable by max batch size so we don't have to wait for a minute for
-	// the test to finish.
-	events := make([]*auditinternal.Event, defaultBatchMaxSize*3)
-	for i := range events {
-		events[i] = &auditinternal.Event{}
-	}
-
-	got := new(int64)
-	want := len(events)
-
-	wg := new(sync.WaitGroup)
-	wg.Add(want)
-	done := make(chan struct{})
-
-	go func() {
-		wg.Wait()
-		// When the expected number of events have been received, close the channel.
-		close(done)
-	}()
-
-	s := httptest.NewServer(newWebhookHandler(t, &auditv1alpha1.EventList{}, func(obj runtime.Object) {
-		events := obj.(*auditv1alpha1.EventList)
-		atomic.AddInt64(got, int64(len(events.Items)))
-		wg.Add(-len(events.Items))
-	}))
-	defer s.Close()
-
-	stopCh := make(chan struct{})
-	defer close(stopCh)
-
-	backend := newTestBatchWebhook(t, s.URL, auditv1alpha1.SchemeGroupVersion)
-
-	// Test the Run codepath. E.g. that the spawned goroutines behave correctly.
-	backend.Run(stopCh)
-
-	backend.ProcessEvents(events...)
-
-	select {
-	case <-done:
-		// Received all the events.
-	case <-time.After(2 * time.Minute):
-		t.Errorf("expected %d events got %d", want, atomic.LoadInt64(got))
-	}
-}
-
-func TestBatchConcurrentRequestsV1Alpha1(t *testing.T) {
-	events := make([]*auditinternal.Event, defaultBatchBufferSize) // Don't drop events
-	for i := range events {
-		events[i] = &auditinternal.Event{}
-	}
-
-	wg := new(sync.WaitGroup)
-	wg.Add(len(events))
-
-	s := httptest.NewServer(newWebhookHandler(t, &auditv1alpha1.EventList{}, func(events runtime.Object) {
-		wg.Add(-len(events.(*auditv1alpha1.EventList).Items))
-
-		// Since the webhook makes concurrent requests, blocking on the webhook response
-		// shouldn't block the webhook from sending more events.
-		//
-		// Wait for all responses to be received before sending the response.
-		wg.Wait()
-	}))
-	defer s.Close()
-
-	stopCh := make(chan struct{})
-	defer close(stopCh)
-
-	backend := newTestBatchWebhook(t, s.URL, auditv1alpha1.SchemeGroupVersion)
-	backend.Run(stopCh)
-
-	backend.ProcessEvents(events...)
-	// Wait for the webhook to receive all events.
-	wg.Wait()
-}

staging/src/k8s.io/kube-aggregator/Godeps/Godeps.json

@@ -1143,6 +1143,10 @@
 			"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
 		},
 		{
+			"ImportPath": "k8s.io/apiserver/plugin/pkg/audit/buffered",
+			"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+		},
+		{
 			"ImportPath": "k8s.io/apiserver/plugin/pkg/audit/log",
 			"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
 		},

staging/src/k8s.io/sample-apiserver/Godeps/Godeps.json

@@ -1111,6 +1111,10 @@
 			"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
 		},
 		{
+			"ImportPath": "k8s.io/apiserver/plugin/pkg/audit/buffered",
+			"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+		},
+		{
 			"ImportPath": "k8s.io/apiserver/plugin/pkg/audit/log",
 			"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
 		},