Merge pull request #6694 from bprashanth/bench

hack/benchmark-integration.sh

+#!/bin/bash
+
+# Copyright 2014 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
+BENCHMARK_REGEX=${BENCHMARK_REGEX:-"."}
+
+source "${KUBE_ROOT}/hack/lib/init.sh"
+
+cleanup() {
+  kube::etcd::cleanup
+  kube::log::status "Benchmark cleanup complete"
+}
+
+runTests() {
+  kube::etcd::start
+  kube::log::status "Running benchmarks"
+  KUBE_GOFLAGS="-tags 'benchmark no-docker' -bench . -benchtime 1s -cpu 4" \
+    KUBE_RACE="-race" \
+    KUBE_TEST_API_VERSIONS="v1beta3" \
+    KUBE_TIMEOUT="-timeout 10m" \
+    KUBE_TEST_ETCD_PREFIXES="registry"\
+    ETCD_CUSTOM_PREFIX="None" \
+    KUBE_TEST_ARGS="-bench-quiet 0 -bench-pods 30 -bench-tasks 1"\
+    "${KUBE_ROOT}/hack/test-go.sh" test/integration
+  cleanup
+}
+
+# Run cleanup to stop etcd on interrupt or other kill signal.
+trap cleanup EXIT
+
+runTests

hack/test-go.sh

@@ -101,6 +101,7 @@ shift $((OPTIND - 1))
 
 # Use eval to preserve embedded quoted strings.
 eval "goflags=(${KUBE_GOFLAGS:-})"
+eval "testargs=(${KUBE_TEST_ARGS:-})"
 
 # Filter out arguments that start with "-" and move them to goflags.
 testcases=()
@@ -134,7 +135,8 @@ runTests() {
       count=0
       for i in $(seq 1 ${iterations}); do
         if go test "${goflags[@]:+${goflags[@]}}" \
-            ${KUBE_RACE} ${KUBE_TIMEOUT} "${pkg}"; then
+            ${KUBE_RACE} ${KUBE_TIMEOUT} "${pkg}" \
+            "${testargs[@]:+${testargs[@]}}"; then
           pass=$((pass + 1))
         else
           fails=$((fails + 1))
@@ -155,7 +157,8 @@ runTests() {
   if [[ ! ${KUBE_COVER} =~ ^[yY]$ ]]; then
     kube::log::status "Running unit tests without code coverage"
     go test "${goflags[@]:+${goflags[@]}}" \
-      ${KUBE_RACE} ${KUBE_TIMEOUT} "${@+${@/#/${KUBE_GO_PACKAGE}/}}"
+      ${KUBE_RACE} ${KUBE_TIMEOUT} "${@+${@/#/${KUBE_GO_PACKAGE}/}}" \
+     "${testargs[@]:+${testargs[@]}}"
     return 0
   fi
 
@@ -177,7 +180,8 @@ runTests() {
           -cover -covermode="${KUBE_COVERMODE}" \
           -coverprofile="${cover_report_dir}/{}/${cover_profile}" \
           "${cover_params[@]+${cover_params[@]}}" \
-          "${KUBE_GO_PACKAGE}/{}"
+          "${KUBE_GO_PACKAGE}/{}" \
+          "${testargs[@]:+${testargs[@]}}"
 
   COMBINED_COVER_PROFILE="${cover_report_dir}/combined-coverage.out"
   {

pkg/controller/replication_controller.go

@@ -169,6 +169,14 @@ func NewReplicationManager(kubeClient client.Interface, burstReplicas int) *Repl
 	return rm
 }
 
+// SetEventRecorder replaces the event recorder used by the replication manager
+// with the given recorder. Only used for testing.
+func (rm *ReplicationManager) SetEventRecorder(recorder record.EventRecorder) {
+	// TODO: Hack. We can't cleanly shutdown the event recorder, so benchmarks
+	// need to pass in a fake.
+	rm.podControl = RealPodControl{rm.kubeClient, recorder}
+}
+
 // Run begins watching and syncing.
 func (rm *ReplicationManager) Run(workers int, stopCh <-chan struct{}) {
 	defer util.HandleCrash()
@@ -178,6 +186,7 @@ func (rm *ReplicationManager) Run(workers int, stopCh <-chan struct{}) {
 		go util.Until(rm.worker, time.Second, stopCh)
 	}
 	<-stopCh
+	glog.Infof("Shutting down RC Manager")
 	rm.queue.ShutDown()
 }
 

pkg/kubectl/resize.go

@@ -101,6 +101,7 @@ type ReplicationControllerResizer struct {
 	c ResizerClient
 }
 
+// RetryParams encapsulates the retry parameters used by kubectl's resizer.
 type RetryParams struct {
 	Interval, Timeout time.Duration
 }

test/integration/auth_test.go

@@ -46,15 +46,14 @@ import (
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/auth/user"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/master"
-	"github.com/GoogleCloudPlatform/kubernetes/pkg/tools/etcdtest"
 	"github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/admission/admit"
 	"github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/auth/authenticator/token/tokentest"
+	"github.com/GoogleCloudPlatform/kubernetes/test/integration/framework"
 )
 
 var nodeResourceName string
 
 func init() {
-	requireEtcd()
 	if api.PreV1Beta3(testapi.Version()) {
 		nodeResourceName = "minions"
 	} else {
@@ -375,15 +374,13 @@ func getTestRequests() []struct {
 //
 // TODO(etune): write a fuzz test of the REST API.
 func TestAuthModeAlwaysAllow(t *testing.T) {
-	deleteAllEtcdKeys()
+	framework.DeleteAllEtcdKeys()
 
 	// Set up a master
-
-	helper, err := master.NewEtcdHelper(newEtcdClient(), testapi.Version(), etcdtest.PathPrefix())
+	helper, err := framework.NewHelper()
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
-
 	var m *master.Master
 	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 		m.Handler.ServeHTTP(w, req)
@@ -516,10 +513,10 @@ func getPreviousResourceVersionKey(url, id string) string {
 }
 
 func TestAuthModeAlwaysDeny(t *testing.T) {
-	deleteAllEtcdKeys()
+	framework.DeleteAllEtcdKeys()
 
 	// Set up a master
-	helper, err := master.NewEtcdHelper(newEtcdClient(), testapi.Version(), etcdtest.PathPrefix())
+	helper, err := framework.NewHelper()
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
@@ -581,12 +578,12 @@ func (allowAliceAuthorizer) Authorize(a authorizer.Attributes) error {
 // the authentication system and authorized to do any actions.
 func TestAliceNotForbiddenOrUnauthorized(t *testing.T) {
 
-	deleteAllEtcdKeys()
+	framework.DeleteAllEtcdKeys()
 
 	// This file has alice and bob in it.
 
 	// Set up a master
-	helper, err := master.NewEtcdHelper(newEtcdClient(), testapi.Version(), etcdtest.PathPrefix())
+	helper, err := framework.NewHelper()
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
@@ -670,12 +667,10 @@ func TestAliceNotForbiddenOrUnauthorized(t *testing.T) {
 // the authentication system but not authorized to do any actions
 // should receive "Forbidden".
 func TestBobIsForbidden(t *testing.T) {
-	deleteAllEtcdKeys()
+	framework.DeleteAllEtcdKeys()
 
 	// This file has alice and bob in it.
-
-	// Set up a master
-	helper, err := master.NewEtcdHelper(newEtcdClient(), testapi.Version(), etcdtest.PathPrefix())
+	helper, err := framework.NewHelper()
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
@@ -731,12 +726,12 @@ func TestBobIsForbidden(t *testing.T) {
 // An authorization module is installed in this scenario for integration
 // test purposes, but requests aren't expected to reach it.
 func TestUnknownUserIsUnauthorized(t *testing.T) {
-	deleteAllEtcdKeys()
+	framework.DeleteAllEtcdKeys()
 
 	// This file has alice and bob in it.
 
 	// Set up a master
-	helper, err := master.NewEtcdHelper(newEtcdClient(), testapi.Version(), etcdtest.PathPrefix())
+	helper, err := framework.NewHelper()
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
@@ -810,10 +805,10 @@ func newAuthorizerWithContents(t *testing.T, contents string) authorizer.Authori
 // TestNamespaceAuthorization tests that authorization can be controlled
 // by namespace.
 func TestNamespaceAuthorization(t *testing.T) {
-	deleteAllEtcdKeys()
+	framework.DeleteAllEtcdKeys()
 
 	// This file has alice and bob in it.
-	helper, err := master.NewEtcdHelper(newEtcdClient(), testapi.Version(), etcdtest.PathPrefix())
+	helper, err := framework.NewHelper()
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
@@ -923,12 +918,12 @@ func TestNamespaceAuthorization(t *testing.T) {
 // TestKindAuthorization tests that authorization can be controlled
 // by namespace.
 func TestKindAuthorization(t *testing.T) {
-	deleteAllEtcdKeys()
+	framework.DeleteAllEtcdKeys()
 
 	// This file has alice and bob in it.
 
 	// Set up a master
-	helper, err := master.NewEtcdHelper(newEtcdClient(), testapi.Version(), etcdtest.PathPrefix())
+	helper, err := framework.NewHelper()
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
@@ -1026,12 +1021,12 @@ func TestKindAuthorization(t *testing.T) {
 // TestReadOnlyAuthorization tests that authorization can be controlled
 // by namespace.
 func TestReadOnlyAuthorization(t *testing.T) {
-	deleteAllEtcdKeys()
+	framework.DeleteAllEtcdKeys()
 
 	// This file has alice and bob in it.
 
 	// Set up a master
-	helper, err := master.NewEtcdHelper(newEtcdClient(), testapi.Version(), etcdtest.PathPrefix())
+	helper, err := framework.NewHelper()
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}

test/integration/benchmark-controller.json

+{
+  "kind": "ReplicationController",
+  "apiVersion": "v1beta3",
+  "metadata": {
+     "name": "test-controller",
+     "namespace": "test",
+     "labels": {"name": "test-controller"}
+  },
+  "spec": {
+    "replicas": 0,
+    "selector": {"name": "test-pod"},
+    "template": {
+       "metadata": {
+          "namespace": "test",
+          "labels": {"name": "test-pod"}
+       },
+       "spec": {
+           "containers": [{
+             "name": "test-container",
+             "image": "kubernetes/pause"
+           }]
+       }
+    }
+  }
+}

test/integration/client_test.go

@@ -34,18 +34,15 @@ import (
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/version"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/watch"
+	"github.com/GoogleCloudPlatform/kubernetes/test/integration/framework"
 )
 
-func init() {
-	requireEtcd()
-}
-
 func TestClient(t *testing.T) {
-	_, s := runAMaster(t)
+	_, s := framework.RunAMaster(t)
 	defer s.Close()
 
 	ns := api.NamespaceDefault
-	deleteAllEtcdKeys()
+	framework.DeleteAllEtcdKeys()
 	client := client.NewOrDie(&client.Config{Host: s.URL, Version: testapi.Version()})
 
 	info, err := client.ServerVersion()
@@ -194,9 +191,9 @@ func TestMultiWatch(t *testing.T) {
 	const watcherCount = 50
 	runtime.GOMAXPROCS(watcherCount)
 
-	deleteAllEtcdKeys()
-	defer deleteAllEtcdKeys()
-	_, s := runAMaster(t)
+	framework.DeleteAllEtcdKeys()
+	defer framework.DeleteAllEtcdKeys()
+	_, s := framework.RunAMaster(t)
 	defer s.Close()
 
 	ns := api.NamespaceDefault

test/integration/etcd_tools_test.go

@@ -28,12 +28,9 @@ import (
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/tools"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/tools/etcdtest"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/watch"
+	"github.com/GoogleCloudPlatform/kubernetes/test/integration/framework"
 )
 
-func init() {
-	requireEtcd()
-}
-
 type stringCodec struct{}
 
 type fakeAPIObject string
@@ -56,9 +53,9 @@ func (c stringCodec) DecodeInto(data []byte, obj runtime.Object) error {
 }
 
 func TestSetObj(t *testing.T) {
-	client := newEtcdClient()
+	client := framework.NewEtcdClient()
 	helper := tools.EtcdHelper{Client: client, Codec: stringCodec{}}
-	withEtcdKey(func(key string) {
+	framework.WithEtcdKey(func(key string) {
 		fakeObject := fakeAPIObject("object")
 		if err := helper.SetObj(key, &fakeObject, nil, 0); err != nil {
 			t.Fatalf("unexpected error: %v", err)
@@ -74,9 +71,9 @@ func TestSetObj(t *testing.T) {
 }
 
 func TestExtractObj(t *testing.T) {
-	client := newEtcdClient()
+	client := framework.NewEtcdClient()
 	helper := tools.EtcdHelper{Client: client, Codec: stringCodec{}}
-	withEtcdKey(func(key string) {
+	framework.WithEtcdKey(func(key string) {
 		_, err := client.Set(key, "object", 0)
 		if err != nil {
 			t.Fatalf("unexpected error: %v", err)
@@ -92,9 +89,9 @@ func TestExtractObj(t *testing.T) {
 }
 
 func TestWatch(t *testing.T) {
-	client := newEtcdClient()
+	client := framework.NewEtcdClient()
 	helper := tools.NewEtcdHelper(client, testapi.Codec(), etcdtest.PathPrefix())
-	withEtcdKey(func(key string) {
+	framework.WithEtcdKey(func(key string) {
 		key = etcdtest.AddPrefix(key)
 		resp, err := client.Set(key, runtime.EncodeOrDie(testapi.Codec(), &api.Pod{ObjectMeta: api.ObjectMeta{Name: "foo"}}), 0)
 		if err != nil {

test/integration/framework/etcd_utils.go

+/*
+Copyright 2014 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// +build integration
+
+package framework
+
+import (
+	"fmt"
+	"math/rand"
+
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/testapi"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/master"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/tools"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/tools/etcdtest"
+	"github.com/coreos/go-etcd/etcd"
+	"github.com/golang/glog"
+)
+
+// If you need to start an etcd instance by hand, you also need to insert a key
+// for this check to pass (*any* key will do, eg:
+//curl -L http://127.0.0.1:4001/v2/keys/message -XPUT -d value="Hello world").
+func init() {
+	RequireEtcd()
+}
+
+func NewEtcdClient() *etcd.Client {
+	return etcd.NewClient([]string{})
+}
+
+func NewHelper() (tools.EtcdHelper, error) {
+	return master.NewEtcdHelper(NewEtcdClient(), testapi.Version(), etcdtest.PathPrefix())
+}
+
+func RequireEtcd() {
+	if _, err := NewEtcdClient().Get("/", false, false); err != nil {
+		glog.Fatalf("unable to connect to etcd for testing: %v", err)
+	}
+}
+
+func WithEtcdKey(f func(string)) {
+	prefix := fmt.Sprintf("/test-%d", rand.Int63())
+	defer NewEtcdClient().Delete(prefix, true)
+	f(prefix)
+}
+
+// DeleteAllEtcdKeys deletes all keys from etcd.
+// TODO: Instead of sprinkling calls to this throughout the code, adjust the
+// prefix in etcdtest package; then just delete everything once at the end
+// of the test run.
+func DeleteAllEtcdKeys() {
+	glog.Infof("Deleting all etcd keys")
+	client := NewEtcdClient()
+	keys, err := client.Get("/", false, false)
+	if err != nil {
+		glog.Fatalf("Unable to list root etcd keys: %v", err)
+	}
+	for _, node := range keys.Node.Nodes {
+		if _, err := client.Delete(node.Key, true); err != nil {
+			glog.Fatalf("Unable delete key: %v", err)
+		}
+	}
+
+}

test/integration/master_benchmark_test.go

+// +build benchmark,!no-etcd,!integration
+
+/*
+Copyright 2014 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package integration
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/fields"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"
+	etcdgeneric "github.com/GoogleCloudPlatform/kubernetes/pkg/registry/generic/etcd"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/tools/etcdtest"
+	"github.com/GoogleCloudPlatform/kubernetes/test/integration/framework"
+	"github.com/golang/glog"
+)
+
+// Command line flag globals, parsed in init and used by the benchmarks:
+//	* pods && !tasks: Start -pods, scale number of parallel tasks with b.N
+//  * !pods && tasks: Start -tasks, scale pods with b.N
+//  * pods && tasks: Ignore b.N, benchmark behaves like a test constrained by -benchtime.
+//  * !pods && !tasks: scale pods and workers with b.N.
+// -workers specifies the number of workers to shard tasks across.
+// Eg: go test bench . -bench-pods 3000 -bench-tasks 100 -bench-tasks 10:
+// Create 100 tasks each listing 3000 pods, and run them 10 at a time.
+var (
+	Workers int
+	Pods    int
+	Tasks   int
+)
+
+const Glog_fatalf = 3
+
+func init() {
+	q := flag.Int("bench-quiet", 3, "Quietness, don't glog severities <= given level during the benchmark.")
+	pods := flag.Int("bench-pods", -1, "Number of pods for the benchmark. If unspecified, uses b.N.")
+	workers := flag.Int("bench-workers", -1, "Number workers for the benchmark. If unspecified, uses tasks.")
+	tasks := flag.Int("bench-tasks", -1, "Number of tasks for the benchmark. These tasks are sharded across the workers. If unspecified, uses b.N.")
+	flag.Parse()
+
+	// Unfortunately this v level goes in the opposite direction as stderrthreshold.
+	flag.Set("v", fmt.Sprintf("%d", *q))
+
+	// We need quiet logs to parse benchmark results, which includes Errorf.
+	flag.Set("logtostderr", "false")
+	flag.Set("stderrthreshold", fmt.Sprintf("%d", Glog_fatalf-*q))
+	Pods = *pods
+	Workers = *workers
+	Tasks = *tasks
+	framework.DeleteAllEtcdKeys()
+}
+
+// getPods returns the cmd line -pods or b.N if -pods wasn't specified.
+// Benchmarks can call getPods to get the number of pods they need to
+// create for a given benchmark.
+func getPods(bN int) int {
+	if Pods < 0 {
+		return bN
+	}
+	return Pods
+}
+
+// getTasks returns the cmd line -workers or b.N if -workers wasn't specified.
+// Benchmarks would call getTasks to get the number of workers required to
+// perform the benchmark in parallel.
+func getTasks(bN int) int {
+	if Tasks < 0 {
+		return bN
+	}
+	return Tasks
+}
+
+// getIterations returns the number of iterations required by each benchmark for
+// go to produce reliable timing results.
+func getIterations(bN int) int {
+	// Anything with constant pods is only linear if we iterate b.N times.
+	if Pods > 0 {
+		return bN
+	}
+	return 1
+}
+
+// startPodsOnNodes creates numPods sharded across numNodes
+func startPodsOnNodes(numPods, numNodes int, restClient *client.Client) {
+	podsPerNode := numPods / numNodes
+	if podsPerNode < 1 {
+		podsPerNode = 1
+	}
+	framework.RunParallel(func(id int) error {
+		return framework.StartPods(podsPerNode, fmt.Sprintf("host.%d", id), restClient)
+	}, numNodes, -1)
+}
+
+// Benchmark pod listing by waiting on `Tasks` listers to list `Pods` pods via `Workers`.
+func BenchmarkPodList(b *testing.B) {
+	b.StopTimer()
+	m := framework.NewMasterComponents(&framework.Config{nil, true, false, 250.0, 500})
+	defer m.Stop(true, true)
+
+	numPods, numTasks, iter := getPods(b.N), getTasks(b.N), getIterations(b.N)
+	podsPerNode := numPods / numTasks
+	if podsPerNode < 1 {
+		podsPerNode = 1
+	}
+	glog.Infof("Starting benchmark: b.N %d, pods %d, workers %d, podsPerNode %d",
+		b.N, numPods, numTasks, podsPerNode)
+
+	startPodsOnNodes(numPods, numTasks, m.RestClient)
+	// Stop the rc manager so it doesn't steal resources
+	m.Stop(false, true)
+
+	b.StartTimer()
+	for i := 0; i < iter; i++ {
+		framework.RunParallel(func(id int) error {
+			host := fmt.Sprintf("host.%d", id)
+			now := time.Now()
+			defer func() {
+				glog.V(3).Infof("Worker %d: Node %v listing pods took %v", id, host, time.Since(now))
+			}()
+			if pods, err := m.RestClient.Pods(framework.TestNS).List(
+				labels.Everything(),
+				fields.OneTermEqualSelector(client.PodHost, host)); err != nil {
+				return err
+			} else if len(pods.Items) < podsPerNode {
+				glog.Fatalf("List retrieved %d pods, which is less than %d", len(pods.Items), podsPerNode)
+			}
+			return nil
+		}, numTasks, Workers)
+	}
+	b.StopTimer()
+}
+
+// Benchmark pod listing by waiting on `Tasks` listers to list `Pods` pods via `Workers`.
+func BenchmarkPodListEtcd(b *testing.B) {
+	b.StopTimer()
+	m := framework.NewMasterComponents(&framework.Config{nil, true, false, 250.0, 500})
+	defer m.Stop(true, true)
+
+	numPods, numTasks, iter := getPods(b.N), getTasks(b.N), getIterations(b.N)
+	podsPerNode := numPods / numTasks
+	if podsPerNode < 1 {
+		podsPerNode = 1
+	}
+
+	startPodsOnNodes(numPods, numTasks, m.RestClient)
+	// Stop the rc manager so it doesn't steal resources
+	m.Stop(false, true)
+
+	glog.Infof("Starting benchmark: b.N %d, pods %d, workers %d, podsPerNode %d",
+		b.N, numPods, numTasks, podsPerNode)
+
+	ctx := api.WithNamespace(api.NewContext(), framework.TestNS)
+	key := etcdgeneric.NamespaceKeyRootFunc(ctx, fmt.Sprintf("%s/pods", etcdtest.PathPrefix()))
+
+	b.StartTimer()
+	for i := 0; i < iter; i++ {
+		framework.RunParallel(func(id int) error {
+			now := time.Now()
+			defer func() {
+				glog.V(3).Infof("Worker %d: listing pods took %v", id, time.Since(now))
+			}()
+			if response, err := m.EtcdHelper.Client.Get(key, true, true); err != nil {
+				return err
+			} else if len(response.Node.Nodes) < podsPerNode {
+				glog.Fatalf("List retrieved %d pods, which is less than %d", len(response.Node.Nodes), podsPerNode)
+			}
+			return nil
+		}, numTasks, Workers)
+	}
+	b.StopTimer()
+}
+
+func TestMain(m *testing.M) {
+	os.Exit(m.Run())
+}

test/integration/metrics_test.go

@@ -30,6 +30,7 @@ import (
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/fields"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"
+	"github.com/GoogleCloudPlatform/kubernetes/test/integration/framework"
 
 	"github.com/golang/glog"
 	"github.com/golang/protobuf/proto"
@@ -38,10 +39,6 @@ import (
 
 const scrapeRequestHeader = "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=compact-text"
 
-func init() {
-	requireEtcd()
-}
-
 func scrapeMetrics(s *httptest.Server) ([]*prometheuspb.MetricFamily, error) {
 	req, err := http.NewRequest("GET", s.URL+"/metrics", nil)
 	if err != nil {
@@ -90,7 +87,7 @@ func checkForExpectedMetrics(t *testing.T, metrics []*prometheuspb.MetricFamily,
 }
 
 func TestMasterProcessMetrics(t *testing.T) {
-	_, s := runAMaster(t)
+	_, s := framework.RunAMaster(t)
 	defer s.Close()
 
 	metrics, err := scrapeMetrics(s)
@@ -107,7 +104,7 @@ func TestMasterProcessMetrics(t *testing.T) {
 }
 
 func TestApiserverMetrics(t *testing.T) {
-	_, s := runAMaster(t)
+	_, s := framework.RunAMaster(t)
 	defer s.Close()
 
 	// Make a request to the apiserver to ensure there's at least one data point

test/integration/scheduler_test.go

@@ -36,19 +36,15 @@ import (
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/client/cache"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/client/record"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/master"
-	"github.com/GoogleCloudPlatform/kubernetes/pkg/tools/etcdtest"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/util/wait"
 	"github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/admission/admit"
 	"github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/scheduler"
 	_ "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/scheduler/algorithmprovider"
 	"github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/scheduler/factory"
+	"github.com/GoogleCloudPlatform/kubernetes/test/integration/framework"
 )
 
-func init() {
-	requireEtcd()
-}
-
 type nodeMutationFunc func(t *testing.T, n *api.Node, nodeStore cache.Store, c *client.Client)
 
 type nodeStateManager struct {
@@ -57,11 +53,11 @@ type nodeStateManager struct {
 }
 
 func TestUnschedulableNodes(t *testing.T) {
-	helper, err := master.NewEtcdHelper(newEtcdClient(), testapi.Version(), etcdtest.PathPrefix())
+	helper, err := framework.NewHelper()
 	if err != nil {
 		t.Fatalf("Couldn't create etcd helper: %v", err)
 	}
-	deleteAllEtcdKeys()
+	framework.DeleteAllEtcdKeys()
 
 	var m *master.Master
 	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {

test/integration/secret_test.go

@@ -30,14 +30,10 @@ import (
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/apiserver"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/master"
-	"github.com/GoogleCloudPlatform/kubernetes/pkg/tools/etcdtest"
 	"github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/admission/admit"
+	"github.com/GoogleCloudPlatform/kubernetes/test/integration/framework"
 )
 
-func init() {
-	requireEtcd()
-}
-
 func deletePodOrErrorf(t *testing.T, c *client.Client, ns, name string) {
 	if err := c.Pods(ns).Delete(name, nil); err != nil {
 		t.Errorf("unable to delete pod %v: %v", name, err)
@@ -51,7 +47,7 @@ func deleteSecretOrErrorf(t *testing.T, c *client.Client, ns, name string) {
 
 // TestSecrets tests apiserver-side behavior of creation of secret objects and their use by pods.
 func TestSecrets(t *testing.T) {
-	helper, err := master.NewEtcdHelper(newEtcdClient(), testapi.Version(), etcdtest.PathPrefix())
+	helper, err := framework.NewHelper()
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
@@ -74,7 +70,7 @@ func TestSecrets(t *testing.T) {
 		AdmissionControl:      admit.NewAlwaysAdmit(),
 	})
 
-	deleteAllEtcdKeys()
+	framework.DeleteAllEtcdKeys()
 	client := client.NewOrDie(&client.Config{Host: s.URL, Version: testapi.Version()})
 	DoTestSecrets(t, client, testapi.Version())
 }