Refactor node e2e tests

Makefile

@@ -101,6 +101,15 @@ test_e2e:
 	hack/e2e-test.sh
 .PHONY: test_e2e
 
+# Build and run node end-to-end tests.
+#
+# Example:
+#   make test_e2e_node
+test_e2e_node:
+	hack/e2e-node-test.sh
+.PHONY: test_e2e_node
+
+
 # Remove all build artifacts.
 #
 # Example:

hack/e2e-node-test.sh

+#!/bin/bash
+
+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Provided for backwards compatibility
+sudo -v
+ginkgo "$(dirname $0)/../test/e2e_node/" -- --alsologtostderr --v 2 --node-name $(hostname) --build-services=true --start-services=true --stop-services=true
+
+exit $?

hack/lib/golang.sh

@@ -87,6 +87,7 @@ kube::golang::test_targets() {
     examples/k8petstore/web-server/src
     github.com/onsi/ginkgo/ginkgo
     test/e2e/e2e.test
+    test/e2e_node/e2e_node.test
   )
   if [ -n "${KUBERNETES_CONTRIB:-}" ]; then
     for contrib in "${KUBERNETES_CONTRIB}"; do

hack/verify-flags/known-flags.txt

@@ -28,6 +28,7 @@ bench-workers
 bind-address
 bind-pods-burst
 bind-pods-qps
+build-services
 cadvisor-port
 cert-dir
 certificate-authority
@@ -161,6 +162,7 @@ ir-user
 jenkins-host
 jenkins-jobs
 k8s-build-output
+k8s-bin-dir
 keep-gogoproto
 km-path
 kube-api-burst
@@ -322,6 +324,7 @@ scheduler-name
 schema-cache-dir
 secure-port
 serialize-image-pulls
+server-start-timeout
 service-account-key-file
 service-account-lookup
 service-account-private-key-file
@@ -343,10 +346,14 @@ skip-generated-rewrite
 skip-munges
 sort-by
 source-file
+ssh-env
 ssh-keyfile
+ssh-options
 ssh-user
+start-services
 static-pods-config
 stats-port
+stop-services
 storage-version
 storage-versions
 streaming-connection-idle-timeout

test/e2e_node/doc.go

@@ -14,4 +14,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+// e2e_node contains e2e tests specific to the node
+// TODO: rename this package e2e-node
 package e2e_node

test/e2e_node/e2e_build.go

+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e_node
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
+
+	"github.com/golang/glog"
+)
+
+var k8sBinDir = flag.String("k8s-bin-dir", "", "Directory containing k8s kubelet and kube-apiserver binaries.")
+
+func buildGo() {
+	glog.Infof("Building k8s binaries...")
+	k8sRoot, err := getK8sRootDir()
+	if err != nil {
+		glog.Fatalf("Failed to locate kubernetes root directory %v.", err)
+	}
+	out, err := exec.Command(filepath.Join(k8sRoot, "hack/build-go.sh")).CombinedOutput()
+	if err != nil {
+		glog.Fatalf("Failed to build go packages %v.  Output:\n%s", err, out)
+	}
+}
+
+func getK8sBin(bin string) (string, error) {
+	// Use commandline specified path
+	if *k8sBinDir != "" {
+		absPath, err := filepath.Abs(*k8sBinDir)
+		if err != nil {
+			return "", err
+		}
+		if _, err := os.Stat(filepath.Join(*k8sBinDir, bin)); err != nil {
+			return "", fmt.Errorf("Could not find kube-apiserver under directory %s.", absPath)
+		}
+		return filepath.Join(absPath, bin), nil
+	}
+
+	path, err := filepath.Abs(filepath.Dir(os.Args[0]))
+	if err != nil {
+		return "", fmt.Errorf("Could not find absolute path of directory containing the tests %s.", filepath.Dir(os.Args[0]))
+	}
+	if _, err := os.Stat(filepath.Join(path, bin)); err == nil {
+		return filepath.Join(path, bin), nil
+	}
+
+	buildOutputDir, err := getK8sBuildOutputDir()
+	if err != nil {
+		return "", err
+	}
+	if _, err := os.Stat(filepath.Join(buildOutputDir, bin)); err == nil {
+		return filepath.Join(buildOutputDir, bin), nil
+	}
+
+	// Give up with error
+	return "", fmt.Errorf("Unable to locate %s.  Can be defined using --k8s-path.", bin)
+}
+
+// TODO: Dedup / merge this with comparable utilities in e2e/util.go
+func getK8sRootDir() (string, error) {
+	// Get the directory of the current executable
+	_, testExec, _, _ := runtime.Caller(0)
+	path := filepath.Dir(testExec)
+
+	// Look for the kubernetes source root directory
+	if strings.Contains(path, "k8s.io/kubernetes") {
+		splitPath := strings.Split(path, "k8s.io/kubernetes")
+		return filepath.Join(splitPath[0], "k8s.io/kubernetes/"), nil
+	}
+
+	return "", fmt.Errorf("Could not find kubernetes source root directory.")
+}
+
+func getK8sBuildOutputDir() (string, error) {
+	k8sRoot, err := getK8sRootDir()
+	if err != nil {
+		return "", err
+	}
+	buildOutputDir := filepath.Join(k8sRoot, "_output/local/go/bin")
+	if _, err := os.Stat(buildOutputDir); err != nil {
+		return "", err
+	}
+	return buildOutputDir, nil
+}
+
+func getK8sNodeTestDir() (string, error) {
+	k8sRoot, err := getK8sRootDir()
+	if err != nil {
+		return "", err
+	}
+	buildOutputDir := filepath.Join(k8sRoot, "test/e2e_node")
+	if _, err := os.Stat(buildOutputDir); err != nil {
+		return "", err
+	}
+	return buildOutputDir, nil
+}
+
+func getKubeletServerBin() string {
+	bin, err := getK8sBin("kubelet")
+	if err != nil {
+		panic(fmt.Sprintf("Could not locate kubelet binary."))
+	}
+	return bin
+}
+
+func getApiServerBin() string {
+	bin, err := getK8sBin("kube-apiserver")
+	if err != nil {
+		panic(fmt.Sprintf("Could not locate kube-apiserver binary."))
+	}
+	return bin
+}

test/e2e_node/e2e_node_suite_test.go

@@ -15,21 +15,31 @@ limitations under the License.
 */
 
 // To run tests in this suite
-// Local: `$ ginkgo -- --logtostderr -v 2`
-// Remote: `$ ginkgo -- --node-name <hostname> --api-server-address=<hostname:api_port> --kubelet-address=<hostname=kubelet_port> --logtostderr -v 2`
+// NOTE: This test suite requires sudo capabilities to run the kubelet and kube-apiserver.
+// $ sudo -v && ginkgo test/e2e_node/ -- --logtostderr --v 2 --node-name `hostname` --start-services
 package e2e_node
 
 import (
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
+	"bytes"
 	"flag"
+	"fmt"
+	"os/exec"
+	"strings"
 	"testing"
+
+	"github.com/golang/glog"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
 )
 
 var kubeletAddress = flag.String("kubelet-address", "http://127.0.0.1:10255", "Host and port of the kubelet")
 var apiServerAddress = flag.String("api-server-address", "http://127.0.0.1:8080", "Host and port of the api server")
-var nodeName = flag.String("node-name", "127.0.0.1", "Name of the node")
+var nodeName = flag.String("node-name", "", "Name of the node")
+var buildServices = flag.Bool("build-services", true, "If true, build local executables")
+var startServices = flag.Bool("start-services", true, "If true, start local node services")
+var stopServices = flag.Bool("stop-services", true, "If true, stop local node services after running tets")
+
+var e2es *e2eService
 
 func TestE2eNode(t *testing.T) {
 	flag.Parse()
@@ -39,8 +49,42 @@ func TestE2eNode(t *testing.T) {
 
 // Setup the kubelet on the node
 var _ = BeforeSuite(func() {
+	if *buildServices {
+		buildGo()
+	}
+	if *nodeName == "" {
+		output, err := exec.Command("hostname").CombinedOutput()
+		if err != nil {
+			glog.Fatal("Could not get node name from hostname %v.  Output:\n%s", err, output)
+		}
+		*nodeName = strings.TrimSpace(fmt.Sprintf("%s", output))
+	}
+
+	if *startServices {
+		e2es = newE2eService()
+		if err := e2es.start(); err != nil {
+			Fail(fmt.Sprintf("Unable to start node services.\n%v", err))
+		}
+		glog.Infof("Node services started.  Running tests...")
+	} else {
+		glog.Infof("Running tests without starting services.")
+	}
 })
 
 // Tear down the kubelet on the node
 var _ = AfterSuite(func() {
+	if e2es != nil && *startServices && *stopServices {
+		glog.Infof("Stopping node services...")
+		e2es.stop()
+		b := &bytes.Buffer{}
+		b.WriteString("-------------------------------------------------------------\n")
+		b.WriteString(fmt.Sprintf("kubelet output:\n%s\n", e2es.kubeletCombinedOut.String()))
+		b.WriteString("-------------------------------------------------------------\n")
+		b.WriteString(fmt.Sprintf("apiserver output:\n%s", e2es.apiServerCombinedOut.String()))
+		b.WriteString("-------------------------------------------------------------\n")
+		b.WriteString(fmt.Sprintf("etcd output:\n%s", e2es.etcdCombinedOut.String()))
+		b.WriteString("-------------------------------------------------------------\n")
+		glog.V(2).Infof(b.String())
+
+	}
 })

test/e2e_node/e2e_remote.go

+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e_node
+
+import (
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"os/exec"
+	"os/user"
+	"path/filepath"
+	"strings"
+
+	"github.com/golang/glog"
+)
+
+var sshOptions = flag.String("ssh-options", "", "Commandline options passed to ssh.")
+var sshEnv = flag.String("ssh-env", "", "Use predefined ssh options for environment.  Options: gce")
+
+var sshOptionsMap map[string]string
+
+const archiveName = "e2e_node_test.tar.gz"
+
+func init() {
+	usr, err := user.Current()
+	if err != nil {
+		glog.Fatal(err)
+	}
+	sshOptionsMap = map[string]string{
+		"gce": fmt.Sprintf("-i %s/.ssh/google_compute_engine -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o CheckHostIP=no -o StrictHostKeyChecking=no", usr.HomeDir),
+	}
+}
+
+// CreateTestArchive builds the local source and creates a tar archive e2e_node_test.tar.gz containing
+// the binaries k8s required for node e2e tests
+func CreateTestArchive() string {
+	// Build the executables
+	buildGo()
+
+	// Build the e2e tests into an executable
+	glog.Infof("Building ginkgo k8s test binaries...")
+	testDir, err := getK8sNodeTestDir()
+	if err != nil {
+		glog.Fatalf("Failed to locate test/e2e_node directory %v.", err)
+	}
+	out, err := exec.Command("ginkgo", "build", testDir).CombinedOutput()
+	if err != nil {
+		glog.Fatalf("Failed to build e2e tests under %s %v.  Output:\n%s", testDir, err, out)
+	}
+	ginkgoTest := filepath.Join(testDir, "e2e_node.test")
+	if _, err := os.Stat(ginkgoTest); err != nil {
+		glog.Fatalf("Failed to locate test binary %s", ginkgoTest)
+	}
+	defer os.Remove(ginkgoTest)
+
+	// Make sure we can find the newly built binaries
+	buildOutputDir, err := getK8sBuildOutputDir()
+	if err != nil {
+		glog.Fatalf("Failed to locate kubernetes build output directory %v", err)
+	}
+	kubelet := filepath.Join(buildOutputDir, "kubelet")
+	if _, err := os.Stat(kubelet); err != nil {
+		glog.Fatalf("Failed to locate binary %s", kubelet)
+	}
+	apiserver := filepath.Join(buildOutputDir, "kube-apiserver")
+	if _, err := os.Stat(apiserver); err != nil {
+		glog.Fatalf("Failed to locate binary %s", apiserver)
+	}
+
+	glog.Infof("Building archive...")
+	tardir, err := ioutil.TempDir("", "node-e2e-archive")
+	if err != nil {
+		glog.Fatalf("Failed to create temporary directory %v.", err)
+	}
+	defer os.RemoveAll(tardir)
+
+	// Copy binaries
+	out, err = exec.Command("cp", ginkgoTest, filepath.Join(tardir, "e2e_node.test")).CombinedOutput()
+	if err != nil {
+		glog.Fatalf("Failed to copy e2e_node.test %v.", err)
+	}
+	out, err = exec.Command("cp", kubelet, filepath.Join(tardir, "kubelet")).CombinedOutput()
+	if err != nil {
+		glog.Fatalf("Failed to copy kubelet %v.", err)
+	}
+	out, err = exec.Command("cp", apiserver, filepath.Join(tardir, "kube-apiserver")).CombinedOutput()
+	if err != nil {
+		glog.Fatalf("Failed to copy kube-apiserver %v.", err)
+	}
+
+	// Build the tar
+	out, err = exec.Command("tar", "-zcvf", archiveName, "-C", tardir, ".").CombinedOutput()
+	if err != nil {
+		glog.Fatalf("Failed to build tar %v.  Output:\n%s", err, out)
+	}
+
+	dir, err := os.Getwd()
+	if err != nil {
+		glog.Fatalf("Failed to get working directory %v.", err)
+	}
+	return filepath.Join(dir, archiveName)
+}
+
+// RunRemote copies the archive file to a /tmp file on host, unpacks it, and runs the e2e_node.test
+func RunRemote(archive string, host string) (string, error) {
+	// Create the temp staging directory
+	tmp := fmt.Sprintf("/tmp/gcloud-e2e-%d", rand.Int31())
+	_, err := runSshCommand("ssh", host, "--", "mkdir", tmp)
+	if err != nil {
+		return "", err
+	}
+	defer func() {
+		output, err := runSshCommand("ssh", host, "--", "rm", "-rf", tmp)
+		if err != nil {
+			glog.Errorf("Failed to cleanup tmp directory %s on host %v.  Output:\n%s", tmp, err, output)
+		}
+	}()
+
+	// Copy the archive to the staging directory
+	_, err = runSshCommand("scp", archive, fmt.Sprintf("%s:%s/", host, tmp))
+	if err != nil {
+		return "", err
+	}
+
+	// Kill any running node processes
+	cmd := getSshCommand(" ; ",
+		"sudo pkill kubelet",
+		"sudo pkill kube-apiserver",
+		"sudo pkill etcd")
+	// No need to log an error if pkill fails since pkill will fail if the commands are not running.
+	// If we are unable to stop existing running k8s processes, we should see messages in the kubelet/apiserver/etcd
+	// logs about failing to bind the required ports.
+	runSshCommand("ssh", host, "--", "sh", "-c", cmd)
+
+	// Extract the archive and run the tests
+	cmd = getSshCommand(" && ",
+		fmt.Sprintf("cd %s", tmp),
+		fmt.Sprintf("tar -xzvf ./%s", archiveName),
+		"./e2e_node.test --logtostderr --v 2 --build-services=false --node-name `hostname`")
+	output, err := runSshCommand("ssh", host, "--", "sh", "-c", cmd)
+	if err != nil {
+		return "", err
+	}
+
+	return output, nil
+}
+
+// getSshCommand handles proper quoting so that multiple commands are executed in the same shell over ssh
+func getSshCommand(sep string, args ...string) string {
+	return fmt.Sprintf("'%s'", strings.Join(args, sep))
+}
+
+// runSshCommand executes the ssh or scp command, adding the flag provided --ssh-options
+func runSshCommand(cmd string, args ...string) (string, error) {
+	if env, found := sshOptionsMap[*sshEnv]; found {
+		args = append(strings.Split(env, " "), args...)
+	}
+	if *sshOptions != "" {
+		args = append(strings.Split(*sshOptions, " "), args...)
+	}
+	output, err := exec.Command(cmd, args...).CombinedOutput()
+	if err != nil {
+		return fmt.Sprintf("%s", output), fmt.Errorf("command %q %q failed with error: %v and output: %q", cmd, args, err, output)
+	}
+	return fmt.Sprintf("%s", output), nil
+}

test/e2e_node/e2e_service.go

+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e_node
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"os/exec"
+	"strings"
+	"time"
+
+	"github.com/golang/glog"
+)
+
+var serverStartTimeout = flag.Duration("server-start-timeout", time.Second*30, "Time to wait for each server to become healthy.")
+
+type e2eService struct {
+	etcdCmd              *exec.Cmd
+	etcdCombinedOut      bytes.Buffer
+	etcdDataDir          string
+	apiServerCmd         *exec.Cmd
+	apiServerCombinedOut bytes.Buffer
+	kubeletCmd           *exec.Cmd
+	kubeletCombinedOut   bytes.Buffer
+}
+
+func newE2eService() *e2eService {
+	return &e2eService{}
+}
+
+func (es *e2eService) start() error {
+	if _, err := getK8sBin("kubelet"); err != nil {
+		return err
+	}
+	if _, err := getK8sBin("kube-apiserver"); err != nil {
+		return err
+	}
+
+	cmd, err := es.startEtcd()
+	if err != nil {
+		return err
+	}
+	es.etcdCmd = cmd
+
+	cmd, err = es.startApiServer()
+	if err != nil {
+		return err
+	}
+	es.apiServerCmd = cmd
+
+	cmd, err = es.startKubeletServer()
+	if err != nil {
+		return err
+	}
+	es.kubeletCmd = cmd
+
+	return nil
+}
+
+func (es *e2eService) stop() {
+	if es.kubeletCmd != nil {
+		err := es.kubeletCmd.Process.Kill()
+		if err != nil {
+			glog.Errorf("Failed to stop kubelet.\n%v", err)
+		}
+	}
+	if es.apiServerCmd != nil {
+		err := es.apiServerCmd.Process.Kill()
+		if err != nil {
+			glog.Errorf("Failed to stop be-apiserver.\n%v", err)
+		}
+	}
+	if es.etcdCmd != nil {
+		err := es.etcdCmd.Process.Kill()
+		if err != nil {
+			glog.Errorf("Failed to stop etcd.\n%v", err)
+		}
+	}
+	if es.etcdDataDir != "" {
+		err := os.RemoveAll(es.etcdDataDir)
+		if err != nil {
+			glog.Errorf("Failed to delete etcd data directory %s.\n%v", es.etcdDataDir, err)
+		}
+	}
+}
+
+func (es *e2eService) startEtcd() (*exec.Cmd, error) {
+	dataDir, err := ioutil.TempDir("", "node-e2e")
+	if err != nil {
+		return nil, err
+	}
+	es.etcdDataDir = dataDir
+	return es.startServer(healthCheckCommand{
+		combinedOut:    &es.etcdCombinedOut,
+		healthCheckUrl: "http://127.0.0.1:4001/v2/keys",
+		command:        "etcd",
+		args:           []string{"--data-dir", dataDir, "--name", "e2e-node"},
+	})
+}
+
+func (es *e2eService) startApiServer() (*exec.Cmd, error) {
+	return es.startServer(
+		healthCheckCommand{
+			combinedOut:    &es.apiServerCombinedOut,
+			healthCheckUrl: "http://127.0.0.1:8080/healthz",
+			command:        "sudo",
+			args: []string{getApiServerBin(),
+				"--v", "2", "--logtostderr", "--log_dir", "./",
+				"--etcd-servers", "http://127.0.0.1:4001",
+				"--insecure-bind-address", "0.0.0.0",
+				"--service-cluster-ip-range", "10.0.0.1/24",
+				"--kubelet-port", "10250"},
+		})
+}
+
+func (es *e2eService) startKubeletServer() (*exec.Cmd, error) {
+	return es.startServer(
+		healthCheckCommand{
+			combinedOut:    &es.kubeletCombinedOut,
+			healthCheckUrl: "http://127.0.0.1:10255/healthz",
+			command:        "sudo",
+			args: []string{getKubeletServerBin(),
+				"--v", "2", "--logtostderr", "--log_dir", "./",
+				"--api-servers", "http://127.0.0.1:8080",
+				"--address", "0.0.0.0",
+				"--port", "10250"},
+		})
+}
+
+func (es *e2eService) startServer(hcc healthCheckCommand) (*exec.Cmd, error) {
+	cmdErrorChan := make(chan error)
+	cmd := exec.Command(hcc.command, hcc.args...)
+	cmd.Stdout = hcc.combinedOut
+	cmd.Stderr = hcc.combinedOut
+	go func() {
+		err := cmd.Run()
+		if err != nil {
+			cmdErrorChan <- fmt.Errorf("%v Exited with status %v.  Output:\n%s", hcc, err, *hcc.combinedOut)
+		}
+		close(cmdErrorChan)
+	}()
+
+	endTime := time.Now().Add(*serverStartTimeout)
+	for endTime.After(time.Now()) {
+		select {
+		case err := <-cmdErrorChan:
+			return nil, err
+		case <-time.After(time.Second):
+			resp, err := http.Get(hcc.healthCheckUrl)
+			if err == nil && resp.StatusCode == http.StatusOK {
+				return cmd, nil
+			}
+		}
+	}
+	return nil, fmt.Errorf("Timeout waiting for service %v", hcc)
+}
+
+type healthCheckCommand struct {
+	healthCheckUrl string
+	command        string
+	args           []string
+	combinedOut    *bytes.Buffer
+}
+
+func (hcc *healthCheckCommand) String() string {
+	return fmt.Sprintf("`%s %s` %s", hcc.command, strings.Join(hcc.args, " "), hcc.healthCheckUrl)
+}

test/e2e_node/environment/conformance.go

@@ -28,8 +28,9 @@ import (
 	"strings"
 
 	"errors"
-	"k8s.io/kubernetes/pkg/kubelet/cadvisor"
 	"os"
+
+	"k8s.io/kubernetes/pkg/kubelet/cadvisor"
 )
 
 const success = "\033[0;32mSUCESS\033[0m"

test/e2e_node/gcloud/gcloud.go

-/*
-Copyright 2015 The Kubernetes Authors All rights reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package gcloud
-
-import (
-	"bytes"
-	"errors"
-	"fmt"
-	"math/rand"
-	"net"
-	"net/http"
-	"os/exec"
-	"path/filepath"
-	"regexp"
-	"strings"
-	"time"
-
-	"github.com/golang/glog"
-)
-
-var freePortRegexp = regexp.MustCompile(".+:([0-9]+)")
-
-type TearDown func() *RunResult
-
-type GCloudClient interface {
-	RunAndWaitTillHealthy(
-		sudo bool, copyBin bool, remotePort string,
-		timeout time.Duration, healthUrl string, bin string, args ...string) (*CmdHandle, error)
-}
-
-type gCloudClientImpl struct {
-	host string
-	zone string
-}
-
-type RunResult struct {
-	out []byte
-	err error
-	cmd string
-}
-
-type CmdHandle struct {
-	TearDown       TearDown
-	CombinedOutput bytes.Buffer
-	Output         chan RunResult
-	LPort          string
-}
-
-func NewGCloudClient(host string, zone string) GCloudClient {
-	return &gCloudClientImpl{host, zone}
-}
-
-func (gc *gCloudClientImpl) Command(cmd string, moreargs ...string) ([]byte, error) {
-	args := append([]string{"compute", "ssh"})
-	if gc.zone != "" {
-		args = append(args, "--zone", gc.zone)
-	}
-	args = append(args, gc.host, "--", cmd)
-	args = append(args, moreargs...)
-	glog.V(2).Infof("Command gcloud %s", strings.Join(args, " "))
-	return exec.Command("gcloud", args...).CombinedOutput()
-}
-
-func (gc *gCloudClientImpl) TunnelCommand(sudo bool, lPort string, rPort string, dir string, cmd string, moreargs ...string) *exec.Cmd {
-	tunnelStr := fmt.Sprintf("-L %s:localhost:%s", lPort, rPort)
-	args := []string{"compute", "ssh"}
-	if gc.zone != "" {
-		args = append(args, "--zone", gc.zone)
-	}
-	args = append(args, "--ssh-flag", tunnelStr, gc.host, "--")
-	args = append(args, "cd", dir, ";")
-	if sudo {
-		args = append(args, "sudo")
-	}
-	args = append(args, cmd)
-	args = append(args, moreargs...)
-	glog.V(2).Infof("Command gcloud %s", strings.Join(args, " "))
-	return exec.Command("gcloud", args...)
-}
-
-func (gc *gCloudClientImpl) CopyToHost(from string, to string) ([]byte, error) {
-	rto := fmt.Sprintf("%s:%s", gc.host, to)
-	args := []string{"compute", "copy-files"}
-	if gc.zone != "" {
-		args = append(args, "--zone", gc.zone)
-	}
-	args = append(args, from, rto)
-	glog.V(2).Infof("Command gcloud %s", strings.Join(args, " "))
-	return exec.Command("gcloud", args...).CombinedOutput()
-}
-
-func (gc *gCloudClientImpl) Run(
-	sudo bool, copyBin bool, remotePort string, bin string, args ...string) *CmdHandle {
-
-	h := &CmdHandle{}
-	h.Output = make(chan RunResult)
-
-	rand.Seed(time.Now().UnixNano())
-
-	// Define where we will copy the temp binary
-	tDir := fmt.Sprintf("/tmp/gcloud-e2e-%d", rand.Int31())
-	_, f := filepath.Split(bin)
-	cmd := f
-	if copyBin {
-		cmd = filepath.Join(tDir, f)
-	}
-	h.LPort = getLocalPort()
-
-	h.TearDown = func() *RunResult {
-		out, err := gc.Command("sudo", "pkill", f)
-		if err != nil {
-			return &RunResult{out, err, fmt.Sprintf("pkill %s", f)}
-		}
-		out, err = gc.Command("rm", "-rf", tDir)
-		if err != nil {
-			return &RunResult{out, err, fmt.Sprintf("rm -rf %s", tDir)}
-		}
-		return &RunResult{}
-	}
-
-	// Run the commands in a Go fn so that this method doesn't block when writing to a channel
-	// to report an error
-	go func() {
-		// Create the tmp directory
-		out, err := gc.Command("mkdir", "-p", tDir)
-
-		// Work around for gcloud flakiness - TODO: debug why gcloud sometimes cannot find credentials for some hosts
-		// If there was an error about credentials, retry making the directory 6 times to see if it can be resolved
-		// This is to help debug if the credential issues are persistent for a given host on a given run, or transient
-		// And if downstream gcloud commands are also impacted
-		for i := 0; i < 6 && err != nil && strings.Contains(string(out), "does not have any valid credentials"); i++ {
-			glog.Warningf("mkdir failed on host %s due to credential issues, retrying in 5 seconds %v %s", gc.host, err, out)
-			time.Sleep(5 * time.Second)
-			out, err = gc.Command("mkdir", "-p", tDir)
-		}
-		if err != nil {
-			glog.Errorf("mkdir failed %v %s", err, out)
-			h.Output <- RunResult{out, err, fmt.Sprintf("mkdir -p %s", tDir)}
-			return
-		}
-
-		// Copy the binary
-		if copyBin {
-			out, err = gc.CopyToHost(bin, tDir)
-			if err != nil {
-				glog.Errorf("copy-files failed %v %s", err, out)
-				h.Output <- RunResult{out, err, fmt.Sprintf("copy-files %s %s", bin, tDir)}
-				return
-			}
-		}
-
-		c := gc.TunnelCommand(sudo, h.LPort, remotePort, tDir, cmd, args...)
-		c.Stdout = &h.CombinedOutput
-		c.Stderr = &h.CombinedOutput
-		go func() {
-			// Start the process
-			err = c.Run()
-			if err != nil {
-				glog.Errorf("command failed %v %s", err, h.CombinedOutput.Bytes())
-				h.Output <- RunResult{h.CombinedOutput.Bytes(), err, fmt.Sprintf("%s %s", cmd, strings.Join(args, " "))}
-				return
-			}
-		}()
-	}()
-	return h
-}
-
-func (gc *gCloudClientImpl) RunAndWaitTillHealthy(
-	sudo bool, copyBin bool,
-	remotePort string, timeout time.Duration, healthUrl string, bin string, args ...string) (*CmdHandle, error) {
-	h := gc.Run(sudo, copyBin, remotePort, bin, args...)
-	eTime := time.Now().Add(timeout)
-	done := false
-	for eTime.After(time.Now()) && !done {
-		select {
-		case r := <-h.Output:
-			glog.V(2).Infof("Error running %s Output:\n%s Error:\n%v", r.cmd, r.out, r.err)
-			return h, r.err
-		case <-time.After(2 * time.Second):
-			resp, err := http.Get(fmt.Sprintf("http://localhost:%s/%s", h.LPort, healthUrl))
-			if err == nil && resp.StatusCode == http.StatusOK {
-				done = true
-				break
-			}
-		}
-	}
-	if !done {
-		return h, errors.New(fmt.Sprintf("Timeout waiting for service to be healthy at http://localhost:%s/%s", h.LPort, healthUrl))
-	}
-	glog.Info("Healthz Success")
-	return h, nil
-}
-
-// GetLocalPort returns a free local port that can be used for ssh tunneling
-func getLocalPort() string {
-	l, _ := net.Listen("tcp", ":0")
-	defer l.Close()
-	return freePortRegexp.FindStringSubmatch(l.Addr().String())[1]
-}

test/e2e_node/kubelet_test.go

@@ -249,7 +249,7 @@ var _ = Describe("Kubelet", func() {
 					Expect(*container.Logs.UsedBytes).NotTo(BeZero(), spew.Sdump(container))
 
 				}
-				Expect(podsList).To(ConsistOf(podNames))
+				Expect(podsList).To(ConsistOf(podNames), spew.Sdump(summary))
 			})
 		})
 

test/e2e_node/runner/run_e2e.go

@@ -19,189 +19,72 @@ limitations under the License.
 package main
 
 import (
-	"bufio"
 	"flag"
 	"fmt"
 	"os"
-	"os/exec"
 	"strings"
-	"sync"
-	"time"
 
-	"runtime"
-
-	"github.com/golang/glog"
-	"k8s.io/kubernetes/test/e2e_node/gcloud"
-	"path/filepath"
+	"k8s.io/kubernetes/test/e2e_node"
 )
 
-type RunFunc func(host string, port string) ([]byte, error)
-
-type Result struct {
-	host   string
-	output []byte
-	err    error
-}
-
-const gray = "\033[1;30m"
-const blue = "\033[0;34m"
-const noColour = "\033[0m"
-
-var u = sync.WaitGroup{}
-var zone = flag.String("zone", "", "gce zone the hosts live in")
 var hosts = flag.String("hosts", "", "hosts to test")
-var wait = flag.Bool("wait", false, "if true, wait for input before running tests")
-var kubeOutputRelPath = flag.String("k8s-build-output", "_output/local/bin/linux/amd64", "Where k8s binary files are written")
-
-var kubeRoot = ""
-
-const buildScriptRelPath = "hack/build-go.sh"
-const ginkgoTestRelPath = "test/e2e_node"
-const healthyTimeoutDuration = time.Minute * 3
 
 func main() {
-	flag.Parse()
-	if *hosts == "" {
-		glog.Fatalf("Must specific --hosts flag")
-	}
-
-	// Figure out the kube root
-	_, path, _, _ := runtime.Caller(0)
-	kubeRoot, _ = filepath.Split(path)
-	kubeRoot = strings.Split(kubeRoot, "/test/e2e_node")[0]
-
-	// Build the go code
-	out, err := exec.Command(filepath.Join(kubeRoot, buildScriptRelPath)).CombinedOutput()
-	if err != nil {
-		glog.Fatalf("Failed to build go packages %s: %v", out, err)
+	// Setup coloring
+	stat, _ := os.Stdout.Stat()
+	useColor := (stat.Mode() & os.ModeCharDevice) != 0
+	blue := ""
+	noColour := ""
+	if useColor {
+		blue = "\033[0;34m"
+		noColour = "\033[0m"
 	}
 
-	// Copy kubelet to each host and run test
-	if *wait {
-		u.Add(1)
+	flag.Parse()
+	if *hosts == "" {
+		fmt.Printf("Must specific --hosts flag")
 	}
+	archive := e2e_node.CreateTestArchive()
+	defer os.Remove(archive)
 
 	results := make(chan *TestResult)
 	hs := strings.Split(*hosts, ",")
 	for _, h := range hs {
-		go func(host string) { results <- runTests(host) }(h)
-	}
-
-	// Maybe wait for user input before running tests
-	if *wait {
-		WaitForUser()
+		fmt.Printf("Starting tests on host %s.", h)
+		go func(host string) {
+			output, err := e2e_node.RunRemote(archive, host)
+			results <- &TestResult{
+				output: output,
+				err:    err,
+				host:   host,
+			}
+		}(h)
 	}
 
 	// Wait for all tests to complete and emit the results
 	errCount := 0
 	for i := 0; i < len(hs); i++ {
 		tr := <-results
-		host := tr.fullhost
+		host := tr.host
+		fmt.Printf("%s================================================================%s\n", blue, noColour)
 		if tr.err != nil {
 			errCount++
-			glog.Infof("%s================================================================%s", blue, noColour)
-			glog.Infof("Failure Finished Host %s Test Suite %s %v", host, tr.testCombinedOutput, tr.err)
-			glog.V(2).Infof("----------------------------------------------------------------")
-			glog.V(5).Infof("Host %s Etcd Logs\n%s%s%s", host, gray, tr.etcdCombinedOutput, noColour)
-			glog.V(5).Infof("----------------------------------------------------------------")
-			glog.V(5).Infof("Host %s Apiserver Logs\n%s%s%s", host, gray, tr.apiServerCombinedOutput, noColour)
-			glog.V(5).Infof("----------------------------------------------------------------")
-			glog.V(2).Infof("Host %s Kubelet Logs\n%s%s%s", host, gray, tr.kubeletCombinedOutput, noColour)
-			glog.Infof("%s================================================================%s", blue, noColour)
+			fmt.Printf("Failure Finished Host %s Test Suite %s %v\n", host, tr.output, tr.err)
 		} else {
-			glog.Infof("================================================================")
-			glog.Infof("Success Finished Host %s Test Suite %s", host, tr.testCombinedOutput)
-			glog.Infof("================================================================")
+			fmt.Printf("Success Finished Host %s Test Suite %s\n", host, tr.output)
 		}
+		fmt.Printf("%s================================================================%s\n", blue, noColour)
 	}
 
 	// Set the exit code if there were failures
 	if errCount > 0 {
-		glog.Errorf("Failure: %d errors encountered.", errCount)
+		fmt.Printf("Failure: %d errors encountered.", errCount)
 		os.Exit(1)
 	}
 }
 
-func WaitForUser() {
-	scanner := bufio.NewScanner(os.Stdin)
-	fmt.Printf("Enter \"y\" to run tests\n")
-	for scanner.Scan() {
-		if strings.ToUpper(scanner.Text()) != "Y\n" {
-			break
-		}
-		fmt.Printf("Enter \"y\" to run tests\n")
-	}
-	u.Done()
-}
-
 type TestResult struct {
-	fullhost                string
-	err                     error
-	testCombinedOutput      string
-	etcdCombinedOutput      string
-	apiServerCombinedOutput string
-	kubeletCombinedOutput   string
-}
-
-func runTests(fullhost string) *TestResult {
-	result := &TestResult{fullhost: fullhost}
-
-	host := strings.Split(fullhost, ".")[0]
-	c := gcloud.NewGCloudClient(host, *zone)
-	// TODO(pwittrock): Come up with something better for bootstrapping the environment.
-	eh, err := c.RunAndWaitTillHealthy(
-		false, false, "4001", healthyTimeoutDuration, "v2/keys/", "etcd", "--data-dir", "./", "--name", "e2e-node")
-	defer func() {
-		eh.TearDown()
-		result.etcdCombinedOutput = fmt.Sprintf("%s", eh.CombinedOutput.Bytes())
-	}()
-	if err != nil {
-		result.err = fmt.Errorf("Host %s failed to run command %v", host, err)
-		return result
-	}
-
-	apiBin := filepath.Join(kubeRoot, *kubeOutputRelPath, "kube-apiserver")
-	ah, err := c.RunAndWaitTillHealthy(
-		true, true, "8080", healthyTimeoutDuration, "healthz", apiBin, "--service-cluster-ip-range",
-		"10.0.0.1/24", "--insecure-bind-address", "0.0.0.0", "--etcd-servers", "http://127.0.0.1:4001",
-		"--v", "2", "--alsologtostderr", "--kubelet-port", "10250")
-	defer func() {
-		ah.TearDown()
-		result.apiServerCombinedOutput = fmt.Sprintf("%s", ah.CombinedOutput.Bytes())
-	}()
-	if err != nil {
-		result.err = fmt.Errorf("Host %s failed to run command %v", host, err)
-		return result
-	}
-
-	kubeletBin := filepath.Join(kubeRoot, *kubeOutputRelPath, "kubelet")
-	// TODO: Used --v 4 or higher and upload to gcs instead of printing to the console
-	// TODO: Copy /var/log/messages and upload to GCS for failed tests
-	kh, err := c.RunAndWaitTillHealthy(
-		true, true, "10255", healthyTimeoutDuration, "healthz", kubeletBin, "--api-servers", "http://127.0.0.1:8080",
-		"--v", "2", "--alsologtostderr", "--address", "0.0.0.0", "--port", "10250")
-	defer func() {
-		kh.TearDown()
-		result.kubeletCombinedOutput = fmt.Sprintf("%s", kh.CombinedOutput.Bytes())
-	}()
-	if err != nil {
-		result.err = fmt.Errorf("Host %s failed to run command %v", host, err)
-	}
-
-	// Run the tests
-	glog.Infof("Kubelet healthy on host %s", host)
-	glog.Infof("Kubelet host %s tunnel running on port %s", host, ah.LPort)
-	u.Wait()
-	glog.Infof("Running ginkgo tests against host %s", host)
-	ginkgoTests := filepath.Join(kubeRoot, ginkgoTestRelPath)
-	out, err := exec.Command(
-		"ginkgo", ginkgoTests, "--",
-		"--kubelet-address", fmt.Sprintf("http://127.0.0.1:%s", kh.LPort),
-		"--api-server-address", fmt.Sprintf("http://127.0.0.1:%s", ah.LPort),
-		"--node-name", fullhost,
-		"--v", "2", "--alsologtostderr").CombinedOutput()
-
-	result.err = err
-	result.testCombinedOutput = fmt.Sprintf("%s", out)
-	return result
+	output string
+	err    error
+	host   string
 }