Test runner harness for node e2e tests

hack/test-go.sh

@@ -37,6 +37,7 @@ kube::test::find_dirs() {
           -o -path './release/*' \
           -o -path './target/*' \
           -o -path './test/e2e/*' \
+          -o -path './test/e2e_node/*' \
           -o -path './test/integration/*' \
         \) -prune \
       \) -name '*_test.go' -print0 | xargs -0n1 dirname | sed 's|^\./||' | sort -u

hack/verify-flags/known-flags.txt

@@ -13,6 +13,7 @@ allow-privileged
 api-burst
 api-prefix
 api-rate
+api-server-host
 api-server-port
 api-servers
 api-token
@@ -143,6 +144,7 @@ kubelet-certificate-authority
 kubelet-client-certificate
 kubelet-client-key
 kubelet-docker-endpoint
+kubelet-host
 kubelet-host-network-sources
 kubelet-https
 kubelet-network-plugin
@@ -154,6 +156,7 @@ kubelet-sync-frequency
 kubelet-timeout
 kube-master
 kubernetes-service-node-port
+k8s-build-output
 label-columns
 last-release-pr
 leave-stdin-open

test/e2e_node/doc.go

+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e_node

test/e2e_node/e2e_node_suite_test.go

+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e_node
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"flag"
+	"testing"
+)
+
+var kubeletHost = flag.String("kubelet-host", "localhost", "Host address of the kubelet")
+var kubeletPort = flag.Int("kubelet-port", 10250, "Kubelet port")
+
+var apiServerHost = flag.String("api-server-host", "localhost", "Host address of the api server")
+var apiServerPort = flag.Int("api-server-port", 8080, "Api server port")
+
+func TestE2eNode(t *testing.T) {
+	flag.Parse()
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "E2eNode Suite")
+}
+
+// Setup the kubelet on the node
+var _ = BeforeSuite(func() {
+})
+
+// Tear down the kubelet on the node
+var _ = AfterSuite(func() {
+})

test/e2e_node/gcloud/gcloud.go

+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gcloud
+
+import (
+	"errors"
+	"fmt"
+	"math/rand"
+	"os/exec"
+
+	"net"
+	"net/http"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/golang/glog"
+)
+
+var freePortRegexp = regexp.MustCompile(".+:([0-9]+)")
+
+type TearDown func()
+
+type GCloudClient interface {
+	CopyAndWaitTillHealthy(sudo bool, remotePort string, timeout time.Duration, healthUrl string, bin string, args ...string) (*CmdHandle, error)
+}
+
+type gCloudClientImpl struct {
+	host string
+	zone string
+}
+
+type RunResult struct {
+	out []byte
+	err error
+	cmd string
+}
+
+type CmdHandle struct {
+	TearDown TearDown
+	Output   chan RunResult
+	LPort    string
+}
+
+func NewGCloudClient(host string, zone string) GCloudClient {
+	return &gCloudClientImpl{host, zone}
+}
+
+func (gc *gCloudClientImpl) Command(cmd string, moreargs ...string) ([]byte, error) {
+	args := append([]string{"compute", "ssh"})
+	if gc.zone != "" {
+		args = append(args, "--zone", gc.zone)
+	}
+	args = append(args, gc.host, "--", cmd)
+	args = append(args, moreargs...)
+	glog.V(2).Infof("Command gcloud %s", strings.Join(args, " "))
+	return exec.Command("gcloud", args...).CombinedOutput()
+}
+
+func (gc *gCloudClientImpl) TunnelCommand(sudo bool, lPort string, rPort string, cmd string, moreargs ...string) ([]byte, error) {
+	tunnelStr := fmt.Sprintf("-L %s:localhost:%s", lPort, rPort)
+	args := []string{"compute", "ssh"}
+	if gc.zone != "" {
+		args = append(args, "--zone", gc.zone)
+	}
+	args = append(args, "--ssh-flag", tunnelStr, gc.host, "--")
+	if sudo {
+		args = append(args, "sudo")
+	}
+	args = append(args, cmd)
+	args = append(args, moreargs...)
+	glog.V(2).Infof("Command gcloud %s", strings.Join(args, " "))
+	return exec.Command("gcloud", args...).CombinedOutput()
+}
+
+func (gc *gCloudClientImpl) CopyToHost(from string, to string) ([]byte, error) {
+	rto := fmt.Sprintf("%s:%s", gc.host, to)
+	args := []string{"compute", "copy-files"}
+	if gc.zone != "" {
+		args = append(args, "--zone", gc.zone)
+	}
+	args = append(args, from, rto)
+	glog.V(2).Infof("Command gcloud %s", strings.Join(args, " "))
+	return exec.Command("gcloud", args...).CombinedOutput()
+}
+
+func (gc *gCloudClientImpl) CopyAndRun(sudo bool, remotePort string, bin string, args ...string) *CmdHandle {
+	h := &CmdHandle{}
+	h.Output = make(chan RunResult)
+
+	rand.Seed(time.Now().UnixNano())
+
+	// Define where we will copy the temp binary
+	tDir := fmt.Sprintf("/tmp/gcloud-e2e-%d", rand.Int31())
+	_, f := filepath.Split(bin)
+	cmd := filepath.Join(tDir, f)
+	h.LPort = getLocalPort()
+
+	h.TearDown = func() {
+		out, err := gc.Command("sudo", "pkill", f)
+		if err != nil {
+			h.Output <- RunResult{out, err, fmt.Sprintf("pkill %s", cmd)}
+			return
+		}
+		out, err = gc.Command("rm", "-rf", tDir)
+		if err != nil {
+			h.Output <- RunResult{out, err, fmt.Sprintf("rm -rf %s", tDir)}
+			return
+		}
+	}
+
+	// Create the tmp directory
+	out, err := gc.Command("mkdir", "-p", tDir)
+	if err != nil {
+		glog.Errorf("mkdir failed %v", err)
+		h.Output <- RunResult{out, err, fmt.Sprintf("mkdir -p %s", tDir)}
+		return h
+	}
+
+	// Copy the binary
+	out, err = gc.CopyToHost(bin, tDir)
+	if err != nil {
+		glog.Errorf("copy-files failed %v", err)
+		h.Output <- RunResult{out, err, fmt.Sprintf("copy-files %s %s", bin, tDir)}
+		return h
+	}
+
+	// Do the setup
+	go func() {
+		// Start the process
+		out, err = gc.TunnelCommand(sudo, h.LPort, remotePort, cmd, args...)
+		if err != nil {
+			glog.Errorf("command failed %v", err)
+			h.Output <- RunResult{out, err, fmt.Sprintf("%s %s", cmd, strings.Join(args, " "))}
+			return
+		}
+	}()
+	return h
+}
+
+func (gc *gCloudClientImpl) CopyAndWaitTillHealthy(
+	sudo bool,
+	remotePort string, timeout time.Duration, healthUrl string, bin string, args ...string) (*CmdHandle, error) {
+	h := gc.CopyAndRun(sudo, remotePort, bin, args...)
+	eTime := time.Now().Add(timeout)
+	done := false
+	for eTime.After(time.Now()) && !done {
+		select {
+		case r := <-h.Output:
+			glog.V(2).Infof("Error running %s Output:\n%s Error:\n%v", r.cmd, r.out, r.err)
+			return h, r.err
+		case <-time.After(2 * time.Second):
+			resp, err := http.Get(fmt.Sprintf("http://localhost:%s/%s", h.LPort, healthUrl))
+			if err == nil && resp.StatusCode == http.StatusOK {
+				done = true
+				break
+			}
+		}
+	}
+	if !done {
+		return h, errors.New(fmt.Sprintf("Timeout waiting for service to be healthy at http://localhost:%s/%s", h.LPort, healthUrl))
+	}
+	glog.Info("Healthz Success")
+	return h, nil
+}
+
+// GetLocalPort returns a free local port that can be used for ssh tunneling
+func getLocalPort() string {
+	l, _ := net.Listen("tcp", ":0")
+	defer l.Close()
+	return freePortRegexp.FindStringSubmatch(l.Addr().String())[1]
+}

test/e2e_node/kubelet_test.go

+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e_node
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+
+	"github.com/golang/glog"
+	. "github.com/onsi/ginkgo"
+)
+
+var _ = Describe("Kubelet", func() {
+	BeforeEach(func() {
+		// Setup the client to talk to the kubelet
+	})
+
+	Describe("checking kubelet status", func() {
+		Context("when retrieving the node status", func() {
+			It("should have the container version", func() {
+
+				// TODO: This is just a place holder, write a real test here
+				resp, err := http.Get(fmt.Sprintf("http://%s:%d/api/v2.0/attributes", *kubeletHost, *kubeletPort))
+				if err != nil {
+					glog.Errorf("Error: %v", err)
+					return
+				}
+				defer resp.Body.Close()
+				body, err := ioutil.ReadAll(resp.Body)
+				if err != nil {
+					glog.Errorf("Error: %v", err)
+					return
+				}
+				glog.Infof("Resp: %s", body)
+			})
+		})
+	})
+})

test/e2e_node/runner/run_e2e.go

+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"bufio"
+	"flag"
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+	"sync"
+	"time"
+
+	"runtime"
+
+	"github.com/golang/glog"
+	"k8s.io/kubernetes/test/e2e_node/gcloud"
+	"path/filepath"
+)
+
+type RunFunc func(host string, port string) ([]byte, error)
+
+type Result struct {
+	host   string
+	output []byte
+	err    error
+}
+
+var u = sync.WaitGroup{}
+var zone = flag.String("zone", "", "gce zone the hosts live in")
+var hosts = flag.String("hosts", "", "hosts to test")
+var wait = flag.Bool("wait", false, "if true, wait for input before running tests")
+var kubeOutputRelPath = flag.String("k8s-build-output", "_output/local/bin/linux/amd64", "Where k8s binary files are written")
+
+var kubeRoot = ""
+
+const buildScriptRelPath = "hack/build-go.sh"
+const ginkoTestRelPath = "test/e2e_node"
+const healthyTimeoutDuration = time.Minute * 3
+
+func main() {
+	flag.Parse()
+	if *hosts == "" {
+		glog.Fatalf("Must specific --hosts flag")
+	}
+
+	// Figure out the kube root
+	_, path, _, _ := runtime.Caller(0)
+	kubeRoot, _ = filepath.Split(path)
+	kubeRoot = strings.Split(kubeRoot, "/test/e2e_node")[0]
+
+	// Build the go code
+	out, err := exec.Command(filepath.Join(kubeRoot, buildScriptRelPath)).CombinedOutput()
+	if err != nil {
+		glog.Fatalf("Failed to build go packages %s: %v", out, err)
+	}
+
+	// Copy kubelet to each host and run test
+	if *wait {
+		u.Add(1)
+	}
+
+	w := sync.WaitGroup{}
+	for _, h := range strings.Split(*hosts, ",") {
+		w.Add(1)
+		go func(host string) {
+			out, err := runTests(host)
+			if err != nil {
+				glog.Infof("Failure Finished Test Suite %s %v", out, err)
+			} else {
+				glog.Infof("Success Finished Test Suite %s", out)
+			}
+			w.Done()
+		}(h)
+	}
+
+	// Maybe wait for user input before running tests
+	if *wait {
+		WaitForUser()
+	}
+
+	// Wait for the tests to finish
+	w.Wait()
+	glog.Infof("All hosts finished")
+}
+
+func WaitForUser() {
+	scanner := bufio.NewScanner(os.Stdin)
+	fmt.Printf("Enter \"y\" to run tests\n")
+	for scanner.Scan() {
+		if strings.ToUpper(scanner.Text()) != "Y\n" {
+			break
+		}
+		fmt.Printf("Enter \"y\" to run tests\n")
+	}
+	u.Done()
+}
+
+func runTests(host string) ([]byte, error) {
+	c := gcloud.NewGCloudClient(host, *zone)
+	// TODO(pwittrock): Come up with something better for bootstrapping the environment.
+	etcdBin := filepath.Join(kubeRoot, "third_party/etcd/etcd")
+	eh, err := c.CopyAndWaitTillHealthy(false, "4001", healthyTimeoutDuration, "v2/keys/", etcdBin)
+	defer func() { eh.TearDown() }()
+	if err != nil {
+		return nil, fmt.Errorf("Host %s failed to run command %v", host, err)
+	}
+
+	apiBin := filepath.Join(kubeRoot, *kubeOutputRelPath, "kube-apiserver")
+	ah, err := c.CopyAndWaitTillHealthy(
+		true, "8080", healthyTimeoutDuration, "healthz", apiBin, "--service-cluster-ip-range",
+		"10.0.0.1/24", "--insecure-bind-address", "0.0.0.0", "--etcd-servers", "http://localhost:4001",
+		"--cluster-name", "kubernetes", "--v", "2", "--kubelet-port", "10250")
+	defer func() { ah.TearDown() }()
+	if err != nil {
+		return nil, fmt.Errorf("Host %s failed to run command %v", host, err)
+	}
+
+	kubeletBin := filepath.Join(kubeRoot, *kubeOutputRelPath, "kubelet")
+	kh, err := c.CopyAndWaitTillHealthy(
+		true, "4194", healthyTimeoutDuration, "healthz", kubeletBin, "--api-servers", "http://localhost:8080",
+		"--logtostderr", "--address", "0.0.0.0", "--port", "10250")
+	defer func() { kh.TearDown() }()
+	if err != nil {
+		return nil, fmt.Errorf("Host %s failed to run command %v", host, err)
+	}
+
+	// Run the tests
+	glog.Infof("Kubelet healthy on host %s", host)
+	glog.Infof("Kubelet host %s tunnel running on port %s", host, ah.LPort)
+	u.Wait()
+	glog.Infof("Running ginkgo tests against host %s", host)
+	ginkoTests := filepath.Join(kubeRoot, ginkoTestRelPath)
+	return exec.Command(
+		"ginkgo", ginkoTests, "--",
+		"--kubelet-host", "localhost", "--kubelet-port", kh.LPort,
+		"--api-server-host", "localhost", "--api-server-port", kh.LPort,
+		"-logtostderr").CombinedOutput()
+}