Implement Go e2e SSH utility and simple test that runs it on all nodes.

test/e2e/ssh.go

+/*
+Copyright 2015 Google Inc. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/fields"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("SSH", func() {
+	BeforeEach(func() {
+		var err error
+		c, err = loadClient()
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	It("should SSH to all nodes and run commands", func() {
+		// When adding more providers here, also implement their functionality
+		// in util.go's getSigner(...).
+		provider := testContext.Provider
+		if !providerIs("gce", "gke") {
+			By(fmt.Sprintf("Skipping SSH test, which is not implemented for %s", provider))
+			return
+		}
+
+		// Get all nodes' external IPs.
+		By("Getting all nodes' SSH-able IP addresses")
+		nodelist, err := c.Nodes().List(labels.Everything(), fields.Everything())
+		if err != nil {
+			Failf("Error getting nodes: %v", err)
+		}
+		hosts := make([]string, 0, len(nodelist.Items))
+		for _, n := range nodelist.Items {
+			for _, addr := range n.Status.Addresses {
+				// Use the first external IP address we find on the node, and
+				// use at most one per node.
+				// NOTE: Until #7412 is fixed this will repeatedly ssh into the
+				// master node and not check any of the minions.
+				if addr.Type == api.NodeExternalIP {
+					hosts = append(hosts, addr.Address+":22")
+					break
+				}
+			}
+		}
+
+		// Fail if any node didn't have an external IP.
+		if len(hosts) != len(nodelist.Items) {
+			Failf("Only found %d external IPs on nodes, but found %d nodes. Nodelist: %v",
+				len(hosts), len(nodelist.Items), nodelist)
+		}
+
+		testCases := []struct {
+			cmd            string
+			checkStdout    bool
+			expectedStdout string
+			expectedStderr string
+			expectedCode   int
+			expectedError  error
+		}{
+			{`echo "Hello"`, true, "Hello", "", 0, nil},
+			// Same as previous, but useful for test output diagnostics.
+			{`echo "Hello from $(whoami)@$(hostname)"`, false, "", "", 0, nil},
+			{`echo "foo" | grep "bar"`, true, "", "", 1, nil},
+			{`echo "Out" && echo "Error" >&2 && exit 7`, true, "Out", "Error", 7, nil},
+		}
+
+		// Run commands on all nodes via SSH.
+		for _, testCase := range testCases {
+			By(fmt.Sprintf("SSH'ing to all nodes and running %s", testCase.cmd))
+			for _, host := range hosts {
+				stdout, stderr, code, err := SSH(testCase.cmd, host, provider)
+				stdout, stderr = strings.TrimSpace(stdout), strings.TrimSpace(stderr)
+				if err != testCase.expectedError {
+					Failf("Ran %s on %s, got error %v, expected %v", testCase.cmd, host, err, testCase.expectedError)
+				}
+				if testCase.checkStdout && stdout != testCase.expectedStdout {
+					Failf("Ran %s on %s, got stdout '%s', expected '%s'", testCase.cmd, host, stdout, testCase.expectedStdout)
+				}
+				if stderr != testCase.expectedStderr {
+					Failf("Ran %s on %s, got stderr '%s', expected '%s'", testCase.cmd, host, stderr, testCase.expectedStderr)
+				}
+				if code != testCase.expectedCode {
+					Failf("Ran %s on %s, got exit code %d, expected %d", testCase.cmd, host, code, testCase.expectedCode)
+				}
+				// Show stdout, stderr for logging purposes.
+				if len(stdout) > 0 {
+					Logf("Got stdout from %s: %s", host, strings.TrimSpace(stdout))
+				}
+				if len(stderr) > 0 {
+					Logf("Got stderr from %s: %s", host, strings.TrimSpace(stderr))
+				}
+			}
+		}
+
+		// Quickly test that SSH itself errors correctly.
+		By("SSH'ing to a nonexistent host")
+		if _, _, _, err = SSH(`echo "hello"`, "i.do.not.exist", provider); err == nil {
+			Failf("Expected error trying to SSH to nonexistent host.")
+		}
+	})
+})

test/e2e/util.go

@@ -19,13 +19,17 @@ package e2e
 import (
 	"bytes"
 	"fmt"
+	"io/ioutil"
 	"math/rand"
+	"os"
 	"os/exec"
 	"path/filepath"
 	"strconv"
 	"strings"
 	"time"
 
+	"code.google.com/p/go-uuid/uuid"
+
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/client/clientcmd"
@@ -34,10 +38,10 @@ import (
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/util/wait"
 
+	"golang.org/x/crypto/ssh"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-
-	"code.google.com/p/go-uuid/uuid"
 )
 
 const (
@@ -626,3 +630,85 @@ func BadEvents(events []*api.Event) int {
 	}
 	return badEvents
 }
+
+// SSH synchronously SSHs to a node running on provider and runs cmd. If there
+// is no error performing the SSH, the stdout, stderr, and exit code are
+// returned.
+func SSH(cmd, host, provider string) (string, string, int, error) {
+	// Get a signer for the provider.
+	signer, err := getSigner(provider)
+	if err != nil {
+		return "", "", 0, fmt.Errorf("error getting signer for provider %s: '%v'", provider, err)
+	}
+
+	// Setup the config, dial the server, and open a session.
+	config := &ssh.ClientConfig{
+		User: os.Getenv("USER"),
+		Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)},
+	}
+	client, err := ssh.Dial("tcp", host, config)
+	if err != nil {
+		return "", "", 0, fmt.Errorf("error getting SSH client to host %s: '%v'", host, err)
+	}
+	session, err := client.NewSession()
+	if err != nil {
+		return "", "", 0, fmt.Errorf("error creating session to host %s: '%v'", host, err)
+	}
+	defer session.Close()
+
+	// Run the command.
+	code := 0
+	var bout, berr bytes.Buffer
+	session.Stdout, session.Stderr = &bout, &berr
+	if err = session.Run(cmd); err != nil {
+		// Check whether the command failed to run or didn't complete.
+		if exiterr, ok := err.(*ssh.ExitError); ok {
+			// If we got an ExitError and the exit code is nonzero, we'll
+			// consider the SSH itself successful (just that the command run
+			// errored on the host).
+			if code = exiterr.ExitStatus(); code != 0 {
+				err = nil
+			}
+		} else {
+			// Some other kind of error happened (e.g. an IOError); consider the
+			// SSH unsuccessful.
+			err = fmt.Errorf("failed running `%s` on %s: '%v'", cmd, host, err)
+		}
+	}
+	return bout.String(), berr.String(), code, err
+}
+
+// getSigner returns an ssh.Signer for the provider ("gce", etc.) that can be
+// used to SSH to their nodes.
+func getSigner(provider string) (ssh.Signer, error) {
+	// Get the directory in which SSH keys are located.
+	keydir := filepath.Join(os.Getenv("HOME"), ".ssh")
+
+	// Select the key itself to use. When implementing more providers here,
+	// please also add them to any SSH tests that are disabled because of signer
+	// support.
+	keyfile := ""
+	switch provider {
+	case "gce", "gke":
+		keyfile = "google_compute_engine"
+	default:
+		return nil, fmt.Errorf("getSigner(...) not implemented for %s", provider)
+	}
+	key := filepath.Join(keydir, keyfile)
+
+	// Create an actual signer.
+	file, err := os.Open(key)
+	if err != nil {
+		return nil, fmt.Errorf("error opening SSH key %s: '%v'", key, err)
+	}
+	defer file.Close()
+	buffer, err := ioutil.ReadAll(file)
+	if err != nil {
+		return nil, fmt.Errorf("error reading SSH key %s: '%v'", key, err)
+	}
+	signer, err := ssh.ParsePrivateKey(buffer)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing SSH key %s: '%v'", key, err)
+	}
+	return signer, nil
+}