Add a simple clusterapi-tester, improve e2e logging.

test/e2e/e2e.go

@@ -187,6 +187,7 @@ var _ = ginkgo.SynchronizedBeforeSuite(func() []byte {
 			dumpAllNamespaceInfo(c, api.NamespaceSystem)
 		}
 		logFailedContainers(api.NamespaceSystem)
+		runKubernetesServiceTestContainer(testContext.RepoRoot, api.NamespaceDefault)
 		Failf("Error waiting for all pods to be running and ready: %v", err)
 	}
 

test/e2e/util.go

@@ -20,6 +20,7 @@ import (
 	"bytes"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"math"
 	"math/rand"
 	"net"
@@ -58,6 +59,7 @@ import (
 	deploymentutil "k8s.io/kubernetes/pkg/util/deployment"
 	"k8s.io/kubernetes/pkg/util/sets"
 	"k8s.io/kubernetes/pkg/util/wait"
+	utilyaml "k8s.io/kubernetes/pkg/util/yaml"
 	"k8s.io/kubernetes/pkg/version"
 	"k8s.io/kubernetes/pkg/watch"
 
@@ -509,6 +511,60 @@ func waitForPodsRunningReady(ns string, minPods int, timeout time.Duration) erro
 	return nil
 }
 
+func podFromManifest(filename string) (*api.Pod, error) {
+	var pod api.Pod
+	Logf("Parsing pod from %v", filename)
+	data, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return nil, err
+	}
+	json, err := utilyaml.ToJSON(data)
+	if err != nil {
+		return nil, err
+	}
+	if err := runtime.DecodeInto(api.Codecs.UniversalDecoder(), json, &pod); err != nil {
+		return nil, err
+	}
+	return &pod, nil
+}
+
+// Run a test container to try and contact the Kubernetes api-server from a pod, wait for it
+// to flip to Ready, log its output and delete it.
+func runKubernetesServiceTestContainer(repoRoot string, ns string) {
+	c, err := loadClient()
+	if err != nil {
+		Logf("Failed to load client")
+		return
+	}
+	path := filepath.Join(repoRoot, "test", "images", "clusterapi-tester", "pod.yaml")
+	p, err := podFromManifest(path)
+	if err != nil {
+		Logf("Failed to parse clusterapi-tester from manifest %v: %v", path, err)
+		return
+	}
+	p.Namespace = ns
+	if _, err := c.Pods(ns).Create(p); err != nil {
+		Logf("Failed to create %v: %v", p.Name, err)
+		return
+	}
+	defer func() {
+		if err := c.Pods(ns).Delete(p.Name, nil); err != nil {
+			Logf("Failed to delete pod %v: %v", p.Name, err)
+		}
+	}()
+	timeout := 5 * time.Minute
+	if err := waitForPodCondition(c, ns, p.Name, "clusterapi-tester", timeout, podRunningReady); err != nil {
+		Logf("Pod %v took longer than %v to enter running/ready: %v", p.Name, timeout, err)
+		return
+	}
+	logs, err := getPodLogs(c, ns, p.Name, p.Spec.Containers[0].Name)
+	if err != nil {
+		Logf("Failed to retrieve logs from %v: %v", p.Name, err)
+	} else {
+		Logf("Output of clusterapi-tester:\n%v", logs)
+	}
+}
+
 func logFailedContainers(ns string) {
 	c, err := loadClient()
 	if err != nil {
@@ -520,16 +576,18 @@ func logFailedContainers(ns string) {
 		Logf("Error getting pods in namespace '%s': %v", ns, err)
 		return
 	}
+	Logf("Running kubectl logs on non-ready containers in %v", ns)
 	for _, pod := range podList.Items {
-		if res, err := podRunningReady(&pod); res && err == nil {
-			Logf("Ignoring Ready pod %v/%v", pod.Namespace, pod.Name)
-		} else {
+		if res, err := podRunningReady(&pod); !res || err != nil {
 			for _, container := range pod.Spec.Containers {
-				logs, err := getPreviousPodLogs(c, ns, pod.Name, container.Name)
+				logs, err := getPodLogs(c, ns, pod.Name, container.Name)
+				if err != nil {
+					logs, err = getPreviousPodLogs(c, ns, pod.Name, container.Name)
 					if err != nil {
 						Logf("Failed to get logs of pod %v, container %v, err: %v", pod.Name, container.Name, err)
 					}
-				By(fmt.Sprintf("Previous logs of %v/%v:%v on node %v", ns, pod.Name, container.Name, pod.Spec.NodeName))
+				}
+				By(fmt.Sprintf("Logs of %v/%v:%v on node %v", ns, pod.Name, container.Name, pod.Spec.NodeName))
 				Logf(logs)
 			}
 		}

test/images/clusterapi-tester/Dockerfile

+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM busybox
+MAINTAINER Prashanth B <beeps@google.com>
+ADD main main
+ENTRYPOINT ["/main"]

test/images/clusterapi-tester/Makefile

+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+all: push
+
+# 0.0 shouldn't clobber any released builds
+TAG = 1.0
+PREFIX = gcr.io/google_containers/clusterapi-tester
+
+main: main.go
+	CGO_ENABLED=0 GOOS=linux godep go build -a -installsuffix cgo -ldflags '-w' -o main ./main.go
+
+container: main
+	docker build -t $(PREFIX):$(TAG) .
+
+push: container
+	gcloud docker push $(PREFIX):$(TAG)
+
+clean:
+	rm -f main

test/images/clusterapi-tester/main.go

+/*
+Copyright 2014 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// A simple pod that first lists all nodes/services through the Kubernetes
+// api, then serves a 200 on /healthz.
+package main
+
+import (
+	"log"
+
+	"fmt"
+	"k8s.io/kubernetes/pkg/api"
+	client "k8s.io/kubernetes/pkg/client/unversioned"
+	"k8s.io/kubernetes/pkg/fields"
+	"k8s.io/kubernetes/pkg/labels"
+	"net/http"
+)
+
+func main() {
+	kubeClient, err := client.NewInCluster()
+	if err != nil {
+		log.Fatalf("Failed to create client: %v", err)
+	}
+	listAll := api.ListOptions{LabelSelector: labels.Everything(), FieldSelector: fields.Everything()}
+	nodes, err := kubeClient.Nodes().List(listAll)
+	if err != nil {
+		log.Fatalf("Failed to list nodes: %v", err)
+	}
+	log.Printf("Nodes:")
+	for _, node := range nodes.Items {
+		log.Printf("\t%v", node.Name)
+	}
+	services, err := kubeClient.Services(api.NamespaceDefault).List(listAll)
+	if err != nil {
+		log.Fatalf("Failed to list services: %v", err)
+	}
+	log.Printf("Services:")
+	for _, svc := range services.Items {
+		log.Printf("\t%v", svc.Name)
+	}
+	log.Printf("Success")
+	http.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintf(w, "Ok")
+	})
+	log.Fatal(http.ListenAndServe(":8080", nil))
+}

test/images/clusterapi-tester/pod.yaml

+apiVersion: v1
+kind: Pod
+metadata:
+  name: clusterapi-tester
+spec:
+  containers:
+  - image: gcr.io/google_containers/clusterapi-tester:1.0
+    name: clusterapi-tester
+    readinessProbe:
+      httpGet:
+        path: /healthz
+        port: 8080
+        scheme: HTTP
+      initialDelaySeconds: 10
+      timeoutSeconds: 5
+      failureThreshold: 3
+      periodSeconds: 10
+      successThreshold: 1
+  restartPolicy: OnFailure