Gather Fluentd logs

cluster/saltbase/salt/fluentd-es/fluentd-es.yaml

@@ -12,8 +12,6 @@ spec:
     resources:
       limits:
         cpu: 100m
-    args:
-    - -q
     volumeMounts:
     - name: varlog
       mountPath: /var/log

test/e2e/es_cluster_logging.go

@@ -219,9 +219,9 @@ func ClusterLevelLoggingWithElasticsearch(f *Framework) {
 	By("Checking to make sure the Fluentd pod are running on each healthy node")
 	label = labels.SelectorFromSet(labels.Set(map[string]string{k8sAppKey: fluentdValue}))
 	options = api.ListOptions{LabelSelector: label}
-	pods, err = f.Client.Pods(api.NamespaceSystem).List(options)
+	fluentdPods, err := f.Client.Pods(api.NamespaceSystem).List(options)
 	Expect(err).NotTo(HaveOccurred())
-	for _, pod := range pods.Items {
+	for _, pod := range fluentdPods.Items {
 		if nodeInNodeList(pod.Spec.NodeName, nodes) {
 			err = waitForPodRunningInNamespace(f.Client, pod.Name, api.NamespaceSystem)
 			Expect(err).NotTo(HaveOccurred())
@@ -231,7 +231,7 @@ func ClusterLevelLoggingWithElasticsearch(f *Framework) {
 	// Check if each healthy node has fluentd running on it
 	for _, node := range nodes.Items {
 		exists := false
-		for _, pod := range pods.Items {
+		for _, pod := range fluentdPods.Items {
 			if pod.Spec.NodeName == node.Name {
 				exists = true
 				break
@@ -432,7 +432,7 @@ func ClusterLevelLoggingWithElasticsearch(f *Framework) {
 	}
 	for n := range missingPerNode {
 		if missingPerNode[n] > 0 {
-			Logf("Node %d is missing %d logs", n, missingPerNode[n])
+			Logf("Node %d %s is missing %d logs", n, nodes.Items[n].Name, missingPerNode[n])
 			opts := &api.PodLogOptions{}
 			body, err = f.Client.Pods(ns).GetLogs(podNames[n], opts).DoRaw()
 			if err != nil {
@@ -440,6 +440,18 @@ func ClusterLevelLoggingWithElasticsearch(f *Framework) {
 				continue
 			}
 			Logf("Pod %s has the following logs: %s", podNames[n], body)
+
+			for _, pod := range fluentdPods.Items {
+				if pod.Spec.NodeName == nodes.Items[n].Name {
+					body, err = f.Client.Pods(api.NamespaceSystem).GetLogs(pod.Name, opts).DoRaw()
+					if err != nil {
+						Logf("Cannot get logs from pod %v", pod.Name)
+						break
+					}
+					Logf("Fluentd Pod %s on node %s has the following logs: %s", pod.Name, nodes.Items[n].Name, body)
+					break
+				}
+			}
 		}
 	}
 	Failf("Failed to find all %d log lines", expected)