Merge pull request #70749 from brancz/log-metrics

pkg/kubelet/kubelet.go

@@ -1311,7 +1311,11 @@ func (kl *Kubelet) StartGarbageCollection() {
 // Note that the modules here must not depend on modules that are not initialized here.
 func (kl *Kubelet) initializeModules() error {
 	// Prometheus metrics.
-	metrics.Register(kl.runtimeCache, collectors.NewVolumeStatsCollector(kl))
+	metrics.Register(
+		kl.runtimeCache,
+		collectors.NewVolumeStatsCollector(kl),
+		collectors.NewLogMetricsCollector(kl.StatsProvider.ListPodStats),
+	)
 
 	// Setup filesystem directories.
 	if err := kl.setupDataDirs(); err != nil {

pkg/kubelet/metrics/collectors/BUILD

@@ -2,7 +2,10 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 
 go_library(
     name = "go_default_library",
-    srcs = ["volume_stats.go"],
+    srcs = [
+        "log_metrics.go",
+        "volume_stats.go",
+    ],
     importpath = "k8s.io/kubernetes/pkg/kubelet/metrics/collectors",
     visibility = ["//visibility:public"],
     deps = [
@@ -19,6 +22,7 @@ go_test(
     name = "go_default_test",
     srcs = [
         "helper_test.go",
+        "log_metrics_test.go",
         "volume_stats_test.go",
     ],
     embed = [":go_default_library"],

pkg/kubelet/metrics/collectors/log_metrics.go

+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package collectors
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"k8s.io/klog"
+
+	statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
+)
+
+var (
+	descLogSize = prometheus.NewDesc(
+		"kubelet_container_log_filesystem_used_bytes",
+		"Bytes used by the container's logs on the filesystem.",
+		[]string{
+			"namespace",
+			"pod",
+			"container",
+		}, nil,
+	)
+)
+
+type logMetricsCollector struct {
+	podStats func() ([]statsapi.PodStats, error)
+}
+
+// NewLogMetricsCollector implements the prometheus.Collector interface and
+// exposes metrics about container's log volume size.
+func NewLogMetricsCollector(podStats func() ([]statsapi.PodStats, error)) prometheus.Collector {
+	return &logMetricsCollector{
+		podStats: podStats,
+	}
+}
+
+// Describe implements the prometheus.Collector interface.
+func (c *logMetricsCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- descLogSize
+}
+
+// Collect implements the prometheus.Collector interface.
+func (c *logMetricsCollector) Collect(ch chan<- prometheus.Metric) {
+	podStats, err := c.podStats()
+	if err != nil {
+		klog.Errorf("failed to get pod stats: %v", err)
+		return
+	}
+
+	for _, ps := range podStats {
+		for _, c := range ps.Containers {
+			if c.Logs.UsedBytes != nil {
+				ch <- prometheus.MustNewConstMetric(
+					descLogSize,
+					prometheus.GaugeValue,
+					float64(*c.Logs.UsedBytes),
+					ps.PodRef.Namespace,
+					ps.PodRef.Name,
+					c.Name,
+				)
+			}
+		}
+	}
+}

pkg/kubelet/metrics/collectors/log_metrics_test.go

+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package collectors
+
+import (
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus"
+
+	statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
+)
+
+func TestNoMetricsCollected(t *testing.T) {
+	ch := make(chan prometheus.Metric)
+
+	collector := &logMetricsCollector{
+		podStats: func() ([]statsapi.PodStats, error) {
+			return []statsapi.PodStats{}, nil
+		},
+	}
+	collector.Collect(ch)
+
+	num := len(ch)
+	if num != 0 {
+		t.Fatalf("Channel expected to be empty, but received %d", num)
+	}
+}
+
+func TestMetricsCollected(t *testing.T) {
+	size := uint64(18)
+	collector := &logMetricsCollector{
+		podStats: func() ([]statsapi.PodStats, error) {
+			return []statsapi.PodStats{
+				{
+					PodRef: statsapi.PodReference{
+						Namespace: "some-namespace",
+						Name:      "podName1",
+					},
+					Containers: []statsapi.ContainerStats{
+						{
+							Name: "containerName1",
+							Logs: &statsapi.FsStats{
+								UsedBytes: &size,
+							},
+						},
+					},
+				},
+			}, nil
+		},
+	}
+
+	err := gatherAndCompare(collector, `
+		# HELP kubelet_container_log_filesystem_used_bytes Bytes used by the container's logs on the filesystem.
+		# TYPE kubelet_container_log_filesystem_used_bytes gauge
+		kubelet_container_log_filesystem_used_bytes{container="containerName1",namespace="some-namespace",pod="podName1"} 18
+`, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+}