Test cases to verify container log stats

pkg/kubelet/kuberuntime/kuberuntime_container.go

@@ -845,7 +845,8 @@ func (m *kubeGenericRuntimeManager) removeContainerLog(containerID string) error
 		return fmt.Errorf("failed to get container status %q: %v", containerID, err)
 	}
 	labeledInfo := getContainerInfoFromLabels(status.Labels)
-	path := BuildContainerLogsDirectory(labeledInfo.PodUID, labeledInfo.ContainerName)
+	annotatedInfo := getContainerInfoFromAnnotations(status.Annotations)
+	path := buildFullContainerLogsPath(labeledInfo.PodUID, labeledInfo.ContainerName, annotatedInfo.RestartCount)
 	if err := m.osInterface.Remove(path); err != nil && !os.IsNotExist(err) {
 		return fmt.Errorf("failed to remove container %q log %q: %v", containerID, path, err)
 	}

pkg/kubelet/kuberuntime/kuberuntime_container_test.go

@@ -61,7 +61,7 @@ func TestRemoveContainer(t *testing.T) {
 	err = m.removeContainer(containerId)
 	assert.NoError(t, err)
 	// Verify container log is removed
-	expectedContainerLogPath := filepath.Join(podLogsRootDirectory, "12345678", "foo_0.log")
+	expectedContainerLogPath := filepath.Join(podLogsRootDirectory, "12345678", "foo", "0.log")
 	expectedContainerLogSymlink := legacyLogSymlink(containerId, "foo", "bar", "new")
 	assert.Equal(t, fakeOS.Removes, []string{expectedContainerLogPath, expectedContainerLogSymlink})
 	// Verify container is removed

pkg/kubelet/stats/cri_stats_provider_test.go

@@ -24,15 +24,24 @@ import (
 	cadvisorfs "github.com/google/cadvisor/fs"
 	cadvisorapiv2 "github.com/google/cadvisor/info/v2"
 	"github.com/stretchr/testify/assert"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
 	runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
 	critest "k8s.io/kubernetes/pkg/kubelet/apis/cri/testing"
 	statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
 	cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing"
 	kubecontainertest "k8s.io/kubernetes/pkg/kubelet/container/testing"
+	"k8s.io/kubernetes/pkg/kubelet/kuberuntime"
 	"k8s.io/kubernetes/pkg/kubelet/leaky"
 	kubepodtest "k8s.io/kubernetes/pkg/kubelet/pod/testing"
 	serverstats "k8s.io/kubernetes/pkg/kubelet/server/stats"
+	"k8s.io/kubernetes/pkg/volume"
+)
+
+const (
+	offsetInodeUsage = iota
+	offsetUsage
 )
 
 func TestCRIListPodStats(t *testing.T) {
@@ -71,18 +80,22 @@ func TestCRIListPodStats(t *testing.T) {
 		sandbox0           = makeFakePodSandbox("sandbox0-name", "sandbox0-uid", "sandbox0-ns")
 		container0         = makeFakeContainer(sandbox0, cName0, 0, false)
 		containerStats0    = makeFakeContainerStats(container0, imageFsMountpoint)
+		containerLogStats0 = makeFakeLogStats(1000)
 		container1         = makeFakeContainer(sandbox0, cName1, 0, false)
 		containerStats1    = makeFakeContainerStats(container1, unknownMountpoint)
+		containerLogStats1 = makeFakeLogStats(2000)
 
 		sandbox1           = makeFakePodSandbox("sandbox1-name", "sandbox1-uid", "sandbox1-ns")
 		container2         = makeFakeContainer(sandbox1, cName2, 0, false)
 		containerStats2    = makeFakeContainerStats(container2, imageFsMountpoint)
+		containerLogStats2 = makeFakeLogStats(3000)
 
 		sandbox2           = makeFakePodSandbox("sandbox2-name", "sandbox2-uid", "sandbox2-ns")
 		container3         = makeFakeContainer(sandbox2, cName3, 0, true)
 		containerStats3    = makeFakeContainerStats(container3, imageFsMountpoint)
 		container4         = makeFakeContainer(sandbox2, cName3, 1, false)
 		containerStats4    = makeFakeContainerStats(container4, imageFsMountpoint)
+		containerLogStats4 = makeFakeLogStats(4000)
 	)
 
 	var (
@@ -135,13 +148,23 @@ func TestCRIListPodStats(t *testing.T) {
 		PersistentVolumes: persistentVolumes,
 	}
 
+	fakeLogStats := map[string]*volume.Metrics{
+		kuberuntime.BuildContainerLogsDirectory(types.UID("sandbox0-uid"), cName0): containerLogStats0,
+		kuberuntime.BuildContainerLogsDirectory(types.UID("sandbox0-uid"), cName1): containerLogStats1,
+		kuberuntime.BuildContainerLogsDirectory(types.UID("sandbox1-uid"), cName2): containerLogStats2,
+		kuberuntime.BuildContainerLogsDirectory(types.UID("sandbox2-uid"), cName3): containerLogStats4,
+	}
+	fakeLogStatsProvider := NewFakeLogMetricsService(fakeLogStats)
+
 	provider := NewCRIStatsProvider(
 		mockCadvisor,
 		resourceAnalyzer,
 		mockPodManager,
 		mockRuntimeCache,
 		fakeRuntimeService,
-		fakeImageService)
+		fakeImageService,
+		fakeLogStatsProvider,
+	)
 
 	stats, err := provider.ListPodStats()
 	assert := assert.New(t)
@@ -157,7 +180,8 @@ func TestCRIListPodStats(t *testing.T) {
 	assert.Equal(sandbox0.CreatedAt, p0.StartTime.UnixNano())
 	assert.Equal(2, len(p0.Containers))
 
-	checkEphemeralStorageStats(assert, p0, ephemeralVolumes, []*runtimeapi.ContainerStats{containerStats0, containerStats1})
+	checkEphemeralStorageStats(assert, p0, ephemeralVolumes, []*runtimeapi.ContainerStats{containerStats0, containerStats1},
+		[]*volume.Metrics{containerLogStats0, containerLogStats1})
 
 	containerStatsMap := make(map[string]statsapi.ContainerStats)
 	for _, s := range p0.Containers {
@@ -168,32 +192,32 @@ func TestCRIListPodStats(t *testing.T) {
 	assert.Equal(container0.CreatedAt, c0.StartTime.UnixNano())
 	checkCRICPUAndMemoryStats(assert, c0, infos[container0.ContainerStatus.Id].Stats[0])
 	checkCRIRootfsStats(assert, c0, containerStats0, &imageFsInfo)
-	checkCRILogsStats(assert, c0, &rootFsInfo)
+	checkCRILogsStats(assert, c0, &rootFsInfo, containerLogStats0)
 	c1 := containerStatsMap[cName1]
 	assert.Equal(container1.CreatedAt, c1.StartTime.UnixNano())
 	checkCRICPUAndMemoryStats(assert, c1, infos[container1.ContainerStatus.Id].Stats[0])
 	checkCRIRootfsStats(assert, c1, containerStats1, nil)
-	checkCRILogsStats(assert, c1, &rootFsInfo)
+	checkCRILogsStats(assert, c1, &rootFsInfo, containerLogStats1)
 	checkCRINetworkStats(assert, p0.Network, infos[sandbox0.PodSandboxStatus.Id].Stats[0].Network)
 
 	p1 := podStatsMap[statsapi.PodReference{Name: "sandbox1-name", UID: "sandbox1-uid", Namespace: "sandbox1-ns"}]
 	assert.Equal(sandbox1.CreatedAt, p1.StartTime.UnixNano())
 	assert.Equal(1, len(p1.Containers))
 
-	checkEphemeralStorageStats(assert, p1, ephemeralVolumes, []*runtimeapi.ContainerStats{containerStats2})
+	checkEphemeralStorageStats(assert, p1, ephemeralVolumes, []*runtimeapi.ContainerStats{containerStats2}, []*volume.Metrics{containerLogStats2})
 	c2 := p1.Containers[0]
 	assert.Equal(cName2, c2.Name)
 	assert.Equal(container2.CreatedAt, c2.StartTime.UnixNano())
 	checkCRICPUAndMemoryStats(assert, c2, infos[container2.ContainerStatus.Id].Stats[0])
 	checkCRIRootfsStats(assert, c2, containerStats2, &imageFsInfo)
-	checkCRILogsStats(assert, c2, &rootFsInfo)
+	checkCRILogsStats(assert, c2, &rootFsInfo, containerLogStats2)
 	checkCRINetworkStats(assert, p1.Network, infos[sandbox1.PodSandboxStatus.Id].Stats[0].Network)
 
 	p2 := podStatsMap[statsapi.PodReference{Name: "sandbox2-name", UID: "sandbox2-uid", Namespace: "sandbox2-ns"}]
 	assert.Equal(sandbox2.CreatedAt, p2.StartTime.UnixNano())
 	assert.Equal(1, len(p2.Containers))
 
-	checkEphemeralStorageStats(assert, p2, ephemeralVolumes, []*runtimeapi.ContainerStats{containerStats4})
+	checkEphemeralStorageStats(assert, p2, ephemeralVolumes, []*runtimeapi.ContainerStats{containerStats4}, []*volume.Metrics{containerLogStats4})
 
 	c3 := p2.Containers[0]
 	assert.Equal(cName3, c3.Name)
@@ -201,7 +225,7 @@ func TestCRIListPodStats(t *testing.T) {
 	checkCRICPUAndMemoryStats(assert, c3, infos[container4.ContainerStatus.Id].Stats[0])
 	checkCRIRootfsStats(assert, c3, containerStats4, &imageFsInfo)
 
-	checkCRILogsStats(assert, c3, &rootFsInfo)
+	checkCRILogsStats(assert, c3, &rootFsInfo, containerLogStats4)
 	checkCRINetworkStats(assert, p2.Network, infos[sandbox2.PodSandboxStatus.Id].Stats[0].Network)
 
 	mockCadvisor.AssertExpectations(t)
@@ -220,6 +244,7 @@ func TestCRIImagesFsStats(t *testing.T) {
 		resourceAnalyzer     = new(fakeResourceAnalyzer)
 		fakeRuntimeService   = critest.NewFakeRuntimeService()
 		fakeImageService     = critest.NewFakeImageService()
+		fakeLogStatsProvider = NewFakeLogMetricsService(nil)
 	)
 
 	mockCadvisor.On("GetDirFsInfo", imageFsMountpoint).Return(imageFsInfo, nil)
@@ -233,7 +258,9 @@ func TestCRIImagesFsStats(t *testing.T) {
 		mockPodManager,
 		mockRuntimeCache,
 		fakeRuntimeService,
-		fakeImageService)
+		fakeImageService,
+		fakeLogStatsProvider,
+	)
 
 	stats, err := provider.ImageFsStats()
 	assert := assert.New(t)
@@ -385,17 +412,21 @@ func checkCRIRootfsStats(assert *assert.Assertions, actual statsapi.ContainerSta
 	assert.Equal(cs.WritableLayer.InodesUsed.Value, *actual.Rootfs.InodesUsed)
 }
 
-func checkCRILogsStats(assert *assert.Assertions, actual statsapi.ContainerStats, rootFsInfo *cadvisorapiv2.FsInfo) {
+func checkCRILogsStats(assert *assert.Assertions, actual statsapi.ContainerStats, rootFsInfo *cadvisorapiv2.FsInfo, logStats *volume.Metrics) {
 	assert.Equal(rootFsInfo.Timestamp, actual.Logs.Time.Time)
 	assert.Equal(rootFsInfo.Available, *actual.Logs.AvailableBytes)
 	assert.Equal(rootFsInfo.Capacity, *actual.Logs.CapacityBytes)
 	assert.Equal(*rootFsInfo.InodesFree, *actual.Logs.InodesFree)
 	assert.Equal(*rootFsInfo.Inodes, *actual.Logs.Inodes)
-	assert.Nil(actual.Logs.UsedBytes)
-	assert.Nil(actual.Logs.InodesUsed)
+	assert.Equal(uint64(logStats.Used.Value()), *actual.Logs.UsedBytes)
+	assert.Equal(uint64(logStats.InodesUsed.Value()), *actual.Logs.InodesUsed)
 }
 
-func checkEphemeralStorageStats(assert *assert.Assertions, actual statsapi.PodStats, volumes []statsapi.VolumeStats, containers []*runtimeapi.ContainerStats) {
+func checkEphemeralStorageStats(assert *assert.Assertions,
+	actual statsapi.PodStats,
+	volumes []statsapi.VolumeStats,
+	containers []*runtimeapi.ContainerStats,
+	containerLogStats []*volume.Metrics) {
 	var totalUsed, inodesUsed uint64
 	for _, container := range containers {
 		totalUsed = totalUsed + container.WritableLayer.UsedBytes.Value
@@ -406,8 +437,13 @@ func checkEphemeralStorageStats(assert *assert.Assertions, actual statsapi.PodSt
 		totalUsed = totalUsed + *volume.FsStats.UsedBytes
 		inodesUsed = inodesUsed + *volume.FsStats.InodesUsed
 	}
-	assert.Equal(int(*actual.EphemeralStorage.UsedBytes), int(totalUsed))
-	assert.Equal(int(*actual.EphemeralStorage.InodesUsed), int(inodesUsed))
+
+	for _, logStats := range containerLogStats {
+		totalUsed = totalUsed + uint64(logStats.Used.Value())
+	}
+
+	assert.Equal(int(totalUsed), int(*actual.EphemeralStorage.UsedBytes))
+	assert.Equal(int(inodesUsed), int(*actual.EphemeralStorage.InodesUsed))
 }
 
 func checkCRINetworkStats(assert *assert.Assertions, actual *statsapi.NetworkStats, expected *cadvisorapiv2.NetworkStats) {
@@ -416,3 +452,10 @@ func checkCRINetworkStats(assert *assert.Assertions, actual *statsapi.NetworkSta
 	assert.Equal(expected.Interfaces[0].TxBytes, *actual.TxBytes)
 	assert.Equal(expected.Interfaces[0].TxErrors, *actual.TxErrors)
 }
+
+func makeFakeLogStats(seed int) *volume.Metrics {
+	m := &volume.Metrics{}
+	m.Used = resource.NewQuantity(int64(seed+offsetUsage), resource.BinarySI)
+	m.InodesUsed = resource.NewQuantity(int64(seed+offsetInodeUsage), resource.BinarySI)
+	return m
+}

pkg/kubelet/stats/log_metrics_provider_test.go

+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package stats
+
+import (
+	"k8s.io/kubernetes/pkg/volume"
+)
+
+type fakeLogMetrics struct {
+	fakeStats map[string]*volume.Metrics
+}
+
+func NewFakeLogMetricsService(stats map[string]*volume.Metrics) LogMetricsService {
+	return &fakeLogMetrics{fakeStats: stats}
+}
+
+func (l *fakeLogMetrics) createLogMetricsProvider(path string) volume.MetricsProvider {
+	return NewFakeMetricsDu(path, l.fakeStats[path])
+}
+
+type fakeMetricsDu struct {
+	fakeStats *volume.Metrics
+}
+
+func NewFakeMetricsDu(path string, stats *volume.Metrics) volume.MetricsProvider {
+	return &fakeMetricsDu{fakeStats: stats}
+}
+
+func (f *fakeMetricsDu) GetMetrics() (*volume.Metrics, error) {
+	return f.fakeStats, nil
+}