enable on-demand metrics for eviction

pkg/kubelet/eviction/eviction_manager.go

@@ -455,7 +455,9 @@ func (m *managerImpl) reclaimNodeLevelResources(resourceToReclaim v1.ResourceNam
 // localStorageEviction checks the EmptyDir volume usage for each pod and determine whether it exceeds the specified limit and needs
 // to be evicted. It also checks every container in the pod, if the container overlay usage exceeds the limit, the pod will be evicted too.
 func (m *managerImpl) localStorageEviction(pods []*v1.Pod) []*v1.Pod {
-	summary, err := m.summaryProvider.Get()
+	// do not update node-level stats as local storage evictions do not utilize them.
+	forceStatsUpdate := false
+	summary, err := m.summaryProvider.Get(forceStatsUpdate)
 	if err != nil {
 		glog.Errorf("Could not get summary provider")
 		return nil

pkg/kubelet/eviction/helpers.go

@@ -712,7 +712,8 @@ func (a byEvictionPriority) Less(i, j int) bool {
 
 // makeSignalObservations derives observations using the specified summary provider.
 func makeSignalObservations(summaryProvider stats.SummaryProvider, capacityProvider CapacityProvider, pods []*v1.Pod) (signalObservations, statsFunc, error) {
-	summary, err := summaryProvider.Get()
+	updateStats := true
+	summary, err := summaryProvider.Get(updateStats)
 	if err != nil {
 		return nil, nil, err
 	}

pkg/kubelet/eviction/helpers_test.go

@@ -920,7 +920,7 @@ type fakeSummaryProvider struct {
 	result *statsapi.Summary
 }
 
-func (f *fakeSummaryProvider) Get() (*statsapi.Summary, error) {
+func (f *fakeSummaryProvider) Get(updateStats bool) (*statsapi.Summary, error) {
 	return f.result, nil
 }
 

pkg/kubelet/server/server_test.go

@@ -176,7 +176,7 @@ func (fk *fakeKubelet) ListVolumesForPod(podUID types.UID) (map[string]volume.Vo
 func (_ *fakeKubelet) RootFsStats() (*statsapi.FsStats, error)    { return nil, nil }
 func (_ *fakeKubelet) ListPodStats() ([]statsapi.PodStats, error) { return nil, nil }
 func (_ *fakeKubelet) ImageFsStats() (*statsapi.FsStats, error)   { return nil, nil }
-func (_ *fakeKubelet) GetCgroupStats(cgroupName string) (*statsapi.ContainerStats, *statsapi.NetworkStats, error) {
+func (_ *fakeKubelet) GetCgroupStats(cgroupName string, updateStats bool) (*statsapi.ContainerStats, *statsapi.NetworkStats, error) {
 	return nil, nil, nil
 }
 

pkg/kubelet/server/stats/handler.go

@@ -50,7 +50,7 @@ type StatsProvider interface {
 	//
 	// GetCgroupStats returns the stats and the networking usage of the cgroup
 	// with the specified cgroupName.
-	GetCgroupStats(cgroupName string) (*statsapi.ContainerStats, *statsapi.NetworkStats, error)
+	GetCgroupStats(cgroupName string, updateStats bool) (*statsapi.ContainerStats, *statsapi.NetworkStats, error)
 	// RootFsStats returns the stats of the node root filesystem.
 	RootFsStats() (*statsapi.FsStats, error)
 
@@ -183,7 +183,9 @@ func (h *handler) handleStats(request *restful.Request, response *restful.Respon
 
 // Handles stats summary requests to /stats/summary
 func (h *handler) handleSummary(request *restful.Request, response *restful.Response) {
-	summary, err := h.summaryProvider.Get()
+	// external calls to the summary API use cached stats
+	forceStatsUpdate := false
+	summary, err := h.summaryProvider.Get(forceStatsUpdate)
 	if err != nil {
 		handleError(response, "/stats/summary", err)
 	} else {

pkg/kubelet/server/stats/summary.go

@@ -25,7 +25,9 @@ import (
 )
 
 type SummaryProvider interface {
-	Get() (*statsapi.Summary, error)
+	// Get provides a new Summary with the stats from Kubelet,
+	// and will update some stats if updateStats is true
+	Get(updateStats bool) (*statsapi.Summary, error)
 }
 
 // summaryProviderImpl implements the SummaryProvider interface.
@@ -41,8 +43,7 @@ func NewSummaryProvider(statsProvider StatsProvider) SummaryProvider {
 	return &summaryProviderImpl{statsProvider}
 }
 
-// Get provides a new Summary with the stats from Kubelet.
-func (sp *summaryProviderImpl) Get() (*statsapi.Summary, error) {
+func (sp *summaryProviderImpl) Get(updateStats bool) (*statsapi.Summary, error) {
 	// TODO(timstclair): Consider returning a best-effort response if any of
 	// the following errors occur.
 	node, err := sp.provider.GetNode()
@@ -50,7 +51,7 @@ func (sp *summaryProviderImpl) Get() (*statsapi.Summary, error) {
 		return nil, fmt.Errorf("failed to get node info: %v", err)
 	}
 	nodeConfig := sp.provider.GetNodeConfig()
-	rootStats, networkStats, err := sp.provider.GetCgroupStats("/")
+	rootStats, networkStats, err := sp.provider.GetCgroupStats("/", updateStats)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get root cgroup stats: %v", err)
 	}
@@ -87,7 +88,7 @@ func (sp *summaryProviderImpl) Get() (*statsapi.Summary, error) {
 		if name == "" {
 			continue
 		}
-		s, _, err := sp.provider.GetCgroupStats(name)
+		s, _, err := sp.provider.GetCgroupStats(name, false)
 		if err != nil {
 			glog.Errorf("Failed to get system container stats for %q: %v", name, err)
 			continue

pkg/kubelet/server/stats/summary_test.go

@@ -69,13 +69,13 @@ func TestSummaryProvider(t *testing.T) {
 		On("ListPodStats").Return(podStats, nil).
 		On("ImageFsStats").Return(imageFsStats, nil).
 		On("RootFsStats").Return(rootFsStats, nil).
-		On("GetCgroupStats", "/").Return(cgroupStatsMap["/"].cs, cgroupStatsMap["/"].ns, nil).
-		On("GetCgroupStats", "/runtime").Return(cgroupStatsMap["/runtime"].cs, cgroupStatsMap["/runtime"].ns, nil).
-		On("GetCgroupStats", "/misc").Return(cgroupStatsMap["/misc"].cs, cgroupStatsMap["/misc"].ns, nil).
-		On("GetCgroupStats", "/kubelet").Return(cgroupStatsMap["/kubelet"].cs, cgroupStatsMap["/kubelet"].ns, nil)
+		On("GetCgroupStats", "/", true).Return(cgroupStatsMap["/"].cs, cgroupStatsMap["/"].ns, nil).
+		On("GetCgroupStats", "/runtime", false).Return(cgroupStatsMap["/runtime"].cs, cgroupStatsMap["/runtime"].ns, nil).
+		On("GetCgroupStats", "/misc", false).Return(cgroupStatsMap["/misc"].cs, cgroupStatsMap["/misc"].ns, nil).
+		On("GetCgroupStats", "/kubelet", false).Return(cgroupStatsMap["/kubelet"].cs, cgroupStatsMap["/kubelet"].ns, nil)
 
 	provider := NewSummaryProvider(mockStatsProvider)
-	summary, err := provider.Get()
+	summary, err := provider.Get(true)
 	assert.NoError(err)
 
 	assert.Equal(summary.Node.NodeName, "test-node")

pkg/kubelet/server/stats/testing/mock_stats_provider.go

@@ -33,13 +33,13 @@ type StatsProvider struct {
 	mock.Mock
 }
 
-// GetCgroupStats provides a mock function with given fields: cgroupName
-func (_m *StatsProvider) GetCgroupStats(cgroupName string) (*v1alpha1.ContainerStats, *v1alpha1.NetworkStats, error) {
-	ret := _m.Called(cgroupName)
+// GetCgroupStats provides a mock function with given fields: cgroupName, updateStats
+func (_m *StatsProvider) GetCgroupStats(cgroupName string, updateStats bool) (*v1alpha1.ContainerStats, *v1alpha1.NetworkStats, error) {
+	ret := _m.Called(cgroupName, updateStats)
 
 	var r0 *v1alpha1.ContainerStats
-	if rf, ok := ret.Get(0).(func(string) *v1alpha1.ContainerStats); ok {
-		r0 = rf(cgroupName)
+	if rf, ok := ret.Get(0).(func(string, bool) *v1alpha1.ContainerStats); ok {
+		r0 = rf(cgroupName, updateStats)
 	} else {
 		if ret.Get(0) != nil {
 			r0 = ret.Get(0).(*v1alpha1.ContainerStats)
@@ -47,8 +47,8 @@ func (_m *StatsProvider) GetCgroupStats(cgroupName string) (*v1alpha1.ContainerS
 	}
 
 	var r1 *v1alpha1.NetworkStats
-	if rf, ok := ret.Get(1).(func(string) *v1alpha1.NetworkStats); ok {
-		r1 = rf(cgroupName)
+	if rf, ok := ret.Get(1).(func(string, bool) *v1alpha1.NetworkStats); ok {
+		r1 = rf(cgroupName, updateStats)
 	} else {
 		if ret.Get(1) != nil {
 			r1 = ret.Get(1).(*v1alpha1.NetworkStats)
@@ -56,8 +56,8 @@ func (_m *StatsProvider) GetCgroupStats(cgroupName string) (*v1alpha1.ContainerS
 	}
 
 	var r2 error
-	if rf, ok := ret.Get(2).(func(string) error); ok {
-		r2 = rf(cgroupName)
+	if rf, ok := ret.Get(2).(func(string, bool) error); ok {
+		r2 = rf(cgroupName, updateStats)
 	} else {
 		r2 = ret.Error(2)
 	}

pkg/kubelet/stats/helper.go

@@ -238,11 +238,17 @@ func isMemoryUnlimited(v uint64) bool {
 
 // getCgroupInfo returns the information of the container with the specified
 // containerName from cadvisor.
-func getCgroupInfo(cadvisor cadvisor.Interface, containerName string) (*cadvisorapiv2.ContainerInfo, error) {
+func getCgroupInfo(cadvisor cadvisor.Interface, containerName string, updateStats bool) (*cadvisorapiv2.ContainerInfo, error) {
+	var maxAge *time.Duration
+	if updateStats {
+		age := 0 * time.Second
+		maxAge = &age
+	}
 	infoMap, err := cadvisor.ContainerInfoV2(containerName, cadvisorapiv2.RequestOptions{
 		IdType:    cadvisorapiv2.TypeName,
 		Count:     2, // 2 samples are needed to compute "instantaneous" CPU
 		Recursive: false,
+		MaxAge:    maxAge,
 	})
 	if err != nil {
 		return nil, fmt.Errorf("failed to get container info for %q: %v", containerName, err)
@@ -256,8 +262,8 @@ func getCgroupInfo(cadvisor cadvisor.Interface, containerName string) (*cadvisor
 
 // getCgroupStats returns the latest stats of the container having the
 // specified containerName from cadvisor.
-func getCgroupStats(cadvisor cadvisor.Interface, containerName string) (*cadvisorapiv2.ContainerStats, error) {
-	info, err := getCgroupInfo(cadvisor, containerName)
+func getCgroupStats(cadvisor cadvisor.Interface, containerName string, updateStats bool) (*cadvisorapiv2.ContainerStats, error) {
+	info, err := getCgroupInfo(cadvisor, containerName, updateStats)
 	if err != nil {
 		return nil, err
 	}

pkg/kubelet/stats/stats_provider.go

@@ -88,8 +88,8 @@ type containerStatsProvider interface {
 
 // GetCgroupStats returns the stats of the cgroup with the cgroupName. Note that
 // this function doesn't generate filesystem stats.
-func (p *StatsProvider) GetCgroupStats(cgroupName string) (*statsapi.ContainerStats, *statsapi.NetworkStats, error) {
-	info, err := getCgroupInfo(p.cadvisor, cgroupName)
+func (p *StatsProvider) GetCgroupStats(cgroupName string, updateStats bool) (*statsapi.ContainerStats, *statsapi.NetworkStats, error) {
+	info, err := getCgroupInfo(p.cadvisor, cgroupName, updateStats)
 	if err != nil {
 		return nil, nil, fmt.Errorf("failed to get cgroup stats for %q: %v", cgroupName, err)
 	}
@@ -113,8 +113,8 @@ func (p *StatsProvider) RootFsStats() (*statsapi.FsStats, error) {
 	}
 
 	// Get the root container stats's timestamp, which will be used as the
-	// imageFs stats timestamp.
-	rootStats, err := getCgroupStats(p.cadvisor, "/")
+	// imageFs stats timestamp.  Dont force a stats update, as we only want the timestamp.
+	rootStats, err := getCgroupStats(p.cadvisor, "/", false)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get root container stats: %v", err)
 	}

pkg/kubelet/stats/stats_provider_test.go

@@ -71,6 +71,7 @@ func TestGetCgroupStats(t *testing.T) {
 	const (
 		cgroupName        = "test-cgroup-name"
 		containerInfoSeed = 1000
+		updateStats       = false
 	)
 	var (
 		mockCadvisor     = new(cadvisortest.Mock)
@@ -87,7 +88,7 @@ func TestGetCgroupStats(t *testing.T) {
 	mockCadvisor.On("ContainerInfoV2", cgroupName, options).Return(containerInfoMap, nil)
 
 	provider := newStatsProvider(mockCadvisor, mockPodManager, mockRuntimeCache, fakeContainerStatsProvider{})
-	cs, ns, err := provider.GetCgroupStats(cgroupName)
+	cs, ns, err := provider.GetCgroupStats(cgroupName, updateStats)
 	assert.NoError(err)
 
 	checkCPUStats(t, "", containerInfoSeed, cs.CPU)
@@ -599,8 +600,8 @@ type fakeResourceAnalyzer struct {
 	podVolumeStats serverstats.PodVolumeStats
 }
 
-func (o *fakeResourceAnalyzer) Start()                          {}
-func (o *fakeResourceAnalyzer) Get() (*statsapi.Summary, error) { return nil, nil }
+func (o *fakeResourceAnalyzer) Start()                              {}
+func (o *fakeResourceAnalyzer) Get(bool) (*statsapi.Summary, error) { return nil, nil }
 func (o *fakeResourceAnalyzer) GetPodVolumeStats(uid types.UID) (serverstats.PodVolumeStats, bool) {
 	return o.podVolumeStats, true
 }