Merge pull request #74788 from feiskyer/windows-network-stats

pkg/kubelet/stats/BUILD

@@ -5,6 +5,8 @@ go_library(
     srcs = [
         "cadvisor_stats_provider.go",
         "cri_stats_provider.go",
+        "cri_stats_provider_others.go",
+        "cri_stats_provider_windows.go",
         "helper.go",
         "log_metrics_provider.go",
         "stats_provider.go",
@@ -32,7 +34,13 @@ go_library(
         "//vendor/github.com/google/cadvisor/info/v1:go_default_library",
         "//vendor/github.com/google/cadvisor/info/v2:go_default_library",
         "//vendor/k8s.io/klog:go_default_library",
-    ],
+    ] + select({
+        "@io_bazel_rules_go//go/platform:windows": [
+            "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+            "//vendor/github.com/Microsoft/hcsshim:go_default_library",
+        ],
+        "//conditions:default": [],
+    }),
 )
 
 filegroup(

pkg/kubelet/stats/cri_stats_provider.go

@@ -26,11 +26,10 @@ import (
 	"time"
 
 	cadvisorfs "github.com/google/cadvisor/fs"
-	"k8s.io/klog"
-
 	cadvisorapiv2 "github.com/google/cadvisor/info/v2"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/klog"
 	internalapi "k8s.io/kubernetes/pkg/kubelet/apis/cri"
 	runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
 	statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
@@ -161,6 +160,13 @@ func (p *criStatsProvider) listPodStats(updateCPUNanoCoreUsage bool) ([]statsapi
 	}
 	caInfos := getCRICadvisorStats(allInfos)
 
+	// get network stats for containers.
+	// This is only used on Windows. For other platforms, (nil, nil) should be returned.
+	containerNetworkStats, err := p.listContainerNetworkStats()
+	if err != nil {
+		return nil, fmt.Errorf("failed to list container network stats: %v", err)
+	}
+
 	for _, stats := range resp {
 		containerID := stats.Attributes.Id
 		container, found := containerMap[containerID]
@@ -184,7 +190,7 @@ func (p *criStatsProvider) listPodStats(updateCPUNanoCoreUsage bool) ([]statsapi
 
 		// Fill available stats for full set of required pod stats
 		cs := p.makeContainerStats(stats, container, &rootFsInfo, fsIDtoInfo, podSandbox.GetMetadata().GetUid(), updateCPUNanoCoreUsage)
-		p.addPodNetworkStats(ps, podSandboxID, caInfos, cs)
+		p.addPodNetworkStats(ps, podSandboxID, caInfos, cs, containerNetworkStats[podSandboxID])
 		p.addPodCPUMemoryStats(ps, types.UID(podSandbox.Metadata.Uid), allInfos, cs)
 
 		// If cadvisor stats is available for the container, use it to populate
@@ -394,16 +400,26 @@ func (p *criStatsProvider) addPodNetworkStats(
 	podSandboxID string,
 	caInfos map[string]cadvisorapiv2.ContainerInfo,
 	cs *statsapi.ContainerStats,
+	netStats *statsapi.NetworkStats,
 ) {
 	caPodSandbox, found := caInfos[podSandboxID]
 	// try get network stats from cadvisor first.
 	if found {
-		ps.Network = cadvisorInfoToNetworkStats(ps.PodRef.Name, &caPodSandbox)
+		networkStats := cadvisorInfoToNetworkStats(ps.PodRef.Name, &caPodSandbox)
+		if networkStats != nil {
+			ps.Network = networkStats
+			return
+		}
+	}
+
+	// Not found from cadvisor, get from netStats.
+	if netStats != nil {
+		ps.Network = netStats
 		return
 	}
 
 	// TODO: sum Pod network stats from container stats.
-	klog.V(4).Infof("Unable to find cadvisor stats for sandbox %q", podSandboxID)
+	klog.V(4).Infof("Unable to find network stats for sandbox %q", podSandboxID)
 }
 
 func (p *criStatsProvider) addPodCPUMemoryStats(

pkg/kubelet/stats/cri_stats_provider_others.go

+// +build !windows
+
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package stats
+
+import (
+	statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
+)
+
+// listContainerNetworkStats returns the network stats of all the running containers.
+// It should return (nil, nil) for platforms other than Windows.
+func (p *criStatsProvider) listContainerNetworkStats() (map[string]*statsapi.NetworkStats, error) {
+	return nil, nil
+}

pkg/kubelet/stats/cri_stats_provider_windows.go

+// +build windows
+
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package stats
+
+import (
+	"time"
+
+	"github.com/Microsoft/hcsshim"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/klog"
+	statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
+)
+
+// listContainerNetworkStats returns the network stats of all the running containers.
+func (p *criStatsProvider) listContainerNetworkStats() (map[string]*statsapi.NetworkStats, error) {
+	containers, err := hcsshim.GetContainers(hcsshim.ComputeSystemQuery{
+		Types: []string{"Container"},
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	stats := make(map[string]*statsapi.NetworkStats)
+	for _, c := range containers {
+		container, err := hcsshim.OpenContainer(c.ID)
+		if err != nil {
+			klog.V(4).Infof("Failed to open container %q with error '%v', continue to get stats for other containers", c.ID, err)
+			continue
+		}
+
+		cstats, err := container.Statistics()
+		if err != nil {
+			klog.V(4).Infof("Failed to get statistics for container %q with error '%v', continue to get stats for other containers", c.ID, err)
+			continue
+		}
+
+		if len(cstats.Network) > 0 {
+			stats[c.ID] = hcsStatsToNetworkStats(cstats.Timestamp, cstats.Network)
+		}
+	}
+
+	return stats, nil
+}
+
+// hcsStatsToNetworkStats converts hcsshim.Statistics.Network to statsapi.NetworkStats
+func hcsStatsToNetworkStats(timestamp time.Time, hcsStats []hcsshim.NetworkStats) *statsapi.NetworkStats {
+	result := &statsapi.NetworkStats{
+		Time:       metav1.NewTime(timestamp),
+		Interfaces: make([]statsapi.InterfaceStats, 0),
+	}
+
+	adapters := sets.NewString()
+	for _, stat := range hcsStats {
+		iStat, err := hcsStatsToInterfaceStats(stat)
+		if err != nil {
+			klog.Warningf("Failed to get HNS endpoint %q with error '%v', continue to get stats for other endpoints", stat.EndpointId, err)
+			continue
+		}
+
+		// Only count each adapter once.
+		if adapters.Has(iStat.Name) {
+			continue
+		}
+
+		result.Interfaces = append(result.Interfaces, *iStat)
+		adapters.Insert(iStat.Name)
+	}
+
+	// TODO(feiskyer): add support of multiple interfaces for getting default interface.
+	if len(result.Interfaces) > 0 {
+		result.InterfaceStats = result.Interfaces[0]
+	}
+
+	return result
+}
+
+// hcsStatsToInterfaceStats converts hcsshim.NetworkStats to statsapi.InterfaceStats.
+func hcsStatsToInterfaceStats(stat hcsshim.NetworkStats) (*statsapi.InterfaceStats, error) {
+	endpoint, err := hcsshim.GetHNSEndpointByID(stat.EndpointId)
+	if err != nil {
+		return nil, err
+	}
+
+	return &statsapi.InterfaceStats{
+		Name:    endpoint.Name,
+		RxBytes: &stat.BytesReceived,
+		TxBytes: &stat.BytesSent,
+	}, nil
+}

pkg/kubelet/stats/helper.go

@@ -20,11 +20,10 @@ import (
 	"fmt"
 	"time"
 
-	"k8s.io/klog"
-
 	cadvisorapiv1 "github.com/google/cadvisor/info/v1"
 	cadvisorapiv2 "github.com/google/cadvisor/info/v2"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/klog"
 	statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
 	"k8s.io/kubernetes/pkg/kubelet/cadvisor"
 )
@@ -158,6 +157,10 @@ func cadvisorInfoToNetworkStats(name string, info *cadvisorapiv2.ContainerInfo)
 		return nil
 	}
 
+	if cstat.Network == nil {
+		return nil
+	}
+
 	iStats := statsapi.NetworkStats{
 		Time: metav1.NewTime(cstat.Timestamp),
 	}

pkg/kubelet/winstats/BUILD

@@ -4,6 +4,7 @@ go_library(
     name = "go_default_library",
     srcs = [
         "doc.go",
+        "network_stats.go",
         "perfcounter_nodestats.go",
         "perfcounters.go",
         "version.go",
@@ -13,6 +14,7 @@ go_library(
     visibility = ["//visibility:public"],
     deps = select({
         "@io_bazel_rules_go//go/platform:windows": [
+            "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
             "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
             "//vendor/github.com/JeffAshton/win_pdh:go_default_library",
             "//vendor/github.com/google/cadvisor/info/v1:go_default_library",

pkg/kubelet/winstats/perfcounter_nodestats.go

@@ -101,8 +101,13 @@ func (p *perfCounterNodeStatsClient) startMonitoring() error {
 		return err
 	}
 
+	networkAdapterCounter, err := newNetworkCounters()
+	if err != nil {
+		return err
+	}
+
 	go wait.Forever(func() {
-		p.collectMetricsData(cpuCounter, memWorkingSetCounter, memCommittedBytesCounter)
+		p.collectMetricsData(cpuCounter, memWorkingSetCounter, memCommittedBytesCounter, networkAdapterCounter)
 	}, perfCounterUpdatePeriod)
 
 	return nil
@@ -138,7 +143,7 @@ func (p *perfCounterNodeStatsClient) getNodeInfo() nodeInfo {
 	return p.nodeInfo
 }
 
-func (p *perfCounterNodeStatsClient) collectMetricsData(cpuCounter, memWorkingSetCounter, memCommittedBytesCounter *perfCounter) {
+func (p *perfCounterNodeStatsClient) collectMetricsData(cpuCounter, memWorkingSetCounter, memCommittedBytesCounter *perfCounter, networkAdapterCounter *networkCounter) {
 	cpuValue, err := cpuCounter.getData()
 	if err != nil {
 		klog.Errorf("Unable to get cpu perf counter data; err: %v", err)
@@ -157,12 +162,19 @@ func (p *perfCounterNodeStatsClient) collectMetricsData(cpuCounter, memWorkingSe
 		return
 	}
 
+	networkAdapterStats, err := networkAdapterCounter.getData()
+	if err != nil {
+		klog.Errorf("Unable to get network adapter perf counter data; err: %v", err)
+		return
+	}
+
 	p.mu.Lock()
 	defer p.mu.Unlock()
 	p.nodeMetrics = nodeMetrics{
 		cpuUsageCoreNanoSeconds:   p.convertCPUValue(cpuValue),
 		memoryPrivWorkingSetBytes: memWorkingSetValue,
 		memoryCommittedBytes:      memCommittedBytesValue,
+		interfaceStats:            networkAdapterStats,
 		timeStamp:                 time.Now(),
 	}
 }

pkg/kubelet/winstats/perfcounters.go

@@ -71,6 +71,7 @@ func newPerfCounter(counter string) (*perfCounter, error) {
 	}, nil
 }
 
+// getData is used for getting data without * in counter name.
 func (p *perfCounter) getData() (uint64, error) {
 	ret := win_pdh.PdhCollectQueryData(p.queryHandle)
 	if ret != win_pdh.ERROR_SUCCESS {
@@ -100,3 +101,36 @@ func (p *perfCounter) getData() (uint64, error) {
 
 	return data, nil
 }
+
+// getData is used for getting data with * in counter name.
+func (p *perfCounter) getDataList() (map[string]uint64, error) {
+	ret := win_pdh.PdhCollectQueryData(p.queryHandle)
+	if ret != win_pdh.ERROR_SUCCESS {
+		return nil, fmt.Errorf("unable to collect data from counter. Error code is %x", ret)
+	}
+
+	var bufSize, bufCount uint32
+	var size = uint32(unsafe.Sizeof(win_pdh.PDH_FMT_COUNTERVALUE_ITEM_DOUBLE{}))
+	var emptyBuf [1]win_pdh.PDH_FMT_COUNTERVALUE_ITEM_DOUBLE // need at least 1 addressable null ptr.
+	data := map[string]uint64{}
+
+	ret = win_pdh.PdhGetFormattedCounterArrayDouble(p.counterHandle, &bufSize, &bufCount, &emptyBuf[0])
+	if ret != win_pdh.PDH_MORE_DATA {
+		return nil, fmt.Errorf("unable to collect data from counter. Error code is %x", ret)
+	}
+
+	filledBuf := make([]win_pdh.PDH_FMT_COUNTERVALUE_ITEM_DOUBLE, bufCount*size)
+	ret = win_pdh.PdhGetFormattedCounterArrayDouble(p.counterHandle, &bufSize, &bufCount, &filledBuf[0])
+	if ret != win_pdh.ERROR_SUCCESS {
+		return nil, fmt.Errorf("unable to collect data from counter. Error code is %x", ret)
+	}
+
+	for i := 0; i < int(bufCount); i++ {
+		c := filledBuf[i]
+		value := uint64(c.FmtValue.DoubleValue)
+		name := win_pdh.UTF16PtrToString(c.SzName)
+		data[name] = value
+	}
+
+	return data, nil
+}

pkg/kubelet/winstats/winstats.go

@@ -58,6 +58,7 @@ type nodeMetrics struct {
 	memoryPrivWorkingSetBytes uint64
 	memoryCommittedBytes      uint64
 	timeStamp                 time.Time
+	interfaceStats            []cadvisorapi.InterfaceStats
 }
 
 type nodeInfo struct {
@@ -109,12 +110,11 @@ func (c *StatsClient) WinVersionInfo() (*cadvisorapi.VersionInfo, error) {
 
 func (c *StatsClient) createRootContainerInfo() (*cadvisorapiv2.ContainerInfo, error) {
 	nodeMetrics, err := c.client.getNodeMetrics()
-
 	if err != nil {
 		return nil, err
 	}
-	var stats []*cadvisorapiv2.ContainerStats
 
+	var stats []*cadvisorapiv2.ContainerStats
 	stats = append(stats, &cadvisorapiv2.ContainerStats{
 		Timestamp: nodeMetrics.timeStamp,
 		Cpu: &cadvisorapi.CpuStats{
@@ -126,6 +126,9 @@ func (c *StatsClient) createRootContainerInfo() (*cadvisorapiv2.ContainerInfo, e
 			WorkingSet: nodeMetrics.memoryPrivWorkingSetBytes,
 			Usage:      nodeMetrics.memoryCommittedBytes,
 		},
+		Network: &cadvisorapiv2.NetworkStats{
+			Interfaces: nodeMetrics.interfaceStats,
+		},
 	})
 
 	nodeInfo := c.client.getNodeInfo()
@@ -134,6 +137,7 @@ func (c *StatsClient) createRootContainerInfo() (*cadvisorapiv2.ContainerInfo, e
 			CreationTime: nodeInfo.startTime,
 			HasCpu:       true,
 			HasMemory:    true,
+			HasNetwork:   true,
 			Memory: cadvisorapiv2.MemorySpec{
 				Limit: nodeInfo.memoryPhysicalCapacityBytes,
 			},

pkg/kubelet/winstats/winstats_test.go

@@ -86,8 +86,9 @@ func TestWinContainerInfos(t *testing.T) {
 	infos := make(map[string]cadvisorapiv2.ContainerInfo)
 	infos["/"] = cadvisorapiv2.ContainerInfo{
 		Spec: cadvisorapiv2.ContainerSpec{
-			HasCpu:    true,
-			HasMemory: true,
+			HasCpu:     true,
+			HasMemory:  true,
+			HasNetwork: true,
 			Memory: cadvisorapiv2.MemorySpec{
 				Limit: 1.6e+10,
 			},
@@ -95,7 +96,11 @@ func TestWinContainerInfos(t *testing.T) {
 		Stats: stats,
 	}
 
-	assert.Equal(t, actualRootInfos, infos)
+	assert.Equal(t, len(actualRootInfos), len(infos))
+	assert.Equal(t, actualRootInfos["/"].Spec, infos["/"].Spec)
+	assert.Equal(t, len(actualRootInfos["/"].Stats), len(infos["/"].Stats))
+	assert.Equal(t, actualRootInfos["/"].Stats[0].Cpu, infos["/"].Stats[0].Cpu)
+	assert.Equal(t, actualRootInfos["/"].Stats[0].Memory, infos["/"].Stats[0].Memory)
 }
 
 func TestWinMachineInfo(t *testing.T) {