Skip to content

K
k3s
Commit a66ee2eb authored by Jing Xu's avatar Jing Xu
Browse files
Options

Add pod-level metric for CPU and memory stats

This PR adds the pod-level metrics for CPU and memory stats. cAdvisor can get all pod cgroup information so we can add this pod-level CPU and memory stats information from the corresponding pod cgroup
parent 6b1b6d50
Show whitespace changes
Inline Side-by-side
Showing with 116 additions and 25 deletions
pkg/kubelet/apis/stats/v1alpha1/types.go
View file @ a66ee2eb
......@@ -86,6 +86,12 @@ type PodStats struct {
// +patchMergeKey=name
// +patchStrategy=merge
Containers []ContainerStats `json:"containers" patchStrategy:"merge" patchMergeKey:"name"`
// Stats pertaining to CPU resources consumed by pod cgroup (which includes all containers' resource usage and pod overhead).
// +optional
CPU *CPUStats `json:"cpu,omitempty"`
// Stats pertaining to memory (RAM) resources consumed by pod cgroup (which includes all containers' resource usage and pod overhead).
// +optional
Memory *MemoryStats `json:"memory,omitempty"`
// Stats pertaining to network resources.
// +optional
Network *NetworkStats `json:"network,omitempty"`
......
pkg/kubelet/cm/cgroup_manager_linux.go
View file @ a66ee2eb
......@@ -45,6 +45,8 @@ const (
libcontainerCgroupfs libcontainerCgroupManagerType = "cgroupfs"
// libcontainerSystemd means use libcontainer with systemd
libcontainerSystemd libcontainerCgroupManagerType = "systemd"
// systemdSuffix is the cgroup name suffix for systemd
systemdSuffix string = ".slice"
)
// hugePageSizeList is useful for converting to the hugetlb canonical unit
......@@ -68,8 +70,8 @@ func ConvertCgroupNameToSystemd(cgroupName CgroupName, outputToCgroupFs bool) st
}
// detect if we are given a systemd style name.
// if so, we do not want to do double encoding.
if strings.HasSuffix(part, ".slice") {
part = strings.TrimSuffix(part, ".slice")
if IsSystemdStyleName(part) {
part = strings.TrimSuffix(part, systemdSuffix)
separatorIndex := strings.LastIndex(part, "-")
if separatorIndex >= 0 && separatorIndex < len(part) {
part = part[separatorIndex+1:]
......@@ -87,8 +89,8 @@ func ConvertCgroupNameToSystemd(cgroupName CgroupName, outputToCgroupFs bool) st
result = "-"
}
// always have a .slice suffix
if !strings.HasSuffix(result, ".slice") {
result = result + ".slice"
if !IsSystemdStyleName(result) {
result = result + systemdSuffix
}
// if the caller desired the result in cgroupfs format...
......@@ -114,6 +116,13 @@ func ConvertCgroupFsNameToSystemd(cgroupfsName string) (string, error) {
return path.Base(cgroupfsName), nil
}
func IsSystemdStyleName(name string) bool {
if strings.HasSuffix(name, systemdSuffix) {
return true
}
return false
}
// libcontainerAdapter provides a simplified interface to libcontainer based on libcontainer type.
type libcontainerAdapter struct {
// cgroupManagerType defines how to interface with libcontainer
......@@ -151,15 +160,18 @@ func (l *libcontainerAdapter) revertName(name string) CgroupName {
if l.cgroupManagerType != libcontainerSystemd {
return CgroupName(name)
}
return CgroupName(RevertFromSystemdToCgroupStyleName(name))
}
func RevertFromSystemdToCgroupStyleName(name string) string {
driverName, err := ConvertCgroupFsNameToSystemd(name)
if err != nil {
panic(err)
}
driverName = strings.TrimSuffix(driverName, ".slice")
driverName = strings.TrimSuffix(driverName, systemdSuffix)
driverName = strings.Replace(driverName, "-", "/", -1)
driverName = strings.Replace(driverName, "_", "-", -1)
return CgroupName(driverName)
return driverName
}
// adaptName converts a CgroupName identifier to a driver specific conversion value.
......
pkg/kubelet/cm/cgroup_manager_unsupported.go
View file @ a66ee2eb
......@@ -77,3 +77,11 @@ func ConvertCgroupFsNameToSystemd(cgroupfsName string) (string, error) {
func ConvertCgroupNameToSystemd(cgroupName CgroupName, outputToCgroupFs bool) string {
return ""
}
func RevertFromSystemdToCgroupStyleName(name string) string {
return ""
}
func IsSystemdStyleName(name string) bool {
return false
}
pkg/kubelet/cm/helpers_linux.go
View file @ a66ee2eb
......@@ -26,6 +26,7 @@ import (
libcontainercgroups "github.com/opencontainers/runc/libcontainer/cgroups"
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/kubernetes/pkg/api/v1/resource"
v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper"
v1qos "k8s.io/kubernetes/pkg/apis/core/v1/helper/qos"
......@@ -222,3 +223,8 @@ func getCgroupProcs(dir string) ([]int, error) {
}
return out, nil
}
// GetPodCgroupNameSuffix returns the last element of the pod CgroupName identifier
func GetPodCgroupNameSuffix(podUID types.UID) string {
return podCgroupNamePrefix + string(podUID)
}
pkg/kubelet/cm/helpers_unsupported.go
View file @ a66ee2eb
......@@ -18,7 +18,10 @@ limitations under the License.
package cm
import "k8s.io/api/core/v1"
import (
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
)
const (
MinShares = 0
......@@ -52,3 +55,8 @@ func GetCgroupSubsystems() (*CgroupSubsystems, error) {
func getCgroupProcs(dir string) ([]int, error) {
return nil, nil
}
// GetPodCgroupNameSuffix returns the last element of the pod CgroupName identifier
func GetPodCgroupNameSuffix(podUID types.UID) string {
return ""
}
pkg/kubelet/cm/pod_container_manager_linux.go
View file @ a66ee2eb
......@@ -104,7 +104,7 @@ func (m *podContainerManagerImpl) GetPodContainerName(pod *v1.Pod) (CgroupName,
case v1.PodQOSBestEffort:
parentContainer = m.qosContainersInfo.BestEffort
}
podContainer := podCgroupNamePrefix + string(pod.UID)
podContainer := GetPodCgroupNameSuffix(pod.UID)
// Get the absolute path of the cgroup
cgroupName := (CgroupName)(path.Join(parentContainer, podContainer))
......
pkg/kubelet/stats/BUILD
View file @ a66ee2eb
......@@ -15,6 +15,7 @@ go_library(
"//pkg/kubelet/apis/cri/v1alpha1/runtime:go_default_library",
"//pkg/kubelet/apis/stats/v1alpha1:go_default_library",
"//pkg/kubelet/cadvisor:go_default_library",
"//pkg/kubelet/cm:go_default_library",
"//pkg/kubelet/container:go_default_library",
"//pkg/kubelet/leaky:go_default_library",
"//pkg/kubelet/network:go_default_library",
......
pkg/kubelet/stats/cadvisor_stats_provider.go
View file @ a66ee2eb
......@@ -18,6 +18,7 @@ package stats
import (
"fmt"
"path"
"sort"
"strings"
......@@ -28,6 +29,7 @@ import (
"k8s.io/apimachinery/pkg/types"
statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
"k8s.io/kubernetes/pkg/kubelet/cadvisor"
"k8s.io/kubernetes/pkg/kubelet/cm"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
"k8s.io/kubernetes/pkg/kubelet/leaky"
"k8s.io/kubernetes/pkg/kubelet/server/stats"
......@@ -89,9 +91,9 @@ func (p *cadvisorStatsProvider) ListPodStats() ([]statsapi.PodStats, error) {
return nil, fmt.Errorf("failed to get root cgroup stats: %v", err)
}
}
// removeTerminatedContainerInfo will also remove pod level cgroups, so save the infos into allInfos first
allInfos := infos
infos = removeTerminatedContainerInfo(infos)
// Map each container to a pod and update the PodStats with container data.
podToStats := map[statsapi.PodReference]*statsapi.PodStats{}
for key, cinfo := range infos {
......@@ -141,6 +143,13 @@ func (p *cadvisorStatsProvider) ListPodStats() ([]statsapi.PodStats, error) {
podStats.VolumeStats = append(vstats.EphemeralVolumes, vstats.PersistentVolumes...)
}
podStats.EphemeralStorage = calcEphemeralStorage(podStats.Containers, ephemeralStats, &rootFsInfo)
// Lookup the pod-level cgroup's CPU and memory stats
podInfo := getcadvisorPodInfoFromPodUID(podUID, allInfos)
if podInfo != nil {
cpu, memory := cadvisorInfoToCPUandMemoryStats(podInfo)
podStats.CPU = cpu
podStats.Memory = memory
}
result = append(result, *podStats)
}
......@@ -243,6 +252,19 @@ func isPodManagedContainer(cinfo *cadvisorapiv2.ContainerInfo) bool {
return managed
}
// getcadvisorPodInfoFromPodUID returns a pod cgroup information by matching the podUID with its CgroupName identifier base name
func getcadvisorPodInfoFromPodUID(podUID types.UID, infos map[string]cadvisorapiv2.ContainerInfo) *cadvisorapiv2.ContainerInfo {
for key, info := range infos {
if cm.IsSystemdStyleName(key) {
key = cm.RevertFromSystemdToCgroupStyleName(key)
}
if cm.GetPodCgroupNameSuffix(podUID) == path.Base(key) {
return &info
}
}
return nil
}
// removeTerminatedContainerInfo returns the specified containerInfo but with
// the stats of the terminated containers removed.
//
......
pkg/kubelet/stats/cadvisor_stats_provider_test.go
View file @ a66ee2eb
......@@ -136,6 +136,8 @@ func TestCadvisorListPodStats(t *testing.T) {
// Pod2 - Namespace2
"/pod2-i": getTestContainerInfo(seedPod2Infra, pName2, namespace2, leaky.PodInfraContainerName),
"/pod2-c0": getTestContainerInfo(seedPod2Container, pName2, namespace2, cName20),
"/kubepods/burstable/podUIDpod0": getTestContainerInfo(seedPod0Infra, pName0, namespace0, leaky.PodInfraContainerName),
"/kubepods/podUIDpod1": getTestContainerInfo(seedPod1Infra, pName1, namespace0, leaky.PodInfraContainerName),
}
freeRootfsInodes := rootfsInodesFree
......@@ -228,6 +230,8 @@ func TestCadvisorListPodStats(t *testing.T) {
assert.EqualValues(t, testTime(creationTime, seedPod0Infra).Unix(), ps.StartTime.Time.Unix())
checkNetworkStats(t, "Pod0", seedPod0Infra, ps.Network)
checkEphemeralStats(t, "Pod0", []int{seedPod0Container0, seedPod0Container1}, []int{seedEphemeralVolume1, seedEphemeralVolume2}, ps.EphemeralStorage)
checkCPUStats(t, "Pod0", seedPod0Infra, ps.CPU)
checkMemoryStats(t, "Pod0", seedPod0Infra, infos["/pod0-i"], ps.Memory)
// Validate Pod1 Results
ps, found = indexPods[prf1]
......
pkg/kubelet/stats/helper.go
View file @ a66ee2eb
......@@ -30,21 +30,15 @@ import (
"k8s.io/kubernetes/pkg/kubelet/network"
)
// cadvisorInfoToContainerStats returns the statsapi.ContainerStats converted
// from the container and filesystem info.
func cadvisorInfoToContainerStats(name string, info *cadvisorapiv2.ContainerInfo, rootFs, imageFs *cadvisorapiv2.FsInfo) *statsapi.ContainerStats {
result := &statsapi.ContainerStats{
StartTime: metav1.NewTime(info.Spec.CreationTime),
Name: name,
}
func cadvisorInfoToCPUandMemoryStats(info *cadvisorapiv2.ContainerInfo) (*statsapi.CPUStats, *statsapi.MemoryStats) {
cstat, found := latestContainerStats(info)
if !found {
return result
return nil, nil
}
var cpuStats *statsapi.CPUStats
var memoryStats *statsapi.MemoryStats
if info.Spec.HasCpu {
cpuStats := statsapi.CPUStats{
cpuStats = &statsapi.CPUStats{
Time: metav1.NewTime(cstat.Timestamp),
}
if cstat.CpuInst != nil {
......@@ -53,13 +47,11 @@ func cadvisorInfoToContainerStats(name string, info *cadvisorapiv2.ContainerInfo
if cstat.Cpu != nil {
cpuStats.UsageCoreNanoSeconds = &cstat.Cpu.Usage.Total
}
result.CPU = &cpuStats
}
if info.Spec.HasMemory {
pageFaults := cstat.Memory.ContainerData.Pgfault
majorPageFaults := cstat.Memory.ContainerData.Pgmajfault
result.Memory = &statsapi.MemoryStats{
memoryStats = &statsapi.MemoryStats{
Time: metav1.NewTime(cstat.Timestamp),
UsageBytes: &cstat.Memory.Usage,
WorkingSetBytes: &cstat.Memory.WorkingSet,
......@@ -70,9 +62,27 @@ func cadvisorInfoToContainerStats(name string, info *cadvisorapiv2.ContainerInfo
// availableBytes = memory limit (if known) - workingset
if !isMemoryUnlimited(info.Spec.Memory.Limit) {
availableBytes := info.Spec.Memory.Limit - cstat.Memory.WorkingSet
result.Memory.AvailableBytes = &availableBytes
memoryStats.AvailableBytes = &availableBytes
}
}
return cpuStats, memoryStats
}
// cadvisorInfoToContainerStats returns the statsapi.ContainerStats converted
// from the container and filesystem info.
func cadvisorInfoToContainerStats(name string, info *cadvisorapiv2.ContainerInfo, rootFs, imageFs *cadvisorapiv2.FsInfo) *statsapi.ContainerStats {
result := &statsapi.ContainerStats{
StartTime: metav1.NewTime(info.Spec.CreationTime),
Name: name,
}
cstat, found := latestContainerStats(info)
if !found {
return result
}
cpu, memory := cadvisorInfoToCPUandMemoryStats(info)
result.CPU = cpu
result.Memory = memory
if rootFs != nil {
// The container logs live on the node rootfs device
......
test/e2e_node/summary_test.go
View file @ a66ee2eb
......@@ -175,6 +175,20 @@ var _ = framework.KubeDescribe("Summary API", func() {
"TxBytes": bounded(10, 10*framework.Mb),
"TxErrors": bounded(0, 1000),
}),
"CPU": ptrMatchAllFields(gstruct.Fields{
"Time": recent(maxStatsAge),
"UsageNanoCores": bounded(100000, 1E9),
"UsageCoreNanoSeconds": bounded(10000000, 1E11),
}),
"Memory": ptrMatchAllFields(gstruct.Fields{
"Time": recent(maxStatsAge),
"AvailableBytes": bounded(1*framework.Kb, 10*framework.Mb),
"UsageBytes": bounded(10*framework.Kb, 20*framework.Mb),
"WorkingSetBytes": bounded(10*framework.Kb, 20*framework.Mb),
"RSSBytes": bounded(1*framework.Kb, framework.Mb),
"PageFaults": bounded(0, 1000000),
"MajorPageFaults": bounded(0, 10),
}),
"VolumeStats": gstruct.MatchAllElements(summaryObjectID, gstruct.Elements{
"test-empty-dir": gstruct.MatchAllFields(gstruct.Fields{
"Name": Equal("test-empty-dir"),
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment