Merge pull request #22487 from vishh/node-status-cpu-hardcap

pkg/kubelet/cm/container_manager.go

@@ -33,6 +33,9 @@ type ContainerManager interface {
 
 	// Returns a NodeConfig that is being used by the container manager.
 	GetNodeConfig() NodeConfig
+
+	// Returns internal Status.
+	Status() Status
 }
 
 type NodeConfig struct {
@@ -41,3 +44,8 @@ type NodeConfig struct {
 	KubeletCgroupsName string
 	ContainerRuntime   string
 }
+
+type Status struct {
+	// Any soft requirements that were unsatisfied.
+	SoftRequirements error
+}

pkg/kubelet/cm/container_manager_linux.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"os"
 	"os/exec"
+	"path"
 	"strconv"
 	"strings"
 	"sync"
@@ -34,6 +35,7 @@ import (
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/api/resource"
 	"k8s.io/kubernetes/pkg/kubelet/cadvisor"
+	"k8s.io/kubernetes/pkg/util"
 	utilerrors "k8s.io/kubernetes/pkg/util/errors"
 	"k8s.io/kubernetes/pkg/util/mount"
 	"k8s.io/kubernetes/pkg/util/oom"
@@ -79,24 +81,35 @@ type containerManagerImpl struct {
 	cadvisorInterface cadvisor.Interface
 	mountUtil         mount.Interface
 	NodeConfig
+	status Status
 	// External containers being managed.
 	systemContainers []*systemContainer
 	periodicTasks    []func()
 }
 
+type features struct {
+	cpuHardcapping bool
+}
+
 var _ ContainerManager = &containerManagerImpl{}
 
 // checks if the required cgroups subsystems are mounted.
 // As of now, only 'cpu' and 'memory' are required.
-func validateSystemRequirements(mountUtil mount.Interface) error {
+// cpu quota is a soft requirement.
+func validateSystemRequirements(mountUtil mount.Interface) (features, error) {
 	const (
 		cgroupMountType = "cgroup"
 		localErr        = "system validation failed"
 	)
+	var (
+		cpuMountPoint string
+		f             features
+	)
 	mountPoints, err := mountUtil.List()
 	if err != nil {
-		return fmt.Errorf("%s - %v", localErr, err)
+		return f, fmt.Errorf("%s - %v", localErr, err)
 	}
+
 	expectedCgroups := sets.NewString("cpu", "cpuacct", "cpuset", "memory")
 	for _, mountPoint := range mountPoints {
 		if mountPoint.Type == cgroupMountType {
@@ -104,14 +117,31 @@ func validateSystemRequirements(mountUtil mount.Interface) error {
 				if expectedCgroups.Has(opt) {
 					expectedCgroups.Delete(opt)
 				}
+				if opt == "cpu" {
+					cpuMountPoint = mountPoint.Path
+				}
 			}
 		}
 	}
 
 	if expectedCgroups.Len() > 0 {
-		return fmt.Errorf("%s - Following Cgroup subsystem not mounted: %v", localErr, expectedCgroups.List())
+		return f, fmt.Errorf("%s - Following Cgroup subsystem not mounted: %v", localErr, expectedCgroups.List())
 	}
-	return nil
+
+	// Check if cpu quota is available.
+	// CPU cgroup is required and so it expected to be mounted at this point.
+	periodExists, err := util.FileExists(path.Join(cpuMountPoint, "cpu.cfs_period_us"))
+	if err != nil {
+		glog.Errorf("failed to detect if CPU cgroup cpu.cfs_period_us is available - %v", err)
+	}
+	quotaExists, err := util.FileExists(path.Join(cpuMountPoint, "cpu.cfs_quota_us"))
+	if err != nil {
+		glog.Errorf("failed to detect if CPU cgroup cpu.cfs_quota_us is available - %v", err)
+	}
+	if quotaExists && periodExists {
+		f.cpuHardcapping = true
+	}
+	return f, nil
 }
 
 // TODO(vmarmol): Add limits to the system containers.
@@ -185,10 +215,13 @@ func setupKernelTunables(option KernelTunableBehavior) error {
 }
 
 func (cm *containerManagerImpl) setupNode() error {
-	if err := validateSystemRequirements(cm.mountUtil); err != nil {
+	f, err := validateSystemRequirements(cm.mountUtil)
+	if err != nil {
 		return err
 	}
-
+	if !f.cpuHardcapping {
+		cm.status.SoftRequirements = fmt.Errorf("CPU hardcapping unsupported")
+	}
 	// TODO: plumb kernel tunable options into container manager, right now, we modify by default
 	if err := setupKernelTunables(KernelTunableModify); err != nil {
 		return err
@@ -312,6 +345,12 @@ func (cm *containerManagerImpl) GetNodeConfig() NodeConfig {
 	return cm.NodeConfig
 }
 
+func (cm *containerManagerImpl) Status() Status {
+	cm.RLock()
+	defer cm.RUnlock()
+	return cm.status
+}
+
 func (cm *containerManagerImpl) Start() error {
 	// Setup the node
 	if err := cm.setupNode(); err != nil {

pkg/kubelet/cm/container_manager_linux_test.go

@@ -20,9 +20,13 @@ package cm
 
 import (
 	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
 	"k8s.io/kubernetes/pkg/util/mount"
 )
@@ -75,7 +79,9 @@ func fakeContainerMgrMountInt() mount.Interface {
 }
 
 func TestCgroupMountValidationSuccess(t *testing.T) {
-	assert.Nil(t, validateSystemRequirements(fakeContainerMgrMountInt()))
+	f, err := validateSystemRequirements(fakeContainerMgrMountInt())
+	assert.Nil(t, err)
+	assert.False(t, f.cpuHardcapping, "cpu hardcapping is expected to be disabled")
 }
 
 func TestCgroupMountValidationMemoryMissing(t *testing.T) {
@@ -98,7 +104,8 @@ func TestCgroupMountValidationMemoryMissing(t *testing.T) {
 			},
 		},
 	}
-	assert.Error(t, validateSystemRequirements(mountInt))
+	_, err := validateSystemRequirements(mountInt)
+	assert.Error(t, err)
 }
 
 func TestCgroupMountValidationMultipleSubsytem(t *testing.T) {
@@ -121,5 +128,37 @@ func TestCgroupMountValidationMultipleSubsytem(t *testing.T) {
 			},
 		},
 	}
-	assert.Nil(t, validateSystemRequirements(mountInt))
+	_, err := validateSystemRequirements(mountInt)
+	assert.Nil(t, err)
+}
+
+func TestSoftRequirementsValidationSuccess(t *testing.T) {
+	req := require.New(t)
+	tempDir, err := ioutil.TempDir("", "")
+	req.NoError(err)
+	req.NoError(ioutil.WriteFile(path.Join(tempDir, "cpu.cfs_period_us"), []byte("0"), os.ModePerm))
+	req.NoError(ioutil.WriteFile(path.Join(tempDir, "cpu.cfs_quota_us"), []byte("0"), os.ModePerm))
+	mountInt := &fakeMountInterface{
+		[]mount.MountPoint{
+			{
+				Device: "cgroup",
+				Type:   "cgroup",
+				Opts:   []string{"rw", "relatime", "cpuset"},
+			},
+			{
+				Device: "cgroup",
+				Type:   "cgroup",
+				Opts:   []string{"rw", "relatime", "cpu"},
+				Path:   tempDir,
+			},
+			{
+				Device: "cgroup",
+				Type:   "cgroup",
+				Opts:   []string{"rw", "relatime", "cpuacct", "memory"},
+			},
+		},
+	}
+	f, err := validateSystemRequirements(mountInt)
+	assert.NoError(t, err)
+	assert.True(t, f.cpuHardcapping, "cpu hardcapping is expected to be enabled")
 }

pkg/kubelet/cm/container_manager_stub.go

@@ -38,6 +38,10 @@ func (cm *containerManagerStub) GetNodeConfig() NodeConfig {
 	return NodeConfig{}
 }
 
+func (cm *containerManagerStub) Status() Status {
+	return Status{}
+}
+
 func NewStubContainerManager() ContainerManager {
 	return &containerManagerStub{}
 }

pkg/kubelet/cm/container_manager_unsupported.go

@@ -43,6 +43,10 @@ func (unsupportedContainerManager) GetNodeConfig() NodeConfig {
 	return NodeConfig{}
 }
 
+func (cm *unsupportedContainerManager) Status() Status {
+	return Status{}
+}
+
 func NewContainerManager(_ mount.Interface, _ cadvisor.Interface, _ NodeConfig) (ContainerManager, error) {
 	return &unsupportedContainerManager{}, nil
 }

pkg/kubelet/kubelet.go

@@ -2964,6 +2964,12 @@ func (kl *Kubelet) setNodeReadyCondition(node *api.Node) {
 		}
 	}
 
+	// Record any soft requirements that were not met in the container manager.
+	status := kl.containerManager.Status()
+	if status.SoftRequirements != nil {
+		newNodeReadyCondition.Message = fmt.Sprintf("%s. WARNING: %s", newNodeReadyCondition.Message, status.SoftRequirements.Error())
+	}
+
 	readyConditionUpdated := false
 	needToRecordEvent := false
 	for i := range node.Status.Conditions {