Skip to content

K
k3s
Commit 059c2aa7 authored by Madhusudan.C.S's avatar Madhusudan.C.S
Browse files
Options

Mitigate node out of disk status oscillation by delaying it.

Implement a flag that defines the frequency at which a node's out of disk condition can change its status. Use this flag to suspend out of disk status changes in the time period specified by the flag, after the status is changed once. Set the flag to 0 in e2e tests so that we can predictably test out of disk node condition. Also, use util.Clock interface for all time related functionality in the kubelet. Calling time functions in unversioned package or time package such as unversioned.Now() or time.Now() makes it really hard to test such code. It also makes the tests flaky and sometimes unnecessarily slow due to time.Sleep() calls used to simulate the time elapsed. So use util.Clock interface instead which can be faked in the tests.
parent 2cd8bff3
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 60 additions and 31 deletions
cluster/gce/config-test.sh
View file @ 059c2aa7
......@@ -78,7 +78,7 @@ KUBEPROXY_TEST_LOG_LEVEL="${KUBEPROXY_TEST_LOG_LEVEL:-$TEST_CLUSTER_LOG_LEVEL}"
TEST_CLUSTER_RESYNC_PERIOD="${TEST_CLUSTER_RESYNC_PERIOD:---min-resync-period=3m}"
KUBELET_TEST_ARGS="--max-pods=110 --serialize-image-pulls=false"
KUBELET_TEST_ARGS="--max-pods=110 --serialize-image-pulls=false --outofdisk-transition-frequency=0"
APISERVER_TEST_ARGS="--runtime-config=extensions/v1beta1"
CONTROLLER_MANAGER_TEST_ARGS="${TEST_CLUSTER_RESYNC_PERIOD}"
SCHEDULER_TEST_ARGS=""
......
cmd/integration/integration.go
View file @ 059c2aa7
......@@ -222,6 +222,7 @@ func startComponents(firstManifestURL, secondManifestURL string) (string, string
10*time.Second, /* MinimumGCAge */
3*time.Second, /* NodeStatusUpdateFrequency */
10*time.Second, /* SyncFrequency */
10*time.Second, /* OutOfDiskTransitionFrequency */
40, /* MaxPods */
cm, net.ParseIP("127.0.0.1"))
......@@ -254,6 +255,7 @@ func startComponents(firstManifestURL, secondManifestURL string) (string, string
10*time.Second, /* MinimumGCAge */
3*time.Second, /* NodeStatusUpdateFrequency */
10*time.Second, /* SyncFrequency */
10*time.Second, /* OutOfDiskTransitionFrequency */
40, /* MaxPods */
cm,
......
cmd/kubelet/app/server.go
View file @ 059c2aa7
......@@ -143,6 +143,7 @@ type KubeletServer struct {
TLSCertFile string
TLSPrivateKeyFile string
ReconcileCIDR bool
OutOfDiskTransitionFrequency time.Duration
// Flags intended for testing
// Is the kubelet containerized?
......@@ -233,6 +234,7 @@ func NewKubeletServer() *KubeletServer {
ReconcileCIDR: true,
KubeAPIQPS: 5.0,
KubeAPIBurst: 10,
OutOfDiskTransitionFrequency: 5 * time.Minute,
ExperimentalFlannelOverlay: experimentalFlannelOverlay,
}
}
......@@ -349,6 +351,7 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) {
fs.Float32Var(&s.KubeAPIQPS, "kube-api-qps", s.KubeAPIQPS, "QPS to use while talking with kubernetes apiserver")
fs.IntVar(&s.KubeAPIBurst, "kube-api-burst", s.KubeAPIBurst, "Burst to use while talking with kubernetes apiserver")
fs.BoolVar(&s.SerializeImagePulls, "serialize-image-pulls", s.SerializeImagePulls, "Pull images one at a time. We recommend *not* changing the default value on nodes that run docker daemon with version < 1.9 or an Aufs storage backend. Issue #10959 has more details. [default=true]")
fs.DurationVar(&s.OutOfDiskTransitionFrequency, "outofdisk-transition-frequency", s.OutOfDiskTransitionFrequency, "Duration for which the kubelet has to wait before transitioning out of out-of-disk node condition status. Default: 5m0s")
fs.BoolVar(&s.ExperimentalFlannelOverlay, "experimental-flannel-overlay", s.ExperimentalFlannelOverlay, "Experimental support for starting the kubelet with the default overlay network (flannel). Assumes flanneld is already running in client mode. [default=false]")
fs.IPVar(&s.NodeIP, "node-ip", s.NodeIP, "IP address of the node. If set, kubelet will use this IP address for the node")
}
......@@ -488,6 +491,7 @@ func (s *KubeletServer) UnsecuredKubeletConfig() (*KubeletConfig, error) {
TLSOptions: tlsOptions,
Writer: writer,
VolumePlugins: ProbeVolumePlugins(s.VolumePluginDir),
OutOfDiskTransitionFrequency: s.OutOfDiskTransitionFrequency,
ExperimentalFlannelOverlay: s.ExperimentalFlannelOverlay,
NodeIP: s.NodeIP,
......@@ -706,7 +710,7 @@ func SimpleKubelet(client *client.Client,
configFilePath string,
cloud cloudprovider.Interface,
osInterface kubecontainer.OSInterface,
fileCheckFrequency, httpCheckFrequency, minimumGCAge, nodeStatusUpdateFrequency, syncFrequency time.Duration,
fileCheckFrequency, httpCheckFrequency, minimumGCAge, nodeStatusUpdateFrequency, syncFrequency, outOfDiskTransitionFrequency time.Duration,
maxPods int,
containerManager cm.ContainerManager, clusterDNS net.IP) *KubeletConfig {
imageGCPolicy := kubelet.ImageGCPolicy{
......@@ -768,6 +772,7 @@ func SimpleKubelet(client *client.Client,
TLSOptions: tlsOptions,
VolumePlugins: volumePlugins,
Writer: &io.StdWriter{},
OutOfDiskTransitionFrequency: outOfDiskTransitionFrequency,
}
return &kcfg
}
......@@ -965,6 +970,7 @@ type KubeletConfig struct {
TLSOptions *server.TLSOptions
Writer io.Writer
VolumePlugins []volume.VolumePlugin
OutOfDiskTransitionFrequency time.Duration
ExperimentalFlannelOverlay bool
NodeIP net.IP
......@@ -1050,6 +1056,7 @@ func CreateAndInitKubelet(kc *KubeletConfig) (k KubeletBootstrap, pc *config.Pod
kc.OOMAdjuster,
kc.SerializeImagePulls,
kc.ContainerManager,
kc.OutOfDiskTransitionFrequency,
kc.ExperimentalFlannelOverlay,
kc.NodeIP,
)
......
docs/admin/kubelet.md
View file @ 059c2aa7
......@@ -118,6 +118,7 @@ kubelet
--node-labels-file="": the path to a yaml or json file containing a series of key pair labels to apply on node registration
--node-status-update-frequency=10s: Specifies how often kubelet posts node status to master. Note: be cautious when changing the constant, it must work with nodeMonitorGracePeriod in nodecontroller. Default: 10s
--oom-score-adj=-999: The oom-score-adj value for kubelet process. Values must be within the range [-1000, 1000]
--outofdisk-transition-frequency=5m0s: Duration for which the kubelet has to wait before transitioning out of out-of-disk node condition status. Default: 5m0s
--pod-cidr="": The CIDR to use for pod IP addresses, only used in standalone mode. In cluster mode, this is obtained from the master.
--pod-infra-container-image="gcr.io/google_containers/pause:2.0": The image whose network/ipc namespaces containers in each pod will use.
--port=10250: The port for the Kubelet to serve on.
......@@ -143,7 +144,7 @@ kubelet
--volume-plugin-dir="/usr/libexec/kubernetes/kubelet-plugins/volume/exec/": <Warning: Alpha feature> The full path of the directory in which to search for additional third party volume plugins
```
###### Auto generated by spf13/cobra on 11-Dec-2015
###### Auto generated by spf13/cobra on 29-Dec-2015
<!-- BEGIN MUNGE: GENERATED_ANALYTICS -->
......
docs/api-reference/extensions/v1beta1/definitions.html
View file @ 059c2aa7
......@@ -4361,7 +4361,7 @@ Populated by the system when a graceful deletion is requested. Read-only. More i
</div>
<div id="footer">
<div id="footer-text">
Last updated 2015-12-18 08:02:33 UTC
Last updated 2015-12-29 09:07:13 UTC
</div>
</div>
</body>
......
docs/api-reference/extensions/v1beta1/operations.html
View file @ 059c2aa7
......@@ -5578,7 +5578,7 @@ span.icon > [class^="icon-"], span.icon > [class*=" icon-"] { cursor: default; }
</div>
<div id="footer">
<div id="footer-text">
Last updated 2015-12-15 06:44:37 UTC
Last updated 2015-12-22 07:40:13 UTC
</div>
</div>
</body>
......
docs/api-reference/v1/definitions.html
View file @ 059c2aa7
......@@ -7001,7 +7001,7 @@ The resulting set of endpoints can be viewed as:<br>
</div>
<div id="footer">
<div id="footer-text">
Last updated 2015-12-18 08:02:17 UTC
Last updated 2015-12-29 09:07:07 UTC
</div>
</div>
</body>
......
docs/api-reference/v1/operations.html
View file @ 059c2aa7
......@@ -25733,7 +25733,7 @@ span.icon > [class^="icon-"], span.icon > [class*=" icon-"] { cursor: default; }
</div>
<div id="footer">
<div id="footer-text">
Last updated 2015-12-22 14:29:57 UTC
Last updated 2015-12-22 07:40:07 UTC
</div>
</div>
</body>
......
hack/verify-flags/known-flags.txt
View file @ 059c2aa7
......@@ -238,6 +238,7 @@ oidc-client-id
oidc-issuer-url
oidc-username-claim
oom-score-adj
outofdisk-transition-frequency
output-base
output-package
output-print-type
......
pkg/kubelet/kubelet.go
View file @ 059c2aa7
......@@ -196,10 +196,10 @@ func NewMainKubelet(
oomAdjuster *oom.OOMAdjuster,
serializeImagePulls bool,
containerManager cm.ContainerManager,
outOfDiskTransitionFrequency time.Duration,
flannelExperimentalOverlay bool,
nodeIP net.IP,
) (*Kubelet, error) {
if rootDirectory == "" {
return nil, fmt.Errorf("invalid root directory %q", rootDirectory)
}
......@@ -315,6 +315,8 @@ func NewMainKubelet(
flannelExperimentalOverlay: flannelExperimentalOverlay,
flannelHelper: NewFlannelHelper(),
nodeIP: nodeIP,
clock: util.RealClock{},
outOfDiskTransitionFrequency: outOfDiskTransitionFrequency,
}
if klet.flannelExperimentalOverlay {
glog.Infof("Flannel is in charge of podCIDR and overlay networking.")
......@@ -419,7 +421,7 @@ func NewMainKubelet(
SystemContainerName: systemContainer,
KubeletContainerName: resourceContainer,
}
klet.runtimeState.setRuntimeSync(time.Now())
klet.runtimeState.setRuntimeSync(klet.clock.Now())
klet.runner = klet.containerRuntime
klet.podManager = kubepod.NewBasicPodManager(kubepod.NewBasicMirrorClient(klet.kubeClient))
......@@ -654,6 +656,16 @@ type Kubelet struct {
// If non-nil, use this IP address for the node
nodeIP net.IP
// clock is an interface that provides time related functionality in a way that makes it
// easy to test the code.
clock util.Clock
// outOfDiskTransitionFrequency specifies the amount of time the kubelet has to be actually
// not out of disk before it can transition the node condition status from out-of-disk to
// not-out-of-disk. This prevents a pod that causes out-of-disk condition from repeatedly
// getting rescheduled onto the node.
outOfDiskTransitionFrequency time.Duration
}
// Validate given node IP belongs to the current host
......@@ -1614,7 +1626,7 @@ func (kl *Kubelet) makePodDataDirs(pod *api.Pod) error {
}
func (kl *Kubelet) syncPod(pod *api.Pod, mirrorPod *api.Pod, runningPod kubecontainer.Pod, updateType kubetypes.SyncPodType) (syncErr error) {
start := time.Now()
start := kl.clock.Now()
var firstSeenTime time.Time
if firstSeenTimeStr, ok := pod.Annotations[kubetypes.ConfigFirstSeenAnnotationKey]; !ok {
glog.V(3).Infof("First seen time not recorded for pod %q", pod.UID)
......@@ -1945,7 +1957,6 @@ func (kl *Kubelet) cleanupTerminatedPods(pods []*api.Pod, runningPods []*kubecon
// pastActiveDeadline returns true if the pod has been active for more than
// ActiveDeadlineSeconds.
func (kl *Kubelet) pastActiveDeadline(pod *api.Pod) bool {
now := unversioned.Now()
if pod.Spec.ActiveDeadlineSeconds != nil {
podStatus, ok := kl.statusManager.GetPodStatus(pod.UID)
if !ok {
......@@ -1953,7 +1964,7 @@ func (kl *Kubelet) pastActiveDeadline(pod *api.Pod) bool {
}
if !podStatus.StartTime.IsZero() {
startTime := podStatus.StartTime.Time
duration := now.Time.Sub(startTime)
duration := kl.clock.Since(startTime)
allowedDuration := time.Duration(*pod.Spec.ActiveDeadlineSeconds) * time.Second
if duration >= allowedDuration {
return true
......@@ -2315,7 +2326,7 @@ func (kl *Kubelet) syncLoop(updates <-chan kubetypes.PodUpdate, handler SyncHand
func (kl *Kubelet) syncLoopIteration(updates <-chan kubetypes.PodUpdate, handler SyncHandler,
syncCh <-chan time.Time, housekeepingCh <-chan time.Time, plegCh <-chan *pleg.PodLifecycleEvent) bool {
kl.syncLoopMonitor.Store(time.Now())
kl.syncLoopMonitor.Store(kl.clock.Now())
select {
case u, open := <-updates:
if !open {
......@@ -2351,7 +2362,7 @@ func (kl *Kubelet) syncLoopIteration(updates <-chan kubetypes.PodUpdate, handler
}
glog.V(2).Infof("SyncLoop (PLEG): %q, event: %#v", format.Pod(pod), e)
// Force the container runtime cache to update.
if err := kl.runtimeCache.ForceUpdateIfOlder(time.Now()); err != nil {
if err := kl.runtimeCache.ForceUpdateIfOlder(kl.clock.Now()); err != nil {
glog.Errorf("SyncLoop: unable to update runtime cache")
// TODO (yujuhong): should we delay the sync until container
// runtime can be updated?
......@@ -2382,7 +2393,7 @@ func (kl *Kubelet) syncLoopIteration(updates <-chan kubetypes.PodUpdate, handler
}
}
}
kl.syncLoopMonitor.Store(time.Now())
kl.syncLoopMonitor.Store(kl.clock.Now())
return true
}
......@@ -2411,7 +2422,7 @@ func (kl *Kubelet) handleMirrorPod(mirrorPod *api.Pod, start time.Time) {
}
func (kl *Kubelet) HandlePodAdditions(pods []*api.Pod) {
start := time.Now()
start := kl.clock.Now()
sort.Sort(podsByCreationTime(pods))
for _, pod := range pods {
kl.podManager.AddPod(pod)
......@@ -2437,7 +2448,7 @@ func (kl *Kubelet) HandlePodAdditions(pods []*api.Pod) {
}
func (kl *Kubelet) HandlePodUpdates(pods []*api.Pod) {
start := time.Now()
start := kl.clock.Now()
for _, pod := range pods {
kl.podManager.UpdatePod(pod)
if kubepod.IsMirrorPod(pod) {
......@@ -2452,7 +2463,7 @@ func (kl *Kubelet) HandlePodUpdates(pods []*api.Pod) {
}
func (kl *Kubelet) HandlePodDeletions(pods []*api.Pod) {
start := time.Now()
start := kl.clock.Now()
for _, pod := range pods {
kl.podManager.DeletePod(pod)
if kubepod.IsMirrorPod(pod) {
......@@ -2469,7 +2480,7 @@ func (kl *Kubelet) HandlePodDeletions(pods []*api.Pod) {
}
func (kl *Kubelet) HandlePodSyncs(pods []*api.Pod) {
start := time.Now()
start := kl.clock.Now()
for _, pod := range pods {
mirrorPod, _ := kl.podManager.GetMirrorPodByPod(pod)
kl.dispatchWork(pod, kubetypes.SyncPodSync, mirrorPod, start)
......@@ -2617,7 +2628,7 @@ func (kl *Kubelet) updateRuntimeUp() {
return
}
kl.oneTimeInitializer.Do(kl.initializeRuntimeDependentModules)
kl.runtimeState.setRuntimeSync(time.Now())
kl.runtimeState.setRuntimeSync(kl.clock.Now())
}
func (kl *Kubelet) reconcileCBR0(podCIDR string) error {
......@@ -2821,7 +2832,7 @@ func (kl *Kubelet) setNodeReadyCondition(node *api.Node) {
// NOTE(aaronlevy): NodeReady condition needs to be the last in the list of node conditions.
// This is due to an issue with version skewed kubelet and master components.
// ref: https://github.com/kubernetes/kubernetes/issues/16961
currentTime := unversioned.Now()
currentTime := unversioned.NewTime(kl.clock.Now())
var newNodeReadyCondition api.NodeCondition
if rs := kl.runtimeState.errors(); len(rs) == 0 {
newNodeReadyCondition = api.NodeCondition{
......@@ -2871,7 +2882,7 @@ func (kl *Kubelet) setNodeReadyCondition(node *api.Node) {
// Set OODcondition for the node.
func (kl *Kubelet) setNodeOODCondition(node *api.Node) {
currentTime := unversioned.Now()
currentTime := unversioned.NewTime(kl.clock.Now())
var nodeOODCondition *api.NodeCondition
// Check if NodeOutOfDisk condition already exists and if it does, just pick it up for update.
......@@ -2885,9 +2896,8 @@ func (kl *Kubelet) setNodeOODCondition(node *api.Node) {
// If the NodeOutOfDisk condition doesn't exist, create one.
if nodeOODCondition == nil {
nodeOODCondition = &api.NodeCondition{
Type: api.NodeOutOfDisk,
Status: api.ConditionUnknown,
LastTransitionTime: currentTime,
Type: api.NodeOutOfDisk,
Status: api.ConditionUnknown,
}
// nodeOODCondition cannot be appended to node.Status.Conditions here because it gets
// copied to the slice. So if we append nodeOODCondition to the slice here none of the
......@@ -2914,11 +2924,18 @@ func (kl *Kubelet) setNodeOODCondition(node *api.Node) {
}
} else {
if nodeOODCondition.Status != api.ConditionFalse {
nodeOODCondition.Status = api.ConditionFalse
nodeOODCondition.Reason = "KubeletHasSufficientDisk"
nodeOODCondition.Message = "kubelet has sufficient disk space available"
nodeOODCondition.LastTransitionTime = currentTime
kl.recordNodeStatusEvent(api.EventTypeNormal, "NodeHasSufficientDisk")
// Update the out of disk condition when the condition status is unknown even if we
// are within the outOfDiskTransitionFrequency duration. We do this to set the
// condition status correctly at kubelet startup.
if nodeOODCondition.Status == api.ConditionUnknown || kl.clock.Since(nodeOODCondition.LastTransitionTime.Time) >= kl.outOfDiskTransitionFrequency {
nodeOODCondition.Status = api.ConditionFalse
nodeOODCondition.Reason = "KubeletHasSufficientDisk"
nodeOODCondition.Message = "kubelet has sufficient disk space available"
nodeOODCondition.LastTransitionTime = currentTime
kl.recordNodeStatusEvent(api.EventTypeNormal, "NodeHasSufficientDisk")
} else {
glog.Infof("Node condition status for OutOfDisk is false, but last transition time is less than %s", kl.outOfDiskTransitionFrequency)
}
}
}
......@@ -3092,7 +3109,7 @@ func GetPhase(spec *api.PodSpec, info []api.ContainerStatus) api.PodPhase {
// after refactoring, modify them later.
func (kl *Kubelet) generatePodStatus(pod *api.Pod) (api.PodStatus, error) {
start := time.Now()
start := kl.clock.Now()
defer func() {
metrics.PodStatusLatency.Observe(metrics.SinceInMicroseconds(start))
}()
......
pkg/kubemark/hollow_kubelet.go
View file @ 059c2aa7
......@@ -70,6 +70,7 @@ func NewHollowKubelet(
1*time.Minute, /* MinimumGCAge */
10*time.Second, /* NodeStatusUpdateFrequency */
10*time.Second, /* SyncFrequency */
5*time.Minute, /* OutOfDiskTransitionFrequency */
40, /* MaxPods */
containerManager,
nil,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment