Skip to content

K
k3s
Commit 2ed490e1 authored by Kubernetes Submit Queue's avatar Kubernetes Submit Queue Committed by GitHub
Browse files
Options

Merge pull request #37255 from jingxu97/Nov/nfshung

Automatic merge from submit-queue remove checking mount point in cleanupOrphanedPodDirs To avoid nfs hung problem, remove the mountpoint checking code in cleanupOrphanedPodDirs(). This removal should still be safe because it checks whether there are still directories under pod's volume and if so, do not delete the pod directory. Note: After removing the mountpoint check code in cleanupOrphanedPodDirs(), the directories might not be cleaned up in such situation. 1. delete pod, kubelet reconciler tries to unmount the volume directory successfully 2. before reconciler tries to delete the volume directory, kubelet gets retarted 3. since under pod directory, there are still volume directors exist (but not mounted), cleanupOrphanedPodDIrs() will not clean them up. Will work on a follow up PR to solve above issue.
parents 4c0781e9 041fa647
Hide whitespace changes
Inline Side-by-side
Showing with 4 additions and 17 deletions
pkg/kubelet/kubelet.go
View file @ 2ed490e1
......@@ -1851,8 +1851,8 @@ func (kl *Kubelet) syncLoopIteration(configCh <-chan kubetypes.PodUpdate, handle
}
case <-housekeepingCh:
if !kl.sourcesReady.AllReady() {
// If the sources aren't ready, skip housekeeping, as we may
// accidentally delete pods from unready sources.
// If the sources aren't ready or volume manager has not yet synced the states,
// skip housekeeping, as we may accidentally delete pods from unready sources.
glog.V(4).Infof("SyncLoop (housekeeping, skipped): sources aren't ready yet.")
} else {
glog.V(4).Infof("SyncLoop (housekeeping)")
......
pkg/kubelet/kubelet_volumes.go
View file @ 2ed490e1
......@@ -25,7 +25,6 @@ import (
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
"k8s.io/kubernetes/pkg/types"
utilerrors "k8s.io/kubernetes/pkg/util/errors"
"k8s.io/kubernetes/pkg/util/mount"
"k8s.io/kubernetes/pkg/util/sets"
"k8s.io/kubernetes/pkg/volume"
volumetypes "k8s.io/kubernetes/pkg/volume/util/types"
......@@ -105,24 +104,12 @@ func (kl *Kubelet) cleanupOrphanedPodDirs(
glog.V(3).Infof("Orphaned pod %q found, but volumes are not cleaned up", uid)
continue
}
// Check whether volume is still mounted on disk. If so, do not delete directory
// If there are still volume directories, do not delete directory
volumePaths, err := kl.getPodVolumePathListFromDisk(uid)
if err != nil {
if err != nil || len(volumePaths) > 0 {
glog.Errorf("Orphaned pod %q found, but error %v occured during reading volume dir from disk", uid, err)
continue
} else if len(volumePaths) > 0 {
for _, path := range volumePaths {
notMount, err := mount.IsNotMountPoint(path)
if err == nil && notMount {
glog.V(2).Infof("Volume path %q is no longer mounted, remove it", path)
os.Remove(path)
} else {
glog.Errorf("Orphaned pod %q found, but it might still mounted with error %v", uid, err)
}
}
continue
}
glog.V(3).Infof("Orphaned pod %q found, removing", uid)
if err := os.RemoveAll(kl.getPodDir(uid)); err != nil {
glog.Errorf("Failed to remove orphaned pod %q dir; err: %v", uid, err)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment