Commit dcaf8e82 authored by Kubernetes Submit Queue's avatar Kubernetes Submit Queue Committed by GitHub

Merge pull request #53167 from dashpole/fix_init_container

Automatic merge from submit-queue (batch tested with PRs 44596, 52708, 53163, 53167, 52692). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>. Do not GC exited containers in running pods This fixes a regression introduced by #45896, and was identified by #52462. This bug causes the kubelet to garbage collect exited containers in a running pod. This manifests in strange and confusing state when viewing the cluster. For example, it can show running pods as having no init container (see #52462), if that container has exited and been removed. This PR solves this problem by only removing containers and sandboxes from terminated pods. The important line change is: ` if cgc.podDeletionProvider.IsPodDeleted(podUID) || evictNonDeletedPods {` ---> `if cgc.podStateProvider.IsPodDeleted(podUID) || (cgc.podStateProvider.IsPodTerminated(podUID) && evictTerminatedPods) {` cc @MrHohn @yujuhong @kubernetes/sig-node-bugs ```release-note BugFix: Exited containers are not Garbage Collected by the kubelet while the pod is running ```
parents 703b0f4b 4300c75d
...@@ -820,6 +820,16 @@ func (kl *Kubelet) podIsTerminated(pod *v1.Pod) bool { ...@@ -820,6 +820,16 @@ func (kl *Kubelet) podIsTerminated(pod *v1.Pod) bool {
return status.Phase == v1.PodFailed || status.Phase == v1.PodSucceeded || (pod.DeletionTimestamp != nil && notRunning(status.ContainerStatuses)) return status.Phase == v1.PodFailed || status.Phase == v1.PodSucceeded || (pod.DeletionTimestamp != nil && notRunning(status.ContainerStatuses))
} }
// IsPodTerminated returns trus if the pod with the provided UID is in a terminated state ("Failed" or "Succeeded")
// or if the pod has been deleted or removed
func (kl *Kubelet) IsPodTerminated(uid types.UID) bool {
pod, podFound := kl.podManager.GetPodByUID(uid)
if !podFound {
return true
}
return kl.podIsTerminated(pod)
}
// IsPodDeleted returns true if the pod is deleted. For the pod to be deleted, either: // IsPodDeleted returns true if the pod is deleted. For the pod to be deleted, either:
// 1. The pod object is deleted // 1. The pod object is deleted
// 2. The pod's status is evicted // 2. The pod's status is evicted
......
...@@ -43,18 +43,25 @@ func (f *fakeHTTP) Get(url string) (*http.Response, error) { ...@@ -43,18 +43,25 @@ func (f *fakeHTTP) Get(url string) (*http.Response, error) {
return nil, f.err return nil, f.err
} }
type fakePodDeletionProvider struct { type fakePodStateProvider struct {
pods map[types.UID]struct{} existingPods map[types.UID]struct{}
runningPods map[types.UID]struct{}
} }
func newFakePodDeletionProvider() *fakePodDeletionProvider { func newFakePodStateProvider() *fakePodStateProvider {
return &fakePodDeletionProvider{ return &fakePodStateProvider{
pods: make(map[types.UID]struct{}), existingPods: make(map[types.UID]struct{}),
runningPods: make(map[types.UID]struct{}),
} }
} }
func (f *fakePodDeletionProvider) IsPodDeleted(uid types.UID) bool { func (f *fakePodStateProvider) IsPodDeleted(uid types.UID) bool {
_, found := f.pods[uid] _, found := f.existingPods[uid]
return !found
}
func (f *fakePodStateProvider) IsPodTerminated(uid types.UID) bool {
_, found := f.runningPods[uid]
return !found return !found
} }
...@@ -79,7 +86,7 @@ func NewFakeKubeRuntimeManager(runtimeService internalapi.RuntimeService, imageS ...@@ -79,7 +86,7 @@ func NewFakeKubeRuntimeManager(runtimeService internalapi.RuntimeService, imageS
return nil, err return nil, err
} }
kubeRuntimeManager.containerGC = NewContainerGC(runtimeService, newFakePodDeletionProvider(), kubeRuntimeManager) kubeRuntimeManager.containerGC = NewContainerGC(runtimeService, newFakePodStateProvider(), kubeRuntimeManager)
kubeRuntimeManager.runtimeName = typedVersion.RuntimeName kubeRuntimeManager.runtimeName = typedVersion.RuntimeName
kubeRuntimeManager.imagePuller = images.NewImageManager( kubeRuntimeManager.imagePuller = images.NewImageManager(
kubecontainer.FilterEventRecorder(recorder), kubecontainer.FilterEventRecorder(recorder),
......
...@@ -33,17 +33,17 @@ import ( ...@@ -33,17 +33,17 @@ import (
// containerGC is the manager of garbage collection. // containerGC is the manager of garbage collection.
type containerGC struct { type containerGC struct {
client internalapi.RuntimeService client internalapi.RuntimeService
manager *kubeGenericRuntimeManager manager *kubeGenericRuntimeManager
podDeletionProvider podDeletionProvider podStateProvider podStateProvider
} }
// NewContainerGC creates a new containerGC. // NewContainerGC creates a new containerGC.
func NewContainerGC(client internalapi.RuntimeService, podDeletionProvider podDeletionProvider, manager *kubeGenericRuntimeManager) *containerGC { func NewContainerGC(client internalapi.RuntimeService, podStateProvider podStateProvider, manager *kubeGenericRuntimeManager) *containerGC {
return &containerGC{ return &containerGC{
client: client, client: client,
manager: manager, manager: manager,
podDeletionProvider: podDeletionProvider, podStateProvider: podStateProvider,
} }
} }
...@@ -201,7 +201,7 @@ func (cgc *containerGC) evictableContainers(minAge time.Duration) (containersByE ...@@ -201,7 +201,7 @@ func (cgc *containerGC) evictableContainers(minAge time.Duration) (containersByE
} }
// evict all containers that are evictable // evict all containers that are evictable
func (cgc *containerGC) evictContainers(gcPolicy kubecontainer.ContainerGCPolicy, allSourcesReady bool, evictNonDeletedPods bool) error { func (cgc *containerGC) evictContainers(gcPolicy kubecontainer.ContainerGCPolicy, allSourcesReady bool, evictTerminatedPods bool) error {
// Separate containers by evict units. // Separate containers by evict units.
evictUnits, err := cgc.evictableContainers(gcPolicy.MinAge) evictUnits, err := cgc.evictableContainers(gcPolicy.MinAge)
if err != nil { if err != nil {
...@@ -211,7 +211,7 @@ func (cgc *containerGC) evictContainers(gcPolicy kubecontainer.ContainerGCPolicy ...@@ -211,7 +211,7 @@ func (cgc *containerGC) evictContainers(gcPolicy kubecontainer.ContainerGCPolicy
// Remove deleted pod containers if all sources are ready. // Remove deleted pod containers if all sources are ready.
if allSourcesReady { if allSourcesReady {
for key, unit := range evictUnits { for key, unit := range evictUnits {
if cgc.podDeletionProvider.IsPodDeleted(key.uid) || evictNonDeletedPods { if cgc.podStateProvider.IsPodDeleted(key.uid) || (cgc.podStateProvider.IsPodTerminated(key.uid) && evictTerminatedPods) {
cgc.removeOldestN(unit, len(unit)) // Remove all. cgc.removeOldestN(unit, len(unit)) // Remove all.
delete(evictUnits, key) delete(evictUnits, key)
} }
...@@ -253,7 +253,7 @@ func (cgc *containerGC) evictContainers(gcPolicy kubecontainer.ContainerGCPolicy ...@@ -253,7 +253,7 @@ func (cgc *containerGC) evictContainers(gcPolicy kubecontainer.ContainerGCPolicy
// 2. contains no containers. // 2. contains no containers.
// 3. belong to a non-existent (i.e., already removed) pod, or is not the // 3. belong to a non-existent (i.e., already removed) pod, or is not the
// most recently created sandbox for the pod. // most recently created sandbox for the pod.
func (cgc *containerGC) evictSandboxes(evictNonDeletedPods bool) error { func (cgc *containerGC) evictSandboxes(evictTerminatedPods bool) error {
containers, err := cgc.manager.getKubeletContainers(true) containers, err := cgc.manager.getKubeletContainers(true)
if err != nil { if err != nil {
return err return err
...@@ -297,7 +297,7 @@ func (cgc *containerGC) evictSandboxes(evictNonDeletedPods bool) error { ...@@ -297,7 +297,7 @@ func (cgc *containerGC) evictSandboxes(evictNonDeletedPods bool) error {
} }
for podUID, sandboxes := range sandboxesByPod { for podUID, sandboxes := range sandboxesByPod {
if cgc.podDeletionProvider.IsPodDeleted(podUID) || evictNonDeletedPods { if cgc.podStateProvider.IsPodDeleted(podUID) || (cgc.podStateProvider.IsPodTerminated(podUID) && evictTerminatedPods) {
// Remove all evictable sandboxes if the pod has been removed. // Remove all evictable sandboxes if the pod has been removed.
// Note that the latest dead sandbox is also removed if there is // Note that the latest dead sandbox is also removed if there is
// already an active one. // already an active one.
...@@ -323,7 +323,7 @@ func (cgc *containerGC) evictPodLogsDirectories(allSourcesReady bool) error { ...@@ -323,7 +323,7 @@ func (cgc *containerGC) evictPodLogsDirectories(allSourcesReady bool) error {
for _, dir := range dirs { for _, dir := range dirs {
name := dir.Name() name := dir.Name()
podUID := types.UID(name) podUID := types.UID(name)
if !cgc.podDeletionProvider.IsPodDeleted(podUID) { if !cgc.podStateProvider.IsPodDeleted(podUID) {
continue continue
} }
err := osInterface.RemoveAll(filepath.Join(podLogsRootDirectory, name)) err := osInterface.RemoveAll(filepath.Join(podLogsRootDirectory, name))
...@@ -357,14 +357,14 @@ func (cgc *containerGC) evictPodLogsDirectories(allSourcesReady bool) error { ...@@ -357,14 +357,14 @@ func (cgc *containerGC) evictPodLogsDirectories(allSourcesReady bool) error {
// * removes oldest dead containers by enforcing gcPolicy.MaxContainers. // * removes oldest dead containers by enforcing gcPolicy.MaxContainers.
// * gets evictable sandboxes which are not ready and contains no containers. // * gets evictable sandboxes which are not ready and contains no containers.
// * removes evictable sandboxes. // * removes evictable sandboxes.
func (cgc *containerGC) GarbageCollect(gcPolicy kubecontainer.ContainerGCPolicy, allSourcesReady bool, evictNonDeletedPods bool) error { func (cgc *containerGC) GarbageCollect(gcPolicy kubecontainer.ContainerGCPolicy, allSourcesReady bool, evictTerminatedPods bool) error {
// Remove evictable containers // Remove evictable containers
if err := cgc.evictContainers(gcPolicy, allSourcesReady, evictNonDeletedPods); err != nil { if err := cgc.evictContainers(gcPolicy, allSourcesReady, evictTerminatedPods); err != nil {
return err return err
} }
// Remove sandboxes with zero containers // Remove sandboxes with zero containers
if err := cgc.evictSandboxes(evictNonDeletedPods); err != nil { if err := cgc.evictSandboxes(evictTerminatedPods); err != nil {
return err return err
} }
......
...@@ -65,9 +65,10 @@ var ( ...@@ -65,9 +65,10 @@ var (
ErrVersionNotSupported = errors.New("Runtime api version is not supported") ErrVersionNotSupported = errors.New("Runtime api version is not supported")
) )
// podDeletionProvider can determine if a pod is deleted // podStateProvider can determine if a pod is deleted ir terminated
type podDeletionProvider interface { type podStateProvider interface {
IsPodDeleted(kubetypes.UID) bool IsPodDeleted(kubetypes.UID) bool
IsPodTerminated(kubetypes.UID) bool
} }
type kubeGenericRuntimeManager struct { type kubeGenericRuntimeManager struct {
...@@ -127,7 +128,7 @@ func NewKubeGenericRuntimeManager( ...@@ -127,7 +128,7 @@ func NewKubeGenericRuntimeManager(
seccompProfileRoot string, seccompProfileRoot string,
containerRefManager *kubecontainer.RefManager, containerRefManager *kubecontainer.RefManager,
machineInfo *cadvisorapi.MachineInfo, machineInfo *cadvisorapi.MachineInfo,
podDeletionProvider podDeletionProvider, podStateProvider podStateProvider,
osInterface kubecontainer.OSInterface, osInterface kubecontainer.OSInterface,
runtimeHelper kubecontainer.RuntimeHelper, runtimeHelper kubecontainer.RuntimeHelper,
httpClient types.HttpGetter, httpClient types.HttpGetter,
...@@ -193,7 +194,7 @@ func NewKubeGenericRuntimeManager( ...@@ -193,7 +194,7 @@ func NewKubeGenericRuntimeManager(
imagePullQPS, imagePullQPS,
imagePullBurst) imagePullBurst)
kubeRuntimeManager.runner = lifecycle.NewHandlerRunner(httpClient, kubeRuntimeManager, kubeRuntimeManager) kubeRuntimeManager.runner = lifecycle.NewHandlerRunner(httpClient, kubeRuntimeManager, kubeRuntimeManager)
kubeRuntimeManager.containerGC = NewContainerGC(runtimeService, podDeletionProvider, kubeRuntimeManager) kubeRuntimeManager.containerGC = NewContainerGC(runtimeService, podStateProvider, kubeRuntimeManager)
kubeRuntimeManager.versionCache = cache.NewObjectCache( kubeRuntimeManager.versionCache = cache.NewObjectCache(
func() (interface{}, error) { func() (interface{}, error) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment