Skip to content

K
k3s
Commit c186afd4 authored by k8s-merge-robot's avatar k8s-merge-robot Committed by GitHub
Browse files
Options

Merge pull request #29122 from ronnielai/image-gc-1-1

Automatic merge from submit-queue Syncing imaging pulling backoff logic - Syncing the backoff logic in the parallel image puller and the sequential image puller to prepare for merging the two pullers into one. - Moving image error definitions under kubelet/images
parents 8bc8cfd1 ee4822e4
Show whitespace changes
Inline Side-by-side
Showing with 46 additions and 37 deletions
pkg/kubelet/container/sync_result.go
View file @ c186afd4
...@@ -29,24 +29,9 @@ import ( ...@@ -29,24 +29,9 @@ import (
var ErrCrashLoopBackOff = errors.New("CrashLoopBackOff") var ErrCrashLoopBackOff = errors.New("CrashLoopBackOff")
var ( var (
// Container image pull failed, kubelet is backing off image pull
ErrImagePullBackOff = errors.New("ImagePullBackOff")
// Unable to inspect image
ErrImageInspect = errors.New("ImageInspectError")
// General image pull error
ErrImagePull = errors.New("ErrImagePull")
// Required Image is absent on host and PullPolicy is NeverPullImage
ErrImageNeverPull = errors.New("ErrImageNeverPull")
// ErrContainerNotFound returned when a container in the given pod with the // ErrContainerNotFound returned when a container in the given pod with the
// given container name was not found, amongst those managed by the kubelet. // given container name was not found, amongst those managed by the kubelet.
ErrContainerNotFound = errors.New("no matching container") ErrContainerNotFound = errors.New("no matching container")
// Get http error when pulling image from registry
RegistryUnavailable = errors.New("RegistryUnavailable")
) )
var ( var (
......
pkg/kubelet/dockertools/docker.go
View file @ c186afd4
...@@ -33,6 +33,7 @@ import ( ...@@ -33,6 +33,7 @@ import (
"k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/credentialprovider" "k8s.io/kubernetes/pkg/credentialprovider"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
"k8s.io/kubernetes/pkg/kubelet/images"
"k8s.io/kubernetes/pkg/kubelet/leaky" "k8s.io/kubernetes/pkg/kubelet/leaky"
"k8s.io/kubernetes/pkg/types" "k8s.io/kubernetes/pkg/types"
utilerrors "k8s.io/kubernetes/pkg/util/errors" utilerrors "k8s.io/kubernetes/pkg/util/errors"
...@@ -142,7 +143,7 @@ func filterHTTPError(err error, image string) error { ...@@ -142,7 +143,7 @@ func filterHTTPError(err error, image string) error {
jerr.Code == http.StatusServiceUnavailable || jerr.Code == http.StatusServiceUnavailable ||
jerr.Code == http.StatusGatewayTimeout) { jerr.Code == http.StatusGatewayTimeout) {
glog.V(2).Infof("Pulling image %q failed: %v", image, err) glog.V(2).Infof("Pulling image %q failed: %v", image, err)
return kubecontainer.RegistryUnavailable return images.RegistryUnavailable
} else { } else {
return err return err
} }
......
pkg/kubelet/dockertools/docker_manager_test.go
View file @ c186afd4
...@@ -44,6 +44,7 @@ import ( ...@@ -44,6 +44,7 @@ import (
"k8s.io/kubernetes/pkg/client/record" "k8s.io/kubernetes/pkg/client/record"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
containertest "k8s.io/kubernetes/pkg/kubelet/container/testing" containertest "k8s.io/kubernetes/pkg/kubelet/container/testing"
"k8s.io/kubernetes/pkg/kubelet/images"
"k8s.io/kubernetes/pkg/kubelet/network" "k8s.io/kubernetes/pkg/kubelet/network"
"k8s.io/kubernetes/pkg/kubelet/network/mock_network" "k8s.io/kubernetes/pkg/kubelet/network/mock_network"
nettest "k8s.io/kubernetes/pkg/kubelet/network/testing" nettest "k8s.io/kubernetes/pkg/kubelet/network/testing"
...@@ -1636,7 +1637,7 @@ func TestSyncPodWithPullPolicy(t *testing.T) { ...@@ -1636,7 +1637,7 @@ func TestSyncPodWithPullPolicy(t *testing.T) {
{kubecontainer.StartContainer, "bar2", nil, ""}, {kubecontainer.StartContainer, "bar2", nil, ""},
{kubecontainer.StartContainer, "bar3", nil, ""}, {kubecontainer.StartContainer, "bar3", nil, ""},
{kubecontainer.StartContainer, "bar4", nil, ""}, {kubecontainer.StartContainer, "bar4", nil, ""},
{kubecontainer.StartContainer, "bar5", kubecontainer.ErrImageNeverPull, {kubecontainer.StartContainer, "bar5", images.ErrImageNeverPull,
"Container image \"pull_never_image\" is not present with pull policy of Never"}, "Container image \"pull_never_image\" is not present with pull policy of Never"},
} }
...@@ -1676,7 +1677,7 @@ func TestSyncPodWithFailure(t *testing.T) { ...@@ -1676,7 +1677,7 @@ func TestSyncPodWithFailure(t *testing.T) {
api.Container{Name: "bar", Image: "realImage", ImagePullPolicy: api.PullAlways}, api.Container{Name: "bar", Image: "realImage", ImagePullPolicy: api.PullAlways},
map[string]error{}, map[string]error{},
[]error{fmt.Errorf("can't pull image")}, []error{fmt.Errorf("can't pull image")},
[]*kubecontainer.SyncResult{{kubecontainer.StartContainer, "bar", kubecontainer.ErrImagePull, "can't pull image"}}, []*kubecontainer.SyncResult{{kubecontainer.StartContainer, "bar", images.ErrImagePull, "can't pull image"}},
}, },
"CreateContainerFailure": { "CreateContainerFailure": {
api.Container{Name: "bar", Image: "alreadyPresent"}, api.Container{Name: "bar", Image: "alreadyPresent"},
......
pkg/kubelet/dockertools/docker_test.go
View file @ c186afd4
...@@ -37,6 +37,7 @@ import ( ...@@ -37,6 +37,7 @@ import (
"k8s.io/kubernetes/pkg/credentialprovider" "k8s.io/kubernetes/pkg/credentialprovider"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
containertest "k8s.io/kubernetes/pkg/kubelet/container/testing" containertest "k8s.io/kubernetes/pkg/kubelet/container/testing"
"k8s.io/kubernetes/pkg/kubelet/images"
"k8s.io/kubernetes/pkg/kubelet/network" "k8s.io/kubernetes/pkg/kubelet/network"
nettest "k8s.io/kubernetes/pkg/kubelet/network/testing" nettest "k8s.io/kubernetes/pkg/kubelet/network/testing"
"k8s.io/kubernetes/pkg/types" "k8s.io/kubernetes/pkg/types"
...@@ -225,7 +226,7 @@ func TestPullWithJSONError(t *testing.T) { ...@@ -225,7 +226,7 @@ func TestPullWithJSONError(t *testing.T) {
"Bad gateway": { "Bad gateway": {
"ubuntu", "ubuntu",
&jsonmessage.JSONError{Code: 502, Message: "<!doctype html>\n<html class=\"no-js\" lang=\"\">\n <head>\n </head>\n <body>\n <h1>Oops, there was an error!</h1>\n <p>We have been contacted of this error, feel free to check out <a href=\"http://status.docker.com/\">status.docker.com</a>\n to see if there is a bigger issue.</p>\n\n </body>\n</html>"}, &jsonmessage.JSONError{Code: 502, Message: "<!doctype html>\n<html class=\"no-js\" lang=\"\">\n <head>\n </head>\n <body>\n <h1>Oops, there was an error!</h1>\n <p>We have been contacted of this error, feel free to check out <a href=\"http://status.docker.com/\">status.docker.com</a>\n to see if there is a bigger issue.</p>\n\n </body>\n</html>"},
kubecontainer.RegistryUnavailable.Error(), images.RegistryUnavailable.Error(),
}, },
} }
for i, test := range tests { for i, test := range tests {
......
pkg/kubelet/images/parallel_image_puller.go
View file @ c186afd4
...@@ -86,7 +86,7 @@ func (puller *parallelImagePuller) pullImage(pod *api.Pod, container *api.Contai ...@@ -86,7 +86,7 @@ func (puller *parallelImagePuller) pullImage(pod *api.Pod, container *api.Contai
if err != nil { if err != nil {
msg := fmt.Sprintf("Failed to inspect image %q: %v", container.Image, err) msg := fmt.Sprintf("Failed to inspect image %q: %v", container.Image, err)
puller.logIt(ref, api.EventTypeWarning, events.FailedToInspectImage, logPrefix, msg, glog.Warning) puller.logIt(ref, api.EventTypeWarning, events.FailedToInspectImage, logPrefix, msg, glog.Warning)
return kubecontainer.ErrImageInspect, msg return ErrImageInspect, msg
} }
if !shouldPullImage(container, present) { if !shouldPullImage(container, present) {
...@@ -97,7 +97,7 @@ func (puller *parallelImagePuller) pullImage(pod *api.Pod, container *api.Contai ...@@ -97,7 +97,7 @@ func (puller *parallelImagePuller) pullImage(pod *api.Pod, container *api.Contai
} else { } else {
msg := fmt.Sprintf("Container image %q is not present with pull policy of Never", container.Image) msg := fmt.Sprintf("Container image %q is not present with pull policy of Never", container.Image)
puller.logIt(ref, api.EventTypeWarning, events.ErrImageNeverPullPolicy, logPrefix, msg, glog.Warning) puller.logIt(ref, api.EventTypeWarning, events.ErrImageNeverPullPolicy, logPrefix, msg, glog.Warning)
return kubecontainer.ErrImageNeverPull, msg return ErrImageNeverPull, msg
} }
} }
...@@ -105,21 +105,20 @@ func (puller *parallelImagePuller) pullImage(pod *api.Pod, container *api.Contai ...@@ -105,21 +105,20 @@ func (puller *parallelImagePuller) pullImage(pod *api.Pod, container *api.Contai
if puller.backOff.IsInBackOffSinceUpdate(backOffKey, puller.backOff.Clock.Now()) { if puller.backOff.IsInBackOffSinceUpdate(backOffKey, puller.backOff.Clock.Now()) {
msg := fmt.Sprintf("Back-off pulling image %q", container.Image) msg := fmt.Sprintf("Back-off pulling image %q", container.Image)
puller.logIt(ref, api.EventTypeNormal, events.BackOffPullImage, logPrefix, msg, glog.Info) puller.logIt(ref, api.EventTypeNormal, events.BackOffPullImage, logPrefix, msg, glog.Info)
return kubecontainer.ErrImagePullBackOff, msg return ErrImagePullBackOff, msg
} }
puller.logIt(ref, api.EventTypeNormal, events.PullingImage, logPrefix, fmt.Sprintf("pulling image %q", container.Image), glog.Info) puller.logIt(ref, api.EventTypeNormal, events.PullingImage, logPrefix, fmt.Sprintf("pulling image %q", container.Image), glog.Info)
if err := puller.runtime.PullImage(spec, pullSecrets); err != nil { if err := puller.runtime.PullImage(spec, pullSecrets); err != nil {
puller.logIt(ref, api.EventTypeWarning, events.FailedToPullImage, logPrefix, fmt.Sprintf("Failed to pull image %q: %v", container.Image, err), glog.Warning) puller.logIt(ref, api.EventTypeWarning, events.FailedToPullImage, logPrefix, fmt.Sprintf("Failed to pull image %q: %v", container.Image, err), glog.Warning)
puller.backOff.Next(backOffKey, puller.backOff.Clock.Now()) puller.backOff.Next(backOffKey, puller.backOff.Clock.Now())
if err == kubecontainer.RegistryUnavailable { if err == RegistryUnavailable {
msg := fmt.Sprintf("image pull failed for %s because the registry is unavailable.", container.Image) msg := fmt.Sprintf("image pull failed for %s because the registry is unavailable.", container.Image)
return err, msg return err, msg
} else { } else {
return kubecontainer.ErrImagePull, err.Error() return ErrImagePull, err.Error()
} }
} }
puller.logIt(ref, api.EventTypeNormal, events.PulledImage, logPrefix, fmt.Sprintf("Successfully pulled image %q", container.Image), glog.Info) puller.logIt(ref, api.EventTypeNormal, events.PulledImage, logPrefix, fmt.Sprintf("Successfully pulled image %q", container.Image), glog.Info)
puller.backOff.DeleteEntry(backOffKey)
puller.backOff.GC() puller.backOff.GC()
return nil, "" return nil, ""
} }
pkg/kubelet/images/serialized_image_puller.go
View file @ c186afd4
...@@ -88,7 +88,7 @@ func (puller *serializedImagePuller) pullImage(pod *api.Pod, container *api.Cont ...@@ -88,7 +88,7 @@ func (puller *serializedImagePuller) pullImage(pod *api.Pod, container *api.Cont
if err != nil { if err != nil {
msg := fmt.Sprintf("Failed to inspect image %q: %v", container.Image, err) msg := fmt.Sprintf("Failed to inspect image %q: %v", container.Image, err)
puller.logIt(ref, api.EventTypeWarning, events.FailedToInspectImage, logPrefix, msg, glog.Warning) puller.logIt(ref, api.EventTypeWarning, events.FailedToInspectImage, logPrefix, msg, glog.Warning)
return kubecontainer.ErrImageInspect, msg return ErrImageInspect, msg
} }
if !shouldPullImage(container, present) { if !shouldPullImage(container, present) {
...@@ -99,15 +99,15 @@ func (puller *serializedImagePuller) pullImage(pod *api.Pod, container *api.Cont ...@@ -99,15 +99,15 @@ func (puller *serializedImagePuller) pullImage(pod *api.Pod, container *api.Cont
} else { } else {
msg := fmt.Sprintf("Container image %q is not present with pull policy of Never", container.Image) msg := fmt.Sprintf("Container image %q is not present with pull policy of Never", container.Image)
puller.logIt(ref, api.EventTypeWarning, events.ErrImageNeverPullPolicy, logPrefix, msg, glog.Warning) puller.logIt(ref, api.EventTypeWarning, events.ErrImageNeverPullPolicy, logPrefix, msg, glog.Warning)
return kubecontainer.ErrImageNeverPull, msg return ErrImageNeverPull, msg
} }
} }
backOffKey := fmt.Sprintf("%s_%s", pod.Name, container.Image) backOffKey := fmt.Sprintf("%s_%s", pod.UID, container.Image)
if puller.backOff.IsInBackOffSinceUpdate(backOffKey, puller.backOff.Clock.Now()) { if puller.backOff.IsInBackOffSinceUpdate(backOffKey, puller.backOff.Clock.Now()) {
msg := fmt.Sprintf("Back-off pulling image %q", container.Image) msg := fmt.Sprintf("Back-off pulling image %q", container.Image)
puller.logIt(ref, api.EventTypeNormal, events.BackOffPullImage, logPrefix, msg, glog.Info) puller.logIt(ref, api.EventTypeNormal, events.BackOffPullImage, logPrefix, msg, glog.Info)
return kubecontainer.ErrImagePullBackOff, msg return ErrImagePullBackOff, msg
} }
// enqueue image pull request and wait for response. // enqueue image pull request and wait for response.
...@@ -123,11 +123,11 @@ func (puller *serializedImagePuller) pullImage(pod *api.Pod, container *api.Cont ...@@ -123,11 +123,11 @@ func (puller *serializedImagePuller) pullImage(pod *api.Pod, container *api.Cont
if err = <-returnChan; err != nil { if err = <-returnChan; err != nil {
puller.logIt(ref, api.EventTypeWarning, events.FailedToPullImage, logPrefix, fmt.Sprintf("Failed to pull image %q: %v", container.Image, err), glog.Warning) puller.logIt(ref, api.EventTypeWarning, events.FailedToPullImage, logPrefix, fmt.Sprintf("Failed to pull image %q: %v", container.Image, err), glog.Warning)
puller.backOff.Next(backOffKey, puller.backOff.Clock.Now()) puller.backOff.Next(backOffKey, puller.backOff.Clock.Now())
if err == kubecontainer.RegistryUnavailable { if err == RegistryUnavailable {
msg := fmt.Sprintf("image pull failed for %s because the registry is unavailable.", container.Image) msg := fmt.Sprintf("image pull failed for %s because the registry is unavailable.", container.Image)
return err, msg return err, msg
} else { } else {
return kubecontainer.ErrImagePull, err.Error() return ErrImagePull, err.Error()
} }
} }
puller.logIt(ref, api.EventTypeNormal, events.PulledImage, logPrefix, fmt.Sprintf("Successfully pulled image %q", container.Image), glog.Info) puller.logIt(ref, api.EventTypeNormal, events.PulledImage, logPrefix, fmt.Sprintf("Successfully pulled image %q", container.Image), glog.Info)
......
pkg/kubelet/images/types.go
View file @ c186afd4
...@@ -16,7 +16,28 @@ limitations under the License. ...@@ -16,7 +16,28 @@ limitations under the License.
package images package images
import "k8s.io/kubernetes/pkg/api" import (
"errors"
"k8s.io/kubernetes/pkg/api"
)
var (
// Container image pull failed, kubelet is backing off image pull
ErrImagePullBackOff = errors.New("ImagePullBackOff")
// Unable to inspect image
ErrImageInspect = errors.New("ImageInspectError")
// General image pull error
ErrImagePull = errors.New("ErrImagePull")
// Required Image is absent on host and PullPolicy is NeverPullImage
ErrImageNeverPull = errors.New("ErrImageNeverPull")
// Get http error when pulling image from registry
RegistryUnavailable = errors.New("RegistryUnavailable")
)
// ImageManager provides an interface to manage the lifecycle of images. // ImageManager provides an interface to manage the lifecycle of images.
// Implementations of this interface are expected to deal with pulling (downloading), // Implementations of this interface are expected to deal with pulling (downloading),
......
pkg/kubelet/kubelet.go
View file @ c186afd4
...@@ -57,6 +57,7 @@ import ( ...@@ -57,6 +57,7 @@ import (
"k8s.io/kubernetes/pkg/kubelet/envvars" "k8s.io/kubernetes/pkg/kubelet/envvars"
"k8s.io/kubernetes/pkg/kubelet/events" "k8s.io/kubernetes/pkg/kubelet/events"
"k8s.io/kubernetes/pkg/kubelet/eviction" "k8s.io/kubernetes/pkg/kubelet/eviction"
"k8s.io/kubernetes/pkg/kubelet/images"
"k8s.io/kubernetes/pkg/kubelet/lifecycle" "k8s.io/kubernetes/pkg/kubelet/lifecycle"
"k8s.io/kubernetes/pkg/kubelet/metrics" "k8s.io/kubernetes/pkg/kubelet/metrics"
"k8s.io/kubernetes/pkg/kubelet/network" "k8s.io/kubernetes/pkg/kubelet/network"
...@@ -2574,9 +2575,9 @@ func (kl *Kubelet) validateContainerLogStatus(podName string, podStatus *api.Pod ...@@ -2574,9 +2575,9 @@ func (kl *Kubelet) validateContainerLogStatus(podName string, podStatus *api.Pod
case waiting != nil: case waiting != nil:
// output some info for the most common pending failures // output some info for the most common pending failures
switch reason := waiting.Reason; reason { switch reason := waiting.Reason; reason {
case kubecontainer.ErrImagePull.Error(): case images.ErrImagePull.Error():
return kubecontainer.ContainerID{}, fmt.Errorf("container %q in pod %q is waiting to start: image can't be pulled", containerName, podName) return kubecontainer.ContainerID{}, fmt.Errorf("container %q in pod %q is waiting to start: image can't be pulled", containerName, podName)
case kubecontainer.ErrImagePullBackOff.Error(): case images.ErrImagePullBackOff.Error():
return kubecontainer.ContainerID{}, fmt.Errorf("container %q in pod %q is waiting to start: trying and failing to pull image", containerName, podName) return kubecontainer.ContainerID{}, fmt.Errorf("container %q in pod %q is waiting to start: trying and failing to pull image", containerName, podName)
default: default:
return kubecontainer.ContainerID{}, fmt.Errorf("container %q in pod %q is waiting to start: %v", containerName, podName, reason) return kubecontainer.ContainerID{}, fmt.Errorf("container %q in pod %q is waiting to start: %v", containerName, podName, reason)
......
test/e2e_node/runtime_conformance_test.go
View file @ c186afd4
...@@ -22,7 +22,7 @@ import ( ...@@ -22,7 +22,7 @@ import (
"time" "time"
"k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/images"
"k8s.io/kubernetes/pkg/util" "k8s.io/kubernetes/pkg/util"
"k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/framework"
...@@ -272,8 +272,8 @@ while true; do sleep 1; done ...@@ -272,8 +272,8 @@ while true; do sleep 1; done
} }
if testCase.waiting && status.State.Waiting != nil { if testCase.waiting && status.State.Waiting != nil {
reason := status.State.Waiting.Reason reason := status.State.Waiting.Reason
return reason == kubecontainer.ErrImagePull.Error() || return reason == images.ErrImagePull.Error() ||
reason == kubecontainer.ErrImagePullBackOff.Error(), nil reason == images.ErrImagePullBackOff.Error(), nil
} }
return false, nil return false, nil
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment