Avoid race between image pull and garbage collect

Hold the pullLock while pulling regular container images, not just the network container. Also: previously, the locks were grabbed at too-low a layer in the callstack, leaving a (narrow) window where images could be pulled and then garbage collected before being used. This change takes the existing pullLock a layer higher in the call stack so the critical regions encompass the entire find+delete and pull+use operation in both cases.

Avoid race between image pull and garbage collect
6f14e2e2 · Angus Lees · 1c9ac79e · 6f14e2e2
Commit 6f14e2e2 authored Dec 31, 2014 by Angus Lees
Hide whitespace changes
Inline Side-by-side

Showing with 10 additions and 4 deletions

kubelet.go pkg/kubelet/kubelet.go +10 -4

No files found.
--- a/pkg/kubelet/kubelet.go
+++ b/pkg/kubelet/kubelet.go
@@ -220,12 +220,13 @@ func (kl *Kubelet) GarbageCollectLoop() {
 }

 func (kl *Kubelet) getUnusedImages() ([]string, error) {
-	kl.pullLock.Lock()
-	defer kl.pullLock.Unlock()
 	return dockertools.GetUnusedImages(kl.dockerClient)
 }

 func (kl *Kubelet) GarbageCollectImages() error {
+	kl.pullLock.Lock()
+	defer kl.pullLock.Unlock()
+
 	images, err := kl.getUnusedImages()
 	if err != nil {
 		return err
@@ -633,6 +634,10 @@ func (kl *Kubelet) createNetworkContainer(pod *api.BoundPod) (dockertools.Docker
 	if err != nil {
 		glog.Errorf("Couldn't make a ref to pod %v, container %v: '%v'", pod.Name, container.Name, err)
 	}
+
+	kl.pullLock.RLock()
+	defer kl.pullLock.RUnlock()
+
 	// TODO: make this a TTL based pull (if image older than X policy, pull)
 	ok, err := kl.dockerPuller.IsImagePresent(container.Image)
 	if err != nil {
@@ -641,6 +646,7 @@ func (kl *Kubelet) createNetworkContainer(pod *api.BoundPod) (dockertools.Docker
 		}
 		return "", err
 	}
+
 	if !ok {
 		if err := kl.pullImage(container.Image, ref); err != nil {
 			return "", err
@@ -653,8 +659,6 @@ func (kl *Kubelet) createNetworkContainer(pod *api.BoundPod) (dockertools.Docker
 }

 func (kl *Kubelet) pullImage(img string, ref *api.ObjectReference) error {
-	kl.pullLock.RLock()
-	defer kl.pullLock.RUnlock()
 	if err := kl.dockerPuller.Pull(img); err != nil {
 		if ref != nil {
 			record.Eventf(ref, "failed", "failed", "Failed to pull image %s", img)
@@ -814,6 +818,8 @@ func (kl *Kubelet) syncPod(pod *api.BoundPod, dockerContainers dockertools.Docke
 		if err != nil {
 			glog.Errorf("Couldn't make a ref to pod %v, container %v: '%v'", pod.Name, container.Name, err)
 		}
+		kl.pullLock.RLock()
+		defer kl.pullLock.RUnlock()
 		if !api.IsPullNever(container.ImagePullPolicy) {
 			present, err := kl.dockerPuller.IsImagePresent(container.Image)
 			latest := dockertools.RequireLatestImage(container.Image)