Ignore "no such container" error when generating pod status

pkg/kubelet/dockertools/manager.go

@@ -2110,10 +2110,22 @@ func (dm *DockerManager) GetPodStatus(uid types.UID, name, namespace string) (*k
 		if dockerName.PodUID != uid {
 			continue
 		}
-
 		result, ip, err := dm.inspectContainer(c.ID, name, namespace)
 		if err != nil {
-			return podStatus, err
+			if _, ok := err.(*docker.NoSuchContainer); ok {
+				// https://github.com/kubernetes/kubernetes/issues/22541
+				// Sometimes when docker's state is corrupt, a container can be listed
+				// but couldn't be inspected. We fake a status for this container so
+				// that we can still return a status for the pod to sync.
+				result = &kubecontainer.ContainerStatus{
+					ID:    kubecontainer.DockerID(c.ID).ContainerID(),
+					Name:  dockerName.ContainerName,
+					State: kubecontainer.ContainerStateUnknown,
+				}
+				glog.Errorf("Unable to inspect container %q: %v", c.ID, err)
+			} else {
+				return podStatus, err
+			}
 		}
 		containerStatuses = append(containerStatuses, result)
 		if ip != "" {

pkg/kubelet/dockertools/manager_test.go

@@ -1887,3 +1887,55 @@ func TestCheckVersionCompatibility(t *testing.T) {
 		}
 	}
 }
+
+func TestGetPodStatusNoSuchContainer(t *testing.T) {
+	const (
+		noSuchContainerID = "nosuchcontainer"
+		infraContainerID  = "9876"
+	)
+	dm, fakeDocker := newTestDockerManager()
+	pod := &api.Pod{
+		ObjectMeta: api.ObjectMeta{
+			UID:       "12345678",
+			Name:      "foo",
+			Namespace: "new",
+		},
+		Spec: api.PodSpec{
+			Containers: []api.Container{{Name: "nosuchcontainer"}},
+		},
+	}
+
+	fakeDocker.SetFakeContainers([]*docker.Container{
+		{
+			ID:   noSuchContainerID,
+			Name: "/k8s_nosuchcontainer_foo_new_12345678_42",
+			State: docker.State{
+				ExitCode:   0,
+				StartedAt:  time.Now(),
+				FinishedAt: time.Now(),
+				Running:    false,
+			},
+		},
+		{
+			ID:   infraContainerID,
+			Name: "/k8s_POD." + strconv.FormatUint(generatePodInfraContainerHash(pod), 16) + "_foo_new_12345678_42",
+			State: docker.State{
+				ExitCode:   0,
+				StartedAt:  time.Now(),
+				FinishedAt: time.Now(),
+				Running:    false,
+			},
+		}})
+
+	fakeDocker.Errors = map[string]error{"inspect": &docker.NoSuchContainer{}}
+	runSyncPod(t, dm, fakeDocker, pod, nil, false)
+
+	// Verify that we will try to start new contrainers even if the inspections
+	// failed.
+	verifyCalls(t, fakeDocker, []string{
+		// Start a new infra container.
+		"create", "start", "inspect_container", "inspect_container",
+		// Start a new container.
+		"create", "start", "inspect_container",
+	})
+}