Add node e2e test for Docker's live-restore

test/e2e_node/BUILD

@@ -41,6 +41,7 @@ go_library(
         "//test/e2e/perftype:go_default_library",
         "//test/e2e_node/perftype:go_default_library",
         "//vendor/github.com/blang/semver:go_default_library",
+        "//vendor/github.com/coreos/go-systemd/util:go_default_library",
         "//vendor/github.com/docker/docker/client:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
         "//vendor/github.com/google/cadvisor/client/v2:go_default_library",

test/e2e_node/docker_test.go

@@ -17,11 +17,16 @@ limitations under the License.
 package e2e_node
 
 import (
+	"fmt"
+	"strings"
+	"time"
+
 	"k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/kubernetes/test/e2e/framework"
 
 	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
 )
 
 var _ = framework.KubeDescribe("Docker features [Feature:Docker]", func() {
@@ -70,4 +75,103 @@ var _ = framework.KubeDescribe("Docker features [Feature:Docker]", func() {
 			}
 		})
 	})
+
+	Context("when live-restore is enabled [Serial] [Slow] [Disruptive]", func() {
+		It("containers should not be disrupted when the daemon shuts down and restarts", func() {
+			const (
+				podName       = "live-restore-test-pod"
+				containerName = "live-restore-test-container"
+			)
+
+			isSupported, err := isDockerLiveRestoreSupported()
+			framework.ExpectNoError(err)
+			if !isSupported {
+				framework.Skipf("Docker live-restore is not supported.")
+			}
+			isEnabled, err := isDockerLiveRestoreEnabled()
+			framework.ExpectNoError(err)
+			if !isEnabled {
+				framework.Skipf("Docker live-restore is not enabled.")
+			}
+
+			By("Create the test pod.")
+			pod := f.PodClient().CreateSync(&v1.Pod{
+				ObjectMeta: metav1.ObjectMeta{Name: podName},
+				Spec: v1.PodSpec{
+					Containers: []v1.Container{{
+						Name:  containerName,
+						Image: "gcr.io/google_containers/nginx-slim:0.7",
+					}},
+				},
+			})
+
+			By("Ensure that the container is running before Docker is down.")
+			Eventually(func() bool {
+				return isContainerRunning(pod.Status.PodIP)
+			}).Should(BeTrue())
+
+			startTime1, err := getContainerStartTime(f, podName, containerName)
+			framework.ExpectNoError(err)
+
+			By("Stop Docker daemon.")
+			framework.ExpectNoError(stopDockerDaemon())
+			isDockerDown := true
+			defer func() {
+				if isDockerDown {
+					By("Start Docker daemon.")
+					framework.ExpectNoError(startDockerDaemon())
+				}
+			}()
+
+			By("Ensure that the container is running after Docker is down.")
+			Consistently(func() bool {
+				return isContainerRunning(pod.Status.PodIP)
+			}).Should(BeTrue())
+
+			By("Start Docker daemon.")
+			framework.ExpectNoError(startDockerDaemon())
+			isDockerDown = false
+
+			By("Ensure that the container is running after Docker has restarted.")
+			Consistently(func() bool {
+				return isContainerRunning(pod.Status.PodIP)
+			}).Should(BeTrue())
+
+			By("Ensure that the container has not been restarted after Docker is restarted.")
+			Consistently(func() bool {
+				startTime2, err := getContainerStartTime(f, podName, containerName)
+				framework.ExpectNoError(err)
+				return startTime1 == startTime2
+			}, 3*time.Second, time.Second).Should(BeTrue())
+		})
+	})
 })
+
+// isContainerRunning returns true if the container is running by checking
+// whether the server is responding, and false otherwise.
+func isContainerRunning(podIP string) bool {
+	output, err := runCommand("curl", podIP)
+	if err != nil {
+		return false
+	}
+	return strings.Contains(output, "Welcome to nginx!")
+}
+
+// getContainerStartTime returns the start time of the container with the
+// containerName of the pod having the podName.
+func getContainerStartTime(f *framework.Framework, podName, containerName string) (time.Time, error) {
+	pod, err := f.PodClient().Get(podName, metav1.GetOptions{})
+	if err != nil {
+		return time.Time{}, fmt.Errorf("failed to get pod %q: %v", podName, err)
+	}
+	for _, status := range pod.Status.ContainerStatuses {
+		if status.Name != containerName {
+			continue
+		}
+		if status.State.Running == nil {
+			return time.Time{}, fmt.Errorf("%v/%v is not running", podName, containerName)
+		}
+		return status.State.Running.StartedAt.Time, nil
+	}
+	return time.Time{}, fmt.Errorf("failed to find %v/%v", podName, containerName)
+}

test/e2e_node/docker_util.go

@@ -21,11 +21,13 @@ import (
 	"fmt"
 
 	"github.com/blang/semver"
+	systemdutil "github.com/coreos/go-systemd/util"
 	"github.com/docker/docker/client"
 )
 
 const (
-	defaultDockerEndpoint = "unix:///var/run/docker.sock"
+	defaultDockerEndpoint  = "unix:///var/run/docker.sock"
+	dockerDaemonConfigName = "/etc/docker/daemon.json"
 )
 
 // getDockerAPIVersion returns the Docker's API version.
@@ -36,7 +38,7 @@ func getDockerAPIVersion() (semver.Version, error) {
 	}
 	version, err := c.ServerVersion(context.Background())
 	if err != nil {
-		return semver.Version{}, fmt.Errorf("failed to get docker info: %v", err)
+		return semver.Version{}, fmt.Errorf("failed to get docker server version: %v", err)
 	}
 	return semver.MustParse(version.APIVersion + ".0"), nil
 }
@@ -60,3 +62,51 @@ func isDockerNoNewPrivilegesSupported() (bool, error) {
 	}
 	return version.GTE(semver.MustParse("1.23.0")), nil
 }
+
+// isDockerLiveRestoreSupported returns true if live-restore is supported in
+// the current Docker version.
+func isDockerLiveRestoreSupported() (bool, error) {
+	version, err := getDockerAPIVersion()
+	if err != nil {
+		return false, err
+	}
+	return version.GTE(semver.MustParse("1.26.0")), nil
+}
+
+// isDockerLiveRestoreEnabled returns true if live-restore is enabled in the
+// Docker.
+func isDockerLiveRestoreEnabled() (bool, error) {
+	c, err := client.NewClient(defaultDockerEndpoint, "", nil, nil)
+	if err != nil {
+		return false, fmt.Errorf("failed to create docker client: %v", err)
+	}
+	info, err := c.Info(context.Background())
+	if err != nil {
+		return false, fmt.Errorf("failed to get docker info: %v", err)
+	}
+	return info.LiveRestoreEnabled, nil
+}
+
+// stopDockerDaemon starts the Docker daemon.
+func startDockerDaemon() error {
+	switch {
+	case systemdutil.IsRunningSystemd():
+		_, err := runCommand("systemctl", "start", "docker")
+		return err
+	default:
+		_, err := runCommand("service", "docker", "start")
+		return err
+	}
+}
+
+// stopDockerDaemon stops the Docker daemon.
+func stopDockerDaemon() error {
+	switch {
+	case systemdutil.IsRunningSystemd():
+		_, err := runCommand("systemctl", "stop", "docker")
+		return err
+	default:
+		_, err := runCommand("service", "docker", "stop")
+		return err
+	}
+}

test/e2e_node/gke_environment_test.go

@@ -342,16 +342,6 @@ var _ = framework.KubeDescribe("GKE system requirements [Conformance] [Feature:G
 	})
 })
 
-// runCommand runs the cmd and returns the combined stdout and stderr, or an
-// error if the command failed.
-func runCommand(cmd ...string) (string, error) {
-	output, err := exec.Command(cmd[0], cmd[1:]...).CombinedOutput()
-	if err != nil {
-		return "", fmt.Errorf("failed to run %q: %s (%s)", strings.Join(cmd, " "), err, output)
-	}
-	return string(output), nil
-}
-
 // getPPID returns the PPID for the pid.
 func getPPID(pid int) (int, error) {
 	statusFile := "/proc/" + strconv.Itoa(pid) + "/status"

test/e2e_node/jenkins/cos-init-live-restore.yaml

+#cloud-config
+
+runcmd:
+  - echo '{"live-restore":true}' > /etc/docker/daemon.json
+  - systemctl restart docker
+  - mount /tmp /tmp -o remount,exec,suid
+  - usermod -a -G docker jenkins
+  - mkdir -p /var/lib/kubelet
+  - mkdir -p /home/kubernetes/containerized_mounter/rootfs
+  - mount --bind /home/kubernetes/containerized_mounter/ /home/kubernetes/containerized_mounter/
+  - mount -o remount, exec /home/kubernetes/containerized_mounter/
+  - wget https://storage.googleapis.com/kubernetes-release/gci-mounter/mounter.tar -O /tmp/mounter.tar
+  - tar xvf /tmp/mounter.tar -C /home/kubernetes/containerized_mounter/rootfs
+  - mkdir -p /home/kubernetes/containerized_mounter/rootfs/var/lib/kubelet
+  - mount --rbind /var/lib/kubelet /home/kubernetes/containerized_mounter/rootfs/var/lib/kubelet
+  - mount --make-rshared /home/kubernetes/containerized_mounter/rootfs/var/lib/kubelet
+  - mount --bind /proc /home/kubernetes/containerized_mounter/rootfs/proc
+  - mount --bind /dev /home/kubernetes/containerized_mounter/rootfs/dev
+  - rm /tmp/mounter.tar

test/e2e_node/jenkins/image-config.yaml

@@ -19,4 +19,4 @@ images:
   cos-beta:
     image_regex: cos-beta-60-9592-70-0 # docker 1.13.1
     project: cos-cloud
-    metadata: "user-data<test/e2e_node/jenkins/gci-init.yaml,gci-update-strategy=update_disabled"
+    metadata: "user-data<test/e2e_node/jenkins/cos-init-live-restore.yaml,gci-update-strategy=update_disabled"

test/e2e_node/util.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
+	"os/exec"
 	"reflect"
 	"strings"
 	"time"
@@ -325,3 +326,13 @@ func newJSONEncoder(groupName string) (runtime.Encoder, error) {
 	// the "best" version supposedly comes first in the list returned from api.Registry.EnabledVersionsForGroup
 	return api.Codecs.EncoderForVersion(info.Serializer, versions[0]), nil
 }
+
+// runCommand runs the cmd and returns the combined stdout and stderr, or an
+// error if the command failed.
+func runCommand(cmd ...string) (string, error) {
+	output, err := exec.Command(cmd[0], cmd[1:]...).CombinedOutput()
+	if err != nil {
+		return "", fmt.Errorf("failed to run %q: %s (%s)", strings.Join(cmd, " "), err, output)
+	}
+	return string(output), nil
+}