Nsenter unit tests

pkg/util/mount/BUILD

@@ -133,6 +133,7 @@ go_test(
         "//vendor/k8s.io/utils/exec/testing:go_default_library",
     ] + select({
         "@io_bazel_rules_go//go/platform:linux": [
+            "//pkg/util/nsenter:go_default_library",
             "//vendor/github.com/golang/glog:go_default_library",
             "//vendor/golang.org/x/sys/unix:go_default_library",
             "//vendor/k8s.io/utils/exec:go_default_library",

pkg/util/nsenter/BUILD

-load("@io_bazel_rules_go//go:def.bzl", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 
 go_library(
     name = "go_default_library",
@@ -92,3 +92,20 @@ filegroup(
     tags = ["automanaged"],
     visibility = ["//visibility:public"],
 )
+
+go_test(
+    name = "go_default_test",
+    srcs = select({
+        "@io_bazel_rules_go//go/platform:linux": [
+            "nsenter_test.go",
+        ],
+        "//conditions:default": [],
+    }),
+    embed = [":go_default_library"],
+    deps = select({
+        "@io_bazel_rules_go//go/platform:linux": [
+            "//vendor/k8s.io/utils/exec:go_default_library",
+        ],
+        "//conditions:default": [],
+    }),
+)

pkg/util/nsenter/nsenter.go

@@ -19,6 +19,8 @@ limitations under the License.
 package nsenter
 
 import (
+	"context"
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -30,9 +32,11 @@ import (
 )
 
 const (
-	hostRootFsPath = "/rootfs"
-	// hostProcMountNsPath is the default mount namespace for rootfs
-	hostProcMountNsPath = "/rootfs/proc/1/ns/mnt"
+	// DefaultHostRootFsPath is path to host's filesystem mounted into container
+	// with kubelet.
+	DefaultHostRootFsPath = "/rootfs"
+	// mountNsPath is the default mount namespace of the host
+	mountNsPath = "/proc/1/ns/mnt"
 	// nsenterPath is the default nsenter command
 	nsenterPath = "nsenter"
 )
@@ -65,30 +69,46 @@ const (
 type Nsenter struct {
 	// a map of commands to their paths on the host filesystem
 	paths map[string]string
+
+	// Path to the host filesystem, typically "/rootfs". Used only for testing.
+	hostRootFsPath string
+
+	// Exec implementation, used only for testing
+	executor exec.Interface
 }
 
 // NewNsenter constructs a new instance of Nsenter
-func NewNsenter() (*Nsenter, error) {
+func NewNsenter(hostRootFsPath string, executor exec.Interface) (*Nsenter, error) {
 	ne := &Nsenter{
-		paths: map[string]string{
-			"mount":       "",
-			"findmnt":     "",
-			"umount":      "",
-			"systemd-run": "",
-			"stat":        "",
-			"touch":       "",
-			"mkdir":       "",
-			"ls":          "",
-			"sh":          "",
-			"chmod":       "",
-		},
+		hostRootFsPath: hostRootFsPath,
+		executor:       executor,
+	}
+	if err := ne.initPaths(); err != nil {
+		return nil, err
+	}
+	return ne, nil
+}
+
+func (ne *Nsenter) initPaths() error {
+	ne.paths = map[string]string{}
+	binaries := []string{
+		"mount",
+		"findmnt",
+		"umount",
+		"systemd-run",
+		"stat",
+		"touch",
+		"mkdir",
+		"sh",
+		"chmod",
+		"realpath",
 	}
 	// search for the required commands in other locations besides /usr/bin
-	for binary := range ne.paths {
+	for _, binary := range binaries {
 		// check for binary under the following directories
 		for _, path := range []string{"/", "/bin", "/usr/sbin", "/usr/bin"} {
 			binPath := filepath.Join(path, binary)
-			if _, err := os.Stat(filepath.Join(hostRootFsPath, binPath)); err != nil {
+			if _, err := os.Stat(filepath.Join(ne.hostRootFsPath, binPath)); err != nil {
 				continue
 			}
 			ne.paths[binary] = binPath
@@ -96,19 +116,19 @@ func NewNsenter() (*Nsenter, error) {
 		}
 		// systemd-run is optional, bailout if we don't find any of the other binaries
 		if ne.paths[binary] == "" && binary != "systemd-run" {
-			return nil, fmt.Errorf("unable to find %v", binary)
+			return fmt.Errorf("unable to find %v", binary)
 		}
 	}
-	return ne, nil
+	return nil
 }
 
 // Exec executes nsenter commands in hostProcMountNsPath mount namespace
 func (ne *Nsenter) Exec(cmd string, args []string) exec.Cmd {
+	hostProcMountNsPath := filepath.Join(ne.hostRootFsPath, mountNsPath)
 	fullArgs := append([]string{fmt.Sprintf("--mount=%s", hostProcMountNsPath), "--"},
 		append([]string{ne.AbsHostPath(cmd)}, args...)...)
 	glog.V(5).Infof("Running nsenter command: %v %v", nsenterPath, fullArgs)
-	exec := exec.New()
-	return exec.Command(nsenterPath, fullArgs...)
+	return ne.executor.Command(nsenterPath, fullArgs...)
 }
 
 // AbsHostPath returns the absolute runnable path for a specified command
@@ -136,6 +156,9 @@ func (ne *Nsenter) SupportsSystemd() (string, bool) {
 //                non/existing/directory does not exist
 // -> It resolves symlinks in /mnt/volume to say /mnt/foo and returns
 //    /mnt/foo/non/existing/directory.
+//
+// BEWARE! EvalSymlinks is not able to detect symlink looks with mustExist=false!
+// If /tmp/link is symlink to /tmp/link, EvalSymlinks(/tmp/link/foo) returns /tmp/link/foo.
 func (ne *Nsenter) EvalSymlinks(pathname string, mustExist bool) (string, error) {
 	var args []string
 	if mustExist {
@@ -157,5 +180,57 @@ func (ne *Nsenter) EvalSymlinks(pathname string, mustExist bool) (string, error)
 // kubelet. It is recommended to resolve symlinks on the host by EvalSymlinks
 // before calling this function
 func (ne *Nsenter) KubeletPath(pathname string) string {
-	return filepath.Join(hostRootFsPath, pathname)
+	return filepath.Join(ne.hostRootFsPath, pathname)
 }
+
+// NewFakeNsenter returns a Nsenter that does not run "nsenter --mount=... --",
+// but runs everything in the same mount namespace as the unit test binary.
+// rootfsPath is supposed to be a symlink, e.g. /tmp/xyz/rootfs -> /.
+// This fake Nsenter is enough for most operations, e.g. to resolve symlinks,
+// but it's not enough to call /bin/mount - unit tests don't run as root.
+func NewFakeNsenter(rootfsPath string) (*Nsenter, error) {
+	executor := &fakeExec{
+		rootfsPath: rootfsPath,
+	}
+	// prepare /rootfs/bin, usr/bin and usr/sbin
+	bin := filepath.Join(rootfsPath, "bin")
+	if err := os.Symlink("/bin", bin); err != nil {
+		return nil, err
+	}
+
+	usr := filepath.Join(rootfsPath, "usr")
+	if err := os.Mkdir(usr, 0755); err != nil {
+		return nil, err
+	}
+	usrbin := filepath.Join(usr, "bin")
+	if err := os.Symlink("/usr/bin", usrbin); err != nil {
+		return nil, err
+	}
+	usrsbin := filepath.Join(usr, "sbin")
+	if err := os.Symlink("/usr/sbin", usrsbin); err != nil {
+		return nil, err
+	}
+
+	return NewNsenter(rootfsPath, executor)
+}
+
+type fakeExec struct {
+	rootfsPath string
+}
+
+func (f fakeExec) Command(cmd string, args ...string) exec.Cmd {
+	// This will intentionaly panic if Nsenter does not provide enough arguments.
+	realCmd := args[2]
+	realArgs := args[3:]
+	return exec.New().Command(realCmd, realArgs...)
+}
+
+func (fakeExec) LookPath(file string) (string, error) {
+	return "", errors.New("not implemented")
+}
+
+func (fakeExec) CommandContext(ctx context.Context, cmd string, args ...string) exec.Cmd {
+	return nil
+}
+
+var _ exec.Interface = fakeExec{}

pkg/util/nsenter/nsenter_test.go

+// +build linux
+
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package nsenter
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"k8s.io/utils/exec"
+)
+
+func TestExec(t *testing.T) {
+	tests := []struct {
+		name           string
+		command        string
+		args           []string
+		expectedOutput string
+		expectError    bool
+	}{
+		{
+			name:           "simple command",
+			command:        "echo",
+			args:           []string{"hello", "world"},
+			expectedOutput: "hello world\n",
+		},
+		{
+			name:        "nozero exit code",
+			command:     "false",
+			expectError: true,
+		},
+	}
+
+	executor := fakeExec{
+		rootfsPath: "/rootfs",
+	}
+	for _, test := range tests {
+		ns := Nsenter{
+			hostRootFsPath: "/rootfs",
+			executor:       executor,
+		}
+		cmd := ns.Exec(test.command, test.args)
+		outBytes, err := cmd.CombinedOutput()
+		out := string(outBytes)
+		if err != nil && !test.expectError {
+			t.Errorf("Test %q: unexpected error: %s", test.name, err)
+		}
+		if err == nil && test.expectError {
+			t.Errorf("Test %q: expected error, got none", test.name)
+		}
+		if test.expectedOutput != out {
+			t.Errorf("test %q: expected output %q, got %q", test.name, test.expectedOutput, out)
+		}
+	}
+}
+
+func TestKubeletPath(t *testing.T) {
+	tests := []struct {
+		rootfs              string
+		hostpath            string
+		expectedKubeletPath string
+	}{
+		{
+			// simple join
+			"/rootfs",
+			"/some/path",
+			"/rootfs/some/path",
+		},
+		{
+			// squash slashes
+			"/rootfs/",
+			"//some/path",
+			"/rootfs/some/path",
+		},
+	}
+
+	for _, test := range tests {
+		ns := Nsenter{
+			hostRootFsPath: test.rootfs,
+		}
+		out := ns.KubeletPath(test.hostpath)
+		if out != test.expectedKubeletPath {
+			t.Errorf("Expected path %q, got %q", test.expectedKubeletPath, out)
+		}
+
+	}
+}
+
+func TestEvalSymlinks(t *testing.T) {
+	tests := []struct {
+		name        string
+		mustExist   bool
+		prepare     func(tmpdir string) (src string, expectedDst string, err error)
+		expectError bool
+	}{
+		{
+			name:      "simple file /src",
+			mustExist: true,
+			prepare: func(tmpdir string) (src string, expectedDst string, err error) {
+				src = filepath.Join(tmpdir, "src")
+				err = ioutil.WriteFile(src, []byte{}, 0644)
+				return src, src, err
+			},
+		},
+		{
+			name:      "non-existing file /src",
+			mustExist: true,
+			prepare: func(tmpdir string) (src string, expectedDst string, err error) {
+				src = filepath.Join(tmpdir, "src")
+				return src, "", nil
+			},
+			expectError: true,
+		},
+		{
+			name:      "non-existing file /src/ with mustExist=false",
+			mustExist: false,
+			prepare: func(tmpdir string) (src string, expectedDst string, err error) {
+				src = filepath.Join(tmpdir, "src")
+				return src, src, nil
+			},
+		},
+		{
+			name:      "non-existing file /existing/path/src with mustExist=false with existing directories",
+			mustExist: false,
+			prepare: func(tmpdir string) (src string, expectedDst string, err error) {
+				src = filepath.Join(tmpdir, "existing/path")
+				if err := os.MkdirAll(src, 0755); err != nil {
+					return "", "", err
+				}
+				src = filepath.Join(src, "src")
+				return src, src, nil
+			},
+		},
+		{
+			name:      "simple symlink /src -> /dst",
+			mustExist: false,
+			prepare: func(tmpdir string) (src string, expectedDst string, err error) {
+				dst := filepath.Join(tmpdir, "dst")
+				if err = ioutil.WriteFile(dst, []byte{}, 0644); err != nil {
+					return "", "", err
+				}
+				src = filepath.Join(tmpdir, "src")
+				err = os.Symlink(dst, src)
+				return src, dst, err
+			},
+		},
+		{
+			name:      "dangling symlink /src -> /non-existing-path",
+			mustExist: true,
+			prepare: func(tmpdir string) (src string, expectedDst string, err error) {
+				dst := filepath.Join(tmpdir, "non-existing-path")
+				src = filepath.Join(tmpdir, "src")
+				err = os.Symlink(dst, src)
+				return src, "", err
+			},
+			expectError: true,
+		},
+		{
+			name:      "dangling symlink /src -> /non-existing-path with mustExist=false",
+			mustExist: false,
+			prepare: func(tmpdir string) (src string, expectedDst string, err error) {
+				dst := filepath.Join(tmpdir, "non-existing-path")
+				src = filepath.Join(tmpdir, "src")
+				err = os.Symlink(dst, src)
+				return src, dst, err
+			},
+		},
+		{
+			name:      "symlink to directory /src/file, where /src is link to /dst",
+			mustExist: true,
+			prepare: func(tmpdir string) (src string, expectedDst string, err error) {
+				dst := filepath.Join(tmpdir, "dst")
+				if err = os.Mkdir(dst, 0755); err != nil {
+					return "", "", err
+				}
+				dstFile := filepath.Join(dst, "file")
+				if err = ioutil.WriteFile(dstFile, []byte{}, 0644); err != nil {
+					return "", "", err
+				}
+
+				src = filepath.Join(tmpdir, "src")
+				if err = os.Symlink(dst, src); err != nil {
+					return "", "", err
+				}
+				srcFile := filepath.Join(src, "file")
+				return srcFile, dstFile, nil
+			},
+		},
+		{
+			name:      "symlink to non-existing directory: /src/file, where /src is link to /dst and dst does not exist",
+			mustExist: true,
+			prepare: func(tmpdir string) (src string, expectedDst string, err error) {
+				dst := filepath.Join(tmpdir, "dst")
+
+				src = filepath.Join(tmpdir, "src")
+				if err = os.Symlink(dst, src); err != nil {
+					return "", "", err
+				}
+				srcFile := filepath.Join(src, "file")
+				return srcFile, "", nil
+			},
+			expectError: true,
+		},
+		{
+			name:      "symlink to non-existing directory: /src/file, where /src is link to /dst and dst does not exist with mustExist=false",
+			mustExist: false,
+			prepare: func(tmpdir string) (src string, expectedDst string, err error) {
+				dst := filepath.Join(tmpdir, "dst")
+				dstFile := filepath.Join(dst, "file")
+
+				src = filepath.Join(tmpdir, "src")
+				if err = os.Symlink(dst, src); err != nil {
+					return "", "", err
+				}
+				srcFile := filepath.Join(src, "file")
+				return srcFile, dstFile, nil
+			},
+		},
+	}
+
+	for _, test := range tests {
+		ns := Nsenter{
+			hostRootFsPath: "/rootfs",
+			executor: fakeExec{
+				rootfsPath: "/rootfs",
+			},
+		}
+
+		tmpdir, err := ioutil.TempDir("", "nsenter-hostpath-")
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer os.RemoveAll(tmpdir)
+
+		src, expectedDst, err := test.prepare(tmpdir)
+		if err != nil {
+			t.Error(err)
+			continue
+		}
+
+		dst, err := ns.EvalSymlinks(src, test.mustExist)
+		if err != nil && !test.expectError {
+			t.Errorf("Test %q: unexpected error: %s", test.name, err)
+		}
+		if err == nil && test.expectError {
+			t.Errorf("Test %q: expected error, got none", test.name)
+		}
+		if dst != expectedDst {
+			t.Errorf("Test %q: expected destination %q, got %q", test.name, expectedDst, dst)
+		}
+	}
+}
+
+func TestNewNsenter(t *testing.T) {
+	// Create a symlink /tmp/xyz/rootfs -> / and use it as rootfs path
+	// It should resolve all binaries correctly, the test runs on Linux
+
+	tmpdir, err := ioutil.TempDir("", "nsenter-hostpath-")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmpdir)
+
+	rootfs := filepath.Join(tmpdir, "rootfs")
+	if err = os.Symlink("/", rootfs); err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = NewNsenter(rootfs, exec.New())
+	if err != nil {
+		t.Errorf("Error: %s", err)
+	}
+}
+
+func TestNewNsenterError(t *testing.T) {
+	// Create empty dir /tmp/xyz/rootfs and use it as rootfs path
+	// It should resolve all binaries correctly, the test runs on Linux
+
+	tmpdir, err := ioutil.TempDir("", "nsenter-hostpath-")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmpdir)
+
+	rootfs := filepath.Join(tmpdir, "rootfs")
+	if err = os.MkdirAll(rootfs, 0755); err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = NewNsenter(rootfs, exec.New())
+	if err == nil {
+		t.Errorf("Expected error, got none")
+	}
+}

pkg/util/nsenter/nsenter_unsupported.go

@@ -22,6 +22,12 @@ import (
 	"k8s.io/utils/exec"
 )
 
+const (
+	// DefaultHostRootFsPath is path to host's filesystem mounted into container
+	// with kubelet.
+	DefaultHostRootFsPath = "/rootfs"
+)
+
 // Nsenter is part of experimental support for running the kubelet
 // in a container.
 type Nsenter struct {
@@ -30,7 +36,7 @@ type Nsenter struct {
 }
 
 // NewNsenter constructs a new instance of Nsenter
-func NewNsenter() (*Nsenter, error) {
+func NewNsenter(hostRootFsPath string, executor exec.Interface) (*Nsenter, error) {
 	return &Nsenter{}, nil
 }