Merge pull request #63143 from jsafrane/containerized-subpath

cmd/kubelet/app/BUILD

@@ -110,6 +110,7 @@ go_library(
         "//pkg/util/io:go_default_library",
         "//pkg/util/mount:go_default_library",
         "//pkg/util/node:go_default_library",
+        "//pkg/util/nsenter:go_default_library",
         "//pkg/util/oom:go_default_library",
         "//pkg/util/rlimit:go_default_library",
         "//pkg/version:go_default_library",
@@ -170,6 +171,7 @@ go_library(
         "//vendor/k8s.io/client-go/tools/record:go_default_library",
         "//vendor/k8s.io/client-go/util/cert:go_default_library",
         "//vendor/k8s.io/client-go/util/certificate:go_default_library",
+        "//vendor/k8s.io/utils/exec:go_default_library",
     ] + select({
         "@io_bazel_rules_go//go/platform:linux": [
             "//vendor/golang.org/x/exp/inotify:go_default_library",

cmd/kubelet/app/server.go

@@ -91,10 +91,12 @@ import (
 	kubeio "k8s.io/kubernetes/pkg/util/io"
 	"k8s.io/kubernetes/pkg/util/mount"
 	nodeutil "k8s.io/kubernetes/pkg/util/node"
+	"k8s.io/kubernetes/pkg/util/nsenter"
 	"k8s.io/kubernetes/pkg/util/oom"
 	"k8s.io/kubernetes/pkg/util/rlimit"
 	"k8s.io/kubernetes/pkg/version"
 	"k8s.io/kubernetes/pkg/version/verflag"
+	"k8s.io/utils/exec"
 )
 
 const (
@@ -361,11 +363,12 @@ func UnsecuredDependencies(s *options.KubeletServer) (*kubelet.Dependencies, err
 	var writer kubeio.Writer = &kubeio.StdWriter{}
 	if s.Containerized {
 		glog.V(2).Info("Running kubelet in containerized mode")
-		mounter, err = mount.NewNsenterMounter()
+		ne, err := nsenter.NewNsenter(nsenter.DefaultHostRootFsPath, exec.New())
 		if err != nil {
 			return nil, err
 		}
-		writer = &kubeio.NsenterWriter{}
+		mounter = mount.NewNsenterMounter(s.RootDirectory, ne)
+		writer = kubeio.NewNsenterWriter(ne)
 	}
 
 	var dockerClientConfig *dockershim.ClientConfig

pkg/kubelet/cm/container_manager_linux_test.go

@@ -92,8 +92,8 @@ func (mi *fakeMountInterface) MakeFile(pathname string) error {
 	return nil
 }
 
-func (mi *fakeMountInterface) ExistsPath(pathname string) bool {
-	return true
+func (mi *fakeMountInterface) ExistsPath(pathname string) (bool, error) {
+	return true, errors.New("not implemented")
 }
 
 func (mi *fakeMountInterface) PrepareSafeSubpath(subPath mount.Subpath) (newHostPath string, cleanupAction func(), err error) {
@@ -120,6 +120,10 @@ func (mi *fakeMountInterface) GetSELinuxSupport(pathname string) (bool, error) {
 	return false, errors.New("not implemented")
 }
 
+func (mi *fakeMountInterface) GetMode(pathname string) (os.FileMode, error) {
+	return 0, errors.New("not implemented")
+}
+
 func fakeContainerMgrMountInt() mount.Interface {
 	return &fakeMountInterface{
 		[]mount.MountPoint{

pkg/kubelet/kubelet_pods.go

@@ -58,7 +58,6 @@ import (
 	"k8s.io/kubernetes/pkg/kubelet/status"
 	kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
 	"k8s.io/kubernetes/pkg/kubelet/util/format"
-	utilfile "k8s.io/kubernetes/pkg/util/file"
 	mountutil "k8s.io/kubernetes/pkg/util/mount"
 	volumeutil "k8s.io/kubernetes/pkg/volume/util"
 	"k8s.io/kubernetes/pkg/volume/util/volumepathhandler"
@@ -179,19 +178,10 @@ func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, h
 				return nil, cleanupAction, fmt.Errorf("unable to provision SubPath `%s`: %v", mount.SubPath, err)
 			}
 
-			fileinfo, err := os.Lstat(hostPath)
-			if err != nil {
-				return nil, cleanupAction, err
-			}
-			perm := fileinfo.Mode()
-
-			volumePath, err := filepath.EvalSymlinks(hostPath)
-			if err != nil {
-				return nil, cleanupAction, err
-			}
+			volumePath := hostPath
 			hostPath = filepath.Join(volumePath, mount.SubPath)
 
-			if subPathExists, err := utilfile.FileOrSymlinkExists(hostPath); err != nil {
+			if subPathExists, err := mounter.ExistsPath(hostPath); err != nil {
 				glog.Errorf("Could not determine if subPath %s exists; will not attempt to change its permissions", hostPath)
 			} else if !subPathExists {
 				// Create the sub path now because if it's auto-created later when referenced, it may have an
@@ -199,10 +189,15 @@ func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, h
 				// when the pod specifies an fsGroup, and if the directory is not created here, Docker will
 				// later auto-create it with the incorrect mode 0750
 				// Make extra care not to escape the volume!
-				if err := mounter.SafeMakeDir(hostPath, volumePath, perm); err != nil {
-					glog.Errorf("failed to mkdir %q: %v", hostPath, err)
+				perm, err := mounter.GetMode(volumePath)
+				if err != nil {
 					return nil, cleanupAction, err
 				}
+				if err := mounter.SafeMakeDir(mount.SubPath, volumePath, perm); err != nil {
+					// Don't pass detailed error back to the user because it could give information about host filesystem
+					glog.Errorf("failed to create subPath directory for volumeMount %q of container %q: %v", mount.Name, container.Name, err)
+					return nil, cleanupAction, fmt.Errorf("failed to create subPath directory for volumeMount %q of container %q", mount.Name, container.Name)
+				}
 			}
 			hostPath, cleanupAction, err = mounter.PrepareSafeSubpath(mountutil.Subpath{
 				VolumeMountIndex: i,

pkg/util/io/writer.go

@@ -50,18 +50,24 @@ func (writer *StdWriter) WriteFile(filename string, data []byte, perm os.FileMod
 // it will not see the mounted device in its own namespace. To work around this
 // limitation one has to first enter hosts namespace (by using 'nsenter') and
 // only then write data.
-type NsenterWriter struct{}
+type NsenterWriter struct {
+	ne *nsenter.Nsenter
+}
+
+// NewNsenterWriter creates a new Writer that allows writing data to file using
+// nsenter command.
+func NewNsenterWriter(ne *nsenter.Nsenter) *NsenterWriter {
+	return &NsenterWriter{
+		ne: ne,
+	}
+}
 
 // WriteFile calls 'nsenter cat - > <the file>' and 'nsenter chmod' to create a
 // file on the host.
 func (writer *NsenterWriter) WriteFile(filename string, data []byte, perm os.FileMode) error {
-	ne, err := nsenter.NewNsenter()
-	if err != nil {
-		return err
-	}
 	echoArgs := []string{"-c", fmt.Sprintf("cat > %s", filename)}
 	glog.V(5).Infof("nsenter: write data to file %s by nsenter", filename)
-	command := ne.Exec("sh", echoArgs)
+	command := writer.ne.Exec("sh", echoArgs)
 	command.SetStdin(bytes.NewBuffer(data))
 	outputBytes, err := command.CombinedOutput()
 	if err != nil {
@@ -71,7 +77,7 @@ func (writer *NsenterWriter) WriteFile(filename string, data []byte, perm os.Fil
 
 	chmodArgs := []string{fmt.Sprintf("%o", perm), filename}
 	glog.V(5).Infof("nsenter: change permissions of file %s to %s", filename, chmodArgs[0])
-	outputBytes, err = ne.Exec("chmod", chmodArgs).CombinedOutput()
+	outputBytes, err = writer.ne.Exec("chmod", chmodArgs).CombinedOutput()
 	if err != nil {
 		glog.Errorf("Output from chmod command: %v", string(outputBytes))
 		return err

pkg/util/mount/BUILD

@@ -71,12 +71,44 @@ go_library(
         "//vendor/github.com/golang/glog:go_default_library",
         "//vendor/k8s.io/utils/exec:go_default_library",
     ] + select({
+        "@io_bazel_rules_go//go/platform:android": [
+            "//pkg/util/nsenter:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:darwin": [
+            "//pkg/util/nsenter:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:dragonfly": [
+            "//pkg/util/nsenter:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:freebsd": [
+            "//pkg/util/nsenter:go_default_library",
+        ],
         "@io_bazel_rules_go//go/platform:linux": [
+            "//pkg/util/file:go_default_library",
             "//pkg/util/io:go_default_library",
             "//pkg/util/nsenter:go_default_library",
             "//vendor/golang.org/x/sys/unix:go_default_library",
             "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         ],
+        "@io_bazel_rules_go//go/platform:nacl": [
+            "//pkg/util/nsenter:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:netbsd": [
+            "//pkg/util/nsenter:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:openbsd": [
+            "//pkg/util/nsenter:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:plan9": [
+            "//pkg/util/nsenter:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:solaris": [
+            "//pkg/util/nsenter:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:windows": [
+            "//pkg/util/file:go_default_library",
+            "//pkg/util/nsenter:go_default_library",
+        ],
         "//conditions:default": [],
     }),
 )
@@ -101,7 +133,9 @@ go_test(
         "//vendor/k8s.io/utils/exec/testing:go_default_library",
     ] + select({
         "@io_bazel_rules_go//go/platform:linux": [
+            "//pkg/util/nsenter:go_default_library",
             "//vendor/github.com/golang/glog:go_default_library",
+            "//vendor/golang.org/x/sys/unix:go_default_library",
             "//vendor/k8s.io/utils/exec:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:windows": [

pkg/util/mount/exec_mount.go

@@ -136,7 +136,7 @@ func (m *execMounter) MakeDir(pathname string) error {
 	return m.wrappedMounter.MakeDir(pathname)
 }
 
-func (m *execMounter) ExistsPath(pathname string) bool {
+func (m *execMounter) ExistsPath(pathname string) (bool, error) {
 	return m.wrappedMounter.ExistsPath(pathname)
 }
 
@@ -163,3 +163,7 @@ func (m *execMounter) GetFSGroup(pathname string) (int64, error) {
 func (m *execMounter) GetSELinuxSupport(pathname string) (bool, error) {
 	return m.wrappedMounter.GetSELinuxSupport(pathname)
 }
+
+func (m *execMounter) GetMode(pathname string) (os.FileMode, error) {
+	return m.wrappedMounter.GetMode(pathname)
+}

pkg/util/mount/exec_mount_test.go

@@ -147,8 +147,8 @@ func (fm *fakeMounter) MakeFile(pathname string) error {
 func (fm *fakeMounter) MakeDir(pathname string) error {
 	return nil
 }
-func (fm *fakeMounter) ExistsPath(pathname string) bool {
-	return false
+func (fm *fakeMounter) ExistsPath(pathname string) (bool, error) {
+	return false, errors.New("not implemented")
 }
 func (fm *fakeMounter) GetFileType(pathname string) (FileType, error) {
 	return FileTypeFile, nil
@@ -176,3 +176,7 @@ func (fm *fakeMounter) GetFSGroup(pathname string) (int64, error) {
 func (fm *fakeMounter) GetSELinuxSupport(pathname string) (bool, error) {
 	return false, errors.New("not implemented")
 }
+
+func (fm *fakeMounter) GetMode(pathname string) (os.FileMode, error) {
+	return 0, errors.New("not implemented")
+}

pkg/util/mount/exec_mount_unsupported.go

@@ -83,8 +83,8 @@ func (mounter *execMounter) MakeFile(pathname string) error {
 	return nil
 }
 
-func (mounter *execMounter) ExistsPath(pathname string) bool {
-	return true
+func (mounter *execMounter) ExistsPath(pathname string) (bool, error) {
+	return true, errors.New("not implemented")
 }
 
 func (mounter *execMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
@@ -110,3 +110,7 @@ func (mounter *execMounter) GetFSGroup(pathname string) (int64, error) {
 func (mounter *execMounter) GetSELinuxSupport(pathname string) (bool, error) {
 	return false, errors.New("not implemented")
 }
+
+func (mounter *execMounter) GetMode(pathname string) (os.FileMode, error) {
+	return 0, errors.New("not implemented")
+}

pkg/util/mount/fake.go

@@ -201,8 +201,8 @@ func (f *FakeMounter) MakeFile(pathname string) error {
 	return nil
 }
 
-func (f *FakeMounter) ExistsPath(pathname string) bool {
-	return false
+func (f *FakeMounter) ExistsPath(pathname string) (bool, error) {
+	return false, errors.New("not implemented")
 }
 
 func (f *FakeMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
@@ -232,3 +232,7 @@ func (f *FakeMounter) GetFSGroup(pathname string) (int64, error) {
 func (f *FakeMounter) GetSELinuxSupport(pathname string) (bool, error) {
 	return false, errors.New("GetSELinuxSupport not implemented")
 }
+
+func (f *FakeMounter) GetMode(pathname string) (os.FileMode, error) {
+	return 0, errors.New("not implemented")
+}

pkg/util/mount/mount.go

@@ -84,16 +84,18 @@ type Interface interface {
 	// MakeDir creates a new directory.
 	// Will operate in the host mount namespace if kubelet is running in a container
 	MakeDir(pathname string) error
-	// SafeMakeDir makes sure that the created directory does not escape given
-	// base directory mis-using symlinks. The directory is created in the same
-	// mount namespace as where kubelet is running. Note that the function makes
-	// sure that it creates the directory somewhere under the base, nothing
-	// else. E.g. if the directory already exists, it may exists outside of the
-	// base due to symlinks.
-	SafeMakeDir(pathname string, base string, perm os.FileMode) error
-	// ExistsPath checks whether the path exists.
-	// Will operate in the host mount namespace if kubelet is running in a container
-	ExistsPath(pathname string) bool
+	// SafeMakeDir creates subdir within given base. It makes sure that the
+	// created directory does not escape given base directory mis-using
+	// symlinks. Note that the function makes sure that it creates the directory
+	// somewhere under the base, nothing else. E.g. if the directory already
+	// exists, it may exist outside of the base due to symlinks.
+	// This method should be used if the directory to create is inside volume
+	// that's under user control. User must not be able to use symlinks to
+	// escape the volume to create directories somewhere else.
+	SafeMakeDir(subdir string, base string, perm os.FileMode) error
+	// Will operate in the host mount namespace if kubelet is running in a container.
+	// Error is returned on any other error than "file not found".
+	ExistsPath(pathname string) (bool, error)
 	// CleanSubPaths removes any bind-mounts created by PrepareSafeSubpath in given
 	// pod volume directory.
 	CleanSubPaths(podDir string, volumeName string) error
@@ -117,6 +119,8 @@ type Interface interface {
 	// GetSELinuxSupport returns true if given path is on a mount that supports
 	// SELinux.
 	GetSELinuxSupport(pathname string) (bool, error)
+	// GetMode returns permissions of the path.
+	GetMode(pathname string) (os.FileMode, error)
 }
 
 type Subpath struct {

pkg/util/mount/mount_linux_test.go

@@ -1193,10 +1193,6 @@ func TestBindSubPath(t *testing.T) {
 					return nil, "", "", err
 				}
 
-				if err := os.MkdirAll(subpathMount, defaultPerm); err != nil {
-					return nil, "", "", err
-				}
-
 				socketFile, socketCreateError := createSocketFile(volpath)
 
 				return mounts, volpath, socketFile, socketCreateError
@@ -1212,10 +1208,6 @@ func TestBindSubPath(t *testing.T) {
 					return nil, "", "", err
 				}
 
-				if err := os.MkdirAll(subpathMount, defaultPerm); err != nil {
-					return nil, "", "", err
-				}
-
 				testFifo := filepath.Join(volpath, "mount_test.fifo")
 				err := syscall.Mkfifo(testFifo, 0)
 				return mounts, volpath, testFifo, err
@@ -1299,7 +1291,7 @@ func TestBindSubPath(t *testing.T) {
 		}
 
 		_, subpathMount := getTestPaths(base)
-		bindPathTarget, err := doBindSubPath(fm, subpath, 1)
+		bindPathTarget, err := doBindSubPath(fm, subpath)
 		if test.expectError {
 			if err == nil {
 				t.Errorf("test %q failed: expected error, got success", test.name)

pkg/util/mount/mount_unsupported.go

@@ -21,8 +21,6 @@ package mount
 import (
 	"errors"
 	"os"
-
-	"github.com/golang/glog"
 )
 
 type Mounter struct {
@@ -110,9 +108,8 @@ func (mounter *Mounter) MakeFile(pathname string) error {
 	return unsupportedErr
 }
 
-func (mounter *Mounter) ExistsPath(pathname string) bool {
-	glog.Errorf("%s", unsupportedErr)
-	return true
+func (mounter *Mounter) ExistsPath(pathname string) (bool, error) {
+	return true, errors.New("not implemented")
 }
 
 func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
@@ -138,3 +135,7 @@ func (mounter *Mounter) GetFSGroup(pathname string) (int64, error) {
 func (mounter *Mounter) GetSELinuxSupport(pathname string) (bool, error) {
 	return false, errors.New("not implemented")
 }
+
+func (mounter *Mounter) GetMode(pathname string) (os.FileMode, error) {
+	return 0, errors.New("not implemented")
+}

pkg/util/mount/mount_windows.go

@@ -29,6 +29,8 @@ import (
 	"syscall"
 
 	"github.com/golang/glog"
+
+	utilfile "k8s.io/kubernetes/pkg/util/file"
 )
 
 // Mounter provides the default implementation of mount.Interface
@@ -147,9 +149,13 @@ func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) {
 	if stat.Mode()&os.ModeSymlink != 0 {
 		target, err := os.Readlink(file)
 		if err != nil {
-			return true, fmt.Errorf("Readlink error: %v", err)
+			return true, fmt.Errorf("readlink error: %v", err)
+		}
+		exists, err := mounter.ExistsPath(target)
+		if err != nil {
+			return true, err
 		}
-		return !mounter.ExistsPath(target), nil
+		return !exists, nil
 	}
 
 	return true, nil
@@ -232,12 +238,8 @@ func (mounter *Mounter) MakeFile(pathname string) error {
 }
 
 // ExistsPath checks whether the path exists
-func (mounter *Mounter) ExistsPath(pathname string) bool {
-	_, err := os.Stat(pathname)
-	if err != nil {
-		return false
-	}
-	return true
+func (mounter *Mounter) ExistsPath(pathname string) (bool, error) {
+	return utilfile.FileExists(pathname)
 }
 
 // check whether hostPath is within volume path
@@ -461,9 +463,23 @@ func (mounter *Mounter) GetSELinuxSupport(pathname string) (bool, error) {
 	return false, nil
 }
 
+func (mounter *Mounter) GetMode(pathname string) (os.FileMode, error) {
+	info, err := os.Stat(pathname)
+	if err != nil {
+		return 0, err
+	}
+	return info.Mode(), nil
+}
+
 // SafeMakeDir makes sure that the created directory does not escape given base directory mis-using symlinks.
-func (mounter *Mounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
-	return doSafeMakeDir(pathname, base, perm)
+func (mounter *Mounter) SafeMakeDir(subdir string, base string, perm os.FileMode) error {
+	realBase, err := filepath.EvalSymlinks(base)
+	if err != nil {
+		return fmt.Errorf("error resolving symlinks in %s: %s", base, err)
+	}
+
+	realFullPath := filepath.Join(realBase, subdir)
+	return doSafeMakeDir(realFullPath, realBase, perm)
 }
 
 func doSafeMakeDir(pathname string, base string, perm os.FileMode) error {

pkg/util/mount/nsenter_mount.go

@@ -22,12 +22,12 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
-	"regexp"
-	"strconv"
 	"strings"
+	"syscall"
 
 	"github.com/golang/glog"
-	utilio "k8s.io/kubernetes/pkg/util/io"
+	"golang.org/x/sys/unix"
+	utilfile "k8s.io/kubernetes/pkg/util/file"
 	"k8s.io/kubernetes/pkg/util/nsenter"
 )
 
@@ -36,13 +36,6 @@ const (
 	hostProcMountsPath = "/rootfs/proc/1/mounts"
 	// hostProcMountinfoPath is the default mount info path for rootfs
 	hostProcMountinfoPath = "/rootfs/proc/1/mountinfo"
-	// hostProcSelfStatusPath is the default path to /proc/self/status on the host
-	hostProcSelfStatusPath = "/rootfs/proc/self/status"
-)
-
-var (
-	// pidRegExp matches "Pid:	<pid>" in /proc/self/status
-	pidRegExp = regexp.MustCompile(`\nPid:\t([0-9]*)\n`)
 )
 
 // Currently, all docker containers receive their own mount namespaces.
@@ -50,14 +43,16 @@ var (
 // the host's mount namespace.
 type NsenterMounter struct {
 	ne *nsenter.Nsenter
+	// rootDir is location of /var/lib/kubelet directory.
+	rootDir string
 }
 
-func NewNsenterMounter() (*NsenterMounter, error) {
-	ne, err := nsenter.NewNsenter()
-	if err != nil {
-		return nil, err
+// NewNsenterMounter creates a new mounter for kubelet that runs as a container.
+func NewNsenterMounter(rootDir string, ne *nsenter.Nsenter) *NsenterMounter {
+	return &NsenterMounter{
+		rootDir: rootDir,
+		ne:      ne,
 	}
-	return &NsenterMounter{ne: ne}, nil
 }
 
 // NsenterMounter implements mount.Interface
@@ -281,42 +276,24 @@ func (mounter *NsenterMounter) MakeFile(pathname string) error {
 	return nil
 }
 
-func (mounter *NsenterMounter) ExistsPath(pathname string) bool {
-	args := []string{pathname}
-	_, err := mounter.ne.Exec("ls", args).CombinedOutput()
-	if err == nil {
-		return true
+func (mounter *NsenterMounter) ExistsPath(pathname string) (bool, error) {
+	// Resolve the symlinks but allow the target not to exist. EvalSymlinks
+	// would return an generic error when the target does not exist.
+	hostPath, err := mounter.ne.EvalSymlinks(pathname, false /* mustExist */)
+	if err != nil {
+		return false, err
 	}
-	return false
+	kubeletpath := mounter.ne.KubeletPath(hostPath)
+	return utilfile.FileExists(kubeletpath)
 }
 
 func (mounter *NsenterMounter) CleanSubPaths(podDir string, volumeName string) error {
 	return doCleanSubPaths(mounter, podDir, volumeName)
 }
 
-// getPidOnHost returns kubelet's pid in the host pid namespace
-func (mounter *NsenterMounter) getPidOnHost(procStatusPath string) (int, error) {
-	// Get the PID from /rootfs/proc/self/status
-	statusBytes, err := utilio.ConsistentRead(procStatusPath, maxListTries)
-	if err != nil {
-		return 0, fmt.Errorf("error reading %s: %s", procStatusPath, err)
-	}
-	matches := pidRegExp.FindSubmatch(statusBytes)
-	if len(matches) < 2 {
-		return 0, fmt.Errorf("cannot parse %s: no Pid:", procStatusPath)
-	}
-	return strconv.Atoi(string(matches[1]))
-}
-
 func (mounter *NsenterMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
-	hostPid, err := mounter.getPidOnHost(hostProcSelfStatusPath)
-	if err != nil {
-		return "", nil, err
-	}
-	glog.V(4).Infof("Kubelet's PID on the host is %d", hostPid)
-
 	// Bind-mount the subpath to avoid using symlinks in subpaths.
-	newHostPath, err = doBindSubPath(mounter, subPath, hostPid)
+	newHostPath, err = doNsEnterBindSubPath(mounter, subPath)
 
 	// There is no action when the container starts. Bind-mount will be cleaned
 	// when container stops by CleanSubPaths.
@@ -324,26 +301,152 @@ func (mounter *NsenterMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath
 	return newHostPath, cleanupAction, err
 }
 
-func (mounter *NsenterMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
-	return doSafeMakeDir(pathname, base, perm)
+func (mounter *NsenterMounter) SafeMakeDir(subdir string, base string, perm os.FileMode) error {
+	fullSubdirPath := filepath.Join(base, subdir)
+	evaluatedSubdirPath, err := mounter.ne.EvalSymlinks(fullSubdirPath, false /* mustExist */)
+	if err != nil {
+		return fmt.Errorf("error resolving symlinks in %s: %s", fullSubdirPath, err)
+	}
+	evaluatedSubdirPath = filepath.Clean(evaluatedSubdirPath)
+
+	evaluatedBase, err := mounter.ne.EvalSymlinks(base, true /* mustExist */)
+	if err != nil {
+		return fmt.Errorf("error resolving symlinks in %s: %s", base, err)
+	}
+	evaluatedBase = filepath.Clean(evaluatedBase)
+
+	rootDir := filepath.Clean(mounter.rootDir)
+	if pathWithinBase(evaluatedBase, rootDir) {
+		// Base is in /var/lib/kubelet. This directory is shared between the
+		// container with kubelet and the host. We don't need to add '/rootfs'.
+		// This is useful when /rootfs is mounted as read-only - we can still
+		// create subpaths for paths in /var/lib/kubelet.
+		return doSafeMakeDir(evaluatedSubdirPath, evaluatedBase, perm)
+	}
+
+	// Base is somewhere on the host's filesystem. Add /rootfs and try to make
+	// the directory there.
+	// This requires /rootfs to be writable.
+	kubeletSubdirPath := mounter.ne.KubeletPath(evaluatedSubdirPath)
+	kubeletBase := mounter.ne.KubeletPath(evaluatedBase)
+	return doSafeMakeDir(kubeletSubdirPath, kubeletBase, perm)
 }
 
 func (mounter *NsenterMounter) GetMountRefs(pathname string) ([]string, error) {
-	hostpath, err := mounter.ne.EvalSymlinks(pathname)
+	hostpath, err := mounter.ne.EvalSymlinks(pathname, true /* mustExist */)
 	if err != nil {
 		return nil, err
 	}
 	return searchMountPoints(hostpath, hostProcMountinfoPath)
 }
 
+func doNsEnterBindSubPath(mounter *NsenterMounter, subpath Subpath) (hostPath string, err error) {
+	// Linux, kubelet runs in a container:
+	// - safely open the subpath
+	// - bind-mount the subpath to target (this can be unsafe)
+	// - check that we mounted the right thing by comparing device ID and inode
+	//   of the subpath (via safely opened fd) and the target (that's under our
+	//   control)
+
+	// Evaluate all symlinks here once for all subsequent functions.
+	evaluatedHostVolumePath, err := mounter.ne.EvalSymlinks(subpath.VolumePath, true /*mustExist*/)
+	if err != nil {
+		return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.VolumePath, err)
+	}
+	evaluatedHostSubpath, err := mounter.ne.EvalSymlinks(subpath.Path, true /*mustExist*/)
+	if err != nil {
+		return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.Path, err)
+	}
+	glog.V(5).Infof("doBindSubPath %q (%q) for volumepath %q", subpath.Path, evaluatedHostSubpath, subpath.VolumePath)
+	subpath.VolumePath = mounter.ne.KubeletPath(evaluatedHostVolumePath)
+	subpath.Path = mounter.ne.KubeletPath(evaluatedHostSubpath)
+
+	// Check the subpath is correct and open it
+	fd, err := safeOpenSubPath(mounter, subpath)
+	if err != nil {
+		return "", err
+	}
+	defer syscall.Close(fd)
+
+	alreadyMounted, bindPathTarget, err := prepareSubpathTarget(mounter, subpath)
+	if err != nil {
+		return "", err
+	}
+	if alreadyMounted {
+		return bindPathTarget, nil
+	}
+
+	success := false
+	defer func() {
+		// Cleanup subpath on error
+		if !success {
+			glog.V(4).Infof("doNsEnterBindSubPath() failed for %q, cleaning up subpath", bindPathTarget)
+			if cleanErr := cleanSubPath(mounter, subpath); cleanErr != nil {
+				glog.Errorf("Failed to clean subpath %q: %v", bindPathTarget, cleanErr)
+			}
+		}
+	}()
+
+	// Leap of faith: optimistically expect that nobody has modified previously
+	// expanded evalSubPath with evil symlinks and bind-mount it.
+	// Mount is done on the host! don't use kubelet path!
+	glog.V(5).Infof("bind mounting %q at %q", evaluatedHostSubpath, bindPathTarget)
+	if err = mounter.Mount(evaluatedHostSubpath, bindPathTarget, "" /*fstype*/, []string{"bind"}); err != nil {
+		return "", fmt.Errorf("error mounting %s: %s", evaluatedHostSubpath, err)
+	}
+
+	// Check that the bind-mount target is the same inode and device as the
+	// source that we keept open, i.e. we mounted the right thing.
+	err = checkDeviceInode(fd, bindPathTarget)
+	if err != nil {
+		return "", fmt.Errorf("error checking bind mount for subpath %s: %s", subpath.VolumePath, err)
+	}
+
+	success = true
+	glog.V(3).Infof("Bound SubPath %s into %s", subpath.Path, bindPathTarget)
+	return bindPathTarget, nil
+}
+
+// checkDeviceInode checks that opened file and path represent the same file.
+func checkDeviceInode(fd int, path string) error {
+	var srcStat, dstStat unix.Stat_t
+	err := unix.Fstat(fd, &srcStat)
+	if err != nil {
+		return fmt.Errorf("error running fstat on subpath FD: %v", err)
+	}
+
+	err = unix.Stat(path, &dstStat)
+	if err != nil {
+		return fmt.Errorf("error running fstat on %s: %v", path, err)
+	}
+
+	if srcStat.Dev != dstStat.Dev {
+		return fmt.Errorf("different device number")
+	}
+	if srcStat.Ino != dstStat.Ino {
+		return fmt.Errorf("different inode")
+	}
+	return nil
+}
+
 func (mounter *NsenterMounter) GetFSGroup(pathname string) (int64, error) {
-	kubeletpath, err := mounter.ne.KubeletPath(pathname)
+	hostPath, err := mounter.ne.EvalSymlinks(pathname, true /* mustExist */)
 	if err != nil {
-		return 0, err
+		return -1, err
 	}
+	kubeletpath := mounter.ne.KubeletPath(hostPath)
 	return getFSGroup(kubeletpath)
 }
 
 func (mounter *NsenterMounter) GetSELinuxSupport(pathname string) (bool, error) {
 	return getSELinuxSupport(pathname, hostProcMountsPath)
 }
+
+func (mounter *NsenterMounter) GetMode(pathname string) (os.FileMode, error) {
+	hostPath, err := mounter.ne.EvalSymlinks(pathname, true /* mustExist */)
+	if err != nil {
+		return 0, err
+	}
+	kubeletpath := mounter.ne.KubeletPath(hostPath)
+	return getMode(kubeletpath)
+}

pkg/util/mount/nsenter_mount_unsupported.go

@@ -21,12 +21,14 @@ package mount
 import (
 	"errors"
 	"os"
+
+	"k8s.io/kubernetes/pkg/util/nsenter"
 )
 
 type NsenterMounter struct{}
 
-func NewNsenterMounter() (*NsenterMounter, error) {
-	return &NsenterMounter{}, nil
+func NewNsenterMounter(rootDir string, ne *nsenter.Nsenter) *NsenterMounter {
+	return &NsenterMounter{}
 }
 
 var _ = Interface(&NsenterMounter{})
@@ -83,8 +85,8 @@ func (*NsenterMounter) MakeFile(pathname string) error {
 	return nil
 }
 
-func (*NsenterMounter) ExistsPath(pathname string) bool {
-	return true
+func (*NsenterMounter) ExistsPath(pathname string) (bool, error) {
+	return true, errors.New("not implemented")
 }
 
 func (*NsenterMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
@@ -110,3 +112,7 @@ func (*NsenterMounter) GetFSGroup(pathname string) (int64, error) {
 func (*NsenterMounter) GetSELinuxSupport(pathname string) (bool, error) {
 	return false, errors.New("not implemented")
 }
+
+func (*NsenterMounter) GetMode(pathname string) (os.FileMode, error) {
+	return 0, errors.New("not implemented")
+}

pkg/util/nsenter/BUILD

-load("@io_bazel_rules_go//go:def.bzl", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 
 go_library(
     name = "go_default_library",
@@ -92,3 +92,20 @@ filegroup(
     tags = ["automanaged"],
     visibility = ["//visibility:public"],
 )
+
+go_test(
+    name = "go_default_test",
+    srcs = select({
+        "@io_bazel_rules_go//go/platform:linux": [
+            "nsenter_test.go",
+        ],
+        "//conditions:default": [],
+    }),
+    embed = [":go_default_library"],
+    deps = select({
+        "@io_bazel_rules_go//go/platform:linux": [
+            "//vendor/k8s.io/utils/exec:go_default_library",
+        ],
+        "//conditions:default": [],
+    }),
+)

pkg/util/nsenter/nsenter.go

@@ -19,6 +19,8 @@ limitations under the License.
 package nsenter
 
 import (
+	"context"
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -30,9 +32,11 @@ import (
 )
 
 const (
-	hostRootFsPath = "/rootfs"
-	// hostProcMountNsPath is the default mount namespace for rootfs
-	hostProcMountNsPath = "/rootfs/proc/1/ns/mnt"
+	// DefaultHostRootFsPath is path to host's filesystem mounted into container
+	// with kubelet.
+	DefaultHostRootFsPath = "/rootfs"
+	// mountNsPath is the default mount namespace of the host
+	mountNsPath = "/proc/1/ns/mnt"
 	// nsenterPath is the default nsenter command
 	nsenterPath = "nsenter"
 )
@@ -65,30 +69,46 @@ const (
 type Nsenter struct {
 	// a map of commands to their paths on the host filesystem
 	paths map[string]string
+
+	// Path to the host filesystem, typically "/rootfs". Used only for testing.
+	hostRootFsPath string
+
+	// Exec implementation, used only for testing
+	executor exec.Interface
 }
 
 // NewNsenter constructs a new instance of Nsenter
-func NewNsenter() (*Nsenter, error) {
+func NewNsenter(hostRootFsPath string, executor exec.Interface) (*Nsenter, error) {
 	ne := &Nsenter{
-		paths: map[string]string{
-			"mount":       "",
-			"findmnt":     "",
-			"umount":      "",
-			"systemd-run": "",
-			"stat":        "",
-			"touch":       "",
-			"mkdir":       "",
-			"ls":          "",
-			"sh":          "",
-			"chmod":       "",
-		},
+		hostRootFsPath: hostRootFsPath,
+		executor:       executor,
+	}
+	if err := ne.initPaths(); err != nil {
+		return nil, err
+	}
+	return ne, nil
+}
+
+func (ne *Nsenter) initPaths() error {
+	ne.paths = map[string]string{}
+	binaries := []string{
+		"mount",
+		"findmnt",
+		"umount",
+		"systemd-run",
+		"stat",
+		"touch",
+		"mkdir",
+		"sh",
+		"chmod",
+		"realpath",
 	}
 	// search for the required commands in other locations besides /usr/bin
-	for binary := range ne.paths {
+	for _, binary := range binaries {
 		// check for binary under the following directories
 		for _, path := range []string{"/", "/bin", "/usr/sbin", "/usr/bin"} {
 			binPath := filepath.Join(path, binary)
-			if _, err := os.Stat(filepath.Join(hostRootFsPath, binPath)); err != nil {
+			if _, err := os.Stat(filepath.Join(ne.hostRootFsPath, binPath)); err != nil {
 				continue
 			}
 			ne.paths[binary] = binPath
@@ -96,19 +116,19 @@ func NewNsenter() (*Nsenter, error) {
 		}
 		// systemd-run is optional, bailout if we don't find any of the other binaries
 		if ne.paths[binary] == "" && binary != "systemd-run" {
-			return nil, fmt.Errorf("unable to find %v", binary)
+			return fmt.Errorf("unable to find %v", binary)
 		}
 	}
-	return ne, nil
+	return nil
 }
 
 // Exec executes nsenter commands in hostProcMountNsPath mount namespace
 func (ne *Nsenter) Exec(cmd string, args []string) exec.Cmd {
+	hostProcMountNsPath := filepath.Join(ne.hostRootFsPath, mountNsPath)
 	fullArgs := append([]string{fmt.Sprintf("--mount=%s", hostProcMountNsPath), "--"},
 		append([]string{ne.AbsHostPath(cmd)}, args...)...)
 	glog.V(5).Infof("Running nsenter command: %v %v", nsenterPath, fullArgs)
-	exec := exec.New()
-	return exec.Command(nsenterPath, fullArgs...)
+	return ne.executor.Command(nsenterPath, fullArgs...)
 }
 
 // AbsHostPath returns the absolute runnable path for a specified command
@@ -128,8 +148,26 @@ func (ne *Nsenter) SupportsSystemd() (string, bool) {
 
 // EvalSymlinks returns the path name on the host after evaluating symlinks on the
 // host.
-func (ne *Nsenter) EvalSymlinks(pathname string) (string, error) {
-	args := []string{"-m", pathname}
+// mustExist makes EvalSymlinks to return error when the path does not
+// exist. When it's false, it evaluates symlinks of the existing part and
+// blindly adds the non-existing part:
+// pathname: /mnt/volume/non/existing/directory
+//     /mnt/volume exists
+//                non/existing/directory does not exist
+// -> It resolves symlinks in /mnt/volume to say /mnt/foo and returns
+//    /mnt/foo/non/existing/directory.
+//
+// BEWARE! EvalSymlinks is not able to detect symlink looks with mustExist=false!
+// If /tmp/link is symlink to /tmp/link, EvalSymlinks(/tmp/link/foo) returns /tmp/link/foo.
+func (ne *Nsenter) EvalSymlinks(pathname string, mustExist bool) (string, error) {
+	var args []string
+	if mustExist {
+		// "realpath -e: all components of the path must exist"
+		args = []string{"-e", pathname}
+	} else {
+		// "realpath -m: no path components need exist or be a directory"
+		args = []string{"-m", pathname}
+	}
 	outBytes, err := ne.Exec("realpath", args).CombinedOutput()
 	if err != nil {
 		glog.Infof("failed to resolve symbolic links on %s: %v", pathname, err)
@@ -139,11 +177,60 @@ func (ne *Nsenter) EvalSymlinks(pathname string) (string, error) {
 }
 
 // KubeletPath returns the path name that can be accessed by containerized
-// kubelet, after evaluating symlinks on the host.
-func (ne *Nsenter) KubeletPath(pathname string) (string, error) {
-	hostpath, err := ne.EvalSymlinks(pathname)
-	if err != nil {
-		return "", err
+// kubelet. It is recommended to resolve symlinks on the host by EvalSymlinks
+// before calling this function
+func (ne *Nsenter) KubeletPath(pathname string) string {
+	return filepath.Join(ne.hostRootFsPath, pathname)
+}
+
+// NewFakeNsenter returns a Nsenter that does not run "nsenter --mount=... --",
+// but runs everything in the same mount namespace as the unit test binary.
+// rootfsPath is supposed to be a symlink, e.g. /tmp/xyz/rootfs -> /.
+// This fake Nsenter is enough for most operations, e.g. to resolve symlinks,
+// but it's not enough to call /bin/mount - unit tests don't run as root.
+func NewFakeNsenter(rootfsPath string) (*Nsenter, error) {
+	executor := &fakeExec{
+		rootfsPath: rootfsPath,
+	}
+	// prepare /rootfs/bin, usr/bin and usr/sbin
+	bin := filepath.Join(rootfsPath, "bin")
+	if err := os.Symlink("/bin", bin); err != nil {
+		return nil, err
 	}
-	return filepath.Join(hostRootFsPath, hostpath), nil
+
+	usr := filepath.Join(rootfsPath, "usr")
+	if err := os.Mkdir(usr, 0755); err != nil {
+		return nil, err
+	}
+	usrbin := filepath.Join(usr, "bin")
+	if err := os.Symlink("/usr/bin", usrbin); err != nil {
+		return nil, err
+	}
+	usrsbin := filepath.Join(usr, "sbin")
+	if err := os.Symlink("/usr/sbin", usrsbin); err != nil {
+		return nil, err
+	}
+
+	return NewNsenter(rootfsPath, executor)
+}
+
+type fakeExec struct {
+	rootfsPath string
 }
+
+func (f fakeExec) Command(cmd string, args ...string) exec.Cmd {
+	// This will intentionaly panic if Nsenter does not provide enough arguments.
+	realCmd := args[2]
+	realArgs := args[3:]
+	return exec.New().Command(realCmd, realArgs...)
+}
+
+func (fakeExec) LookPath(file string) (string, error) {
+	return "", errors.New("not implemented")
+}
+
+func (fakeExec) CommandContext(ctx context.Context, cmd string, args ...string) exec.Cmd {
+	return nil
+}
+
+var _ exec.Interface = fakeExec{}

pkg/util/nsenter/nsenter_test.go

+// +build linux
+
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package nsenter
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"k8s.io/utils/exec"
+)
+
+func TestExec(t *testing.T) {
+	tests := []struct {
+		name           string
+		command        string
+		args           []string
+		expectedOutput string
+		expectError    bool
+	}{
+		{
+			name:           "simple command",
+			command:        "echo",
+			args:           []string{"hello", "world"},
+			expectedOutput: "hello world\n",
+		},
+		{
+			name:        "nozero exit code",
+			command:     "false",
+			expectError: true,
+		},
+	}
+
+	executor := fakeExec{
+		rootfsPath: "/rootfs",
+	}
+	for _, test := range tests {
+		ns := Nsenter{
+			hostRootFsPath: "/rootfs",
+			executor:       executor,
+		}
+		cmd := ns.Exec(test.command, test.args)
+		outBytes, err := cmd.CombinedOutput()
+		out := string(outBytes)
+		if err != nil && !test.expectError {
+			t.Errorf("Test %q: unexpected error: %s", test.name, err)
+		}
+		if err == nil && test.expectError {
+			t.Errorf("Test %q: expected error, got none", test.name)
+		}
+		if test.expectedOutput != out {
+			t.Errorf("test %q: expected output %q, got %q", test.name, test.expectedOutput, out)
+		}
+	}
+}
+
+func TestKubeletPath(t *testing.T) {
+	tests := []struct {
+		rootfs              string
+		hostpath            string
+		expectedKubeletPath string
+	}{
+		{
+			// simple join
+			"/rootfs",
+			"/some/path",
+			"/rootfs/some/path",
+		},
+		{
+			// squash slashes
+			"/rootfs/",
+			"//some/path",
+			"/rootfs/some/path",
+		},
+	}
+
+	for _, test := range tests {
+		ns := Nsenter{
+			hostRootFsPath: test.rootfs,
+		}
+		out := ns.KubeletPath(test.hostpath)
+		if out != test.expectedKubeletPath {
+			t.Errorf("Expected path %q, got %q", test.expectedKubeletPath, out)
+		}
+
+	}
+}
+
+func TestEvalSymlinks(t *testing.T) {
+	tests := []struct {
+		name        string
+		mustExist   bool
+		prepare     func(tmpdir string) (src string, expectedDst string, err error)
+		expectError bool
+	}{
+		{
+			name:      "simple file /src",
+			mustExist: true,
+			prepare: func(tmpdir string) (src string, expectedDst string, err error) {
+				src = filepath.Join(tmpdir, "src")
+				err = ioutil.WriteFile(src, []byte{}, 0644)
+				return src, src, err
+			},
+		},
+		{
+			name:      "non-existing file /src",
+			mustExist: true,
+			prepare: func(tmpdir string) (src string, expectedDst string, err error) {
+				src = filepath.Join(tmpdir, "src")
+				return src, "", nil
+			},
+			expectError: true,
+		},
+		{
+			name:      "non-existing file /src/ with mustExist=false",
+			mustExist: false,
+			prepare: func(tmpdir string) (src string, expectedDst string, err error) {
+				src = filepath.Join(tmpdir, "src")
+				return src, src, nil
+			},
+		},
+		{
+			name:      "non-existing file /existing/path/src with mustExist=false with existing directories",
+			mustExist: false,
+			prepare: func(tmpdir string) (src string, expectedDst string, err error) {
+				src = filepath.Join(tmpdir, "existing/path")
+				if err := os.MkdirAll(src, 0755); err != nil {
+					return "", "", err
+				}
+				src = filepath.Join(src, "src")
+				return src, src, nil
+			},
+		},
+		{
+			name:      "simple symlink /src -> /dst",
+			mustExist: false,
+			prepare: func(tmpdir string) (src string, expectedDst string, err error) {
+				dst := filepath.Join(tmpdir, "dst")
+				if err = ioutil.WriteFile(dst, []byte{}, 0644); err != nil {
+					return "", "", err
+				}
+				src = filepath.Join(tmpdir, "src")
+				err = os.Symlink(dst, src)
+				return src, dst, err
+			},
+		},
+		{
+			name:      "dangling symlink /src -> /non-existing-path",
+			mustExist: true,
+			prepare: func(tmpdir string) (src string, expectedDst string, err error) {
+				dst := filepath.Join(tmpdir, "non-existing-path")
+				src = filepath.Join(tmpdir, "src")
+				err = os.Symlink(dst, src)
+				return src, "", err
+			},
+			expectError: true,
+		},
+		{
+			name:      "dangling symlink /src -> /non-existing-path with mustExist=false",
+			mustExist: false,
+			prepare: func(tmpdir string) (src string, expectedDst string, err error) {
+				dst := filepath.Join(tmpdir, "non-existing-path")
+				src = filepath.Join(tmpdir, "src")
+				err = os.Symlink(dst, src)
+				return src, dst, err
+			},
+		},
+		{
+			name:      "symlink to directory /src/file, where /src is link to /dst",
+			mustExist: true,
+			prepare: func(tmpdir string) (src string, expectedDst string, err error) {
+				dst := filepath.Join(tmpdir, "dst")
+				if err = os.Mkdir(dst, 0755); err != nil {
+					return "", "", err
+				}
+				dstFile := filepath.Join(dst, "file")
+				if err = ioutil.WriteFile(dstFile, []byte{}, 0644); err != nil {
+					return "", "", err
+				}
+
+				src = filepath.Join(tmpdir, "src")
+				if err = os.Symlink(dst, src); err != nil {
+					return "", "", err
+				}
+				srcFile := filepath.Join(src, "file")
+				return srcFile, dstFile, nil
+			},
+		},
+		{
+			name:      "symlink to non-existing directory: /src/file, where /src is link to /dst and dst does not exist",
+			mustExist: true,
+			prepare: func(tmpdir string) (src string, expectedDst string, err error) {
+				dst := filepath.Join(tmpdir, "dst")
+
+				src = filepath.Join(tmpdir, "src")
+				if err = os.Symlink(dst, src); err != nil {
+					return "", "", err
+				}
+				srcFile := filepath.Join(src, "file")
+				return srcFile, "", nil
+			},
+			expectError: true,
+		},
+		{
+			name:      "symlink to non-existing directory: /src/file, where /src is link to /dst and dst does not exist with mustExist=false",
+			mustExist: false,
+			prepare: func(tmpdir string) (src string, expectedDst string, err error) {
+				dst := filepath.Join(tmpdir, "dst")
+				dstFile := filepath.Join(dst, "file")
+
+				src = filepath.Join(tmpdir, "src")
+				if err = os.Symlink(dst, src); err != nil {
+					return "", "", err
+				}
+				srcFile := filepath.Join(src, "file")
+				return srcFile, dstFile, nil
+			},
+		},
+	}
+
+	for _, test := range tests {
+		ns := Nsenter{
+			hostRootFsPath: "/rootfs",
+			executor: fakeExec{
+				rootfsPath: "/rootfs",
+			},
+		}
+
+		tmpdir, err := ioutil.TempDir("", "nsenter-hostpath-")
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer os.RemoveAll(tmpdir)
+
+		src, expectedDst, err := test.prepare(tmpdir)
+		if err != nil {
+			t.Error(err)
+			continue
+		}
+
+		dst, err := ns.EvalSymlinks(src, test.mustExist)
+		if err != nil && !test.expectError {
+			t.Errorf("Test %q: unexpected error: %s", test.name, err)
+		}
+		if err == nil && test.expectError {
+			t.Errorf("Test %q: expected error, got none", test.name)
+		}
+		if dst != expectedDst {
+			t.Errorf("Test %q: expected destination %q, got %q", test.name, expectedDst, dst)
+		}
+	}
+}
+
+func TestNewNsenter(t *testing.T) {
+	// Create a symlink /tmp/xyz/rootfs -> / and use it as rootfs path
+	// It should resolve all binaries correctly, the test runs on Linux
+
+	tmpdir, err := ioutil.TempDir("", "nsenter-hostpath-")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmpdir)
+
+	rootfs := filepath.Join(tmpdir, "rootfs")
+	if err = os.Symlink("/", rootfs); err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = NewNsenter(rootfs, exec.New())
+	if err != nil {
+		t.Errorf("Error: %s", err)
+	}
+}
+
+func TestNewNsenterError(t *testing.T) {
+	// Create empty dir /tmp/xyz/rootfs and use it as rootfs path
+	// It should resolve all binaries correctly, the test runs on Linux
+
+	tmpdir, err := ioutil.TempDir("", "nsenter-hostpath-")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmpdir)
+
+	rootfs := filepath.Join(tmpdir, "rootfs")
+	if err = os.MkdirAll(rootfs, 0755); err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = NewNsenter(rootfs, exec.New())
+	if err == nil {
+		t.Errorf("Expected error, got none")
+	}
+}

pkg/util/nsenter/nsenter_unsupported.go

@@ -22,6 +22,12 @@ import (
 	"k8s.io/utils/exec"
 )
 
+const (
+	// DefaultHostRootFsPath is path to host's filesystem mounted into container
+	// with kubelet.
+	DefaultHostRootFsPath = "/rootfs"
+)
+
 // Nsenter is part of experimental support for running the kubelet
 // in a container.
 type Nsenter struct {
@@ -30,7 +36,7 @@ type Nsenter struct {
 }
 
 // NewNsenter constructs a new instance of Nsenter
-func NewNsenter() (*Nsenter, error) {
+func NewNsenter(hostRootFsPath string, executor exec.Interface) (*Nsenter, error) {
 	return &Nsenter{}, nil
 }
 

pkg/util/removeall/removeall_test.go

@@ -75,8 +75,8 @@ func (mounter *fakeMounter) MakeFile(pathname string) error {
 	return nil
 }
 
-func (mounter *fakeMounter) ExistsPath(pathname string) bool {
-	return true
+func (mounter *fakeMounter) ExistsPath(pathname string) (bool, error) {
+	return true, errors.New("not implemented")
 }
 
 func (mounter *fakeMounter) PrepareSafeSubpath(subPath mount.Subpath) (newHostPath string, cleanupAction func(), err error) {
@@ -103,6 +103,10 @@ func (mounter *fakeMounter) GetSELinuxSupport(pathname string) (bool, error) {
 	return false, errors.New("not implemented")
 }
 
+func (mounter *fakeMounter) GetMode(pathname string) (os.FileMode, error) {
+	return 0, errors.New("not implemented")
+}
+
 func (mounter *fakeMounter) IsLikelyNotMountPoint(file string) (bool, error) {
 	name := path.Base(file)
 	if strings.HasPrefix(name, "mount") {

pkg/volume/host_path/host_path.go

@@ -350,7 +350,8 @@ type fileTypeChecker struct {
 }
 
 func (ftc *fileTypeChecker) Exists() bool {
-	return ftc.mounter.ExistsPath(ftc.path)
+	exists, err := ftc.mounter.ExistsPath(ftc.path)
+	return exists && err == nil
 }
 
 func (ftc *fileTypeChecker) IsFile() bool {

pkg/volume/host_path/host_path_test.go

@@ -369,8 +369,8 @@ func (fftc *fakeFileTypeChecker) MakeDir(pathname string) error {
 	return nil
 }
 
-func (fftc *fakeFileTypeChecker) ExistsPath(pathname string) bool {
-	return true
+func (fftc *fakeFileTypeChecker) ExistsPath(pathname string) (bool, error) {
+	return true, nil
 }
 
 func (fftc *fakeFileTypeChecker) GetFileType(_ string) (utilmount.FileType, error) {
@@ -401,6 +401,10 @@ func (fftc *fakeFileTypeChecker) GetSELinuxSupport(pathname string) (bool, error
 	return false, errors.New("not implemented")
 }
 
+func (fftc *fakeFileTypeChecker) GetMode(pathname string) (os.FileMode, error) {
+	return 0, errors.New("not implemented")
+}
+
 func setUp() error {
 	err := os.MkdirAll("/tmp/ExistingFolder", os.FileMode(0755))
 	if err != nil {