Avoid deleted iSCSI LUNs in the kernel

pkg/volume/iscsi/BUILD

@@ -18,6 +18,7 @@ go_library(
     importpath = "k8s.io/kubernetes/pkg/volume/iscsi",
     deps = [
         "//pkg/features:go_default_library",
+        "//pkg/util/keymutex:go_default_library",
         "//pkg/util/mount:go_default_library",
         "//pkg/util/strings:go_default_library",
         "//pkg/volume:go_default_library",

pkg/volume/iscsi/attacher.go

@@ -26,14 +26,16 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/kubernetes/pkg/features"
+	"k8s.io/kubernetes/pkg/util/keymutex"
 	"k8s.io/kubernetes/pkg/util/mount"
 	"k8s.io/kubernetes/pkg/volume"
 	volumeutil "k8s.io/kubernetes/pkg/volume/util"
 )
 
 type iscsiAttacher struct {
-	host    volume.VolumeHost
-	manager diskManager
+	host        volume.VolumeHost
+	targetLocks keymutex.KeyMutex
+	manager     diskManager
 }
 
 var _ volume.Attacher = &iscsiAttacher{}
@@ -42,8 +44,9 @@ var _ volume.AttachableVolumePlugin = &iscsiPlugin{}
 
 func (plugin *iscsiPlugin) NewAttacher() (volume.Attacher, error) {
 	return &iscsiAttacher{
-		host:    plugin.host,
-		manager: &ISCSIUtil{},
+		host:        plugin.host,
+		targetLocks: plugin.targetLocks,
+		manager:     &ISCSIUtil{},
 	}, nil
 }
 
@@ -66,7 +69,7 @@ func (attacher *iscsiAttacher) VolumesAreAttached(specs []*volume.Spec, nodeName
 }
 
 func (attacher *iscsiAttacher) WaitForAttach(spec *volume.Spec, devicePath string, pod *v1.Pod, timeout time.Duration) (string, error) {
-	mounter, err := volumeSpecToMounter(spec, attacher.host, pod)
+	mounter, err := volumeSpecToMounter(spec, attacher.host, attacher.targetLocks, pod)
 	if err != nil {
 		glog.Warningf("failed to get iscsi mounter: %v", err)
 		return "", err
@@ -76,7 +79,7 @@ func (attacher *iscsiAttacher) WaitForAttach(spec *volume.Spec, devicePath strin
 
 func (attacher *iscsiAttacher) GetDeviceMountPath(
 	spec *volume.Spec) (string, error) {
-	mounter, err := volumeSpecToMounter(spec, attacher.host, nil)
+	mounter, err := volumeSpecToMounter(spec, attacher.host, attacher.targetLocks, nil)
 	if err != nil {
 		glog.Warningf("failed to get iscsi mounter: %v", err)
 		return "", err
@@ -157,7 +160,7 @@ func (detacher *iscsiDetacher) UnmountDevice(deviceMountPath string) error {
 	return nil
 }
 
-func volumeSpecToMounter(spec *volume.Spec, host volume.VolumeHost, pod *v1.Pod) (*iscsiDiskMounter, error) {
+func volumeSpecToMounter(spec *volume.Spec, host volume.VolumeHost, targetLocks keymutex.KeyMutex, pod *v1.Pod) (*iscsiDiskMounter, error) {
 	var secret map[string]string
 	readOnly, fsType, err := getISCSIVolumeInfo(spec)
 	if err != nil {
@@ -165,7 +168,7 @@ func volumeSpecToMounter(spec *volume.Spec, host volume.VolumeHost, pod *v1.Pod)
 	}
 	var podUID types.UID
 	if pod != nil {
-		secret, err = createSecretMap(spec, &iscsiPlugin{host: host}, pod.Namespace)
+		secret, err = createSecretMap(spec, &iscsiPlugin{host: host, targetLocks: targetLocks}, pod.Namespace)
 		if err != nil {
 			return nil, err
 		}
@@ -173,7 +176,7 @@ func volumeSpecToMounter(spec *volume.Spec, host volume.VolumeHost, pod *v1.Pod)
 	}
 	iscsiDisk, err := createISCSIDisk(spec,
 		podUID,
-		&iscsiPlugin{host: host},
+		&iscsiPlugin{host: host, targetLocks: targetLocks},
 		&ISCSIUtil{},
 		secret,
 	)
@@ -214,7 +217,8 @@ func volumeSpecToUnmounter(mounter mount.Interface, host volume.VolumeHost) *isc
 		iscsiDisk: &iscsiDisk{
 			plugin: &iscsiPlugin{},
 		},
-		mounter: mounter,
-		exec:    exec,
+		mounter:    mounter,
+		exec:       exec,
+		deviceUtil: volumeutil.NewDeviceHandler(volumeutil.NewIOHandler()),
 	}
 }

pkg/volume/iscsi/iscsi.go

@@ -27,6 +27,7 @@ import (
 	"k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/kubernetes/pkg/util/keymutex"
 	"k8s.io/kubernetes/pkg/util/mount"
 	utilstrings "k8s.io/kubernetes/pkg/util/strings"
 	"k8s.io/kubernetes/pkg/volume"
@@ -36,11 +37,12 @@ import (
 
 // This is the primary entrypoint for volume plugins.
 func ProbeVolumePlugins() []volume.VolumePlugin {
-	return []volume.VolumePlugin{&iscsiPlugin{nil}}
+	return []volume.VolumePlugin{&iscsiPlugin{}}
 }
 
 type iscsiPlugin struct {
-	host volume.VolumeHost
+	host        volume.VolumeHost
+	targetLocks keymutex.KeyMutex
 }
 
 var _ volume.VolumePlugin = &iscsiPlugin{}
@@ -53,6 +55,7 @@ const (
 
 func (plugin *iscsiPlugin) Init(host volume.VolumeHost) error {
 	plugin.host = host
+	plugin.targetLocks = keymutex.NewKeyMutex()
 	return nil
 }
 
@@ -176,8 +179,9 @@ func (plugin *iscsiPlugin) newUnmounterInternal(volName string, podUID types.UID
 			plugin:          plugin,
 			MetricsProvider: volume.NewMetricsStatFS(plugin.host.GetPodVolumeDir(podUID, utilstrings.EscapeQualifiedNameForDisk(iscsiPluginName), volName)),
 		},
-		mounter: mounter,
-		exec:    exec,
+		mounter:    mounter,
+		exec:       exec,
+		deviceUtil: ioutil.NewDeviceHandler(ioutil.NewIOHandler()),
 	}, nil
 }
 
@@ -281,7 +285,7 @@ func (iscsi *iscsiDisk) GetPath() string {
 }
 
 func (iscsi *iscsiDisk) iscsiGlobalMapPath(spec *volume.Spec) (string, error) {
-	mounter, err := volumeSpecToMounter(spec, iscsi.plugin.host, nil /* pod */)
+	mounter, err := volumeSpecToMounter(spec, iscsi.plugin.host, iscsi.plugin.targetLocks, nil /* pod */)
 	if err != nil {
 		glog.Warningf("failed to get iscsi mounter: %v", err)
 		return "", err
@@ -337,8 +341,9 @@ func (b *iscsiDiskMounter) SetUpAt(dir string, fsGroup *int64) error {
 
 type iscsiDiskUnmounter struct {
 	*iscsiDisk
-	mounter mount.Interface
-	exec    mount.Exec
+	mounter    mount.Interface
+	exec       mount.Exec
+	deviceUtil ioutil.DeviceUtil
 }
 
 var _ volume.Unmounter = &iscsiDiskUnmounter{}

pkg/volume/util/device_util.go

@@ -20,6 +20,8 @@ package util
 type DeviceUtil interface {
 	FindMultipathDeviceForDevice(disk string) string
 	FindSlaveDevicesOnMultipath(disk string) []string
+	GetISCSIPortalHostMapForTarget(targetIqn string) (map[string]int, error)
+	FindDevicesForISCSILun(targetIqn string, lun int) ([]string, error)
 }
 
 type deviceHandler struct {

pkg/volume/util/device_util_linux.go

@@ -20,7 +20,10 @@ package util
 
 import (
 	"errors"
+	"fmt"
+	"github.com/golang/glog"
 	"path"
+	"strconv"
 	"strings"
 )
 
@@ -80,3 +83,206 @@ func (handler *deviceHandler) FindSlaveDevicesOnMultipath(dm string) []string {
 	}
 	return devices
 }
+
+// GetISCSIPortalHostMapForTarget given a target iqn, find all the scsi hosts logged into
+// that target. Returns a map of iSCSI portals (string) to SCSI host numbers (integers).
+// For example: {
+//    "192.168.30.7:3260": 2,
+//    "192.168.30.8:3260": 3,
+// }
+func (handler *deviceHandler) GetISCSIPortalHostMapForTarget(targetIqn string) (map[string]int, error) {
+	portalHostMap := make(map[string]int)
+	io := handler.get_io
+
+	// Iterate over all the iSCSI hosts in sysfs
+	sysPath := "/sys/class/iscsi_host"
+	hostDirs, err := io.ReadDir(sysPath)
+	if err != nil {
+		return nil, err
+	}
+	for _, hostDir := range hostDirs {
+		// iSCSI hosts are always of the format "host%d"
+		// See drivers/scsi/hosts.c in Linux
+		hostName := hostDir.Name()
+		if !strings.HasPrefix(hostName, "host") {
+			continue
+		}
+		hostNumber, err := strconv.Atoi(strings.TrimPrefix(hostName, "host"))
+		if err != nil {
+			glog.Errorf("Could not get number from iSCSI host: %s", hostName)
+			continue
+		}
+
+		// Iterate over the children of the iscsi_host device
+		// We are looking for the associated session
+		devicePath := sysPath + "/" + hostName + "/device"
+		deviceDirs, err := io.ReadDir(devicePath)
+		if err != nil {
+			return nil, err
+		}
+		for _, deviceDir := range deviceDirs {
+			// Skip over files that aren't the session
+			// Sessions are of the format "session%u"
+			// See drivers/scsi/scsi_transport_iscsi.c in Linux
+			sessionName := deviceDir.Name()
+			if !strings.HasPrefix(sessionName, "session") {
+				continue
+			}
+
+			sessionPath := devicePath + "/" + sessionName
+
+			// Read the target name for the iSCSI session
+			targetNamePath := sessionPath + "/iscsi_session/" + sessionName + "/targetname"
+			targetName, err := io.ReadFile(targetNamePath)
+			if err != nil {
+				return nil, err
+			}
+
+			// Ignore hosts that don't matchthe target we were looking for.
+			if strings.TrimSpace(string(targetName)) != targetIqn {
+				continue
+			}
+
+			// Iterate over the children of the iSCSI session looking
+			// for the iSCSI connection.
+			dirs2, err := io.ReadDir(sessionPath)
+			if err != nil {
+				return nil, err
+			}
+			for _, dir2 := range dirs2 {
+				// Skip over files that aren't the connection
+				// Connections are of the format "connection%d:%u"
+				// See drivers/scsi/scsi_transport_iscsi.c in Linux
+				dirName := dir2.Name()
+				if !strings.HasPrefix(dirName, "connection") {
+					continue
+				}
+
+				connectionPath := sessionPath + "/" + dirName + "/iscsi_connection/" + dirName
+
+				// Read the current and persistent portal information for the connection.
+				addrPath := connectionPath + "/address"
+				addr, err := io.ReadFile(addrPath)
+				if err != nil {
+					return nil, err
+				}
+
+				portPath := connectionPath + "/port"
+				port, err := io.ReadFile(portPath)
+				if err != nil {
+					return nil, err
+				}
+
+				persistentAddrPath := connectionPath + "/persistent_address"
+				persistentAddr, err := io.ReadFile(persistentAddrPath)
+				if err != nil {
+					return nil, err
+				}
+
+				persistentPortPath := connectionPath + "/persistent_port"
+				persistentPort, err := io.ReadFile(persistentPortPath)
+				if err != nil {
+					return nil, err
+				}
+
+				// Add entries to the map for both the current and persistent portals
+				// pointing to the SCSI host for those connections
+				portal := strings.TrimSpace(string(addr)) + ":" +
+					strings.TrimSpace(string(port))
+				portalHostMap[portal] = hostNumber
+
+				persistentPortal := strings.TrimSpace(string(persistentAddr)) + ":" +
+					strings.TrimSpace(string(persistentPort))
+				portalHostMap[persistentPortal] = hostNumber
+			}
+		}
+	}
+
+	return portalHostMap, nil
+}
+
+// FindDevicesForISCSILun given an iqn, and lun number, find all the devices
+// corresponding to that LUN.
+func (handler *deviceHandler) FindDevicesForISCSILun(targetIqn string, lun int) ([]string, error) {
+	devices := make([]string, 0)
+	io := handler.get_io
+
+	// Iterate over all the iSCSI hosts in sysfs
+	sysPath := "/sys/class/iscsi_host"
+	hostDirs, err := io.ReadDir(sysPath)
+	if err != nil {
+		return nil, err
+	}
+	for _, hostDir := range hostDirs {
+		// iSCSI hosts are always of the format "host%d"
+		// See drivers/scsi/hosts.c in Linux
+		hostName := hostDir.Name()
+		if !strings.HasPrefix(hostName, "host") {
+			continue
+		}
+		hostNumber, err := strconv.Atoi(strings.TrimPrefix(hostName, "host"))
+		if err != nil {
+			glog.Errorf("Could not get number from iSCSI host: %s", hostName)
+			continue
+		}
+
+		// Iterate over the children of the iscsi_host device
+		// We are looking for the associated session
+		devicePath := sysPath + "/" + hostName + "/device"
+		deviceDirs, err := io.ReadDir(devicePath)
+		if err != nil {
+			return nil, err
+		}
+		for _, deviceDir := range deviceDirs {
+			// Skip over files that aren't the session
+			// Sessions are of the format "session%u"
+			// See drivers/scsi/scsi_transport_iscsi.c in Linux
+			sessionName := deviceDir.Name()
+			if !strings.HasPrefix(sessionName, "session") {
+				continue
+			}
+
+			// Read the target name for the iSCSI session
+			targetNamePath := devicePath + "/" + sessionName + "/iscsi_session/" + sessionName + "/targetname"
+			targetName, err := io.ReadFile(targetNamePath)
+			if err != nil {
+				return nil, err
+			}
+
+			// Only if the session matches the target we were looking for,
+			// add it to the map
+			if strings.TrimSpace(string(targetName)) != targetIqn {
+				continue
+			}
+
+			// The list of block devices on the scsi bus will be in a
+			// directory called "target%d:%d:%d".
+			// See drivers/scsi/scsi_scan.c in Linux
+			// We assume the channel/bus and device/controller are always zero for iSCSI
+			targetPath := devicePath + "/" + sessionName + fmt.Sprintf("/target%d:0:0", hostNumber)
+
+			// The block device for a given lun will be "%d:%d:%d:%d" --
+			// host:channel:bus:LUN
+			blockDevicePath := targetPath + fmt.Sprintf("/%d:0:0:%d", hostNumber, lun)
+
+			// If the LUN doesn't exist on this bus, continue on
+			_, err = io.Lstat(blockDevicePath)
+			if err != nil {
+				continue
+			}
+
+			// Read the block directory, there should only be one child --
+			// the block device "sd*"
+			path := blockDevicePath + "/block"
+			dirs, err := io.ReadDir(path)
+			if err != nil {
+				return nil, err
+			}
+			if 0 < len(dirs) {
+				devices = append(devices, dirs[0].Name())
+			}
+		}
+	}
+
+	return devices, nil
+}

pkg/volume/util/device_util_linux_test.go

@@ -22,12 +22,41 @@ import (
 	"errors"
 	"os"
 	"reflect"
+	"regexp"
 	"testing"
 	"time"
 )
 
 type mockOsIOHandler struct{}
 
+func (handler *mockOsIOHandler) ReadFile(filename string) ([]byte, error) {
+	portPattern := regexp.MustCompile("^/sys/class/iscsi_host/(host\\d)/device/session\\d/connection\\d:0/iscsi_connection/connection\\d:0/(?:persistent_)?port$")
+	if portPattern.MatchString(filename) {
+		return []byte("3260"), nil
+	}
+	addressPattern := regexp.MustCompile("^/sys/class/iscsi_host/(host\\d)/device/session\\d/connection\\d:0/iscsi_connection/connection\\d:0/(?:persistent_)?address$")
+	matches := addressPattern.FindStringSubmatch(filename)
+	if nil != matches {
+		switch matches[1] {
+		case "host2":
+			return []byte("10.0.0.1"), nil
+		case "host3":
+			return []byte("10.0.0.2"), nil
+		}
+	}
+	targetNamePattern := regexp.MustCompile("^/sys/class/iscsi_host/(host\\d)/device/session\\d/iscsi_session/session\\d/targetname$")
+	matches = targetNamePattern.FindStringSubmatch(filename)
+	if nil != matches {
+		switch matches[1] {
+		case "host2":
+			return []byte("target1"), nil
+		case "host3":
+			return []byte("target2"), nil
+		}
+	}
+	return nil, errors.New("Not Implemented for Mock")
+}
+
 func (handler *mockOsIOHandler) ReadDir(dirname string) ([]os.FileInfo, error) {
 	switch dirname {
 	case "/sys/block/dm-1/slaves":
@@ -46,14 +75,81 @@ func (handler *mockOsIOHandler) ReadDir(dirname string) ([]os.FileInfo, error) {
 			name: "dm-1",
 		}
 		return []os.FileInfo{f1, f2}, nil
+	case "/sys/class/iscsi_host":
+		f1 := &fakeFileInfo{
+			name: "host2",
+		}
+		f2 := &fakeFileInfo{
+			name: "host3",
+		}
+		f3 := &fakeFileInfo{
+			name: "ignore",
+		}
+		return []os.FileInfo{f1, f2, f3}, nil
+	case "/sys/class/iscsi_host/host2/device":
+		f1 := &fakeFileInfo{
+			name: "session1",
+		}
+		f2 := &fakeFileInfo{
+			name: "ignore",
+		}
+		return []os.FileInfo{f1, f2}, nil
+	case "/sys/class/iscsi_host/host3/device":
+		f1 := &fakeFileInfo{
+			name: "session2",
+		}
+		f2 := &fakeFileInfo{
+			name: "ignore",
+		}
+		return []os.FileInfo{f1, f2}, nil
+	case "/sys/class/iscsi_host/host2/device/session1":
+		f1 := &fakeFileInfo{
+			name: "connection1:0",
+		}
+		f2 := &fakeFileInfo{
+			name: "ignore",
+		}
+		return []os.FileInfo{f1, f2}, nil
+	case "/sys/class/iscsi_host/host3/device/session2":
+		f1 := &fakeFileInfo{
+			name: "connection2:0",
+		}
+		f2 := &fakeFileInfo{
+			name: "ignore",
+		}
+		return []os.FileInfo{f1, f2}, nil
+	case "/sys/class/iscsi_host/host2/device/session1/target2:0:0/2:0:0:1/block":
+		f1 := &fakeFileInfo{
+			name: "sda",
+		}
+		return []os.FileInfo{f1}, nil
+	case "/sys/class/iscsi_host/host2/device/session1/target2:0:0/2:0:0:2/block":
+		f1 := &fakeFileInfo{
+			name: "sdc",
+		}
+		return []os.FileInfo{f1}, nil
+	case "/sys/class/iscsi_host/host3/device/session2/target3:0:0/3:0:0:1/block":
+		f1 := &fakeFileInfo{
+			name: "sdb",
+		}
+		return []os.FileInfo{f1}, nil
+	case "/sys/class/iscsi_host/host3/device/session2/target3:0:0/3:0:0:2/block":
+		f1 := &fakeFileInfo{
+			name: "sdd",
+		}
+		return []os.FileInfo{f1}, nil
 	}
-	return nil, nil
+	return nil, errors.New("Not Implemented for Mock")
 }
 
 func (handler *mockOsIOHandler) Lstat(name string) (os.FileInfo, error) {
 	links := map[string]string{
 		"/sys/block/dm-1/slaves/sda": "sda",
 		"/dev/sda":                   "sda",
+		"/sys/class/iscsi_host/host2/device/session1/target2:0:0/2:0:0:1": "2:0:0:1",
+		"/sys/class/iscsi_host/host2/device/session1/target2:0:0/2:0:0:2": "2:0:0:2",
+		"/sys/class/iscsi_host/host3/device/session2/target3:0:0/3:0:0:1": "3:0:0:1",
+		"/sys/class/iscsi_host/host3/device/session2/target3:0:0/3:0:0:2": "3:0:0:2",
 	}
 	if dev, ok := links[name]; ok {
 		return &fakeFileInfo{name: dev}, nil
@@ -158,3 +254,37 @@ func TestFindSlaveDevicesOnMultipath(t *testing.T) {
 		t.Fatalf("mpio device not found '' expected got [%s]", dev)
 	}
 }
+
+func TestGetISCSIPortalHostMapForTarget(t *testing.T) {
+	mockDeviceUtil := NewDeviceHandler(&mockOsIOHandler{})
+	portalHostMap, err := mockDeviceUtil.GetISCSIPortalHostMapForTarget("target1")
+	if nil != err {
+		t.Fatalf("error getting scsi hosts for target: %v", err)
+	}
+	if nil == portalHostMap {
+		t.Fatal("no portal host map returned")
+	}
+	if 1 != len(portalHostMap) {
+		t.Fatalf("wrong number of map entries in portal host map: %d", len(portalHostMap))
+	}
+	if 2 != portalHostMap["10.0.0.1:3260"] {
+		t.Fatalf("incorrect entry in portal host map: %v", portalHostMap)
+	}
+}
+
+func TestFindDevicesForISCSILun(t *testing.T) {
+	mockDeviceUtil := NewDeviceHandler(&mockOsIOHandler{})
+	devices, err := mockDeviceUtil.FindDevicesForISCSILun("target1", 1)
+	if nil != err {
+		t.Fatalf("error getting devices for lun: %v", err)
+	}
+	if nil == devices {
+		t.Fatal("no devices returned")
+	}
+	if 1 != len(devices) {
+		t.Fatalf("wrong number of devices: %d", len(devices))
+	}
+	if "sda" != devices[0] {
+		t.Fatalf("incorrect device %v", devices)
+	}
+}

pkg/volume/util/device_util_unsupported.go

@@ -28,3 +28,15 @@ func (handler *deviceHandler) FindSlaveDevicesOnMultipath(disk string) []string
 	out := []string{}
 	return out
 }
+
+// GetISCSIPortalHostMapForTarget unsupported returns nil
+func (handler *deviceHandler) GetISCSIPortalHostMapForTarget(targetIqn string) (map[string]int, error) {
+	portalHostMap := make(map[string]int)
+	return portalHostMap, nil
+}
+
+// FindDevicesForISCSILun unsupported returns nil
+func (handler *deviceHandler) FindDevicesForISCSILun(targetIqn string, lun int) ([]string, error) {
+	devices := []string{}
+	return devices, nil
+}

pkg/volume/util/io_util.go

@@ -24,6 +24,7 @@ import (
 
 // IoUtil is a mockable util for common IO operations
 type IoUtil interface {
+	ReadFile(filename string) ([]byte, error)
 	ReadDir(dirname string) ([]os.FileInfo, error)
 	Lstat(name string) (os.FileInfo, error)
 	EvalSymlinks(path string) (string, error)
@@ -36,6 +37,9 @@ func NewIOHandler() IoUtil {
 	return &osIOHandler{}
 }
 
+func (handler *osIOHandler) ReadFile(filename string) ([]byte, error) {
+	return ioutil.ReadFile(filename)
+}
 func (handler *osIOHandler) ReadDir(dirname string) ([]os.FileInfo, error) {
 	return ioutil.ReadDir(dirname)
 }