Merge pull request #73739 from jsafrane/parallelize-iscsi

test/e2e/framework/volume_util.go

@@ -81,6 +81,9 @@ const (
 	// PodCleanupTimeout is a waiting period for pod to be cleaned up and unmount its volumes so we
 	// don't tear down containers with NFS/Ceph/Gluster server too early.
 	PodCleanupTimeout = 20 * time.Second
+
+	// Template for iSCSI IQN.
+	iSCSIIQNTemplate = "iqn.2003-01.io.k8s:e2e.%s"
 )
 
 // VolumeTestConfig is a struct for configuration of one tests. The test consist of:
@@ -104,6 +107,8 @@ type VolumeTestConfig struct {
 	ServerVolumes map[string]string
 	// Message to wait for before starting clients
 	ServerReadyMessage string
+	// Use HostNetwork for the server
+	ServerHostNetwork bool
 	// Wait for the pod to terminate successfully
 	// False indicates that the pod is long running
 	WaitForCompletion bool
@@ -183,20 +188,30 @@ func NewGlusterfsServer(cs clientset.Interface, namespace string) (config Volume
 }
 
 // NewISCSIServer is an iSCSI-specific wrapper for CreateStorageServer.
-func NewISCSIServer(cs clientset.Interface, namespace string) (config VolumeTestConfig, pod *v1.Pod, ip string) {
+func NewISCSIServer(cs clientset.Interface, namespace string) (config VolumeTestConfig, pod *v1.Pod, ip, iqn string) {
+	// Generate cluster-wide unique IQN
+	iqn = fmt.Sprintf(iSCSIIQNTemplate, namespace)
+
 	config = VolumeTestConfig{
 		Namespace:   namespace,
 		Prefix:      "iscsi",
 		ServerImage: imageutils.GetE2EImage(imageutils.VolumeISCSIServer),
-		ServerPorts: []int{3260},
+		ServerArgs:  []string{iqn},
 		ServerVolumes: map[string]string{
 			// iSCSI container needs to insert modules from the host
 			"/lib/modules": "/lib/modules",
+			// iSCSI container needs to configure kernel
+			"/sys/kernel": "/sys/kernel",
+			// iSCSI source "block devices" must be available on the host
+			"/srv/iscsi": "/srv/iscsi",
 		},
-		ServerReadyMessage: "Configuration restored from /etc/target/saveconfig.json",
+		ServerReadyMessage: "iscsi target started",
+		ServerHostNetwork:  true,
 	}
 	pod, ip = CreateStorageServer(cs, config)
-	return config, pod, ip
+	// Make sure the client runs on the same node as server so we don't need to open any firewalls.
+	config.ClientNodeName = pod.Spec.NodeName
+	return config, pod, ip, iqn
 }
 
 // NewRBDServer is a CephRBD-specific wrapper for CreateStorageServer.
@@ -312,6 +327,7 @@ func StartVolumeServer(client clientset.Interface, config VolumeTestConfig) *v1.
 		},
 
 		Spec: v1.PodSpec{
+			HostNetwork: config.ServerHostNetwork,
 			Containers: []v1.Container{
 				{
 					Name:  serverPodName,

test/e2e/storage/drivers/in_tree.go

@@ -330,6 +330,7 @@ type iSCSIVolume struct {
 	serverPod *v1.Pod
 	serverIP  string
 	f         *framework.Framework
+	iqn       string
 }
 
 var _ testsuites.TestDriver = &iSCSIDriver{}
@@ -374,11 +375,10 @@ func (i *iSCSIDriver) GetVolumeSource(readOnly bool, fsType string, volume tests
 
 	volSource := v1.VolumeSource{
 		ISCSI: &v1.ISCSIVolumeSource{
-			TargetPortal: iv.serverIP + ":3260",
-			// from test/images/volume/iscsi/initiatorname.iscsi
-			IQN:      "iqn.2003-01.org.linux-iscsi.f21.x8664:sn.4b0aae584f7c",
-			Lun:      0,
-			ReadOnly: readOnly,
+			TargetPortal: "127.0.0.1:3260",
+			IQN:          iv.iqn,
+			Lun:          0,
+			ReadOnly:     readOnly,
 		},
 	}
 	if fsType != "" {
@@ -393,8 +393,8 @@ func (i *iSCSIDriver) GetPersistentVolumeSource(readOnly bool, fsType string, vo
 
 	pvSource := v1.PersistentVolumeSource{
 		ISCSI: &v1.ISCSIPersistentVolumeSource{
-			TargetPortal: iv.serverIP + ":3260",
-			IQN:          "iqn.2003-01.org.linux-iscsi.f21.x8664:sn.4b0aae584f7c",
+			TargetPortal: "127.0.0.1:3260",
+			IQN:          iv.iqn,
 			Lun:          0,
 			ReadOnly:     readOnly,
 		},
@@ -418,11 +418,13 @@ func (i *iSCSIDriver) CreateVolume(config *testsuites.PerTestConfig, volType tes
 	cs := f.ClientSet
 	ns := f.Namespace
 
-	c, serverPod, serverIP := framework.NewISCSIServer(cs, ns.Name)
+	c, serverPod, serverIP, iqn := framework.NewISCSIServer(cs, ns.Name)
 	config.ServerConfig = &c
+	config.ClientNodeName = c.ClientNodeName
 	return &iSCSIVolume{
 		serverPod: serverPod,
 		serverIP:  serverIP,
+		iqn:       iqn,
 		f:         f,
 	}
 }

test/e2e/storage/testsuites/volumes.go

@@ -168,7 +168,7 @@ func (t *volumesTestSuite) defineTests(driver TestDriver, pattern testpatterns.T
 		init()
 		defer cleanup()
 
-		testScriptInPod(f, l.resource.volType, l.resource.volSource, l.config.ClientNodeSelector)
+		testScriptInPod(f, l.resource.volType, l.resource.volSource, l.config)
 	})
 }
 
@@ -176,7 +176,7 @@ func testScriptInPod(
 	f *framework.Framework,
 	volumeType string,
 	source *v1.VolumeSource,
-	nodeSelector map[string]string) {
+	config *PerTestConfig) {
 
 	const (
 		volPath = "/vol1"
@@ -217,7 +217,8 @@ func testScriptInPod(
 				},
 			},
 			RestartPolicy: v1.RestartPolicyNever,
-			NodeSelector:  nodeSelector,
+			NodeSelector:  config.ClientNodeSelector,
+			NodeName:      config.ClientNodeName,
 		},
 	}
 	By(fmt.Sprintf("Creating pod %s", pod.Name))

test/images/volume/iscsi/Dockerfile

@@ -16,20 +16,8 @@ FROM BASEIMAGE
 
 CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/
 
-RUN yum install -y iscsi-initiator-utils targetcli net-tools strace procps-ng psmisc && yum clean all
-ADD run_iscsid.sh /usr/local/bin/
-ADD initiatorname.iscsi /etc/iscsi/
+RUN yum install -y targetcli && yum clean all
+ADD run_iscsi_target.sh /usr/local/bin/
 ADD block.tar.gz /
 
-# This JSON file was generated by targetcli with these commands:
-# /backstores/fileio create block /block
-# /iscsi create
-# # Enable demo mode (no authentication!):
-# /iscsi/iqn.2003-01.org.linux-iscsi.f21.x8664:sn.4b0aae584f7c/tpg1 set attribute authentication=0 demo_mode_write_protect=0 generate_node_acls=1 cache_dynamic_acls=1
-# /iscsi/iqn.2003-01.org.linux-iscsi.f21.x8664:sn.4b0aae584f7c/tpg1/luns create /backstores/fileio/block
-# saveconfig
-ADD saveconfig.json /etc/target/
-
-EXPOSE 3260/tcp
-
-ENTRYPOINT ["/usr/local/bin/run_iscsid.sh"]
+ENTRYPOINT ["/usr/local/bin/run_iscsi_target.sh"]

test/images/volume/iscsi/VERSION

-1.0
+2.0

test/images/volume/iscsi/create_block.sh

@@ -40,7 +40,7 @@ mkfs.ext2 block
 
 # Add index.html to it
 mount -o loop block $MNTDIR
-echo "Hello from iSCSI" > $MNTDIR/index.html
+echo "Hello from iscsi" > $MNTDIR/index.html
 umount $MNTDIR
 
 rm block.tar.gz 2>/dev/null || :

test/images/volume/iscsi/initiatorname.iscsi

-InitiatorName=iqn.1994-05.com.redhat:eb59fbe2c4c5

test/images/volume/iscsi/run_iscsid.sh → test/images/volume/iscsi/run_iscsi_target.sh

 #!/usr/bin/env bash
 
-# Copyright 2015 The Kubernetes Authors.
+# Copyright 2018 The Kubernetes Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,31 +14,52 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# This script does not run any daemon, it only configures iSCSI target (=server)
+# in kernel. It is possible to run this script multiple times on a single
+# node, each run will create its own IQN and LUN.
+
+# Kubernetes must provide unique name.
+IQN=$1
+
+# targetcli synchronizes over dbus, however it does not work in
+# containers. Use flock instead
+LOCK=/srv/iscsi/targetcli.lock
+
 function start()
 {
-    # targetcli need dbus
+    # targetcli need dbus. It may not run on the host, so start a private one
     mkdir /run/dbus
     dbus-daemon  --system
 
-    # clear any previous configuration
-    targetcli clearconfig confirm=True
+    # Create new IQN (iSCSI Qualified Name)
+    flock $LOCK targetcli /iscsi create "$IQN"
+    # Run it in demo mode, i.e. no authentication
+    flock $LOCK targetcli /iscsi/"$IQN"/tpg1 set attribute authentication=0 demo_mode_write_protect=0 generate_node_acls=1 cache_dynamic_acls=1
 
-    # restore configuration from saveconfig.json
-    targetcli restoreconfig
+    # Create unique "block volume" (i.e. flat file) on the *host*.
+    # Having it in the container confuses kernel from some reason
+    # and it's not able to server multiple LUNs from different
+    # containers.
+    # /srv/iscsi must be bind-mount from the host.
+    cp /block /srv/iscsi/"$IQN"
 
-    # maximum log level
-    iscsid -f -d 8
+    # Make the block volume available through our IQN as LUN 0
+    flock $LOCK targetcli /backstores/fileio create block-"$IQN" /srv/iscsi/"$IQN"
+    flock $LOCK targetcli /iscsi/"$IQN"/tpg1/luns create /backstores/fileio/block-"$IQN"
 
-    echo "iscsid started"
+    echo "iscsi target started"
 }
 
 function stop()
 {
-    echo "Stopping iscsid"
-    killall iscsid
-    targetcli clearconfig confirm=True
-
-    echo "iscsid stopped"
+    echo "stopping iscsi target"
+    # Remove IQN
+    flock $LOCK targetcli /iscsi/"$IQN"/tpg1/luns/ delete 0
+    flock $LOCK targetcli /iscsi delete "$IQN"
+    # Remove block device mapping
+    flock $LOCK targetcli /backstores/fileio delete block-"$IQN"
+    /bin/rm -f /srv/iscsi/"$IQN"
+    echo "iscsi target stopped"
     exit 0
 }
 
@@ -47,5 +68,5 @@ trap stop TERM
 start
 
 while true; do
-    sleep 5
+    sleep 1
 done

test/images/volume/iscsi/saveconfig.json

-{
-  "fabric_modules": [],
-  "storage_objects": [
-    {
-      "attributes": {
-        "block_size": 512,
-        "emulate_3pc": 1,
-        "emulate_caw": 1,
-        "emulate_dpo": 0,
-        "emulate_fua_read": 0,
-        "emulate_fua_write": 1,
-        "emulate_model_alias": 1,
-        "emulate_rest_reord": 0,
-        "emulate_tas": 1,
-        "emulate_tpu": 0,
-        "emulate_tpws": 0,
-        "emulate_ua_intlck_ctrl": 0,
-        "emulate_write_cache": 1,
-        "enforce_pr_isids": 1,
-        "force_pr_aptpl": 0,
-        "is_nonrot": 0,
-        "max_unmap_block_desc_count": 1,
-        "max_unmap_lba_count": 8192,
-        "max_write_same_len": 4096,
-        "optimal_sectors": 16384,
-        "pi_prot_format": 0,
-        "pi_prot_type": 0,
-        "queue_depth": 128,
-        "unmap_granularity": 1,
-        "unmap_granularity_alignment": 0
-      },
-      "dev": "block",
-      "name": "block",
-      "plugin": "fileio",
-      "size": 126877696,
-      "write_back": true,
-      "wwn": "521c57aa-9d9b-4e5d-ab1a-527487f92a33"
-    }
-  ],
-  "targets": [
-    {
-      "fabric": "iscsi",
-      "tpgs": [
-        {
-          "attributes": {
-            "authentication": 0,
-            "cache_dynamic_acls": 1,
-            "default_cmdsn_depth": 64,
-            "default_erl": 0,
-            "demo_mode_discovery": 1,
-            "demo_mode_write_protect": 0,
-            "generate_node_acls": 1,
-            "login_timeout": 15,
-            "netif_timeout": 2,
-            "prod_mode_write_protect": 0,
-            "t10_pi": 0
-          },
-          "enable": true,
-          "luns": [
-            {
-              "index": 0,
-              "storage_object": "/backstores/fileio/block"
-            }
-          ],
-          "node_acls": [],
-          "parameters": {
-            "AuthMethod": "CHAP,None",
-            "DataDigest": "CRC32C,None",
-            "DataPDUInOrder": "Yes",
-            "DataSequenceInOrder": "Yes",
-            "DefaultTime2Retain": "20",
-            "DefaultTime2Wait": "2",
-            "ErrorRecoveryLevel": "0",
-            "FirstBurstLength": "65536",
-            "HeaderDigest": "CRC32C,None",
-            "IFMarkInt": "2048~65535",
-            "IFMarker": "No",
-            "ImmediateData": "Yes",
-            "InitialR2T": "Yes",
-            "MaxBurstLength": "262144",
-            "MaxConnections": "1",
-            "MaxOutstandingR2T": "1",
-            "MaxRecvDataSegmentLength": "8192",
-            "MaxXmitDataSegmentLength": "262144",
-            "OFMarkInt": "2048~65535",
-            "OFMarker": "No",
-            "TargetAlias": "LIO Target"
-          },
-          "portals": [
-            {
-              "ip_address": "0.0.0.0",
-              "iser": false,
-              "port": 3260
-            }
-          ],
-          "tag": 1
-        }
-      ],
-      "wwn": "iqn.2003-01.org.linux-iscsi.f21.x8664:sn.4b0aae584f7c"
-    }
-  ]
-}

test/utils/image/manifest.go

@@ -238,9 +238,9 @@ func initImageConfigs() map[int]Config {
 	configs[ServeHostname] = Config{e2eRegistry, "serve-hostname", "1.1"}
 	configs[TestWebserver] = Config{e2eRegistry, "test-webserver", "1.0"}
 	configs[VolumeNFSServer] = Config{e2eRegistry, "volume/nfs", "1.0"}
-	configs[VolumeISCSIServer] = Config{e2eRegistry, "volume/iscsi", "1.0"}
+	configs[VolumeISCSIServer] = Config{"quay.io", "jsafrane/iscsi-test", "2.0"}
 	configs[VolumeGlusterServer] = Config{e2eRegistry, "volume/gluster", "1.0"}
-	configs[VolumeRBDServer] = Config{e2eRegistry, "volume/rbd", "1.0.1"}
+	configs[VolumeRBDServer] = Config{"quay.io", "jsafrane/rbd-test", "2.0"}
 	return configs
 }