Skip to content

K
k3s
Commit b18dba37 authored by Jan Safranek's avatar Jan Safranek
Browse files
Options

Parallelize iSCSI tests

iSCSI target (=the server) is implemented in Linux kernel. The "iSCSI server" pod is not a real server, it just configures the kernel on the host. In order to run iSCSI tests in parallel, we need to be able to run multiple such pods on a single node, serving different LUNs to different tests. The "server pod" must run with HostNetwork=true to achieve that. Each pod then creates its own IQN with namespace name, so it can't collide with other server pods running in another namespaces on the same node.
parent a93f803f
Hide whitespace changes
Inline Side-by-side
Showing with 76 additions and 151 deletions
test/e2e/framework/volume_util.go
View file @ b18dba37
...@@ -81,6 +81,9 @@ const ( ...@@ -81,6 +81,9 @@ const (
// PodCleanupTimeout is a waiting period for pod to be cleaned up and unmount its volumes so we // PodCleanupTimeout is a waiting period for pod to be cleaned up and unmount its volumes so we
// don't tear down containers with NFS/Ceph/Gluster server too early. // don't tear down containers with NFS/Ceph/Gluster server too early.
PodCleanupTimeout = 20 * time.Second PodCleanupTimeout = 20 * time.Second
// Template for iSCSI IQN.
iSCSIIQNTemplate = "iqn.2003-01.io.k8s:e2e.%s"
) )
// VolumeTestConfig is a struct for configuration of one tests. The test consist of: // VolumeTestConfig is a struct for configuration of one tests. The test consist of:
...@@ -104,6 +107,8 @@ type VolumeTestConfig struct { ...@@ -104,6 +107,8 @@ type VolumeTestConfig struct {
ServerVolumes map[string]string ServerVolumes map[string]string
// Message to wait for before starting clients // Message to wait for before starting clients
ServerReadyMessage string ServerReadyMessage string
// Use HostNetwork for the server
ServerHostNetwork bool
// Wait for the pod to terminate successfully // Wait for the pod to terminate successfully
// False indicates that the pod is long running // False indicates that the pod is long running
WaitForCompletion bool WaitForCompletion bool
...@@ -183,20 +188,30 @@ func NewGlusterfsServer(cs clientset.Interface, namespace string) (config Volume ...@@ -183,20 +188,30 @@ func NewGlusterfsServer(cs clientset.Interface, namespace string) (config Volume
} }
// NewISCSIServer is an iSCSI-specific wrapper for CreateStorageServer. // NewISCSIServer is an iSCSI-specific wrapper for CreateStorageServer.
func NewISCSIServer(cs clientset.Interface, namespace string) (config VolumeTestConfig, pod *v1.Pod, ip string) { func NewISCSIServer(cs clientset.Interface, namespace string) (config VolumeTestConfig, pod *v1.Pod, ip, iqn string) {
// Generate cluster-wide unique IQN
iqn = fmt.Sprintf(iSCSIIQNTemplate, namespace)
config = VolumeTestConfig{ config = VolumeTestConfig{
Namespace: namespace, Namespace: namespace,
Prefix: "iscsi", Prefix: "iscsi",
ServerImage: imageutils.GetE2EImage(imageutils.VolumeISCSIServer), ServerImage: imageutils.GetE2EImage(imageutils.VolumeISCSIServer),
ServerPorts: []int{3260}, ServerArgs: []string{iqn},
ServerVolumes: map[string]string{ ServerVolumes: map[string]string{
// iSCSI container needs to insert modules from the host // iSCSI container needs to insert modules from the host
"/lib/modules": "/lib/modules", "/lib/modules": "/lib/modules",
// iSCSI container needs to configure kernel
"/sys/kernel": "/sys/kernel",
// iSCSI source "block devices" must be available on the host
"/srv/iscsi": "/srv/iscsi",
}, },
ServerReadyMessage: "Configuration restored from /etc/target/saveconfig.json", ServerReadyMessage: "iscsi target started",
ServerHostNetwork: true,
} }
pod, ip = CreateStorageServer(cs, config) pod, ip = CreateStorageServer(cs, config)
return config, pod, ip // Make sure the client runs on the same node as server so we don't need to open any firewalls.
config.ClientNodeName = pod.Spec.NodeName
return config, pod, ip, iqn
} }
// NewRBDServer is a CephRBD-specific wrapper for CreateStorageServer. // NewRBDServer is a CephRBD-specific wrapper for CreateStorageServer.
...@@ -312,6 +327,7 @@ func StartVolumeServer(client clientset.Interface, config VolumeTestConfig) *v1. ...@@ -312,6 +327,7 @@ func StartVolumeServer(client clientset.Interface, config VolumeTestConfig) *v1.
}, },
Spec: v1.PodSpec{ Spec: v1.PodSpec{
HostNetwork: config.ServerHostNetwork,
Containers: []v1.Container{ Containers: []v1.Container{
{ {
Name: serverPodName, Name: serverPodName,
......
test/e2e/storage/drivers/in_tree.go
View file @ b18dba37
...@@ -330,6 +330,7 @@ type iSCSIVolume struct { ...@@ -330,6 +330,7 @@ type iSCSIVolume struct {
serverPod *v1.Pod serverPod *v1.Pod
serverIP string serverIP string
f *framework.Framework f *framework.Framework
iqn string
} }
var _ testsuites.TestDriver = &iSCSIDriver{} var _ testsuites.TestDriver = &iSCSIDriver{}
...@@ -374,11 +375,10 @@ func (i *iSCSIDriver) GetVolumeSource(readOnly bool, fsType string, volume tests ...@@ -374,11 +375,10 @@ func (i *iSCSIDriver) GetVolumeSource(readOnly bool, fsType string, volume tests
volSource := v1.VolumeSource{ volSource := v1.VolumeSource{
ISCSI: &v1.ISCSIVolumeSource{ ISCSI: &v1.ISCSIVolumeSource{
TargetPortal: iv.serverIP + ":3260", TargetPortal: "127.0.0.1:3260",
// from test/images/volume/iscsi/initiatorname.iscsi IQN: iv.iqn,
IQN: "iqn.2003-01.org.linux-iscsi.f21.x8664:sn.4b0aae584f7c", Lun: 0,
Lun: 0, ReadOnly: readOnly,
ReadOnly: readOnly,
}, },
} }
if fsType != "" { if fsType != "" {
...@@ -393,8 +393,8 @@ func (i *iSCSIDriver) GetPersistentVolumeSource(readOnly bool, fsType string, vo ...@@ -393,8 +393,8 @@ func (i *iSCSIDriver) GetPersistentVolumeSource(readOnly bool, fsType string, vo
pvSource := v1.PersistentVolumeSource{ pvSource := v1.PersistentVolumeSource{
ISCSI: &v1.ISCSIPersistentVolumeSource{ ISCSI: &v1.ISCSIPersistentVolumeSource{
TargetPortal: iv.serverIP + ":3260", TargetPortal: "127.0.0.1:3260",
IQN: "iqn.2003-01.org.linux-iscsi.f21.x8664:sn.4b0aae584f7c", IQN: iv.iqn,
Lun: 0, Lun: 0,
ReadOnly: readOnly, ReadOnly: readOnly,
}, },
...@@ -418,11 +418,13 @@ func (i *iSCSIDriver) CreateVolume(config *testsuites.PerTestConfig, volType tes ...@@ -418,11 +418,13 @@ func (i *iSCSIDriver) CreateVolume(config *testsuites.PerTestConfig, volType tes
cs := f.ClientSet cs := f.ClientSet
ns := f.Namespace ns := f.Namespace
c, serverPod, serverIP := framework.NewISCSIServer(cs, ns.Name) c, serverPod, serverIP, iqn := framework.NewISCSIServer(cs, ns.Name)
config.ServerConfig = &c config.ServerConfig = &c
config.ClientNodeName = c.ClientNodeName
return &iSCSIVolume{ return &iSCSIVolume{
serverPod: serverPod, serverPod: serverPod,
serverIP: serverIP, serverIP: serverIP,
iqn: iqn,
f: f, f: f,
} }
} }
......
test/e2e/storage/testsuites/volumes.go
View file @ b18dba37
...@@ -168,7 +168,7 @@ func (t *volumesTestSuite) defineTests(driver TestDriver, pattern testpatterns.T ...@@ -168,7 +168,7 @@ func (t *volumesTestSuite) defineTests(driver TestDriver, pattern testpatterns.T
init() init()
defer cleanup() defer cleanup()
testScriptInPod(f, l.resource.volType, l.resource.volSource, l.config.ClientNodeSelector) testScriptInPod(f, l.resource.volType, l.resource.volSource, l.config)
}) })
} }
...@@ -176,7 +176,7 @@ func testScriptInPod( ...@@ -176,7 +176,7 @@ func testScriptInPod(
f *framework.Framework, f *framework.Framework,
volumeType string, volumeType string,
source *v1.VolumeSource, source *v1.VolumeSource,
nodeSelector map[string]string) { config *PerTestConfig) {
const ( const (
volPath = "/vol1" volPath = "/vol1"
...@@ -217,7 +217,8 @@ func testScriptInPod( ...@@ -217,7 +217,8 @@ func testScriptInPod(
}, },
}, },
RestartPolicy: v1.RestartPolicyNever, RestartPolicy: v1.RestartPolicyNever,
NodeSelector: nodeSelector, NodeSelector: config.ClientNodeSelector,
NodeName: config.ClientNodeName,
}, },
} }
By(fmt.Sprintf("Creating pod %s", pod.Name)) By(fmt.Sprintf("Creating pod %s", pod.Name))
......
test/images/volume/iscsi/Dockerfile
View file @ b18dba37
...@@ -16,20 +16,8 @@ FROM BASEIMAGE ...@@ -16,20 +16,8 @@ FROM BASEIMAGE
CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/ CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/
RUN yum install -y iscsi-initiator-utils targetcli net-tools strace procps-ng psmisc && yum clean all RUN yum install -y targetcli && yum clean all
ADD run_iscsid.sh /usr/local/bin/ ADD run_iscsi_target.sh /usr/local/bin/
ADD initiatorname.iscsi /etc/iscsi/
ADD block.tar.gz / ADD block.tar.gz /
# This JSON file was generated by targetcli with these commands: ENTRYPOINT ["/usr/local/bin/run_iscsi_target.sh"]
# /backstores/fileio create block /block
# /iscsi create
# # Enable demo mode (no authentication!):
# /iscsi/iqn.2003-01.org.linux-iscsi.f21.x8664:sn.4b0aae584f7c/tpg1 set attribute authentication=0 demo_mode_write_protect=0 generate_node_acls=1 cache_dynamic_acls=1
# /iscsi/iqn.2003-01.org.linux-iscsi.f21.x8664:sn.4b0aae584f7c/tpg1/luns create /backstores/fileio/block
# saveconfig
ADD saveconfig.json /etc/target/
EXPOSE 3260/tcp
ENTRYPOINT ["/usr/local/bin/run_iscsid.sh"]
test/images/volume/iscsi/create_block.sh
View file @ b18dba37
...@@ -40,7 +40,7 @@ mkfs.ext2 block ...@@ -40,7 +40,7 @@ mkfs.ext2 block
# Add index.html to it # Add index.html to it
mount -o loop block $MNTDIR mount -o loop block $MNTDIR
echo "Hello from iSCSI" > $MNTDIR/index.html echo "Hello from iscsi" > $MNTDIR/index.html
umount $MNTDIR umount $MNTDIR
rm block.tar.gz 2>/dev/null || : rm block.tar.gz 2>/dev/null || :
......
test/images/volume/iscsi/initiatorname.iscsi deleted 100644 → 0
View file @ a93f803f
InitiatorName=iqn.1994-05.com.redhat:eb59fbe2c4c5
test/images/volume/iscsi/run_iscsid.sh → test/images/volume/iscsi/run_iscsi_target.sh
View file @ b18dba37
#!/usr/bin/env bash #!/usr/bin/env bash
# Copyright 2015 The Kubernetes Authors. # Copyright 2018 The Kubernetes Authors.
# #
# Licensed under the Apache License, Version 2.0 (the "License"); # Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License. # you may not use this file except in compliance with the License.
...@@ -14,31 +14,52 @@ ...@@ -14,31 +14,52 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
# This script does not run any daemon, it only configures iSCSI target (=server)
# in kernel. It is possible to run this script multiple times on a single
# node, each run will create its own IQN and LUN.
# Kubernetes must provide unique name.
IQN=$1
# targetcli synchronizes over dbus, however it does not work in
# containers. Use flock instead
LOCK=/srv/iscsi/targetcli.lock
function start() function start()
{ {
# targetcli need dbus # targetcli need dbus. It may not run on the host, so start a private one
mkdir /run/dbus mkdir /run/dbus
dbus-daemon --system dbus-daemon --system
# clear any previous configuration # Create new IQN (iSCSI Qualified Name)
targetcli clearconfig confirm=True flock $LOCK targetcli /iscsi create "$IQN"
# Run it in demo mode, i.e. no authentication
flock $LOCK targetcli /iscsi/"$IQN"/tpg1 set attribute authentication=0 demo_mode_write_protect=0 generate_node_acls=1 cache_dynamic_acls=1
# restore configuration from saveconfig.json # Create unique "block volume" (i.e. flat file) on the *host*.
targetcli restoreconfig # Having it in the container confuses kernel from some reason
# and it's not able to server multiple LUNs from different
# containers.
# /srv/iscsi must be bind-mount from the host.
cp /block /srv/iscsi/"$IQN"
# maximum log level # Make the block volume available through our IQN as LUN 0
iscsid -f -d 8 flock $LOCK targetcli /backstores/fileio create block-"$IQN" /srv/iscsi/"$IQN"
flock $LOCK targetcli /iscsi/"$IQN"/tpg1/luns create /backstores/fileio/block-"$IQN"
echo "iscsid started" echo "iscsi target started"
} }
function stop() function stop()
{ {
echo "Stopping iscsid" echo "stopping iscsi target"
killall iscsid # Remove IQN
targetcli clearconfig confirm=True flock $LOCK targetcli /iscsi/"$IQN"/tpg1/luns/ delete 0
flock $LOCK targetcli /iscsi delete "$IQN"
echo "iscsid stopped" # Remove block device mapping
flock $LOCK targetcli /backstores/fileio delete block-"$IQN"
/bin/rm -f /srv/iscsi/"$IQN"
echo "iscsi target stopped"
exit 0 exit 0
} }
...@@ -47,5 +68,5 @@ trap stop TERM ...@@ -47,5 +68,5 @@ trap stop TERM
start start
while true; do while true; do
sleep 5 sleep 1
done done
test/images/volume/iscsi/saveconfig.json deleted 100644 → 0
View file @ a93f803f
{
"fabric_modules": [],
"storage_objects": [
{
"attributes": {
"block_size": 512,
"emulate_3pc": 1,
"emulate_caw": 1,
"emulate_dpo": 0,
"emulate_fua_read": 0,
"emulate_fua_write": 1,
"emulate_model_alias": 1,
"emulate_rest_reord": 0,
"emulate_tas": 1,
"emulate_tpu": 0,
"emulate_tpws": 0,
"emulate_ua_intlck_ctrl": 0,
"emulate_write_cache": 1,
"enforce_pr_isids": 1,
"force_pr_aptpl": 0,
"is_nonrot": 0,
"max_unmap_block_desc_count": 1,
"max_unmap_lba_count": 8192,
"max_write_same_len": 4096,
"optimal_sectors": 16384,
"pi_prot_format": 0,
"pi_prot_type": 0,
"queue_depth": 128,
"unmap_granularity": 1,
"unmap_granularity_alignment": 0
},
"dev": "block",
"name": "block",
"plugin": "fileio",
"size": 126877696,
"write_back": true,
"wwn": "521c57aa-9d9b-4e5d-ab1a-527487f92a33"
}
],
"targets": [
{
"fabric": "iscsi",
"tpgs": [
{
"attributes": {
"authentication": 0,
"cache_dynamic_acls": 1,
"default_cmdsn_depth": 64,
"default_erl": 0,
"demo_mode_discovery": 1,
"demo_mode_write_protect": 0,
"generate_node_acls": 1,
"login_timeout": 15,
"netif_timeout": 2,
"prod_mode_write_protect": 0,
"t10_pi": 0
},
"enable": true,
"luns": [
{
"index": 0,
"storage_object": "/backstores/fileio/block"
}
],
"node_acls": [],
"parameters": {
"AuthMethod": "CHAP,None",
"DataDigest": "CRC32C,None",
"DataPDUInOrder": "Yes",
"DataSequenceInOrder": "Yes",
"DefaultTime2Retain": "20",
"DefaultTime2Wait": "2",
"ErrorRecoveryLevel": "0",
"FirstBurstLength": "65536",
"HeaderDigest": "CRC32C,None",
"IFMarkInt": "2048~65535",
"IFMarker": "No",
"ImmediateData": "Yes",
"InitialR2T": "Yes",
"MaxBurstLength": "262144",
"MaxConnections": "1",
"MaxOutstandingR2T": "1",
"MaxRecvDataSegmentLength": "8192",
"MaxXmitDataSegmentLength": "262144",
"OFMarkInt": "2048~65535",
"OFMarker": "No",
"TargetAlias": "LIO Target"
},
"portals": [
{
"ip_address": "0.0.0.0",
"iser": false,
"port": 3260
}
],
"tag": 1
}
],
"wwn": "iqn.2003-01.org.linux-iscsi.f21.x8664:sn.4b0aae584f7c"
}
]
}
test/utils/image/manifest.go
View file @ b18dba37
...@@ -238,7 +238,7 @@ func initImageConfigs() map[int]Config { ...@@ -238,7 +238,7 @@ func initImageConfigs() map[int]Config {
configs[ServeHostname] = Config{e2eRegistry, "serve-hostname", "1.1"} configs[ServeHostname] = Config{e2eRegistry, "serve-hostname", "1.1"}
configs[TestWebserver] = Config{e2eRegistry, "test-webserver", "1.0"} configs[TestWebserver] = Config{e2eRegistry, "test-webserver", "1.0"}
configs[VolumeNFSServer] = Config{e2eRegistry, "volume/nfs", "1.0"} configs[VolumeNFSServer] = Config{e2eRegistry, "volume/nfs", "1.0"}
configs[VolumeISCSIServer] = Config{e2eRegistry, "volume/iscsi", "1.0"} configs[VolumeISCSIServer] = Config{e2eRegistry, "volume/iscsi", "2.0"}
configs[VolumeGlusterServer] = Config{e2eRegistry, "volume/gluster", "1.0"} configs[VolumeGlusterServer] = Config{e2eRegistry, "volume/gluster", "1.0"}
configs[VolumeRBDServer] = Config{e2eRegistry, "volume/rbd", "1.0.1"} configs[VolumeRBDServer] = Config{e2eRegistry, "volume/rbd", "1.0.1"}
return configs return configs
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment