From 62426772f5cd7afa9b3e48c20a4702e943a479c5 Mon Sep 17 00:00:00 2001
From: Xiubo Li <xiubli@redhat.com>
Date: Mon, 14 Sep 2020 09:21:01 +0800
Subject: [PATCH] qa: add ceph iscsi test suite support

Currently this only supports the gwcli tests on CentOS/RHEL 8 with
2 ceph iscsi gateways.

Fixes: https://tracker.ceph.com/issues/47439
Signed-off-by: Xiubo Li <xiubli@redhat.com>
---
 qa/suites/rbd/iscsi/%                         |   0
 qa/suites/rbd/iscsi/.qa                       |   1 +
 qa/suites/rbd/iscsi/base/.qa                  |   1 +
 qa/suites/rbd/iscsi/base/install.yaml         |  21 +++
 qa/suites/rbd/iscsi/cluster/+                 |   0
 qa/suites/rbd/iscsi/cluster/.qa               |   1 +
 qa/suites/rbd/iscsi/cluster/fixed-3.yaml      |   4 +
 qa/suites/rbd/iscsi/cluster/openstack.yaml    |   8 +
 qa/suites/rbd/iscsi/pool/.qa                  |   1 +
 qa/suites/rbd/iscsi/pool/datapool.yaml        |  12 ++
 .../supported-random-distro$/centos_8.yaml    |   1 +
 .../supported-random-distro$/rhel_8.yaml      |   1 +
 qa/suites/rbd/iscsi/workloads/.qa             |   1 +
 qa/suites/rbd/iscsi/workloads/ceph_iscsi.yaml |  13 ++
 qa/tasks/ceph_iscsi.py                        | 141 ++++++++++++++++++
 src/test/cli-integration/rbd/gwcli_client.t   |  30 ++++
 src/test/cli-integration/rbd/gwcli_create.t   |  74 +++++++++
 src/test/cli-integration/rbd/gwcli_delete.t   |  27 ++++
 18 files changed, 337 insertions(+)
 create mode 100644 qa/suites/rbd/iscsi/%
 create mode 120000 qa/suites/rbd/iscsi/.qa
 create mode 120000 qa/suites/rbd/iscsi/base/.qa
 create mode 100644 qa/suites/rbd/iscsi/base/install.yaml
 create mode 100644 qa/suites/rbd/iscsi/cluster/+
 create mode 120000 qa/suites/rbd/iscsi/cluster/.qa
 create mode 100644 qa/suites/rbd/iscsi/cluster/fixed-3.yaml
 create mode 100644 qa/suites/rbd/iscsi/cluster/openstack.yaml
 create mode 120000 qa/suites/rbd/iscsi/pool/.qa
 create mode 100644 qa/suites/rbd/iscsi/pool/datapool.yaml
 create mode 120000 qa/suites/rbd/iscsi/supported-random-distro$/centos_8.yaml
 create mode 120000 qa/suites/rbd/iscsi/supported-random-distro$/rhel_8.yaml
 create mode 120000 qa/suites/rbd/iscsi/workloads/.qa
 create mode 100644 qa/suites/rbd/iscsi/workloads/ceph_iscsi.yaml
 create mode 100644 qa/tasks/ceph_iscsi.py
 create mode 100644 src/test/cli-integration/rbd/gwcli_client.t
 create mode 100644 src/test/cli-integration/rbd/gwcli_create.t
 create mode 100644 src/test/cli-integration/rbd/gwcli_delete.t

diff --git a/qa/suites/rbd/iscsi/% b/qa/suites/rbd/iscsi/%
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/qa/suites/rbd/iscsi/.qa b/qa/suites/rbd/iscsi/.qa
new file mode 120000
index 00000000000..a602a0353e7
--- /dev/null
+++ b/qa/suites/rbd/iscsi/.qa
@@ -0,0 +1 @@
+../.qa/
\ No newline at end of file
diff --git a/qa/suites/rbd/iscsi/base/.qa b/qa/suites/rbd/iscsi/base/.qa
new file mode 120000
index 00000000000..a602a0353e7
--- /dev/null
+++ b/qa/suites/rbd/iscsi/base/.qa
@@ -0,0 +1 @@
+../.qa/
\ No newline at end of file
diff --git a/qa/suites/rbd/iscsi/base/install.yaml b/qa/suites/rbd/iscsi/base/install.yaml
new file mode 100644
index 00000000000..e35affe7e24
--- /dev/null
+++ b/qa/suites/rbd/iscsi/base/install.yaml
@@ -0,0 +1,21 @@
+use_shaman: True
+tasks:
+- install:
+    extra_packages:
+      - iscsi-initiator-utils
+      - device-mapper-multipath
+- ceph:
+- install:
+    packages:
+      rpm:
+      - tcmu-runner
+    project: tcmu-runner
+    sha1: latest
+- install:
+    packages:
+      rpm:
+      - ceph-iscsi
+    shaman:
+      force_noarch: True
+    project: ceph-iscsi
+    sha1: latest
diff --git a/qa/suites/rbd/iscsi/cluster/+ b/qa/suites/rbd/iscsi/cluster/+
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/qa/suites/rbd/iscsi/cluster/.qa b/qa/suites/rbd/iscsi/cluster/.qa
new file mode 120000
index 00000000000..a602a0353e7
--- /dev/null
+++ b/qa/suites/rbd/iscsi/cluster/.qa
@@ -0,0 +1 @@
+../.qa/
\ No newline at end of file
diff --git a/qa/suites/rbd/iscsi/cluster/fixed-3.yaml b/qa/suites/rbd/iscsi/cluster/fixed-3.yaml
new file mode 100644
index 00000000000..bf1eb6018b4
--- /dev/null
+++ b/qa/suites/rbd/iscsi/cluster/fixed-3.yaml
@@ -0,0 +1,4 @@
+roles:
+- [mon.a, osd.0, osd.1, osd.2, a_gateway.0]
+- [mon.b, osd.3, osd.4, osd.5, b_client.0]
+- [mon.c, mgr.x, osd.6, osd.7, c_gateway.1]
diff --git a/qa/suites/rbd/iscsi/cluster/openstack.yaml b/qa/suites/rbd/iscsi/cluster/openstack.yaml
new file mode 100644
index 00000000000..40fef4770b9
--- /dev/null
+++ b/qa/suites/rbd/iscsi/cluster/openstack.yaml
@@ -0,0 +1,8 @@
+openstack:
+  - machine:
+      disk: 40 # GB
+      ram: 8000 # MB
+      cpus: 1
+    volumes: # attached to each instance
+      count: 4
+      size: 30 # GB
diff --git a/qa/suites/rbd/iscsi/pool/.qa b/qa/suites/rbd/iscsi/pool/.qa
new file mode 120000
index 00000000000..a602a0353e7
--- /dev/null
+++ b/qa/suites/rbd/iscsi/pool/.qa
@@ -0,0 +1 @@
+../.qa/
\ No newline at end of file
diff --git a/qa/suites/rbd/iscsi/pool/datapool.yaml b/qa/suites/rbd/iscsi/pool/datapool.yaml
new file mode 100644
index 00000000000..45b00378b83
--- /dev/null
+++ b/qa/suites/rbd/iscsi/pool/datapool.yaml
@@ -0,0 +1,12 @@
+overrides:
+  ceph:
+    log-ignorelist:
+      - overall HEALTH_
+      - \(CACHE_POOL_NEAR_FULL\)
+      - \(CACHE_POOL_NO_HIT_SET\)
+tasks:
+- exec:
+    a_gateway.0:
+      - sudo ceph osd pool create rbd 4
+      - sudo ceph osd pool create datapool 4
+      - sudo ceph osd pool application enable datapool rbd
diff --git a/qa/suites/rbd/iscsi/supported-random-distro$/centos_8.yaml b/qa/suites/rbd/iscsi/supported-random-distro$/centos_8.yaml
new file mode 120000
index 00000000000..750bfb4908f
--- /dev/null
+++ b/qa/suites/rbd/iscsi/supported-random-distro$/centos_8.yaml
@@ -0,0 +1 @@
+../../../../distros/supported-all-distro/centos_8.yaml
\ No newline at end of file
diff --git a/qa/suites/rbd/iscsi/supported-random-distro$/rhel_8.yaml b/qa/suites/rbd/iscsi/supported-random-distro$/rhel_8.yaml
new file mode 120000
index 00000000000..046a2f699ff
--- /dev/null
+++ b/qa/suites/rbd/iscsi/supported-random-distro$/rhel_8.yaml
@@ -0,0 +1 @@
+../../../../distros/supported-all-distro/rhel_8.yaml
\ No newline at end of file
diff --git a/qa/suites/rbd/iscsi/workloads/.qa b/qa/suites/rbd/iscsi/workloads/.qa
new file mode 120000
index 00000000000..a602a0353e7
--- /dev/null
+++ b/qa/suites/rbd/iscsi/workloads/.qa
@@ -0,0 +1 @@
+../.qa/
\ No newline at end of file
diff --git a/qa/suites/rbd/iscsi/workloads/ceph_iscsi.yaml b/qa/suites/rbd/iscsi/workloads/ceph_iscsi.yaml
new file mode 100644
index 00000000000..026c6ce71da
--- /dev/null
+++ b/qa/suites/rbd/iscsi/workloads/ceph_iscsi.yaml
@@ -0,0 +1,13 @@
+tasks:
+- ceph_iscsi:
+    gateways: [a_gateway.0, c_gateway.1]
+    clients: [b_client.0]
+- cram:
+    parallel: False
+    clients:
+      a_gateway.0:
+      - src/test/cli-integration/rbd/gwcli_create.t
+      b_client.0:
+      - src/test/cli-integration/rbd/gwcli_client.t
+      c_gateway.1:
+      - src/test/cli-integration/rbd/gwcli_delete.t
diff --git a/qa/tasks/ceph_iscsi.py b/qa/tasks/ceph_iscsi.py
new file mode 100644
index 00000000000..a8870743a09
--- /dev/null
+++ b/qa/tasks/ceph_iscsi.py
@@ -0,0 +1,141 @@
+"""
+Run ceph-iscsi cluster setup
+"""
+import logging
+import contextlib
+from io import StringIO
+from teuthology.exceptions import CommandFailedError, ConnectionLostError
+from teuthology.orchestra import run
+from textwrap import dedent
+
+log = logging.getLogger(__name__)
+
+class IscsiSetup(object):
+    def __init__(self, ctx, config):
+        self.ctx = ctx
+        self.config = config
+        self.target_iqn = "iqn.2003-01.com.redhat.iscsi-gw:ceph-gw"
+        self.client_iqn = "iqn.1994-05.com.redhat:client"
+        self.trusted_ip_list = []
+        self.background_procs = []
+
+    def run_daemon(self, remote, cmds):
+        p = remote.run(args=['sudo', 'adjust-ulimits', 'daemon-helper', 'kill', cmds],
+                          wait=False, stdin=run.PIPE, stdout=StringIO())
+        self.background_procs.append(p)
+
+    def _kill_background(self, p):
+        if p.stdin:
+            p.stdin.close()
+            try:
+                p.wait()
+            except (CommandFailedError, ConnectionLostError):
+                pass
+
+    def kill_backgrounds(self):
+        for p in self.background_procs:
+            self._kill_background(p)
+        self.background_procs = []
+
+    def _setup_iscsi_gateway_cfg(self, role):
+        # setup the iscsi-gateway.cfg file, we only set the
+        # clust_name and trusted_ip_list and all the others
+        # as default
+        ips = ','.join(self.trusted_ip_list)
+        conf = dedent(f'''\
+[config]
+cluster_name = ceph
+pool = rbd
+api_secure = false
+api_port = 5000
+trusted_ip_list = {ips}
+''')
+        path = "/etc/ceph/iscsi-gateway.cfg"
+        (remote,) = (self.ctx.cluster.only(role).remotes.keys())
+        remote.sudo_write_file(path, conf)
+
+    def _setup_gateway(self, role):
+        """Spawned task that setups the gateway"""
+        (remote,) = (self.ctx.cluster.only(role).remotes.keys())
+
+        self._setup_iscsi_gateway_cfg(role)
+
+        self.run_daemon(remote, "/usr/bin/tcmu-runner")
+        self.run_daemon(remote, "/usr/bin/rbd-target-gw")
+        self.run_daemon(remote, "/usr/bin/rbd-target-api")
+
+    def setup_gateways(self):
+        for role in self.config['gateways']:
+            (remote,) = (self.ctx.cluster.only(role).remotes.keys())
+            self.trusted_ip_list.append(remote.ip_address)
+
+        for role in self.config['gateways']:
+            self._setup_gateway(role)
+
+    def _setup_client(self, role):
+        """Spawned task that setups the gateway"""
+        (remote,) = (self.ctx.cluster.only(role).remotes.keys())
+
+        # copy the "iscsi-gateway.cfg" to client and will be
+        # used to get the IPs
+        self._setup_iscsi_gateway_cfg(role)
+
+        conf = dedent(f'''
+InitiatorName={self.client_iqn}
+''')
+        path = "/etc/iscsi/initiatorname.iscsi"
+        remote.sudo_write_file(path, conf, mkdir=True)
+
+        # the restart is needed after the above change is applied
+        remote.run(args=['sudo', 'systemctl', 'restart', 'iscsid'])
+
+        remote.run(args=['sudo', 'modprobe', 'dm_multipath'])
+        remote.run(args=['sudo', 'mpathconf', '--enable'])
+        conf = dedent('''\
+devices {
+        device {
+                vendor                 "LIO-ORG"
+                product                "LIO-ORG"
+                hardware_handler       "1 alua"
+                path_grouping_policy   "failover"
+                path_selector          "queue-length 0"
+                failback               60
+                path_checker           tur
+                prio                   alua
+                prio_args              exclusive_pref_bit
+                fast_io_fail_tmo       25
+                no_path_retry          queue
+        }
+}
+''')
+        path = "/etc/multipath.conf"
+        remote.sudo_write_file(path, conf, append=True)
+        remote.run(args=['sudo', 'systemctl', 'start', 'multipathd'])
+
+    def setup_clients(self):
+        for role in self.config['clients']:
+            self._setup_client(role)
+
+@contextlib.contextmanager
+def task(ctx, config):
+    """
+    Run ceph iscsi setup.
+
+    Specify the list of gateways to run ::
+
+      tasks:
+        ceph_iscsi:
+          gateways: [a_gateway.0, c_gateway.1]
+          clients: [b_client.0]
+
+    """
+    log.info('Setting ceph iscsi cluster...')
+    iscsi = IscsiSetup(ctx, config)
+    iscsi.setup_gateways()
+    iscsi.setup_clients()
+
+    try:
+        yield
+    finally:
+        log.info('Ending ceph iscsi daemons')
+        iscsi.kill_backgrounds()
diff --git a/src/test/cli-integration/rbd/gwcli_client.t b/src/test/cli-integration/rbd/gwcli_client.t
new file mode 100644
index 00000000000..f636d540d89
--- /dev/null
+++ b/src/test/cli-integration/rbd/gwcli_client.t
@@ -0,0 +1,30 @@
+Login to the target
+===================
+  $ IP=`cat /etc/ceph/iscsi-gateway.cfg |grep 'trusted_ip_list' | awk -F'[, ]' '{print $3}'`
+  > sudo iscsiadm -m discovery -t st -p $IP -l 2&> /dev/null
+  $ sleep 10
+  $ sudo ls /dev/disk/by-path/ |grep 'iscsi-iqn.2003-01.com.redhat.iscsi-gw:ceph-gw' |wc -l
+  2
+
+Make filesystem
+===============
+  $ device=`sudo multipath -l | grep 'LIO-ORG,TCMU device' | awk '{print $1}'`
+  > sudo mkfs.xfs /dev/mapper/$device -f | grep 'meta-data=/dev/mapper/mpath' | awk '{print $2}'
+  isize=512
+
+Write/Read test
+===============
+  $ device=`sudo multipath -l | grep 'LIO-ORG,TCMU device' | awk '{print $1}'`
+  > sudo dd if=/dev/random of=/tmp/iscsi_tmpfile bs=1 count=1K status=none
+  > sudo dd if=/tmp/iscsi_tmpfile of=/dev/mapper/$device bs=1 count=1K status=none
+  > sudo dd if=/dev/mapper/$device of=/tmp/iscsi_tmpfile1 bs=1 count=1K status=none
+  $ sudo diff /tmp/iscsi_tmpfile /tmp/iscsi_tmpfile1
+
+Logout the targets
+==================
+  $ IP=`cat /etc/ceph/iscsi-gateway.cfg |grep 'trusted_ip_list' | awk -F'[, ]' '{print $3}'`
+  > sudo iscsiadm -m node -T iqn.2003-01.com.redhat.iscsi-gw:ceph-gw -p $IP:3260 -u | grep 'successful' | awk -F']' '{print $2}'
+   successful.
+  $ IP=`cat /etc/ceph/iscsi-gateway.cfg |grep 'trusted_ip_list' | awk -F'[, ]' '{print $4}'`
+  > sudo iscsiadm -m node -T iqn.2003-01.com.redhat.iscsi-gw:ceph-gw -p $IP:3260 -u | grep 'successful' | awk -F']' '{print $2}'
+   successful.
diff --git a/src/test/cli-integration/rbd/gwcli_create.t b/src/test/cli-integration/rbd/gwcli_create.t
new file mode 100644
index 00000000000..e23ed895e7b
--- /dev/null
+++ b/src/test/cli-integration/rbd/gwcli_create.t
@@ -0,0 +1,74 @@
+Dismiss the "could not load preferences file .gwcli/prefs.bin" warning
+======================================================================
+  $ sudo gwcli ls >/dev/null 2>&1
+
+Create a datapool/block0 disk
+=============================
+  $ sudo gwcli disks/ create pool=datapool image=block0 size=300M wwn=36001405da17b74481464e9fa968746d3
+  $ sudo gwcli ls disks/ | grep 'o- disks' | awk -F'[' '{print $2}'
+  300M, Disks: 1]
+  $ sudo gwcli ls disks/ | grep 'o- datapool' | awk -F'[' '{print $2}'
+  datapool (300M)]
+  $ sudo gwcli ls disks/ | grep 'o- block0' | awk -F'[' '{print $2}'
+  datapool/block0 (Unknown, 300M)]
+
+Create the target IQN
+=====================
+  $ sudo gwcli iscsi-targets/ create target_iqn=iqn.2003-01.com.redhat.iscsi-gw:ceph-gw
+  $ sudo gwcli ls iscsi-targets/ | grep 'o- iscsi-targets' | awk -F'[' '{print $2}'
+  DiscoveryAuth: None, Targets: 1]
+  $ sudo gwcli ls iscsi-targets/ | grep 'o- iqn.2003-01.com.redhat.iscsi-gw:ceph-gw' | awk -F'[' '{print $2}'
+  Auth: None, Gateways: 0]
+  $ sudo gwcli ls iscsi-targets/ | grep 'o- disks' | awk -F'[' '{print $2}'
+  Disks: 0]
+  $ sudo gwcli ls iscsi-targets/ | grep 'o- gateways' | awk -F'[' '{print $2}'
+  Up: 0/0, Portals: 0]
+  $ sudo gwcli ls iscsi-targets/ | grep 'o- host-groups' | awk -F'[' '{print $2}'
+  Groups : 0]
+  $ sudo gwcli ls iscsi-targets/ | grep 'o- hosts' | awk -F'[' '{print $2}'
+  Auth: ACL_ENABLED, Hosts: 0]
+
+Create the first gateway
+========================
+  $ HOST=`python3 -c "import socket; print(socket.getfqdn())"`
+  > IP=`hostname -i | awk '{print $1}'`
+  > sudo gwcli iscsi-targets/iqn.2003-01.com.redhat.iscsi-gw:ceph-gw/gateways create ip_addresses=$IP gateway_name=$HOST
+  $ sudo gwcli ls iscsi-targets/ | grep 'o- gateways' | awk -F'[' '{print $2}'
+  Up: 1/1, Portals: 1]
+
+Create the second gateway
+========================
+  $ IP=`cat /etc/ceph/iscsi-gateway.cfg |grep 'trusted_ip_list' | awk -F'[, ]' '{print $3}'`
+  > if [ "$IP" != `hostname -i | awk '{print $1}'` ]; then
+  >   HOST=`python3 -c "import socket; print(socket.getfqdn('$IP'))"`
+  >   sudo gwcli iscsi-targets/iqn.2003-01.com.redhat.iscsi-gw:ceph-gw/gateways create ip_addresses=$IP gateway_name=$HOST
+  > fi
+  $ IP=`cat /etc/ceph/iscsi-gateway.cfg |grep 'trusted_ip_list' | awk -F'[, ]' '{print $4}'`
+  > if [ "$IP" != `hostname -i | awk '{print $1}'` ]; then
+  >   HOST=`python3 -c "import socket; print(socket.getfqdn('$IP'))"`
+  >   sudo gwcli iscsi-targets/iqn.2003-01.com.redhat.iscsi-gw:ceph-gw/gateways create ip_addresses=$IP gateway_name=$HOST
+  > fi
+  $ sudo gwcli ls iscsi-targets/ | grep 'o- gateways' | awk -F'[' '{print $2}'
+  Up: 2/2, Portals: 2]
+
+Attach the disk
+===============
+  $ sudo gwcli iscsi-targets/iqn.2003-01.com.redhat.iscsi-gw:ceph-gw/disks/ add disk=datapool/block0
+  $ sudo gwcli ls iscsi-targets/ | grep 'o- disks' | awk -F'[' '{print $2}'
+  Disks: 1]
+
+Create a host
+=============
+  $ sudo gwcli iscsi-targets/iqn.2003-01.com.redhat.iscsi-gw:ceph-gw/hosts create client_iqn=iqn.1994-05.com.redhat:client
+  $ sudo gwcli ls iscsi-targets/ | grep 'o- hosts' | awk -F'[' '{print $2}'
+  Auth: ACL_ENABLED, Hosts: 1]
+  $ sudo gwcli ls iscsi-targets/ | grep 'o- iqn.1994-05.com.redhat:client' | awk -F'[' '{print $2}'
+  Auth: None, Disks: 0(0.00Y)]
+
+Map the LUN
+===========
+  $ sudo gwcli iscsi-targets/iqn.2003-01.com.redhat.iscsi-gw:ceph-gw/hosts/iqn.1994-05.com.redhat:client disk disk=datapool/block0
+  $ sudo gwcli ls iscsi-targets/ | grep 'o- hosts' | awk -F'[' '{print $2}'
+  Auth: ACL_ENABLED, Hosts: 1]
+  $ sudo gwcli ls iscsi-targets/ | grep 'o- iqn.1994-05.com.redhat:client' | awk -F'[' '{print $2}'
+  Auth: None, Disks: 1(300M)]
diff --git a/src/test/cli-integration/rbd/gwcli_delete.t b/src/test/cli-integration/rbd/gwcli_delete.t
new file mode 100644
index 00000000000..5aef9d0e960
--- /dev/null
+++ b/src/test/cli-integration/rbd/gwcli_delete.t
@@ -0,0 +1,27 @@
+Dismiss the "could not load preferences file .gwcli/prefs.bin" warning
+======================================================================
+  $ sudo gwcli ls >/dev/null 2>&1
+
+Delete the host
+===============
+  $ sudo gwcli iscsi-targets/iqn.2003-01.com.redhat.iscsi-gw:ceph-gw/hosts delete client_iqn=iqn.1994-05.com.redhat:client
+  $ sudo gwcli ls iscsi-targets/ | grep 'o- hosts' | awk -F'[' '{print $2}'
+  Auth: ACL_ENABLED, Hosts: 0]
+
+Delete the iscsi-targets disk
+=============================
+  $ sudo gwcli iscsi-targets/iqn.2003-01.com.redhat.iscsi-gw:ceph-gw/disks/ delete disk=datapool/block0
+  $ sudo gwcli ls iscsi-targets/ | grep 'o- disks' | awk -F'[' '{print $2}'
+  Disks: 0]
+
+Delete the target IQN
+=====================
+  $ sudo gwcli iscsi-targets/ delete target_iqn=iqn.2003-01.com.redhat.iscsi-gw:ceph-gw
+  $ sudo gwcli ls iscsi-targets/ | grep 'o- iscsi-targets' | awk -F'[' '{print $2}'
+  DiscoveryAuth: None, Targets: 0]
+
+Delete the disks
+================
+  $ sudo gwcli disks/ delete image_id=datapool/block0
+  $ sudo gwcli ls disks/ | grep 'o- disks' | awk -F'[' '{print $2}'
+  0.00Y, Disks: 0]
-- 
2.39.5