--- /dev/null
+../.qa/
\ No newline at end of file
--- /dev/null
+../.qa/
\ No newline at end of file
--- /dev/null
+use_shaman: True
+tasks:
+- install:
+ extra_packages:
+ - iscsi-initiator-utils
+ - device-mapper-multipath
+- ceph:
+- install:
+ packages:
+ rpm:
+ - tcmu-runner
+ project: tcmu-runner
+ sha1: latest
+- install:
+ packages:
+ rpm:
+ - ceph-iscsi
+ shaman:
+ force_noarch: True
+ project: ceph-iscsi
+ sha1: latest
--- /dev/null
+../.qa/
\ No newline at end of file
--- /dev/null
+roles:
+- [mon.a, osd.0, osd.1, osd.2, a_gateway.0]
+- [mon.b, osd.3, osd.4, osd.5, b_client.0]
+- [mon.c, mgr.x, osd.6, osd.7, c_gateway.1]
--- /dev/null
+openstack:
+ - machine:
+ disk: 40 # GB
+ ram: 8000 # MB
+ cpus: 1
+ volumes: # attached to each instance
+ count: 4
+ size: 30 # GB
--- /dev/null
+../.qa/
\ No newline at end of file
--- /dev/null
+overrides:
+ ceph:
+ log-ignorelist:
+ - overall HEALTH_
+ - \(CACHE_POOL_NEAR_FULL\)
+ - \(CACHE_POOL_NO_HIT_SET\)
+tasks:
+- exec:
+ a_gateway.0:
+ - sudo ceph osd pool create rbd 4
+ - sudo ceph osd pool create datapool 4
+ - sudo ceph osd pool application enable datapool rbd
--- /dev/null
+../../../../distros/supported-all-distro/centos_8.yaml
\ No newline at end of file
--- /dev/null
+../../../../distros/supported-all-distro/rhel_8.yaml
\ No newline at end of file
--- /dev/null
+../.qa/
\ No newline at end of file
--- /dev/null
+tasks:
+- ceph_iscsi:
+ gateways: [a_gateway.0, c_gateway.1]
+ clients: [b_client.0]
+- cram:
+ parallel: False
+ clients:
+ a_gateway.0:
+ - src/test/cli-integration/rbd/gwcli_create.t
+ b_client.0:
+ - src/test/cli-integration/rbd/gwcli_client.t
+ c_gateway.1:
+ - src/test/cli-integration/rbd/gwcli_delete.t
--- /dev/null
+"""
+Run ceph-iscsi cluster setup
+"""
+import logging
+import contextlib
+from io import StringIO
+from teuthology.exceptions import CommandFailedError, ConnectionLostError
+from teuthology.orchestra import run
+from textwrap import dedent
+
+log = logging.getLogger(__name__)
+
+class IscsiSetup(object):
+ def __init__(self, ctx, config):
+ self.ctx = ctx
+ self.config = config
+ self.target_iqn = "iqn.2003-01.com.redhat.iscsi-gw:ceph-gw"
+ self.client_iqn = "iqn.1994-05.com.redhat:client"
+ self.trusted_ip_list = []
+ self.background_procs = []
+
+ def run_daemon(self, remote, cmds):
+ p = remote.run(args=['sudo', 'adjust-ulimits', 'daemon-helper', 'kill', cmds],
+ wait=False, stdin=run.PIPE, stdout=StringIO())
+ self.background_procs.append(p)
+
+ def _kill_background(self, p):
+ if p.stdin:
+ p.stdin.close()
+ try:
+ p.wait()
+ except (CommandFailedError, ConnectionLostError):
+ pass
+
+ def kill_backgrounds(self):
+ for p in self.background_procs:
+ self._kill_background(p)
+ self.background_procs = []
+
+ def _setup_iscsi_gateway_cfg(self, role):
+ # setup the iscsi-gateway.cfg file, we only set the
+ # clust_name and trusted_ip_list and all the others
+ # as default
+ ips = ','.join(self.trusted_ip_list)
+ conf = dedent(f'''\
+[config]
+cluster_name = ceph
+pool = rbd
+api_secure = false
+api_port = 5000
+trusted_ip_list = {ips}
+''')
+ path = "/etc/ceph/iscsi-gateway.cfg"
+ (remote,) = (self.ctx.cluster.only(role).remotes.keys())
+ remote.sudo_write_file(path, conf)
+
+ def _setup_gateway(self, role):
+ """Spawned task that setups the gateway"""
+ (remote,) = (self.ctx.cluster.only(role).remotes.keys())
+
+ self._setup_iscsi_gateway_cfg(role)
+
+ self.run_daemon(remote, "/usr/bin/tcmu-runner")
+ self.run_daemon(remote, "/usr/bin/rbd-target-gw")
+ self.run_daemon(remote, "/usr/bin/rbd-target-api")
+
+ def setup_gateways(self):
+ for role in self.config['gateways']:
+ (remote,) = (self.ctx.cluster.only(role).remotes.keys())
+ self.trusted_ip_list.append(remote.ip_address)
+
+ for role in self.config['gateways']:
+ self._setup_gateway(role)
+
+ def _setup_client(self, role):
+ """Spawned task that setups the gateway"""
+ (remote,) = (self.ctx.cluster.only(role).remotes.keys())
+
+ # copy the "iscsi-gateway.cfg" to client and will be
+ # used to get the IPs
+ self._setup_iscsi_gateway_cfg(role)
+
+ conf = dedent(f'''
+InitiatorName={self.client_iqn}
+''')
+ path = "/etc/iscsi/initiatorname.iscsi"
+ remote.sudo_write_file(path, conf, mkdir=True)
+
+ # the restart is needed after the above change is applied
+ remote.run(args=['sudo', 'systemctl', 'restart', 'iscsid'])
+
+ remote.run(args=['sudo', 'modprobe', 'dm_multipath'])
+ remote.run(args=['sudo', 'mpathconf', '--enable'])
+ conf = dedent('''\
+devices {
+ device {
+ vendor "LIO-ORG"
+ product "LIO-ORG"
+ hardware_handler "1 alua"
+ path_grouping_policy "failover"
+ path_selector "queue-length 0"
+ failback 60
+ path_checker tur
+ prio alua
+ prio_args exclusive_pref_bit
+ fast_io_fail_tmo 25
+ no_path_retry queue
+ }
+}
+''')
+ path = "/etc/multipath.conf"
+ remote.sudo_write_file(path, conf, append=True)
+ remote.run(args=['sudo', 'systemctl', 'start', 'multipathd'])
+
+ def setup_clients(self):
+ for role in self.config['clients']:
+ self._setup_client(role)
+
+@contextlib.contextmanager
+def task(ctx, config):
+ """
+ Run ceph iscsi setup.
+
+ Specify the list of gateways to run ::
+
+ tasks:
+ ceph_iscsi:
+ gateways: [a_gateway.0, c_gateway.1]
+ clients: [b_client.0]
+
+ """
+ log.info('Setting ceph iscsi cluster...')
+ iscsi = IscsiSetup(ctx, config)
+ iscsi.setup_gateways()
+ iscsi.setup_clients()
+
+ try:
+ yield
+ finally:
+ log.info('Ending ceph iscsi daemons')
+ iscsi.kill_backgrounds()
--- /dev/null
+Login to the target
+===================
+ $ IP=`cat /etc/ceph/iscsi-gateway.cfg |grep 'trusted_ip_list' | awk -F'[, ]' '{print $3}'`
+ > sudo iscsiadm -m discovery -t st -p $IP -l 2&> /dev/null
+ $ sleep 10
+ $ sudo ls /dev/disk/by-path/ |grep 'iscsi-iqn.2003-01.com.redhat.iscsi-gw:ceph-gw' |wc -l
+ 2
+
+Make filesystem
+===============
+ $ device=`sudo multipath -l | grep 'LIO-ORG,TCMU device' | awk '{print $1}'`
+ > sudo mkfs.xfs /dev/mapper/$device -f | grep 'meta-data=/dev/mapper/mpath' | awk '{print $2}'
+ isize=512
+
+Write/Read test
+===============
+ $ device=`sudo multipath -l | grep 'LIO-ORG,TCMU device' | awk '{print $1}'`
+ > sudo dd if=/dev/random of=/tmp/iscsi_tmpfile bs=1 count=1K status=none
+ > sudo dd if=/tmp/iscsi_tmpfile of=/dev/mapper/$device bs=1 count=1K status=none
+ > sudo dd if=/dev/mapper/$device of=/tmp/iscsi_tmpfile1 bs=1 count=1K status=none
+ $ sudo diff /tmp/iscsi_tmpfile /tmp/iscsi_tmpfile1
+
+Logout the targets
+==================
+ $ IP=`cat /etc/ceph/iscsi-gateway.cfg |grep 'trusted_ip_list' | awk -F'[, ]' '{print $3}'`
+ > sudo iscsiadm -m node -T iqn.2003-01.com.redhat.iscsi-gw:ceph-gw -p $IP:3260 -u | grep 'successful' | awk -F']' '{print $2}'
+ successful.
+ $ IP=`cat /etc/ceph/iscsi-gateway.cfg |grep 'trusted_ip_list' | awk -F'[, ]' '{print $4}'`
+ > sudo iscsiadm -m node -T iqn.2003-01.com.redhat.iscsi-gw:ceph-gw -p $IP:3260 -u | grep 'successful' | awk -F']' '{print $2}'
+ successful.
--- /dev/null
+Dismiss the "could not load preferences file .gwcli/prefs.bin" warning
+======================================================================
+ $ sudo gwcli ls >/dev/null 2>&1
+
+Create a datapool/block0 disk
+=============================
+ $ sudo gwcli disks/ create pool=datapool image=block0 size=300M wwn=36001405da17b74481464e9fa968746d3
+ $ sudo gwcli ls disks/ | grep 'o- disks' | awk -F'[' '{print $2}'
+ 300M, Disks: 1]
+ $ sudo gwcli ls disks/ | grep 'o- datapool' | awk -F'[' '{print $2}'
+ datapool (300M)]
+ $ sudo gwcli ls disks/ | grep 'o- block0' | awk -F'[' '{print $2}'
+ datapool/block0 (Unknown, 300M)]
+
+Create the target IQN
+=====================
+ $ sudo gwcli iscsi-targets/ create target_iqn=iqn.2003-01.com.redhat.iscsi-gw:ceph-gw
+ $ sudo gwcli ls iscsi-targets/ | grep 'o- iscsi-targets' | awk -F'[' '{print $2}'
+ DiscoveryAuth: None, Targets: 1]
+ $ sudo gwcli ls iscsi-targets/ | grep 'o- iqn.2003-01.com.redhat.iscsi-gw:ceph-gw' | awk -F'[' '{print $2}'
+ Auth: None, Gateways: 0]
+ $ sudo gwcli ls iscsi-targets/ | grep 'o- disks' | awk -F'[' '{print $2}'
+ Disks: 0]
+ $ sudo gwcli ls iscsi-targets/ | grep 'o- gateways' | awk -F'[' '{print $2}'
+ Up: 0/0, Portals: 0]
+ $ sudo gwcli ls iscsi-targets/ | grep 'o- host-groups' | awk -F'[' '{print $2}'
+ Groups : 0]
+ $ sudo gwcli ls iscsi-targets/ | grep 'o- hosts' | awk -F'[' '{print $2}'
+ Auth: ACL_ENABLED, Hosts: 0]
+
+Create the first gateway
+========================
+ $ HOST=`python3 -c "import socket; print(socket.getfqdn())"`
+ > IP=`hostname -i | awk '{print $1}'`
+ > sudo gwcli iscsi-targets/iqn.2003-01.com.redhat.iscsi-gw:ceph-gw/gateways create ip_addresses=$IP gateway_name=$HOST
+ $ sudo gwcli ls iscsi-targets/ | grep 'o- gateways' | awk -F'[' '{print $2}'
+ Up: 1/1, Portals: 1]
+
+Create the second gateway
+========================
+ $ IP=`cat /etc/ceph/iscsi-gateway.cfg |grep 'trusted_ip_list' | awk -F'[, ]' '{print $3}'`
+ > if [ "$IP" != `hostname -i | awk '{print $1}'` ]; then
+ > HOST=`python3 -c "import socket; print(socket.getfqdn('$IP'))"`
+ > sudo gwcli iscsi-targets/iqn.2003-01.com.redhat.iscsi-gw:ceph-gw/gateways create ip_addresses=$IP gateway_name=$HOST
+ > fi
+ $ IP=`cat /etc/ceph/iscsi-gateway.cfg |grep 'trusted_ip_list' | awk -F'[, ]' '{print $4}'`
+ > if [ "$IP" != `hostname -i | awk '{print $1}'` ]; then
+ > HOST=`python3 -c "import socket; print(socket.getfqdn('$IP'))"`
+ > sudo gwcli iscsi-targets/iqn.2003-01.com.redhat.iscsi-gw:ceph-gw/gateways create ip_addresses=$IP gateway_name=$HOST
+ > fi
+ $ sudo gwcli ls iscsi-targets/ | grep 'o- gateways' | awk -F'[' '{print $2}'
+ Up: 2/2, Portals: 2]
+
+Attach the disk
+===============
+ $ sudo gwcli iscsi-targets/iqn.2003-01.com.redhat.iscsi-gw:ceph-gw/disks/ add disk=datapool/block0
+ $ sudo gwcli ls iscsi-targets/ | grep 'o- disks' | awk -F'[' '{print $2}'
+ Disks: 1]
+
+Create a host
+=============
+ $ sudo gwcli iscsi-targets/iqn.2003-01.com.redhat.iscsi-gw:ceph-gw/hosts create client_iqn=iqn.1994-05.com.redhat:client
+ $ sudo gwcli ls iscsi-targets/ | grep 'o- hosts' | awk -F'[' '{print $2}'
+ Auth: ACL_ENABLED, Hosts: 1]
+ $ sudo gwcli ls iscsi-targets/ | grep 'o- iqn.1994-05.com.redhat:client' | awk -F'[' '{print $2}'
+ Auth: None, Disks: 0(0.00Y)]
+
+Map the LUN
+===========
+ $ sudo gwcli iscsi-targets/iqn.2003-01.com.redhat.iscsi-gw:ceph-gw/hosts/iqn.1994-05.com.redhat:client disk disk=datapool/block0
+ $ sudo gwcli ls iscsi-targets/ | grep 'o- hosts' | awk -F'[' '{print $2}'
+ Auth: ACL_ENABLED, Hosts: 1]
+ $ sudo gwcli ls iscsi-targets/ | grep 'o- iqn.1994-05.com.redhat:client' | awk -F'[' '{print $2}'
+ Auth: None, Disks: 1(300M)]
--- /dev/null
+Dismiss the "could not load preferences file .gwcli/prefs.bin" warning
+======================================================================
+ $ sudo gwcli ls >/dev/null 2>&1
+
+Delete the host
+===============
+ $ sudo gwcli iscsi-targets/iqn.2003-01.com.redhat.iscsi-gw:ceph-gw/hosts delete client_iqn=iqn.1994-05.com.redhat:client
+ $ sudo gwcli ls iscsi-targets/ | grep 'o- hosts' | awk -F'[' '{print $2}'
+ Auth: ACL_ENABLED, Hosts: 0]
+
+Delete the iscsi-targets disk
+=============================
+ $ sudo gwcli iscsi-targets/iqn.2003-01.com.redhat.iscsi-gw:ceph-gw/disks/ delete disk=datapool/block0
+ $ sudo gwcli ls iscsi-targets/ | grep 'o- disks' | awk -F'[' '{print $2}'
+ Disks: 0]
+
+Delete the target IQN
+=====================
+ $ sudo gwcli iscsi-targets/ delete target_iqn=iqn.2003-01.com.redhat.iscsi-gw:ceph-gw
+ $ sudo gwcli ls iscsi-targets/ | grep 'o- iscsi-targets' | awk -F'[' '{print $2}'
+ DiscoveryAuth: None, Targets: 0]
+
+Delete the disks
+================
+ $ sudo gwcli disks/ delete image_id=datapool/block0
+ $ sudo gwcli ls disks/ | grep 'o- disks' | awk -F'[' '{print $2}'
+ 0.00Y, Disks: 0]
projects / ceph.git / commitdiff