From cbd2c71398bca7b7c21d1c9307d0abf5db6acb3c Mon Sep 17 00:00:00 2001 From: Deepika Upadhyay Date: Wed, 13 Oct 2021 11:49:02 +0530 Subject: [PATCH] qa/tasks: adapt ceph_iscsi.py task to ceph_iscsi_client * we use setup_iscsi_client.py to deploy iscsi client services, configuring intiator and mulitpath this is done by qa task ceph_iscsi_client * qa/cephadm: adds remotes ip addresses to iscsi gateway, * rename poolname: iscsi >> datapool, which we usually use for tests and expresses type of pool more clearly. Signed-off-by: Deepika Upadhyay --- qa/tasks/ceph_iscsi.py | 141 ---------------------------------- qa/tasks/ceph_iscsi_client.py | 56 ++++++++++++++ qa/tasks/cephadm.py | 39 ++++++++-- 3 files changed, 88 insertions(+), 148 deletions(-) delete mode 100644 qa/tasks/ceph_iscsi.py create mode 100644 qa/tasks/ceph_iscsi_client.py diff --git a/qa/tasks/ceph_iscsi.py b/qa/tasks/ceph_iscsi.py deleted file mode 100644 index a8870743a09f1..0000000000000 --- a/qa/tasks/ceph_iscsi.py +++ /dev/null @@ -1,141 +0,0 @@ -""" -Run ceph-iscsi cluster setup -""" -import logging -import contextlib -from io import StringIO -from teuthology.exceptions import CommandFailedError, ConnectionLostError -from teuthology.orchestra import run -from textwrap import dedent - -log = logging.getLogger(__name__) - -class IscsiSetup(object): - def __init__(self, ctx, config): - self.ctx = ctx - self.config = config - self.target_iqn = "iqn.2003-01.com.redhat.iscsi-gw:ceph-gw" - self.client_iqn = "iqn.1994-05.com.redhat:client" - self.trusted_ip_list = [] - self.background_procs = [] - - def run_daemon(self, remote, cmds): - p = remote.run(args=['sudo', 'adjust-ulimits', 'daemon-helper', 'kill', cmds], - wait=False, stdin=run.PIPE, stdout=StringIO()) - self.background_procs.append(p) - - def _kill_background(self, p): - if p.stdin: - p.stdin.close() - try: - p.wait() - except (CommandFailedError, ConnectionLostError): - pass - - def kill_backgrounds(self): - for p in self.background_procs: - self._kill_background(p) - self.background_procs = [] - - def _setup_iscsi_gateway_cfg(self, role): - # setup the iscsi-gateway.cfg file, we only set the - # clust_name and trusted_ip_list and all the others - # as default - ips = ','.join(self.trusted_ip_list) - conf = dedent(f'''\ -[config] -cluster_name = ceph -pool = rbd -api_secure = false -api_port = 5000 -trusted_ip_list = {ips} -''') - path = "/etc/ceph/iscsi-gateway.cfg" - (remote,) = (self.ctx.cluster.only(role).remotes.keys()) - remote.sudo_write_file(path, conf) - - def _setup_gateway(self, role): - """Spawned task that setups the gateway""" - (remote,) = (self.ctx.cluster.only(role).remotes.keys()) - - self._setup_iscsi_gateway_cfg(role) - - self.run_daemon(remote, "/usr/bin/tcmu-runner") - self.run_daemon(remote, "/usr/bin/rbd-target-gw") - self.run_daemon(remote, "/usr/bin/rbd-target-api") - - def setup_gateways(self): - for role in self.config['gateways']: - (remote,) = (self.ctx.cluster.only(role).remotes.keys()) - self.trusted_ip_list.append(remote.ip_address) - - for role in self.config['gateways']: - self._setup_gateway(role) - - def _setup_client(self, role): - """Spawned task that setups the gateway""" - (remote,) = (self.ctx.cluster.only(role).remotes.keys()) - - # copy the "iscsi-gateway.cfg" to client and will be - # used to get the IPs - self._setup_iscsi_gateway_cfg(role) - - conf = dedent(f''' -InitiatorName={self.client_iqn} -''') - path = "/etc/iscsi/initiatorname.iscsi" - remote.sudo_write_file(path, conf, mkdir=True) - - # the restart is needed after the above change is applied - remote.run(args=['sudo', 'systemctl', 'restart', 'iscsid']) - - remote.run(args=['sudo', 'modprobe', 'dm_multipath']) - remote.run(args=['sudo', 'mpathconf', '--enable']) - conf = dedent('''\ -devices { - device { - vendor "LIO-ORG" - product "LIO-ORG" - hardware_handler "1 alua" - path_grouping_policy "failover" - path_selector "queue-length 0" - failback 60 - path_checker tur - prio alua - prio_args exclusive_pref_bit - fast_io_fail_tmo 25 - no_path_retry queue - } -} -''') - path = "/etc/multipath.conf" - remote.sudo_write_file(path, conf, append=True) - remote.run(args=['sudo', 'systemctl', 'start', 'multipathd']) - - def setup_clients(self): - for role in self.config['clients']: - self._setup_client(role) - -@contextlib.contextmanager -def task(ctx, config): - """ - Run ceph iscsi setup. - - Specify the list of gateways to run :: - - tasks: - ceph_iscsi: - gateways: [a_gateway.0, c_gateway.1] - clients: [b_client.0] - - """ - log.info('Setting ceph iscsi cluster...') - iscsi = IscsiSetup(ctx, config) - iscsi.setup_gateways() - iscsi.setup_clients() - - try: - yield - finally: - log.info('Ending ceph iscsi daemons') - iscsi.kill_backgrounds() diff --git a/qa/tasks/ceph_iscsi_client.py b/qa/tasks/ceph_iscsi_client.py new file mode 100644 index 0000000000000..189b7fa31fef8 --- /dev/null +++ b/qa/tasks/ceph_iscsi_client.py @@ -0,0 +1,56 @@ +""" +Set up ceph-iscsi client. +""" +import logging +import contextlib +from textwrap import dedent + +log = logging.getLogger(__name__) + + +@contextlib.contextmanager +def task(ctx, config): + """ + Set up ceph-iscsi client. + + tasks: + ceph_iscsi_client: + clients: [client.1] + """ + log.info('Setting up ceph-iscsi client...') + for role in config['clients']: + (remote,) = (ctx.cluster.only(role).remotes.keys()) + + conf = dedent(''' + InitiatorName=iqn.1994-05.com.redhat:client + ''') + path = "/etc/iscsi/initiatorname.iscsi" + remote.sudo_write_file(path, conf, mkdir=True) + + # the restart is needed after the above change is applied + remote.run(args=['sudo', 'systemctl', 'restart', 'iscsid']) + + remote.run(args=['sudo', 'modprobe', 'dm_multipath']) + remote.run(args=['sudo', 'mpathconf', '--enable']) + conf = dedent(''' + devices { + device { + vendor "LIO-ORG" + product "TCMU device" + hardware_handler "1 alua" + path_grouping_policy "failover" + path_selector "queue-length 0" + failback 60 + path_checker tur + prio alua + prio_args exclusive_pref_bit + fast_io_fail_tmo 25 + no_path_retry queue + } + } + ''') + path = "/etc/multipath.conf" + remote.sudo_write_file(path, conf, append=True) + remote.run(args=['sudo', 'systemctl', 'start', 'multipathd']) + + yield diff --git a/qa/tasks/cephadm.py b/qa/tasks/cephadm.py index 8bd7f220c82f1..b72e8b0e2a674 100644 --- a/qa/tasks/cephadm.py +++ b/qa/tasks/cephadm.py @@ -19,6 +19,7 @@ from teuthology import contextutil from teuthology.orchestra import run from teuthology.orchestra.daemon import DaemonGroup from teuthology.config import config as teuth_config +from textwrap import dedent # these items we use from ceph.py should probably eventually move elsewhere from tasks.ceph import get_mons, healthy @@ -67,6 +68,18 @@ def build_initial_config(ctx, config): return conf +def distribute_iscsi_gateway_cfg(ctx, conf_data): + """ + Distribute common gateway config to get the IPs. + These will help in iscsi clients with finding trusted_ip_list. + """ + log.info('Distributing iscsi-gateway.cfg...') + for remote, roles in ctx.cluster.remotes.items(): + remote.write_file( + path='/etc/ceph/iscsi-gateway.cfg', + data=conf_data, + sudo=True) + def update_archive_setting(ctx, key, value): """ Add logs directory to job's info log file @@ -905,32 +918,44 @@ def ceph_iscsi(ctx, config): nodes = [] daemons = {} + ips = [] + for remote, roles in ctx.cluster.remotes.items(): for role in [r for r in roles - if teuthology.is_type('iscsi', cluster_name)(r)]: + if teuthology.is_type('iscsi', cluster_name)(r)]: c_, _, id_ = teuthology.split_role(role) log.info('Adding %s on %s' % (role, remote.shortname)) nodes.append(remote.shortname + '=' + id_) daemons[role] = (remote, id_) + ips.append(remote.ip_address) + trusted_ip_list = ','.join(ips) if nodes: - poolname = 'iscsi' - # ceph osd pool create iscsi 3 3 replicated + poolname = 'datapool' + # ceph osd pool create datapool 3 3 replicated _shell(ctx, cluster_name, remote, [ 'ceph', 'osd', 'pool', 'create', poolname, '3', '3', 'replicated'] ) _shell(ctx, cluster_name, remote, [ - 'ceph', 'osd', 'pool', 'application', 'enable', - poolname, 'rbd'] + 'rbd', 'pool', 'init', poolname] ) - # ceph orch apply iscsi iscsi user password + # ceph orch apply iscsi datapool (admin)user (admin)password _shell(ctx, cluster_name, remote, [ 'ceph', 'orch', 'apply', 'iscsi', - poolname, 'user', 'password', + poolname, 'admin', 'admin', + '--trusted_ip_list', trusted_ip_list, '--placement', str(len(nodes)) + ';' + ';'.join(nodes)] ) + + # used by iscsi client to identify valid gateway ip's + conf_data = dedent(f""" + [config] + trusted_ip_list = {trusted_ip_list} + """) + distribute_iscsi_gateway_cfg(ctx, conf_data) + for role, i in daemons.items(): remote, id_ = i ctx.daemons.register_daemon( -- 2.39.5