From 3e72dcb9f478421f004f704d05a2af3ede7f7eec Mon Sep 17 00:00:00 2001 From: Warren Usui Date: Tue, 15 Oct 2013 14:43:39 -0700 Subject: [PATCH] Added two new tasks. tgt starts up the tgt service. iscsi starts up the iscsi service and logins to an rbd image using the tgt service (either locally or remotely). The iscsi service runs some simple tests, and then sets up the isci-image to be useable by rbd test scripts. Later workunits can perform further testing on the isci-image interface. In order to add the new tasks, common_fs_utils.py was formed from code extracted out of rbd.py. Rbd.py and iscsi.py both call the functions in this module. Fixes: #6433 Signed-off-by: Warren Usui Reviewed-by: Josh Durgin --- teuthology/task/common_fs_utils.py | 132 ++++++++++++++++ teuthology/task/iscsi.py | 232 +++++++++++++++++++++++++++++ teuthology/task/rbd.py | 131 ++-------------- teuthology/task/tgt.py | 165 ++++++++++++++++++++ 4 files changed, 538 insertions(+), 122 deletions(-) create mode 100644 teuthology/task/common_fs_utils.py create mode 100644 teuthology/task/iscsi.py create mode 100644 teuthology/task/tgt.py diff --git a/teuthology/task/common_fs_utils.py b/teuthology/task/common_fs_utils.py new file mode 100644 index 0000000000..b963e98d42 --- /dev/null +++ b/teuthology/task/common_fs_utils.py @@ -0,0 +1,132 @@ +""" +Common filesystem related utilities. Originally this +code was part of rbd.py. It was broken out so that it +could be used by other modules (tgt.py and iscsi.py for instance). +""" +import logging +import contextlib +from teuthology import misc as teuthology + +log = logging.getLogger(__name__) + + +def default_image_name(role): + """ + Image name used by rbd and iscsi + """ + return 'testimage.{role}'.format(role=role) + + +@contextlib.contextmanager +def generic_mkfs(ctx, config, devname_rtn): + """ + Create a filesystem (either rbd or tgt, depending on devname_rtn) + + Rbd for example, now makes the following calls: + - rbd.create_image: [client.0] + - rbd.modprobe: [client.0] + - rbd.dev_create: [client.0] + - common_fs_utils.generic_mkfs: [client.0] + - common_fs_utils.generic_mount: + client.0: testimage.client.0 + """ + assert isinstance(config, list) or isinstance(config, dict), \ + "task mkfs must be configured with a list or dictionary" + if isinstance(config, dict): + images = config.items() + else: + images = [(role, None) for role in config] + + for role, properties in images: + if properties is None: + properties = {} + (remote,) = ctx.cluster.only(role).remotes.keys() + image = properties.get('image_name', default_image_name(role)) + fs_type = properties.get('fs_type', 'ext3') + remote.run( + args=[ + 'sudo', + 'mkfs', + '-t', fs_type, + devname_rtn(ctx, image), + ], + ) + yield + + +@contextlib.contextmanager +def generic_mount(ctx, config, devname_rtn): + """ + Generic Mount an rbd or tgt image. + + Rbd for example, now makes the following calls: + - rbd.create_image: [client.0] + - rbd.modprobe: [client.0] + - rbd.dev_create: [client.0] + - common_fs_utils.generic_mkfs: [client.0] + - common_fs_utils.generic_mount: + client.0: testimage.client.0 + """ + assert isinstance(config, list) or isinstance(config, dict), \ + "task mount must be configured with a list or dictionary" + if isinstance(config, dict): + role_images = config.items() + else: + role_images = [(role, None) for role in config] + + def strip_client_prefix(role): + """ + Extract the number from the name of a client role + """ + prefix = 'client.' + assert role.startswith(prefix) + id_ = role[len(prefix):] + return id_ + + testdir = teuthology.get_testdir(ctx) + + mnt_template = '{tdir}/mnt.{id}' + mounted = [] + for role, image in role_images: + if image is None: + image = default_image_name(role) + (remote,) = ctx.cluster.only(role).remotes.keys() + id_ = strip_client_prefix(role) + mnt = mnt_template.format(tdir=testdir, id=id_) + mounted.append((remote, mnt)) + remote.run( + args=[ + 'mkdir', + '--', + mnt, + ] + ) + + remote.run( + args=[ + 'sudo', + 'mount', + devname_rtn(ctx, image), + mnt, + ], + ) + + try: + yield + finally: + log.info("Unmounting rbd images... %s", mounted) + for remote, mnt in mounted: + remote.run( + args=[ + 'sudo', + 'umount', + mnt, + ], + ) + remote.run( + args=[ + 'rmdir', + '--', + mnt, + ] + ) diff --git a/teuthology/task/iscsi.py b/teuthology/task/iscsi.py new file mode 100644 index 0000000000..3487d1d4b2 --- /dev/null +++ b/teuthology/task/iscsi.py @@ -0,0 +1,232 @@ +""" +Handle iscsi adm commands for tgt connections. +""" +import logging +import contextlib +import socket + +from cStringIO import StringIO +from teuthology import misc as teuthology +from teuthology import contextutil +from teuthology.task.common_fs_utils import generic_mkfs +from teuthology.task.common_fs_utils import generic_mount +from ..orchestra import run + +log = logging.getLogger(__name__) + + +def _get_remote(remotes, client): + """ + Get remote object that is associated with the client specified. + """ + for rem in remotes: + if client in remotes[rem]: + return rem + + +def _get_remote_name(remotes, client): + """ + Get remote name that is associated with the client specified. + """ + rem_name = _get_remote(remotes, client).name + rem_name = rem_name[rem_name.find('@') + 1:] + return rem_name + + +def tgt_devname_get(ctx, test_image): + """ + Get the name of the newly created device by following the by-path + link (which is symbolically linked to the appropriate /dev/sd* file). + """ + remotes = ctx.cluster.only(teuthology.is_type('client')).remotes + rem_name = _get_remote_name(remotes, test_image) + lnkpath = '/dev/disk/by-path/ip-%s:3260-iscsi-rbd-lun-1' % \ + socket.gethostbyname(rem_name) + return lnkpath + + +def tgt_devname_rtn(ctx, test_image): + """ + Wrapper passed to common_fs_util functions. + """ + image = test_image[test_image.find('.') + 1:] + return tgt_devname_get(ctx, image) + + +def file_io_test(rem, file_from, lnkpath): + """ + dd to the iscsi inteface, read it, and compare with original + """ + rem.run( + args=[ + 'sudo', + 'dd', + 'if=%s' % file_from, + 'of=%s' % lnkpath, + 'bs=1024', + 'conv=fsync', + ]) + proc = rem.run(args=['mktemp'], stdout=StringIO(),) + tfile2 = proc.stdout.getvalue().strip() + rem.run( + args=[ + 'sudo', + 'rbd', + 'export', + 'iscsi-image', + run.Raw('-'), + run.Raw('>'), + tfile2, + ]) + proc = rem.run( + args=[ + 'ls', + '-l', + file_from, + run.Raw('|'), + 'awk', + '{print $5}', ], + stdout=StringIO(), + ) + size = proc.stdout.getvalue().strip() + rem.run( + args=[ + 'cmp', + '-n', + size, + file_from, + tfile2, + ]) + rem.run(args=['rm', tfile2]) + + +def general_io_test(ctx, rem, image_name): + """ + Do simple I/O tests to the iscsi interface before putting a + filesystem on it. + """ + test_phrase = 'The time has come the walrus said to speak of many things.' + lnkpath = tgt_devname_get(ctx, image_name) + proc = rem.run(args=['mktemp'], stdout=StringIO(),) + tfile1 = proc.stdout.getvalue().strip() + rem.run( + args=[ + 'echo', + test_phrase, + run.Raw('>'), + tfile1, + ]) + file_io_test(rem, tfile1, lnkpath) + rem.run(args=['rm', tfile1]) + file_io_test(rem, '/bin/ls', lnkpath) + + +@contextlib.contextmanager +def start_iscsi_initiators(ctx, tgt_link): + """ + This is the sub-task that assigns an rbd to an iscsiadm control and + performs a login (thereby creating a /dev/sd device). It performs + a logout when finished. + """ + remotes = ctx.cluster.only(teuthology.is_type('client')).remotes + tgtd_list = [] + for role, host in tgt_link: + rem = _get_remote(remotes, role) + rem_name = _get_remote_name(remotes, host) + rem.run( + args=[ + 'sudo', + 'iscsiadm', + '-m', + 'discovery', + '-t', + 'st', + '-p', + rem_name, + ]) + proc = rem.run( + args=[ + 'sudo', + 'iscsiadm', + '-m', + 'node', + '--login', + ]) + if proc.exitstatus == 0: + tgtd_list.append((rem, rem_name)) + rem.run( + args=[ + 'udevadm', + 'settle', + ]) + general_io_test(ctx, rem, host) + with contextutil.nested( + lambda: generic_mkfs(ctx=ctx, config={host: {'fs_type': 'xfs'}}, + devname_rtn=tgt_devname_rtn), + lambda: generic_mount(ctx=ctx, config={host: None}, + devname_rtn=tgt_devname_rtn), + ): + pass + try: + yield + finally: + for rem_info in tgtd_list: + rem = rem_info[0] + rem_name = rem_info[1] + rem.run( + args=[ + 'sudo', + 'iscsiadm', + '-m', + 'node', + '--logout', + ]) + rem.run( + args=[ + 'sudo', + 'iscsiadm', + '-m', + 'discovery', + '-p', + rem_name, + '-o', + 'delete', + ]) + + +@contextlib.contextmanager +def task(ctx, config): + """ + handle iscsi admin login after a tgt connection has been established. + + Assume a default host client of client.0 and a sending client of + client.0 if not specified otherwise. + + Sample tests could be: + + iscsi: + + This sets up a tgt link from client.0 to client.0 + + iscsi: [client.1, client.2] + + This sets up a tgt link from client.1 to client.0 and a tgt link + from client.2 to client.0 + + iscsi: + client.0: client.1 + client.1: client.0 + + This sets up a tgt link from client.0 to client.1 and a tgt link + from client.1 to client.0 + + Note that the iscsi image name is iscsi-image, so this only works + for one image being tested at any one time. + """ + try: + pairs = config.items() + except AttributeError: + pairs = [('client.0', 'client.0')] + with contextutil.nested( + lambda: start_iscsi_initiators(ctx=ctx, tgt_link=pairs),): + yield diff --git a/teuthology/task/rbd.py b/teuthology/task/rbd.py index 6e8112732c..71302f047b 100644 --- a/teuthology/task/rbd.py +++ b/teuthology/task/rbd.py @@ -10,15 +10,12 @@ from ..orchestra import run from teuthology import misc as teuthology from teuthology import contextutil from teuthology.parallel import parallel +from teuthology.task.common_fs_utils import generic_mkfs +from teuthology.task.common_fs_utils import generic_mount +from teuthology.task.common_fs_utils import default_image_name log = logging.getLogger(__name__) -def default_image_name(role): - """ - Currently just append role to 'testimage.' string - """ - return 'testimage.{role}'.format(role=role) - @contextlib.contextmanager def create_image(ctx, config): """ @@ -216,121 +213,9 @@ def dev_create(ctx, config): ], ) -@contextlib.contextmanager -def mkfs(ctx, config): - """ - Create a filesystem on a block device. - - For example:: - - tasks: - - ceph: - - rbd.create_image: [client.0] - - rbd.modprobe: [client.0] - - rbd.dev_create: [client.0] - - rbd.mkfs: - client.0: - fs_type: xfs - """ - assert isinstance(config, list) or isinstance(config, dict), \ - "task mkfs must be configured with a list or dictionary" - if isinstance(config, dict): - images = config.items() - else: - images = [(role, None) for role in config] - - for role, properties in images: - if properties is None: - properties = {} - (remote,) = ctx.cluster.only(role).remotes.keys() - image = properties.get('image_name', default_image_name(role)) - fs = properties.get('fs_type', 'ext3') - remote.run( - args=[ - 'sudo', - 'mkfs', - '-t', fs, - '/dev/rbd/rbd/{image}'.format(image=image), - ], - ) - yield - -@contextlib.contextmanager -def mount(ctx, config): - """ - Mount an rbd image. - - For example:: - - tasks: - - ceph: - - rbd.create_image: [client.0] - - rbd.modprobe: [client.0] - - rbd.mkfs: [client.0] - - rbd.mount: - client.0: testimage.client.0 - """ - assert isinstance(config, list) or isinstance(config, dict), \ - "task mount must be configured with a list or dictionary" - if isinstance(config, dict): - role_images = config.items() - else: - role_images = [(role, None) for role in config] - - def strip_client_prefix(role): - """Currently just removes 'client.' from start of role name""" - PREFIX = 'client.' - assert role.startswith(PREFIX) - id_ = role[len(PREFIX):] - return id_ - testdir = teuthology.get_testdir(ctx) - - mnt_template = '{tdir}/mnt.{id}' - mounted = [] - for role, image in role_images: - if image is None: - image = default_image_name(role) - (remote,) = ctx.cluster.only(role).remotes.keys() - id_ = strip_client_prefix(role) - mnt = mnt_template.format(tdir=testdir, id=id_) - mounted.append((remote, mnt)) - remote.run( - args=[ - 'mkdir', - '--', - mnt, - ] - ) - - remote.run( - args=[ - 'sudo', - 'mount', - '/dev/rbd/rbd/{image}'.format(image=image), - mnt, - ], - ) - - try: - yield - finally: - log.info("Unmounting rbd images... %s", mounted) - for remote, mnt in mounted: - remote.run( - args=[ - 'sudo', - 'umount', - mnt, - ], - ) - remote.run( - args=[ - 'rmdir', - '--', - mnt, - ] - ) +def rbd_devname_rtn(ctx, image): + return '/dev/rbd/rbd/{image}'.format(image=image) def canonical_path(ctx, role, path): """ @@ -613,7 +498,9 @@ def task(ctx, config): lambda: create_image(ctx=ctx, config=norm_config), lambda: modprobe(ctx=ctx, config=norm_config), lambda: dev_create(ctx=ctx, config=role_images), - lambda: mkfs(ctx=ctx, config=norm_config), - lambda: mount(ctx=ctx, config=role_images), + lambda: generic_mkfs(ctx=ctx, config=norm_config, + devname_rtn=rbd_devname_rtn), + lambda: generic_mount(ctx=ctx, config=role_images, + devname_rtn=rbd_devname_rtn), ): yield diff --git a/teuthology/task/tgt.py b/teuthology/task/tgt.py new file mode 100644 index 0000000000..58a80e5c15 --- /dev/null +++ b/teuthology/task/tgt.py @@ -0,0 +1,165 @@ +""" +Task to handle tgt + +Assumptions made: + The ceph-extras tgt package may need to get installed. + The open-iscsi package needs to get installed. +""" +import logging +import contextlib + +from teuthology import misc as teuthology +from teuthology import contextutil + +log = logging.getLogger(__name__) + + +@contextlib.contextmanager +def start_tgt_remotes(ctx, start_tgtd): + """ + This subtask starts up a tgtd on the clients specified + """ + remotes = ctx.cluster.only(teuthology.is_type('client')).remotes + tgtd_list = [] + for rem, roles in remotes.iteritems(): + for _id in roles: + if _id in start_tgtd: + if not rem in tgtd_list: + tgtd_list.append(rem) + rem.run( + args=[ + 'rbd', + 'create', + 'iscsi-image', + '--size', + '500', + ]) + rem.run( + args=[ + 'sudo', + 'tgtadm', + '--lld', + 'iscsi', + '--mode', + 'target', + '--op', + 'new', + '--tid', + '1', + '--targetname', + 'rbd', + ]) + rem.run( + args=[ + 'sudo', + 'tgtadm', + '--lld', + 'iscsi', + '--mode', + 'logicalunit', + '--op', + 'new', + '--tid', + '1', + '--lun', + '1', + '--backing-store', + 'iscsi-image', + '--bstype', + 'rbd', + ]) + rem.run( + args=[ + 'sudo', + 'tgtadm', + '--lld', + 'iscsi', + '--op', + 'bind', + '--mode', + 'target', + '--tid', + '1', + '-I', + 'ALL', + ]) + try: + yield + + finally: + for rem in tgtd_list: + rem.run( + args=[ + 'sudo', + 'tgtadm', + '--lld', + 'iscsi', + '--mode', + 'target', + '--op', + 'delete', + '--force', + '--tid', + '1', + ]) + rem.run( + args=[ + 'rbd', + 'snap', + 'purge', + 'iscsi-image', + ]) + rem.run( + args=[ + 'sudo', + 'rbd', + 'rm', + 'iscsi-image', + ]) + + +@contextlib.contextmanager +def task(ctx, config): + """ + Start up tgt. + + To start on on all clients:: + + tasks: + - ceph: + - tgt: + + To start on certain clients:: + + tasks: + - ceph: + - tgt: [client.0, client.3] + + or + + tasks: + - ceph: + - tgt: + client.0: + client.3: + + The general flow of things here is: + 1. Find clients on which tgt is supposed to run (start_tgtd) + 2. Remotely start up tgt daemon + On cleanup: + 3. Stop tgt daemon + + The iscsi administration is handled by the iscsi task. + """ + start_tgtd = [] + remotes = ctx.cluster.only(teuthology.is_type('client')).remotes + log.info(remotes) + if config == None: + start_tgtd = ['client.{id}'.format(id=id_) + for id_ in teuthology.all_roles_of_type(ctx.cluster, 'client')] + else: + start_tgtd = config + log.info(start_tgtd) + with contextutil.nested( + lambda: start_tgt_remotes(ctx=ctx, start_tgtd=start_tgtd),): + yield -- 2.39.5