From d91f028e26f9329c51be59185af902db285d1cf8 Mon Sep 17 00:00:00 2001 From: John Spray Date: Wed, 2 Jul 2014 16:43:16 +0100 Subject: [PATCH] task: refactor kclient into KernelMount Additionally make both kclient and ceph_fuse tasks yield a collection of CephFSMount objects so that subsequent tasks can retrieve them. Signed-off-by: John Spray --- teuthology/misc.py | 20 ------ teuthology/task/ceph_fuse.py | 13 ++-- teuthology/task/cephfs/filesystem.py | 26 -------- teuthology/task/cephfs/fuse_mount.py | 16 +---- teuthology/task/cephfs/kernel_mount.py | 82 ++++++++++++++++++++++++ teuthology/task/cephfs/mount.py | 44 +++++++++++++ teuthology/task/kclient.py | 80 ++++++----------------- teuthology/task/mds_journal_migration.py | 20 ++++-- 8 files changed, 171 insertions(+), 130 deletions(-) create mode 100644 teuthology/task/cephfs/kernel_mount.py create mode 100644 teuthology/task/cephfs/mount.py diff --git a/teuthology/misc.py b/teuthology/misc.py index 4d295b5337..3757c5e124 100644 --- a/teuthology/misc.py +++ b/teuthology/misc.py @@ -894,26 +894,6 @@ def reconnect(ctx, timeout, remotes=None): time.sleep(1) -def write_secret_file(ctx, remote, role, keyring, filename): - """ - Stash the kerying in the filename specified. - """ - testdir = get_testdir(ctx) - remote.run( - args=[ - 'adjust-ulimits', - 'ceph-coverage', - '{tdir}/archive/coverage'.format(tdir=testdir), - 'ceph-authtool', - '--name={role}'.format(role=role), - '--print-key', - keyring, - run.Raw('>'), - filename, - ], - ) - - def get_clients(ctx, roles): """ return all remote roles that are clients. diff --git a/teuthology/task/ceph_fuse.py b/teuthology/task/ceph_fuse.py index e49d1c969d..306a4f1112 100644 --- a/teuthology/task/ceph_fuse.py +++ b/teuthology/task/ceph_fuse.py @@ -77,28 +77,29 @@ def task(ctx, config): clients = list(teuthology.get_clients(ctx=ctx, roles=config.keys())) - fuse_mounts = [] + fuse_mounts = {} for id_, remote in clients: client_config = config.get("client.%s" % id_) if client_config is None: client_config = {} fuse_mount = FuseMount(client_config, testdir, id_, remote) - fuse_mounts.append(fuse_mount) + fuse_mounts[id_] = fuse_mount fuse_mount.mount() - for mount in fuse_mounts: + for mount in fuse_mounts.values(): mount.wait_until_mounted() + ctx.mounts = fuse_mounts try: - yield + yield fuse_mounts finally: log.info('Unmounting ceph-fuse clients...') - for mount in fuse_mounts: + for mount in fuse_mounts.values(): mount.umount() run.wait([m.fuse_daemon for m in fuse_mounts.values()], timeout=600) - for mount in fuse_mounts: + for mount in fuse_mounts.values(): mount.cleanup() diff --git a/teuthology/task/cephfs/filesystem.py b/teuthology/task/cephfs/filesystem.py index 5025752c6a..6295ac04d3 100644 --- a/teuthology/task/cephfs/filesystem.py +++ b/teuthology/task/cephfs/filesystem.py @@ -2,7 +2,6 @@ from StringIO import StringIO import json import logging -import os from teuthology import misc from teuthology.task import ceph_manager @@ -41,8 +40,6 @@ class Filesystem(object): self.client_id = client_list[0] self.client_remote = list(misc.get_clients(ctx=ctx, roles=["client.{0}".format(self.client_id)]))[0][1] - self.test_files = ['a', 'b', 'c'] - def mds_stop(self): mds = self._ctx.daemons.get_daemon('mds', self.mds_id) mds.stop() @@ -60,29 +57,6 @@ class Filesystem(object): self.mds_manager.raw_cluster_cmd_result('fs', 'rm', "default", "--yes-i-really-mean-it") self.mds_manager.raw_cluster_cmd_result('fs', 'new', "default", "metadata", "data") - @property - def _mount_path(self): - return os.path.join(misc.get_testdir(self._ctx), 'mnt.{0}'.format(self.client_id)) - - def create_files(self): - for suffix in self.test_files: - log.info("Creating file {0}".format(suffix)) - self.client_remote.run(args=[ - 'sudo', 'touch', os.path.join(self._mount_path, suffix) - ]) - - def check_files(self): - """ - This will raise a CommandFailedException if expected files are not present - """ - for suffix in self.test_files: - log.info("Checking file {0}".format(suffix)) - r = self.client_remote.run(args=[ - 'sudo', 'ls', os.path.join(self._mount_path, suffix) - ], check_status=False) - if r.exitstatus != 0: - raise RuntimeError("Expected file {0} not found".format(suffix)) - def get_metadata_object(self, object_type, object_id): """ Retrieve an object from the metadata pool, pass it through diff --git a/teuthology/task/cephfs/fuse_mount.py b/teuthology/task/cephfs/fuse_mount.py index 93aeca341d..67078c6b79 100644 --- a/teuthology/task/cephfs/fuse_mount.py +++ b/teuthology/task/cephfs/fuse_mount.py @@ -6,28 +6,18 @@ import logging from teuthology import misc from ...orchestra import run +from teuthology.task.cephfs.mount import CephFSMount log = logging.getLogger(__name__) -class FuseMount(object): +class FuseMount(CephFSMount): def __init__(self, client_config, test_dir, client_id, client_remote): - """ - :param client_config: Configuration dictionary for this particular client - :param test_dir: Global teuthology test dir - :param client_id: Client ID, the 'foo' in client.foo - :param client_remote: Remote instance for the host where client will run - """ + super(FuseMount, self).__init__(test_dir, client_id, client_remote) self.client_config = client_config - self.test_dir = test_dir - self.client_id = client_id - self.client_remote = client_remote self.fuse_daemon = None - self.mountpoint = os.path.join(self.test_dir, 'mnt.{id}'.format(id=self.client_id)) - - def mount(self): log.info("Client client.%s config is %s" % (self.client_id, self.client_config)) diff --git a/teuthology/task/cephfs/kernel_mount.py b/teuthology/task/cephfs/kernel_mount.py new file mode 100644 index 0000000000..6e47bb45ca --- /dev/null +++ b/teuthology/task/cephfs/kernel_mount.py @@ -0,0 +1,82 @@ +import logging +import os + +from teuthology.orchestra import run +from teuthology.task.cephfs.mount import CephFSMount + +log = logging.getLogger(__name__) + + +class KernelMount(CephFSMount): + def __init__(self, mons, test_dir, client_id, client_remote): + super(KernelMount, self).__init__(test_dir, client_id, client_remote) + self.mons = mons + + def write_secret_file(self, remote, role, keyring, filename): + """ + Stash the keyring in the filename specified. + """ + remote.run( + args=[ + 'adjust-ulimits', + 'ceph-coverage', + '{tdir}/archive/coverage'.format(tdir=self.test_dir), + 'ceph-authtool', + '--name={role}'.format(role=role), + '--print-key', + keyring, + run.Raw('>'), + filename, + ], + ) + + def mount(self): + log.info('Mounting kclient client.{id} at {remote} {mnt}...'.format( + id=self.client_id, remote=self.client_remote, mnt=self.mountpoint)) + + keyring = '/etc/ceph/ceph.client.{id}.keyring'.format(id=self.client_id) + secret = '{tdir}/data/client.{id}.secret'.format(tdir=self.test_dir, id=self.client_id) + self.write_secret_file(self.client_remote, 'client.{id}'.format(id=self.client_id), + keyring, secret) + + self.client_remote.run( + args=[ + 'mkdir', + '--', + self.mountpoint, + ], + ) + + self.client_remote.run( + args=[ + 'sudo', + 'adjust-ulimits', + 'ceph-coverage', + '{tdir}/archive/coverage'.format(tdir=self.test_dir), + '/sbin/mount.ceph', + '{mons}:/'.format(mons=','.join(self.mons)), + self.mountpoint, + '-v', + '-o', + 'name={id},secretfile={secret}'.format(id=self.client_id, + secret=secret), + ], + ) + + def umount(self): + log.debug('Unmounting client client.{id}...'.format(id=self.client_id)) + mnt = os.path.join(self.test_dir, 'mnt.{id}'.format(id=self.client_id)) + self.client_remote.run( + args=[ + 'sudo', + 'umount', + mnt, + ], + ) + self.client_remote.run( + args=[ + 'rmdir', + '--', + mnt, + ], + ) diff --git a/teuthology/task/cephfs/mount.py b/teuthology/task/cephfs/mount.py new file mode 100644 index 0000000000..337b451104 --- /dev/null +++ b/teuthology/task/cephfs/mount.py @@ -0,0 +1,44 @@ + +import logging +import os + +log = logging.getLogger(__name__) + + +class CephFSMount(object): + def __init__(self, test_dir, client_id, client_remote): + """ + :param test_dir: Global teuthology test dir + :param client_id: Client ID, the 'foo' in client.foo + :param client_remote: Remote instance for the host where client will run + """ + + self.test_dir = test_dir + self.client_id = client_id + self.client_remote = client_remote + + self.mountpoint = os.path.join(self.test_dir, 'mnt.{id}'.format(id=self.client_id)) + self.test_files = ['a', 'b', 'c'] + + @property + def _mount_path(self): + return os.path.join(self.test_dir, 'mnt.{0}'.format(self.client_id)) + + def create_files(self): + for suffix in self.test_files: + log.info("Creating file {0}".format(suffix)) + self.client_remote.run(args=[ + 'sudo', 'touch', os.path.join(self._mount_path, suffix) + ]) + + def check_files(self): + """ + This will raise a CommandFailedException if expected files are not present + """ + for suffix in self.test_files: + log.info("Checking file {0}".format(suffix)) + r = self.client_remote.run(args=[ + 'sudo', 'ls', os.path.join(self._mount_path, suffix) + ], check_status=False) + if r.exitstatus != 0: + raise RuntimeError("Expected file {0} not found".format(suffix)) diff --git a/teuthology/task/kclient.py b/teuthology/task/kclient.py index 2229fe07ea..e7b2e0f3ed 100644 --- a/teuthology/task/kclient.py +++ b/teuthology/task/kclient.py @@ -3,9 +3,9 @@ Mount/unmount a ``kernel`` client. """ import contextlib import logging -import os -from teuthology import misc as teuthology +from teuthology import misc +from teuthology.task.cephfs.kernel_mount import KernelMount log = logging.getLogger(__name__) @@ -43,69 +43,29 @@ def task(ctx, config): if config is None: config = ['client.{id}'.format(id=id_) - for id_ in teuthology.all_roles_of_type(ctx.cluster, 'client')] - clients = list(teuthology.get_clients(ctx=ctx, roles=config)) + for id_ in misc.all_roles_of_type(ctx.cluster, 'client')] + clients = list(misc.get_clients(ctx=ctx, roles=config)) - testdir = teuthology.get_testdir(ctx) + test_dir = misc.get_testdir(ctx) - for id_, remote in clients: - mnt = os.path.join(testdir, 'mnt.{id}'.format(id=id_)) - log.info('Mounting kclient client.{id} at {remote} {mnt}...'.format( - id=id_, remote=remote, mnt=mnt)) - - # figure mon ips - remotes_and_roles = ctx.cluster.remotes.items() - roles = [roles for (remote_, roles) in remotes_and_roles] - ips = [host for (host, port) in (remote_.ssh.get_transport().getpeername() for (remote_, roles) in remotes_and_roles)] - mons = teuthology.get_mons(roles, ips).values() + # Assemble mon addresses + remotes_and_roles = ctx.cluster.remotes.items() + roles = [roles for (remote_, roles) in remotes_and_roles] + ips = [host for (host, port) in + (remote_.ssh.get_transport().getpeername() for (remote_, roles) in remotes_and_roles)] + mons = misc.get_mons(roles, ips).values() - keyring = '/etc/ceph/ceph.client.{id}.keyring'.format(id=id_) - secret = '{tdir}/data/client.{id}.secret'.format(tdir=testdir, id=id_) - teuthology.write_secret_file(ctx, remote, 'client.{id}'.format(id=id_), - keyring, secret) - - remote.run( - args=[ - 'mkdir', - '--', - mnt, - ], - ) + mounts = {} + for id_, remote in clients: + kernel_mount = KernelMount(mons, test_dir, id_, remote) + mounts[id_] = kernel_mount - remote.run( - args=[ - 'sudo', - 'adjust-ulimits', - 'ceph-coverage', - '{tdir}/archive/coverage'.format(tdir=testdir), - '/sbin/mount.ceph', - '{mons}:/'.format(mons=','.join(mons)), - mnt, - '-v', - '-o', - 'name={id},secretfile={secret}'.format(id=id_, - secret=secret), - ], - ) + kernel_mount.mount() + ctx.mounts = mounts try: - yield + yield mounts finally: log.info('Unmounting kernel clients...') - for id_, remote in clients: - log.debug('Unmounting client client.{id}...'.format(id=id_)) - mnt = os.path.join(testdir, 'mnt.{id}'.format(id=id_)) - remote.run( - args=[ - 'sudo', - 'umount', - mnt, - ], - ) - remote.run( - args=[ - 'rmdir', - '--', - mnt, - ], - ) + for mount in mounts.values(): + mount.umount() diff --git a/teuthology/task/mds_journal_migration.py b/teuthology/task/mds_journal_migration.py index f80726d48c..31de29c430 100644 --- a/teuthology/task/mds_journal_migration.py +++ b/teuthology/task/mds_journal_migration.py @@ -1,6 +1,7 @@ import contextlib import logging +from teuthology import misc from teuthology.task.ceph import write_conf from teuthology.task.ceph_fuse import task as ceph_fuse_ctx @@ -21,6 +22,13 @@ def task(ctx, config): successful completion the filesystem will be running with a journal in the new format. """ + # Pick one client to use + client_list = list(misc.all_roles_of_type(ctx.cluster, 'client')) + try: + client_id = client_list[0] + except IndexError: + raise RuntimeError("This task requires at least one client") + fs = Filesystem(ctx, config) old_journal_version = JOURNAL_FORMAT_LEGACY new_journal_version = JOURNAL_FORMAT_RESILIENT @@ -41,9 +49,10 @@ def task(ctx, config): fs.mds_restart() # Do some client work so that the log is populated with something. - with ceph_fuse_ctx(ctx, None): - fs.create_files() - fs.check_files() # sanity, this should always pass + with ceph_fuse_ctx(ctx, None) as client_mounts: + mount = client_mounts[client_id] + mount.create_files() + mount.check_files() # sanity, this should always pass # Modify the ceph.conf to ask the MDS to use the new journal format. ctx.ceph.conf['mds']['mds journal format'] = new_journal_version @@ -54,8 +63,9 @@ def task(ctx, config): # Check that files created in the initial client workload are still visible # in a client mount. - with ceph_fuse_ctx(ctx, None): - fs.check_files() + with ceph_fuse_ctx(ctx, None) as client_mounts: + mount = client_mounts[client_id] + mount.check_files() # Verify that the journal really has been rewritten. journal_version = fs.get_journal_version() -- 2.39.5