From 78a3c234188fd0656c37353c6408f493a33c05a1 Mon Sep 17 00:00:00 2001 From: Tommi Virtanen Date: Thu, 16 Jun 2011 14:34:19 -0700 Subject: [PATCH] Move non-ceph logic out of the ceph task: base dir, archive transfer. --- teuthology/run.py | 6 +- teuthology/task/ceph.py | 79 +------------------------ teuthology/task/internal.py | 111 ++++++++++++++++++++++++++++++++++++ 3 files changed, 117 insertions(+), 79 deletions(-) diff --git a/teuthology/run.py b/teuthology/run.py index a1414b27eb..11b5fd7240 100644 --- a/teuthology/run.py +++ b/teuthology/run.py @@ -92,7 +92,11 @@ def main(): ctx.summary = {} - ctx.config['tasks'].insert(0, {'internal.check_conflict': None}) + ctx.config['tasks'][:0] = [ + {'internal.check_conflict': None}, + {'internal.base': None}, + {'internal.archive': None}, + ] from teuthology.run_tasks import run_tasks try: diff --git a/teuthology/task/ceph.py b/teuthology/task/ceph.py index e5d06d2d03..8b7bef86aa 100644 --- a/teuthology/task/ceph.py +++ b/teuthology/task/ceph.py @@ -3,10 +3,8 @@ from cStringIO import StringIO import contextlib import logging import os -import tarfile from teuthology import misc as teuthology -from teuthology import safepath from teuthology import contextutil from orchestra import run @@ -612,12 +610,11 @@ def task(ctx, config): flavor = 'gcov' coverage_dir = '/tmp/cephtest/archive/coverage' - log.info('Creating directories...') + log.info('Creating coverage directory...') run.wait( ctx.cluster.run( args=[ 'install', '-d', '-m0755', '--', - '/tmp/cephtest/archive', coverage_dir, ], wait=False, @@ -646,77 +643,3 @@ def task(ctx, config): ): healthy(ctx=ctx, config=None) yield - - if ctx.archive is not None: - - log.info('Transferring archived files...') - logdir = os.path.join(ctx.archive, 'remote') - os.mkdir(logdir) - for remote in ctx.cluster.remotes.iterkeys(): - path = os.path.join(logdir, remote.shortname) - os.mkdir(path) - log.debug('Transferring archived files from %s to %s', remote.shortname, path) - proc = remote.run( - args=[ - 'tar', - 'c', - '-f', '-', - '-C', '/tmp/cephtest/archive', - '--', - '.', - ], - stdout=run.PIPE, - wait=False, - ) - tar = tarfile.open(mode='r|', fileobj=proc.stdout) - while True: - ti = tar.next() - if ti is None: - break - - if ti.isdir(): - # ignore silently; easier to just create leading dirs below - pass - elif ti.isfile(): - sub = safepath.munge(ti.name) - safepath.makedirs(root=path, path=os.path.dirname(sub)) - tar.makefile(ti, targetpath=os.path.join(path, sub)) - else: - if ti.isdev(): - type_ = 'device' - elif ti.issym(): - type_ = 'symlink' - elif ti.islnk(): - type_ = 'hard link' - else: - type_ = 'unknown' - log.info('Ignoring tar entry: %r type %r', ti.name, type_) - continue - proc.exitstatus.get() - - log.info('Removing archived files...') - run.wait( - ctx.cluster.run( - args=[ - 'rm', - '-rf', - '--', - '/tmp/cephtest/archive', - ], - wait=False, - ), - ) - - log.info('Tidying up after the test...') - # if this fails, one of the above cleanups is flawed; don't - # just cram an rm -rf here - run.wait( - ctx.cluster.run( - args=[ - 'rmdir', - '--', - '/tmp/cephtest', - ], - wait=False, - ), - ) diff --git a/teuthology/task/internal.py b/teuthology/task/internal.py index 17b4e426b8..937f1d4c1b 100644 --- a/teuthology/task/internal.py +++ b/teuthology/task/internal.py @@ -1,10 +1,45 @@ +import contextlib import gevent import logging +import os +import tarfile +from teuthology import safepath from orchestra import run log = logging.getLogger(__name__) +@contextlib.contextmanager +def base(ctx, config): + log.info('Creating base directory...') + run.wait( + ctx.cluster.run( + args=[ + 'mkdir', '-m0755', '--', + '/tmp/cephtest', + ], + wait=False, + ) + ) + + try: + yield + finally: + log.info('Tidying up after the test...') + # if this fails, one of the earlier cleanups is flawed; don't + # just cram an rm -rf here + run.wait( + ctx.cluster.run( + args=[ + 'rmdir', + '--', + '/tmp/cephtest', + ], + wait=False, + ), + ) + + def check_conflict(ctx, config): log.info('Checking for old test directory...') processes = ctx.cluster.run( @@ -23,3 +58,79 @@ def check_conflict(ctx, config): failed = True if failed: raise RuntimeError('Stale jobs detected, aborting.') + +@contextlib.contextmanager +def archive(ctx, config): + log.info('Creating archive directory...') + run.wait( + ctx.cluster.run( + args=[ + 'install', '-d', '-m0755', '--', + '/tmp/cephtest/archive', + ], + wait=False, + ) + ) + + try: + yield + finally: + if ctx.archive is not None: + + log.info('Transferring archived files...') + logdir = os.path.join(ctx.archive, 'remote') + os.mkdir(logdir) + for remote in ctx.cluster.remotes.iterkeys(): + path = os.path.join(logdir, remote.shortname) + os.mkdir(path) + log.debug('Transferring archived files from %s to %s', remote.shortname, path) + proc = remote.run( + args=[ + 'tar', + 'c', + '-f', '-', + '-C', '/tmp/cephtest/archive', + '--', + '.', + ], + stdout=run.PIPE, + wait=False, + ) + tar = tarfile.open(mode='r|', fileobj=proc.stdout) + while True: + ti = tar.next() + if ti is None: + break + + if ti.isdir(): + # ignore silently; easier to just create leading dirs below + pass + elif ti.isfile(): + sub = safepath.munge(ti.name) + safepath.makedirs(root=path, path=os.path.dirname(sub)) + tar.makefile(ti, targetpath=os.path.join(path, sub)) + else: + if ti.isdev(): + type_ = 'device' + elif ti.issym(): + type_ = 'symlink' + elif ti.islnk(): + type_ = 'hard link' + else: + type_ = 'unknown' + log.info('Ignoring tar entry: %r type %r', ti.name, type_) + continue + proc.exitstatus.get() + + log.info('Removing archived files...') + run.wait( + ctx.cluster.run( + args=[ + 'rm', + '-rf', + '--', + '/tmp/cephtest/archive', + ], + wait=False, + ), + ) -- 2.39.5