From: Tommi Virtanen Date: Tue, 7 Jun 2011 21:47:30 +0000 (-0700) Subject: Archive logs if given --archive=PATH. Clean up after a test run. X-Git-Tag: v0.94.10~27^2^2~364^2~1758 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=e6ae6bddf36f68aecf41a54f02e521960a1b3d90;p=ceph.git Archive logs if given --archive=PATH. Clean up after a test run. --- diff --git a/teuthology/run.py b/teuthology/run.py index d25fc4352663..1c983b1e7bcc 100644 --- a/teuthology/run.py +++ b/teuthology/run.py @@ -34,6 +34,11 @@ def parse_args(): default={}, help='config file to read', ) + parser.add_argument( + '--archive', + metavar='DIR', + help='path to archive results in', + ) args = parser.parse_args() return args diff --git a/teuthology/safepath.py b/teuthology/safepath.py new file mode 100644 index 000000000000..b8115a25edca --- /dev/null +++ b/teuthology/safepath.py @@ -0,0 +1,42 @@ +import errno +import os + +def munge(path): + """ + Munge a potentially hostile path name to be safe to use. + + This very definitely changes the meaning of the path, + but it only does that for unsafe paths. + """ + # explicitly ignoring windows as a platform + segments = path.split('/') + # filter out empty segments like foo//bar + segments = [s for s in segments if s!=''] + # filter out no-op segments like foo/./bar + segments = [s for s in segments if s!='.'] + # all leading dots become underscores; makes .. safe too + for idx, seg in enumerate(segments): + if seg.startswith('.'): + segments[idx] = '_'+seg[1:] + # empty string, "/", "//", etc + if not segments: + segments = ['_'] + return '/'.join(segments) + + +def makedirs(root, path): + """ + os.makedirs gets confused if the path contains '..', and root might. + + This relies on the fact that `path` has been normalized by munge(). + """ + segments = path.split('/') + for seg in segments: + root = os.path.join(root, seg) + try: + os.mkdir(root) + except OSError as e: + if e.errno == errno.EEXIST: + pass + else: + raise diff --git a/teuthology/task/ceph.py b/teuthology/task/ceph.py index e5676a27a48a..ac90bed15c86 100644 --- a/teuthology/task/ceph.py +++ b/teuthology/task/ceph.py @@ -4,8 +4,10 @@ import contextlib import logging import os import gevent +import tarfile from teuthology import misc as teuthology +from teuthology import safepath from orchestra import run log = logging.getLogger(__name__) @@ -421,3 +423,122 @@ def task(ctx, config): for id_, proc in mon_daemons.iteritems(): proc.stdin.close() run.wait(mon_daemons.itervalues()) + + log.info('Removing uninteresting files...') + run.wait( + ctx.cluster.run( + args=[ + 'rm', + '-rf', + '--', + '/tmp/cephtest/binary', + '/tmp/cephtest/class_tmp', + '/tmp/cephtest/daemon-helper', + '/tmp/cephtest/ceph.conf', + '/tmp/cephtest/ceph.keyring', + '/tmp/cephtest/temp.keyring', + '/tmp/cephtest/data', + ], + wait=False, + ), + ) + + if ctx.archive is not None: + os.mkdir(ctx.archive) + + log.info('Compressing logs...') + run.wait( + ctx.cluster.run( + args=[ + 'find', + '/tmp/cephtest/log', + '-name', + '*.log', + '-print0', + run.Raw('|'), + 'xargs', + '-0', + '--no-run-if-empty', + '--', + 'bzip2', + '-9', + '--', + ], + wait=False, + ), + ) + + log.info('Transferring logs...') + for logtype in ['log', 'profiling-logger']: + logdir = os.path.join(ctx.archive, logtype) + os.mkdir(logdir) + for remote in ctx.cluster.remotes.iterkeys(): + path = os.path.join(logdir, remote.shortname) + os.mkdir(path) + log.debug('Transferring logs for %s from %s to %s', remote.shortname, logtype, path) + proc = remote.run( + args=[ + 'tar', + 'c', + '-f', '-', + '-C', '/tmp/cephtest', + '-C', logtype, + '--', + '.', + ], + stdout=run.PIPE, + wait=False, + ) + tar = tarfile.open(mode='r|', fileobj=proc.stdout) + while True: + ti = tar.next() + if ti is None: + break + + if ti.isdir(): + # ignore silently; easier to just create leading dirs below + pass + elif ti.isfile(): + sub = safepath.munge(ti.name) + safepath.makedirs(root=path, path=os.path.dirname(sub)) + tar.makefile(ti, targetpath=os.path.join(path, sub)) + else: + if ti.isdev(): + type_ = 'device' + elif ti.issym(): + type_ = 'symlink' + elif ti.islnk(): + type_ = 'hard link' + else: + type_ = 'unknown' + log.info('Ignoring tar entry: %r type %r', ti.name, type_) + continue + proc.exitstatus.get() + + log.info('Removing log files...') + run.wait( + ctx.cluster.run( + args=[ + 'rm', + '-rf', + '--', + '/tmp/cephtest/log', + '/tmp/cephtest/profiling-logger', + ], + wait=False, + ), + ) + + log.info('Tidying up after the test...') + # if this fails, one of the above cleanups is flawed; don't + # just cram an rm -rf here + run.wait( + ctx.cluster.run( + args=[ + 'rmdir', + '--', + '/tmp/cephtest', + ], + wait=False, + ), + ) diff --git a/teuthology/test/__init__.py b/teuthology/test/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/teuthology/test/test_safepath.py b/teuthology/test/test_safepath.py new file mode 100644 index 000000000000..4f05f10d2d70 --- /dev/null +++ b/teuthology/test/test_safepath.py @@ -0,0 +1,56 @@ +from nose.tools import eq_ as eq + +from .. import safepath + +def test_simple(): + got = safepath.munge('foo') + eq(got, 'foo') + +def test_empty(): + # really odd corner case + got = safepath.munge('') + eq(got, '_') + +def test_slash(): + got = safepath.munge('/') + eq(got, '_') + +def test_slashslash(): + got = safepath.munge('//') + eq(got, '_') + +def test_absolute(): + got = safepath.munge('/evil') + eq(got, 'evil') + +def test_absolute_subdir(): + got = safepath.munge('/evil/here') + eq(got, 'evil/here') + +def test_dot_leading(): + got = safepath.munge('./foo') + eq(got, 'foo') + +def test_dot_middle(): + got = safepath.munge('evil/./foo') + eq(got, 'evil/foo') + +def test_dot_trailing(): + got = safepath.munge('evil/foo/.') + eq(got, 'evil/foo') + +def test_dotdot(): + got = safepath.munge('../evil/foo') + eq(got, '_./evil/foo') + +def test_dotdot_subdir(): + got = safepath.munge('evil/../foo') + eq(got, 'evil/_./foo') + +def test_hidden(): + got = safepath.munge('.evil') + eq(got, '_evil') + +def test_hidden_subdir(): + got = safepath.munge('foo/.evil') + eq(got, 'foo/_evil')