internal: move pulling archive w/ tar to helper

diff --git a/teuthology/misc.py b/teuthology/misc.py
index 33f9b5169738622de52eb364873628c3250be421..bc025ee02c07a48301293d6cfc3212a0f2c99a03 100644 (file)
--- a/teuthology/misc.py
+++ b/teuthology/misc.py
@@ -5,6 +5,7 @@ import logging
 import configobj
 import getpass
 import socket
+import tarfile
 import time
 import urllib2
 import urlparse
@@ -12,6 +13,7 @@ import yaml
 import json
 import subprocess
 
+from teuthology import safepath
 from .orchestra import run
 
 log = logging.getLogger(__name__)
@@ -239,6 +241,51 @@ def get_file(remote, path):
     data = proc.stdout.getvalue()
     return data
 
+def pull_directory(remote, remotedir, localdir):
+    """
+    Copy a remote directory to a local directory.
+    """
+    os.mkdir(localdir)
+    log.debug('Transferring archived files from %s:%s to %s',
+              remote.shortname, remotedir, localdir)
+    proc = remote.run(
+        args=[
+            'tar',
+            'c',
+            '-f', '-',
+            '-C', remotedir,
+            '--',
+            '.',
+            ],
+        stdout=run.PIPE,
+        wait=False,
+        )
+    tar = tarfile.open(mode='r|', fileobj=proc.stdout)
+    while True:
+        ti = tar.next()
+        if ti is None:
+            break
+
+        if ti.isdir():
+            # ignore silently; easier to just create leading dirs below
+            pass
+        elif ti.isfile():
+            sub = safepath.munge(ti.name)
+            safepath.makedirs(root=localdir, path=os.path.dirname(sub))
+            tar.makefile(ti, targetpath=os.path.join(localdir, sub))
+        else:
+            if ti.isdev():
+                type_ = 'device'
+            elif ti.issym():
+                type_ = 'symlink'
+            elif ti.islnk():
+                type_ = 'hard link'
+            else:
+                type_ = 'unknown'
+                log.info('Ignoring tar entry: %r type %r', ti.name, type_)
+                continue
+    proc.exitstatus.get()
+
 def get_scratch_devices(remote):
     """
     Read the scratch disk list from remote host

diff --git a/teuthology/task/internal.py b/teuthology/task/internal.py
index 0ad0b03caade67f8f178bf88f1075b8d92d3edd4..3ff0a20063e1ff5cb9df5a6a3ce39787a87e9076 100644 (file)
--- a/teuthology/task/internal.py
+++ b/teuthology/task/internal.py
@@ -3,13 +3,11 @@ import contextlib
 import gevent
 import logging
 import os
-import tarfile
 import time
 import yaml
 
 from teuthology import lock
 from teuthology import misc as teuthology
-from teuthology import safepath
 from ..orchestra import run
 
 log = logging.getLogger(__name__)
@@ -190,51 +188,12 @@ def archive(ctx, config):
         yield
     finally:
         if ctx.archive is not None:
-
             log.info('Transferring archived files...')
             logdir = os.path.join(ctx.archive, 'remote')
             os.mkdir(logdir)
             for remote in ctx.cluster.remotes.iterkeys():
                 path = os.path.join(logdir, remote.shortname)
-                os.mkdir(path)
-                log.debug('Transferring archived files from %s to %s', remote.shortname, path)
-                proc = remote.run(
-                    args=[
-                        'tar',
-                        'c',
-                        '-f', '-',
-                        '-C', '/tmp/cephtest/archive',
-                        '--',
-                        '.',
-                        ],
-                    stdout=run.PIPE,
-                    wait=False,
-                    )
-                tar = tarfile.open(mode='r|', fileobj=proc.stdout)
-                while True:
-                    ti = tar.next()
-                    if ti is None:
-                        break
-
-                    if ti.isdir():
-                        # ignore silently; easier to just create leading dirs below
-                        pass
-                    elif ti.isfile():
-                        sub = safepath.munge(ti.name)
-                        safepath.makedirs(root=path, path=os.path.dirname(sub))
-                        tar.makefile(ti, targetpath=os.path.join(path, sub))
-                    else:
-                        if ti.isdev():
-                            type_ = 'device'
-                        elif ti.issym():
-                            type_ = 'symlink'
-                        elif ti.islnk():
-                            type_ = 'hard link'
-                        else:
-                            type_ = 'unknown'
-                        log.info('Ignoring tar entry: %r type %r', ti.name, type_)
-                        continue
-                proc.exitstatus.get()
+                teuthology.pull_directory(remote, '/tmp/cephtest/archive', path)
 
         log.info('Removing archive directory...')
         run.wait(