From: Dan Mick <dan.mick@redhat.com>
Date: Wed, 2 Nov 2016 04:27:13 +0000 (-0700)
Subject: Use remote.get_tar_stream in misc.pull_directory
X-Git-Tag: 1.1.0~511^2
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=eb736241cb0c03203b3b4c95f36ef4280cd729de;p=teuthology.git

Use remote.get_tar_stream in misc.pull_directory

Avoids remote and local temp files, and subsequent read/write
thrashing of local dir.  Since some of those directories are
multiple tens or hundreds of gigabytes, the extra file load
is important.

Signed-off-by: Dan Mick <dan.mick@redhat.com>
---

diff --git a/teuthology/misc.py b/teuthology/misc.py
index 18ac78fab..dd5629c27 100644
--- a/teuthology/misc.py
+++ b/teuthology/misc.py
@@ -19,7 +19,6 @@ import urlparse
 import yaml
 import json
 import re
-import tempfile
 import pprint
 
 from teuthology import safepath
@@ -778,33 +777,31 @@ def pull_directory(remote, remotedir, localdir):
               remote.shortname, remotedir, localdir)
     if not os.path.exists(localdir):
         os.mkdir(localdir)
-    _, local_tarfile = tempfile.mkstemp(dir=localdir)
-    remote.get_tar(remotedir, local_tarfile, sudo=True)
-    with open(local_tarfile, 'r+') as fb1:
-        tar = tarfile.open(mode='r|gz', fileobj=fb1)
-        while True:
-            ti = tar.next()
-            if ti is None:
-                break
+    r = remote.get_tar_stream(remotedir, sudo=True)
+    tar = tarfile.open(mode='r|gz', fileobj=r.stdout)
+    while True:
+        ti = tar.next()
+        if ti is None:
+            break
 
-            if ti.isdir():
-                # ignore silently; easier to just create leading dirs below
-                pass
-            elif ti.isfile():
-                sub = safepath.munge(ti.name)
-                safepath.makedirs(root=localdir, path=os.path.dirname(sub))
-                tar.makefile(ti, targetpath=os.path.join(localdir, sub))
+        if ti.isdir():
+            # ignore silently; easier to just create leading dirs below
+            # XXX this mean empty dirs are not transferred
+            pass
+        elif ti.isfile():
+            sub = safepath.munge(ti.name)
+            safepath.makedirs(root=localdir, path=os.path.dirname(sub))
+            tar.makefile(ti, targetpath=os.path.join(localdir, sub))
+        else:
+            if ti.isdev():
+                type_ = 'device'
+            elif ti.issym():
+                type_ = 'symlink'
+            elif ti.islnk():
+                type_ = 'hard link'
             else:
-                if ti.isdev():
-                    type_ = 'device'
-                elif ti.issym():
-                    type_ = 'symlink'
-                elif ti.islnk():
-                    type_ = 'hard link'
-                else:
-                    type_ = 'unknown'
-                log.info('Ignoring tar entry: %r type %r', ti.name, type_)
-    os.remove(local_tarfile)
+                type_ = 'unknown'
+            log.info('Ignoring tar entry: %r type %r', ti.name, type_)
 
 
 def pull_directory_tarball(remote, remotedir, localfile):