Rewrite most file-retrieval functions

Signed-off-by: Zack Cerza <zack.cerza@inktank.com>

diff --git a/teuthology/misc.py b/teuthology/misc.py
index afe58e0129d30c7127b40dde4995e39abaaa6aca..185743fd32d6911eed61c4ceae8cc4dd288f1940 100644 (file)
--- a/teuthology/misc.py
+++ b/teuthology/misc.py
@@ -611,11 +611,17 @@ def create_file(remote, path, data="", permissions=str(644), sudo=False):
         append_lines_to_file(remote, path, data, sudo)
 
 
-def get_file(remote, path, sudo=False):
+def get_file(remote, path, sudo=False, dest_dir='/tmp'):
     """
-    Copy_remote wrapper.
+    Get the contents of a remote file. Do not use for large files; use
+    Remote.get_file() instead.
     """
-    return remote.get_file(path, sudo)
+    local_path = remote.get_file(path, sudo=sudo, dest_dir=dest_dir)
+    with open(local_path) as file_obj:
+        file_data = file_obj.read()
+    os.remove(local_path)
+    return file_data
+
 
 def pull_directory(remote, remotedir, localdir):
     """
@@ -628,7 +634,7 @@ def pull_directory(remote, remotedir, localdir):
     _, local_tarfile = tempfile.mkstemp(dir=localdir)
     remote.get_tar(remotedir, local_tarfile, sudo=True)
     with open(local_tarfile, 'r+') as fb1:
-        tar = tarfile.open(mode='r|', fileobj=fb1)
+        tar = tarfile.open(mode='r|gz', fileobj=fb1)
         while True:
             ti = tar.next()
             if ti is None:
@@ -661,7 +667,7 @@ def pull_directory_tarball(remote, remotedir, localfile):
     """
     log.debug('Transferring archived files from %s:%s to %s',
               remote.shortname, remotedir, localfile)
-    remote.get_tar(remotedir, localfile, zip_flag=True, sudo=True)
+    remote.get_tar(remotedir, localfile, sudo=True)
 
 
 def get_wwn_id_map(remote, devs):

diff --git a/teuthology/orchestra/remote.py b/teuthology/orchestra/remote.py
index be69a3de0064c457913cc6922033e859feece629..a80c3a31bc6166ea2704a5a9595d02eafbae6017 100644 (file)
--- a/teuthology/orchestra/remote.py
+++ b/teuthology/orchestra/remote.py
@@ -10,7 +10,8 @@ import re
 import logging
 from cStringIO import StringIO
 from teuthology import lockstatus as ls
-import paramiko
+import os
+import tempfile
 
 try:
     import libvirt
@@ -116,13 +117,11 @@ class Remote(object):
         Returns: the name of the temp file created using
                  tempfile.mkstemp
         """
-        py_cmd = """
-        import os; import tempfile; import sys;
-        (fd,fname) = tempfile.mkstemp();
-        os.close(fd);
-        sys.stdout.write(fname.rstrip());
-        sys.stdout.flush()'
-        """.replace('\n', ' ')
+        py_cmd = "import os; import tempfile; import sys;" + \
+            "(fd,fname) = tempfile.mkstemp();" + \
+            "os.close(fd);" + \
+            "sys.stdout.write(fname.rstrip());" + \
+            "sys.stdout.flush()"
         args = [
             'python',
             '-c',
@@ -149,66 +148,68 @@ class Remote(object):
             args=args,
             )
 
-    def _sftp_get_file(self, file_path):
+    def _sftp_get_file(self, remote_path, local_path):
         """
-        Use the Paramiko SFTPClient to copy the data from the remote
-        file.  Returns the file's content.
+        Use the paramiko.SFTPClient to get a file. Returns the local filename.
         """
-        conn = self.connect()
-        transport = conn.get_transport()
-        sftp = paramiko.SFTPClient.from_transport(transport)
-        with sftp.open(file_path, 'rb') as file_sftp:
-            result = file_sftp.read()
-        return result
+        sftp = self.ssh.open_sftp()
+        sftp.get(remote_path, local_path)
+        return local_path
 
-    def _sftp_copy_file(self, file_path, to_path):
+    def _sftp_open_file(self, remote_path):
+        """
+        Use the paramiko.SFTPClient to open a file. Returns a
+        paramiko.SFTPFile object.
+        """
         sftp = self.ssh.open_sftp()
-        sftp.get(file_path, to_path)
+        return sftp.open(remote_path)
 
     def remove(self, path):
         self.run(args=['rm', '-fr', path])
 
-    def get_file(self, path, sudo=False):
+    def get_file(self, path, sudo=False, dest_dir='/tmp'):
         """
-        Read a file from the remote host into memory.
+        Fetch a remote file, and return its local filename.
         """
-        if not sudo:
-            return self._sftp_get_file(path)
-        temp_file_path = self.mktemp()
-        args = [
-            'sudo',
-            'cp',
-            path,
-            temp_file_path,
-            ]
-        self.run(args=args)
-        self.chmod(temp_file_path, '0666')
-        ret = self._sftp_get_file(temp_file_path)
-        self.remove(temp_file_path)
-        return ret
+        if sudo:
+            orig_path = path
+            path = self.mktemp()
+            args = [
+                'sudo',
+                'cp',
+                orig_path,
+                path,
+                ]
+            self.run(args=args)
+            self.chmod(path, '0666')
+        (fd, local_temp_path) = tempfile.mkstemp(dir=dest_dir)
+        os.close(fd)
+        self._sftp_get_file(path, local_temp_path)
+        if sudo:
+            self.remove(path)
+        return local_temp_path
 
-    def get_tar(self, path, to_path, sudo=False, zip_flag=False):
+    def get_tar(self, path, to_path, sudo=False):
         """
-        Tar a remote file.
+        Tar a remote directory and copy it locally
         """
-        zip_fld = lambda x: 'cz' if x else 'c'
-        temp_file_path = self.mktemp()
+        remote_temp_path = self.mktemp()
         args = []
         if sudo:
             args.append('sudo')
         args.extend([
             'tar',
-            zip_fld(zip_flag),
-            '-f', temp_file_path,
+            'cz',
+            '-f', remote_temp_path,
             '-C', path,
             '--',
             '.',
             ])
         self.run(args=args)
         if sudo:
-            self.chmod(temp_file_path, '0666')
-        self._sftp_copy_file(temp_file_path, to_path)
-        self.remove(temp_file_path)
+            self.chmod(remote_temp_path, '0666')
+        self._sftp_get_file(remote_temp_path, to_path)
+        self.remove(remote_temp_path)
 
 
 def getShortName(name):