Use SFTPClienti get for long reads/writes

Modified remote.py to use the paramiko SFTPClient get
method to extract long files (mostly tar files) from
the remote host.  The code formerly saved the data
in a long local string which was very inefficient.

Fixes: 8261
Signed-off-by: Warren Usui <warren.usui@inktank.com>

diff --git a/teuthology/misc.py b/teuthology/misc.py
index a28e80a537fc4d4bd41e89f46eda5f88edf8f2bb..bef136429af10bb41f56a3c40fa5f7cd3579b6d1 100644 (file)
--- a/teuthology/misc.py
+++ b/teuthology/misc.py
@@ -625,11 +625,9 @@ def pull_directory(remote, remotedir, localdir):
               remote.shortname, remotedir, localdir)
     if not os.path.exists(localdir):
         os.mkdir(localdir)
-    result = remote.get_tar(remotedir, sudo=True)
     _, local_tarfile = tempfile.mkstemp()
+    remote.get_tar(remotedir, local_tarfile, sudo=True)
     with open(local_tarfile, 'r+') as fb1:
-        fb1.write(result)
-        fb1.seek(0)
         tar = tarfile.open(mode='r|', fileobj=fb1)
         while True:
             ti = tar.next()
@@ -663,9 +661,7 @@ def pull_directory_tarball(remote, remotedir, localfile):
     """
     log.debug('Transferring archived files from %s:%s to %s',
               remote.shortname, remotedir, localfile)
-    tardata = remote.get_tar(remotedir, zip_flag=True, sudo=True)
-    with open(localfile, 'w') as out:
-        out.write(tardata)
+    remote.get_tar(remotedir, localfile, zip_flag=True, sudo=True)
 
 
 def get_wwn_id_map(remote, devs):

diff --git a/teuthology/orchestra/remote.py b/teuthology/orchestra/remote.py
index e24d865e40b240c4fd5230974e33dd7afcdaefe4..5e8be0c843c6182e1d16029fddc9342f0b9cc0b1 100644 (file)
--- a/teuthology/orchestra/remote.py
+++ b/teuthology/orchestra/remote.py
@@ -73,6 +73,10 @@ class Remote(object):
     def hostname(self):
         return self.name.split('@')[1]
 
+    @property
+    def username(self):
+        return self.name.split('@')[0]
+
     @property
     def is_online(self):
         if self.ssh is None:
@@ -154,6 +158,14 @@ class Remote(object):
             result = file_sftp.read()
         return result
 
+    def _sftp_copy_file(self, file_path, to_path):
+        client = paramiko.SSHClient()
+        client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        client.load_system_host_keys()
+        client.connect(self.hostname, username=self.username)
+        sftp = client.open_sftp()
+        sftp.get(file_path, to_path)
+
     def remove(self, path):
         self.run(args=['rm', '-fr', path]) 
 
@@ -164,7 +176,6 @@ class Remote(object):
         if not sudo:
             return self._sftp_get_file(path)
         temp_file_path = self.mktemp()
-        self.chmod(temp_file_path, '0666')
         args = [
             'sudo',
             'cp',
@@ -172,17 +183,17 @@ class Remote(object):
             temp_file_path,
             ]
         self.run(args=args)
+        self.chmod(temp_file_path, '0666')
         ret = self._sftp_get_file(temp_file_path) 
         self.remove(temp_file_path)
         return ret
 
-    def get_tar(self, path, sudo=False, zip_flag=False):
+    def get_tar(self, path, to_path, sudo=False, zip_flag=False):
         """
         Tar a remote file.
         """
         zip_fld = lambda x: 'cz' if x else 'c'
         temp_file_path = self.mktemp()
-        self.chmod(temp_file_path, '0666')
         args = []
         if sudo:
             args.append('sudo')
@@ -195,10 +206,11 @@ class Remote(object):
             '.',
             ])
         self.run(args=args)
-        ret = self._sftp_get_file(temp_file_path)
+        if sudo:
+            self.chmod(temp_file_path, '0666')
+        self._sftp_copy_file(temp_file_path, to_path)
         self.remove(temp_file_path)
-        return ret
-                
+
 
 def getShortName(name):
     """