From a1838b2a0f45aa134f2b1b5d03cbf7bac8ec2920 Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Mon, 5 May 2014 16:03:39 -0500 Subject: [PATCH] Rewrite most file-retrieval functions Signed-off-by: Zack Cerza --- teuthology/misc.py | 16 ++++-- teuthology/orchestra/remote.py | 89 +++++++++++++++++----------------- 2 files changed, 56 insertions(+), 49 deletions(-) diff --git a/teuthology/misc.py b/teuthology/misc.py index afe58e0129d30..185743fd32d69 100644 --- a/teuthology/misc.py +++ b/teuthology/misc.py @@ -611,11 +611,17 @@ def create_file(remote, path, data="", permissions=str(644), sudo=False): append_lines_to_file(remote, path, data, sudo) -def get_file(remote, path, sudo=False): +def get_file(remote, path, sudo=False, dest_dir='/tmp'): """ - Copy_remote wrapper. + Get the contents of a remote file. Do not use for large files; use + Remote.get_file() instead. """ - return remote.get_file(path, sudo) + local_path = remote.get_file(path, sudo=sudo, dest_dir=dest_dir) + with open(local_path) as file_obj: + file_data = file_obj.read() + os.remove(local_path) + return file_data + def pull_directory(remote, remotedir, localdir): """ @@ -628,7 +634,7 @@ def pull_directory(remote, remotedir, localdir): _, local_tarfile = tempfile.mkstemp(dir=localdir) remote.get_tar(remotedir, local_tarfile, sudo=True) with open(local_tarfile, 'r+') as fb1: - tar = tarfile.open(mode='r|', fileobj=fb1) + tar = tarfile.open(mode='r|gz', fileobj=fb1) while True: ti = tar.next() if ti is None: @@ -661,7 +667,7 @@ def pull_directory_tarball(remote, remotedir, localfile): """ log.debug('Transferring archived files from %s:%s to %s', remote.shortname, remotedir, localfile) - remote.get_tar(remotedir, localfile, zip_flag=True, sudo=True) + remote.get_tar(remotedir, localfile, sudo=True) def get_wwn_id_map(remote, devs): diff --git a/teuthology/orchestra/remote.py b/teuthology/orchestra/remote.py index be69a3de0064c..a80c3a31bc616 100644 --- a/teuthology/orchestra/remote.py +++ b/teuthology/orchestra/remote.py @@ -10,7 +10,8 @@ import re import logging from cStringIO import StringIO from teuthology import lockstatus as ls -import paramiko +import os +import tempfile try: import libvirt @@ -116,13 +117,11 @@ class Remote(object): Returns: the name of the temp file created using tempfile.mkstemp """ - py_cmd = """ - import os; import tempfile; import sys; - (fd,fname) = tempfile.mkstemp(); - os.close(fd); - sys.stdout.write(fname.rstrip()); - sys.stdout.flush()' - """.replace('\n', ' ') + py_cmd = "import os; import tempfile; import sys;" + \ + "(fd,fname) = tempfile.mkstemp();" + \ + "os.close(fd);" + \ + "sys.stdout.write(fname.rstrip());" + \ + "sys.stdout.flush()" args = [ 'python', '-c', @@ -149,66 +148,68 @@ class Remote(object): args=args, ) - def _sftp_get_file(self, file_path): + def _sftp_get_file(self, remote_path, local_path): """ - Use the Paramiko SFTPClient to copy the data from the remote - file. Returns the file's content. + Use the paramiko.SFTPClient to get a file. Returns the local filename. """ - conn = self.connect() - transport = conn.get_transport() - sftp = paramiko.SFTPClient.from_transport(transport) - with sftp.open(file_path, 'rb') as file_sftp: - result = file_sftp.read() - return result + sftp = self.ssh.open_sftp() + sftp.get(remote_path, local_path) + return local_path - def _sftp_copy_file(self, file_path, to_path): + def _sftp_open_file(self, remote_path): + """ + Use the paramiko.SFTPClient to open a file. Returns a + paramiko.SFTPFile object. + """ sftp = self.ssh.open_sftp() - sftp.get(file_path, to_path) + return sftp.open(remote_path) def remove(self, path): self.run(args=['rm', '-fr', path]) - def get_file(self, path, sudo=False): + def get_file(self, path, sudo=False, dest_dir='/tmp'): """ - Read a file from the remote host into memory. + Fetch a remote file, and return its local filename. """ - if not sudo: - return self._sftp_get_file(path) - temp_file_path = self.mktemp() - args = [ - 'sudo', - 'cp', - path, - temp_file_path, - ] - self.run(args=args) - self.chmod(temp_file_path, '0666') - ret = self._sftp_get_file(temp_file_path) - self.remove(temp_file_path) - return ret + if sudo: + orig_path = path + path = self.mktemp() + args = [ + 'sudo', + 'cp', + orig_path, + path, + ] + self.run(args=args) + self.chmod(path, '0666') + (fd, local_temp_path) = tempfile.mkstemp(dir=dest_dir) + os.close(fd) + self._sftp_get_file(path, local_temp_path) + if sudo: + self.remove(path) + return local_temp_path - def get_tar(self, path, to_path, sudo=False, zip_flag=False): + def get_tar(self, path, to_path, sudo=False): """ - Tar a remote file. + Tar a remote directory and copy it locally """ - zip_fld = lambda x: 'cz' if x else 'c' - temp_file_path = self.mktemp() + remote_temp_path = self.mktemp() args = [] if sudo: args.append('sudo') args.extend([ 'tar', - zip_fld(zip_flag), - '-f', temp_file_path, + 'cz', + '-f', remote_temp_path, '-C', path, '--', '.', ]) self.run(args=args) if sudo: - self.chmod(temp_file_path, '0666') - self._sftp_copy_file(temp_file_path, to_path) - self.remove(temp_file_path) + self.chmod(remote_temp_path, '0666') + self._sftp_get_file(remote_temp_path, to_path) + self.remove(remote_temp_path) def getShortName(name): -- 2.39.5