From 0268487057288d9c0a1ac9c6cf6d75218eada1e1 Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Mon, 5 May 2014 16:57:01 -0500 Subject: [PATCH] Revert "Revert "Handle raw data I/O."" This reverts commit 0c8a3e2334631f5fc20cb7933d0005353ea6a46e. --- teuthology/misc.py | 148 +++++++++++++-------------------- teuthology/orchestra/remote.py | 78 +++++++++++++++++ teuthology/task/kernel.py | 2 +- 3 files changed, 139 insertions(+), 89 deletions(-) diff --git a/teuthology/misc.py b/teuthology/misc.py index c075e56d6dd51..27e9b8b2e20c0 100644 --- a/teuthology/misc.py +++ b/teuthology/misc.py @@ -18,6 +18,8 @@ import urlparse import yaml import json import re +import tempfile +import paramiko from teuthology import safepath from .orchestra import run @@ -554,7 +556,7 @@ def remove_lines_from_file(remote, path, line_is_valid_test, # get a temp file path on the remote host to write to, # we don't want to blow away the remote file and then have the # network drop out - temp_file_path = remote_mktemp(remote) + temp_file_path = remote.remote_mktemp() # write out the data to a temp file write_file(remote, temp_file_path, out_data) @@ -570,7 +572,7 @@ def append_lines_to_file(remote, path, lines, sudo=False): Remove_lines_from_list. """ - temp_file_path = remote_mktemp(remote) + temp_file_path = remote.remote_mktemp() data = get_file(remote, path, sudo) @@ -585,26 +587,6 @@ def append_lines_to_file(remote, path, lines, sudo=False): move_file(remote, temp_file_path, path) -def remote_mktemp(remote, sudo=False): - """ - Make a temporary file on a remote system - """ - args = [] - if sudo: - args.append('sudo') - args.extend([ - 'python', - '-c', - 'import os; import tempfile; (fd,fname) = tempfile.mkstemp(); os.close(fd); print fname.rstrip()' - ]) - proc = remote.run( - args=args, - stdout=StringIO(), - ) - data = proc.stdout.getvalue() - return data - - def create_file(remote, path, data="", permissions=str(644), sudo=False): """ Create a file on the remote host. @@ -630,25 +612,35 @@ def create_file(remote, path, data="", permissions=str(644), sudo=False): append_lines_to_file(remote, path, data, sudo) -def get_file(remote, path, sudo=False): +def do_remote_sftp(remote, tempf, sudo=False): """ - Read a file from remote host into memory. + Make sure file is aways readble if root, and use SFTPClient to + copy across data. """ - args = [] if sudo: - args.append('sudo') - args.extend([ - 'cat', - '--', - path, - ]) - proc = remote.run( - args=args, - stdout=StringIO(), - ) - data = proc.stdout.getvalue() - return data + args = [] + args.extend([ + 'sudo', + 'chmod', + '0666', + tempf, + ]) + remote.run( + args=args, + stdout=StringIO(), + ) + conn = remote.connect() + transport = conn.get_transport() + sftp = paramiko.SFTPClient.from_transport(transport) + with sftp.open(tempf, 'rb') as file_sftp: + result = file_sftp.read() + return result +def get_file(remote, path, sudo=False): + """ + Copy_remote wrapper. + """ + return remote.copy_remote(path, sudo) def pull_directory(remote, remotedir, localdir): """ @@ -658,44 +650,36 @@ def pull_directory(remote, remotedir, localdir): remote.shortname, remotedir, localdir) if not os.path.exists(localdir): os.mkdir(localdir) - proc = remote.run( - args=[ - 'sudo', - 'tar', - 'c', - '-f', '-', - '-C', remotedir, - '--', - '.', - ], - stdout=run.PIPE, - wait=False, - ) - tar = tarfile.open(mode='r|', fileobj=proc.stdout) - while True: - ti = tar.next() - if ti is None: - break + result = remote.tar_remote(remotedir, sudo=True) + _, local_tarfile = tempfile.mkstemp() + with open(local_tarfile, 'r+') as fb1: + fb1.write(result) + fb1.seek(0) + tar = tarfile.open(mode='r|', fileobj=fb1) + while True: + ti = tar.next() + if ti is None: + break - if ti.isdir(): - # ignore silently; easier to just create leading dirs below - pass - elif ti.isfile(): - sub = safepath.munge(ti.name) - safepath.makedirs(root=localdir, path=os.path.dirname(sub)) - tar.makefile(ti, targetpath=os.path.join(localdir, sub)) - else: - if ti.isdev(): - type_ = 'device' - elif ti.issym(): - type_ = 'symlink' - elif ti.islnk(): - type_ = 'hard link' + if ti.isdir(): + # ignore silently; easier to just create leading dirs below + pass + elif ti.isfile(): + sub = safepath.munge(ti.name) + safepath.makedirs(root=localdir, path=os.path.dirname(sub)) + tar.makefile(ti, targetpath=os.path.join(localdir, sub)) else: - type_ = 'unknown' - log.info('Ignoring tar entry: %r type %r', ti.name, type_) - continue - proc.exitstatus.get() + if ti.isdev(): + type_ = 'device' + elif ti.issym(): + type_ = 'symlink' + elif ti.islnk(): + type_ = 'hard link' + else: + type_ = 'unknown' + log.info('Ignoring tar entry: %r type %r', ti.name, type_) + continue + os.remove(local_tarfile) def pull_directory_tarball(remote, remotedir, localfile): @@ -704,21 +688,9 @@ def pull_directory_tarball(remote, remotedir, localfile): """ log.debug('Transferring archived files from %s:%s to %s', remote.shortname, remotedir, localfile) - out = open(localfile, 'w') - proc = remote.run( - args=[ - 'sudo', - 'tar', - 'cz', - '-f', '-', - '-C', remotedir, - '--', - '.', - ], - stdout=out, - wait=False, - ) - proc.exitstatus.get() + tardata = remote.tar_remote(remotedir, zip_flag=True, sudo=True) + with open(localfile, 'w') as out: + out.write(tardata) def get_wwn_id_map(remote, devs): diff --git a/teuthology/orchestra/remote.py b/teuthology/orchestra/remote.py index 79dbc2d413b03..905fc6f88ab4d 100644 --- a/teuthology/orchestra/remote.py +++ b/teuthology/orchestra/remote.py @@ -8,7 +8,9 @@ import time import pexpect import re import logging +from cStringIO import StringIO from teuthology import lockstatus as ls +import paramiko try: import libvirt @@ -107,6 +109,82 @@ class Remote(object): r.remote = self return r + def remote_mktemp(self, sudo=False): + """ + Make a remote temporary file + """ + args = [] + if sudo: + args.append('sudo') + args.extend([ + 'python', + '-c', + 'import os; import tempfile; import sys; (fd,fname) = tempfile.mkstemp(); os.close(fd); sys.stdout.write(fname.rstrip()); sys.stdout.flush()' + ]) + proc = self.run( + args=args, + stdout=StringIO(), + ) + data = proc.stdout.getvalue() + return data + + def _set_remote_perms(self, tempf, perms): + args = [] + args.extend([ + 'sudo', + 'chmod', + perms, + tempf, + ]) + self.run( + args=args, + stdout=StringIO(), + ) + + def _do_sftp_cmd(self, args, tempf, sudo=False): + self.run( + args=args, + stdout=StringIO(), + ) + if sudo: + self._set_remote_perms(tempf, '0666') + conn = self.connect() + transport = conn.get_transport() + sftp = paramiko.SFTPClient.from_transport(transport) + with sftp.open(tempf, 'rb') as file_sftp: + result = file_sftp.read() + return result + + def copy_remote(self, path, sudo=False): + """ + Read a file from the remote host into memory. + """ + tempf = self.remote_mktemp() + args = [ + 'sudo', + 'cp', + path, + tempf, + ] + return self._do_sftp_cmd(args, tempf, sudo) + + def tar_remote(self, path, sudo=False, zip_flag=False): + """ + Tar a remote file. + """ + zip_fld = lambda x: 'cz' if x else 'c' + tempf = self.remote_mktemp() + args = [ + 'sudo', + 'tar', + zip_fld(zip_flag), + '-f', tempf, + '-C', path, + '--', + '.', + ] + return self._do_sftp_cmd(args, tempf, sudo) + def getShortName(name): """ diff --git a/teuthology/task/kernel.py b/teuthology/task/kernel.py index 7d63190dcc04f..d54e79ed9da15 100644 --- a/teuthology/task/kernel.py +++ b/teuthology/task/kernel.py @@ -649,7 +649,7 @@ def update_grub_rpm(remote, newversion): newgrub = generate_legacy_grub_entry(remote, newversion) for line in newgrub: data += line + '\n' - temp_file_path = teuthology.remote_mktemp(remote) + temp_file_path = remote.remote_mktemp() teuthology.sudo_write_file(remote, temp_file_path, StringIO(data), '755') teuthology.move_file(remote, temp_file_path, '/boot/grub/grub.conf', True) else: -- 2.39.5