Handle raw data I/O.

Paramiko 1.13.0 checks data in the Channel and fails if
invalid UTF-8 characters are sent.  The teuthology/misc.py
functions that piped cat output (get_file) and piped tar
output (pull_directory and pull_directory_tarball) formerly
did not work for Paramiko 1.13.0.

Code was changed to use SFTPClient to copy raw data.  The
remote_mktemp and function was changed to be a method of the
remote object.  Remote object methods to copy files and write
tar files were also added.

In misc.py, get_file() was changed to be a wrapper around the
remote object call.  Pull_directory() and pull_directory_tarball
were changed to use the remote object methods as well.

The remote methods used to send the data makes use of the
SFTPClient interface in Paramiko.

The remote_mktemp function was also modified slightly to insure
that new-lines do not appear in temp file names.

Fixes: 8116
Signed-off-by: Warren Usui <warren.usui@inktank.com>

diff --git a/teuthology/misc.py b/teuthology/misc.py
index c075e56d6dd51b858fbd84d3a5689496383d6a33..27e9b8b2e20c02de40381ae8b6f2051c7bcaafeb 100644 (file)
--- a/teuthology/misc.py
+++ b/teuthology/misc.py
@@ -18,6 +18,8 @@ import urlparse
 import yaml
 import json
 import re
+import tempfile
+import paramiko
 
 from teuthology import safepath
 from .orchestra import run
@@ -554,7 +556,7 @@ def remove_lines_from_file(remote, path, line_is_valid_test,
     # get a temp file path on the remote host to write to,
     # we don't want to blow away the remote file and then have the
     # network drop out
-    temp_file_path = remote_mktemp(remote)
+    temp_file_path = remote.remote_mktemp()
 
     # write out the data to a temp file
     write_file(remote, temp_file_path, out_data)
@@ -570,7 +572,7 @@ def append_lines_to_file(remote, path, lines, sudo=False):
     Remove_lines_from_list.
     """
 
-    temp_file_path = remote_mktemp(remote)
+    temp_file_path = remote.remote_mktemp()
 
     data = get_file(remote, path, sudo)
 
@@ -585,26 +587,6 @@ def append_lines_to_file(remote, path, lines, sudo=False):
     move_file(remote, temp_file_path, path)
 
 
-def remote_mktemp(remote, sudo=False):
-    """
-    Make a temporary file on a remote system
-    """
-    args = []
-    if sudo:
-        args.append('sudo')
-    args.extend([
-        'python',
-        '-c',
-        'import os; import tempfile; (fd,fname) = tempfile.mkstemp(); os.close(fd); print fname.rstrip()'
-        ])
-    proc = remote.run(
-        args=args,
-        stdout=StringIO(),
-        )
-    data = proc.stdout.getvalue()
-    return data
-
-
 def create_file(remote, path, data="", permissions=str(644), sudo=False):
     """
     Create a file on the remote host.
@@ -630,25 +612,35 @@ def create_file(remote, path, data="", permissions=str(644), sudo=False):
         append_lines_to_file(remote, path, data, sudo)
 
 
-def get_file(remote, path, sudo=False):
+def do_remote_sftp(remote, tempf, sudo=False):
     """
-    Read a file from remote host into memory.
+    Make sure file is aways readble if root, and use SFTPClient to
+    copy across data.
     """
-    args = []
     if sudo:
-        args.append('sudo')
-    args.extend([
-        'cat',
-        '--',
-        path,
-        ])
-    proc = remote.run(
-        args=args,
-        stdout=StringIO(),
-        )
-    data = proc.stdout.getvalue()
-    return data
+        args = []
+        args.extend([
+            'sudo',
+            'chmod',
+            '0666',
+            tempf,
+            ])
+        remote.run(
+            args=args,
+            stdout=StringIO(),
+            )
+    conn = remote.connect()
+    transport = conn.get_transport()
+    sftp = paramiko.SFTPClient.from_transport(transport)
+    with sftp.open(tempf, 'rb') as file_sftp:
+        result = file_sftp.read()
+    return result
 
+def get_file(remote, path, sudo=False):
+    """
+    Copy_remote wrapper.
+    """
+    return remote.copy_remote(path, sudo)
 
 def pull_directory(remote, remotedir, localdir):
     """
@@ -658,44 +650,36 @@ def pull_directory(remote, remotedir, localdir):
               remote.shortname, remotedir, localdir)
     if not os.path.exists(localdir):
         os.mkdir(localdir)
-    proc = remote.run(
-        args=[
-            'sudo',
-            'tar',
-            'c',
-            '-f', '-',
-            '-C', remotedir,
-            '--',
-            '.',
-            ],
-        stdout=run.PIPE,
-        wait=False,
-        )
-    tar = tarfile.open(mode='r|', fileobj=proc.stdout)
-    while True:
-        ti = tar.next()
-        if ti is None:
-            break
+    result = remote.tar_remote(remotedir, sudo=True)
+    _, local_tarfile = tempfile.mkstemp()
+    with open(local_tarfile, 'r+') as fb1:
+        fb1.write(result)
+        fb1.seek(0)
+        tar = tarfile.open(mode='r|', fileobj=fb1)
+        while True:
+            ti = tar.next()
+            if ti is None:
+                break
 
-        if ti.isdir():
-            # ignore silently; easier to just create leading dirs below
-            pass
-        elif ti.isfile():
-            sub = safepath.munge(ti.name)
-            safepath.makedirs(root=localdir, path=os.path.dirname(sub))
-            tar.makefile(ti, targetpath=os.path.join(localdir, sub))
-        else:
-            if ti.isdev():
-                type_ = 'device'
-            elif ti.issym():
-                type_ = 'symlink'
-            elif ti.islnk():
-                type_ = 'hard link'
+            if ti.isdir():
+                # ignore silently; easier to just create leading dirs below
+                pass
+            elif ti.isfile():
+                sub = safepath.munge(ti.name)
+                safepath.makedirs(root=localdir, path=os.path.dirname(sub))
+                tar.makefile(ti, targetpath=os.path.join(localdir, sub))
             else:
-                type_ = 'unknown'
-                log.info('Ignoring tar entry: %r type %r', ti.name, type_)
-                continue
-    proc.exitstatus.get()
+                if ti.isdev():
+                    type_ = 'device'
+                elif ti.issym():
+                    type_ = 'symlink'
+                elif ti.islnk():
+                    type_ = 'hard link'
+                else:
+                    type_ = 'unknown'
+                    log.info('Ignoring tar entry: %r type %r', ti.name, type_)
+                    continue
+    os.remove(local_tarfile)
 
 
 def pull_directory_tarball(remote, remotedir, localfile):
@@ -704,21 +688,9 @@ def pull_directory_tarball(remote, remotedir, localfile):
     """
     log.debug('Transferring archived files from %s:%s to %s',
               remote.shortname, remotedir, localfile)
-    out = open(localfile, 'w')
-    proc = remote.run(
-        args=[
-            'sudo',
-            'tar',
-            'cz',
-            '-f', '-',
-            '-C', remotedir,
-            '--',
-            '.',
-            ],
-        stdout=out,
-        wait=False,
-        )
-    proc.exitstatus.get()
+    tardata = remote.tar_remote(remotedir, zip_flag=True, sudo=True)
+    with open(localfile, 'w') as out:
+        out.write(tardata)
 
 
 def get_wwn_id_map(remote, devs):

diff --git a/teuthology/orchestra/remote.py b/teuthology/orchestra/remote.py
index 79dbc2d413b03384965a66f27a7c2b328bd6f183..905fc6f88ab4de29c0bb931fa25000ecfa2a70c2 100644 (file)
--- a/teuthology/orchestra/remote.py
+++ b/teuthology/orchestra/remote.py
@@ -8,7 +8,9 @@ import time
 import pexpect
 import re
 import logging
+from cStringIO import StringIO
 from teuthology import lockstatus as ls
+import paramiko
 
 try:
     import libvirt
@@ -107,6 +109,82 @@ class Remote(object):
         r.remote = self
         return r
 
+    def remote_mktemp(self, sudo=False):
+        """
+        Make a remote temporary file 
+        """
+        args = []
+        if sudo:
+            args.append('sudo')
+        args.extend([
+            'python',
+            '-c',
+            'import os; import tempfile; import sys; (fd,fname) = tempfile.mkstemp(); os.close(fd); sys.stdout.write(fname.rstrip()); sys.stdout.flush()'
+            ])
+        proc = self.run(
+            args=args,
+            stdout=StringIO(),
+            )
+        data = proc.stdout.getvalue()
+        return data
+
+    def _set_remote_perms(self, tempf, perms):
+        args = []
+        args.extend([
+            'sudo',
+            'chmod',
+            perms,
+            tempf,
+            ])
+        self.run(
+            args=args,
+            stdout=StringIO(),
+            )
+
+    def _do_sftp_cmd(self, args, tempf, sudo=False):
+        self.run(
+            args=args,
+            stdout=StringIO(),
+            )
+        if sudo:
+            self._set_remote_perms(tempf, '0666')
+        conn = self.connect()
+        transport = conn.get_transport()
+        sftp = paramiko.SFTPClient.from_transport(transport)
+        with sftp.open(tempf, 'rb') as file_sftp:
+            result = file_sftp.read()
+        return result
+
+    def copy_remote(self, path, sudo=False):
+        """
+        Read a file from the remote host into memory.
+        """
+        tempf = self.remote_mktemp()
+        args = [
+            'sudo',
+            'cp',
+            path,
+            tempf,
+            ]
+        return self._do_sftp_cmd(args, tempf, sudo)
+
+    def tar_remote(self, path, sudo=False, zip_flag=False):
+        """
+        Tar a remote file.
+        """
+        zip_fld = lambda x: 'cz' if x else 'c'
+        tempf = self.remote_mktemp()
+        args = [
+            'sudo',
+            'tar',
+            zip_fld(zip_flag),
+            '-f', tempf,
+            '-C', path,
+            '--',
+            '.',
+            ]
+        return self._do_sftp_cmd(args, tempf, sudo)
+                
 
 def getShortName(name):
     """

diff --git a/teuthology/task/kernel.py b/teuthology/task/kernel.py
index 7d63190dcc04f291fba2a22e57d2395fe4f817e8..d54e79ed9da153f4a2e9a87dbde82a3ac4fcd5c4 100644 (file)
--- a/teuthology/task/kernel.py
+++ b/teuthology/task/kernel.py
@@ -649,7 +649,7 @@ def update_grub_rpm(remote, newversion):
         newgrub = generate_legacy_grub_entry(remote, newversion)
         for line in newgrub:
             data += line + '\n'
-        temp_file_path = teuthology.remote_mktemp(remote)
+        temp_file_path = remote.remote_mktemp()
         teuthology.sudo_write_file(remote, temp_file_path, StringIO(data), '755')
         teuthology.move_file(remote, temp_file_path, '/boot/grub/grub.conf', True)
     else: