From 6242d9f649a66e8f0a9d8b3964e10d84fbfd6aad Mon Sep 17 00:00:00 2001 From: Kyr Shatskyy Date: Sun, 15 Mar 2020 00:33:17 +0100 Subject: [PATCH] py3: get rid of cStringIO Signed-off-by: Kyr Shatskyy --- scripts/test/script.py | 3 +- teuthology/kill.py | 10 +++-- teuthology/misc.py | 21 ++++++----- teuthology/orchestra/remote.py | 11 ++++-- teuthology/orchestra/run.py | 47 ++++++++++++++++++++---- teuthology/orchestra/test/test_remote.py | 8 ++-- teuthology/orchestra/test/test_run.py | 29 ++++++++------- teuthology/packaging.py | 7 +++- teuthology/provision/cloud/base.py | 2 +- teuthology/provision/cloud/util.py | 5 ++- teuthology/repo_utils.py | 13 +++++-- teuthology/suite/placeholder.py | 2 +- teuthology/suite/run.py | 4 +- teuthology/suite/test/test_run_.py | 18 ++++++--- teuthology/task/__init__.py | 2 + teuthology/task/install/deb.py | 2 +- teuthology/task/mpi.py | 1 + teuthology/task/pcp.py | 2 + teuthology/task/selinux.py | 2 +- 19 files changed, 126 insertions(+), 63 deletions(-) diff --git a/scripts/test/script.py b/scripts/test/script.py index 43f24548cb..db88f8f5ee 100644 --- a/scripts/test/script.py +++ b/scripts/test/script.py @@ -1,5 +1,6 @@ import subprocess from pytest import raises +from six import ensure_str class Script(object): @@ -7,7 +8,7 @@ class Script(object): def test_help(self): args = (self.script_name, '--help') - out = subprocess.check_output(args) + out = ensure_str(subprocess.check_output(args)) assert out.startswith('usage') def test_invalid(self): diff --git a/teuthology/kill.py b/teuthology/kill.py index c6fd61293a..f61e6a150b 100755 --- a/teuthology/kill.py +++ b/teuthology/kill.py @@ -8,6 +8,8 @@ import tempfile import logging import getpass +from six import ensure_str + from teuthology import beanstalk from teuthology import report from teuthology.config import config @@ -220,7 +222,7 @@ def nuke_targets(targets_dict, owner): for target in targets: to_nuke.append(misc.decanonicalize_hostname(target)) - target_file = tempfile.NamedTemporaryFile(delete=False) + target_file = tempfile.NamedTemporaryFile(delete=False, mode='w+t') target_file.write(yaml.safe_dump(targets_dict)) target_file.close() @@ -238,9 +240,9 @@ def nuke_targets(targets_dict, owner): nuke_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - for line in iter(proc.stdout.readline, ''): - line = line.replace('\r', '').replace('\n', '') - log.info(line) + for line in iter(proc.stdout.readline, b''): + line = line.replace(b'\r', b'').replace(b'\n', b'') + log.info(ensure_str(line)) sys.stdout.flush() os.unlink(target_file.name) diff --git a/teuthology/misc.py b/teuthology/misc.py index 46847bfe01..1ab68f009e 100644 --- a/teuthology/misc.py +++ b/teuthology/misc.py @@ -29,7 +29,7 @@ from teuthology.config import config from teuthology.contextutil import safe_while from teuthology.orchestra.opsys import DEFAULT_OS_VERSION -from six import reraise +from six import (reraise, ensure_str) log = logging.getLogger(__name__) @@ -647,7 +647,7 @@ def remove_lines_from_file(remote, path, line_is_valid_test, on when the main site goes up and down. """ # read in the specified file - in_data = get_file(remote, path, False) + in_data = ensure_str(get_file(remote, path, False)) out_data = "" first_line = True @@ -685,7 +685,7 @@ def append_lines_to_file(remote, path, lines, sudo=False): temp_file_path = remote.mktemp() - data = get_file(remote, path, sudo) + data = ensure_str(get_file(remote, path, sudo)) # add the additional data and write it back out, using a temp file # in case of connectivity of loss, and then mv it to the @@ -705,7 +705,7 @@ def prepend_lines_to_file(remote, path, lines, sudo=False): temp_file_path = remote.mktemp() - data = get_file(remote, path, sudo) + data = ensure_str(get_file(remote, path, sudo)) # add the additional data and write it back out, using a temp file # in case of connectivity of loss, and then mv it to the @@ -810,7 +810,7 @@ def get_scratch_devices(remote): """ devs = [] try: - file_data = get_file(remote, "/scratch_devs") + file_data = ensure_str(get_file(remote, "/scratch_devs")) devs = file_data.split() except Exception: devs = remote.sh('ls /dev/[sv]d?').strip().split('\n') @@ -1136,11 +1136,11 @@ def _ssh_keyscan(hostname): ) p.wait() for line in p.stderr.readlines(): - line = line.strip() + line = ensure_str(line.strip()) if line and not line.startswith('#'): log.error(line) for line in p.stdout.readlines(): - host, key = line.strip().split(' ', 1) + host, key = ensure_str(line.strip()).split(' ', 1) return key @@ -1310,15 +1310,16 @@ def sh(command, log_limit=1024, cwd=None, env=None): truncated = False with proc.stdout: for line in iter(proc.stdout.readline, b''): + line = ensure_str(line) lines.append(line) line = line.strip() if len(line) > log_limit: truncated = True - log.debug(str(line)[:log_limit] + + log.debug(line[:log_limit] + "... (truncated to the first " + str(log_limit) + " characters)") else: - log.debug(str(line)) + log.debug(line) output = "".join(lines) if proc.wait() != 0: if truncated: @@ -1331,4 +1332,4 @@ def sh(command, log_limit=1024, cwd=None, env=None): cmd=command, output=output ) - return output.decode('utf-8') + return output diff --git a/teuthology/orchestra/remote.py b/teuthology/orchestra/remote.py index 721d809ce0..f03447d9ad 100644 --- a/teuthology/orchestra/remote.py +++ b/teuthology/orchestra/remote.py @@ -1,6 +1,9 @@ """ Support for paramiko remote objects. """ + +from six import ensure_str + import teuthology.lock.query import teuthology.lock.util from teuthology.orchestra import run @@ -13,7 +16,7 @@ from teuthology.misc import host_shortname import time import re import logging -from cStringIO import StringIO +from io import BytesIO import os import pwd import tempfile @@ -236,12 +239,12 @@ class Remote(object): remote_date = remote.sh('date') """ if 'stdout' not in kwargs: - kwargs['stdout'] = StringIO() + kwargs['stdout'] = BytesIO() if 'args' not in kwargs: kwargs['args'] = script proc=self.run(**kwargs) - return proc.stdout.getvalue() - + out=proc.stdout.getvalue() + return ensure_str(out) def sh_file(self, script, label="script", sudo=False, **kwargs): """ diff --git a/teuthology/orchestra/run.py b/teuthology/orchestra/run.py index 8992ec31b7..4cb5f94203 100644 --- a/teuthology/orchestra/run.py +++ b/teuthology/orchestra/run.py @@ -1,7 +1,9 @@ """ Paramiko run support """ -from cStringIO import StringIO + +import io + from paramiko import ChannelFile import gevent @@ -11,6 +13,8 @@ import pipes import logging import shutil +from teuthology.util.compat import PY3 + from teuthology.contextutil import safe_while from teuthology.exceptions import (CommandCrashedError, CommandFailedError, ConnectionLostError) @@ -268,15 +272,39 @@ def copy_to_log(f, logger, loglevel=logging.INFO, capture=None): # Work-around for http://tracker.ceph.com/issues/8313 if isinstance(f, ChannelFile): f._flags += ChannelFile.FLAG_BINARY - for line in f: if capture: - capture.write(line) + #out = ensure_str(line) + if PY3: + if isinstance(line, str): + out = line.encode() + else: + out = line.decode() + if isinstance(capture, io.StringIO): + capture.write(out) + elif isinstance(capture, io.BytesIO): + capture.write(out.encode()) + else: + if isinstance(line, str): + out = line.decode() + else: + out = line.encode() + if isinstance(capture, io.StringIO): + capture.write(out) + elif isinstance(capture, io.BytesIO): + capture.write(out.encode()) + else: + # isinstance does not work with cStringIO.StringIO and + # fails with error: + # TypeError: isinstance() arg 2 must be a class, type, + # or tuple of classes and types + capture.write(out.encode()) line = line.rstrip() # Second part of work-around for http://tracker.ceph.com/issues/8313 try: - line = unicode(line, 'utf-8', 'replace').encode('utf-8') - logger.log(loglevel, line.decode('utf-8')) + if isinstance(line, bytes): + line = line.decode('utf-8', 'replace') + logger.log(loglevel, line) except (UnicodeDecodeError, UnicodeEncodeError): logger.exception("Encountered unprintable line in command output") @@ -286,8 +314,13 @@ def copy_and_close(src, fdst): copyfileobj call wrapper. """ if src is not None: - if isinstance(src, basestring): - src = StringIO(src) + if isinstance(src, bytes): + src = io.BytesIO(src) + elif isinstance(src, str): + if PY3: + src = io.StringIO(src) + else: + src = io.BytesIO(src) shutil.copyfileobj(src, fdst) fdst.close() diff --git a/teuthology/orchestra/test/test_remote.py b/teuthology/orchestra/test/test_remote.py index 6a77201634..9a94bfc43c 100644 --- a/teuthology/orchestra/test/test_remote.py +++ b/teuthology/orchestra/test/test_remote.py @@ -1,6 +1,6 @@ from mock import patch, Mock, MagicMock -from cStringIO import StringIO +from io import BytesIO from teuthology.orchestra import remote from teuthology.orchestra import opsys @@ -74,7 +74,7 @@ class TestRemote(object): 'hostname', '--fqdn', ] - stdout = StringIO('test_hostname') + stdout = BytesIO(b'test_hostname') stdout.seek(0) proc = RemoteProcess( client=self.m_ssh, @@ -97,7 +97,7 @@ class TestRemote(object): 'uname', '-m', ] - stdout = StringIO('test_arch') + stdout = BytesIO(b'test_arch') stdout.seek(0) proc = RemoteProcess( client=self.m_ssh, @@ -116,7 +116,7 @@ class TestRemote(object): assert m_run.called_once_with( client=self.m_ssh, args=args, - stdout=StringIO(), + stdout=BytesIO(), name=r.shortname, ) assert r.arch == 'test_arch' diff --git a/teuthology/orchestra/test/test_run.py b/teuthology/orchestra/test/test_run.py index acbf85514d..f87909cf21 100644 --- a/teuthology/orchestra/test/test_run.py +++ b/teuthology/orchestra/test/test_run.py @@ -1,4 +1,4 @@ -from StringIO import StringIO +from io import BytesIO import paramiko import socket @@ -6,20 +6,23 @@ import socket from mock import MagicMock, patch from pytest import raises +from six import ensure_str, ensure_binary + from teuthology.orchestra import run from teuthology.exceptions import (CommandCrashedError, CommandFailedError, ConnectionLostError) - def set_buffer_contents(buf, contents): buf.seek(0) - if isinstance(contents, basestring): + if isinstance(contents, bytes): buf.write(contents) elif isinstance(contents, (list, tuple)): buf.writelines(contents) + elif isinstance(contents, str): + buf.write(ensure_binary(contents)) else: raise TypeError( - "% is is a %s; should be a string, list or tuple" % ( + "%s is a %s; should be a byte string, list or tuple" % ( contents, type(contents) ) ) @@ -46,7 +49,7 @@ class TestRun(object): self.m_stdout_buf = self.m_channelfile(self.m_channel()) self.m_stderr_buf = self.m_channelfile(self.m_channel()) """ - class M_ChannelFile(StringIO): + class M_ChannelFile(BytesIO): channel = MagicMock(spec=paramiko.Channel)() self.m_channelfile = M_ChannelFile @@ -99,29 +102,29 @@ class TestRun(object): output = 'foo\nbar' set_buffer_contents(self.m_stdout_buf, output) self.m_stdout_buf.channel.recv_exit_status.return_value = 0 - stdout = StringIO() + stdout = BytesIO() proc = run.run( client=self.m_ssh, args=['foo', 'bar baz'], stdout=stdout, ) assert proc.stdout is stdout - assert proc.stdout.read() == output - assert proc.stdout.getvalue() == output + assert ensure_str(proc.stdout.read()) == output + assert ensure_str(proc.stdout.getvalue()) == output def test_capture_stderr_newline(self): output = 'foo\nbar\n' set_buffer_contents(self.m_stderr_buf, output) self.m_stderr_buf.channel.recv_exit_status.return_value = 0 - stderr = StringIO() + stderr = BytesIO() proc = run.run( client=self.m_ssh, args=['foo', 'bar baz'], stderr=stderr, ) assert proc.stderr is stderr - assert proc.stderr.read() == output - assert proc.stderr.getvalue() == output + assert ensure_str(proc.stderr.read()) == output + assert ensure_str(proc.stderr.getvalue()) == output def test_status_bad(self): self.m_stdout_buf.channel.recv_exit_status.return_value = 42 @@ -223,7 +226,7 @@ class TestRun(object): def test_stdout_pipe(self): self.m_stdout_buf.channel.recv_exit_status.return_value = 0 - lines = ['one\n', 'two', ''] + lines = [b'one\n', b'two', b''] set_buffer_contents(self.m_stdout_buf, lines) proc = run.run( client=self.m_ssh, @@ -241,7 +244,7 @@ class TestRun(object): def test_stderr_pipe(self): self.m_stdout_buf.channel.recv_exit_status.return_value = 0 - lines = ['one\n', 'two', ''] + lines = [b'one\n', b'two', b''] set_buffer_contents(self.m_stderr_buf, lines) proc = run.run( client=self.m_ssh, diff --git a/teuthology/packaging.py b/teuthology/packaging.py index 5ea9bc2958..22eddfb456 100644 --- a/teuthology/packaging.py +++ b/teuthology/packaging.py @@ -6,8 +6,11 @@ import requests from teuthology.util.compat import urljoin, urlencode from collections import OrderedDict -from cStringIO import StringIO - +from teuthology.util.compat import PY3 +if PY3: + from io import StringIO +else: + from io import BytesIO as StringIO from teuthology import repo_utils from teuthology.config import config diff --git a/teuthology/provision/cloud/base.py b/teuthology/provision/cloud/base.py index e06972e072..fd2ef972a2 100644 --- a/teuthology/provision/cloud/base.py +++ b/teuthology/provision/cloud/base.py @@ -1,5 +1,6 @@ import logging from copy import deepcopy +from six import string_types as basestring from libcloud.compute.providers import get_driver from libcloud.compute.types import Provider as lc_Provider @@ -8,7 +9,6 @@ import teuthology.orchestra.remote import teuthology.provision.cloud from teuthology.misc import canonicalize_hostname, decanonicalize_hostname - log = logging.getLogger(__name__) diff --git a/teuthology/provision/cloud/util.py b/teuthology/provision/cloud/util.py index b5345cb45b..c8170a87d4 100644 --- a/teuthology/provision/cloud/util.py +++ b/teuthology/provision/cloud/util.py @@ -4,15 +4,16 @@ import dateutil.parser import json import os -from teuthology.util.flock import FileLock +from six import ensure_str +from teuthology.util.flock import FileLock def get_user_ssh_pubkey(path='~/.ssh/id_rsa.pub'): full_path = os.path.expanduser(path) if not os.path.exists(full_path): return with open(full_path, 'rb') as f: - return f.read().strip() + return ensure_str(f.read()).strip() def combine_dicts(list_of_dicts, func): diff --git a/teuthology/repo_utils.py b/teuthology/repo_utils.py index 20910fc27a..a9c391fee8 100644 --- a/teuthology/repo_utils.py +++ b/teuthology/repo_utils.py @@ -3,6 +3,9 @@ import os import re import shutil import subprocess + +from six import ensure_str + import time from teuthology import misc @@ -60,10 +63,12 @@ def ls_remote(url, ref): :returns: The sha1 if found; else None """ + sha1 = None cmd = "git ls-remote {} {}".format(url, ref) result = subprocess.check_output( cmd, shell=True).split() - sha1 = result[0] if result else None + if result: + sha1 = ensure_str(result[0]) log.debug("{} -> {}".format(cmd, sha1)) return sha1 @@ -127,7 +132,7 @@ def clone_repo(repo_url, dest_path, branch, shallow=True): stderr=subprocess.STDOUT) not_found_str = "Remote branch %s not found" % branch - out = proc.stdout.read() + out = ensure_str(proc.stdout.read()) result = proc.wait() # Newer git versions will bail if the branch is not found, but older ones # will not. Fortunately they both output similar text. @@ -224,7 +229,7 @@ def fetch(repo_path): stdout=subprocess.PIPE, stderr=subprocess.STDOUT) if proc.wait() != 0: - out = proc.stdout.read() + out = ensure_str(proc.stdout.read()) log.error(out) raise GitError("git fetch failed!") @@ -252,7 +257,7 @@ def fetch_branch(repo_path, branch, shallow=True): stderr=subprocess.STDOUT) if proc.wait() != 0: not_found_str = "fatal: couldn't find remote ref %s" % branch - out = proc.stdout.read() + out = ensure_str(proc.stdout.read()) log.error(out) if not_found_str in out.lower(): raise BranchNotFoundError(branch) diff --git a/teuthology/suite/placeholder.py b/teuthology/suite/placeholder.py index 9af2ffb78e..bdf9cf929b 100644 --- a/teuthology/suite/placeholder.py +++ b/teuthology/suite/placeholder.py @@ -27,7 +27,7 @@ def substitute_placeholders(input_dict, values_dict): input_dict = copy.deepcopy(input_dict) def _substitute(input_dict, values_dict): - for key, value in input_dict.items(): + for key, value in list(input_dict.items()): if isinstance(value, dict): _substitute(value, values_dict) elif isinstance(value, Placeholder): diff --git a/teuthology/suite/run.py b/teuthology/suite/run.py index 4aa3cf2ada..6d216dfa18 100644 --- a/teuthology/suite/run.py +++ b/teuthology/suite/run.py @@ -361,7 +361,7 @@ class Run(object): base_frag_paths = [ util.strip_fragment_path(x) for x in fragment_paths ] - limit = self.args.limit + limit = self.args.limit or 0 if limit > 0 and len(jobs_to_schedule) >= limit: log.info( 'Stopped after {limit} jobs due to --limit={limit}'.format( @@ -548,7 +548,7 @@ class Run(object): log.debug("Base job config:\n%s" % self.base_config) with open(base_yaml_path, 'w+b') as base_yaml: - base_yaml.write(str(self.base_config)) + base_yaml.write(str(self.base_config).encode()) if jobs_to_schedule: self.write_rerun_memo() diff --git a/teuthology/suite/test/test_run_.py b/teuthology/suite/test/test_run_.py index b89a710d9d..503b25b373 100644 --- a/teuthology/suite/test/test_run_.py +++ b/teuthology/suite/test/test_run_.py @@ -6,7 +6,13 @@ import contextlib from datetime import datetime from mock import patch, call, ANY, DEFAULT -from StringIO import StringIO +from teuthology.util.compat import PY3 +if PY3: + from io import StringIO + from io import BytesIO +else: + from io import BytesIO as StringIO + from io import BytesIO from teuthology.config import config, YamlConfig from teuthology.exceptions import ScheduleFailError @@ -201,7 +207,7 @@ class TestScheduleSuite(object): @patch('teuthology.suite.util.has_packages_for_distro') @patch('teuthology.suite.util.get_package_versions') @patch('teuthology.suite.util.get_install_task_flavor') - @patch('__builtin__.open') + @patch('teuthology.suite.run.open') @patch('teuthology.suite.run.build_matrix') @patch('teuthology.suite.util.git_ls_remote') @patch('teuthology.suite.util.package_version_for_hash') @@ -236,7 +242,7 @@ class TestScheduleSuite(object): m_open.side_effect = [ StringIO(frag1_read_output), StringIO(frag2_read_output), - contextlib.closing(StringIO()) + contextlib.closing(BytesIO()) ] m_get_install_task_flavor.return_value = 'basic' m_get_package_versions.return_value = dict() @@ -277,7 +283,7 @@ class TestScheduleSuite(object): @patch('teuthology.suite.util.has_packages_for_distro') @patch('teuthology.suite.util.get_package_versions') @patch('teuthology.suite.util.get_install_task_flavor') - @patch('__builtin__.open', create=True) + @patch('teuthology.suite.run.open', create=True) @patch('teuthology.suite.run.build_matrix') @patch('teuthology.suite.util.git_ls_remote') @patch('teuthology.suite.util.package_version_for_hash') @@ -332,7 +338,7 @@ class TestScheduleSuite(object): @patch('teuthology.suite.util.has_packages_for_distro') @patch('teuthology.suite.util.get_package_versions') @patch('teuthology.suite.util.get_install_task_flavor') - @patch('__builtin__.open', create=True) + @patch('teuthology.suite.run.open', create=True) @patch('teuthology.suite.run.build_matrix') @patch('teuthology.suite.util.git_ls_remote') @patch('teuthology.suite.util.package_version_for_hash') @@ -369,7 +375,7 @@ class TestScheduleSuite(object): m_open.side_effect = [ StringIO('field: val\n') for i in range(NUM_FAILS+1) ] + [ - contextlib.closing(StringIO()) + contextlib.closing(BytesIO()) ] m_get_install_task_flavor.return_value = 'basic' m_get_package_versions.return_value = dict() diff --git a/teuthology/task/__init__.py b/teuthology/task/__init__.py index d21ff509e8..7bbf6c5e8b 100644 --- a/teuthology/task/__init__.py +++ b/teuthology/task/__init__.py @@ -1,5 +1,7 @@ import logging +from six import string_types as basestring + from teuthology.misc import deep_merge from teuthology.orchestra.cluster import Cluster diff --git a/teuthology/task/install/deb.py b/teuthology/task/install/deb.py index be4f344da4..0a31b0d11e 100644 --- a/teuthology/task/install/deb.py +++ b/teuthology/task/install/deb.py @@ -1,7 +1,7 @@ import logging import os -from cStringIO import StringIO +from io import StringIO from teuthology.orchestra import run diff --git a/teuthology/task/mpi.py b/teuthology/task/mpi.py index 4a752b1213..b6348c02ed 100644 --- a/teuthology/task/mpi.py +++ b/teuthology/task/mpi.py @@ -6,6 +6,7 @@ import logging import re from teuthology import misc as teuthology +from six import string_types as basestring log = logging.getLogger(__name__) diff --git a/teuthology/task/pcp.py b/teuthology/task/pcp.py index 53dcd51f2a..60aceab8ae 100644 --- a/teuthology/task/pcp.py +++ b/teuthology/task/pcp.py @@ -14,6 +14,8 @@ from teuthology.orchestra import run from teuthology import misc +from six import string_types as basestring + from teuthology.task import Task log = logging.getLogger(__name__) diff --git a/teuthology/task/selinux.py b/teuthology/task/selinux.py index bf699f63df..27009073a9 100644 --- a/teuthology/task/selinux.py +++ b/teuthology/task/selinux.py @@ -1,7 +1,7 @@ import logging import os -from cStringIO import StringIO +from io import StringIO from teuthology.exceptions import SELinuxError from teuthology.misc import get_archive_dir -- 2.39.5