import subprocess
from pytest import raises
+from six import ensure_str
class Script(object):
def test_help(self):
args = (self.script_name, '--help')
- out = subprocess.check_output(args)
+ out = ensure_str(subprocess.check_output(args))
assert out.startswith('usage')
def test_invalid(self):
import logging
import getpass
+from six import ensure_str
+
from teuthology import beanstalk
from teuthology import report
from teuthology.config import config
for target in targets:
to_nuke.append(misc.decanonicalize_hostname(target))
- target_file = tempfile.NamedTemporaryFile(delete=False)
+ target_file = tempfile.NamedTemporaryFile(delete=False, mode='w+t')
target_file.write(yaml.safe_dump(targets_dict))
target_file.close()
nuke_args,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
- for line in iter(proc.stdout.readline, ''):
- line = line.replace('\r', '').replace('\n', '')
- log.info(line)
+ for line in iter(proc.stdout.readline, b''):
+ line = line.replace(b'\r', b'').replace(b'\n', b'')
+ log.info(ensure_str(line))
sys.stdout.flush()
os.unlink(target_file.name)
from teuthology.contextutil import safe_while
from teuthology.orchestra.opsys import DEFAULT_OS_VERSION
-from six import reraise
+from six import (reraise, ensure_str)
log = logging.getLogger(__name__)
on when the main site goes up and down.
"""
# read in the specified file
- in_data = get_file(remote, path, False)
+ in_data = ensure_str(get_file(remote, path, False))
out_data = ""
first_line = True
temp_file_path = remote.mktemp()
- data = get_file(remote, path, sudo)
+ data = ensure_str(get_file(remote, path, sudo))
# add the additional data and write it back out, using a temp file
# in case of connectivity of loss, and then mv it to the
temp_file_path = remote.mktemp()
- data = get_file(remote, path, sudo)
+ data = ensure_str(get_file(remote, path, sudo))
# add the additional data and write it back out, using a temp file
# in case of connectivity of loss, and then mv it to the
"""
devs = []
try:
- file_data = get_file(remote, "/scratch_devs")
+ file_data = ensure_str(get_file(remote, "/scratch_devs"))
devs = file_data.split()
except Exception:
devs = remote.sh('ls /dev/[sv]d?').strip().split('\n')
)
p.wait()
for line in p.stderr.readlines():
- line = line.strip()
+ line = ensure_str(line.strip())
if line and not line.startswith('#'):
log.error(line)
for line in p.stdout.readlines():
- host, key = line.strip().split(' ', 1)
+ host, key = ensure_str(line.strip()).split(' ', 1)
return key
truncated = False
with proc.stdout:
for line in iter(proc.stdout.readline, b''):
+ line = ensure_str(line)
lines.append(line)
line = line.strip()
if len(line) > log_limit:
truncated = True
- log.debug(str(line)[:log_limit] +
+ log.debug(line[:log_limit] +
"... (truncated to the first " + str(log_limit) +
" characters)")
else:
- log.debug(str(line))
+ log.debug(line)
output = "".join(lines)
if proc.wait() != 0:
if truncated:
cmd=command,
output=output
)
- return output.decode('utf-8')
+ return output
"""
Support for paramiko remote objects.
"""
+
+from six import ensure_str
+
import teuthology.lock.query
import teuthology.lock.util
from teuthology.orchestra import run
import time
import re
import logging
-from cStringIO import StringIO
+from io import BytesIO
import os
import pwd
import tempfile
remote_date = remote.sh('date')
"""
if 'stdout' not in kwargs:
- kwargs['stdout'] = StringIO()
+ kwargs['stdout'] = BytesIO()
if 'args' not in kwargs:
kwargs['args'] = script
proc=self.run(**kwargs)
- return proc.stdout.getvalue()
-
+ out=proc.stdout.getvalue()
+ return ensure_str(out)
def sh_file(self, script, label="script", sudo=False, **kwargs):
"""
"""
Paramiko run support
"""
-from cStringIO import StringIO
+
+import io
+
from paramiko import ChannelFile
import gevent
import logging
import shutil
+from teuthology.util.compat import PY3
+
from teuthology.contextutil import safe_while
from teuthology.exceptions import (CommandCrashedError, CommandFailedError,
ConnectionLostError)
# Work-around for http://tracker.ceph.com/issues/8313
if isinstance(f, ChannelFile):
f._flags += ChannelFile.FLAG_BINARY
-
for line in f:
if capture:
- capture.write(line)
+ #out = ensure_str(line)
+ if PY3:
+ if isinstance(line, str):
+ out = line.encode()
+ else:
+ out = line.decode()
+ if isinstance(capture, io.StringIO):
+ capture.write(out)
+ elif isinstance(capture, io.BytesIO):
+ capture.write(out.encode())
+ else:
+ if isinstance(line, str):
+ out = line.decode()
+ else:
+ out = line.encode()
+ if isinstance(capture, io.StringIO):
+ capture.write(out)
+ elif isinstance(capture, io.BytesIO):
+ capture.write(out.encode())
+ else:
+ # isinstance does not work with cStringIO.StringIO and
+ # fails with error:
+ # TypeError: isinstance() arg 2 must be a class, type,
+ # or tuple of classes and types
+ capture.write(out.encode())
line = line.rstrip()
# Second part of work-around for http://tracker.ceph.com/issues/8313
try:
- line = unicode(line, 'utf-8', 'replace').encode('utf-8')
- logger.log(loglevel, line.decode('utf-8'))
+ if isinstance(line, bytes):
+ line = line.decode('utf-8', 'replace')
+ logger.log(loglevel, line)
except (UnicodeDecodeError, UnicodeEncodeError):
logger.exception("Encountered unprintable line in command output")
copyfileobj call wrapper.
"""
if src is not None:
- if isinstance(src, basestring):
- src = StringIO(src)
+ if isinstance(src, bytes):
+ src = io.BytesIO(src)
+ elif isinstance(src, str):
+ if PY3:
+ src = io.StringIO(src)
+ else:
+ src = io.BytesIO(src)
shutil.copyfileobj(src, fdst)
fdst.close()
from mock import patch, Mock, MagicMock
-from cStringIO import StringIO
+from io import BytesIO
from teuthology.orchestra import remote
from teuthology.orchestra import opsys
'hostname',
'--fqdn',
]
- stdout = StringIO('test_hostname')
+ stdout = BytesIO(b'test_hostname')
stdout.seek(0)
proc = RemoteProcess(
client=self.m_ssh,
'uname',
'-m',
]
- stdout = StringIO('test_arch')
+ stdout = BytesIO(b'test_arch')
stdout.seek(0)
proc = RemoteProcess(
client=self.m_ssh,
assert m_run.called_once_with(
client=self.m_ssh,
args=args,
- stdout=StringIO(),
+ stdout=BytesIO(),
name=r.shortname,
)
assert r.arch == 'test_arch'
-from StringIO import StringIO
+from io import BytesIO
import paramiko
import socket
from mock import MagicMock, patch
from pytest import raises
+from six import ensure_str, ensure_binary
+
from teuthology.orchestra import run
from teuthology.exceptions import (CommandCrashedError, CommandFailedError,
ConnectionLostError)
-
def set_buffer_contents(buf, contents):
buf.seek(0)
- if isinstance(contents, basestring):
+ if isinstance(contents, bytes):
buf.write(contents)
elif isinstance(contents, (list, tuple)):
buf.writelines(contents)
+ elif isinstance(contents, str):
+ buf.write(ensure_binary(contents))
else:
raise TypeError(
- "% is is a %s; should be a string, list or tuple" % (
+ "%s is a %s; should be a byte string, list or tuple" % (
contents, type(contents)
)
)
self.m_stdout_buf = self.m_channelfile(self.m_channel())
self.m_stderr_buf = self.m_channelfile(self.m_channel())
"""
- class M_ChannelFile(StringIO):
+ class M_ChannelFile(BytesIO):
channel = MagicMock(spec=paramiko.Channel)()
self.m_channelfile = M_ChannelFile
output = 'foo\nbar'
set_buffer_contents(self.m_stdout_buf, output)
self.m_stdout_buf.channel.recv_exit_status.return_value = 0
- stdout = StringIO()
+ stdout = BytesIO()
proc = run.run(
client=self.m_ssh,
args=['foo', 'bar baz'],
stdout=stdout,
)
assert proc.stdout is stdout
- assert proc.stdout.read() == output
- assert proc.stdout.getvalue() == output
+ assert ensure_str(proc.stdout.read()) == output
+ assert ensure_str(proc.stdout.getvalue()) == output
def test_capture_stderr_newline(self):
output = 'foo\nbar\n'
set_buffer_contents(self.m_stderr_buf, output)
self.m_stderr_buf.channel.recv_exit_status.return_value = 0
- stderr = StringIO()
+ stderr = BytesIO()
proc = run.run(
client=self.m_ssh,
args=['foo', 'bar baz'],
stderr=stderr,
)
assert proc.stderr is stderr
- assert proc.stderr.read() == output
- assert proc.stderr.getvalue() == output
+ assert ensure_str(proc.stderr.read()) == output
+ assert ensure_str(proc.stderr.getvalue()) == output
def test_status_bad(self):
self.m_stdout_buf.channel.recv_exit_status.return_value = 42
def test_stdout_pipe(self):
self.m_stdout_buf.channel.recv_exit_status.return_value = 0
- lines = ['one\n', 'two', '']
+ lines = [b'one\n', b'two', b'']
set_buffer_contents(self.m_stdout_buf, lines)
proc = run.run(
client=self.m_ssh,
def test_stderr_pipe(self):
self.m_stdout_buf.channel.recv_exit_status.return_value = 0
- lines = ['one\n', 'two', '']
+ lines = [b'one\n', b'two', b'']
set_buffer_contents(self.m_stderr_buf, lines)
proc = run.run(
client=self.m_ssh,
from teuthology.util.compat import urljoin, urlencode
from collections import OrderedDict
-from cStringIO import StringIO
-
+from teuthology.util.compat import PY3
+if PY3:
+ from io import StringIO
+else:
+ from io import BytesIO as StringIO
from teuthology import repo_utils
from teuthology.config import config
import logging
from copy import deepcopy
+from six import string_types as basestring
from libcloud.compute.providers import get_driver
from libcloud.compute.types import Provider as lc_Provider
import teuthology.provision.cloud
from teuthology.misc import canonicalize_hostname, decanonicalize_hostname
-
log = logging.getLogger(__name__)
import json
import os
-from teuthology.util.flock import FileLock
+from six import ensure_str
+from teuthology.util.flock import FileLock
def get_user_ssh_pubkey(path='~/.ssh/id_rsa.pub'):
full_path = os.path.expanduser(path)
if not os.path.exists(full_path):
return
with open(full_path, 'rb') as f:
- return f.read().strip()
+ return ensure_str(f.read()).strip()
def combine_dicts(list_of_dicts, func):
import re
import shutil
import subprocess
+
+from six import ensure_str
+
import time
from teuthology import misc
:returns: The sha1 if found; else None
"""
+ sha1 = None
cmd = "git ls-remote {} {}".format(url, ref)
result = subprocess.check_output(
cmd, shell=True).split()
- sha1 = result[0] if result else None
+ if result:
+ sha1 = ensure_str(result[0])
log.debug("{} -> {}".format(cmd, sha1))
return sha1
stderr=subprocess.STDOUT)
not_found_str = "Remote branch %s not found" % branch
- out = proc.stdout.read()
+ out = ensure_str(proc.stdout.read())
result = proc.wait()
# Newer git versions will bail if the branch is not found, but older ones
# will not. Fortunately they both output similar text.
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
if proc.wait() != 0:
- out = proc.stdout.read()
+ out = ensure_str(proc.stdout.read())
log.error(out)
raise GitError("git fetch failed!")
stderr=subprocess.STDOUT)
if proc.wait() != 0:
not_found_str = "fatal: couldn't find remote ref %s" % branch
- out = proc.stdout.read()
+ out = ensure_str(proc.stdout.read())
log.error(out)
if not_found_str in out.lower():
raise BranchNotFoundError(branch)
input_dict = copy.deepcopy(input_dict)
def _substitute(input_dict, values_dict):
- for key, value in input_dict.items():
+ for key, value in list(input_dict.items()):
if isinstance(value, dict):
_substitute(value, values_dict)
elif isinstance(value, Placeholder):
base_frag_paths = [
util.strip_fragment_path(x) for x in fragment_paths
]
- limit = self.args.limit
+ limit = self.args.limit or 0
if limit > 0 and len(jobs_to_schedule) >= limit:
log.info(
'Stopped after {limit} jobs due to --limit={limit}'.format(
log.debug("Base job config:\n%s" % self.base_config)
with open(base_yaml_path, 'w+b') as base_yaml:
- base_yaml.write(str(self.base_config))
+ base_yaml.write(str(self.base_config).encode())
if jobs_to_schedule:
self.write_rerun_memo()
from datetime import datetime
from mock import patch, call, ANY, DEFAULT
-from StringIO import StringIO
+from teuthology.util.compat import PY3
+if PY3:
+ from io import StringIO
+ from io import BytesIO
+else:
+ from io import BytesIO as StringIO
+ from io import BytesIO
from teuthology.config import config, YamlConfig
from teuthology.exceptions import ScheduleFailError
@patch('teuthology.suite.util.has_packages_for_distro')
@patch('teuthology.suite.util.get_package_versions')
@patch('teuthology.suite.util.get_install_task_flavor')
- @patch('__builtin__.open')
+ @patch('teuthology.suite.run.open')
@patch('teuthology.suite.run.build_matrix')
@patch('teuthology.suite.util.git_ls_remote')
@patch('teuthology.suite.util.package_version_for_hash')
m_open.side_effect = [
StringIO(frag1_read_output),
StringIO(frag2_read_output),
- contextlib.closing(StringIO())
+ contextlib.closing(BytesIO())
]
m_get_install_task_flavor.return_value = 'basic'
m_get_package_versions.return_value = dict()
@patch('teuthology.suite.util.has_packages_for_distro')
@patch('teuthology.suite.util.get_package_versions')
@patch('teuthology.suite.util.get_install_task_flavor')
- @patch('__builtin__.open', create=True)
+ @patch('teuthology.suite.run.open', create=True)
@patch('teuthology.suite.run.build_matrix')
@patch('teuthology.suite.util.git_ls_remote')
@patch('teuthology.suite.util.package_version_for_hash')
@patch('teuthology.suite.util.has_packages_for_distro')
@patch('teuthology.suite.util.get_package_versions')
@patch('teuthology.suite.util.get_install_task_flavor')
- @patch('__builtin__.open', create=True)
+ @patch('teuthology.suite.run.open', create=True)
@patch('teuthology.suite.run.build_matrix')
@patch('teuthology.suite.util.git_ls_remote')
@patch('teuthology.suite.util.package_version_for_hash')
m_open.side_effect = [
StringIO('field: val\n') for i in range(NUM_FAILS+1)
] + [
- contextlib.closing(StringIO())
+ contextlib.closing(BytesIO())
]
m_get_install_task_flavor.return_value = 'basic'
m_get_package_versions.return_value = dict()
import logging
+from six import string_types as basestring
+
from teuthology.misc import deep_merge
from teuthology.orchestra.cluster import Cluster
import logging
import os
-from cStringIO import StringIO
+from io import StringIO
from teuthology.orchestra import run
import re
from teuthology import misc as teuthology
+from six import string_types as basestring
log = logging.getLogger(__name__)
from teuthology import misc
+from six import string_types as basestring
+
from teuthology.task import Task
log = logging.getLogger(__name__)
import logging
import os
-from cStringIO import StringIO
+from io import StringIO
from teuthology.exceptions import SELinuxError
from teuthology.misc import get_archive_dir
projects / teuthology.git / commitdiff