stamp = datetime.datetime.now().strftime("%y%m%d%H%M")
is_vm = lambda x: x.startswith('vpm') or x.startswith('ubuntu@vpm')
-is_arm = lambda x: x.startswith('tala') or x.startswith('ubuntu@tala') or x.startswith('saya') or x.startswith('ubuntu@saya')
+is_arm = lambda x: x.startswith('tala') or x.startswith(
+ 'ubuntu@tala') or x.startswith('saya') or x.startswith('ubuntu@saya')
def config_file(string):
class MergeConfig(argparse.Action):
+
def __call__(self, parser, namespace, values, option_string=None):
config_dict = getattr(namespace, self.dest)
for new in values:
# FIXME this ideally should use os.path.expanduser() in the future, in case
# $HOME isn't /home/$USER - e.g. on a Mac. However, since we're executing
# this on the server side, it won't work properly.
- return ctx.teuthology_config.get('test_path', '/home/%s/cephtest' % test_user)
+ return ctx.teuthology_config.get('test_path', '/home/%s/cephtest' %
+ test_user)
def get_test_user(ctx):
bindir_url = urlparse.urljoin(BASE, 'sha1/{sha1}/'.format(sha1=sha1))
return (sha1, bindir_url)
+
def feed_many_stdins(fp, processes):
while True:
data = fp.read(8192)
for proc in processes:
proc.stdin.write(data)
+
def feed_many_stdins_and_close(fp, processes):
feed_many_stdins(fp, processes)
for proc in processes:
proc.stdin.close()
+
def get_mons(roles, ips):
mons = {}
mon_ports = {}
assert mons
return mons
+
def generate_caps(type_):
defaults = dict(
osd=dict(
yield subsystem
yield capability
+
def skeleton_config(ctx, roles, ips):
"""
Returns a ConfigObj that's prefilled with a skeleton config.
conf[role]['mds standby for name'] = standby_mds
return conf
+
def roles_of_type(roles_for_host, type_):
prefix = '{type}.'.format(type=type_)
for name in roles_for_host:
id_ = name[len(prefix):]
yield id_
+
def all_roles(cluster):
for _, roles_for_host in cluster.remotes.iteritems():
for name in roles_for_host:
yield name
+
def all_roles_of_type(cluster, type_):
prefix = '{type}.'.format(type=type_)
for _, roles_for_host in cluster.remotes.iteritems():
id_ = name[len(prefix):]
yield id_
+
def is_type(type_):
"""
Returns a matcher function for whether role is of type given.
"""
prefix = '{type}.'.format(type=type_)
+
def _is_type(role):
return role.startswith(prefix)
return _is_type
+
def num_instances_of_type(cluster, type_):
remotes_and_roles = cluster.remotes.items()
roles = [roles for (remote, roles) in remotes_and_roles]
prefix = '{type}.'.format(type=type_)
- num = sum(sum(1 for role in hostroles if role.startswith(prefix)) for hostroles in roles)
+ num = sum(sum(1 for role in hostroles if role.startswith(prefix))
+ for hostroles in roles)
return num
+
def create_simple_monmap(ctx, remote, conf):
"""
Writes a simple monmap based on current ceph.conf into <tmpdir>/monmap.
for (name, addr) in addresses:
args.extend(('--add', name, addr))
args.extend([
- '--print',
- '{tdir}/monmap'.format(tdir=testdir),
- ])
+ '--print',
+ '{tdir}/monmap'.format(tdir=testdir),
+ ])
r = remote.run(
args=args,
stdout=StringIO()
)
monmap_output = r.stdout.getvalue()
- fsid = re.search("generated fsid (.+)$", monmap_output, re.MULTILINE).group(1)
+ fsid = re.search("generated fsid (.+)$",
+ monmap_output, re.MULTILINE).group(1)
return fsid
+
def write_file(remote, path, data):
remote.run(
args=[
stdin=data,
)
+
def sudo_write_file(remote, path, data, perms=None):
permargs = []
if perms:
- permargs=[run.Raw('&&'), 'sudo', 'chmod', perms, path]
+ permargs = [run.Raw('&&'), 'sudo', 'chmod', perms, path]
remote.run(
args=[
'sudo',
stdin=data,
)
+
def move_file(remote, from_path, to_path, sudo=False):
# need to stat the file first, to make sure we
if sudo:
args.append('sudo')
args.extend([
- 'stat',
- '-c',
- '\"%a\"',
- to_path
- ])
+ 'stat',
+ '-c',
+ '\"%a\"',
+ to_path
+ ])
proc = remote.run(
args=args,
stdout=StringIO(),
if sudo:
args.append('sudo')
args.extend([
- 'mv',
- '--',
- from_path,
- to_path,
- ])
+ 'mv',
+ '--',
+ from_path,
+ to_path,
+ ])
proc = remote.run(
args=args,
stdout=StringIO(),
if sudo:
args.append('sudo')
args.extend([
- 'chmod',
- perms,
- to_path,
- ])
+ 'chmod',
+ perms,
+ to_path,
+ ])
proc = remote.run(
args=args,
stdout=StringIO(),
)
-def remove_lines_from_file(remote, path, line_is_valid_test, string_to_test_for):
+def remove_lines_from_file(remote, path, line_is_valid_test,
+ string_to_test_for):
# read in the specified file
in_data = get_file(remote, path, False)
out_data = ""
# then do a 'mv' to the actual file location
move_file(remote, temp_file_path, path)
+
def append_lines_to_file(remote, path, lines, sudo=False):
temp_file_path = remote_mktemp(remote)
# then do a 'mv' to the actual file location
move_file(remote, temp_file_path, path)
+
def remote_mktemp(remote, sudo=False):
args = []
if sudo:
args.append('sudo')
args.extend([
- 'python',
- '-c',
- 'import os; import tempfile; (fd,fname) = tempfile.mkstemp(); os.close(fd); print fname.rstrip()'
- ])
+ 'python',
+ '-c',
+ 'import os; import tempfile; (fd,fname) = tempfile.mkstemp(); os.close(fd); print fname.rstrip()'
+ ])
proc = remote.run(
args=args,
stdout=StringIO(),
if sudo:
args.append('sudo')
args.extend([
- 'cat',
- '--',
- path,
- ])
+ 'cat',
+ '--',
+ path,
+ ])
proc = remote.run(
args=args,
stdout=StringIO(),
data = proc.stdout.getvalue()
return data
+
def pull_directory(remote, remotedir, localdir):
"""
Copy a remote directory to a local directory.
continue
proc.exitstatus.get()
+
def pull_directory_tarball(remote, remotedir, localfile):
"""
Copy a remote directory to a local tarball.
# returns map of devices to device id links:
# /dev/sdb: /dev/disk/by-id/wwn-0xf00bad
+
+
def get_wwn_id_map(remote, devs):
stdout = None
try:
stdout = r.stdout.getvalue()
except Exception:
log.error('Failed to get wwn devices! Using /dev/sd* devices...')
- return dict((d,d) for d in devs)
+ return dict((d, d) for d in devs)
devmap = {}
# lines will be:
- # lrwxrwxrwx 1 root root 9 Jan 22 14:58 /dev/disk/by-id/wwn-0x50014ee002ddecaf -> ../../sdb
+ # lrwxrwxrwx 1 root root 9 Jan 22 14:58
+ # /dev/disk/by-id/wwn-0x50014ee002ddecaf -> ../../sdb
for line in stdout.splitlines():
comps = line.split(' ')
# comps[-1] should be:
# ../../sdb
rdev = comps[-1]
# translate to /dev/sdb
- dev='/dev/{d}'.format(d=rdev.split('/')[-1])
+ dev = '/dev/{d}'.format(d=rdev.split('/')[-1])
# comps[-3] should be:
# /dev/disk/by-id/wwn-0x50014ee002ddecaf
return devmap
+
def get_scratch_devices(remote):
"""
Read the scratch disk list from remote host
devs = file_data.split()
except Exception:
r = remote.run(
- args=['ls', run.Raw('/dev/[sv]d?')],
- stdout=StringIO()
- )
+ args=['ls', run.Raw('/dev/[sv]d?')],
+ stdout=StringIO()
+ )
devs = r.stdout.getvalue().strip().split('\n')
- #Remove root device (vm guests) from the disk list
+ # Remove root device (vm guests) from the disk list
for dev in devs:
if 'vda' in dev:
devs.remove(dev)
break
time.sleep(1)
+
def wait_until_osds_up(ctx, cluster, remote):
"""Wait until all Ceph OSDs are booted."""
num_osds = num_instances_of_type(cluster, 'osd')
break
time.sleep(1)
+
def wait_until_fuse_mounted(remote, fuse, mountpoint):
while True:
proc = remote.run(
fstype = proc.stdout.getvalue().rstrip('\n')
if fstype == 'fuseblk':
break
- log.debug('ceph-fuse not yet mounted, got fs type {fstype!r}'.format(fstype=fstype))
+ log.debug('ceph-fuse not yet mounted, got fs type {fstype!r}'.format(
+ fstype=fstype))
# it shouldn't have exited yet; exposes some trivial problems
assert not fuse.exitstatus.ready()
time.sleep(5)
log.info('ceph-fuse is mounted on %s', mountpoint)
+
def reconnect(ctx, timeout, remotes=None):
"""
Connect to all the machines in ctx.cluster.
else:
need_reconnect.remove(remote)
- log.debug('waited {elapsed}'.format(elapsed=str(time.time() - starttime)))
+ log.debug('waited {elapsed}'.format(
+ elapsed=str(time.time() - starttime)))
time.sleep(1)
+
def write_secret_file(ctx, remote, role, keyring, filename):
testdir = get_testdir(ctx)
remote.run(
],
)
+
def get_clients(ctx, roles):
for role in roles:
assert isinstance(role, basestring)
(remote,) = ctx.cluster.only(role).remotes.iterkeys()
yield (id_, remote)
+
def get_user():
return getpass.getuser() + '@' + socket.gethostname()
for new in g:
ctx.teuthology_config.update(new)
+
def get_mon_names(ctx):
mons = []
for remote, roles in ctx.cluster.remotes.items():
return mons
# return the "first" mon (alphanumerically, for lack of anything better)
+
+
def get_first_mon(ctx, config):
firstmon = sorted(get_mon_names(ctx))[0]
assert firstmon
return firstmon
+
def replace_all_with_clients(cluster, config):
"""
Converts a dict containing a key all to one
norm_config['client.{id}'.format(id=client)] = config['all']
return norm_config
+
def deep_merge(a, b):
if a is None:
return b
return a
return b
+
def get_valgrind_args(testdir, name, preamble, v):
"""
Build a command line for running valgrind.
log.debug('running %s under valgrind with args %s', name, args)
return args
+
def stop_daemons_of_type(ctx, type_):
log.info('Shutting down %s daemons...' % type_)
exc_info = (None, None, None)
if exc_info != (None, None, None):
raise exc_info[0], exc_info[1], exc_info[2]
+
def get_system_type(remote, distro=False):
"""
Return this system type (deb or rpm) or Distro.
"""
r = remote.run(
args=[
- 'sudo','lsb_release', '-is',
+ 'sudo', 'lsb_release', '-is',
],
- stdout=StringIO(),
+ stdout=StringIO(),
)
system_value = r.stdout.getvalue().strip()
log.debug("System to be installed: %s" % system_value)
if distro:
return system_value.lower()
- if system_value in ['Ubuntu','Debian']:
+ if system_value in ['Ubuntu', 'Debian']:
return "deb"
- if system_value in ['CentOS','Fedora','RedHatEnterpriseServer']:
+ if system_value in ['CentOS', 'Fedora', 'RedHatEnterpriseServer']:
return "rpm"
return system_value
+
def get_distro(ctx):
try:
os_type = ctx.config.get('os_type', ctx.os_type)
except AttributeError:
return ctx.os_type
+
def get_distro_version(ctx):
default_os_version = dict(
ubuntu="12.04",
except (KeyError, AttributeError):
return os_version
+
def get_multi_machine_types(machinetype):
"""
Converts machine type string to list based on common deliminators
"""
machinetypes = []
- machine_type_deliminator = [',',' ','\t']
+ machine_type_deliminator = [',', ' ', '\t']
for deliminator in machine_type_deliminator:
if deliminator in machinetype:
machinetypes = machinetype.split(deliminator)
projects / ceph.git / commitdiff