+"""
+Miscellaneous teuthology functions.
+Used by other modules, but mostly called from tasks.
+"""
from cStringIO import StringIO
import argparse
def config_file(string):
+ """
+ Create a config file
+
+ :param string: name of yaml file used for config.
+ :returns: Dictionary of configuration information.
+ """
config_dict = {}
try:
with file(string) as f:
class MergeConfig(argparse.Action):
-
+ """
+ Used by scripts to mergeg configurations. (nuke, run, and
+ schedule, for example)
+ """
def __call__(self, parser, namespace, values, option_string=None):
+ """
+ Perform merges of all the day in the config dictionaries.
+ """
config_dict = getattr(namespace, self.dest)
for new in values:
deep_merge(config_dict, new)
def get_testdir(ctx):
+ """
+ :returns: A test directory
+ """
if 'test_path' in ctx.teuthology_config:
return ctx.teuthology_config['test_path']
test_user = get_test_user(ctx)
def get_archive_dir(ctx):
+ """
+ :returns: archive directory (a subdirectory of the test directory)
+ """
test_dir = get_testdir(ctx)
return os.path.normpath(os.path.join(test_dir, 'archive'))
def get_http_log_path(archive_dir, job_id=None):
+ """
+ :param archive_dir: directory to be searched
+ :param job_id: id of job that terminates the name of the log path
+ :returns: http log path
+ """
http_base = config.archive_server
if not http_base:
return None
def get_ceph_binary_url(package=None,
branch=None, tag=None, sha1=None, dist=None,
flavor=None, format=None, arch=None):
+ """
+ return the url of the ceph binary found on gitbuildder.
+ """
BASE = 'http://gitbuilder.ceph.com/{package}-{format}-{dist}-{arch}-{flavor}/'.format(
package=package,
flavor=flavor,
def feed_many_stdins(fp, processes):
+ """
+ :param fp: input file
+ :param processes: list of processes to be written to.
+ """
while True:
data = fp.read(8192)
if not data:
def feed_many_stdins_and_close(fp, processes):
+ """
+ Feed many and then close processes.
+
+ :param fp: input file
+ :param processes: list of processes to be written to.
+ """
feed_many_stdins(fp, processes)
for proc in processes:
proc.stdin.close()
def get_mons(roles, ips):
+ """
+ Get monitors and their associated ports
+ """
mons = {}
mon_ports = {}
mon_id = 0
def generate_caps(type_):
+ """
+ Each call will return the next capability for each system type
+ (essentially a subset of possible role values). Valid types are osd,
+ mds and client.
+ """
defaults = dict(
osd=dict(
mon='allow *',
def skeleton_config(ctx, roles, ips):
"""
- Returns a ConfigObj that's prefilled with a skeleton config.
+ Returns a ConfigObj that is prefilled with a skeleton config.
Use conf[section][key]=value or conf.merge to change it.
def roles_of_type(roles_for_host, type_):
+ """
+ Generator of roles.
+
+ Each call returns the next possible role of the type specified.
+ :param roles_for host: list of roles possible
+ :param type_: type of role
+ """
prefix = '{type}.'.format(type=type_)
for name in roles_for_host:
if not name.startswith(prefix):
def all_roles(cluster):
+ """
+ Generator of role values. Each call returns another role.
+
+ :param cluster: Cluster extracted from the ctx.
+ """
for _, roles_for_host in cluster.remotes.iteritems():
for name in roles_for_host:
yield name
def all_roles_of_type(cluster, type_):
+ """
+ Generator of role values. Each call returns another role of the
+ type specified.
+
+ :param cluster: Cluster extracted from the ctx.
+ :type_: role type
+ """
prefix = '{type}.'.format(type=type_)
for _, roles_for_host in cluster.remotes.iteritems():
for name in roles_for_host:
prefix = '{type}.'.format(type=type_)
def _is_type(role):
+ """
+ Return type based on the starting role name. This should
+ probably be improved in the future.
+ """
return role.startswith(prefix)
return _is_type
def num_instances_of_type(cluster, type_):
+ """
+ Total the number of instances of the role type specified in all remotes.
+
+ :param cluster: Cluster extracted from ctx.
+ :param type_: role
+ """
remotes_and_roles = cluster.remotes.items()
roles = [roles for (remote, roles) in remotes_and_roles]
prefix = '{type}.'.format(type=type_)
:return the FSID (as a string) of the newly created monmap
"""
def gen_addresses():
+ """
+ Monitor address generator.
+
+ Each invocation returns the next monitor address
+ """
for section, data in conf.iteritems():
PREFIX = 'mon.'
if not section.startswith(PREFIX):
def write_file(remote, path, data):
+ """
+ Write data to a remote file
+
+ :param remote: Remote site.
+ :param path: Path on the remote being written to.
+ :param data: Data to be written.
+ """
remote.run(
args=[
'python',
def sudo_write_file(remote, path, data, perms=None):
+ """
+ Write data to a remote file as super user
+
+ :param remote: Remote site.
+ :param path: Path on the remote being written to.
+ :param data: Data to be written.
+ :param perms: Permissions on the file being written
+ """
permargs = []
if perms:
permargs = [run.Raw('&&'), 'sudo', 'chmod', perms, path]
def move_file(remote, from_path, to_path, sudo=False):
+ """
+ Move a file from one path to another on a remote site
- # need to stat the file first, to make sure we
- # maintain the same permissions
+ The file needs to be stat'ed first, to make sure we
+ maintain the same permissions
+ """
args = []
if sudo:
args.append('sudo')
def delete_file(remote, path, sudo=False, force=False):
+ """
+ rm a file on a remote site.
+ """
args = []
if sudo:
args.append('sudo')
def remove_lines_from_file(remote, path, line_is_valid_test,
string_to_test_for):
+ """
+ Remove lines from a file. This involves reading the file in, removing
+ the appropriate lines, saving the file, and then replacing the original
+ file with the new file. Intermediate files are used to prevent data loss
+ on when the main site goes up and down.
+ """
# read in the specified file
in_data = get_file(remote, path, False)
out_data = ""
def append_lines_to_file(remote, path, lines, sudo=False):
+ """
+ Append lines to a file.
+ An intermediate file is used in the same manner as in
+ Remove_lines_from_list.
+ """
+
temp_file_path = remote_mktemp(remote)
data = get_file(remote, path, sudo)
def remote_mktemp(remote, sudo=False):
+ """
+ Make a temporary file on a remote system
+ """
args = []
if sudo:
args.append('sudo')
)
proc.exitstatus.get()
-# returns map of devices to device id links:
-# /dev/sdb: /dev/disk/by-id/wwn-0xf00bad
-
def get_wwn_id_map(remote, devs):
+ """
+ Extract ww_id_map information from ls output on the associated devs.
+
+ Sample dev information: /dev/sdb: /dev/disk/by-id/wwn-0xf00bad
+
+ :returns: map of devices to device id links
+ """
stdout = None
try:
r = remote.run(
def wait_until_fuse_mounted(remote, fuse, mountpoint):
+ """
+ Check to make sure that fuse is mounted on mountpoint. If not,
+ sleep for 5 seconds and check again.
+ """
while True:
proc = remote.run(
args=[
def write_secret_file(ctx, remote, role, keyring, filename):
+ """
+ Stash the kerying in the filename specified.
+ """
testdir = get_testdir(ctx)
remote.run(
args=[
def get_clients(ctx, roles):
+ """
+ return all remote roles that are clients.
+ """
for role in roles:
assert isinstance(role, basestring)
PREFIX = 'client.'
def get_user():
+ """
+ Return the username in the format user@host.
+ """
return getpass.getuser() + '@' + socket.gethostname()
def read_config(ctx):
+ """
+ read the default teuthology yaml configuration file.
+ """
ctx.teuthology_config = {}
filename = os.path.join(os.environ['HOME'], '.teuthology.yaml')
def get_mon_names(ctx):
+ """
+ :returns: a list of monitor names
+ """
mons = []
for remote, roles in ctx.cluster.remotes.items():
for role in roles:
mons.append(role)
return mons
-# return the "first" mon (alphanumerically, for lack of anything better)
-
def get_first_mon(ctx, config):
+ """
+ return the "first" mon (alphanumerically, for lack of anything better)
+ """
firstmon = sorted(get_mon_names(ctx))[0]
assert firstmon
return firstmon
def deep_merge(a, b):
+ """
+ Deep Merge. If a and b are both lists, all elements in b are
+ added into a. If a and b are both dictionaries, elements in b are
+ recursively added to a.
+ :param a: object items will be merged into
+ :param b: object items will be merged from
+ """
if a is None:
return b
if b is None:
def stop_daemons_of_type(ctx, type_):
+ """
+ :param type_: type of daemons to be stopped.
+ """
log.info('Shutting down %s daemons...' % type_)
exc_info = (None, None, None)
for daemon in ctx.daemons.iter_daemons_of_role(type_):
def get_distro(ctx):
+ """
+ Get the name of the distro that we are using (usually the os_type).
+ """
try:
os_type = ctx.config.get('os_type', ctx.os_type)
except AttributeError:
def get_distro_version(ctx):
+ """
+ Get the verstion of the distro that we are using (release number).
+ """
default_os_version = dict(
ubuntu="12.04",
fedora="18",
projects / teuthology.git / commitdiff