From e0ed09cfa996b789d65d077fd3105cc5a54164c4 Mon Sep 17 00:00:00 2001 From: Warren Usui Date: Fri, 14 Feb 2014 09:57:43 -0800 Subject: [PATCH] Add docstrings to the orchestra code. Also fix minor formatting issues (mostly column start locations). Fixes: 7003 Signed-off-by: Warren Usui --- teuthology/orchestra/cluster.py | 15 +++-- teuthology/orchestra/connection.py | 19 ++++++ teuthology/orchestra/monkey.py | 9 ++- teuthology/orchestra/remote.py | 94 +++++++++++++++++++++++++----- teuthology/orchestra/run.py | 63 +++++++++++++++++++- 5 files changed, 177 insertions(+), 23 deletions(-) diff --git a/teuthology/orchestra/cluster.py b/teuthology/orchestra/cluster.py index 7c55fa0ff6d79..5dca131179fe5 100644 --- a/teuthology/orchestra/cluster.py +++ b/teuthology/orchestra/cluster.py @@ -1,3 +1,7 @@ +""" +Cluster definition +part of context, Cluster is used to save connection information. +""" import teuthology.misc class Cluster(object): @@ -15,9 +19,9 @@ class Cluster(object): self.add(remote, roles) def __repr__(self): - remotes = [(k, v) for k,v in self.remotes.items()] + remotes = [(k, v) for k, v in self.remotes.items()] remotes.sort(key=lambda tup: tup[0].name) - remotes = '{' + ', '.join('{remote!r}: {roles!r}'.format(remote=k, roles=v) for k,v in remotes) + '}' + remotes = '{' + ', '.join('{remote!r}: {roles!r}'.format(remote=k, roles=v) for k, v in remotes) + '}' return '{classname}(remotes={remotes})'.format( classname=self.__class__.__name__, remotes=remotes, @@ -26,11 +30,14 @@ class Cluster(object): def __str__(self): remotes = list(self.remotes.items()) remotes.sort(key=lambda tup: tup[0].name) - remotes = ((k, ','.join(v)) for k,v in remotes) - remotes = ('{k}[{v}]'.format(k=k, v=v) for k,v in remotes) + remotes = ((k, ','.join(v)) for k, v in remotes) + remotes = ('{k}[{v}]'.format(k=k, v=v) for k, v in remotes) return ' '.join(remotes) def add(self, remote, roles): + """ + Add roles to the list of remotes. + """ if remote in self.remotes: raise RuntimeError( 'Remote {new!r} already found in remotes: {old!r}'.format( diff --git a/teuthology/orchestra/connection.py b/teuthology/orchestra/connection.py index b674f84bbb13b..9317d695bd2b8 100644 --- a/teuthology/orchestra/connection.py +++ b/teuthology/orchestra/connection.py @@ -1,3 +1,6 @@ +""" +Connection utilities +""" import base64 import paramiko import os @@ -5,6 +8,9 @@ from ..config import config def split_user(user_at_host): + """ + break apart user@host fields into user and host. + """ try: user, host = user_at_host.rsplit('@', 1) except ValueError: @@ -15,6 +21,9 @@ def split_user(user_at_host): def create_key(keytype, key): + """ + Create an ssh-rsa or ssh-dss key. + """ if keytype == 'ssh-rsa': return paramiko.rsakey.RSAKey(data=base64.decodestring(key)) elif keytype == 'ssh-dss': @@ -25,6 +34,16 @@ def create_key(keytype, key): def connect(user_at_host, host_key=None, keep_alive=False, _SSHClient=None, _create_key=None): + """ + ssh connection routine. + + :param user_at_host: user@host + :param host_key: ssh key + :param keep_alive: keep_alive indicator + :param _SSHClient: client, default is paramiko ssh client + :param _create_key: routine to create a key (defaults to local reate_key) + :return: ssh connection. + """ user, host = split_user(user_at_host) if _SSHClient is None: _SSHClient = paramiko.SSHClient diff --git a/teuthology/orchestra/monkey.py b/teuthology/orchestra/monkey.py index 33c692ad8af6b..cd6104b9c8d4b 100644 --- a/teuthology/orchestra/monkey.py +++ b/teuthology/orchestra/monkey.py @@ -1,10 +1,13 @@ +""" +Monkey patches (paramiko support) +""" import logging log = logging.getLogger(__name__) def patch_001_paramiko_deprecation(): """ - Silence an an unhelpful DeprecationWarning triggered by Paramiko. + Silence an an unhelpful Deprecation Warning triggered by Paramiko. Not strictly a monkeypatch. """ @@ -40,8 +43,8 @@ def patch_all(): """ Run all the patch_* functions in this module. """ - monkeys = [(k,v) for (k,v) in globals().iteritems() if k.startswith('patch_') and k != 'patch_all'] + monkeys = [(k, v) for (k, v) in globals().iteritems() if k.startswith('patch_') and k != 'patch_all'] monkeys.sort() - for k,v in monkeys: + for k, v in monkeys: log.debug('Patching %s', k) v() diff --git a/teuthology/orchestra/remote.py b/teuthology/orchestra/remote.py index e74c2e373b7a1..1c6a4f743f3c7 100644 --- a/teuthology/orchestra/remote.py +++ b/teuthology/orchestra/remote.py @@ -1,3 +1,6 @@ +"" +Support for paramiko remote objects. +""" from . import run from teuthology import misc import time @@ -32,6 +35,9 @@ class Remote(object): @property def shortname(self): + """ + shortname decorator + """ name = self._shortname if name is None: name = self.name @@ -39,6 +45,9 @@ class Remote(object): @property def system_type(self): + """ + System type decorator + """ return misc.get_system_type(self) def __str__(self): @@ -62,12 +71,17 @@ class Remote(object): def getShortName(name): + """ + Extract the name portion from remote name strings. + """ hn = name.split('@')[-1] p = re.compile('([^.]+)\.?.*') return p.match(hn).groups()[0] class PhysicalConsole(): - + """ + Physical Console (set from getRemoteConsole) + """ def __init__(self, name, ipmiuser, ipmipass, ipmidomain, logfile=None, timeout=20): self.name = name self.shortname = getShortName(name) @@ -78,8 +92,11 @@ class PhysicalConsole(): self.ipmidomain = ipmidomain def _exec(self, cmd): + """ + Run the cmd specified using ipmitool. + """ if not self.ipmiuser or not self.ipmipass or not self.ipmidomain: - log.error('Must set ipmi_user, ipmi_password, and ipmi_domain in .teuthology.yaml') + log.error('Must set ipmi_user, ipmi_password, and ipmi_domain in .teuthology.yaml') log.debug('pexpect command: ipmitool -H {s}.{dn} -I lanplus -U {ipmiuser} -P {ipmipass} {cmd}'.format( cmd=cmd, s=self.shortname, @@ -107,6 +124,9 @@ class PhysicalConsole(): self._exec('sol deactivate') def _wait_for_login(self, timeout=None, attempts=6): + """ + Wait for login. Retry if timeouts occur on commands. + """ log.debug('Waiting for login prompt on {s}'.format(s=self.shortname)) # wait for login prompt to indicate boot completed t = timeout @@ -126,14 +146,16 @@ class PhysicalConsole(): if r == 0: return def check_power(self, state, timeout=None): - # check power - total_timeout = timeout - if not total_timeout: - total_timeout = self.timeout - t = 1 - total = t - ta = time.time() - while total < total_timeout: + """ + Check power. Retry if EOF encountered on power check read. + """ + total_timeout = timeout + if not total_timeout: + total_timeout = self.timeout + t = 1 + total = t + ta = time.time() + while total < total_timeout: c = self._exec('power status') r = c.expect(['Chassis Power is {s}'.format(s=state), pexpect.EOF, pexpect.TIMEOUT], timeout=t) tb = time.time() @@ -148,10 +170,12 @@ class PhysicalConsole(): ta = tb t *= 2 total += t - return False + return False - # returns True if console is at login prompt def check_status(self, timeout=None): + """ + Check status. Returns True if console is at login prompt + """ try : # check for login prompt at console self._wait_for_login(timeout) @@ -161,6 +185,9 @@ class PhysicalConsole(): return False def power_cycle(self): + """ + Power cycle and wait for login. + """ log.info('Power cycling {s}'.format(s=self.shortname)) child = self._exec('power cycle') child.expect('Chassis Power Control: Cycle', timeout=self.timeout) @@ -168,6 +195,10 @@ class PhysicalConsole(): log.info('Power cycle for {s} completed'.format(s=self.shortname)) def hard_reset(self): + """ + Perform physical hard reset. Retry if EOF returned from read + and wait for login when complete. + """ log.info('Performing hard reset of {s}'.format(s=self.shortname)) start = time.time() while time.time() - start < self.timeout: @@ -179,6 +210,9 @@ class PhysicalConsole(): log.info('Hard reset for {s} completed'.format(s=self.shortname)) def power_on(self): + """ + Physical power on. Loop checking cmd return. + """ log.info('Power on {s}'.format(s=self.shortname)) start = time.time() while time.time() - start < self.timeout: @@ -191,6 +225,9 @@ class PhysicalConsole(): log.info('Power on for {s} completed'.format(s=self.shortname)) def power_off(self): + """ + Physical power off. Loop checking cmd return. + """ log.info('Power off {s}'.format(s=self.shortname)) start = time.time() while time.time() - start < self.timeout: @@ -203,6 +240,11 @@ class PhysicalConsole(): log.info('Power off for {s} completed'.format(s=self.shortname)) def power_off_for_interval(self, interval=30): + """ + Physical power off for an interval. Wait for login when complete. + + :param interval: Length of power-off period. + """ log.info('Power off {s} for {i} seconds'.format(s=self.shortname, i=interval)) child = self._exec('power off') child.expect('Chassis Power Control: Down/Off', timeout=self.timeout) @@ -215,7 +257,9 @@ class PhysicalConsole(): log.info('Power off for {i} seconds completed'.format(s=self.shortname, i=interval)) class VirtualConsole(): - + """ + Virtual Console (set from getRemoteConsole) + """ def __init__(self, name, ipmiuser, ipmipass, ipmidomain, logfile=None, timeout=20): if libvirt is None: raise RuntimeError("libvirt not found") @@ -235,26 +279,47 @@ class VirtualConsole(): return def check_power(self, state, timeout=None): + """ + Return true if vm domain state indicates power is on. + """ return self.vm_domain.info[0] in [libvirt.VIR_DOMAIN_RUNNING, libvirt.VIR_DOMAIN_BLOCKED, libvirt.VIR_DOMAIN_PAUSED] def check_status(self, timeout=None): + """ + Return true if running. + """ return self.vm_domain.info()[0] == libvirt.VIR_DOMAIN_RUNNING def power_cycle(self): + """ + Simiulate virtual machine power cycle + """ self.vm_domain.info().destroy() self.vm_domain.info().create() def hard_reset(self): + """ + Simiulate hard reset + """ self.vm_domain.info().destroy() def power_on(self): + """ + Simiulate power on + """ self.vm_domain.info().create() def power_off(self): + """ + Simiulate power off + """ self.vm_domain.info().destroy() def power_off_for_interval(self, interval=30): + """ + Simiulate power off for an interval. + """ log.info('Power off {s} for {i} seconds'.format(s=self.shortname, i=interval)) self.vm_domain.info().destroy() time.sleep(interval) @@ -262,6 +327,9 @@ class VirtualConsole(): log.info('Power off for {i} seconds completed'.format(s=self.shortname, i=interval)) def getRemoteConsole(name, ipmiuser, ipmipass, ipmidomain, logfile=None, timeout=20): + """ + Return either VirtualConsole or PhysicalConsole depending on name. + """ if misc.is_vm(name): return VirtualConsole(name, ipmiuser, ipmipass, ipmidomain, logfile, timeout) return PhysicalConsole(name, ipmiuser, ipmipass, ipmidomain, logfile, timeout) diff --git a/teuthology/orchestra/run.py b/teuthology/orchestra/run.py index 7bc5047668851..16987f24d9810 100644 --- a/teuthology/orchestra/run.py +++ b/teuthology/orchestra/run.py @@ -1,3 +1,6 @@ +""" +Paramiko run support +""" from cStringIO import StringIO import gevent @@ -9,6 +12,9 @@ import shutil log = logging.getLogger(__name__) class RemoteProcess(object): + """ + Remote process object used to keep track of attributes of a process. + """ __slots__ = [ 'command', 'stdin', 'stdout', 'stderr', 'exitstatus', 'exited', # for orchestra.remote.Remote to place a backreference @@ -23,6 +29,9 @@ class RemoteProcess(object): self.exited = exited class Raw(object): + """ + Raw objects are passed to remote objects and are not processed locally. + """ def __init__(self, value): self.value = value @@ -33,7 +42,13 @@ class Raw(object): ) def quote(args): + """ + Internal quote wrapper. + """ def _quote(args): + """ + Handle quoted string, testing for raw charaters. + """ for a in args: if isinstance(a, Raw): yield a.value @@ -64,13 +79,21 @@ def execute(client, args): (in_, out, err) = client.exec_command(cmd) def get_exitstatus(): + """ + Get exit status. + + When -1 on connection loss *and* signals occur, this + maps to more pythonic None + """ status = out.channel.recv_exit_status() - # -1 on connection loss *and* signals; map to more pythonic None if status == -1: status = None return status def exitstatus_ready(): + """ + out.channel exit wrapper. + """ return out.channel.exit_status_ready() r = RemoteProcess( @@ -86,6 +109,9 @@ def execute(client, args): return r def copy_to_log(f, logger, host, loglevel=logging.INFO): + """ + Interface to older xreadlines api. + """ # i can't seem to get fudge to fake an iterable, so using this old # api for now for line in f.xreadlines(): @@ -93,6 +119,9 @@ def copy_to_log(f, logger, host, loglevel=logging.INFO): logger.log(loglevel, '[' + host + ']: ' + line) def copy_and_close(src, fdst): + """ + copyfileobj call wrapper. + """ if src is not None: if isinstance(src, basestring): src = StringIO(src) @@ -100,6 +129,12 @@ def copy_and_close(src, fdst): fdst.close() def copy_file_to(f, dst, host): + """ + Copy file + :param f: file to be copied. + :param dst: destination + :param host: original host location + """ if hasattr(dst, 'log'): # looks like a Logger to me; not using isinstance to make life # easier for unit tests @@ -111,6 +146,9 @@ def copy_file_to(f, dst, host): class CommandFailedError(Exception): + """ + Exception thrown on command failure + """ def __init__(self, command, exitstatus, node=None): self.command = command self.exitstatus = exitstatus @@ -125,6 +163,9 @@ class CommandFailedError(Exception): class CommandCrashedError(Exception): + """ + Exception thrown on crash + """ def __init__(self, command): self.command = command @@ -135,6 +176,9 @@ class CommandCrashedError(Exception): class ConnectionLostError(Exception): + """ + Exception thrown when the connection is lost + """ def __init__(self, command): self.command = command @@ -155,6 +199,9 @@ def spawn_asyncresult(fn, *args, **kwargs): """ r = gevent.event.AsyncResult() def wrapper(): + """ + Internal wrapper. + """ try: value = fn(*args, **kwargs) except Exception as e: @@ -166,6 +213,9 @@ def spawn_asyncresult(fn, *args, **kwargs): return r class Sentinel(object): + """ + Sentinel -- used to define PIPE file-like object. + """ def __init__(self, name): self.name = name @@ -186,6 +236,9 @@ class KludgeFile(object): return getattr(self._wrapped, name) def close(self): + """ + Close and shutdown. + """ self._wrapped.close() self._wrapped.channel.shutdown_write() @@ -222,7 +275,7 @@ def run( if logger is None: logger = log - (host,port) = client.get_transport().getpeername() + (host, port) = client.get_transport().getpeername() g_err = None if stderr is not PIPE: if stderr is None: @@ -242,6 +295,10 @@ def run( assert not wait, "Using PIPE for stdout without wait=False would deadlock." def _check_status(status): + """ + get values needed if uninitialized. Handle ssh issues when checking + the status. + """ if g_err is not None: g_err.get() if g_out is not None: @@ -263,7 +320,7 @@ def run( # signal; sadly SSH does not tell us which signal raise CommandCrashedError(command=r.command) if status != 0: - (host,port) = client.get_transport().getpeername() + (host, port) = client.get_transport().getpeername() raise CommandFailedError(command=r.command, exitstatus=status, node=host) return status -- 2.39.5