task/thrashosds: Ipmi checking/setup in thrashosds

We don't need to setup the ipmi console on runs that
don't use powercycling, so delay setup of the RemoteConsole
with ipmi to the thrashosd task and only then if the powercycle
config is set.  This avoids spurious test failures from flaky
ipmi.

Signed-off-by: Sam Lang <sam.lang@inktank.com>

diff --git a/teuthology/task/internal.py b/teuthology/task/internal.py
index 7195e31cb1fdda8772e51eb2254d2d2f50c53af6..e10bfdc4fc6aa97865f710707d0f259803c3293d 100644 (file)
--- a/teuthology/task/internal.py
+++ b/teuthology/task/internal.py
@@ -152,29 +152,13 @@ def connect(ctx, config):
     from ..orchestra import cluster
     remotes = []
     for t, key in ctx.config['targets'].iteritems():
-        console = None
-        if 'ipmi_user' in ctx.teuthology_config:
-            host = t.split('@')[-1]
-            shortname = host.split('.')[0]
-            console = remote.RemoteConsole(name=host,
-                                           ipmiuser=ctx.teuthology_config['ipmi_user'],
-                                           ipmipass=ctx.teuthology_config['ipmi_password'],
-                                           ipmidomain=ctx.teuthology_config['ipmi_domain'])
-            cname = '{host}.{domain}'.format(host=shortname, domain=ctx.teuthology_config['ipmi_domain'])
-            log.debug('checking console status of %s' % cname)
-            if not console.check_status():
-                log.info('Failed to get console status for %s, disabling console...' % cname)
-                console=None
-            else:
-                log.debug('console ready on %s' % cname)
-
         log.debug('connecting to %s', t)
         remotes.append(
             remote.Remote(name=t,
                           ssh=connection.connect(user_at_host=t,
                                                  host_key=key,
                                                  keep_alive=True),
-                          console=console))
+                          console=None))
     ctx.cluster = cluster.Cluster()
     if 'roles' in ctx.config:
         for rem, roles in zip(remotes, ctx.config['roles']):

diff --git a/teuthology/task/thrashosds.py b/teuthology/task/thrashosds.py
index 50628a3e6010e4a0190117bc01d16d8b6cfaed5f..f8b74a900a58ca804a6c822e75f828d7ae07dd8c 100644 (file)
--- a/teuthology/task/thrashosds.py
+++ b/teuthology/task/thrashosds.py
@@ -87,6 +87,37 @@ def task(ctx, config):
         config = {}
     assert isinstance(config, dict), \
         'thrashosds task only accepts a dict for configuration'
+
+    if 'powercycle' in config:
+
+        if 'ipmi_user' in ctx.teuthology_config:
+            for t, key in ctx.config['targets'].iteritems():
+                host = t.split('@')[-1]
+                shortname = host.split('.')[0]
+                from ..orchestra import remote as oremote
+                console = oremote.RemoteConsole(name=host,
+                                           ipmiuser=ctx.teuthology_config['ipmi_user'],
+                                           ipmipass=ctx.teuthology_config['ipmi_password'],
+                                           ipmidomain=ctx.teuthology_config['ipmi_domain'])
+                cname = '{host}.{domain}'.format(host=shortname, domain=ctx.teuthology_config['ipmi_domain'])
+                log.debug('checking console status of %s' % cname)
+                if not console.check_status():
+                    log.info('Failed to get console status for %s, disabling console...' % cname)
+                    console=None
+                else:
+                    # find the remote for this console and add it
+                    remotes = [r for r in ctx.cluster.remotes.keys() if r.name == t]
+                    if len(remotes) != 1:
+                        raise Exception('Too many (or too few) remotes found for target {t}'.format(t=t))
+                    remotes[0].console = console
+                    log.debug('console ready on %s' % cname)
+
+            # check that all osd remotes have a valid console
+            osds = ctx.cluster.only(teuthology.is_type('osd'))
+            for remote, _ in osds.remotes.iteritems():
+                if not remote.console:
+                    raise Exception('IPMI console required for powercycling, but not available on osd role: {r}'.format(r=remote.name))
+
     log.info('Beginning thrashosds...')
     first_mon = teuthology.get_first_mon(ctx, config)
     (mon,) = ctx.cluster.only(first_mon).remotes.iterkeys()