From f31791e35d8eeaba2355cf2af6c2cab8b6149c6b Mon Sep 17 00:00:00 2001 From: Jos Collin Date: Mon, 5 Aug 2019 16:22:10 +0530 Subject: [PATCH] qa/tasks: introduce Thrasher base class * Introduced a Thrasher base class. * Updated thrashers to inherit from Thrasher. * Replaced the magic variable e with Thrasher.exception as per the discussion. Now the exception variable sets by default as the thrashers are inheriting from the Thrasher class. Fixes: https://github.com/ceph/ceph/pull/28378#discussion_r309337928 Fixes: https://tracker.ceph.com/issues/41133 Signed-off-by: Jos Collin --- qa/tasks/ceph_manager.py | 19 +++++++++++++++++-- qa/tasks/daemonwatchdog.py | 4 ++-- qa/tasks/mds_thrash.py | 10 +++++----- qa/tasks/mon_thrash.py | 7 ++++--- qa/tasks/rbd_mirror_thrash.py | 13 ++++++++----- qa/tasks/thrasher.py | 16 ++++++++++++++++ qa/tasks/thrashosds.py | 2 +- 7 files changed, 53 insertions(+), 18 deletions(-) create mode 100644 qa/tasks/thrasher.py diff --git a/qa/tasks/ceph_manager.py b/qa/tasks/ceph_manager.py index 4d9ac33a902..2e3d9cdd4b8 100644 --- a/qa/tasks/ceph_manager.py +++ b/qa/tasks/ceph_manager.py @@ -22,6 +22,7 @@ from teuthology.contextutil import safe_while from teuthology.orchestra.remote import Remote from teuthology.orchestra import run from teuthology.exceptions import CommandFailedError +from tasks.thrasher import Thrasher try: from subprocess import DEVNULL # py3k @@ -100,11 +101,12 @@ class PoolType: ERASURE_CODED = 3 -class Thrasher: +class OSDThrasher(Thrasher): """ Object used to thrash Ceph """ def __init__(self, manager, config, logger): + super(OSDThrasher, self).__init__() self.ceph_manager = manager self.cluster = manager.cluster self.ceph_manager.wait_for_clean() @@ -955,6 +957,19 @@ class Thrasher: val -= prob return None + def do_thrash(self): + """ + _do_thrash() wrapper. + """ + try: + self._do_thrash() + except Exception as e: + # See _run exception comment for MDSThrasher + self.exception = e + self.logger.exception("exception:") + # Allow successful completion so gevent doesn't see an exception. + # The DaemonWatchdog will observe the error and tear down the test. + def log_exc(func): @wraps(func) def wrapper(self): @@ -1047,7 +1062,7 @@ class Thrasher: self.ceph_manager.raw_cluster_cmd('osd', 'unset', 'nodeep-scrub') @log_exc - def do_thrash(self): + def _do_thrash(self): """ Loop to select random actions to thrash ceph manager with. """ diff --git a/qa/tasks/daemonwatchdog.py b/qa/tasks/daemonwatchdog.py index e53f182d5ee..1c54d3b622e 100644 --- a/qa/tasks/daemonwatchdog.py +++ b/qa/tasks/daemonwatchdog.py @@ -23,7 +23,7 @@ class DaemonWatchdog(Greenlet): """ def __init__(self, ctx, config, thrashers): - Greenlet.__init__(self) + super(DaemonWatchdog, self).__init__() self.ctx = ctx self.config = config self.e = None @@ -106,7 +106,7 @@ class DaemonWatchdog(Greenlet): del daemon_failure_time[name] for thrasher in self.thrashers: - if thrasher.e is not None: + if thrasher.exception is not None: self.log("thrasher on fs.{name} failed".format(name=thrasher.fs.name)) bark = True diff --git a/qa/tasks/mds_thrash.py b/qa/tasks/mds_thrash.py index 05475338972..a381c3056e6 100644 --- a/qa/tasks/mds_thrash.py +++ b/qa/tasks/mds_thrash.py @@ -14,10 +14,11 @@ from gevent.event import Event from teuthology import misc as teuthology from tasks.cephfs.filesystem import MDSCluster, Filesystem +from tasks.thrasher import Thrasher log = logging.getLogger(__name__) -class MDSThrasher(Greenlet): +class MDSThrasher(Greenlet, Thrasher, object): """ MDSThrasher:: @@ -97,11 +98,10 @@ class MDSThrasher(Greenlet): """ def __init__(self, ctx, manager, config, fs, max_mds): - Greenlet.__init__(self) + super(MDSThrasher, self).__init__() self.config = config self.ctx = ctx - self.e = None self.logger = log.getChild('fs.[{f}]'.format(f = fs.name)) self.fs = fs self.manager = manager @@ -136,7 +136,7 @@ class MDSThrasher(Greenlet): # File "/usr/lib/python2.7/traceback.py", line 13, in _print # file.write(str+terminator) # 2017-02-03T14:34:01.261 CRITICAL:root:IOError - self.e = e + self.exception = e self.logger.exception("exception:") # allow successful completion so gevent doesn't see an exception... @@ -425,7 +425,7 @@ def task(ctx, config): finally: log.info('joining mds_thrasher') thrasher.stop() - if thrasher.e: + if thrasher.exception is not None: raise RuntimeError('error during thrashing') thrasher.join() log.info('done joining') diff --git a/qa/tasks/mon_thrash.py b/qa/tasks/mon_thrash.py index b4023a97873..b166f759b4a 100644 --- a/qa/tasks/mon_thrash.py +++ b/qa/tasks/mon_thrash.py @@ -11,6 +11,7 @@ import json import math from teuthology import misc as teuthology from tasks.cephfs.filesystem import MDSCluster +from tasks.thrasher import Thrasher log = logging.getLogger(__name__) @@ -21,7 +22,7 @@ def _get_mons(ctx): mons = [f[len('mon.'):] for f in teuthology.get_mon_names(ctx)] return mons -class MonitorThrasher: +class MonitorThrasher(Thrasher): """ How it works:: @@ -85,11 +86,11 @@ class MonitorThrasher: - mon/workloadgen.sh """ def __init__(self, ctx, manager, config, logger): + super(MonitorThrasher, self).__init__() self.ctx = ctx self.manager = manager self.manager.wait_for_clean() - self.e = None self.stopping = False self.logger = logger self.config = config @@ -228,7 +229,7 @@ class MonitorThrasher: self._do_thrash() except Exception as e: # See _run exception comment for MDSThrasher - self.e = e + self.exception = e self.logger.exception("exception:") # Allow successful completion so gevent doesn't see an exception. # The DaemonWatchdog will observe the error and tear down the test. diff --git a/qa/tasks/rbd_mirror_thrash.py b/qa/tasks/rbd_mirror_thrash.py index 75e1cdab638..7ca1a6d1d70 100644 --- a/qa/tasks/rbd_mirror_thrash.py +++ b/qa/tasks/rbd_mirror_thrash.py @@ -17,11 +17,12 @@ from teuthology import misc from teuthology.exceptions import CommandFailedError from teuthology.task import Task from teuthology.orchestra import run +from tasks.thrasher import Thrasher log = logging.getLogger(__name__) -class RBDMirrorThrasher(Greenlet): +class RBDMirrorThrasher(Greenlet, Thrasher, object): """ RBDMirrorThrasher:: @@ -63,14 +64,13 @@ class RBDMirrorThrasher(Greenlet): """ def __init__(self, ctx, config, cluster, daemons): - Greenlet.__init__(self) + super(RBDMirrorThrasher, self).__init__() self.ctx = ctx self.config = config self.cluster = cluster self.daemons = daemons - self.e = None self.logger = log self.name = 'thrasher.rbd_mirror.[{cluster}]'.format(cluster = cluster) self.stopping = Event() @@ -85,8 +85,11 @@ class RBDMirrorThrasher(Greenlet): try: self.do_thrash() except Exception as e: - self.e = e + # See _run exception comment for MDSThrasher + self.exception = e self.logger.exception("exception:") + # Allow successful completion so gevent doesn't see an exception. + # The DaemonWatchdog will observe the error and tear down the test. def log(self, x): """Write data to logger assigned to this RBDMirrorThrasher""" @@ -211,7 +214,7 @@ def task(ctx, config): finally: log.info('joining rbd_mirror_thrash') thrasher.stop() - if thrasher.e: + if thrasher.exception is not None: raise RuntimeError('error during thrashing') thrasher.join() log.info('done joining') diff --git a/qa/tasks/thrasher.py b/qa/tasks/thrasher.py new file mode 100644 index 00000000000..fc783eaa697 --- /dev/null +++ b/qa/tasks/thrasher.py @@ -0,0 +1,16 @@ +""" +Thrasher base class +""" +class Thrasher(object): + + def __init__(self): + super(Thrasher, self).__init__() + self.exception = None + + @property + def exception(self): + return self._exception + + @exception.setter + def exception(self, e): + self._exception = e diff --git a/qa/tasks/thrashosds.py b/qa/tasks/thrashosds.py index aa8d9c37024..c65f79c2b88 100644 --- a/qa/tasks/thrashosds.py +++ b/qa/tasks/thrashosds.py @@ -200,7 +200,7 @@ def task(ctx, config): 'true') log.info('Beginning thrashosds...') - thrash_proc = ceph_manager.Thrasher( + thrash_proc = ceph_manager.OSDThrasher( cluster_manager, config, logger=log.getChild('thrasher') -- 2.39.5