From: Joao Eduardo Luis Date: Tue, 12 Feb 2013 15:31:28 +0000 (+0000) Subject: task: mon_thrash: Thrash multiple monitors and 'maintain-quorum' option X-Git-Tag: 1.1.0~2290 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=c1b75c6b0ca7d5cab8dcd03bd5d37f8f6bb3aacb;p=teuthology.git task: mon_thrash: Thrash multiple monitors and 'maintain-quorum' option We now add a new option 'thrash-many' that by being set to true will break the default behaviour of killing only one monitor at a time. Instead, this option will select up to the maximum number of killable monitors to kill in each round. We also add a new 'maintain-quorum' option that will limit the amount of monitors that can be killed in each thrashing round. If set to true, this option will limit the amount of killable monitors up to (n/2-1). This means that if we are running a configuration that only has up to two configured monitors, if 'maintain-quorum' is set to true, this task won't run as there are no killable monitors -- in such a scenario, this option should be set to false. Furthermore, if 'store-thrash' is set to true, then 'maintain-quorum' must also be set to true, as we cannot let the task to thrash all the monitor stores, or we wouldn't be able to sync from other monitors, nor can we let quorum be dropped, or we won't be able to resync our way into quorum. Signed-off-by: Joao Eduardo Luis --- diff --git a/teuthology/task/mon_thrash.py b/teuthology/task/mon_thrash.py index 9ed48196d..bf2f9943d 100644 --- a/teuthology/task/mon_thrash.py +++ b/teuthology/task/mon_thrash.py @@ -5,6 +5,7 @@ import random import time import gevent import json +import math from teuthology import misc as teuthology log = logging.getLogger(__name__) @@ -37,6 +38,21 @@ class MonitorThrasher: being thrashed (default: False) store-thrash-probability Probability of thrashing a monitor's store (default: 50) + thrash-many Thrash multiple monitors instead of just one. If + 'maintain-quorum' is set to False, then we will + thrash up to as many monitors as there are + available. (default: False) + maintain-quorum Always maintain quorum, taking care on how many + monitors we kill during the thrashing. If we + happen to only have one or two monitors configured, + if this option is set to True, then we won't run + this task as we cannot guarantee maintenance of + quorum. Setting it to false however would allow the + task to run with as many as just one single monitor. + (default: True) + + Note: if 'store-thrash' is set to True, then 'maintain-quorum' must also + be set to True. For example:: @@ -48,6 +64,8 @@ class MonitorThrasher: store-thrash: true store-thrash-probability: 40 seed: 31337 + maintain-quorum: true + thrash-many: true - ceph-fuse: - workunit: clients: @@ -79,6 +97,12 @@ class MonitorThrasher: self.revive_delay = float(self.config.get('revive_delay', 10.0)) self.thrash_delay = float(self.config.get('thrash_delay', 0.0)) + self.thrash_many = self.config.get('thrash-many', False) + self.maintain_quorum = self.config.get('maintain-quorum', True) + + assert self.max_killable() > 0, \ + 'Unable to kill at least one monitor with the current config.' + """ Store thrashing """ self.store_thrash = self.config.get('store-thrash', False) self.store_thrash_probability = int( @@ -86,6 +110,8 @@ class MonitorThrasher: if self.store_thrash: assert self.store_thrash_probability > 0, \ 'store-thrash is set, probability must be > 0' + assert self.maintain_quorum, \ + 'store-thrash = true must imply maintain-quorum = true' self.thread = gevent.spawn(self.do_thrash) @@ -118,12 +144,20 @@ class MonitorThrasher: self.log('reviving mon.{id}'.format(id=mon)) self.manager.revive_mon(mon) + def max_killable(self): + m = len(_get_mons(self.ctx)) + if self.maintain_quorum: + return max(math.ceil(m/2.0)-1,0) + else: + return m def do_thrash(self): self.log('start thrashing') self.log('seed: {s}, revive delay: {r}, thrash delay: {t} '\ + 'thrash many: {tm}, maintain quorum: {mq} '\ 'store thrash: {st}, probability: {stp}'.format( s=self.random_seed,r=self.revive_delay,t=self.thrash_delay, + tm=self.thrash_many, mq=self.maintain_quorum, st=self.store_thrash,stp=self.store_thrash_probability)) while not self.stopping: @@ -135,27 +169,35 @@ class MonitorThrasher: assert s['state'] == 'leader' or s['state'] == 'peon' assert len(s['quorum']) == len(mons) - self.log('pick a monitor to kill') - to_kill = self.rng.choice(mons) + kill_up_to = self.rng.randrange(1, self.max_killable()+1) + mons_to_kill = self.rng.sample(mons, kill_up_to) + self.log('monitors to thrash: {m}'.format(m=mons_to_kill)) - if self.should_thrash_store(): - self.thrash_store(to_kill) + for mon in mons_to_kill: + self.log('thrashing mon.{m}'.format(m=mon)) - self.kill_mon(to_kill) + """ we only thrash stores if we are maintaining quorum """ + if self.should_thrash_store() and self.maintain_quorum: + self.thrash_store(mon) - self.manager.wait_for_mon_quorum_size(len(mons)-1) - for m in mons: - if m == to_kill: - continue - s = self.manager.get_mon_status(m) - assert s['state'] == 'leader' or s['state'] == 'peon' - assert len(s['quorum']) == len(mons)-1 + self.kill_mon(mon) + + if self.maintain_quorum: + self.manager.wait_for_mon_quorum_size(len(mons)-len(mons_to_kill)) + for m in mons: + if m in mons_to_kill: + continue + s = self.manager.get_mon_status(m) + assert s['state'] == 'leader' or s['state'] == 'peon' + assert len(s['quorum']) == len(mons)-len(mons_to_kill) - self.log('waiting for {delay} secs before reviving mon.{id}'.format( - delay=self.revive_delay,id=to_kill)) + self.log('waiting for {delay} secs before reviving monitors'.format( + delay=self.revive_delay)) time.sleep(self.revive_delay) - self.revive_mon(to_kill) + for mon in mons_to_kill: + self.revive_mon(mon) + self.manager.wait_for_mon_quorum_size(len(mons)) for m in mons: s = self.manager.get_mon_status(m)