From f82d4a7b86f49ca862f46e14135a0cf1f08a20f9 Mon Sep 17 00:00:00 2001 From: Samuel Just Date: Thu, 8 Nov 2012 16:22:40 -0800 Subject: [PATCH] Add divergent_priors test Tests scenario where merge_old_entry encounters a divergent entry where the prior_version is prior to log_tail. This is a problem since it will go into the missing set, but won't be re-added to the missing set during read_log() if the node restarts prior to recovering the object. Signed-off-by: Samuel Just --- teuthology/task/ceph_manager.py | 32 ++++++ teuthology/task/divergent_priors.py | 165 ++++++++++++++++++++++++++++ 2 files changed, 197 insertions(+) create mode 100644 teuthology/task/divergent_priors.py diff --git a/teuthology/task/ceph_manager.py b/teuthology/task/ceph_manager.py index 4266a870f669c..aa18d5b9798c5 100644 --- a/teuthology/task/ceph_manager.py +++ b/teuthology/task/ceph_manager.py @@ -185,6 +185,38 @@ class CephManager: ) return proc.stdout.getvalue() + def get_pg_primary(self, pool, pgnum): + """ + get primary for pool, pgnum (e.g. (data, 0)->0 + """ + poolnum = self.get_pool_num(pool) + output = self.raw_cluster_cmd("pg", "dump", '--format=json') + j = json.loads('\n'.join(output.split('\n')[1:])) + pg_str = "%d.%d" % (poolnum, pgnum) + for pg in j['pg_stats']: + if pg['pgid'] == pg_str: + return int(pg['acting'][0]) + assert False + + def get_pool_num(self, pool): + """ + get number for pool (e.g., data -> 2) + """ + out = self.raw_cluster_cmd('--', 'osd','dump','--format=json') + j = json.loads('\n'.join(out.split('\n')[1:])) + for i in j['pools']: + if i['pool_name'] == pool: + return int(i['pool']) + assert False + + def set_config(self, osdnum, **argdict): + return self.raw_cluster_cmd( + 'tell', "osd.%d"%(int(osdnum),), + 'injectargs', + " ".join( + [("--" + conf.replace("_", "-") + " " + str(val)) for (conf,val) in + argdict.iteritems()])) + def raw_cluster_status(self): return self.raw_cluster_cmd('-s') diff --git a/teuthology/task/divergent_priors.py b/teuthology/task/divergent_priors.py new file mode 100644 index 0000000000000..5a34f352d9d2b --- /dev/null +++ b/teuthology/task/divergent_priors.py @@ -0,0 +1,165 @@ +import logging +import ceph_manager +from teuthology import misc as teuthology +import time + + +log = logging.getLogger(__name__) + + +def rados(remote, cmd, wait=True): + log.info("rados %s" % ' '.join(cmd)) + pre = [ + 'LD_LIBRARY_PATH=/tmp/cephtest/binary/usr/local/lib', + '/tmp/cephtest/enable-coredump', + '/tmp/cephtest/binary/usr/local/bin/ceph-coverage', + '/tmp/cephtest/archive/coverage', + '/tmp/cephtest/binary/usr/local/bin/rados', + '-c', '/tmp/cephtest/ceph.conf', + ]; + pre.extend(cmd) + proc = remote.run( + args=pre, + check_status=False, + wait=wait + ) + if wait: + return proc.exitstatus + else: + return proc + +def task(ctx, config): + """ + Test handling of divergent entries with prior_version + prior to log_tail + + config: none + + Requires 3 osds. + """ + if config is None: + config = {} + assert isinstance(config, dict), \ + 'divergent_priors task only accepts a dict for configuration' + first_mon = teuthology.get_first_mon(ctx, config) + (mon,) = ctx.cluster.only(first_mon).remotes.iterkeys() + + manager = ceph_manager.CephManager( + mon, + ctx=ctx, + logger=log.getChild('ceph_manager'), + ) + ctx.manager = manager + + while len(manager.get_osd_status()['up']) < 3: + time.sleep(10) + manager.raw_cluster_cmd('tell', 'osd.0', 'flush_pg_stats') + manager.raw_cluster_cmd('tell', 'osd.1', 'flush_pg_stats') + manager.raw_cluster_cmd('tell', 'osd.2', 'flush_pg_stats') + manager.raw_cluster_cmd('osd', 'set', 'noout') + manager.raw_cluster_cmd('osd', 'set', 'noin') + manager.raw_cluster_cmd('osd', 'set', 'nodown') + manager.wait_for_clean() + + # something that is always there + dummyfile = '/etc/fstab' + dummyfile2 = '/etc/resolv.conf' + + # create 1 pg pool + log.info('creating foo') + manager.raw_cluster_cmd('osd', 'pool', 'create', 'foo', '1') + + osds = [0, 1, 2] + for i in osds: + manager.set_config(i, osd_min_pg_log_entries=1) + + # determine primary + divergent = manager.get_pg_primary('foo', 0) + log.info("primary and soon to be divergent is %d", divergent) + non_divergent = [0,1,2] + non_divergent.remove(divergent) + + log.info('writing initial objects') + # write 1000 objects + for i in range(1000): + rados(mon, ['-p', 'foo', 'put', 'existing_%d' % i, dummyfile]) + + manager.wait_for_clean() + + # blackhole non_divergent + log.info("blackholing osds %s", str(non_divergent)) + for i in non_divergent: + manager.set_config(i, filestore_blackhole='') + + # write 1 (divergent) object + log.info('writing divergent object existing_0') + rados( + mon, ['-p', 'foo', 'put', 'existing_0', dummyfile2], + wait=False) + time.sleep(10) + mon.run( + args=['killall', '-9', 'rados'], + wait=True, + check_status=False) + + # kill all the osds + log.info('killing all the osds') + for i in osds: + manager.kill_osd(i) + for i in osds: + manager.mark_down_osd(i) + for i in osds: + manager.mark_out_osd(i) + + # bring up non-divergent + log.info("bringing up non_divergent %s", str(non_divergent)) + for i in non_divergent: + manager.revive_osd(i) + for i in non_divergent: + manager.mark_in_osd(i) + + log.info('making log long to prevent backfill') + for i in non_divergent: + manager.set_config(i, osd_min_pg_log_entries=100000) + + # write 1 non-divergent object (ensure that old divergent one is divergent) + log.info('writing non-divergent object existing_1') + rados(mon, ['-p', 'foo', 'put', 'existing_1', dummyfile2]) + + manager.wait_for_recovery() + + # ensure no recovery + log.info('delay recovery') + for i in non_divergent: + manager.set_config(i, osd_recovery_delay_start=100000) + + # bring in our divergent friend + log.info("revive divergent %d", divergent) + manager.revive_osd(divergent) + + while len(manager.get_osd_status()['up']) < 3: + time.sleep(10) + + log.info('delay recovery divergent') + manager.set_config(divergent, osd_recovery_delay_start=100000) + log.info('mark divergent in') + manager.mark_in_osd(divergent) + + log.info('wait for peering') + rados(mon, ['-p', 'foo', 'put', 'foo', dummyfile]) + + log.info("killing divergent %d", divergent) + manager.kill_osd(divergent) + log.info("reviving divergent %d", divergent) + manager.revive_osd(divergent) + + log.info('allowing recovery') + for i in non_divergent: + manager.set_config(i, osd_recovery_delay_start=0) + + log.info('reading existing_0') + exit_status = rados(mon, + ['-p', 'foo', 'get', 'existing_0', + '-o', '/tmp/existing']) + assert exit_status is 0 + log.info("success") -- 2.39.5