From 6b8f1c6bcef53c15fcc928f253ab09c168449f44 Mon Sep 17 00:00:00 2001 From: David Zafman Date: Wed, 24 Apr 2013 15:50:28 -0700 Subject: [PATCH] repair_test.py: Additional test cases Test repair with more than 1 damaged object and with different types of damage Regression test for bug #4778 Signed-off-by: David Zafman Reviewed-by: Samuel Just --- teuthology/task/repair_test.py | 111 ++++++++++++++++++++++++++++++--- 1 file changed, 103 insertions(+), 8 deletions(-) diff --git a/teuthology/task/repair_test.py b/teuthology/task/repair_test.py index 49ba575b473ab..1dd8f2fdefa6c 100644 --- a/teuthology/task/repair_test.py +++ b/teuthology/task/repair_test.py @@ -1,4 +1,5 @@ import logging +import time import ceph_manager from teuthology import misc as teuthology @@ -55,9 +56,15 @@ def mdataerr(ctx): ['injectmdataerr', pool, obj]) return ret -def gen_repair_test(corrupter, chooser, scrub_type): +def omaperr(ctx): + def ret(osd, pool, obj): + log.info("injecting omap err on object") + return ctx.manager.osd_admin_socket(osd, ['setomapval', pool, obj, 'badkey', 'badval']); + return ret + +def gen_repair_test_1(corrupter, chooser, scrub_type): def ret(ctx, config, pool): - log.info("starting repair test") + log.info("starting repair test type 1") victim_osd = chooser(pool, 0) # create object @@ -86,10 +93,96 @@ def gen_repair_test(corrupter, chooser, scrub_type): log.info("done") return ret +def gen_repair_test_2(chooser): + def ret(ctx, config, pool): + log.info("starting repair test type 2") + victim_osd = chooser(pool, 0) + first_mon = teuthology.get_first_mon(ctx, config) + (mon,) = ctx.cluster.only(first_mon).remotes.iterkeys() + + # create object + log.info("doing put and setomapval") + ctx.manager.do_put(pool, 'file1', '/etc/hosts') + ctx.manager.do_rados(mon, ['-p', pool, 'setomapval', 'file1', 'key', 'val']) + ctx.manager.do_put(pool, 'file2', '/etc/hosts') + ctx.manager.do_put(pool, 'file3', '/etc/hosts') + ctx.manager.do_put(pool, 'file4', '/etc/hosts') + ctx.manager.do_put(pool, 'file5', '/etc/hosts') + ctx.manager.do_rados(mon, ['-p', pool, 'setomapval', 'file5', 'key', 'val']) + ctx.manager.do_put(pool, 'file6', '/etc/hosts') + + # corrupt object + log.info("corrupting object") + omaperr(ctx)(victim_osd, pool, 'file1') + + # verify inconsistent + log.info("scrubbing") + ctx.manager.do_pg_scrub(pool, 0, 'deep-scrub') + + assert ctx.manager.pg_inconsistent(pool, 0) + + # Regression test for bug #4778, should still + # be inconsistent after scrub + ctx.manager.do_pg_scrub(pool, 0, 'scrub') + + assert ctx.manager.pg_inconsistent(pool, 0) + + # Additional corruptions including 2 types for file1 + log.info("corrupting more objects") + dataerr(ctx)(victim_osd, pool, 'file1') + mdataerr(ctx)(victim_osd, pool, 'file2') + trunc(ctx)(victim_osd, pool, 'file3') + omaperr(ctx)(victim_osd, pool, 'file6') + + # see still inconsistent + log.info("scrubbing") + ctx.manager.do_pg_scrub(pool, 0, 'deep-scrub') + + assert ctx.manager.pg_inconsistent(pool, 0) + + # repair + log.info("repairing") + ctx.manager.do_pg_scrub(pool, 0, "repair") + + # Let repair clear inconsistent flag + time.sleep(10) + + # verify consistent + assert not ctx.manager.pg_inconsistent(pool, 0) + + # In the future repair might determine state of + # inconsistency itself, verify with a deep-scrub + log.info("scrubbing") + ctx.manager.do_pg_scrub(pool, 0, 'deep-scrub') + + # verify consistent + assert not ctx.manager.pg_inconsistent(pool, 0) + + log.info("done") + return ret + def task(ctx, config): """ Test [deep] repair in several situations: Repair [Truncate, Data EIO, MData EIO] on [Primary|Replica] + + The config should be as follows: + + Must include the log-whitelist below + Must enable filestore_debug_inject_read_err config + + example: + + tasks: + - chef: + - install: + - ceph: + log-whitelist: ['candidate had a read error', 'deep-scrub 0 missing, 1 inconsistent objects', 'deep-scrub 0 missing, 4 inconsistent objects', 'deep-scrub 1 errors', 'deep-scrub 4 errors', '!= known omap_digest', 'repair 0 missing, 1 inconsistent objects', 'repair 0 missing, 4 inconsistent objects', 'repair 1 errors, 1 fixed', 'repair 4 errors, 4 fixed', 'scrub 0 missing, 1 inconsistent', 'scrub 1 errors', 'size 1 != known size'] + conf: + osd: + filestore debug inject read err: true + - repair_test: + """ if config is None: config = {} @@ -106,12 +199,14 @@ def task(ctx, config): ) tests = [ - gen_repair_test(mdataerr(ctx), choose_primary(ctx), "scrub"), - gen_repair_test(mdataerr(ctx), choose_replica(ctx), "scrub"), - gen_repair_test(dataerr(ctx), choose_primary(ctx), "deep-scrub"), - gen_repair_test(dataerr(ctx), choose_replica(ctx), "deep-scrub"), - gen_repair_test(trunc(ctx), choose_primary(ctx), "scrub"), - gen_repair_test(trunc(ctx), choose_replica(ctx), "scrub") + gen_repair_test_1(mdataerr(ctx), choose_primary(ctx), "scrub"), + gen_repair_test_1(mdataerr(ctx), choose_replica(ctx), "scrub"), + gen_repair_test_1(dataerr(ctx), choose_primary(ctx), "deep-scrub"), + gen_repair_test_1(dataerr(ctx), choose_replica(ctx), "deep-scrub"), + gen_repair_test_1(trunc(ctx), choose_primary(ctx), "scrub"), + gen_repair_test_1(trunc(ctx), choose_replica(ctx), "scrub"), + gen_repair_test_2(choose_primary(ctx)), + gen_repair_test_2(choose_replica(ctx)) ] for test in tests: -- 2.39.5