repair_test.py: Additional test cases

Test repair with more than 1 damaged object and with different types of damage
Regression test for bug #4778

Signed-off-by: David Zafman <david.zafman@inktank.com>
Reviewed-by: Samuel Just <sam.just@inktank.com>

diff --git a/teuthology/task/repair_test.py b/teuthology/task/repair_test.py
index 49ba575b473ab92acc1d3f592660defab45a3dd4..1dd8f2fdefa6c04465d136225046b4f8cda40416 100644 (file)
--- a/teuthology/task/repair_test.py
+++ b/teuthology/task/repair_test.py
@@ -1,4 +1,5 @@
 import logging
+import time
 
 import ceph_manager
 from teuthology import misc as teuthology
@@ -55,9 +56,15 @@ def mdataerr(ctx):
             ['injectmdataerr', pool, obj])
     return ret
 
-def gen_repair_test(corrupter, chooser, scrub_type):
+def omaperr(ctx):
+    def ret(osd, pool, obj):
+        log.info("injecting omap err on object")
+        return ctx.manager.osd_admin_socket(osd, ['setomapval', pool, obj, 'badkey', 'badval']);
+    return ret
+
+def gen_repair_test_1(corrupter, chooser, scrub_type):
     def ret(ctx, config, pool):
-        log.info("starting repair test")
+        log.info("starting repair test type 1")
         victim_osd = chooser(pool, 0)
 
         # create object
@@ -86,10 +93,96 @@ def gen_repair_test(corrupter, chooser, scrub_type):
         log.info("done")
     return ret
 
+def gen_repair_test_2(chooser):
+    def ret(ctx, config, pool):
+        log.info("starting repair test type 2")
+        victim_osd = chooser(pool, 0)
+        first_mon = teuthology.get_first_mon(ctx, config)
+        (mon,) = ctx.cluster.only(first_mon).remotes.iterkeys()
+
+        # create object
+        log.info("doing put and setomapval")
+        ctx.manager.do_put(pool, 'file1', '/etc/hosts')
+        ctx.manager.do_rados(mon, ['-p', pool, 'setomapval', 'file1', 'key', 'val'])
+        ctx.manager.do_put(pool, 'file2', '/etc/hosts')
+        ctx.manager.do_put(pool, 'file3', '/etc/hosts')
+        ctx.manager.do_put(pool, 'file4', '/etc/hosts')
+        ctx.manager.do_put(pool, 'file5', '/etc/hosts')
+        ctx.manager.do_rados(mon, ['-p', pool, 'setomapval', 'file5', 'key', 'val'])
+        ctx.manager.do_put(pool, 'file6', '/etc/hosts')
+
+        # corrupt object
+        log.info("corrupting object")
+        omaperr(ctx)(victim_osd, pool, 'file1')
+
+        # verify inconsistent
+        log.info("scrubbing")
+        ctx.manager.do_pg_scrub(pool, 0, 'deep-scrub')
+
+        assert ctx.manager.pg_inconsistent(pool, 0)
+
+        # Regression test for bug #4778, should still
+        # be inconsistent after scrub
+        ctx.manager.do_pg_scrub(pool, 0, 'scrub')
+
+        assert ctx.manager.pg_inconsistent(pool, 0)
+
+        # Additional corruptions including 2 types for file1
+        log.info("corrupting more objects")
+        dataerr(ctx)(victim_osd, pool, 'file1')
+        mdataerr(ctx)(victim_osd, pool, 'file2')
+        trunc(ctx)(victim_osd, pool, 'file3')
+        omaperr(ctx)(victim_osd, pool, 'file6')
+
+        # see still inconsistent
+        log.info("scrubbing")
+        ctx.manager.do_pg_scrub(pool, 0, 'deep-scrub')
+
+        assert ctx.manager.pg_inconsistent(pool, 0)
+
+        # repair
+        log.info("repairing")
+        ctx.manager.do_pg_scrub(pool, 0, "repair")
+
+        # Let repair clear inconsistent flag
+        time.sleep(10)
+
+        # verify consistent
+        assert not ctx.manager.pg_inconsistent(pool, 0)
+
+        # In the future repair might determine state of
+        # inconsistency itself, verify with a deep-scrub
+        log.info("scrubbing")
+        ctx.manager.do_pg_scrub(pool, 0, 'deep-scrub')
+
+        # verify consistent
+        assert not ctx.manager.pg_inconsistent(pool, 0)
+
+        log.info("done")
+    return ret
+
 def task(ctx, config):
     """
     Test [deep] repair in several situations:
       Repair [Truncate, Data EIO, MData EIO] on [Primary|Replica]
+
+    The config should be as follows:
+
+      Must include the log-whitelist below
+      Must enable filestore_debug_inject_read_err config
+
+    example:
+
+    tasks:
+    - chef:
+    - install:
+    - ceph:
+        log-whitelist: ['candidate had a read error', 'deep-scrub 0 missing, 1 inconsistent objects', 'deep-scrub 0 missing, 4 inconsistent objects', 'deep-scrub 1 errors', 'deep-scrub 4 errors', '!= known omap_digest', 'repair 0 missing, 1 inconsistent objects', 'repair 0 missing, 4 inconsistent objects', 'repair 1 errors, 1 fixed', 'repair 4 errors, 4 fixed', 'scrub 0 missing, 1 inconsistent', 'scrub 1 errors', 'size 1 != known size']
+        conf:
+          osd:
+            filestore debug inject read err: true
+    - repair_test:
+
     """
     if config is None:
         config = {}
@@ -106,12 +199,14 @@ def task(ctx, config):
             )
 
     tests = [
-        gen_repair_test(mdataerr(ctx), choose_primary(ctx), "scrub"),
-        gen_repair_test(mdataerr(ctx), choose_replica(ctx), "scrub"),
-        gen_repair_test(dataerr(ctx), choose_primary(ctx), "deep-scrub"),
-        gen_repair_test(dataerr(ctx), choose_replica(ctx), "deep-scrub"),
-        gen_repair_test(trunc(ctx), choose_primary(ctx), "scrub"),
-        gen_repair_test(trunc(ctx), choose_replica(ctx), "scrub")
+        gen_repair_test_1(mdataerr(ctx), choose_primary(ctx), "scrub"),
+        gen_repair_test_1(mdataerr(ctx), choose_replica(ctx), "scrub"),
+        gen_repair_test_1(dataerr(ctx), choose_primary(ctx), "deep-scrub"),
+        gen_repair_test_1(dataerr(ctx), choose_replica(ctx), "deep-scrub"),
+        gen_repair_test_1(trunc(ctx), choose_primary(ctx), "scrub"),
+        gen_repair_test_1(trunc(ctx), choose_replica(ctx), "scrub"),
+        gen_repair_test_2(choose_primary(ctx)),
+        gen_repair_test_2(choose_replica(ctx))
         ]
 
     for test in tests: