osd_recovery: inject a recovery delay

Signed-off-by: Samuel Just <sam.just@inktank.com>

diff --git a/teuthology/task/ceph_manager.py b/teuthology/task/ceph_manager.py
index a2ff979aee9e04a2eb19324d8b1c6de08eb16071..1b96d1fac23b099a2594b3f3c1ad9498aa35df7d 100644 (file)
--- a/teuthology/task/ceph_manager.py
+++ b/teuthology/task/ceph_manager.py
@@ -308,6 +308,13 @@ class CephManager:
                 return int(i['pool'])
         assert False
 
+    def kick_recovery_wq(self, osdnum):
+        return self.raw_cluster_cmd(
+            'tell', "osd.%d" % (int(osdnum),),
+            'debug',
+            'kick_recovery_wq',
+            '0')
+
     def set_config(self, osdnum, **argdict):
         return self.raw_cluster_cmd(
             'tell', "osd.%d" % (int(osdnum),),

diff --git a/teuthology/task/osd_recovery.py b/teuthology/task/osd_recovery.py
index 591aa6e62c06c074c9620b97b827ab3f3a1d4d6d..e4c28a7563ef41a9d0c80484dccdc2d6f5f91ecd 100644 (file)
--- a/teuthology/task/osd_recovery.py
+++ b/teuthology/task/osd_recovery.py
@@ -135,6 +135,11 @@ def test_incomplete_pgs(ctx, config):
 
     log.info('Testing incomplete pgs...')
 
+    for i in range(4):
+        manager.set_config(
+            i,
+            osd_recovery_delay_start=1000)
+
     # move data off of osd.0, osd.1
     manager.raw_cluster_cmd('osd', 'out', '0', '1')
     manager.raw_cluster_cmd('tell', 'osd.0', 'flush_pg_stats')
@@ -186,5 +191,8 @@ def test_incomplete_pgs(ctx, config):
         time.sleep(2)
     log.info('all are up!')
 
+    for i in range(4):
+        manager.kick_recovery_wq(i)
+
     # cluster must recover
     manager.wait_for_clean()