dump_stuck: fix race with osd start

Occasionally we don't wait long enough for the osd to start and
mark itself up.  Keep trying until flush succeeds.

Fixes: #5431
Signed-off-by: Sage Weil <sage@inktank.com>

diff --git a/teuthology/task/dump_stuck.py b/teuthology/task/dump_stuck.py
index d6fd1b0987d659e94650230565d613119acf038d..d7dbf7dafbbe97e9a7930718c56feaba42819ca6 100644 (file)
--- a/teuthology/task/dump_stuck.py
+++ b/teuthology/task/dump_stuck.py
@@ -112,9 +112,14 @@ def task(ctx, config):
     for id_ in teuthology.all_roles_of_type(ctx.cluster, 'osd'):
         manager.revive_osd(id_)
         manager.mark_in_osd(id_)
-    time.sleep(timeout)
-    manager.raw_cluster_cmd('tell', 'osd.0', 'flush_pg_stats')
-    manager.raw_cluster_cmd('tell', 'osd.1', 'flush_pg_stats')
+    while True:
+        try:
+            manager.raw_cluster_cmd('tell', 'osd.0', 'flush_pg_stats')
+            manager.raw_cluster_cmd('tell', 'osd.1', 'flush_pg_stats')
+            break
+        except:
+            log.debug('osds must not be started yet, waiting...')
+            time.sleep(1)
     manager.wait_for_clean(timeout)
 
     check_stuck(