From: Sage Weil <sage@redhat.com>
Date: Mon, 20 Jan 2020 15:45:00 +0000 (-0600)
Subject: qa/tasks/ceph_manager: fix post-osd-kill pg peered check
X-Git-Tag: v15.1.0~145^2
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=refs%2Fpull%2F32737%2Fhead;p=ceph.git

qa/tasks/ceph_manager: fix post-osd-kill pg peered check

This was asserting that all PGs are active or peered, but that assertion
could fail if the concurrent workload created a new pool.

Switch to a loop that checks several times for the condition to be true.

Fixes: https://tracker.ceph.com/issues/43656
Signed-off-by: Sage Weil <sage@redhat.com>
---

diff --git a/qa/tasks/ceph_manager.py b/qa/tasks/ceph_manager.py
index f83b5d58a796..d7257c80b4f8 100644
--- a/qa/tasks/ceph_manager.py
+++ b/qa/tasks/ceph_manager.py
@@ -842,9 +842,16 @@ class OSDThrasher(Thrasher):
                     self.log("chose to kill {n} OSDs".format(n=most_killable))
                     for i in range(1, most_killable):
                         self.kill_osd(mark_out=True)
-                    time.sleep(15)
-                    assert self.ceph_manager.all_active_or_peered(), \
-                            'not all PGs are active or peered 15 seconds after marking out OSDs'
+                    time.sleep(10)
+                    # try a few times since there might be a concurrent pool
+                    # creation or deletion
+                    with safe_while(
+                            sleep=5, tries=5,
+                            action='check for active or peered') as proceed:
+                        while proceed():
+                            if self.ceph_manager.all_active_or_peered():
+                                break
+                            self.log('not all PGs are active or peered')
                 else: # chose to revive OSDs, bring up a random fraction of the dead ones
                     self.log("chose to revive osds")
                     for i in range(1, int(rand_val * len(self.dead_osds))):