From d7d1912b4b6a84010d43919eef907dd3a118d4ff Mon Sep 17 00:00:00 2001 From: Kamoltat Date: Mon, 16 Nov 2020 08:52:12 +0000 Subject: [PATCH] qa/mgr/test_progress: fix bug 48217 Fixes a failing test case regarding osd coming back after being marked out. The old test case wasn't accounting for a specific event, therefore this resulted in the failure. The fix basically accounts for a specific event of osd being marked in/out. Fixes: https://tracker.ceph.com/issues/48217 Signed-off-by: Kamoltat (cherry picked from commit 52fe9dbdae87b65a99a6c3bac3519c9528f22ea2) Conflicts: qa/tasks/mgr/test_progress.py - trivial fix --- qa/tasks/mgr/test_progress.py | 90 ++++++++++++++++++++++++++++++----- 1 file changed, 77 insertions(+), 13 deletions(-) diff --git a/qa/tasks/mgr/test_progress.py b/qa/tasks/mgr/test_progress.py index bf2d559bbddd..8c06dd0e0da3 100644 --- a/qa/tasks/mgr/test_progress.py +++ b/qa/tasks/mgr/test_progress.py @@ -45,6 +45,65 @@ class TestProgress(MgrTestCase): log.info(json.dumps(p, indent=2)) return p['events'] + def _completed_events(self): + """ + This function returns all events that are completed + """ + p = self._get_progress() + log.info(json.dumps(p, indent=2)) + return p['completed'] + + def is_osd_marked_out(self, ev): + return ev['message'].endswith('marked out') + + def is_osd_marked_in(self, ev): + return ev['message'].endswith('marked in') + + def _get_osd_in_out_events(self, marked='both'): + """ + Return the event that deals with OSDs being + marked in, out or both + """ + + marked_in_events = [] + marked_out_events = [] + + events_in_progress = self._events_in_progress() + for ev in events_in_progress: + if self.is_osd_marked_out(ev): + marked_out_events.append(ev) + elif self.is_osd_marked_in(ev): + marked_in_events.append(ev) + + if marked == 'both': + return [marked_in_events] + [marked_out_events] + elif marked == 'in': + return marked_in_events + else: + return marked_out_events + + def _osd_in_out_events_count(self, marked='both'): + """ + Count the number of on going recovery events that deals with + OSDs being marked in, out or both. + """ + events_in_progress = self._events_in_progress() + marked_in_count = 0 + marked_out_count = 0 + + for ev in events_in_progress: + if self.is_osd_marked_out(ev): + marked_out_count += 1 + elif self.is_osd_marked_in(ev): + marked_in_count += 1 + + if marked == 'both': + return marked_in_count + marked_out_count + elif marked == 'in': + return marked_in_count + else: + return marked_out_count + def _setup_pool(self, size=None): self.mgr_cluster.mon_manager.create_pool(self.POOL) if size is not None: @@ -106,9 +165,10 @@ class TestProgress(MgrTestCase): 'osd', 'out', str(osd_id)) # Wait for a progress event to pop up - self.wait_until_equal(lambda: len(self._all_events()), 1, - timeout=self.EVENT_CREATION_PERIOD) - ev = self._all_events()[0] + self.wait_until_equal(lambda: self._osd_in_out_events_count('out'), 1, + timeout=self.EVENT_CREATION_PERIOD*2, + period=1) + ev = self._get_osd_in_out_events('out')[0] log.info(json.dumps(ev, indent=1)) self.assertIn("Rebalancing after osd.0 marked out", ev['message']) @@ -123,16 +183,20 @@ class TestProgress(MgrTestCase): # First Event should complete promptly self.wait_until_true(lambda: self._is_complete(initial_event['id']), timeout=self.EVENT_CREATION_PERIOD) - - - # Wait for progress event marked in to pop up - self.wait_until_equal(lambda: len(self._events_in_progress()), 1, - timeout=self.EVENT_CREATION_PERIOD) - - new_event = self._all_events()[0] - log.info(json.dumps(new_event, indent=1)) - self.assertIn("Rebalancing after osd.0 marked in", new_event['message']) - + + try: + # Wait for progress event marked in to pop up + self.wait_until_equal(lambda: self._osd_in_out_events_count('in'), 1, + timeout=self.EVENT_CREATION_PERIOD*2, + period=1) + except RuntimeError as ex: + if not "Timed out after" in str(ex): + raise ex + + log.info("There was no PGs affected by osd being marked in") + return None + + new_event = self._get_osd_in_out_events('in')[0] return new_event def _no_events_anywhere(self): -- 2.47.3