qa/mgr/test_progress: fix bug 48217

Fixes a failing test case regarding osd coming back
after being marked out. The old test case wasn't accounting
for a specific event, therefore this resulted in the failure.
The fix basically accounts for a specific event of osd being
marked in/out.

Fixes: https://tracker.ceph.com/issues/48217
Signed-off-by: Kamoltat <ksirivad@redhat.com>
(cherry picked from commit 52fe9dbdae87b65a99a6c3bac3519c9528f22ea2)

Conflicts:
	qa/tasks/mgr/test_progress.py - trivial fix

diff --git a/qa/tasks/mgr/test_progress.py b/qa/tasks/mgr/test_progress.py
index bf2d559bbdddc8ec3b01e1c93a93bb2782c9cc27..8c06dd0e0da3b84b74ac64adbc3686065c9e2b8c 100644 (file)
--- a/qa/tasks/mgr/test_progress.py
+++ b/qa/tasks/mgr/test_progress.py
@@ -45,6 +45,65 @@ class TestProgress(MgrTestCase):
         log.info(json.dumps(p, indent=2))
         return p['events']
 
+    def _completed_events(self):
+        """
+        This function returns all events that are completed
+        """
+        p = self._get_progress()
+        log.info(json.dumps(p, indent=2))
+        return p['completed']
+
+    def is_osd_marked_out(self, ev):
+        return ev['message'].endswith('marked out')
+
+    def is_osd_marked_in(self, ev):
+        return ev['message'].endswith('marked in')
+
+    def _get_osd_in_out_events(self, marked='both'):
+        """
+        Return the event that deals with OSDs being
+        marked in, out or both
+        """
+
+        marked_in_events = []
+        marked_out_events = []
+
+        events_in_progress = self._events_in_progress()
+        for ev in events_in_progress:
+            if self.is_osd_marked_out(ev):
+                marked_out_events.append(ev)
+            elif self.is_osd_marked_in(ev):
+                marked_in_events.append(ev)
+
+        if marked == 'both':
+            return [marked_in_events] + [marked_out_events]
+        elif marked == 'in':
+            return marked_in_events
+        else:
+            return marked_out_events
+
+    def _osd_in_out_events_count(self, marked='both'):
+        """
+        Count the number of on going recovery events that deals with
+        OSDs being marked in, out or both.
+        """
+        events_in_progress = self._events_in_progress()
+        marked_in_count = 0
+        marked_out_count = 0
+
+        for ev in events_in_progress:
+            if self.is_osd_marked_out(ev):
+                marked_out_count += 1
+            elif self.is_osd_marked_in(ev):
+                marked_in_count += 1
+
+        if marked == 'both':
+            return marked_in_count + marked_out_count
+        elif marked == 'in':
+            return marked_in_count
+        else:
+            return marked_out_count
+
     def _setup_pool(self, size=None):
         self.mgr_cluster.mon_manager.create_pool(self.POOL)
         if size is not None:
@@ -106,9 +165,10 @@ class TestProgress(MgrTestCase):
                 'osd', 'out', str(osd_id))
 
         # Wait for a progress event to pop up
-        self.wait_until_equal(lambda: len(self._all_events()), 1,
-                              timeout=self.EVENT_CREATION_PERIOD)
-        ev = self._all_events()[0]
+        self.wait_until_equal(lambda: self._osd_in_out_events_count('out'), 1,
+                              timeout=self.EVENT_CREATION_PERIOD*2,
+                              period=1)
+        ev = self._get_osd_in_out_events('out')[0]
         log.info(json.dumps(ev, indent=1))
         self.assertIn("Rebalancing after osd.0 marked out", ev['message'])
         
@@ -123,16 +183,20 @@ class TestProgress(MgrTestCase):
         # First Event should complete promptly
         self.wait_until_true(lambda: self._is_complete(initial_event['id']),
                              timeout=self.EVENT_CREATION_PERIOD)
-
-            
-        # Wait for progress event marked in to pop up
-        self.wait_until_equal(lambda: len(self._events_in_progress()), 1,
-                              timeout=self.EVENT_CREATION_PERIOD)
-
-        new_event = self._all_events()[0]
-        log.info(json.dumps(new_event, indent=1))
-        self.assertIn("Rebalancing after osd.0 marked in", new_event['message'])    
-        
+       
+        try:
+            # Wait for progress event marked in to pop up
+            self.wait_until_equal(lambda: self._osd_in_out_events_count('in'), 1,
+                                  timeout=self.EVENT_CREATION_PERIOD*2,
+                                  period=1)
+        except RuntimeError as ex:
+            if not "Timed out after" in str(ex):
+                raise ex
+
+            log.info("There was no PGs affected by osd being marked in")
+            return None
+
+        new_event = self._get_osd_in_out_events('in')[0]
         return new_event
 
     def _no_events_anywhere(self):