mgr/progress: introduce turn off/on feature

progress module can be turned off/on by using
the commands: 'progress off' and 'progress on'

As well as refractoring teuthology test suite
to prevent future bugs that can possibly occur

fixes: https://tracker.ceph.com/issues/47238

Signed-off-by: kamoltat <ksirivad@redhat.com>
(cherry picked from commit 993bb02b30cf73a1c1c70da1ef266be8373d56dd)

 Conflicts:
	PendingReleaseNotes - add release notes about this feature
	qa/tasks/mgr/test_progress.py - replace helper functions that is neeeded
					for dealing with more than 1 type of events
				      - remove `period` in wait_until_equal()
	src/pybind/mgr/progress/module.py - remove code that deals with mypy in master
	qa/suites/rados/singleton/all/pg-autoscaler-progress-off.yaml - remove log-ignorelist

update some files

Signed-off-by: kamoltat <ksirivad@redhat.com>

diff --git a/PendingReleaseNotes b/PendingReleaseNotes
index e8ecd9cf32df8f551bae3861043e1b6112c6933c..b626ee5fd19343782531b6013b697546de007008 100644 (file)
--- a/PendingReleaseNotes
+++ b/PendingReleaseNotes
@@ -1,3 +1,9 @@
+14.2.15
+-------
+
+* MGR: progress module can now be turned on/off, using the commands:
+  ``ceph progress on`` and ``ceph progress off``.
+
 14.2.13
 -------
 

diff --git a/qa/suites/rados/singleton/all/pg-autoscaler-progress-off.yaml b/qa/suites/rados/singleton/all/pg-autoscaler-progress-off.yaml
new file mode 100644 (file)
index 0000000..2784b7e
--- /dev/null
+++ b/qa/suites/rados/singleton/all/pg-autoscaler-progress-off.yaml
@@ -0,0 +1,42 @@
+roles:
+- - mon.a
+  - mgr.x
+  - osd.0
+  - osd.1
+  - osd.2
+  - osd.3
+  - client.0
+- - mon.b
+  - mon.c
+  - osd.4
+  - osd.5
+  - osd.6
+  - osd.7
+openstack:
+  - volumes: # attached to each instance
+      count: 4
+      size: 10 # GB
+tasks:
+- install:
+- ceph:
+    create_rbd_pool: false
+    log-whitelist:
+      - overall HEALTH_
+      - \(OSDMAP_FLAGS\)
+      - \(OSD_
+      - \(PG_
+      - \(POOL_
+      - \(CACHE_POOL_
+      - \(OBJECT_
+      - \(SLOW_OPS\)
+      - \(REQUEST_SLOW\)
+      - \(TOO_FEW_PGS\)
+      - slow request
+- exec:
+    client.0:
+    - ceph progress off
+
+- workunit:
+    clients:
+      all:
+        - mon/pg_autoscaler.sh

diff --git a/qa/tasks/mgr/test_progress.py b/qa/tasks/mgr/test_progress.py
index dad329a06f39886adb5a1e634685d9f762776bed..4cd881b5c5675c851f7a79e0f545189eb521c8d4 100644 (file)
--- a/qa/tasks/mgr/test_progress.py
+++ b/qa/tasks/mgr/test_progress.py
@@ -106,6 +106,14 @@ class TestProgress(MgrTestCase):
 
         return ev
 
+    def _no_events_anywhere(self):
+        """
+        Whether there are any live or completed events
+        """
+        p = self._get_progress()
+        total_events = len(p['events']) + len(p['completed'])
+        return total_events == 0
+
     def _is_quiet(self):
         """
         Whether any progress events are live.
@@ -224,3 +232,49 @@ class TestProgress(MgrTestCase):
         time.sleep(self.EVENT_CREATION_PERIOD)
 
         self.assertEqual(len(self._all_events()), osd_count - pool_size)
+
+    def test_turn_off_module(self):
+        """
+        When the the module is turned off, there should not
+        be any on going events or completed events.
+        Also module should not accept any kind of Remote Event
+        coming in from other module, however, once it is turned
+        back, on creating an event should be working as it is.
+        """
+
+        pool_size = 3
+        self._setup_pool(size=pool_size)
+        self._write_some_data(self.WRITE_PERIOD)
+
+        self.mgr_cluster.mon_manager.raw_cluster_cmd("progress", "off")
+
+        self.mgr_cluster.mon_manager.raw_cluster_cmd(
+                'osd', 'out', '0')
+
+        time.sleep(self.EVENT_CREATION_PERIOD)
+
+        self.mgr_cluster.mon_manager.raw_cluster_cmd(
+                    'osd', 'in', '0')
+
+        time.sleep(self.EVENT_CREATION_PERIOD)
+
+        self.assertTrue(self._is_quiet())
+
+        self.mgr_cluster.mon_manager.raw_cluster_cmd("progress", "on")
+
+        self._write_some_data(self.WRITE_PERIOD)
+
+        self.mgr_cluster.mon_manager.raw_cluster_cmd(
+                'osd', 'out', '0')
+
+        # Wait for a progress event to pop up
+        self.wait_until_equal(lambda: len(self._all_events()), 1,
+                              timeout=self.EVENT_CREATION_PERIOD*2)
+
+        ev = self._all_events()[0]
+
+        log.info(json.dumps(ev, indent=1))
+
+        self.wait_until_true(lambda: self._is_complete(ev['id']),
+                             timeout=self.RECOVERY_PERIOD)
+        self.assertTrue(self._is_quiet())

diff --git a/src/pybind/mgr/progress/module.py b/src/pybind/mgr/progress/module.py
index 85ed2f0269fc19c488854dbc174ca67504ee84a7..adee58749a11e25e283f758952d98b271de23fa4 100644 (file)
--- a/src/pybind/mgr/progress/module.py
+++ b/src/pybind/mgr/progress/module.py
@@ -304,7 +304,14 @@ class Module(MgrModule):
          "perm": "r"},
         {"cmd": "progress clear",
          "desc": "Reset progress tracking",
+         "perm": "rw"},
+        {"cmd": "progress on",
+         "desc": "Enable progress tracking",
+         "perm": "rw"},
+        {"cmd": "progress off",
+         "desc": "Disable progress tracking",
          "perm": "rw"}
+
     ]
 
     MODULE_OPTIONS = [
@@ -322,6 +329,11 @@ class Module(MgrModule):
             'desc': 'how frequently to persist completed events',
             'runtime': True,
         },
+        {
+            'name': 'enabled',
+            'default': True,
+            'type': 'bool',
+        }
     ]
 
     def __init__(self, *args, **kwargs):
@@ -444,7 +456,8 @@ class Module(MgrModule):
 
     def notify(self, notify_type, notify_data):
         self._ready.wait()
-
+        if not self.enabled:
+            return
         if notify_type == "osd_map":
             old_osdmap = self._latest_osdmap
             self._latest_osdmap = self.get_osdmap()
@@ -454,7 +467,7 @@ class Module(MgrModule):
             ))
             self._osdmap_changed(old_osdmap, self._latest_osdmap)
         elif notify_type == "pg_summary":
-            # if there are no events we will skip this here to avoid 
+            # if there are no events we will skip this here to avoid
             # expensive get calls
             if len(self._events) == 0:
                 return
@@ -538,6 +551,9 @@ class Module(MgrModule):
         """
         For calling from other mgr modules
         """
+        if not self.enabled:
+            return
+
         if refs is None:
             refs = []
         try:
@@ -572,6 +588,8 @@ class Module(MgrModule):
         """
         For calling from other mgr modules
         """
+        if not self.enabled:
+            return
         try:
             ev = self._events[ev_id]
             ev.set_progress(1.0)
@@ -597,6 +615,12 @@ class Module(MgrModule):
         except KeyError:
             self.log.warn("fail: ev {0} does not exist".format(ev_id))
 
+    def on(self):
+        self.set_module_option('enabled', True)
+
+    def off(self):
+        self.set_module_option('enabled', False)
+
     def _handle_ls(self):
         if len(self._events) or len(self._completed_events):
             out = ""
@@ -623,12 +647,14 @@ class Module(MgrModule):
             'completed': [ev.to_json() for ev in self._completed_events]
         }
 
-    def _handle_clear(self):
+    def clear(self):
         self._events = {}
         self._completed_events = []
         self._dirty = True
         self._save()
 
+    def _handle_clear(self):
+        self.clear()
         return 0, "", ""
 
     def handle_command(self, _, cmd):
@@ -641,6 +667,17 @@ class Module(MgrModule):
             # that never finishes)
             return self._handle_clear()
         elif cmd['prefix'] == "progress json":
-            return 0, json.dumps(self._json(), indent=2), ""
+            return 0, json.dumps(self._json(), indent=4, sort_keys=True), ""
+        elif cmd['prefix'] == "progress on":
+            if self.enabled:
+                return 0, "", "progress already enabled!"
+            self.on()
+            return 0, "", "progress enabled"
+        elif cmd['prefix'] == "progress off":
+            if not self.enabled:
+                return 0, "", "progress already disabled!"
+            self.off()
+            self.clear()
+            return 0, "", "progress disabled"
         else:
             raise NotImplementedError(cmd['prefix'])