From: Kamoltat Date: Wed, 30 Sep 2020 05:44:23 +0000 (+0000) Subject: mgr/progress: introduce turn off/on feature X-Git-Tag: v14.2.15~2^2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=c847a2f6a1bdb9ce5cc747fbb75a4cd660d17cca;p=ceph.git mgr/progress: introduce turn off/on feature progress module can be turned off/on by using the commands: 'progress off' and 'progress on' As well as refractoring teuthology test suite to prevent future bugs that can possibly occur fixes: https://tracker.ceph.com/issues/47238 Signed-off-by: kamoltat (cherry picked from commit 993bb02b30cf73a1c1c70da1ef266be8373d56dd) Conflicts: PendingReleaseNotes - add release notes about this feature qa/tasks/mgr/test_progress.py - replace helper functions that is neeeded for dealing with more than 1 type of events - remove `period` in wait_until_equal() src/pybind/mgr/progress/module.py - remove code that deals with mypy in master qa/suites/rados/singleton/all/pg-autoscaler-progress-off.yaml - remove log-ignorelist update some files Signed-off-by: kamoltat --- diff --git a/PendingReleaseNotes b/PendingReleaseNotes index e8ecd9cf32df..b626ee5fd193 100644 --- a/PendingReleaseNotes +++ b/PendingReleaseNotes @@ -1,3 +1,9 @@ +14.2.15 +------- + +* MGR: progress module can now be turned on/off, using the commands: + ``ceph progress on`` and ``ceph progress off``. + 14.2.13 ------- diff --git a/qa/suites/rados/singleton/all/pg-autoscaler-progress-off.yaml b/qa/suites/rados/singleton/all/pg-autoscaler-progress-off.yaml new file mode 100644 index 000000000000..2784b7e35c3c --- /dev/null +++ b/qa/suites/rados/singleton/all/pg-autoscaler-progress-off.yaml @@ -0,0 +1,42 @@ +roles: +- - mon.a + - mgr.x + - osd.0 + - osd.1 + - osd.2 + - osd.3 + - client.0 +- - mon.b + - mon.c + - osd.4 + - osd.5 + - osd.6 + - osd.7 +openstack: + - volumes: # attached to each instance + count: 4 + size: 10 # GB +tasks: +- install: +- ceph: + create_rbd_pool: false + log-whitelist: + - overall HEALTH_ + - \(OSDMAP_FLAGS\) + - \(OSD_ + - \(PG_ + - \(POOL_ + - \(CACHE_POOL_ + - \(OBJECT_ + - \(SLOW_OPS\) + - \(REQUEST_SLOW\) + - \(TOO_FEW_PGS\) + - slow request +- exec: + client.0: + - ceph progress off + +- workunit: + clients: + all: + - mon/pg_autoscaler.sh diff --git a/qa/tasks/mgr/test_progress.py b/qa/tasks/mgr/test_progress.py index dad329a06f39..4cd881b5c567 100644 --- a/qa/tasks/mgr/test_progress.py +++ b/qa/tasks/mgr/test_progress.py @@ -106,6 +106,14 @@ class TestProgress(MgrTestCase): return ev + def _no_events_anywhere(self): + """ + Whether there are any live or completed events + """ + p = self._get_progress() + total_events = len(p['events']) + len(p['completed']) + return total_events == 0 + def _is_quiet(self): """ Whether any progress events are live. @@ -224,3 +232,49 @@ class TestProgress(MgrTestCase): time.sleep(self.EVENT_CREATION_PERIOD) self.assertEqual(len(self._all_events()), osd_count - pool_size) + + def test_turn_off_module(self): + """ + When the the module is turned off, there should not + be any on going events or completed events. + Also module should not accept any kind of Remote Event + coming in from other module, however, once it is turned + back, on creating an event should be working as it is. + """ + + pool_size = 3 + self._setup_pool(size=pool_size) + self._write_some_data(self.WRITE_PERIOD) + + self.mgr_cluster.mon_manager.raw_cluster_cmd("progress", "off") + + self.mgr_cluster.mon_manager.raw_cluster_cmd( + 'osd', 'out', '0') + + time.sleep(self.EVENT_CREATION_PERIOD) + + self.mgr_cluster.mon_manager.raw_cluster_cmd( + 'osd', 'in', '0') + + time.sleep(self.EVENT_CREATION_PERIOD) + + self.assertTrue(self._is_quiet()) + + self.mgr_cluster.mon_manager.raw_cluster_cmd("progress", "on") + + self._write_some_data(self.WRITE_PERIOD) + + self.mgr_cluster.mon_manager.raw_cluster_cmd( + 'osd', 'out', '0') + + # Wait for a progress event to pop up + self.wait_until_equal(lambda: len(self._all_events()), 1, + timeout=self.EVENT_CREATION_PERIOD*2) + + ev = self._all_events()[0] + + log.info(json.dumps(ev, indent=1)) + + self.wait_until_true(lambda: self._is_complete(ev['id']), + timeout=self.RECOVERY_PERIOD) + self.assertTrue(self._is_quiet()) diff --git a/src/pybind/mgr/progress/module.py b/src/pybind/mgr/progress/module.py index 85ed2f0269fc..adee58749a11 100644 --- a/src/pybind/mgr/progress/module.py +++ b/src/pybind/mgr/progress/module.py @@ -304,7 +304,14 @@ class Module(MgrModule): "perm": "r"}, {"cmd": "progress clear", "desc": "Reset progress tracking", + "perm": "rw"}, + {"cmd": "progress on", + "desc": "Enable progress tracking", + "perm": "rw"}, + {"cmd": "progress off", + "desc": "Disable progress tracking", "perm": "rw"} + ] MODULE_OPTIONS = [ @@ -322,6 +329,11 @@ class Module(MgrModule): 'desc': 'how frequently to persist completed events', 'runtime': True, }, + { + 'name': 'enabled', + 'default': True, + 'type': 'bool', + } ] def __init__(self, *args, **kwargs): @@ -444,7 +456,8 @@ class Module(MgrModule): def notify(self, notify_type, notify_data): self._ready.wait() - + if not self.enabled: + return if notify_type == "osd_map": old_osdmap = self._latest_osdmap self._latest_osdmap = self.get_osdmap() @@ -454,7 +467,7 @@ class Module(MgrModule): )) self._osdmap_changed(old_osdmap, self._latest_osdmap) elif notify_type == "pg_summary": - # if there are no events we will skip this here to avoid + # if there are no events we will skip this here to avoid # expensive get calls if len(self._events) == 0: return @@ -538,6 +551,9 @@ class Module(MgrModule): """ For calling from other mgr modules """ + if not self.enabled: + return + if refs is None: refs = [] try: @@ -572,6 +588,8 @@ class Module(MgrModule): """ For calling from other mgr modules """ + if not self.enabled: + return try: ev = self._events[ev_id] ev.set_progress(1.0) @@ -597,6 +615,12 @@ class Module(MgrModule): except KeyError: self.log.warn("fail: ev {0} does not exist".format(ev_id)) + def on(self): + self.set_module_option('enabled', True) + + def off(self): + self.set_module_option('enabled', False) + def _handle_ls(self): if len(self._events) or len(self._completed_events): out = "" @@ -623,12 +647,14 @@ class Module(MgrModule): 'completed': [ev.to_json() for ev in self._completed_events] } - def _handle_clear(self): + def clear(self): self._events = {} self._completed_events = [] self._dirty = True self._save() + def _handle_clear(self): + self.clear() return 0, "", "" def handle_command(self, _, cmd): @@ -641,6 +667,17 @@ class Module(MgrModule): # that never finishes) return self._handle_clear() elif cmd['prefix'] == "progress json": - return 0, json.dumps(self._json(), indent=2), "" + return 0, json.dumps(self._json(), indent=4, sort_keys=True), "" + elif cmd['prefix'] == "progress on": + if self.enabled: + return 0, "", "progress already enabled!" + self.on() + return 0, "", "progress enabled" + elif cmd['prefix'] == "progress off": + if not self.enabled: + return 0, "", "progress already disabled!" + self.off() + self.clear() + return 0, "", "progress disabled" else: raise NotImplementedError(cmd['prefix'])