From: Pere Diaz Bou Date: Thu, 15 Apr 2021 07:55:24 +0000 (+0200) Subject: mgr/dashboard/rbd_mirroring: test and refactor get_daemons_and_pools X-Git-Tag: v17.1.0~2066^2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=6ee60349d198119d546e87c2baae72c0e7d29849;p=ceph.git mgr/dashboard/rbd_mirroring: test and refactor get_daemons_and_pools Signed-off-by: Pere Diaz Bou --- diff --git a/src/pybind/mgr/dashboard/controllers/rbd_mirroring.py b/src/pybind/mgr/dashboard/controllers/rbd_mirroring.py index 0c41d71375c..832c97fda97 100644 --- a/src/pybind/mgr/dashboard/controllers/rbd_mirroring.py +++ b/src/pybind/mgr/dashboard/controllers/rbd_mirroring.py @@ -45,146 +45,160 @@ def RbdMirroringTask(name, metadata, wait_for): # noqa: N802 return composed_decorator -@ViewCache() -def get_daemons_and_pools(): # pylint: disable=R0915 - def get_daemons(): - daemons = [] - for hostname, server in CephService.get_service_map('rbd-mirror').items(): - for service in server['services']: - id = service['id'] # pylint: disable=W0622 - metadata = service['metadata'] - status = service['status'] or {} - - try: - status = json.loads(status['json']) - except (ValueError, KeyError): - status = {} - - instance_id = metadata['instance_id'] - if id == instance_id: - # new version that supports per-cluster leader elections - id = metadata['id'] - - # extract per-daemon service data and health - daemon = { - 'id': id, - 'instance_id': instance_id, - 'version': metadata['ceph_version'], - 'server_hostname': hostname, - 'service': service, - 'server': server, - 'metadata': metadata, - 'status': status - } - daemon = dict(daemon, **get_daemon_health(daemon)) - daemons.append(daemon) - - return sorted(daemons, key=lambda k: k['instance_id']) - - def get_daemon_health(daemon): - health = { - 'health_color': 'info', - 'health': 'Unknown' - } - for _, pool_data in daemon['status'].items(): - if (health['health'] != 'error' - and [k for k, v in pool_data.get('callouts', {}).items() - if v['level'] == 'error']): - health = { - 'health_color': 'error', - 'health': 'Error' - } - elif (health['health'] != 'error' - and [k for k, v in pool_data.get('callouts', {}).items() - if v['level'] == 'warning']): - health = { - 'health_color': 'warning', - 'health': 'Warning' - } - elif health['health_color'] == 'info': - health = { - 'health_color': 'success', - 'health': 'OK' - } - return health - - def get_pools(daemons): # pylint: disable=R0912, R0915 - pool_names = [pool['pool_name'] for pool in CephService.get_pool_list('rbd') - if pool.get('type', 1) == 1] - pool_stats = {} - rbdctx = rbd.RBD() - for pool_name in pool_names: - logger.debug("Constructing IOCtx %s", pool_name) - try: - ioctx = mgr.rados.open_ioctx(pool_name) - except TypeError: - logger.exception("Failed to open pool %s", pool_name) - continue +def get_daemons(): + daemons = [] + for hostname, server in CephService.get_service_map('rbd-mirror').items(): + for service in server['services']: + id = service['id'] # pylint: disable=W0622 + metadata = service['metadata'] + status = service['status'] or {} try: - mirror_mode = rbdctx.mirror_mode_get(ioctx) - peer_uuids = [x['uuid'] for x in rbdctx.mirror_peer_list(ioctx)] - except: # noqa pylint: disable=W0702 - logger.exception("Failed to query mirror settings %s", pool_name) - mirror_mode = None - peer_uuids = [] - - stats = {} - if mirror_mode == rbd.RBD_MIRROR_MODE_DISABLED: - mirror_mode = "disabled" - stats['health_color'] = "info" - stats['health'] = "Disabled" - elif mirror_mode == rbd.RBD_MIRROR_MODE_IMAGE: - mirror_mode = "image" - elif mirror_mode == rbd.RBD_MIRROR_MODE_POOL: - mirror_mode = "pool" - else: - mirror_mode = "unknown" - stats['health_color'] = "warning" - stats['health'] = "Warning" - - pool_stats[pool_name] = dict(stats, **{ - 'mirror_mode': mirror_mode, - 'peer_uuids': peer_uuids - }) - - for daemon in daemons: - for _, pool_data in daemon['status'].items(): - stats = pool_stats.get(pool_data['name'], None) # type: ignore - if stats is None: - continue - - if pool_data.get('leader', False): - # leader instance stores image counts - stats['leader_id'] = daemon['metadata']['instance_id'] - stats['image_local_count'] = pool_data.get('image_local_count', 0) - stats['image_remote_count'] = pool_data.get('image_remote_count', 0) - - if (stats.get('health_color', '') != 'error' - and pool_data.get('image_error_count', 0) > 0): - stats['health_color'] = 'error' - stats['health'] = 'Error' - elif (stats.get('health_color', '') != 'error' - and pool_data.get('image_warning_count', 0) > 0): - stats['health_color'] = 'warning' - stats['health'] = 'Warning' - elif stats.get('health', None) is None: - stats['health_color'] = 'success' - stats['health'] = 'OK' - - for _, stats in pool_stats.items(): - if stats['mirror_mode'] == 'disabled': + status = json.loads(status['json']) + except (ValueError, KeyError): + status = {} + + instance_id = metadata['instance_id'] + if id == instance_id: + # new version that supports per-cluster leader elections + id = metadata['id'] + + # extract per-daemon service data and health + daemon = { + 'id': id, + 'instance_id': instance_id, + 'version': metadata['ceph_version'], + 'server_hostname': hostname, + 'service': service, + 'server': server, + 'metadata': metadata, + 'status': status + } + daemon = dict(daemon, **get_daemon_health(daemon)) + daemons.append(daemon) + + return sorted(daemons, key=lambda k: k['instance_id']) + + +def get_daemon_health(daemon): + health = { + 'health_color': 'info', + 'health': 'Unknown' + } + for _, pool_data in daemon['status'].items(): + if (health['health'] != 'error' + and [k for k, v in pool_data.get('callouts', {}).items() + if v['level'] == 'error']): + health = { + 'health_color': 'error', + 'health': 'Error' + } + elif (health['health'] != 'error' + and [k for k, v in pool_data.get('callouts', {}).items() + if v['level'] == 'warning']): + health = { + 'health_color': 'warning', + 'health': 'Warning' + } + elif health['health_color'] == 'info': + health = { + 'health_color': 'success', + 'health': 'OK' + } + return health + + +def get_pools(daemons): # pylint: disable=R0912, R0915 + pool_names = [pool['pool_name'] for pool in CephService.get_pool_list('rbd') + if pool.get('type', 1) == 1] + pool_stats = _get_pool_stats(pool_names) + _update_pool_stats(daemons, pool_stats) + return pool_stats + + +def _update_pool_stats(daemons, pool_stats): + _update_pool_stats_with_daemons(daemons, pool_stats) + for _, stats in pool_stats.items(): + if stats['mirror_mode'] == 'disabled': + continue + if stats.get('health', None) is None: + # daemon doesn't know about pool + stats['health_color'] = 'error' + stats['health'] = 'Error' + elif stats.get('leader_id', None) is None: + # no daemons are managing the pool as leader instance + stats['health_color'] = 'warning' + stats['health'] = 'Warning' + + +def _update_pool_stats_with_daemons(daemons, pool_stats): + for daemon in daemons: + for _, pool_data in daemon['status'].items(): + stats = pool_stats.get(pool_data['name'], None) # type: ignore + if stats is None: continue - if stats.get('health', None) is None: - # daemon doesn't know about pool + + if pool_data.get('leader', False): + # leader instance stores image counts + stats['leader_id'] = daemon['metadata']['instance_id'] + stats['image_local_count'] = pool_data.get('image_local_count', 0) + stats['image_remote_count'] = pool_data.get('image_remote_count', 0) + + if (stats.get('health_color', '') != 'error' + and pool_data.get('image_error_count', 0) > 0): stats['health_color'] = 'error' stats['health'] = 'Error' - elif stats.get('leader_id', None) is None: - # no daemons are managing the pool as leader instance + elif (stats.get('health_color', '') != 'error' + and pool_data.get('image_warning_count', 0) > 0): stats['health_color'] = 'warning' stats['health'] = 'Warning' - return pool_stats + elif stats.get('health', None) is None: + stats['health_color'] = 'success' + stats['health'] = 'OK' + + +def _get_pool_stats(pool_names): + pool_stats = {} + rbdctx = rbd.RBD() + for pool_name in pool_names: + logger.debug("Constructing IOCtx %s", pool_name) + try: + ioctx = mgr.rados.open_ioctx(pool_name) + except TypeError: + logger.exception("Failed to open pool %s", pool_name) + continue + try: + mirror_mode = rbdctx.mirror_mode_get(ioctx) + peer_uuids = [x['uuid'] for x in rbdctx.mirror_peer_list(ioctx)] + except: # noqa pylint: disable=W0702 + logger.exception("Failed to query mirror settings %s", pool_name) + mirror_mode = None + peer_uuids = [] + + stats = {} + if mirror_mode == rbd.RBD_MIRROR_MODE_DISABLED: + mirror_mode = "disabled" + stats['health_color'] = "info" + stats['health'] = "Disabled" + elif mirror_mode == rbd.RBD_MIRROR_MODE_IMAGE: + mirror_mode = "image" + elif mirror_mode == rbd.RBD_MIRROR_MODE_POOL: + mirror_mode = "pool" + else: + mirror_mode = "unknown" + stats['health_color'] = "warning" + stats['health'] = "Warning" + + pool_stats[pool_name] = dict(stats, **{ + 'mirror_mode': mirror_mode, + 'peer_uuids': peer_uuids + }) + return pool_stats + + +@ViewCache() +def get_daemons_and_pools(): # pylint: disable=R0915 daemons = get_daemons() return { 'daemons': daemons, diff --git a/src/pybind/mgr/dashboard/tests/test_rbd_mirroring.py b/src/pybind/mgr/dashboard/tests/test_rbd_mirroring.py index 9f83bad83e1..acb8336de7e 100644 --- a/src/pybind/mgr/dashboard/tests/test_rbd_mirroring.py +++ b/src/pybind/mgr/dashboard/tests/test_rbd_mirroring.py @@ -1,6 +1,9 @@ from __future__ import absolute_import import json +import unittest + +import rbd try: import mock @@ -8,7 +11,8 @@ except ImportError: import unittest.mock as mock from .. import mgr -from ..controllers.rbd_mirroring import RbdMirroring, RbdMirroringPoolBootstrap, RbdMirroringSummary +from ..controllers.rbd_mirroring import RbdMirroring, \ + RbdMirroringPoolBootstrap, RbdMirroringSummary, get_daemons, get_pools from ..controllers.summary import Summary from ..services import progress from . import ControllerTestCase # pylint: disable=no-name-in-module @@ -31,7 +35,7 @@ _status = { 'image_remote_count': 6, 'image_error_count': 7, 'image_warning_count': 8, - 'name': 'pool_name' + 'name': 'rbd' } } @@ -47,6 +51,113 @@ mock_osd_map = { } +class GetDaemonAndPoolsTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + mgr.list_servers.return_value = mock_list_servers + mgr.get_metadata = mock.Mock(return_value=mock_get_metadata) + mgr.get_daemon_status.return_value = mock_get_daemon_status + mgr.get.side_effect = lambda key: { + 'osd_map': mock_osd_map, + 'health': {'json': '{"status": 1}'}, + 'fs_map': {'filesystems': []}, + 'mgr_map': { + 'services': { + 'dashboard': 'https://ceph.dev:11000/' + }, + } + }[key] + mgr.url_prefix = '' + mgr.get_mgr_id.return_value = 0 + mgr.have_mon_connection.return_value = True + mgr.version = 'ceph version 13.1.0-534-g23d3751b89 ' \ + '(23d3751b897b31d2bda57aeaf01acb5ff3c4a9cd) ' \ + 'nautilus (dev)' + + progress.get_progress_tasks = mock.MagicMock() + progress.get_progress_tasks.return_value = ([], []) + + @mock.patch('rbd.RBD') + def test_get_pools_unknown(self, mock_rbd): + mock_rbd_instance = mock_rbd.return_value + mock_rbd_instance.mirror_mode_get.side_effect = Exception + daemons = get_daemons() + res = get_pools(daemons) + self.assertTrue(res['rbd']['mirror_mode'] == "unknown") + + @mock.patch('rbd.RBD') + def test_get_pools_mode(self, mock_rbd): + + daemons = get_daemons() + mock_rbd_instance = mock_rbd.return_value + testcases = [ + (rbd.RBD_MIRROR_MODE_DISABLED, "disabled"), + (rbd.RBD_MIRROR_MODE_IMAGE, "image"), + (rbd.RBD_MIRROR_MODE_POOL, "pool"), + ] + mock_rbd_instance.mirror_peer_list.return_value = [] + for mirror_mode, expected in testcases: + mock_rbd_instance.mirror_mode_get.return_value = mirror_mode + res = get_pools(daemons) + self.assertTrue(res['rbd']['mirror_mode'] == expected) + + @mock.patch('rbd.RBD') + def test_get_pools_health(self, mock_rbd): + + mock_rbd_instance = mock_rbd.return_value + mock_rbd_instance.mirror_peer_list.return_value = [] + test_cases = self._get_pool_test_cases() + for new_status, mirror_mode, expected_output in test_cases: + _status[1].update(new_status) + daemon_status = { + 'json': json.dumps(_status) + } + mgr.get_daemon_status.return_value = daemon_status + daemons = get_daemons() + mock_rbd_instance.mirror_mode_get.return_value = mirror_mode + res = get_pools(daemons) + for k, v in expected_output.items(): + self.assertTrue(v == res['rbd'][k]) + mgr.get_daemon_status.return_value = mock_get_daemon_status # reset return value + + def _get_pool_test_cases(self): + test_cases = [ + ( + { + 'image_error_count': 7, + }, + rbd.RBD_MIRROR_MODE_IMAGE, + { + 'health_color': 'warning', + 'health': 'Warning' + } + ), + ( + { + 'image_error_count': 7, + }, + rbd.RBD_MIRROR_MODE_DISABLED, + { + 'health_color': 'error', + 'health': 'Error' + } + ), + ( + { + 'image_error_count': 0, + 'image_warning_count': 0, + 'leader_id': 1 + }, + rbd.RBD_MIRROR_MODE_DISABLED, + { + 'health_color': 'info', + 'health': 'Disabled' + } + ), + ] + return test_cases + + class RbdMirroringControllerTest(ControllerTestCase): @classmethod