From d3c0bf7f15d88d9909771d12f7e1e4fc26734cbb Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Fri, 23 Feb 2018 12:40:17 +0100 Subject: [PATCH] mgr/dashboard_v2: Add rbd_mirroring to `/api/summary` Also: * Made `get_daemons_and_pools` a function independant of `RbdMirroring`. * Added tests Signed-off-by: Sebastian Wagner --- .../dashboard_v2/controllers/rbd_mirroring.py | 277 +++++++++--------- .../mgr/dashboard_v2/controllers/summary.py | 27 ++ .../dashboard_v2/tests/test_rbd_mirroring.py | 27 +- .../mgr/dashboard_v2/tests/test_summary.py | 2 + 4 files changed, 192 insertions(+), 141 deletions(-) diff --git a/src/pybind/mgr/dashboard_v2/controllers/rbd_mirroring.py b/src/pybind/mgr/dashboard_v2/controllers/rbd_mirroring.py index 6fdcf9d417a32..6fc90892dae2e 100644 --- a/src/pybind/mgr/dashboard_v2/controllers/rbd_mirroring.py +++ b/src/pybind/mgr/dashboard_v2/controllers/rbd_mirroring.py @@ -14,6 +14,144 @@ from ..tools import ApiController, AuthRequired, BaseController, ViewCache from .. import logger +@ViewCache() +def get_daemons_and_pools(mgr): # pylint: disable=R0915 + def get_daemons(): + daemons = [] + for hostname, server in CephService.get_service_map('rbd-mirror').items(): + for service in server['services']: + id = service['id'] # pylint: disable=W0622 + metadata = service['metadata'] + status = service['status'] + + try: + status = json.loads(status['json']) + except ValueError: + status = {} + + instance_id = metadata['instance_id'] + if id == instance_id: + # new version that supports per-cluster leader elections + id = metadata['id'] + + # extract per-daemon service data and health + daemon = { + 'id': id, + 'instance_id': instance_id, + 'version': metadata['ceph_version'], + 'server_hostname': hostname, + 'service': service, + 'server': server, + 'metadata': metadata, + 'status': status + } + daemon = dict(daemon, **get_daemon_health(daemon)) + daemons.append(daemon) + + return sorted(daemons, key=lambda k: k['instance_id']) + + def get_daemon_health(daemon): + health = { + 'health_color': 'info', + 'health': 'Unknown' + } + for _, pool_data in daemon['status'].items(): # TODO: simplify + if (health['health'] != 'error' and + [k for k, v in pool_data.get('callouts', {}).items() + if v['level'] == 'error']): + health = { + 'health_color': 'error', + 'health': 'Error' + } + elif (health['health'] != 'error' and + [k for k, v in pool_data.get('callouts', {}).items() + if v['level'] == 'warning']): + health = { + 'health_color': 'warning', + 'health': 'Warning' + } + elif health['health_color'] == 'info': + health = { + 'health_color': 'success', + 'health': 'OK' + } + return health + + def get_pools(daemons): # pylint: disable=R0912, R0915 + pool_names = [pool['pool_name'] for pool in CephService.get_pool_list('rbd')] + pool_stats = {} + rbdctx = rbd.RBD() + for pool_name in pool_names: + logger.debug("Constructing IOCtx %s", pool_name) + try: + ioctx = mgr.rados.open_ioctx(pool_name) + except TypeError: + logger.exception("Failed to open pool %s", pool_name) + continue + + try: + mirror_mode = rbdctx.mirror_mode_get(ioctx) + except: # noqa pylint: disable=W0702 + logger.exception("Failed to query mirror mode %s", pool_name) + + stats = {} + if mirror_mode == rbd.RBD_MIRROR_MODE_DISABLED: + continue + elif mirror_mode == rbd.RBD_MIRROR_MODE_IMAGE: + mirror_mode = "image" + elif mirror_mode == rbd.RBD_MIRROR_MODE_POOL: + mirror_mode = "pool" + else: + mirror_mode = "unknown" + stats['health_color'] = "warning" + stats['health'] = "Warning" + + pool_stats[pool_name] = dict(stats, **{ + 'mirror_mode': mirror_mode + }) + + for daemon in daemons: + for _, pool_data in daemon['status'].items(): + stats = pool_stats.get(pool_data['name'], None) + if stats is None: + continue + + if pool_data.get('leader', False): + # leader instance stores image counts + stats['leader_id'] = daemon['metadata']['instance_id'] + stats['image_local_count'] = pool_data.get('image_local_count', 0) + stats['image_remote_count'] = pool_data.get('image_remote_count', 0) + + if (stats.get('health_color', '') != 'error' and + pool_data.get('image_error_count', 0) > 0): + stats['health_color'] = 'error' + stats['health'] = 'Error' + elif (stats.get('health_color', '') != 'error' and + pool_data.get('image_warning_count', 0) > 0): + stats['health_color'] = 'warning' + stats['health'] = 'Warning' + elif stats.get('health', None) is None: + stats['health_color'] = 'success' + stats['health'] = 'OK' + + for _, stats in pool_stats.items(): + if stats.get('health', None) is None: + # daemon doesn't know about pool + stats['health_color'] = 'error' + stats['health'] = 'Error' + elif stats.get('leader_id', None) is None: + # no daemons are managing the pool as leader instance + stats['health_color'] = 'warning' + stats['health'] = 'Warning' + return pool_stats + + daemons = get_daemons() + return { + 'daemons': daemons, + 'pools': get_pools(daemons) + } + + @ApiController('rbdmirror') @AuthRequired() class RbdMirror(BaseController): @@ -27,143 +165,6 @@ class RbdMirror(BaseController): status, content_data = self._get_content_data() return {'status': status, 'content_data': content_data} - @ViewCache() - def _get_daemons_and_pools(self): # pylint: disable=R0915 - def get_daemons(): - daemons = [] - for hostname, server in CephService.get_service_map('rbd-mirror').items(): - for service in server['services']: - id = service['id'] # pylint: disable=W0622 - metadata = service['metadata'] - status = service['status'] - - try: - status = json.loads(status['json']) - except ValueError: - status = {} - - instance_id = metadata['instance_id'] - if id == instance_id: - # new version that supports per-cluster leader elections - id = metadata['id'] - - # extract per-daemon service data and health - daemon = { - 'id': id, - 'instance_id': instance_id, - 'version': metadata['ceph_version'], - 'server_hostname': hostname, - 'service': service, - 'server': server, - 'metadata': metadata, - 'status': status - } - daemon = dict(daemon, **get_daemon_health(daemon)) - daemons.append(daemon) - - return sorted(daemons, key=lambda k: k['instance_id']) - - def get_daemon_health(daemon): - health = { - 'health_color': 'info', - 'health': 'Unknown' - } - for _, pool_data in daemon['status'].items(): # TODO: simplify - if (health['health'] != 'error' and - [k for k, v in pool_data.get('callouts', {}).items() - if v['level'] == 'error']): - health = { - 'health_color': 'error', - 'health': 'Error' - } - elif (health['health'] != 'error' and - [k for k, v in pool_data.get('callouts', {}).items() - if v['level'] == 'warning']): - health = { - 'health_color': 'warning', - 'health': 'Warning' - } - elif health['health_color'] == 'info': - health = { - 'health_color': 'success', - 'health': 'OK' - } - return health - - def get_pools(daemons): # pylint: disable=R0912, R0915 - pool_names = [pool['pool_name'] for pool in CephService.get_pool_list('rbd')] - pool_stats = {} - rbdctx = rbd.RBD() - for pool_name in pool_names: - logger.debug("Constructing IOCtx %s", pool_name) - try: - ioctx = self.mgr.rados.open_ioctx(pool_name) - except TypeError: - logger.exception("Failed to open pool %s", pool_name) - continue - - try: - mirror_mode = rbdctx.mirror_mode_get(ioctx) - except: # noqa pylint: disable=W0702 - logger.exception("Failed to query mirror mode %s", pool_name) - - stats = {} - if mirror_mode == rbd.RBD_MIRROR_MODE_DISABLED: - continue - elif mirror_mode == rbd.RBD_MIRROR_MODE_IMAGE: - mirror_mode = "image" - elif mirror_mode == rbd.RBD_MIRROR_MODE_POOL: - mirror_mode = "pool" - else: - mirror_mode = "unknown" - stats['health_color'] = "warning" - stats['health'] = "Warning" - - pool_stats[pool_name] = dict(stats, **{ - 'mirror_mode': mirror_mode - }) - - for daemon in daemons: - for _, pool_data in daemon['status'].items(): - stats = pool_stats.get(pool_data['name'], None) - if stats is None: - continue - - if pool_data.get('leader', False): - # leader instance stores image counts - stats['leader_id'] = daemon['metadata']['instance_id'] - stats['image_local_count'] = pool_data.get('image_local_count', 0) - stats['image_remote_count'] = pool_data.get('image_remote_count', 0) - - if (stats.get('health_color', '') != 'error' and - pool_data.get('image_error_count', 0) > 0): - stats['health_color'] = 'error' - stats['health'] = 'Error' - elif (stats.get('health_color', '') != 'error' and - pool_data.get('image_warning_count', 0) > 0): - stats['health_color'] = 'warning' - stats['health'] = 'Warning' - elif stats.get('health', None) is None: - stats['health_color'] = 'success' - stats['health'] = 'OK' - - for _, stats in pool_stats.items(): - if stats.get('health', None) is None: - # daemon doesn't know about pool - stats['health_color'] = 'error' - stats['health'] = 'Error' - elif stats.get('leader_id', None) is None: - # no daemons are managing the pool as leader instance - stats['health_color'] = 'warning' - stats['health'] = 'Warning' - return pool_stats - - daemons = get_daemons() - return { - 'daemons': daemons, - 'pools': get_pools(daemons) - } - @ViewCache() def _get_pool_datum(self, pool_name): data = {} @@ -246,7 +247,7 @@ class RbdMirror(BaseController): return value pool_names = [pool['pool_name'] for pool in CephService.get_pool_list('rbd')] - _, data = self._get_daemons_and_pools() + _, data = get_daemons_and_pools(self.mgr) if data is None: logger.warning("Failed to get rbd-mirror daemons list") data = {} diff --git a/src/pybind/mgr/dashboard_v2/controllers/summary.py b/src/pybind/mgr/dashboard_v2/controllers/summary.py index a3c0ed0fbd6f6..68fa09893cf00 100644 --- a/src/pybind/mgr/dashboard_v2/controllers/summary.py +++ b/src/pybind/mgr/dashboard_v2/controllers/summary.py @@ -5,8 +5,10 @@ import json import cherrypy +from ..controllers.rbd_mirroring import get_daemons_and_pools from ..tools import AuthRequired, ApiController, BaseController from ..services.ceph_service import CephService +from .. import logger @ApiController('summary') @@ -30,6 +32,30 @@ class Summary(BaseController): for f in fsmap['filesystems'] ] + def _rbd_mirroring(self): + _, data = get_daemons_and_pools(self.mgr) + + if isinstance(data, Exception): + logger.exception("Failed to get rbd-mirror daemons and pools") + raise type(data)(str(data)) + else: + daemons = data.get('daemons', []) + pools = data.get('pools', {}) + + warnings = 0 + errors = 0 + for daemon in daemons: + if daemon['health_color'] == 'error': + errors += 1 + elif daemon['health_color'] == 'warning': + warnings += 1 + for _, pool in pools.items(): + if pool['health_color'] == 'error': + errors += 1 + elif pool['health_color'] == 'warning': + warnings += 1 + return {'warnings': warnings, 'errors': errors} + @cherrypy.expose @cherrypy.tools.json_out() def default(self): @@ -37,6 +63,7 @@ class Summary(BaseController): 'rbd_pools': self._rbd_pool_data(), 'health_status': self._health_status(), 'filesystems': self._filesystems(), + 'rbd_mirroring': self._rbd_mirroring(), 'mgr_id': self.mgr.get_mgr_id(), 'have_mon_connection': self.mgr.have_mon_connection() } diff --git a/src/pybind/mgr/dashboard_v2/tests/test_rbd_mirroring.py b/src/pybind/mgr/dashboard_v2/tests/test_rbd_mirroring.py index f1b5f01d27390..39468dab122de 100644 --- a/src/pybind/mgr/dashboard_v2/tests/test_rbd_mirroring.py +++ b/src/pybind/mgr/dashboard_v2/tests/test_rbd_mirroring.py @@ -5,9 +5,10 @@ import cherrypy from cherrypy.test.helper import CPWebCase from ..controllers.auth import Auth +from ..controllers.summary import Summary +from ..controllers.rbd_mirroring import RbdMirror from ..services import Service from ..tools import SessionExpireAtBrowserCloseTool -from ..controllers.rbd_mirroring import RbdMirror from .helper import ControllerTestCase @@ -62,14 +63,26 @@ class RbdMirroringControllerTest(ControllerTestCase, CPWebCase): mgr_mock.list_servers.return_value = mock_list_servers mgr_mock.get_metadata.return_value = mock_get_metadata mgr_mock.get_daemon_status.return_value = mock_get_daemon_status - mgr_mock.get.return_value = mock_osd_map + mgr_mock.get.side_effect = lambda key: { + 'osd_map': mock_osd_map, + 'health': {'json': '{"status": 1}'}, + 'fs_map': {'filesystems': []}, + + }[key] mgr_mock.url_prefix = '' + mgr_mock.get_mgr_id.return_value = 0 + mgr_mock.have_mon_connection.return_value = True - RbdMirror.mgr = mgr_mock Service.mgr = mgr_mock + + RbdMirror.mgr = mgr_mock RbdMirror._cp_config['tools.authenticate.on'] = False # pylint: disable=protected-access + Summary.mgr = mgr_mock + Summary._cp_config['tools.authenticate.on'] = False # pylint: disable=protected-access + cherrypy.tree.mount(RbdMirror(), '/api/test/rbdmirror') + cherrypy.tree.mount(Summary(), '/api/test/summary') def __init__(self, *args, **kwargs): super(RbdMirroringControllerTest, self).__init__(*args, dashboard_port=54583, **kwargs) @@ -82,3 +95,11 @@ class RbdMirroringControllerTest(ControllerTestCase, CPWebCase): self.assertEqual(result['status'], 0) for k in ['daemons', 'pools', 'image_error', 'image_syncing', 'image_ready']: self.assertIn(k, result['content_data']) + + @mock.patch('dashboard_v2.controllers.rbd_mirroring.rbd') + def test_summary(self, rbd_mock): # pylint: disable=W0613 + """We're also testing `summary`, as it also uses code from `rbd_mirroring.py`""" + data = self._get('/api/test/summary') + self.assertStatus(200) + summary = data['rbd_mirroring'] + self.assertEqual(summary, {'errors': 0, 'warnings': 1}) diff --git a/src/pybind/mgr/dashboard_v2/tests/test_summary.py b/src/pybind/mgr/dashboard_v2/tests/test_summary.py index 91a9f7a34df0c..d98d1b4a0269b 100644 --- a/src/pybind/mgr/dashboard_v2/tests/test_summary.py +++ b/src/pybind/mgr/dashboard_v2/tests/test_summary.py @@ -13,8 +13,10 @@ class SummaryTest(ControllerTestCase): self.assertIn('rbd_pools', data) self.assertIn('mgr_id', data) self.assertIn('have_mon_connection', data) + self.assertIn('rbd_mirroring', data) self.assertIsNotNone(data['filesystems']) self.assertIsNotNone(data['health_status']) self.assertIsNotNone(data['rbd_pools']) self.assertIsNotNone(data['mgr_id']) self.assertIsNotNone(data['have_mon_connection']) + self.assertEqual(data['rbd_mirroring'], {'errors': 0, 'warnings': 0}) -- 2.39.5