From 50b7d42fe59aa8fd5194fa81395725b5b5cedd64 Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Mon, 29 Oct 2018 16:07:27 -0600 Subject: [PATCH] mgr/dashboard: Replace dashboard service This splits out the collection of health and log data from the /api/dashboard/health controller into /api/health/{full,minimal} and /api/logs/all. /health/full contains all the data (minus logs) that /dashboard/health did, whereas /health/minimal contains only what is needed for the health component to function. /logs/all contains exactly what the logs portion of /dashboard/health did. By using /health/minimal, on a vstart cluster we pull ~1.4KB of data every 5s, where we used to pull ~6KB; those numbers would get larger with larger clusters. Once we split out log data, that will drop to ~0.4KB. Fixes: http://tracker.ceph.com/issues/36675 Signed-off-by: Zack Cerza --- qa/suites/rados/mgr/tasks/dashboard.yaml | 3 +- qa/tasks/mgr/dashboard/test_dashboard.py | 81 ------ qa/tasks/mgr/dashboard/test_health.py | 261 ++++++++++++++++++ qa/tasks/mgr/dashboard/test_logs.py | 38 +++ qa/tasks/mgr/dashboard/test_pool.py | 4 +- .../mgr/dashboard/controllers/dashboard.py | 128 --------- .../mgr/dashboard/controllers/health.py | 193 +++++++++++++ src/pybind/mgr/dashboard/controllers/logs.py | 51 ++++ .../crushmap/crushmap.component.spec.ts | 12 +- .../cluster/crushmap/crushmap.component.ts | 6 +- .../app/ceph/cluster/logs/logs.component.ts | 6 +- .../dashboard/health/health.component.html | 118 ++++---- .../dashboard/health/health.component.spec.ts | 6 +- .../ceph/dashboard/health/health.component.ts | 20 +- .../src/app/shared/api/health.service.spec.ts | 40 +++ .../src/app/shared/api/health.service.ts | 19 ++ ...d.service.spec.ts => logs.service.spec.ts} | 16 +- .../{dashboard.service.ts => logs.service.ts} | 6 +- 18 files changed, 700 insertions(+), 308 deletions(-) delete mode 100644 qa/tasks/mgr/dashboard/test_dashboard.py create mode 100644 qa/tasks/mgr/dashboard/test_health.py create mode 100644 qa/tasks/mgr/dashboard/test_logs.py delete mode 100644 src/pybind/mgr/dashboard/controllers/dashboard.py create mode 100644 src/pybind/mgr/dashboard/controllers/health.py create mode 100644 src/pybind/mgr/dashboard/controllers/logs.py create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/api/health.service.spec.ts create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/api/health.service.ts rename src/pybind/mgr/dashboard/frontend/src/app/shared/api/{dashboard.service.spec.ts => logs.service.spec.ts} (63%) rename src/pybind/mgr/dashboard/frontend/src/app/shared/api/{dashboard.service.ts => logs.service.ts} (70%) diff --git a/qa/suites/rados/mgr/tasks/dashboard.yaml b/qa/suites/rados/mgr/tasks/dashboard.yaml index 95eba5723de..6bb4a9a03b2 100644 --- a/qa/suites/rados/mgr/tasks/dashboard.yaml +++ b/qa/suites/rados/mgr/tasks/dashboard.yaml @@ -30,8 +30,9 @@ tasks: - tasks.mgr.dashboard.test_auth - tasks.mgr.dashboard.test_cephfs - tasks.mgr.dashboard.test_cluster_configuration - - tasks.mgr.dashboard.test_dashboard + - tasks.mgr.dashboard.test_health - tasks.mgr.dashboard.test_host + - tasks.mgr.dashboard.test_logs - tasks.mgr.dashboard.test_monitor - tasks.mgr.dashboard.test_osd - tasks.mgr.dashboard.test_perf_counters diff --git a/qa/tasks/mgr/dashboard/test_dashboard.py b/qa/tasks/mgr/dashboard/test_dashboard.py deleted file mode 100644 index 82578aae1f9..00000000000 --- a/qa/tasks/mgr/dashboard/test_dashboard.py +++ /dev/null @@ -1,81 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import absolute_import - -from .helper import DashboardTestCase - - -class DashboardTest(DashboardTestCase): - CEPHFS = True - - def test_health(self): - data = self._get("/api/dashboard/health") - self.assertStatus(200) - - self.assertIn('health', data) - self.assertIn('mon_status', data) - self.assertIn('fs_map', data) - self.assertIn('osd_map', data) - self.assertIn('clog', data) - self.assertIn('audit_log', data) - self.assertIn('pools', data) - self.assertIn('mgr_map', data) - self.assertIn('df', data) - self.assertIn('scrub_status', data) - self.assertIn('pg_info', data) - self.assertIn('client_perf', data) - self.assertIn('hosts', data) - self.assertIn('rgw', data) - self.assertIn('iscsi_daemons', data) - self.assertIsNotNone(data['health']) - self.assertIsNotNone(data['mon_status']) - self.assertIsNotNone(data['fs_map']) - self.assertIsNotNone(data['osd_map']) - self.assertIsNotNone(data['clog']) - self.assertIsNotNone(data['audit_log']) - self.assertIsNotNone(data['pools']) - self.assertIsNotNone(data['scrub_status']) - self.assertIsNotNone(data['pg_info']) - self.assertIsNotNone(data['client_perf']) - self.assertIsNotNone(data['hosts']) - self.assertIsNotNone(data['rgw']) - self.assertIsNotNone(data['iscsi_daemons']) - - cluster_pools = self.ceph_cluster.mon_manager.list_pools() - self.assertEqual(len(cluster_pools), len(data['pools'])) - for pool in data['pools']: - self.assertIn(pool['pool_name'], cluster_pools) - - self.assertIsNotNone(data['mgr_map']) - self.assertIsNotNone(data['df']) - - - @DashboardTestCase.RunAs('test', 'test', ['pool-manager']) - def test_health_permissions(self): - data = self._get("/api/dashboard/health") - self.assertStatus(200) - - self.assertIn('health', data) - self.assertNotIn('mon_status', data) - self.assertNotIn('fs_map', data) - self.assertNotIn('osd_map', data) - self.assertNotIn('clog', data) - self.assertNotIn('audit_log', data) - self.assertIn('pools', data) - self.assertNotIn('mgr_map', data) - self.assertIn('df', data) - self.assertNotIn('scrub_status', data) - self.assertNotIn('pg_info', data) - self.assertIn('client_perf', data) - self.assertNotIn('hosts', data) - self.assertNotIn('rgw', data) - self.assertNotIn('iscsi_daemons', data) - self.assertIsNotNone(data['health']) - self.assertIsNotNone(data['pools']) - self.assertIsNotNone(data['client_perf']) - - cluster_pools = self.ceph_cluster.mon_manager.list_pools() - self.assertEqual(len(cluster_pools), len(data['pools'])) - for pool in data['pools']: - self.assertIn(pool['pool_name'], cluster_pools) - - self.assertIsNotNone(data['df']) diff --git a/qa/tasks/mgr/dashboard/test_health.py b/qa/tasks/mgr/dashboard/test_health.py new file mode 100644 index 00000000000..421567c8039 --- /dev/null +++ b/qa/tasks/mgr/dashboard/test_health.py @@ -0,0 +1,261 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import + +from .helper import DashboardTestCase, JAny, JLeaf, JList, JObj + + +class HealthTest(DashboardTestCase): + CEPHFS = True + + def test_minimal_health(self): + data = self._get('/api/health/minimal') + self.assertStatus(200) + schema = JObj({ + 'client_perf': JObj({ + 'read_bytes_sec': int, + 'read_op_per_sec': int, + 'recovering_bytes_per_sec': int, + 'write_bytes_sec': int, + 'write_op_per_sec': int + }), + 'df': JObj({ + 'stats': JObj({ + 'total_avail_bytes': int, + 'total_bytes': int, + 'total_objects': int, + 'total_used_bytes': int, + }) + }), + 'fs_map': JObj({ + 'filesystems': JList( + JObj({ + 'mdsmap': JObj({ + 'info': JObj( + {}, + allow_unknown=True, + unknown_schema=JObj({ + 'state': str + }) + ) + }) + }), + ), + 'standbys': JList(JObj({})), + }), + 'health': JObj({ + 'checks': JList(str), + 'status': str, + }), + 'hosts': int, + 'iscsi_daemons': int, + 'mgr_map': JObj({ + 'active_name': str, + 'standbys': JList(JLeaf(dict)) + }), + 'mon_status': JObj({ + 'monmap': JObj({ + 'mons': JList(JLeaf(dict)), + }), + 'quorum': JList(int) + }), + 'osd_map': JObj({ + 'osds': JList( + JObj({ + 'in': int, + 'up': int, + })), + }), + 'pg_info': JObj({ + 'pgs_per_osd': int, + 'statuses': JObj({}, allow_unknown=True, unknown_schema=int) + }), + 'pools': JList(JLeaf(dict)), + 'rgw': int, + 'scrub_status': str + }) + self.assertSchema(data, schema) + + def test_full_health(self): + data = self._get('/api/health/full') + self.assertStatus(200) + schema = JObj({ + 'client_perf': JObj({ + 'read_bytes_sec': int, + 'read_op_per_sec': int, + 'recovering_bytes_per_sec': int, + 'write_bytes_sec': int, + 'write_op_per_sec': int + }), + 'df': JObj({ + 'pools': JList(JObj({ + 'stats': JObj({ + 'wr': int, + 'quota_objects': int, + 'bytes_used': int, + 'max_avail': int, + 'rd': int, + 'rd_bytes': int, + 'objects': int, + 'percent_used': float, + 'kb_used': int, + 'quota_bytes': int, + 'raw_bytes_used': int, + 'wr_bytes': int, + 'dirty': int + }), + 'name': str, + 'id': int + })), + 'stats': JObj({ + 'total_avail_bytes': int, + 'total_bytes': int, + 'total_objects': int, + 'total_percent_used': float, + 'total_used_bytes': int + }) + }), + 'fs_map': JObj({ + 'compat': JObj({ + 'compat': JObj({}, allow_unknown=True, unknown_schema=str), + 'incompat': JObj( + {}, allow_unknown=True, unknown_schema=str), + 'ro_compat': JObj( + {}, allow_unknown=True, unknown_schema=str) + }), + 'default_fscid': int, + 'epoch': int, + 'feature_flags': JObj( + {}, allow_unknown=True, unknown_schema=bool), + 'filesystems': JList( + JObj({ + 'id': int, + 'mdsmap': JObj({ + # TODO: Expand mdsmap schema + 'info': JObj( + {}, + allow_unknown=True, + unknown_schema=JObj({ + 'state': str + }, allow_unknown=True) + ) + }, allow_unknown=True) + }), + ), + 'standbys': JList(JObj({}, allow_unknown=True)), + }), + 'health': JObj({ + 'checks': JList(str), + 'status': str, + }), + 'hosts': int, + 'iscsi_daemons': int, + 'mgr_map': JObj({ + 'active_addrs': JObj({ + 'addrvec': JList(JObj({ + 'addr': str, + 'nonce': int, + 'type': str + })) + }), + 'active_change': str, # timestamp + 'active_gid': int, + 'active_name': str, + 'always_on_modules': JObj( + {}, + allow_unknown=True, unknown_schema=JList(str) + ), + 'available': bool, + 'available_modules': JList(JObj({ + 'can_run': bool, + 'error_string': str, + 'name': str + })), + 'epoch': int, + 'modules': JList(str), + 'services': JObj( + {'dashboard': str}, # This module should always be present + allow_unknown=True, unknown_schema=str + ), + 'standbys': JList(JObj({ + 'available_modules': JList(JObj({ + 'can_run': bool, + 'error_string': str, + 'name': str + })), + 'gid': int, + 'name': str + })) + }), + 'mon_status': JObj({ + 'election_epoch': int, + 'extra_probe_peers': JList(JAny(none=True)), + 'feature_map': JObj( + {}, allow_unknown=True, unknown_schema=JList(JObj({ + 'features': str, + 'num': int, + 'release': str + })) + ), + 'features': JObj({ + 'quorum_con': str, + 'quorum_mon': JList(str), + 'required_con': str, + 'required_mon': JList(str) + }), + 'monmap': JObj({ + # TODO: expand on monmap schema + 'mons': JList(JLeaf(dict)), + }, allow_unknown=True), + 'name': str, + 'outside_quorum': JList(int), + 'quorum': JList(int), + 'quorum_age': str, + 'rank': int, + 'state': str, + # TODO: What type should be expected here? + 'sync_provider': JList(JAny(none=True)) + }), + 'osd_map': JObj({ + # TODO: define schema for crush map and osd_metadata, among + # others + 'osds': JList( + JObj({ + 'in': int, + 'up': int, + }, allow_unknown=True)), + }, allow_unknown=True), + 'pg_info': JObj({ + 'pgs_per_osd': int, + 'statuses': JObj({}, allow_unknown=True, unknown_schema=int) + }), + 'pools': JList(JLeaf(dict)), + 'rgw': int, + 'scrub_status': str + }) + self.assertSchema(data, schema) + + cluster_pools = self.ceph_cluster.mon_manager.list_pools() + self.assertEqual(len(cluster_pools), len(data['pools'])) + for pool in data['pools']: + self.assertIn(pool['pool_name'], cluster_pools) + + @DashboardTestCase.RunAs('test', 'test', ['pool-manager']) + def test_health_permissions(self): + data = self._get('/api/health/full') + self.assertStatus(200) + + schema = JObj({ + 'client_perf': JObj({}, allow_unknown=True), + 'df': JObj({}, allow_unknown=True), + 'health': JObj({ + 'checks': JList(str), + 'status': str + }), + 'pools': JList(JLeaf(dict)), + }) + self.assertSchema(data, schema) + + cluster_pools = self.ceph_cluster.mon_manager.list_pools() + self.assertEqual(len(cluster_pools), len(data['pools'])) + for pool in data['pools']: + self.assertIn(pool['pool_name'], cluster_pools) diff --git a/qa/tasks/mgr/dashboard/test_logs.py b/qa/tasks/mgr/dashboard/test_logs.py new file mode 100644 index 00000000000..17d5d830c99 --- /dev/null +++ b/qa/tasks/mgr/dashboard/test_logs.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import + +from .helper import DashboardTestCase, JList, JObj + + +class LogsTest(DashboardTestCase): + CEPHFS = True + + def test_logs(self): + data = self._get("/api/logs/all") + self.assertStatus(200) + log_entry_schema = JList(JObj({ + 'addrs': JObj({ + 'addrvec': JList(JObj({ + 'addr': str, + 'nonce': int, + 'type': str + })) + }), + 'channel': str, + 'message': str, + 'name': str, + 'priority': str, + 'rank': str, + 'seq': int, + 'stamp': str + })) + schema = JObj({ + 'audit_log': log_entry_schema, + 'clog': log_entry_schema + }) + self.assertSchema(data, schema) + + @DashboardTestCase.RunAs('test', 'test', ['pool-manager']) + def test_log_perms(self): + self._get("/api/logs/all") + self.assertStatus(403) diff --git a/qa/tasks/mgr/dashboard/test_pool.py b/qa/tasks/mgr/dashboard/test_pool.py index 91b63810d9e..92276d22806 100644 --- a/qa/tasks/mgr/dashboard/test_pool.py +++ b/qa/tasks/mgr/dashboard/test_pool.py @@ -47,7 +47,7 @@ class PoolTest(DashboardTestCase): log.exception("test_pool_create: pool=%s", pool) raise - health = self._get('/api/dashboard/health')['health'] + health = self._get('/api/health/minimal')['health'] self.assertEqual(health['status'], 'HEALTH_OK', msg='health={}'.format(health)) def _get_pool(self, pool_name): @@ -86,7 +86,7 @@ class PoolTest(DashboardTestCase): # Feel free to test it locally. prop = 'pg_num' pgp_prop = 'pg_placement_num' - health = lambda: self._get('/api/dashboard/health')['health']['status'] == 'HEALTH_OK' + health = lambda: self._get('/api/health/minimal')['health']['status'] == 'HEALTH_OK' t = 0; while (int(value) != pool[pgp_prop] or not health()) and t < 180: time.sleep(2) diff --git a/src/pybind/mgr/dashboard/controllers/dashboard.py b/src/pybind/mgr/dashboard/controllers/dashboard.py deleted file mode 100644 index 75fc5f66d00..00000000000 --- a/src/pybind/mgr/dashboard/controllers/dashboard.py +++ /dev/null @@ -1,128 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import absolute_import - -import collections -import json - -from . import ApiController, Endpoint, BaseController -from .. import mgr -from ..security import Permission, Scope -from ..services.ceph_service import CephService -from ..services.tcmu_service import TcmuService -from ..tools import NotificationQueue - - -LOG_BUFFER_SIZE = 30 - - -@ApiController('/dashboard') -class Dashboard(BaseController): - def __init__(self): - super(Dashboard, self).__init__() - - self._log_initialized = False - - self.log_buffer = collections.deque(maxlen=LOG_BUFFER_SIZE) - self.audit_buffer = collections.deque(maxlen=LOG_BUFFER_SIZE) - - def append_log(self, log_struct): - if log_struct['channel'] == "audit": - self.audit_buffer.appendleft(log_struct) - else: - self.log_buffer.appendleft(log_struct) - - def load_buffer(self, buf, channel_name): - lines = CephService.send_command('mon', 'log last', channel=channel_name, - num=LOG_BUFFER_SIZE) - for l in lines: - buf.appendleft(l) - - @Endpoint() - def health(self): - if not self._log_initialized: - self._log_initialized = True - - self.load_buffer(self.log_buffer, "cluster") - self.load_buffer(self.audit_buffer, "audit") - - NotificationQueue.register(self.append_log, 'clog') - - result = { - "health": self.health_data(), - } - - if self._has_permissions(Permission.READ, Scope.LOG): - result['clog'] = list(self.log_buffer) - result['audit_log'] = list(self.audit_buffer) - - if self._has_permissions(Permission.READ, Scope.MONITOR): - result['mon_status'] = self.mon_status() - - if self._has_permissions(Permission.READ, Scope.CEPHFS): - result['fs_map'] = mgr.get('fs_map') - - if self._has_permissions(Permission.READ, Scope.OSD): - osd_map = self.osd_map() - # Not needed, skip the effort of transmitting this to UI - del osd_map['pg_temp'] - result['osd_map'] = osd_map - result['scrub_status'] = CephService.get_scrub_status() - result['pg_info'] = CephService.get_pg_info() - - if self._has_permissions(Permission.READ, Scope.MANAGER): - result['mgr_map'] = mgr.get("mgr_map") - - if self._has_permissions(Permission.READ, Scope.POOL): - pools = CephService.get_pool_list_with_stats() - result['pools'] = pools - - df = mgr.get("df") - df['stats']['total_objects'] = sum( - [p['stats']['objects'] for p in df['pools']]) - result['df'] = df - - result['client_perf'] = CephService.get_client_perf() - - if self._has_permissions(Permission.READ, Scope.HOSTS): - result['hosts'] = len(mgr.list_servers()) - - if self._has_permissions(Permission.READ, Scope.RGW): - result['rgw'] = len(CephService.get_service_list('rgw')) - - if self._has_permissions(Permission.READ, Scope.ISCSI): - result['iscsi_daemons'] = TcmuService.get_iscsi_daemons_amount() - - return result - - def mon_status(self): - mon_status_data = mgr.get("mon_status") - return json.loads(mon_status_data['json']) - - def osd_map(self): - osd_map = mgr.get("osd_map") - - assert osd_map is not None - - osd_map['tree'] = mgr.get("osd_map_tree") - osd_map['crush'] = mgr.get("osd_map_crush") - osd_map['crush_map_text'] = mgr.get("osd_map_crush_map_text") - osd_map['osd_metadata'] = mgr.get("osd_metadata") - - return osd_map - - def health_data(self): - health_data = mgr.get("health") - health = json.loads(health_data['json']) - - # Transform the `checks` dict into a list for the convenience - # of rendering from javascript. - checks = [] - for k, v in health['checks'].items(): - v['type'] = k - checks.append(v) - - checks = sorted(checks, key=lambda c: c['severity']) - - health['checks'] = checks - - return health diff --git a/src/pybind/mgr/dashboard/controllers/health.py b/src/pybind/mgr/dashboard/controllers/health.py new file mode 100644 index 00000000000..4e67f2303e3 --- /dev/null +++ b/src/pybind/mgr/dashboard/controllers/health.py @@ -0,0 +1,193 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import + +import json + +from . import ApiController, Endpoint, BaseController + +from .. import mgr +from ..security import Permission, Scope +from ..services.ceph_service import CephService +from ..services.tcmu_service import TcmuService + + +class HealthData(object): + """ + A class to be used in combination with BaseController to allow either + "full" or "minimal" sets of health data to be collected. + + To function properly, it needs BaseCollector._has_permissions to be passed + in as ``auth_callback``. + """ + + def __init__(self, auth_callback, minimal=True): + self._has_permissions = auth_callback + self._minimal = minimal + + @staticmethod + def _partial_dict(orig, keys): + return {k: orig[k] for k in keys} + + def all_health(self): + result = { + "health": self.basic_health(), + } + + if self._has_permissions(Permission.READ, Scope.MONITOR): + result['mon_status'] = self.mon_status() + + if self._has_permissions(Permission.READ, Scope.CEPHFS): + result['fs_map'] = self.fs_map() + + if self._has_permissions(Permission.READ, Scope.OSD): + result['osd_map'] = self.osd_map() + result['scrub_status'] = self.scrub_status() + result['pg_info'] = self.pg_info() + + if self._has_permissions(Permission.READ, Scope.MANAGER): + result['mgr_map'] = self.mgr_map() + + if self._has_permissions(Permission.READ, Scope.POOL): + result['pools'] = self.pools() + result['df'] = self.df() + result['client_perf'] = self.client_perf() + + if self._has_permissions(Permission.READ, Scope.HOSTS): + result['hosts'] = self.host_count() + + if self._has_permissions(Permission.READ, Scope.RGW): + result['rgw'] = self.rgw_count() + + if self._has_permissions(Permission.READ, Scope.ISCSI): + result['iscsi_daemons'] = self.iscsi_daemons() + + return result + + def basic_health(self): + health_data = mgr.get("health") + health = json.loads(health_data['json']) + + # Transform the `checks` dict into a list for the convenience + # of rendering from javascript. + checks = [] + for k, v in health['checks'].items(): + v['type'] = k + checks.append(v) + + checks = sorted(checks, key=lambda c: c['severity']) + health['checks'] = checks + return health + + def client_perf(self): + result = CephService.get_client_perf() + if self._minimal: + result = self._partial_dict( + result, + ['read_bytes_sec', 'read_op_per_sec', + 'recovering_bytes_per_sec', 'write_bytes_sec', + 'write_op_per_sec'] + ) + return result + + def df(self): + df = mgr.get('df') + df['stats']['total_objects'] = sum( + [p['stats']['objects'] for p in df['pools']]) + if self._minimal: + df = dict(stats=self._partial_dict( + df['stats'], + ['total_avail_bytes', 'total_bytes', 'total_objects', + 'total_used_bytes'] + )) + return df + + def fs_map(self): + fs_map = mgr.get('fs_map') + if self._minimal: + fs_map = self._partial_dict(fs_map, ['filesystems', 'standbys']) + fs_map['standbys'] = [{}] * len(fs_map['standbys']) + fs_map['filesystems'] = [self._partial_dict(item, ['mdsmap']) for + item in fs_map['filesystems']] + for fs in fs_map['filesystems']: + mdsmap_info = fs['mdsmap']['info'] + min_mdsmap_info = dict() + for k, v in mdsmap_info.items(): + min_mdsmap_info[k] = self._partial_dict(v, ['state']) + fs['mdsmap'] = dict(info=min_mdsmap_info) + return fs_map + + def host_count(self): + return len(mgr.list_servers()) + + def iscsi_daemons(self): + return TcmuService.get_iscsi_daemons_amount() + + def mgr_map(self): + mgr_map = mgr.get('mgr_map') + if self._minimal: + mgr_map = self._partial_dict(mgr_map, ['active_name', 'standbys']) + mgr_map['standbys'] = [{}] * len(mgr_map['standbys']) + return mgr_map + + def mon_status(self): + mon_status = json.loads(mgr.get('mon_status')['json']) + if self._minimal: + mon_status = self._partial_dict(mon_status, ['monmap', 'quorum']) + mon_status['monmap'] = self._partial_dict( + mon_status['monmap'], ['mons'] + ) + mon_status['monmap']['mons'] = [{}] * \ + len(mon_status['monmap']['mons']) + return mon_status + + def osd_map(self): + osd_map = mgr.get('osd_map') + assert osd_map is not None + # Not needed, skip the effort of transmitting this to UI + del osd_map['pg_temp'] + if self._minimal: + osd_map = self._partial_dict(osd_map, ['osds']) + osd_map['osds'] = [ + self._partial_dict(item, ['in', 'up']) + for item in osd_map['osds'] + ] + else: + osd_map['tree'] = mgr.get('osd_map_tree') + osd_map['crush'] = mgr.get('osd_map_crush') + osd_map['crush_map_text'] = mgr.get('osd_map_crush_map_text') + osd_map['osd_metadata'] = mgr.get('osd_metadata') + return osd_map + + def pg_info(self): + pg_info = CephService.get_pg_info() + if self._minimal: + pg_info = self._partial_dict(pg_info, ['pgs_per_osd', 'statuses']) + return pg_info + + def pools(self): + pools = CephService.get_pool_list_with_stats() + if self._minimal: + pools = [{}] * len(pools) + return pools + + def rgw_count(self): + return len(CephService.get_service_list('rgw')) + + def scrub_status(self): + return CephService.get_scrub_status() + + +@ApiController('/health') +class Health(BaseController): + def __init__(self): + super(Health, self).__init__() + self.health_full = HealthData(self._has_permissions, minimal=False) + self.health_minimal = HealthData(self._has_permissions, minimal=True) + + @Endpoint() + def full(self): + return self.health_full.all_health() + + @Endpoint() + def minimal(self): + return self.health_minimal.all_health() diff --git a/src/pybind/mgr/dashboard/controllers/logs.py b/src/pybind/mgr/dashboard/controllers/logs.py new file mode 100644 index 00000000000..9dc5286f3bd --- /dev/null +++ b/src/pybind/mgr/dashboard/controllers/logs.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import + +import collections + +from . import ApiController, Endpoint, BaseController, ReadPermission +from ..security import Scope +from ..services.ceph_service import CephService +from ..tools import NotificationQueue + + +LOG_BUFFER_SIZE = 30 + + +@ApiController('/logs', Scope.LOG) +class Logs(BaseController): + def __init__(self): + super(Logs, self).__init__() + self._log_initialized = False + self.log_buffer = collections.deque(maxlen=LOG_BUFFER_SIZE) + self.audit_buffer = collections.deque(maxlen=LOG_BUFFER_SIZE) + + def append_log(self, log_struct): + if log_struct['channel'] == 'audit': + self.audit_buffer.appendleft(log_struct) + else: + self.log_buffer.appendleft(log_struct) + + def load_buffer(self, buf, channel_name): + lines = CephService.send_command( + 'mon', 'log last', channel=channel_name, num=LOG_BUFFER_SIZE) + for l in lines: + buf.appendleft(l) + + def initialize_buffers(self): + if not self._log_initialized: + self._log_initialized = True + + self.load_buffer(self.log_buffer, 'cluster') + self.load_buffer(self.audit_buffer, 'audit') + + NotificationQueue.register(self.append_log, 'clog') + + @Endpoint() + @ReadPermission + def all(self): + self.initialize_buffers() + return dict( + clog=list(self.log_buffer), + audit_log=list(self.audit_buffer), + ) diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/crushmap/crushmap.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/crushmap/crushmap.component.spec.ts index 7c959569145..cf884056ec4 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/crushmap/crushmap.component.spec.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/crushmap/crushmap.component.spec.ts @@ -8,7 +8,7 @@ import { TreeModule } from 'ng2-tree'; import { TabsModule } from 'ngx-bootstrap/tabs'; import { configureTestBed } from '../../../../testing/unit-test-helper'; -import { DashboardService } from '../../../shared/api/dashboard.service'; +import { HealthService } from '../../../shared/api/health.service'; import { SharedModule } from '../../../shared/shared.module'; import { CrushmapComponent } from './crushmap.component'; @@ -19,7 +19,7 @@ describe('CrushmapComponent', () => { configureTestBed({ imports: [HttpClientTestingModule, TreeModule, TabsModule.forRoot(), SharedModule], declarations: [CrushmapComponent], - providers: [DashboardService] + providers: [HealthService] }); beforeEach(() => { @@ -39,21 +39,21 @@ describe('CrushmapComponent', () => { }); describe('test tree', () => { - let dashboardService: DashboardService; + let healthService: HealthService; const prepareGetHealth = (nodes: object[]) => { - spyOn(dashboardService, 'getHealth').and.returnValue( + spyOn(healthService, 'getFullHealth').and.returnValue( of({ osd_map: { tree: { nodes: nodes } } }) ); fixture.detectChanges(); }; beforeEach(() => { - dashboardService = debugElement.injector.get(DashboardService); + healthService = debugElement.injector.get(HealthService); }); it('should display "No nodes!" if ceph tree nodes is empty array', () => { prepareGetHealth([]); - expect(dashboardService.getHealth).toHaveBeenCalled(); + expect(healthService.getFullHealth).toHaveBeenCalled(); expect(component.tree.value).toEqual('No nodes!'); }); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/crushmap/crushmap.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/crushmap/crushmap.component.ts index 1e843a3f702..945b1ed5fb4 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/crushmap/crushmap.component.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/crushmap/crushmap.component.ts @@ -2,7 +2,7 @@ import { Component, OnInit } from '@angular/core'; import { NodeEvent, TreeModel } from 'ng2-tree'; -import { DashboardService } from '../../../shared/api/dashboard.service'; +import { HealthService } from '../../../shared/api/health.service'; @Component({ selector: 'cd-crushmap', @@ -14,10 +14,10 @@ export class CrushmapComponent implements OnInit { metadata: any; metadataKeyMap: { [key: number]: number } = {}; - constructor(private dashboardService: DashboardService) {} + constructor(private healthService: HealthService) {} ngOnInit() { - this.dashboardService.getHealth().subscribe((data: any) => { + this.healthService.getFullHealth().subscribe((data: any) => { this.tree = this._abstractTreeData(data); }); } diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/logs/logs.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/logs/logs.component.ts index 19d3ec97506..d3a9a341ad7 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/logs/logs.component.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/logs/logs.component.ts @@ -1,6 +1,6 @@ import { Component, OnDestroy, OnInit } from '@angular/core'; -import { DashboardService } from '../../../shared/api/dashboard.service'; +import { LogsService } from '../../../shared/api/logs.service'; @Component({ selector: 'cd-logs', @@ -11,7 +11,7 @@ export class LogsComponent implements OnInit, OnDestroy { contentData: any; interval: number; - constructor(private dashboardService: DashboardService) {} + constructor(private logsService: LogsService) {} ngOnInit() { this.getInfo(); @@ -25,7 +25,7 @@ export class LogsComponent implements OnInit, OnDestroy { } getInfo() { - this.dashboardService.getHealth().subscribe((data: any) => { + this.logsService.getLogs().subscribe((data: any) => { this.contentData = data; }); } diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard/health/health.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard/health/health.component.html index efe3657e3f1..54d9e40bd15 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard/health/health.component.html +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard/health/health.component.html @@ -1,34 +1,34 @@ -
+ *ngIf="healthData.health?.status + || healthData.mon_status + || healthData.osd_map + || healthData.mgr_map + || healthData.hosts != null + || healthData.rgw != null + || healthData.fs_map + || healthData.iscsi_daemons != null"> - + [contentClass]="healthData.health?.checks?.length > 0 ? 'content-highlight text-area-size-2' : 'content-highlight'" + *ngIf="healthData.health?.status"> +

→ See Logs for more details.

    -
  • +
  • {{ check.type }}: {{ check.summary.message }}
- {{ contentData.health.status }} + {{ healthData.health.status }}
- -
- {{ contentData.health.status }} + +
+ {{ healthData.health.status }}
@@ -51,15 +51,15 @@ link="/monitor" class="col-sm-6 col-md-4 col-lg-3" contentClass="content-highlight" - *ngIf="contentData.mon_status"> - {{ contentData.mon_status | monSummary }} + *ngIf="healthData.mon_status"> + {{ healthData.mon_status | monSummary }} @@ -71,8 +71,8 @@ i18n-cardTitle class="col-sm-6 col-md-4 col-lg-3" contentClass="content-highlight text-area-size-2" - *ngIf="contentData.mgr_map"> - + {{ result.content }} @@ -84,8 +84,8 @@ link="/hosts" class="col-sm-6 col-md-4 col-lg-3" contentClass="content-medium content-highlight" - *ngIf="contentData.hosts != null"> - {{ contentData.hosts }} total + *ngIf="healthData.hosts != null"> + {{ healthData.hosts }} total - {{ contentData.rgw }} total + *ngIf="healthData.rgw != null"> + {{ healthData.rgw }} total @@ -112,15 +112,15 @@ link="/block/iscsi" class="col-sm-6 col-md-4 col-lg-3" contentClass="content-medium content-highlight" - *ngIf="contentData.iscsi_daemons != null"> - {{ contentData.iscsi_daemons }} total + *ngIf="healthData.iscsi_daemons != null"> + {{ healthData.iscsi_daemons }} total + *ngIf="healthData.client_perf || healthData.scrub_status">
- {{ (contentData.client_perf.read_op_per_sec + contentData.client_perf.write_op_per_sec) | round:1 }} + *ngIf="healthData.client_perf"> + {{ (healthData.client_perf.read_op_per_sec + healthData.client_perf.write_op_per_sec) | round:1 }} - {{ ((contentData.client_perf.read_bytes_sec + contentData.client_perf.write_bytes_sec) | dimlessBinary) + '/s' }} + *ngIf="healthData.client_perf"> + {{ ((healthData.client_perf.read_bytes_sec + healthData.client_perf.write_bytes_sec) | dimlessBinary) + '/s' }} - + - + N/A @@ -164,8 +164,8 @@ class="cd-col-5" cardClass="card-medium" contentClass="content-medium content-highlight" - *ngIf="contentData.client_perf"> - {{ (contentData.client_perf.recovering_bytes_per_sec | dimlessBinary) + '/s' }} + *ngIf="healthData.client_perf"> + {{ (healthData.client_perf.recovering_bytes_per_sec | dimlessBinary) + '/s' }} - {{ contentData.scrub_status }} + *ngIf="healthData.scrub_status"> + {{ healthData.scrub_status }}
@@ -182,10 +182,10 @@ + *ngIf="healthData.pools + || healthData.df + || healthData.df?.stats?.total_objects != null + || healthData.pg_info">
- {{ contentData.pools.length }} + *ngIf="healthData.pools"> + {{ healthData.pools.length }} - + @@ -216,8 +216,8 @@ class="cd-col-5" cardClass="card-medium" contentClass="content-medium content-highlight" - *ngIf="contentData.df?.stats?.total_objects != null"> - {{ contentData.df?.stats?.total_objects }} + *ngIf="healthData.df?.stats?.total_objects != null"> + {{ healthData.df?.stats?.total_objects }} - {{ contentData.pg_info.pgs_per_osd | dimless }} + *ngIf="healthData.pg_info"> + {{ healthData.pg_info.pgs_per_osd | dimless }} + *ngIf="healthData.pg_info">

→ See Logs for more details.

    -
  • +
  • {{ pgStatesText.key }}: {{ pgStatesText.value }}
@@ -250,7 +250,7 @@ triggers="" #pgStatusTarget="bs-popover" placement="bottom"> - diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard/health/health.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard/health/health.component.spec.ts index 44593d6ac97..f4de0412581 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard/health/health.component.spec.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard/health/health.component.spec.ts @@ -7,9 +7,8 @@ import { PopoverModule } from 'ngx-bootstrap/popover'; import { of } from 'rxjs'; import { configureTestBed, i18nProviders } from '../../../../testing/unit-test-helper'; -import { DashboardService } from '../../../shared/api/dashboard.service'; +import { HealthService } from '../../../shared/api/health.service'; import { SharedModule } from '../../../shared/shared.module'; -import { LogColorPipe } from '../log-color.pipe'; import { MdsSummaryPipe } from '../mds-summary.pipe'; import { MgrSummaryPipe } from '../mgr-summary.pipe'; import { MonSummaryPipe } from '../mon-summary.pipe'; @@ -47,7 +46,6 @@ describe('HealthComponent', () => { MdsSummaryPipe, MgrSummaryPipe, PgStatusStylePipe, - LogColorPipe, PgStatusPipe ], schemas: [NO_ERRORS_SCHEMA], @@ -57,7 +55,7 @@ describe('HealthComponent', () => { beforeEach(() => { fixture = TestBed.createComponent(HealthComponent); component = fixture.componentInstance; - getHealthSpy = spyOn(TestBed.get(DashboardService), 'getHealth'); + getHealthSpy = spyOn(TestBed.get(HealthService), 'getMinimalHealth'); }); it('should create', () => { diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard/health/health.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard/health/health.component.ts index eb65a16f7ad..5ec12fee396 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard/health/health.component.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/dashboard/health/health.component.ts @@ -3,7 +3,7 @@ import { Component, OnDestroy, OnInit } from '@angular/core'; import { I18n } from '@ngx-translate/i18n-polyfill'; import * as _ from 'lodash'; -import { DashboardService } from '../../../shared/api/dashboard.service'; +import { HealthService } from '../../../shared/api/health.service'; @Component({ selector: 'cd-health', @@ -11,15 +11,15 @@ import { DashboardService } from '../../../shared/api/dashboard.service'; styleUrls: ['./health.component.scss'] }) export class HealthComponent implements OnInit, OnDestroy { - contentData: any; + healthData: any; interval: number; - constructor(private dashboardService: DashboardService, private i18n: I18n) {} + constructor(private healthService: HealthService, private i18n: I18n) {} ngOnInit() { - this.getInfo(); + this.getHealth(); this.interval = window.setInterval(() => { - this.getInfo(); + this.getHealth(); }, 5000); } @@ -27,9 +27,9 @@ export class HealthComponent implements OnInit, OnDestroy { clearInterval(this.interval); } - getInfo() { - this.dashboardService.getHealth().subscribe((data: any) => { - this.contentData = data; + getHealth() { + this.healthService.getMinimalHealth().subscribe((data: any) => { + this.healthData = data; }); } @@ -38,9 +38,9 @@ export class HealthComponent implements OnInit, OnDestroy { const ratioData = []; ratioLabels.push(this.i18n('Writes')); - ratioData.push(this.contentData.client_perf.write_op_per_sec); + ratioData.push(this.healthData.client_perf.write_op_per_sec); ratioLabels.push(this.i18n('Reads')); - ratioData.push(this.contentData.client_perf.read_op_per_sec); + ratioData.push(this.healthData.client_perf.read_op_per_sec); chart.dataset[0].data = ratioData; chart.labels = ratioLabels; diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/health.service.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/health.service.spec.ts new file mode 100644 index 00000000000..8273640188e --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/health.service.spec.ts @@ -0,0 +1,40 @@ +import { HttpClientTestingModule, HttpTestingController } from '@angular/common/http/testing'; +import { TestBed } from '@angular/core/testing'; + +import { configureTestBed } from '../../../testing/unit-test-helper'; +import { HealthService } from './health.service'; + +describe('HealthService', () => { + let service: HealthService; + let httpTesting: HttpTestingController; + + configureTestBed({ + providers: [HealthService], + imports: [HttpClientTestingModule] + }); + + beforeEach(() => { + service = TestBed.get(HealthService); + httpTesting = TestBed.get(HttpTestingController); + }); + + afterEach(() => { + httpTesting.verify(); + }); + + it('should be created', () => { + expect(service).toBeTruthy(); + }); + + it('should call getFullHealth', () => { + service.getFullHealth().subscribe(); + const req = httpTesting.expectOne('api/health/full'); + expect(req.request.method).toBe('GET'); + }); + + it('should call getMinimalHealth', () => { + service.getMinimalHealth().subscribe(); + const req = httpTesting.expectOne('api/health/minimal'); + expect(req.request.method).toBe('GET'); + }); +}); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/health.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/health.service.ts new file mode 100644 index 00000000000..e3ac39342e5 --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/health.service.ts @@ -0,0 +1,19 @@ +import { HttpClient } from '@angular/common/http'; +import { Injectable } from '@angular/core'; + +import { ApiModule } from './api.module'; + +@Injectable({ + providedIn: ApiModule +}) +export class HealthService { + constructor(private http: HttpClient) {} + + getFullHealth() { + return this.http.get('api/health/full'); + } + + getMinimalHealth() { + return this.http.get('api/health/minimal'); + } +} diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/dashboard.service.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/logs.service.spec.ts similarity index 63% rename from src/pybind/mgr/dashboard/frontend/src/app/shared/api/dashboard.service.spec.ts rename to src/pybind/mgr/dashboard/frontend/src/app/shared/api/logs.service.spec.ts index dcfde3073d5..8e7777a63f4 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/dashboard.service.spec.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/logs.service.spec.ts @@ -2,19 +2,19 @@ import { HttpClientTestingModule, HttpTestingController } from '@angular/common/ import { TestBed } from '@angular/core/testing'; import { configureTestBed } from '../../../testing/unit-test-helper'; -import { DashboardService } from './dashboard.service'; +import { LogsService } from './logs.service'; -describe('DashboardService', () => { - let service: DashboardService; +describe('LogsService', () => { + let service: LogsService; let httpTesting: HttpTestingController; configureTestBed({ - providers: [DashboardService], + providers: [LogsService], imports: [HttpClientTestingModule] }); beforeEach(() => { - service = TestBed.get(DashboardService); + service = TestBed.get(LogsService); httpTesting = TestBed.get(HttpTestingController); }); @@ -26,9 +26,9 @@ describe('DashboardService', () => { expect(service).toBeTruthy(); }); - it('should call getHealth', () => { - service.getHealth().subscribe(); - const req = httpTesting.expectOne('api/dashboard/health'); + it('should call getLogs', () => { + service.getLogs().subscribe(); + const req = httpTesting.expectOne('api/logs/all'); expect(req.request.method).toBe('GET'); }); }); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/dashboard.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/logs.service.ts similarity index 70% rename from src/pybind/mgr/dashboard/frontend/src/app/shared/api/dashboard.service.ts rename to src/pybind/mgr/dashboard/frontend/src/app/shared/api/logs.service.ts index 04bdece8cf6..81b868a5fec 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/dashboard.service.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/logs.service.ts @@ -6,10 +6,10 @@ import { ApiModule } from './api.module'; @Injectable({ providedIn: ApiModule }) -export class DashboardService { +export class LogsService { constructor(private http: HttpClient) {} - getHealth() { - return this.http.get('api/dashboard/health'); + getLogs() { + return this.http.get('api/logs/all'); } } -- 2.39.5