From d9404d68130c9cd4371b0ff90b66d5230c323d3b Mon Sep 17 00:00:00 2001 From: alfonsomthd Date: Wed, 9 Jan 2019 08:58:54 +0100 Subject: [PATCH] mgr/dashboard: add columns to Pools table MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit * Added columns: Usage, Read bytes, Write bytes, Read ops, Write ops. * TableKeyValueComponent: Bugfix in method: _insertFlattenObjects * Updated jest global mock: Now window.getComputedStyle returns an object that has getPropertyValue defined (otherwise jest pool tests involving chart.js threw error). See: https://developer.mozilla.org/en-US/docs/Web/API/Window/getComputedStyle Fixes: https://tracker.ceph.com/issues/37717 Fixes: https://tracker.ceph.com/issues/34320 Signed-off-by: Alfonso Martínez --- qa/tasks/mgr/dashboard/test_pool.py | 22 +++++- .../pool/pool-list/pool-list.component.html | 10 ++- .../pool-list/pool-list.component.spec.ts | 33 ++++++++- .../pool/pool-list/pool-list.component.ts | 68 ++++++++++++++++--- .../frontend/src/app/ceph/pool/pool-stat.ts | 5 ++ .../frontend/src/app/ceph/pool/pool.ts | 14 ++-- .../table-key-value.component.spec.ts | 16 +++-- .../table-key-value.component.ts | 30 +++++--- .../dashboard/frontend/src/jestGlobalMocks.ts | 6 +- .../frontend/src/locale/messages.xlf | 25 ++++++- src/pybind/mgr/dashboard/module.py | 15 ++++ .../mgr/dashboard/services/ceph_service.py | 15 +--- 12 files changed, 210 insertions(+), 49 deletions(-) create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-stat.ts diff --git a/qa/tasks/mgr/dashboard/test_pool.py b/qa/tasks/mgr/dashboard/test_pool.py index 9851397815e..590c3516073 100644 --- a/qa/tasks/mgr/dashboard/test_pool.py +++ b/qa/tasks/mgr/dashboard/test_pool.py @@ -6,7 +6,7 @@ import logging import six import time -from .helper import DashboardTestCase, JObj, JList +from .helper import DashboardTestCase, JAny, JList, JObj log = logging.getLogger(__name__) @@ -22,6 +22,21 @@ class PoolTest(DashboardTestCase): 'flags_names': str, }, allow_unknown=True) + pool_list_stat_schema = JObj(sub_elems={ + 'latest': int, + 'rate': float, + 'series': JList(JAny(none=False)), + }) + + pool_list_stats_schema = JObj(sub_elems={ + 'bytes_used': pool_list_stat_schema, + 'max_avail': pool_list_stat_schema, + 'rd_bytes': pool_list_stat_schema, + 'wr_bytes': pool_list_stat_schema, + 'rd': pool_list_stat_schema, + 'wr': pool_list_stat_schema, + }, allow_unknown=True) + def _pool_create(self, data): try: self._task_post('/api/pool/', data) @@ -152,13 +167,14 @@ class PoolTest(DashboardTestCase): cluster_pools = self.ceph_cluster.mon_manager.list_pools() self.assertEqual(len(cluster_pools), len(data)) + self.assertSchemaBody(JList(self.pool_schema)) for pool in data: self.assertIn('pool_name', pool) self.assertIn('type', pool) self.assertIn('application_metadata', pool) self.assertIn('flags', pool) self.assertIn('pg_status', pool) - self.assertIn('stats', pool) + self.assertSchema(pool['stats'], self.pool_list_stats_schema) self.assertIn('flags_names', pool) self.assertIn(pool['pool_name'], cluster_pools) @@ -170,7 +186,7 @@ class PoolTest(DashboardTestCase): self.assertIn('type', pool) self.assertIn('flags', pool) self.assertNotIn('pg_status', pool) - self.assertIn('stats', pool) + self.assertSchema(pool['stats'], self.pool_list_stats_schema) self.assertNotIn('flags_names', pool) def test_pool_create(self): diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.html index 16e48e0464a..66d00f4631c 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.html +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.html @@ -18,7 +18,7 @@ @@ -33,6 +33,14 @@ + + + + + { describe('transformPoolsData', () => { it('transforms pools data correctly', () => { - const pools = [{ stats: { rate: 0 }, pg_status: { 'active+clean': 8, down: 2 } }]; - const expected = [{ pg_status: '8 active+clean, 2 down' }]; + const pools = [ + { + stats: { rd_bytes: { latest: 6, rate: 4, series: [[0, 2], [1, 6]] } }, + pg_status: { 'active+clean': 8, down: 2 } + } + ]; + const expected = [ + { + cdIsBinary: true, + pg_status: '8 active+clean, 2 down', + stats: { + bytes_used: { latest: 0, rate: 0, series: [] }, + max_avail: { latest: 0, rate: 0, series: [] }, + rd: { latest: 0, rate: 0, series: [] }, + rd_bytes: { latest: 6, rate: 4, series: [2, 6] }, + wr: { latest: 0, rate: 0, series: [] }, + wr_bytes: { latest: 0, rate: 0, series: [] } + } + } + ]; expect(component.transformPoolsData(pools)).toEqual(expected); }); @@ -212,7 +230,7 @@ describe('PoolListComponent', () => { }); describe('transformPgStatus', () => { - it('returns ststus groups correctly', () => { + it('returns status groups correctly', () => { const pgStatus = { 'active+clean': 8 }; const expected = '8 active+clean'; @@ -240,4 +258,13 @@ describe('PoolListComponent', () => { expect(component.transformPgStatus(pgStatus)).toEqual(expected); }); }); + + describe('getPoolDetails', () => { + it('returns pool details corretly', () => { + const pool = { prop1: 1, cdIsBinary: true, prop2: 2, cdExecuting: true, prop3: 3 }; + const expected = { prop1: 1, prop2: 2, prop3: 3 }; + + expect(component.getPoolDetails(pool)).toEqual(expected); + }); + }); }); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.ts index bf58ce52263..46a921b0548 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.ts @@ -1,4 +1,4 @@ -import { Component, OnInit, ViewChild } from '@angular/core'; +import { Component, OnInit, TemplateRef, ViewChild } from '@angular/core'; import { I18n } from '@ngx-translate/i18n-polyfill'; import * as _ from 'lodash'; @@ -15,6 +15,7 @@ import { CdTableSelection } from '../../../shared/models/cd-table-selection'; import { ExecutingTask } from '../../../shared/models/executing-task'; import { FinishedTask } from '../../../shared/models/finished-task'; import { Permissions } from '../../../shared/models/permissions'; +import { DimlessPipe } from '../../../shared/pipes/dimless.pipe'; import { AuthStorageService } from '../../../shared/services/auth-storage.service'; import { TaskListService } from '../../../shared/services/task-list.service'; import { TaskWrapperService } from '../../../shared/services/task-wrapper.service'; @@ -30,6 +31,8 @@ import { Pool } from '../pool'; export class PoolListComponent implements OnInit { @ViewChild(TableComponent) table: TableComponent; + @ViewChild('poolUsageTpl') + poolUsageTpl: TemplateRef; pools: Pool[] = []; columns: CdTableColumn[]; @@ -47,7 +50,8 @@ export class PoolListComponent implements OnInit { private taskListService: TaskListService, private modalService: BsModalService, private i18n: I18n, - private pgCategoryService: PgCategoryService + private pgCategoryService: PgCategoryService, + private dimlessPipe: DimlessPipe ) { this.permissions = this.authStorageService.getPermissions(); this.tableActions = [ @@ -70,11 +74,14 @@ export class PoolListComponent implements OnInit { name: this.i18n('Delete') } ]; + } + + ngOnInit() { this.columns = [ { prop: 'pool_name', name: this.i18n('Name'), - flexGrow: 3, + flexGrow: 4, cellTransformation: CellTemplate.executing }, { @@ -85,12 +92,12 @@ export class PoolListComponent implements OnInit { { prop: 'application_metadata', name: this.i18n('Applications'), - flexGrow: 3 + flexGrow: 2 }, { prop: 'pg_status', name: this.i18n('PG Status'), - flexGrow: 1, + flexGrow: 3, cellClass: ({ row, column, value }): any => { return this.getPgStatusCellClass({ row, column, value }); } @@ -115,12 +122,37 @@ export class PoolListComponent implements OnInit { { prop: 'crush_rule', name: this.i18n('Crush Ruleset'), - flexGrow: 2 + flexGrow: 3 + }, + { name: this.i18n('Usage'), cellTemplate: this.poolUsageTpl, flexGrow: 3 }, + { + prop: 'stats.rd_bytes.series', + name: this.i18n('Read bytes'), + cellTransformation: CellTemplate.sparkline, + flexGrow: 3 + }, + { + prop: 'stats.wr_bytes.series', + name: this.i18n('Write bytes'), + cellTransformation: CellTemplate.sparkline, + flexGrow: 3 + }, + { + prop: 'stats.rd.rate', + name: this.i18n('Read ops'), + flexGrow: 1, + pipe: this.dimlessPipe, + cellTransformation: CellTemplate.perSecond + }, + { + prop: 'stats.wr.rate', + name: this.i18n('Write ops'), + flexGrow: 1, + pipe: this.dimlessPipe, + cellTransformation: CellTemplate.perSecond } ]; - } - ngOnInit() { this.taskListService.init( () => this.poolService.getList(), undefined, @@ -161,9 +193,21 @@ export class PoolListComponent implements OnInit { } transformPoolsData(pools: any) { - _.map(pools, (pool: object) => { - delete pool['stats']; + const requiredStats = ['bytes_used', 'max_avail', 'rd_bytes', 'wr_bytes', 'rd', 'wr']; + const emptyStat = { latest: 0, rate: 0, series: [] }; + + _.forEach(pools, (pool: Pool) => { pool['pg_status'] = this.transformPgStatus(pool['pg_status']); + const stats = {}; + _.forEach(requiredStats, (stat) => { + stats[stat] = pool.stats && pool.stats[stat] ? pool.stats[stat] : emptyStat; + }); + pool['stats'] = stats; + + ['rd_bytes', 'wr_bytes'].forEach((stat) => { + pool.stats[stat].series = pool.stats[stat].series.map((point) => point[1]); + }); + pool.cdIsBinary = true; }); return pools; @@ -177,4 +221,8 @@ export class PoolListComponent implements OnInit { return strings.join(', '); } + + getPoolDetails(pool: object) { + return _.omit(pool, ['cdExecuting', 'cdIsBinary']); + } } diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-stat.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-stat.ts new file mode 100644 index 00000000000..66b5852131c --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-stat.ts @@ -0,0 +1,5 @@ +export class PoolStat { + latest: number; + rate: number; + series: number[]; +} diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool.ts index 4241565d8b5..3ff9d5e9a16 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool.ts @@ -1,4 +1,5 @@ import { ExecutingTask } from '../../shared/models/executing-task'; +import { PoolStat } from './pool-stat'; export class Pool { cache_target_full_ratio_micro: number; @@ -7,7 +8,6 @@ export class Pool { flags_names: string; tier_of: number; hit_set_grade_decay_rate: number; - pg_placement_num: number; use_gmt_hitset: boolean; last_force_op_resend_preluminous: string; quota_max_bytes: number; @@ -22,10 +22,8 @@ export class Pool { target_max_objects: number; pg_num: number; type: string; - grade_table: any[]; pool_name: string; cache_min_evict_age: number; - snap_mode: string; cache_mode: string; min_size: number; cache_target_dirty_high_ratio_micro: number; @@ -55,12 +53,20 @@ export class Pool { hit_set_count: number; flags: number; target_max_bytes: number; - snap_epoch: number; hit_set_search_last_n: number; last_change: string; min_write_recency_for_promote: number; read_tier: number; pg_status: string; + stats?: { + bytes_used?: PoolStat; + max_avail?: PoolStat; + rd_bytes?: PoolStat; + wr_bytes?: PoolStat; + rd?: PoolStat; + wr?: PoolStat; + }; + cdIsBinary?: boolean; constructor(name) { this.pool_name = name; diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/datatable/table-key-value/table-key-value.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/datatable/table-key-value/table-key-value.component.spec.ts index ab6bd4bcb57..7ff6dd7e175 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/shared/datatable/table-key-value/table-key-value.component.spec.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/datatable/table-key-value/table-key-value.component.spec.ts @@ -199,9 +199,9 @@ describe('TableKeyValueComponent', () => { ]; expect(component._insertFlattenObjects(v)).toEqual([ { key: 'no', value: 'change' }, + { key: 'first layer', value: 'something' }, { key: 'first second l3_1', value: 33 }, - { key: 'first second l3_2', value: 44 }, - { key: 'first layer', value: 'something' } + { key: 'first second l3_2', value: 44 } ]); }); @@ -217,7 +217,9 @@ describe('TableKeyValueComponent', () => { sub3: 56 } }, - someKey: 0 + someKey: 0, + additionalKeyContainingObject: { type: 'none' }, + keyWithEmptyObject: {} }; component.renderObjects = true; }); @@ -225,12 +227,14 @@ describe('TableKeyValueComponent', () => { it('with parent key', () => { component.ngOnInit(); expect(component.tableData).toEqual([ + { key: 'someKey', value: 0 }, + { key: 'keyWithEmptyObject', value: '' }, { key: 'options someSetting1', value: 38 }, { key: 'options anotherSetting2', value: 'somethingElse' }, { key: 'options suboptions sub1', value: 12 }, { key: 'options suboptions sub2', value: 34 }, { key: 'options suboptions sub3', value: 56 }, - { key: 'someKey', value: 0 } + { key: 'additionalKeyContainingObject type', value: 'none' } ]); }); @@ -238,12 +242,14 @@ describe('TableKeyValueComponent', () => { component.appendParentKey = false; component.ngOnInit(); expect(component.tableData).toEqual([ + { key: 'someKey', value: 0 }, + { key: 'keyWithEmptyObject', value: '' }, { key: 'someSetting1', value: 38 }, { key: 'anotherSetting2', value: 'somethingElse' }, { key: 'sub1', value: 12 }, { key: 'sub2', value: 34 }, { key: 'sub3', value: 56 }, - { key: 'someKey', value: 0 } + { key: 'type', value: 'none' } ]); }); }); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/datatable/table-key-value/table-key-value.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/datatable/table-key-value/table-key-value.component.ts index 5439aab4931..c4fb5f9d1a7 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/shared/datatable/table-key-value/table-key-value.component.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/datatable/table-key-value/table-key-value.component.ts @@ -139,18 +139,32 @@ export class TableKeyValueComponent implements OnInit, OnChanges { } _insertFlattenObjects(temp: any[]) { + const itemsToRemoveIndexes = []; + const itemsToAdd = []; temp.forEach((v, i) => { if (_.isObject(v.value)) { - temp.splice(i, 1); - this._makePairs(v.value).forEach((item) => { - if (this.appendParentKey) { - item.key = v.key + ' ' + item.key; - } - temp.splice(i, 0, item); - i++; - }); + if (_.isEmpty(v.value)) { + temp[i]['value'] = ''; + } else { + itemsToRemoveIndexes.push(i); + this._makePairs(v.value).forEach((item) => { + if (this.appendParentKey) { + item.key = v.key + ' ' + item.key; + } + itemsToAdd.push(item); + i++; + }); + } } }); + + _.remove(temp, (item, itemIndex) => { + return _.includes(itemsToRemoveIndexes, itemIndex); + }); + itemsToAdd.forEach((item) => { + temp.push(item); + }); + return temp; } diff --git a/src/pybind/mgr/dashboard/frontend/src/jestGlobalMocks.ts b/src/pybind/mgr/dashboard/frontend/src/jestGlobalMocks.ts index d56cceaab12..3ccce9a1d28 100644 --- a/src/pybind/mgr/dashboard/frontend/src/jestGlobalMocks.ts +++ b/src/pybind/mgr/dashboard/frontend/src/jestGlobalMocks.ts @@ -11,5 +11,9 @@ const mock = () => { Object.defineProperty(window, 'localStorage', { value: mock() }); Object.defineProperty(window, 'sessionStorage', { value: mock() }); Object.defineProperty(window, 'getComputedStyle', { - value: () => ['-webkit-appearance'] + value: () => ({ + getPropertyValue: (prop) => { + return ''; + } + }) }); diff --git a/src/pybind/mgr/dashboard/frontend/src/locale/messages.xlf b/src/pybind/mgr/dashboard/frontend/src/locale/messages.xlf index 659fed32cf4..d87c86aa1d0 100644 --- a/src/pybind/mgr/dashboard/frontend/src/locale/messages.xlf +++ b/src/pybind/mgr/dashboard/frontend/src/locale/messages.xlf @@ -761,7 +761,7 @@ app/ceph/pool/pool-list/pool-list.component.html - 40 + 48 app/ceph/rgw/rgw-daemon-list/rgw-daemon-list.component.html @@ -3520,6 +3520,10 @@ src/app/ceph/cluster/osd/osd-list/osd-list.component.ts 1 + + src/app/ceph/pool/pool-list/pool-list.component.ts + 1 + Standby daemons @@ -4052,6 +4056,10 @@ src/app/ceph/cluster/osd/osd-list/osd-list.component.ts 1 + + src/app/ceph/pool/pool-list/pool-list.component.ts + 1 + Writes bytes @@ -4066,6 +4074,10 @@ src/app/ceph/cluster/osd/osd-list/osd-list.component.ts 1 + + src/app/ceph/pool/pool-list/pool-list.component.ts + 1 + Write ops @@ -4073,6 +4085,10 @@ src/app/ceph/cluster/osd/osd-list/osd-list.component.ts 1 + + src/app/ceph/pool/pool-list/pool-list.component.ts + 1 + Mark OSD @@ -4346,6 +4362,13 @@ 1 + + Write bytes + + src/app/ceph/pool/pool-list/pool-list.component.ts + 1 + + bucket diff --git a/src/pybind/mgr/dashboard/module.py b/src/pybind/mgr/dashboard/module.py index 35560e7bc27..d8f587f28c1 100644 --- a/src/pybind/mgr/dashboard/module.py +++ b/src/pybind/mgr/dashboard/module.py @@ -4,6 +4,7 @@ ceph dashboard mgr plugin (based on CherryPy) """ from __future__ import absolute_import +import collections import errno from distutils.version import StrictVersion from distutils.util import strtobool @@ -11,6 +12,7 @@ import os import socket import tempfile import threading +import time from uuid import uuid4 from OpenSSL import crypto @@ -252,6 +254,9 @@ class Module(MgrModule, CherryPyConfig): ] MODULE_OPTIONS.extend(options_schema_list()) + __pool_stats = collections.defaultdict(lambda: collections.defaultdict( + lambda: collections.deque(maxlen=10))) + def __init__(self, *args, **kwargs): super(Module, self).__init__(*args, **kwargs) CherryPyConfig.__init__(self) @@ -376,6 +381,16 @@ class Module(MgrModule, CherryPyConfig): def notify(self, notify_type, notify_id): NotificationQueue.new_notification(notify_type, notify_id) + def get_updated_pool_stats(self): + df = self.get('df') + pool_stats = dict([(p['id'], p['stats']) for p in df['pools']]) + now = time.time() + for pool_id, stats in pool_stats.items(): + for stat_name, stat_val in stats.items(): + self.__pool_stats[pool_id][stat_name].append((now, stat_val)) + + return self.__pool_stats + class StandbyModule(MgrStandbyModule, CherryPyConfig): def __init__(self, *args, **kwargs): diff --git a/src/pybind/mgr/dashboard/services/ceph_service.py b/src/pybind/mgr/dashboard/services/ceph_service.py index 91b4f351ef6..805ec6adc39 100644 --- a/src/pybind/mgr/dashboard/services/ceph_service.py +++ b/src/pybind/mgr/dashboard/services/ceph_service.py @@ -1,9 +1,6 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import -import time -import collections -from collections import defaultdict import json import rados @@ -103,15 +100,7 @@ class CephService(object): pools_w_stats = [] pg_summary = mgr.get("pg_summary") - pool_stats = defaultdict(lambda: defaultdict( - lambda: collections.deque(maxlen=10))) - - df = mgr.get("df") - pool_stats_dict = dict([(p['id'], p['stats']) for p in df['pools']]) - now = time.time() - for pool_id, stats in pool_stats_dict.items(): - for stat_name, stat_val in stats.items(): - pool_stats[pool_id][stat_name].appendleft((now, stat_val)) + pool_stats = mgr.get_updated_pool_stats() for pool in pools: pool['pg_status'] = pg_summary['by_pool'][pool['pool'].__str__()] @@ -120,7 +109,7 @@ class CephService(object): def get_rate(series): if len(series) >= 2: - return differentiate(*series[0:1]) + return differentiate(*list(series)[-2:]) return 0 for stat_name, stat_series in stats.items(): -- 2.39.5