From b4a9dc17a3de90379964443d26b29f1759824f28 Mon Sep 17 00:00:00 2001 From: Ernesto Puerta Date: Thu, 25 Jun 2020 11:17:22 +0200 Subject: [PATCH] mgr/dashboard: fix pool usage calculation Currently Dashboard Pool usage calculation does not match the output of 'ceph df' command. Fixes: https://tracker.ceph.com/issues/45185 Signed-off-by: Ernesto Puerta --- qa/tasks/mgr/dashboard/helper.py | 16 ++++++++++++++++ qa/tasks/mgr/dashboard/test_health.py | 3 ++- qa/tasks/mgr/dashboard/test_pool.py | 6 ++++-- src/mon/PGMap.cc | 3 ++- .../ceph/pool/pool-list/pool-list.component.html | 7 ++++--- .../pool/pool-list/pool-list.component.spec.ts | 8 ++++++-- .../ceph/pool/pool-list/pool-list.component.ts | 14 +++++++++++--- .../frontend/src/app/ceph/pool/pool-stat.ts | 2 ++ .../usage-bar/usage-bar.component.html | 2 +- .../components/usage-bar/usage-bar.component.ts | 4 +++- 10 files changed, 51 insertions(+), 14 deletions(-) diff --git a/qa/tasks/mgr/dashboard/helper.py b/qa/tasks/mgr/dashboard/helper.py index 1acca15373bd4..44f919b11190e 100644 --- a/qa/tasks/mgr/dashboard/helper.py +++ b/qa/tasks/mgr/dashboard/helper.py @@ -18,6 +18,9 @@ log = logging.getLogger(__name__) class DashboardTestCase(MgrTestCase): + # Display full error diffs + maxDiff = None + MGRS_REQUIRED = 2 MDSS_REQUIRED = 1 REQUIRE_FILESYSTEM = True @@ -467,6 +470,7 @@ JList = namedtuple('JList', ['elem_typ']) JTuple = namedtuple('JList', ['elem_typs']) +JUnion = namedtuple('JUnion', ['elem_typs']) class JObj(namedtuple('JObj', ['sub_elems', 'allow_unknown', 'none', 'unknown_schema'])): def __new__(cls, sub_elems, allow_unknown=False, none=False, unknown_schema=None): @@ -496,6 +500,10 @@ def _validate_json(val, schema, path=[]): ... ds = JObj({'a': int, 'b': str, 'c': JList(int)}) ... _validate_json(d, ds) True + >>> _validate_json({'num': 1}, JObj({'num': JUnion([int,float])})) + True + >>> _validate_json({'num': 'a'}, JObj({'num': JUnion([int,float])})) + False """ if isinstance(schema, JAny): if not schema.none and val is None: @@ -514,6 +522,14 @@ def _validate_json(val, schema, path=[]): if isinstance(schema, JTuple): return all(_validate_json(val[i], typ, path + [i]) for i, typ in enumerate(schema.elem_typs)) + if isinstance(schema, JUnion): + for typ in schema.elem_typs: + try: + if _validate_json(val, typ, path): + return True + except _ValError: + pass + return False if isinstance(schema, JObj): if val is None and schema.none: return True diff --git a/qa/tasks/mgr/dashboard/test_health.py b/qa/tasks/mgr/dashboard/test_health.py index 08551e488ab99..e4091874425c4 100644 --- a/qa/tasks/mgr/dashboard/test_health.py +++ b/qa/tasks/mgr/dashboard/test_health.py @@ -169,7 +169,8 @@ class HealthTest(DashboardTestCase): 'wr_bytes': int, 'compress_bytes_used': int, 'compress_under_bytes': int, - 'stored_raw': int + 'stored_raw': int, + 'avail_raw': int }), 'name': str, 'id': int diff --git a/qa/tasks/mgr/dashboard/test_pool.py b/qa/tasks/mgr/dashboard/test_pool.py index c2c25bc1aafe3..2d34bc75ce136 100644 --- a/qa/tasks/mgr/dashboard/test_pool.py +++ b/qa/tasks/mgr/dashboard/test_pool.py @@ -6,7 +6,7 @@ import six import time from contextlib import contextmanager -from .helper import DashboardTestCase, JAny, JList, JObj +from .helper import DashboardTestCase, JAny, JList, JObj, JUnion log = logging.getLogger(__name__) @@ -23,14 +23,16 @@ class PoolTest(DashboardTestCase): }, allow_unknown=True) pool_list_stat_schema = JObj(sub_elems={ - 'latest': int, + 'latest': JUnion([int,float]), 'rate': float, 'rates': JList(JAny(none=False)), }) pool_list_stats_schema = JObj(sub_elems={ + 'avail_raw': pool_list_stat_schema, 'bytes_used': pool_list_stat_schema, 'max_avail': pool_list_stat_schema, + 'percent_used': pool_list_stat_schema, 'rd_bytes': pool_list_stat_schema, 'wr_bytes': pool_list_stat_schema, 'rd': pool_list_stat_schema, diff --git a/src/mon/PGMap.cc b/src/mon/PGMap.cc index 24af54ba0ccaa..0f686d0a93960 100644 --- a/src/mon/PGMap.cc +++ b/src/mon/PGMap.cc @@ -960,6 +960,7 @@ void PGMapDigest::dump_object_stat_sum( f->dump_int("compress_under_bytes", statfs.data_compressed_original); // Stored by user amplified by replication f->dump_int("stored_raw", stored_raw); + f->dump_unsigned("avail_raw", avail); } } else { tbl << stringify(byte_u_t(stored_normalized)); @@ -1178,7 +1179,7 @@ void PGMap::apply_incremental(CephContext *cct, const Incremental& inc) auto pool_statfs_iter = pool_statfs.find(std::make_pair(update_pool, update_osd)); - if (pg_pool_sum.count(update_pool)) { + if (pg_pool_sum.count(update_pool)) { pool_stat_t &pool_sum_ref = pg_pool_sum[update_pool]; if (pool_statfs_iter == pool_statfs.end()) { pool_statfs.emplace(std::make_pair(update_pool, update_osd), statfs_inc); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.html index 1bc7dbcdb7421..bd313df7de1b0 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.html +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.html @@ -50,8 +50,9 @@ - + diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.spec.ts index 3b1de559f3028..73ce4df019242 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.spec.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.spec.ts @@ -309,6 +309,8 @@ describe('PoolListComponent', () => { stats: { bytes_used: { latest: 0, rate: 0, rates: [] }, max_avail: { latest: 0, rate: 0, rates: [] }, + avail_raw: { latest: 0, rate: 0, rates: [] }, + percent_used: { latest: 0, rate: 0, rates: [] }, rd: { latest: 0, rate: 0, rates: [] }, rd_bytes: { latest: 0, rate: 0, rates: [] }, wr: { latest: 0, rate: 0, rates: [] }, @@ -328,7 +330,8 @@ describe('PoolListComponent', () => { pool = _.merge(pool, { stats: { bytes_used: { latest: 5, rate: 0, rates: [] }, - max_avail: { latest: 15, rate: 0, rates: [] }, + avail_raw: { latest: 15, rate: 0, rates: [] }, + percent_used: { latest: 0.25, rate: 0, rates: [] }, rd_bytes: { latest: 6, rate: 4, @@ -345,7 +348,8 @@ describe('PoolListComponent', () => { pg_status: '8 active+clean, 2 down', stats: { bytes_used: { latest: 5, rate: 0, rates: [] }, - max_avail: { latest: 15, rate: 0, rates: [] }, + avail_raw: { latest: 15, rate: 0, rates: [] }, + percent_used: { latest: 0.25, rate: 0, rates: [] }, rd_bytes: { latest: 6, rate: 4, rates: [2, 6] } }, usage: 0.25 diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.ts index b0d63284b0ac3..4f70d1fd08d0d 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.ts @@ -241,7 +241,16 @@ export class PoolListComponent extends ListWithDetails implements OnInit { } transformPoolsData(pools: any) { - const requiredStats = ['bytes_used', 'max_avail', 'rd_bytes', 'wr_bytes', 'rd', 'wr']; + const requiredStats = [ + 'bytes_used', + 'max_avail', + 'avail_raw', + 'percent_used', + 'rd_bytes', + 'wr_bytes', + 'rd', + 'wr' + ]; const emptyStat: PoolStat = { latest: 0, rate: 0, rates: [] }; _.forEach(pools, (pool: Pool) => { @@ -251,8 +260,7 @@ export class PoolListComponent extends ListWithDetails implements OnInit { stats[stat] = pool.stats && pool.stats[stat] ? pool.stats[stat] : emptyStat; }); pool['stats'] = stats; - const avail = stats.bytes_used.latest + stats.max_avail.latest; - pool['usage'] = avail > 0 ? stats.bytes_used.latest / avail : avail; + pool['usage'] = stats.percent_used.latest; if ( !pool.cdExecuting && diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-stat.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-stat.ts index f746bf3866ea5..9820be94a8de9 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-stat.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-stat.ts @@ -7,6 +7,8 @@ export class PoolStat { export class PoolStats { bytes_used?: PoolStat; max_avail?: PoolStat; + avail_raw?: PoolStat; + percent_used?: PoolStat; rd_bytes?: PoolStat; wr_bytes?: PoolStat; rd?: PoolStat; diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/usage-bar/usage-bar.component.html b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/usage-bar/usage-bar.component.html index 42f995efdbbf8..0ecf79f1cdf9a 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/usage-bar/usage-bar.component.html +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/usage-bar/usage-bar.component.html @@ -17,7 +17,7 @@
- {{ usedPercentage }}% + {{ usedPercentage | number: '1.0-' + decimals }}%
0 ? Math.round((this.used / this.total) * 100) : 0; + this.usedPercentage = this.total > 0 ? (this.used / this.total) * 100 : 0; this.freePercentage = 100 - this.usedPercentage; } } -- 2.39.5