From cc2072312f35e0a2edc1c9d57f9c2b555be49ca4 Mon Sep 17 00:00:00 2001 From: Ernesto Puerta Date: Thu, 25 Jun 2020 11:17:22 +0200 Subject: [PATCH] mgr/dashboard: fix pool usage calculation Currently Dashboard Pool usage calculation does not match the output of 'ceph df' command. Fixes: https://tracker.ceph.com/issues/45185 Signed-off-by: Ernesto Puerta (cherry picked from commit b4a9dc17a3de90379964443d26b29f1759824f28) Conflicts: qa/tasks/mgr/dashboard/helper.py src/mon/PGMap.cc src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.html src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.spec.ts src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.ts src/pybind/mgr/dashboard/frontend/src/app/shared/components/usage-bar/usage-bar.component.ts: - Keep UsageBar component totalBytes/usedBytes names - Bring new UsageBar option decimal Signed-off-by: Ernesto Puerta --- qa/tasks/mgr/dashboard/helper.py | 16 ++++++++++++++++ qa/tasks/mgr/dashboard/test_health.py | 3 ++- qa/tasks/mgr/dashboard/test_pool.py | 6 ++++-- src/mon/PGMap.cc | 3 ++- .../ceph/pool/pool-list/pool-list.component.html | 7 ++++--- .../pool/pool-list/pool-list.component.spec.ts | 14 +++++++++++--- .../ceph/pool/pool-list/pool-list.component.ts | 14 +++++++++++--- .../frontend/src/app/ceph/pool/pool-stat.ts | 2 ++ .../usage-bar/usage-bar.component.html | 2 +- .../components/usage-bar/usage-bar.component.ts | 4 +++- 10 files changed, 56 insertions(+), 15 deletions(-) diff --git a/qa/tasks/mgr/dashboard/helper.py b/qa/tasks/mgr/dashboard/helper.py index 854169fb61099..9627a8428f84e 100644 --- a/qa/tasks/mgr/dashboard/helper.py +++ b/qa/tasks/mgr/dashboard/helper.py @@ -18,6 +18,9 @@ log = logging.getLogger(__name__) class DashboardTestCase(MgrTestCase): + # Display full error diffs + maxDiff = None + # Increased x3 (20 -> 60) TIMEOUT_HEALTH_CLEAR = 60 @@ -433,6 +436,7 @@ JList = namedtuple('JList', ['elem_typ']) JTuple = namedtuple('JList', ['elem_typs']) +JUnion = namedtuple('JUnion', ['elem_typs']) class JObj(namedtuple('JObj', ['sub_elems', 'allow_unknown', 'none', 'unknown_schema'])): def __new__(cls, sub_elems, allow_unknown=False, none=False, unknown_schema=None): @@ -462,6 +466,10 @@ def _validate_json(val, schema, path=[]): ... ds = JObj({'a': int, 'b': str, 'c': JList(int)}) ... _validate_json(d, ds) True + >>> _validate_json({'num': 1}, JObj({'num': JUnion([int,float])})) + True + >>> _validate_json({'num': 'a'}, JObj({'num': JUnion([int,float])})) + False """ if isinstance(schema, JAny): if not schema.none and val is None: @@ -480,6 +488,14 @@ def _validate_json(val, schema, path=[]): if isinstance(schema, JTuple): return all(_validate_json(val[i], typ, path + [i]) for i, typ in enumerate(schema.elem_typs)) + if isinstance(schema, JUnion): + for typ in schema.elem_typs: + try: + if _validate_json(val, typ, path): + return True + except _ValError: + pass + return False if isinstance(schema, JObj): if val is None and schema.none: return True diff --git a/qa/tasks/mgr/dashboard/test_health.py b/qa/tasks/mgr/dashboard/test_health.py index 65ea21b16da65..a9334edc35dd1 100644 --- a/qa/tasks/mgr/dashboard/test_health.py +++ b/qa/tasks/mgr/dashboard/test_health.py @@ -161,7 +161,8 @@ class HealthTest(DashboardTestCase): 'wr_bytes': int, 'compress_bytes_used': int, 'compress_under_bytes': int, - 'stored_raw': int + 'stored_raw': int, + 'avail_raw': int }), 'name': str, 'id': int diff --git a/qa/tasks/mgr/dashboard/test_pool.py b/qa/tasks/mgr/dashboard/test_pool.py index 58b78a4dee496..59d86d3b0e604 100644 --- a/qa/tasks/mgr/dashboard/test_pool.py +++ b/qa/tasks/mgr/dashboard/test_pool.py @@ -6,7 +6,7 @@ import six import time from contextlib import contextmanager -from .helper import DashboardTestCase, JAny, JList, JObj +from .helper import DashboardTestCase, JAny, JList, JObj, JUnion log = logging.getLogger(__name__) @@ -23,14 +23,16 @@ class PoolTest(DashboardTestCase): }, allow_unknown=True) pool_list_stat_schema = JObj(sub_elems={ - 'latest': int, + 'latest': JUnion([int,float]), 'rate': float, 'rates': JList(JAny(none=False)), }) pool_list_stats_schema = JObj(sub_elems={ + 'avail_raw': pool_list_stat_schema, 'bytes_used': pool_list_stat_schema, 'max_avail': pool_list_stat_schema, + 'percent_used': pool_list_stat_schema, 'rd_bytes': pool_list_stat_schema, 'wr_bytes': pool_list_stat_schema, 'rd': pool_list_stat_schema, diff --git a/src/mon/PGMap.cc b/src/mon/PGMap.cc index ce6abdf537c58..952e1ae19d8b1 100644 --- a/src/mon/PGMap.cc +++ b/src/mon/PGMap.cc @@ -936,6 +936,7 @@ void PGMapDigest::dump_object_stat_sum( f->dump_int("compress_under_bytes", statfs.data_compressed_original); // Stored by user amplified by replication f->dump_int("stored_raw", pool_stat.get_user_bytes(1.0, per_pool)); + f->dump_unsigned("avail_raw", avail); } } else { tbl << stringify(byte_u_t(stored_normalized)); @@ -1146,7 +1147,7 @@ void PGMap::apply_incremental(CephContext *cct, const Incremental& inc) auto pool_statfs_iter = pool_statfs.find(std::make_pair(update_pool, update_osd)); - if (pg_pool_sum.count(update_pool)) { + if (pg_pool_sum.count(update_pool)) { pool_stat_t &pool_sum_ref = pg_pool_sum[update_pool]; if (pool_statfs_iter == pool_statfs.end()) { pool_statfs.emplace(std::make_pair(update_pool, update_osd), statfs_inc); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.html index 124a670d2353c..0a83df29efc19 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.html +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.html @@ -23,9 +23,10 @@ - + diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.spec.ts index 3a365223ffd90..3f917e20a8c59 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.spec.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.spec.ts @@ -302,6 +302,8 @@ describe('PoolListComponent', () => { stats: { bytes_used: { latest: 0, rate: 0, rates: [] }, max_avail: { latest: 0, rate: 0, rates: [] }, + avail_raw: { latest: 0, rate: 0, rates: [] }, + percent_used: { latest: 0, rate: 0, rates: [] }, rd: { latest: 0, rate: 0, rates: [] }, rd_bytes: { latest: 0, rate: 0, rates: [] }, wr: { latest: 0, rate: 0, rates: [] }, @@ -321,8 +323,13 @@ describe('PoolListComponent', () => { pool = _.merge(pool, { stats: { bytes_used: { latest: 5, rate: 0, rates: [] }, - max_avail: { latest: 15, rate: 0, rates: [] }, - rd_bytes: { latest: 6, rate: 4, rates: [[0, 2], [1, 6]] } + avail_raw: { latest: 15, rate: 0, rates: [] }, + percent_used: { latest: 0.25, rate: 0, rates: [] }, + rd_bytes: { + latest: 6, + rate: 4, + rates: [[0, 2], [1, 6]] + } }, pg_status: { 'active+clean': 8, down: 2 } }); @@ -331,7 +338,8 @@ describe('PoolListComponent', () => { pg_status: '8 active+clean, 2 down', stats: { bytes_used: { latest: 5, rate: 0, rates: [] }, - max_avail: { latest: 15, rate: 0, rates: [] }, + avail_raw: { latest: 15, rate: 0, rates: [] }, + percent_used: { latest: 0.25, rate: 0, rates: [] }, rd_bytes: { latest: 6, rate: 4, rates: [2, 6] } }, usage: 0.25 diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.ts index 935bbf224c06f..4b2940dcea74d 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.ts @@ -239,7 +239,16 @@ export class PoolListComponent implements OnInit { } transformPoolsData(pools: any) { - const requiredStats = ['bytes_used', 'max_avail', 'rd_bytes', 'wr_bytes', 'rd', 'wr']; + const requiredStats = [ + 'bytes_used', + 'max_avail', + 'avail_raw', + 'percent_used', + 'rd_bytes', + 'wr_bytes', + 'rd', + 'wr' + ]; const emptyStat = { latest: 0, rate: 0, rates: [] }; _.forEach(pools, (pool: Pool) => { @@ -249,8 +258,7 @@ export class PoolListComponent implements OnInit { stats[stat] = pool.stats && pool.stats[stat] ? pool.stats[stat] : emptyStat; }); pool['stats'] = stats; - const avail = stats.bytes_used.latest + stats.max_avail.latest; - pool['usage'] = avail > 0 ? stats.bytes_used.latest / avail : avail; + pool['usage'] = stats.percent_used.latest; if ( !pool.cdExecuting && diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-stat.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-stat.ts index f746bf3866ea5..9820be94a8de9 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-stat.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-stat.ts @@ -7,6 +7,8 @@ export class PoolStat { export class PoolStats { bytes_used?: PoolStat; max_avail?: PoolStat; + avail_raw?: PoolStat; + percent_used?: PoolStat; rd_bytes?: PoolStat; wr_bytes?: PoolStat; rd?: PoolStat; diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/usage-bar/usage-bar.component.html b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/usage-bar/usage-bar.component.html index 93348f16acb5e..65d9a89d4d8f1 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/usage-bar/usage-bar.component.html +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/usage-bar/usage-bar.component.html @@ -17,7 +17,7 @@
- {{ usedPercentage }}% + {{ usedPercentage | number: '1.0-' + decimals }}%
0 ? (this.usedBytes / this.totalBytes) * 100 : 0; this.freePercentage = 100 - this.usedPercentage; this.freeBytes = this.totalBytes - this.usedBytes; } -- 2.39.5