mgr/dashboard: fix pool usage calculation

Currently Dashboard Pool usage calculation does not match the output of
'ceph df' command.

Fixes: https://tracker.ceph.com/issues/45185
Signed-off-by: Ernesto Puerta <epuertat@redhat.com>

diff --git a/qa/tasks/mgr/dashboard/helper.py b/qa/tasks/mgr/dashboard/helper.py
index 1acca15373bd4f3c9cb2bde09cf7dfc575f00c1a..44f919b11190eb17b559ed60a7ba7f83108fca54 100644 (file)
--- a/qa/tasks/mgr/dashboard/helper.py
+++ b/qa/tasks/mgr/dashboard/helper.py
@@ -18,6 +18,9 @@ log = logging.getLogger(__name__)
 
 
 class DashboardTestCase(MgrTestCase):
+    # Display full error diffs
+    maxDiff = None
+
     MGRS_REQUIRED = 2
     MDSS_REQUIRED = 1
     REQUIRE_FILESYSTEM = True
@@ -467,6 +470,7 @@ JList = namedtuple('JList', ['elem_typ'])
 
 JTuple = namedtuple('JList', ['elem_typs'])
 
+JUnion = namedtuple('JUnion', ['elem_typs'])
 
 class JObj(namedtuple('JObj', ['sub_elems', 'allow_unknown', 'none', 'unknown_schema'])):
     def __new__(cls, sub_elems, allow_unknown=False, none=False, unknown_schema=None):
@@ -496,6 +500,10 @@ def _validate_json(val, schema, path=[]):
     ... ds = JObj({'a': int, 'b': str, 'c': JList(int)})
     ... _validate_json(d, ds)
     True
+    >>> _validate_json({'num': 1}, JObj({'num': JUnion([int,float])}))
+    True
+    >>> _validate_json({'num': 'a'}, JObj({'num': JUnion([int,float])}))
+    False
     """
     if isinstance(schema, JAny):
         if not schema.none and val is None:
@@ -514,6 +522,14 @@ def _validate_json(val, schema, path=[]):
     if isinstance(schema, JTuple):
         return all(_validate_json(val[i], typ, path + [i])
                    for i, typ in enumerate(schema.elem_typs))
+    if isinstance(schema, JUnion):
+        for typ in schema.elem_typs:
+            try:
+                if _validate_json(val, typ, path):
+                    return True
+            except _ValError:
+                pass
+        return False
     if isinstance(schema, JObj):
         if val is None and schema.none:
             return True

diff --git a/qa/tasks/mgr/dashboard/test_health.py b/qa/tasks/mgr/dashboard/test_health.py
index 08551e488ab99164f79afe1e1cd9302c75c30b1d..e4091874425c437e21cbf6ed67fb49ef83e4012a 100644 (file)
--- a/qa/tasks/mgr/dashboard/test_health.py
+++ b/qa/tasks/mgr/dashboard/test_health.py
@@ -169,7 +169,8 @@ class HealthTest(DashboardTestCase):
                         'wr_bytes': int,
                         'compress_bytes_used': int,
                         'compress_under_bytes': int,
-                        'stored_raw': int
+                        'stored_raw': int,
+                        'avail_raw': int
                     }),
                     'name': str,
                     'id': int

diff --git a/qa/tasks/mgr/dashboard/test_pool.py b/qa/tasks/mgr/dashboard/test_pool.py
index c2c25bc1aafe3e86afd204fdf73abe14256d69e5..2d34bc75ce136686dd37e40bd48a540b4361b3e7 100644 (file)
--- a/qa/tasks/mgr/dashboard/test_pool.py
+++ b/qa/tasks/mgr/dashboard/test_pool.py
@@ -6,7 +6,7 @@ import six
 import time
 from contextlib import contextmanager
 
-from .helper import DashboardTestCase, JAny, JList, JObj
+from .helper import DashboardTestCase, JAny, JList, JObj, JUnion
 
 log = logging.getLogger(__name__)
 
@@ -23,14 +23,16 @@ class PoolTest(DashboardTestCase):
     }, allow_unknown=True)
 
     pool_list_stat_schema = JObj(sub_elems={
-        'latest': int,
+        'latest': JUnion([int,float]),
         'rate': float,
         'rates': JList(JAny(none=False)),
     })
 
     pool_list_stats_schema = JObj(sub_elems={
+        'avail_raw': pool_list_stat_schema,
         'bytes_used': pool_list_stat_schema,
         'max_avail': pool_list_stat_schema,
+        'percent_used': pool_list_stat_schema,
         'rd_bytes': pool_list_stat_schema,
         'wr_bytes': pool_list_stat_schema,
         'rd': pool_list_stat_schema,

diff --git a/src/mon/PGMap.cc b/src/mon/PGMap.cc
index 24af54ba0ccaad47b457062a88e2f04483682b71..0f686d0a939603b763dd7db30843107e20b02e89 100644 (file)
--- a/src/mon/PGMap.cc
+++ b/src/mon/PGMap.cc
@@ -960,6 +960,7 @@ void PGMapDigest::dump_object_stat_sum(
       f->dump_int("compress_under_bytes", statfs.data_compressed_original);
       // Stored by user amplified by replication
       f->dump_int("stored_raw", stored_raw);
+      f->dump_unsigned("avail_raw", avail);
     }
   } else {
     tbl << stringify(byte_u_t(stored_normalized));
@@ -1178,7 +1179,7 @@ void PGMap::apply_incremental(CephContext *cct, const Incremental& inc)
 
     auto pool_statfs_iter =
       pool_statfs.find(std::make_pair(update_pool, update_osd));
-    if (pg_pool_sum.count(update_pool)) { 
+    if (pg_pool_sum.count(update_pool)) {
       pool_stat_t &pool_sum_ref = pg_pool_sum[update_pool];
       if (pool_statfs_iter == pool_statfs.end()) {
         pool_statfs.emplace(std::make_pair(update_pool, update_osd), statfs_inc);

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.html
index 1bc7dbcdb74211a4153d9a840f7717f94ab56820..bd313df7de1b0b14d7aa68706fc2351a83a2201b 100644 (file)
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.html
@@ -50,8 +50,9 @@
 
 <ng-template #poolUsageTpl
              let-row="row">
-  <cd-usage-bar *ngIf="row.stats?.max_avail?.latest"
-                [total]="row.stats.bytes_used.latest + row.stats.max_avail.latest"
-                [used]="row.stats.bytes_used.latest">
+  <cd-usage-bar *ngIf="row.stats?.avail_raw?.latest"
+                [total]="row.stats.bytes_used.latest + row.stats.avail_raw.latest"
+                [used]="row.stats.bytes_used.latest"
+                decimals="2">
   </cd-usage-bar>
 </ng-template>

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.spec.ts
index 3b1de559f302851b234def2b1e4712486a86d1a5..73ce4df0192424a600bef8e3c6d0de3c22d73162 100644 (file)
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.spec.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.spec.ts
@@ -309,6 +309,8 @@ describe('PoolListComponent', () => {
           stats: {
             bytes_used: { latest: 0, rate: 0, rates: [] },
             max_avail: { latest: 0, rate: 0, rates: [] },
+            avail_raw: { latest: 0, rate: 0, rates: [] },
+            percent_used: { latest: 0, rate: 0, rates: [] },
             rd: { latest: 0, rate: 0, rates: [] },
             rd_bytes: { latest: 0, rate: 0, rates: [] },
             wr: { latest: 0, rate: 0, rates: [] },
@@ -328,7 +330,8 @@ describe('PoolListComponent', () => {
       pool = _.merge(pool, {
         stats: {
           bytes_used: { latest: 5, rate: 0, rates: [] },
-          max_avail: { latest: 15, rate: 0, rates: [] },
+          avail_raw: { latest: 15, rate: 0, rates: [] },
+          percent_used: { latest: 0.25, rate: 0, rates: [] },
           rd_bytes: {
             latest: 6,
             rate: 4,
@@ -345,7 +348,8 @@ describe('PoolListComponent', () => {
           pg_status: '8 active+clean, 2 down',
           stats: {
             bytes_used: { latest: 5, rate: 0, rates: [] },
-            max_avail: { latest: 15, rate: 0, rates: [] },
+            avail_raw: { latest: 15, rate: 0, rates: [] },
+            percent_used: { latest: 0.25, rate: 0, rates: [] },
             rd_bytes: { latest: 6, rate: 4, rates: [2, 6] }
           },
           usage: 0.25

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.ts
index b0d63284b0ac317c0c44a917a2973c6f6d071c0c..4f70d1fd08d0d9ca124d2c012b957f7590e27b58 100644 (file)
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.ts
@@ -241,7 +241,16 @@ export class PoolListComponent extends ListWithDetails implements OnInit {
   }
 
   transformPoolsData(pools: any) {
-    const requiredStats = ['bytes_used', 'max_avail', 'rd_bytes', 'wr_bytes', 'rd', 'wr'];
+    const requiredStats = [
+      'bytes_used',
+      'max_avail',
+      'avail_raw',
+      'percent_used',
+      'rd_bytes',
+      'wr_bytes',
+      'rd',
+      'wr'
+    ];
     const emptyStat: PoolStat = { latest: 0, rate: 0, rates: [] };
 
     _.forEach(pools, (pool: Pool) => {
@@ -251,8 +260,7 @@ export class PoolListComponent extends ListWithDetails implements OnInit {
         stats[stat] = pool.stats && pool.stats[stat] ? pool.stats[stat] : emptyStat;
       });
       pool['stats'] = stats;
-      const avail = stats.bytes_used.latest + stats.max_avail.latest;
-      pool['usage'] = avail > 0 ? stats.bytes_used.latest / avail : avail;
+      pool['usage'] = stats.percent_used.latest;
 
       if (
         !pool.cdExecuting &&

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-stat.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-stat.ts
index f746bf3866ea52d1c95cadc5246e5c584878baf1..9820be94a8de9fb3530d87e994afe0f06504df12 100644 (file)
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-stat.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-stat.ts
@@ -7,6 +7,8 @@ export class PoolStat {
 export class PoolStats {
   bytes_used?: PoolStat;
   max_avail?: PoolStat;
+  avail_raw?: PoolStat;
+  percent_used?: PoolStat;
   rd_bytes?: PoolStat;
   wr_bytes?: PoolStat;
   rd?: PoolStat;

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/usage-bar/usage-bar.component.html b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/usage-bar/usage-bar.component.html
index 42f995efdbbf86f940fc658460602ecb50064f07..0ecf79f1cdf9a1c87fa452e1f0211a35f3c8936c 100644 (file)
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/usage-bar/usage-bar.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/usage-bar/usage-bar.component.html
@@ -17,7 +17,7 @@
   <div class="progress-bar bg-info"
        role="progressbar"
        [style.width]="usedPercentage + '%'">
-    <span>{{ usedPercentage }}%</span>
+    <span>{{ usedPercentage | number: '1.0-' + decimals }}%</span>
   </div>
   <div class="progress-bar bg-freespace"
        role="progressbar"

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/usage-bar/usage-bar.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/usage-bar/usage-bar.component.ts
index 99b6aa60df69f1b2ca73c74df45696e17857beb2..fb0dbd9a978504fb6b8f469c97863a56cdf5e146 100644 (file)
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/usage-bar/usage-bar.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/usage-bar/usage-bar.component.ts
@@ -12,12 +12,14 @@ export class UsageBarComponent implements OnChanges {
   used: number;
   @Input()
   isBinary = true;
+  @Input()
+  decimals = 0;
 
   usedPercentage: number;
   freePercentage: number;
 
   ngOnChanges() {
-    this.usedPercentage = this.total > 0 ? Math.round((this.used / this.total) * 100) : 0;
+    this.usedPercentage = this.total > 0 ? (this.used / this.total) * 100 : 0;
     this.freePercentage = 100 - this.usedPercentage;
   }
 }