mgr/dashboard: fix pool usage calculation

Currently Dashboard Pool usage calculation does not match the output of
'ceph df' command.

Fixes: https://tracker.ceph.com/issues/45185
Signed-off-by: Ernesto Puerta <epuertat@redhat.com>
(cherry picked from commit b4a9dc17a3de90379964443d26b29f1759824f28)

Conflicts:
	qa/tasks/mgr/dashboard/test_pool.py: relative import
	src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.html: change place of new chunk

diff --git a/qa/tasks/mgr/dashboard/helper.py b/qa/tasks/mgr/dashboard/helper.py
index 1acca15373bd4f3c9cb2bde09cf7dfc575f00c1a..44f919b11190eb17b559ed60a7ba7f83108fca54 100644 (file)
--- a/qa/tasks/mgr/dashboard/helper.py
+++ b/qa/tasks/mgr/dashboard/helper.py
@@ -18,6 +18,9 @@ log = logging.getLogger(__name__)
 
 
 class DashboardTestCase(MgrTestCase):
+    # Display full error diffs
+    maxDiff = None
+
     MGRS_REQUIRED = 2
     MDSS_REQUIRED = 1
     REQUIRE_FILESYSTEM = True
@@ -467,6 +470,7 @@ JList = namedtuple('JList', ['elem_typ'])
 
 JTuple = namedtuple('JList', ['elem_typs'])
 
+JUnion = namedtuple('JUnion', ['elem_typs'])
 
 class JObj(namedtuple('JObj', ['sub_elems', 'allow_unknown', 'none', 'unknown_schema'])):
     def __new__(cls, sub_elems, allow_unknown=False, none=False, unknown_schema=None):
@@ -496,6 +500,10 @@ def _validate_json(val, schema, path=[]):
     ... ds = JObj({'a': int, 'b': str, 'c': JList(int)})
     ... _validate_json(d, ds)
     True
+    >>> _validate_json({'num': 1}, JObj({'num': JUnion([int,float])}))
+    True
+    >>> _validate_json({'num': 'a'}, JObj({'num': JUnion([int,float])}))
+    False
     """
     if isinstance(schema, JAny):
         if not schema.none and val is None:
@@ -514,6 +522,14 @@ def _validate_json(val, schema, path=[]):
     if isinstance(schema, JTuple):
         return all(_validate_json(val[i], typ, path + [i])
                    for i, typ in enumerate(schema.elem_typs))
+    if isinstance(schema, JUnion):
+        for typ in schema.elem_typs:
+            try:
+                if _validate_json(val, typ, path):
+                    return True
+            except _ValError:
+                pass
+        return False
     if isinstance(schema, JObj):
         if val is None and schema.none:
             return True

diff --git a/qa/tasks/mgr/dashboard/test_health.py b/qa/tasks/mgr/dashboard/test_health.py
index e7bfb4fabf001f1c65b81520aa152013f0854cb5..2cc35e377b13f6c07a0a2e56adda0d8704036ae3 100644 (file)
--- a/qa/tasks/mgr/dashboard/test_health.py
+++ b/qa/tasks/mgr/dashboard/test_health.py
@@ -169,7 +169,8 @@ class HealthTest(DashboardTestCase):
                         'wr_bytes': int,
                         'compress_bytes_used': int,
                         'compress_under_bytes': int,
-                        'stored_raw': int
+                        'stored_raw': int,
+                        'avail_raw': int
                     }),
                     'name': str,
                     'id': int

diff --git a/qa/tasks/mgr/dashboard/test_pool.py b/qa/tasks/mgr/dashboard/test_pool.py
index bf40ac2062638d36bd695483064a1a585bde698d..58b7fa8fed4c0653e4d9336e86d72acdafd68e18 100644 (file)
--- a/qa/tasks/mgr/dashboard/test_pool.py
+++ b/qa/tasks/mgr/dashboard/test_pool.py
@@ -6,7 +6,7 @@ import six
 import time
 from contextlib import contextmanager
 
-from tasks.mgr.dashboard.helper import DashboardTestCase, JAny, JList, JObj
+from .helper import DashboardTestCase, JAny, JList, JObj, JUnion
 
 log = logging.getLogger(__name__)
 
@@ -23,14 +23,16 @@ class PoolTest(DashboardTestCase):
     }, allow_unknown=True)
 
     pool_list_stat_schema = JObj(sub_elems={
-        'latest': int,
+        'latest': JUnion([int,float]),
         'rate': float,
         'rates': JList(JAny(none=False)),
     })
 
     pool_list_stats_schema = JObj(sub_elems={
+        'avail_raw': pool_list_stat_schema,
         'bytes_used': pool_list_stat_schema,
         'max_avail': pool_list_stat_schema,
+        'percent_used': pool_list_stat_schema,
         'rd_bytes': pool_list_stat_schema,
         'wr_bytes': pool_list_stat_schema,
         'rd': pool_list_stat_schema,

diff --git a/src/mon/PGMap.cc b/src/mon/PGMap.cc
index 57796651c6e3d4a5d36f820b363216e97f06ab2b..1488bcb5c8e74f7f6e79aacf4cd2aa953b6ce2fb 100644 (file)
--- a/src/mon/PGMap.cc
+++ b/src/mon/PGMap.cc
@@ -958,6 +958,7 @@ void PGMapDigest::dump_object_stat_sum(
       f->dump_int("compress_under_bytes", statfs.data_compressed_original);
       // Stored by user amplified by replication
       f->dump_int("stored_raw", stored_raw);
+      f->dump_unsigned("avail_raw", avail);
     }
   } else {
     tbl << stringify(byte_u_t(stored_normalized));
@@ -1176,7 +1177,7 @@ void PGMap::apply_incremental(CephContext *cct, const Incremental& inc)
 
     auto pool_statfs_iter =
       pool_statfs.find(std::make_pair(update_pool, update_osd));
-    if (pg_pool_sum.count(update_pool)) { 
+    if (pg_pool_sum.count(update_pool)) {
       pool_stat_t &pool_sum_ref = pg_pool_sum[update_pool];
       if (pool_statfs_iter == pool_statfs.end()) {
         pool_statfs.emplace(std::make_pair(update_pool, update_osd), statfs_inc);

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.html
index 40f68d8b64a3fafdc471887ef5c092b399a68654..5ecf20f33204e654d350bd42e00597e272147da8 100644 (file)
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.html
@@ -29,9 +29,10 @@
 
     <ng-template #poolUsageTpl
                  let-row="row">
-      <cd-usage-bar *ngIf="row.stats?.max_avail?.latest"
-                    [total]="row.stats.bytes_used.latest + row.stats.max_avail.latest"
-                    [used]="row.stats.bytes_used.latest">
+      <cd-usage-bar *ngIf="row.stats?.avail_raw?.latest"
+                    [total]="row.stats.bytes_used.latest + row.stats.avail_raw.latest"
+                    [used]="row.stats.bytes_used.latest"
+                    decimals="2">
       </cd-usage-bar>
     </ng-template>
   </tab>

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.spec.ts
index dbe4ec6f209194af3a9501bf774574559017875c..165ed7f34e83e9d0b852e84f4d649bf824d35c65 100644 (file)
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.spec.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.spec.ts
@@ -309,6 +309,8 @@ describe('PoolListComponent', () => {
           stats: {
             bytes_used: { latest: 0, rate: 0, rates: [] },
             max_avail: { latest: 0, rate: 0, rates: [] },
+            avail_raw: { latest: 0, rate: 0, rates: [] },
+            percent_used: { latest: 0, rate: 0, rates: [] },
             rd: { latest: 0, rate: 0, rates: [] },
             rd_bytes: { latest: 0, rate: 0, rates: [] },
             wr: { latest: 0, rate: 0, rates: [] },
@@ -328,7 +330,8 @@ describe('PoolListComponent', () => {
       pool = _.merge(pool, {
         stats: {
           bytes_used: { latest: 5, rate: 0, rates: [] },
-          max_avail: { latest: 15, rate: 0, rates: [] },
+          avail_raw: { latest: 15, rate: 0, rates: [] },
+          percent_used: { latest: 0.25, rate: 0, rates: [] },
           rd_bytes: {
             latest: 6,
             rate: 4,
@@ -345,7 +348,8 @@ describe('PoolListComponent', () => {
           pg_status: '8 active+clean, 2 down',
           stats: {
             bytes_used: { latest: 5, rate: 0, rates: [] },
-            max_avail: { latest: 15, rate: 0, rates: [] },
+            avail_raw: { latest: 15, rate: 0, rates: [] },
+            percent_used: { latest: 0.25, rate: 0, rates: [] },
             rd_bytes: { latest: 6, rate: 4, rates: [2, 6] }
           },
           usage: 0.25

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.ts
index 61fa3585183f4f363213b2383c988b94d10b784b..93958f1b5339eb61f79e04cacb3afe8dee07404e 100644 (file)
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.ts
@@ -241,7 +241,16 @@ export class PoolListComponent extends ListWithDetails implements OnInit {
   }
 
   transformPoolsData(pools: any) {
-    const requiredStats = ['bytes_used', 'max_avail', 'rd_bytes', 'wr_bytes', 'rd', 'wr'];
+    const requiredStats = [
+      'bytes_used',
+      'max_avail',
+      'avail_raw',
+      'percent_used',
+      'rd_bytes',
+      'wr_bytes',
+      'rd',
+      'wr'
+    ];
     const emptyStat: PoolStat = { latest: 0, rate: 0, rates: [] };
 
     _.forEach(pools, (pool: Pool) => {
@@ -251,8 +260,7 @@ export class PoolListComponent extends ListWithDetails implements OnInit {
         stats[stat] = pool.stats && pool.stats[stat] ? pool.stats[stat] : emptyStat;
       });
       pool['stats'] = stats;
-      const avail = stats.bytes_used.latest + stats.max_avail.latest;
-      pool['usage'] = avail > 0 ? stats.bytes_used.latest / avail : avail;
+      pool['usage'] = stats.percent_used.latest;
 
       if (
         !pool.cdExecuting &&

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-stat.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-stat.ts
index f746bf3866ea52d1c95cadc5246e5c584878baf1..9820be94a8de9fb3530d87e994afe0f06504df12 100644 (file)
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-stat.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-stat.ts
@@ -7,6 +7,8 @@ export class PoolStat {
 export class PoolStats {
   bytes_used?: PoolStat;
   max_avail?: PoolStat;
+  avail_raw?: PoolStat;
+  percent_used?: PoolStat;
   rd_bytes?: PoolStat;
   wr_bytes?: PoolStat;
   rd?: PoolStat;

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/usage-bar/usage-bar.component.html b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/usage-bar/usage-bar.component.html
index ebab648817b09d61f9d6ede8592454e4f78957d5..d4ba37b00536d95400e3c48bf883e2f19b56b6df 100644 (file)
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/usage-bar/usage-bar.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/usage-bar/usage-bar.component.html
@@ -17,7 +17,7 @@
   <div class="progress-bar bg-info"
        role="progressbar"
        [style.width]="usedPercentage + '%'">
-    <span>{{ usedPercentage }}%</span>
+    <span>{{ usedPercentage | number: '1.0-' + decimals }}%</span>
   </div>
   <div class="progress-bar bg-freespace"
        role="progressbar"

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/usage-bar/usage-bar.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/usage-bar/usage-bar.component.ts
index 40e110e3f27229a311319969d5e8b7e21bd98717..22b485c552575e6b7de15d3efda382bb4ce34e48 100644 (file)
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/usage-bar/usage-bar.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/usage-bar/usage-bar.component.ts
@@ -12,6 +12,8 @@ export class UsageBarComponent implements OnChanges {
   used: number;
   @Input()
   isBinary = true;
+  @Input()
+  decimals = 0;
 
   usedPercentage: number;
   freePercentage: number;
@@ -19,7 +21,7 @@ export class UsageBarComponent implements OnChanges {
   constructor() {}
 
   ngOnChanges() {
-    this.usedPercentage = this.total > 0 ? Math.round((this.used / this.total) * 100) : 0;
+    this.usedPercentage = this.total > 0 ? (this.used / this.total) * 100 : 0;
     this.freePercentage = 100 - this.usedPercentage;
   }
 }