mgr/dashboard: fix pool usage calculation

Currently Dashboard Pool usage calculation does not match the output of
'ceph df' command.

Fixes: https://tracker.ceph.com/issues/45185
Signed-off-by: Ernesto Puerta <epuertat@redhat.com>
(cherry picked from commit b4a9dc17a3de90379964443d26b29f1759824f28)

Conflicts:
	qa/tasks/mgr/dashboard/helper.py
	src/mon/PGMap.cc
	src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.html
	src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.spec.ts
	src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.ts
	src/pybind/mgr/dashboard/frontend/src/app/shared/components/usage-bar/usage-bar.component.ts:
        - Keep UsageBar component totalBytes/usedBytes names
        - Bring new UsageBar option decimal
Signed-off-by: Ernesto Puerta <epuertat@redhat.com>

diff --git a/qa/tasks/mgr/dashboard/helper.py b/qa/tasks/mgr/dashboard/helper.py
index 854169fb61099ab70e8af0ed23d9a388f9d9c296..9627a8428f84e8115e5da6c97c549148135bff34 100644 (file)
--- a/qa/tasks/mgr/dashboard/helper.py
+++ b/qa/tasks/mgr/dashboard/helper.py
@@ -18,6 +18,9 @@ log = logging.getLogger(__name__)
 
 
 class DashboardTestCase(MgrTestCase):
+    # Display full error diffs
+    maxDiff = None
+
     # Increased x3 (20 -> 60)
     TIMEOUT_HEALTH_CLEAR = 60
 
@@ -433,6 +436,7 @@ JList = namedtuple('JList', ['elem_typ'])
 
 JTuple = namedtuple('JList', ['elem_typs'])
 
+JUnion = namedtuple('JUnion', ['elem_typs'])
 
 class JObj(namedtuple('JObj', ['sub_elems', 'allow_unknown', 'none', 'unknown_schema'])):
     def __new__(cls, sub_elems, allow_unknown=False, none=False, unknown_schema=None):
@@ -462,6 +466,10 @@ def _validate_json(val, schema, path=[]):
     ... ds = JObj({'a': int, 'b': str, 'c': JList(int)})
     ... _validate_json(d, ds)
     True
+    >>> _validate_json({'num': 1}, JObj({'num': JUnion([int,float])}))
+    True
+    >>> _validate_json({'num': 'a'}, JObj({'num': JUnion([int,float])}))
+    False
     """
     if isinstance(schema, JAny):
         if not schema.none and val is None:
@@ -480,6 +488,14 @@ def _validate_json(val, schema, path=[]):
     if isinstance(schema, JTuple):
         return all(_validate_json(val[i], typ, path + [i])
                    for i, typ in enumerate(schema.elem_typs))
+    if isinstance(schema, JUnion):
+        for typ in schema.elem_typs:
+            try:
+                if _validate_json(val, typ, path):
+                    return True
+            except _ValError:
+                pass
+        return False
     if isinstance(schema, JObj):
         if val is None and schema.none:
             return True

diff --git a/qa/tasks/mgr/dashboard/test_health.py b/qa/tasks/mgr/dashboard/test_health.py
index 65ea21b16da655f8d5834351db65995ec62e7a3c..a9334edc35dd1f9771adb4c57113fda5bf4cff2e 100644 (file)
--- a/qa/tasks/mgr/dashboard/test_health.py
+++ b/qa/tasks/mgr/dashboard/test_health.py
@@ -161,7 +161,8 @@ class HealthTest(DashboardTestCase):
                         'wr_bytes': int,
                         'compress_bytes_used': int,
                         'compress_under_bytes': int,
-                        'stored_raw': int
+                        'stored_raw': int,
+                        'avail_raw': int
                     }),
                     'name': str,
                     'id': int

diff --git a/qa/tasks/mgr/dashboard/test_pool.py b/qa/tasks/mgr/dashboard/test_pool.py
index 58b78a4dee49602d48312b534681d58e0172cca3..59d86d3b0e604378a8f460cdafeebbac435e88e6 100644 (file)
--- a/qa/tasks/mgr/dashboard/test_pool.py
+++ b/qa/tasks/mgr/dashboard/test_pool.py
@@ -6,7 +6,7 @@ import six
 import time
 from contextlib import contextmanager
 
-from .helper import DashboardTestCase, JAny, JList, JObj
+from .helper import DashboardTestCase, JAny, JList, JObj, JUnion
 
 log = logging.getLogger(__name__)
 
@@ -23,14 +23,16 @@ class PoolTest(DashboardTestCase):
     }, allow_unknown=True)
 
     pool_list_stat_schema = JObj(sub_elems={
-        'latest': int,
+        'latest': JUnion([int,float]),
         'rate': float,
         'rates': JList(JAny(none=False)),
     })
 
     pool_list_stats_schema = JObj(sub_elems={
+        'avail_raw': pool_list_stat_schema,
         'bytes_used': pool_list_stat_schema,
         'max_avail': pool_list_stat_schema,
+        'percent_used': pool_list_stat_schema,
         'rd_bytes': pool_list_stat_schema,
         'wr_bytes': pool_list_stat_schema,
         'rd': pool_list_stat_schema,

diff --git a/src/mon/PGMap.cc b/src/mon/PGMap.cc
index ce6abdf537c5841e7a8c272f094d8a36f8277591..952e1ae19d8b1829d89681970ab5e02db92f01b2 100644 (file)
--- a/src/mon/PGMap.cc
+++ b/src/mon/PGMap.cc
@@ -936,6 +936,7 @@ void PGMapDigest::dump_object_stat_sum(
       f->dump_int("compress_under_bytes", statfs.data_compressed_original);
       // Stored by user amplified by replication
       f->dump_int("stored_raw", pool_stat.get_user_bytes(1.0, per_pool));
+      f->dump_unsigned("avail_raw", avail);
     }
   } else {
     tbl << stringify(byte_u_t(stored_normalized));
@@ -1146,7 +1147,7 @@ void PGMap::apply_incremental(CephContext *cct, const Incremental& inc)
 
     auto pool_statfs_iter =
       pool_statfs.find(std::make_pair(update_pool, update_osd));
-    if (pg_pool_sum.count(update_pool)) { 
+    if (pg_pool_sum.count(update_pool)) {
       pool_stat_t &pool_sum_ref = pg_pool_sum[update_pool];
       if (pool_statfs_iter == pool_statfs.end()) {
         pool_statfs.emplace(std::make_pair(update_pool, update_osd), statfs_inc);

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.html
index 124a670d2353cb0d899c11e754407c76fe26f592..0a83df29efc196e75ff744267e43d01b730ffb43 100644 (file)
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.html
@@ -23,9 +23,10 @@
 
     <ng-template #poolUsageTpl
                  let-row="row">
-      <cd-usage-bar *ngIf="row.stats?.max_avail?.latest"
-                    [totalBytes]="row.stats.bytes_used.latest + row.stats.max_avail.latest"
-                    [usedBytes]="row.stats.bytes_used.latest">
+      <cd-usage-bar *ngIf="row.stats?.avail_raw?.latest"
+                    [totalBytes]="row.stats.bytes_used.latest + row.stats.avail_raw.latest"
+                    [usedBytes]="row.stats.bytes_used.latest"
+                    decimals="2">
       </cd-usage-bar>
     </ng-template>
   </tab>

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.spec.ts
index 3a365223ffd90dec8b5bc506d4bd1c01770451a0..3f917e20a8c5907fdb60fc53bd24087a6a2cadc1 100644 (file)
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.spec.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.spec.ts
@@ -302,6 +302,8 @@ describe('PoolListComponent', () => {
           stats: {
             bytes_used: { latest: 0, rate: 0, rates: [] },
             max_avail: { latest: 0, rate: 0, rates: [] },
+            avail_raw: { latest: 0, rate: 0, rates: [] },
+            percent_used: { latest: 0, rate: 0, rates: [] },
             rd: { latest: 0, rate: 0, rates: [] },
             rd_bytes: { latest: 0, rate: 0, rates: [] },
             wr: { latest: 0, rate: 0, rates: [] },
@@ -321,8 +323,13 @@ describe('PoolListComponent', () => {
       pool = _.merge(pool, {
         stats: {
           bytes_used: { latest: 5, rate: 0, rates: [] },
-          max_avail: { latest: 15, rate: 0, rates: [] },
-          rd_bytes: { latest: 6, rate: 4, rates: [[0, 2], [1, 6]] }
+          avail_raw: { latest: 15, rate: 0, rates: [] },
+          percent_used: { latest: 0.25, rate: 0, rates: [] },
+          rd_bytes: {
+            latest: 6,
+            rate: 4,
+            rates: [[0, 2], [1, 6]]
+          }
         },
         pg_status: { 'active+clean': 8, down: 2 }
       });
@@ -331,7 +338,8 @@ describe('PoolListComponent', () => {
           pg_status: '8 active+clean, 2 down',
           stats: {
             bytes_used: { latest: 5, rate: 0, rates: [] },
-            max_avail: { latest: 15, rate: 0, rates: [] },
+            avail_raw: { latest: 15, rate: 0, rates: [] },
+            percent_used: { latest: 0.25, rate: 0, rates: [] },
             rd_bytes: { latest: 6, rate: 4, rates: [2, 6] }
           },
           usage: 0.25

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.ts
index 935bbf224c06f3e4410435aaca119c6c70495bb0..4b2940dcea74de89b1497d11210d656f60d73927 100644 (file)
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-list/pool-list.component.ts
@@ -239,7 +239,16 @@ export class PoolListComponent implements OnInit {
   }
 
   transformPoolsData(pools: any) {
-    const requiredStats = ['bytes_used', 'max_avail', 'rd_bytes', 'wr_bytes', 'rd', 'wr'];
+    const requiredStats = [
+      'bytes_used',
+      'max_avail',
+      'avail_raw',
+      'percent_used',
+      'rd_bytes',
+      'wr_bytes',
+      'rd',
+      'wr'
+    ];
     const emptyStat = { latest: 0, rate: 0, rates: [] };
 
     _.forEach(pools, (pool: Pool) => {
@@ -249,8 +258,7 @@ export class PoolListComponent implements OnInit {
         stats[stat] = pool.stats && pool.stats[stat] ? pool.stats[stat] : emptyStat;
       });
       pool['stats'] = stats;
-      const avail = stats.bytes_used.latest + stats.max_avail.latest;
-      pool['usage'] = avail > 0 ? stats.bytes_used.latest / avail : avail;
+      pool['usage'] = stats.percent_used.latest;
 
       if (
         !pool.cdExecuting &&

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-stat.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-stat.ts
index f746bf3866ea52d1c95cadc5246e5c584878baf1..9820be94a8de9fb3530d87e994afe0f06504df12 100644 (file)
--- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-stat.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/pool/pool-stat.ts
@@ -7,6 +7,8 @@ export class PoolStat {
 export class PoolStats {
   bytes_used?: PoolStat;
   max_avail?: PoolStat;
+  avail_raw?: PoolStat;
+  percent_used?: PoolStat;
   rd_bytes?: PoolStat;
   wr_bytes?: PoolStat;
   rd?: PoolStat;

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/usage-bar/usage-bar.component.html b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/usage-bar/usage-bar.component.html
index 93348f16acb5eeaf2e94a6ea01fb15973b93e453..65d9a89d4d8f1e1483e6dc5eba03126a038236a6 100644 (file)
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/usage-bar/usage-bar.component.html
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/usage-bar/usage-bar.component.html
@@ -17,7 +17,7 @@
   <div class="progress-bar progress-bar-info"
        role="progressbar"
        [style.width]="usedPercentage + '%'">
-    <span>{{ usedPercentage }}%</span>
+    <span>{{ usedPercentage | number: '1.0-' + decimals }}%</span>
   </div>
   <div class="progress-bar progress-bar-freespace"
        role="progressbar"

diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/usage-bar/usage-bar.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/usage-bar/usage-bar.component.ts
index e8b4ddc148eedb7012ffb39bdd4422c94e1a2701..0a3b12e24e5e2abee32dfa3c6a4fe3780ea29b12 100644 (file)
--- a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/usage-bar/usage-bar.component.ts
+++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/usage-bar/usage-bar.component.ts
@@ -10,6 +10,8 @@ export class UsageBarComponent implements OnChanges {
   totalBytes: number;
   @Input()
   usedBytes: number;
+  @Input()
+  decimals = 0;
 
   usedPercentage: number;
   freePercentage: number;
@@ -18,7 +20,7 @@ export class UsageBarComponent implements OnChanges {
   constructor() {}
 
   ngOnChanges() {
-    this.usedPercentage = Math.round((this.usedBytes / this.totalBytes) * 100);
+    this.usedPercentage = this.totalBytes > 0 ? (this.usedBytes / this.totalBytes) * 100 : 0;
     this.freePercentage = 100 - this.usedPercentage;
     this.freeBytes = this.totalBytes - this.usedBytes;
   }