mgr: Add pg_stats and pool_stats direct access

More efficient to get just what you need out of the PGMap

mgr/progress: Use pg_stats instead of pg_dump
mgr/diskprediction_cloud: Use pg_stats instead of pg_dump
mgr/balancer: Use pg_stats and pool_stats instead of pg_dump

Fixes: https://tracker.ceph.com/issues/43556
Signed-off-by: David Zafman <dzafman@redhat.com>
(cherry picked from commit 69b9e0472d99f7e5278b6579a2d73ffa83ecf1b1)

Conflicts:
	src/pybind/mgr/balancer/module.py
- src/pybind/mgr/balancer/module.py - nautilus version does not have the
  "plan.mode == 'upmap'" conditional blocks, so the changes inside these blocks
  were omitted
- the MappingState ctor call in plan_create is inside a MsPlan() call in
  master (nautilus: Plan() call)
	src/pybind/mgr/progress/module.py
- adapted for some post-nautilus cleanups

diff --git a/src/mgr/ActivePyModules.cc b/src/mgr/ActivePyModules.cc
index 460b337ca0777c460e7bf52970305de216fa5a2a..ee022e875e969703375fba8f5b256ae3216811d0 100644 (file)
--- a/src/mgr/ActivePyModules.cc
+++ b/src/mgr/ActivePyModules.cc
@@ -358,6 +358,20 @@ PyObject *ActivePyModules::get_python(const std::string &what)
         pg_map.dump_pool_stats_full(osd_map, nullptr, &f, true);
       });
     return f.get();
+  } else if (what == "pg_stats") {
+    cluster_state.with_pgmap(
+        [&f, &tstate](const PGMap &pg_map) {
+      PyEval_RestoreThread(tstate);
+      pg_map.dump_pg_stats(&f, false);
+    });
+    return f.get();
+  } else if (what == "pool_stats") {
+    cluster_state.with_pgmap(
+        [&f, &tstate](const PGMap &pg_map) {
+      PyEval_RestoreThread(tstate);
+      pg_map.dump_pool_stats(&f);
+    });
+    return f.get();
   } else if (what == "osd_stats") {
     cluster_state.with_pgmap(
         [&f, &tstate](const PGMap &pg_map) {

diff --git a/src/pybind/mgr/balancer/module.py b/src/pybind/mgr/balancer/module.py
index 2592305a935c296709f99627d85a2bd40bee2ce5..942bde38c4bbdc903098dfd62bc78628d721f7a4 100644 (file)
--- a/src/pybind/mgr/balancer/module.py
+++ b/src/pybind/mgr/balancer/module.py
@@ -1,4 +1,3 @@
-
 """
 Balance PG distribution across OSDs.
 """
@@ -18,18 +17,19 @@ import datetime
 TIME_FORMAT = '%Y-%m-%d_%H:%M:%S'
 
 class MappingState:
-    def __init__(self, osdmap, pg_dump, desc=''):
+    def __init__(self, osdmap, raw_pg_stats, raw_pool_stats, desc=''):
         self.desc = desc
         self.osdmap = osdmap
         self.osdmap_dump = self.osdmap.dump()
         self.crush = osdmap.get_crush()
         self.crush_dump = self.crush.dump()
-        self.pg_dump = pg_dump
+        self.raw_pg_stats = raw_pg_stats
+        self.raw_pool_stats = raw_pool_stats
         self.pg_stat = {
-            i['pgid']: i['stat_sum'] for i in pg_dump.get('pg_stats', [])
+            i['pgid']: i['stat_sum'] for i in raw_pg_stats.get('pg_stats', [])
         }
         osd_poolids = [p['pool'] for p in self.osdmap_dump.get('pools', [])]
-        pg_poolids = [p['poolid'] for p in pg_dump.get('pool_stats', [])]
+        pg_poolids = [p['poolid'] for p in raw_pool_stats.get('pool_stats', [])]
         self.poolids = set(osd_poolids) & set(pg_poolids)
         self.pg_up = {}
         self.pg_up_by_poolid = {}
@@ -63,7 +63,8 @@ class Plan:
         self.inc.set_osd_reweights(self.osd_weights)
         self.inc.set_crush_compat_weight_set_weights(self.compat_ws)
         return MappingState(self.initial.osdmap.apply_incremental(self.inc),
-                            self.initial.pg_dump,
+                            self.initial.raw_pg_stats,
+                            self.initial.raw_pool_stats,
                             'plan %s final' % self.name)
 
     def dump(self):
@@ -437,7 +438,8 @@ class Module(MgrModule):
                     return (-errno.EPERM, '', warn)
             elif command['mode'] == 'crush-compat':
                 ms = MappingState(self.get_osdmap(),
-                                  self.get("pg_dump"),
+                                  self.get("pg_stats"),
+                                  self.get("pool_stats"),
                                   'initialize compat weight-set')
                 self.get_compat_weight_set_weights(ms) # ignore error
             self.set_module_option('mode', command['mode'])
@@ -514,13 +516,14 @@ class Module(MgrModule):
                     if option not in valid_pool_names:
                          return (-errno.EINVAL, '', 'option "%s" not a plan or a pool' % option)
                     pools.append(option)
-                    ms = MappingState(osdmap, self.get("pg_dump"), 'pool "%s"' % option)
+                    ms = MappingState(osdmap, self.get("pg_stats"), self.get("pool_stats"), 'pool "%s"' % option)
                 else:
                     pools = plan.pools
                     ms = plan.final_state()
             else:
                 ms = MappingState(self.get_osdmap(),
-                                  self.get("pg_dump"),
+                                  self.get("pg_stats"),
+                                  self.get("pool_stats"),
                                   'current cluster')
             return (0, self.evaluate(ms, pools, verbose=verbose), '')
         elif command['prefix'] == 'balancer optimize':
@@ -667,7 +670,8 @@ class Module(MgrModule):
     def plan_create(self, name, osdmap, pools):
         plan = Plan(name,
                     MappingState(osdmap,
-                                 self.get("pg_dump"),
+                                 self.get("pg_stats"),
+                                 self.get("pool_stats"),
                                  'plan %s initial' % name),
                     pools)
         return plan

diff --git a/src/pybind/mgr/diskprediction_cloud/agent/metrics/ceph_cluster.py b/src/pybind/mgr/diskprediction_cloud/agent/metrics/ceph_cluster.py
index 7c13d10afb42c006c0a9dda32bec742b5d673ced..2491644a89affc4378f756132e4510410a21247e 100644 (file)
--- a/src/pybind/mgr/diskprediction_cloud/agent/metrics/ceph_cluster.py
+++ b/src/pybind/mgr/diskprediction_cloud/agent/metrics/ceph_cluster.py
@@ -89,7 +89,7 @@ class CephClusterAgent(MetricsAgent):
         else:\r
             c_data.fields['osd_bytes_used_percentage'] = 0.0000\r
 \r
-        pg_stats = obj_api.module.get('pg_dump').get('pg_stats', [])\r
+        pg_stats = obj_api.module.get('pg_stats').get('pg_stats', [])\r
         num_bytes = 0\r
         num_object = 0\r
         num_object_degraded = 0\r

diff --git a/src/pybind/mgr/mgr_module.py b/src/pybind/mgr/mgr_module.py
index 40fc8433dfe0ba7fd60b98e7c94d110131730b5e..4e7e923b71901f7f1b30bb36bead84f2723b6670 100644 (file)
--- a/src/pybind/mgr/mgr_module.py
+++ b/src/pybind/mgr/mgr_module.py
@@ -701,7 +701,8 @@ class MgrModule(ceph_module.BaseMgrModule):
         :param str data_name: Valid things to fetch are osd_crush_map_text,
                 osd_map, osd_map_tree, osd_map_crush, config, mon_map, fs_map,
                 osd_metadata, pg_summary, io_rate, pg_dump, df, osd_stats,
-                health, mon_status, devices, device <devid>.
+                health, mon_status, devices, device <devid>, pg_stats,
+                pool_stats.
 
         Note:
             All these structures have their own JSON representations: experiment

diff --git a/src/pybind/mgr/progress/module.py b/src/pybind/mgr/progress/module.py
index 5db15c716c56509b40ad205b3475f76cd07d948a..ce81aaee3bbd27a69ed8bcbcb8ca6e5a8cc97cdd 100644 (file)
--- a/src/pybind/mgr/progress/module.py
+++ b/src/pybind/mgr/progress/module.py
@@ -191,10 +191,10 @@ class PgRecoveryEvent(Event):
     def evacuating_osds(self):
         return self. _evacuate_osds
 
-    def pg_update(self, pg_dump, log):
+    def pg_update(self, raw_pg_stats, log):
         # FIXME: O(pg_num) in python
         # FIXME: far more fields getting pythonized than we really care about
-        pg_to_state = dict([(p['pgid'], p) for p in pg_dump['pg_stats']])
+        pg_to_state = dict([(p['pgid'], p) for p in raw_pg_stats['pg_stats']])
 
         if self._original_bytes_recovered is None:
             self._original_bytes_recovered = {}
@@ -409,7 +409,7 @@ class Module(MgrModule):
             which_pgs=affected_pgs,
             evacuate_osds=[osd_id]
         )
-        ev.pg_update(self.get("pg_dump"), self.log)
+        ev.pg_update(self.get("pg_stats"), self.log)
         self._events[ev.id] = ev
 
     def _osd_in(self, osd_id):
@@ -454,7 +454,7 @@ class Module(MgrModule):
             ))
             self._osdmap_changed(old_osdmap, self._latest_osdmap)
         elif notify_type == "pg_summary":
-            data = self.get("pg_dump")
+            data = self.get("pg_stats")
             for ev_id, ev in self._events.items():
                 if isinstance(ev, PgRecoveryEvent):
                     ev.pg_update(data, self.log)

diff --git a/src/pybind/mgr/selftest/module.py b/src/pybind/mgr/selftest/module.py
index a5a2cc10273b167f9ff810676e4acc898462e919..5a1a8a5fb8d363c65d72ffa42b3835afa8cdebc3 100644 (file)
--- a/src/pybind/mgr/selftest/module.py
+++ b/src/pybind/mgr/selftest/module.py
@@ -237,6 +237,8 @@ class Module(MgrModule):
                 "pg_status",
                 "pg_dump",
                 "df",
+                "pg_stats",
+                "pool_stats",
                 "osd_stats",
                 "health",
                 "mon_status",