mgr/devicehealth: extract+store wear level from metrics scraping

When we scrape and store health metrics for a device, extract the wear
level from the JSON.  If present, also store it in the config-key
per-device metadata.

Signed-off-by: Sage Weil <sage@newdream.net>
(cherry picked from commit 4840507cfcdd5182003671994d0bc9604d072e3e)

 Conflicts:
	src/pybind/mgr/devicehealth/module.py
 - type annotations for _get_device_metrics

diff --git a/src/mgr/ActivePyModules.cc b/src/mgr/ActivePyModules.cc
index 02935c8b5f849f9f1c857a9b782b6442f2219a57..06b632957d5f2299b60bbde4ec1cd8a14349274a 100644 (file)
--- a/src/mgr/ActivePyModules.cc
+++ b/src/mgr/ActivePyModules.cc
@@ -1279,6 +1279,36 @@ void ActivePyModules::set_uri(const std::string& module_name,
   modules.at(module_name)->set_uri(uri);
 }
 
+void ActivePyModules::set_device_wear_level(const std::string& devid,
+                                           float wear_level)
+{
+  // update mgr state
+  map<string,string> meta;
+  daemon_state.with_device(
+    devid,
+    [wear_level, &meta] (DeviceState& dev) {
+      dev.set_wear_level(wear_level);
+      meta = dev.metadata;
+    });
+
+  // tell mon
+  json_spirit::Object json_object;
+  for (auto& i : meta) {
+    json_spirit::Config::add(json_object, i.first, i.second);
+  }
+  bufferlist json;
+  json.append(json_spirit::write(json_object));
+  const string cmd =
+    "{"
+    "\"prefix\": \"config-key set\", "
+    "\"key\": \"device/" + devid + "\""
+    "}";
+
+  Command set_cmd;
+  set_cmd.run(&monc, cmd, json);
+  set_cmd.wait();
+}
+
 MetricQueryID ActivePyModules::add_osd_perf_query(
     const OSDPerfMetricQuery &query,
     const std::optional<OSDPerfMetricLimit> &limit)

diff --git a/src/mgr/ActivePyModules.h b/src/mgr/ActivePyModules.h
index 09c7041a610130b9729625b6eefedddd8f7d6a8e..f4feec4307a92c10a1ae3e28e75e32f5dfc1641c 100644 (file)
--- a/src/mgr/ActivePyModules.h
+++ b/src/mgr/ActivePyModules.h
@@ -159,6 +159,7 @@ public:
   void config_notify();
 
   void set_uri(const std::string& module_name, const std::string &uri);
+  void set_device_wear_level(const std::string& devid, float wear_level);
 
   int handle_command(
     const ModuleCommand& module_command,

diff --git a/src/mgr/BaseMgrModule.cc b/src/mgr/BaseMgrModule.cc
index f368381e4fc9ba7dbdf0856c75eab6ef17c2c8c5..f1ebb94ae93c07d9f9142ae6bc3edb086dc985a7 100644 (file)
--- a/src/mgr/BaseMgrModule.cc
+++ b/src/mgr/BaseMgrModule.cc
@@ -689,6 +689,23 @@ ceph_set_uri(BaseMgrModule *self, PyObject *args)
   Py_RETURN_NONE;
 }
 
+static PyObject*
+ceph_set_wear_level(BaseMgrModule *self, PyObject *args)
+{
+  char *devid = nullptr;
+  float wear_level;
+  if (!PyArg_ParseTuple(args, "sf:ceph_set_wear_level",
+                       &devid, &wear_level)) {
+    return nullptr;
+  }
+
+  PyThreadState *tstate = PyEval_SaveThread();
+  self->py_modules->set_device_wear_level(devid, wear_level);
+  PyEval_RestoreThread(tstate);
+
+  Py_RETURN_NONE;
+}
+
 static PyObject*
 ceph_have_mon_connection(BaseMgrModule *self, PyObject *args)
 {
@@ -1452,6 +1469,9 @@ PyMethodDef BaseMgrModule_methods[] = {
   {"_ceph_set_uri", (PyCFunction)ceph_set_uri, METH_VARARGS,
     "Advertize a service URI served by this module"},
 
+  {"_ceph_set_device_wear_level", (PyCFunction)ceph_set_wear_level, METH_VARARGS,
+   "Set device wear_level value"},
+
   {"_ceph_have_mon_connection", (PyCFunction)ceph_have_mon_connection,
     METH_NOARGS, "Find out whether this mgr daemon currently has "
                  "a connection to a monitor"},

diff --git a/src/mgr/MgrContext.h b/src/mgr/MgrContext.h
index c6e647365b505317b75de76c9107b53e63a6b103..a5490bef3d6768e7a4b251a728f66522e1d96add 100644 (file)
--- a/src/mgr/MgrContext.h
+++ b/src/mgr/MgrContext.h
@@ -35,6 +35,12 @@ public:
         &outbl, &outs, &cond);
   }
 
+  void run(MonClient *monc, const std::string &command, const ceph::buffer::list &inbl)
+  {
+    monc->start_mon_command({command}, inbl,
+        &outbl, &outs, &cond);
+  }
+
   virtual void wait()
   {
     r = cond.wait();

diff --git a/src/pybind/mgr/ceph_module.pyi b/src/pybind/mgr/ceph_module.pyi
index 77ed08c033b6d2834f11ad6d12cbb83c7dfc2823..06b7725f86777343f5497731452137733478d179 100644 (file)
--- a/src/pybind/mgr/ceph_module.pyi
+++ b/src/pybind/mgr/ceph_module.pyi
@@ -62,6 +62,7 @@ class BaseMgrModule(object):
     def _ceph_get_store(self, key):...
     def _ceph_get_osdmap(self):...
     def _ceph_set_uri(self, uri):...
+    def _ceph_set_device_wear_level(self, devid, val):...
     def _ceph_have_mon_connection(self):...
     def _ceph_update_progress_event(self, evid, desc, progress, add_to_ceph_s):...
     def _ceph_complete_progress_event(self, evid):...

diff --git a/src/pybind/mgr/devicehealth/module.py b/src/pybind/mgr/devicehealth/module.py
index 06ad97763bfb9df691dfd7026d0ff8ac2ace6308..54b1f5b0dfc16d16d75ad4847b2aa31bf6de8df5 100644 (file)
--- a/src/pybind/mgr/devicehealth/module.py
+++ b/src/pybind/mgr/devicehealth/module.py
@@ -25,6 +25,29 @@ HEALTH_MESSAGES = {
 MAX_SAMPLES = 500
 
 
+def get_ata_wear_level(data: Dict[Any,Any]) -> Optional[float]:
+    """
+    Extract wear level (as float) from smartctl -x --json output for SATA SSD
+    """
+    for page in data.get("ata_device_statistics", {}).get("pages", []):
+        if page.get("number") != 7:
+            continue
+        for item in page.get("table", []):
+            if item["offset"] == 8:
+                return item["value"] / 100.0
+    return None
+
+
+def get_nvme_wear_level(data: Dict[Any,Any]) -> Optional[float]:
+    """
+    Extract wear level (as float) from smartctl -x --json output for NVME SSD
+    """
+    pct_used = data.get("nvme_smart_health_information_log", {}).get("percentage_used")
+    if pct_used is None:
+        return None
+    return pct_used / 100.0
+
+
 class Module(MgrModule):
     MODULE_OPTIONS = [
         Option(
@@ -450,6 +473,22 @@ class Module(MgrModule):
                 ioctx.remove_omap_keys(op, tuple(erase))
             ioctx.operate_write_op(op, devid)
 
+        # extract wear level?
+        wear_level = get_ata_wear_level(data)
+        if wear_level is None:
+            wear_level = get_nvme_wear_level(data)
+        dev_data = self.get(f"device {devid}") or {}
+        if wear_level is not None:
+            if dev_data.get(wear_level) != str(wear_level):
+                dev_data["wear_level"] = str(wear_level)
+                self.log.debug(f"updating {devid} wear level to {wear_level}")
+                self.set_device_wear_level(devid, wear_level)
+        else:
+            if "wear_level" in dev_data:
+                del dev_data["wear_level"]
+                self.log.debug(f"removing {devid} wear level")
+                self.set_device_wear_level(devid, -1.0)
+
     def _get_device_metrics(self, devid: str,
                             sample: Optional[str] = None,
                             min_sample: Optional[str] = None) -> Dict[str, Dict[str, Any]]:

diff --git a/src/pybind/mgr/mgr_module.py b/src/pybind/mgr/mgr_module.py
index d29d96eed4cf91ba98117de3730d3c3c8eedeb38..abe4079b289b3b415002f2fd6985cc731924579d 100644 (file)
--- a/src/pybind/mgr/mgr_module.py
+++ b/src/pybind/mgr/mgr_module.py
@@ -1615,6 +1615,9 @@ class MgrModule(ceph_module.BaseMgrModule, MgrModuleLoggingMixin):
         """
         return self._ceph_set_uri(uri)
 
+    def set_device_wear_level(self, devid: str, wear_level: float) -> None:
+        return self._ceph_set_device_wear_level(devid, wear_level)
+
     def have_mon_connection(self) -> bool:
         """
         Check whether this ceph-mgr daemon has an open connection