mgr: add new API to fetch labeled perf schema

The manager clients such as telemetry fetches the performance counter
schema using the 'get_unlabled_perf_schema' API. The information from
these schemas are then used to fetch the latest values of these perf
counters. The schema from the current API does not expose the
information about labeled performance counters.

The representation format of the schema using the current API is not
capable of supporting labeled counters as it is. This is because of the
presence of NULL delimiters which are used to segregate the key name and
labels in the counter path. Because of these NULL delimiters, the
labeled counters would only have the key name in the emitted perf
schema.

Hence a new API 'get_perf_schema' has been introduced to support
labeled counters.

The new schema format from this API looks like:
```json
{
 "osd_scrub_sh_repl": [
  {
    "labels": {
      "level": "shallow",
      "pooltype": "replicated"
    },
    "counters": {
      "successful_scrubs_elapsed": {
        "description": "",
        "nick": "",
        ...
      }
    }
  }
 ]
}
```

Two new subfields 'labels' and 'counters' have been added to each
counter. This format is similar to what one would expect when they run
the command `ceph tell osd.0 counter dump`

Signed-off-by: Naveen Naidu <naveen.naidu@ibm.com>

diff --git a/doc/mgr/modules.rst b/doc/mgr/modules.rst
index b94eeaf70981835bb4978b6d344c4b4ae07fd017..b07f6ad335e8bda4b72be705c5c76aa57e208c9c 100644 (file)
--- a/doc/mgr/modules.rst
+++ b/doc/mgr/modules.rst
@@ -508,6 +508,7 @@ function. This will result in a circular locking exception.
 .. automethod:: MgrModule.get_unlabeled_perf_schema
 .. automethod:: MgrModule.get_unlabeled_counter
 .. automethod:: MgrModule.get_latest_unlabeled_counter
+.. automethod:: MgrModule.get_perf_schema   
 .. automethod:: MgrModule.get_mgr_id
 .. automethod:: MgrModule.get_daemon_health_metrics
 

diff --git a/src/mgr/ActivePyModules.cc b/src/mgr/ActivePyModules.cc
index 33a82e53c4d9968d60670e3477cb3461b1348a14..c197692c8c95d24a4d433e053b8c39127663aee5 100644 (file)
--- a/src/mgr/ActivePyModules.cc
+++ b/src/mgr/ActivePyModules.cc
@@ -995,6 +995,179 @@ PyObject* ActivePyModules::get_unlabeled_perf_schema_python(
   return f.get();
 }
 
+// Holds a list of label pairs for a counter, [(level, shallow), (pooltype, replicated)]
+typedef std::vector<pair<std::string_view, std::string_view>> perf_counter_label_pairs;
+
+PyObject* ActivePyModules::get_perf_schema_python(
+    const std::string& svc_type,
+    const std::string& svc_id)
+{
+  without_gil_t no_gil;
+  std::lock_guard l(lock);
+
+  DaemonStateCollection daemons;
+
+  if (svc_type == "") {
+    daemons = daemon_state.get_all();
+  } else if (svc_id.empty()) {
+    daemons = daemon_state.get_by_service(svc_type);
+  } else {
+    auto key = DaemonKey{svc_type, svc_id};
+    // so that the below can be a loop in all cases
+    auto got = daemon_state.get(key);
+    if (got != nullptr) {
+      daemons[key] = got;
+    }
+  }
+
+  auto f = with_gil(no_gil, [&] { return PyFormatter(); });
+
+  auto dump_sub_counter_information = [](PyFormatter *f, PerfCounterType type) {
+    // Labels can also have "." in them, eg (notice, client.4620):
+    // "mds_client_metrics-cephfs^@client^@client.4620^@rank^@0^@.avg_metadata_latency"
+    // Hence search for the last occurence of "." to get sub counter name
+    size_t pos = type.path.rfind('.');
+    std::string sub_counter_name = type.path.substr(pos + 1, type.path.length());
+    Formatter::ObjectSection counter_section(*f, sub_counter_name);
+    f->create_unique("description", type.description);
+    if (!type.nick.empty()) {
+      f->dump_string("nick", type.nick);
+    }
+    f->dump_unsigned("type", type.type);
+    f->dump_unsigned("priority", type.priority);
+    f->dump_unsigned("units", type.unit);
+  };
+
+  auto dump_counter_with_labels = [&dump_sub_counter_information](
+                                     PyFormatter *f, auto key_labels,
+                                     auto type) {
+    f->open_object_section("");         // counter should be enclosed by array
+
+    for (Formatter::ObjectSection labels_section{*f, "labels"};
+        const auto &label : key_labels) {
+      f->dump_string(label.first, label.second);
+    }
+
+    f->open_object_section("counters");
+    dump_sub_counter_information(f, type);
+  };
+
+
+  if (!daemons.empty()) {
+    for (auto &[key, state] : daemons) {
+      std::lock_guard l(state->lock);
+      with_gil(no_gil, [&, key = ceph::to_string(key), state = state] {
+       std::string_view key_name, prev_key_name;
+       perf_counter_label_pairs prev_key_labels;
+       Formatter::ObjectSection counter_section(
+           f, key.c_str());  // Main Object Section
+       std::optional<Formatter::ArraySection> array_section;
+
+       for (const auto &[counter_name_with_labels, _] :
+            state->perf_counters.instances) {
+         /*
+              The path of the counter can either be:
+                - labeled counter path: "osd_scrub_sh_repl^@level^@shallow^@pooltype^@replicated^@.successful_scrubs_elapsed"
+                - unlabeled counter path: "osd.stat_bytes"
+
+              For the above counters:
+                - key_names are: 'osd_scrub_sh_repl' and 'osd'
+                - counter names are: 'successful_scrubs_elapsed' and 'stat_bytes'
+
+          */
+         auto type = state->perf_counters.types[counter_name_with_labels];
+
+         // create a vector of labels i.e [(level, shallow), (pooltype, replicated)]
+         perf_counter_label_pairs key_labels;
+         auto labels =
+             ceph::perf_counters::key_labels(counter_name_with_labels);
+         std::copy_if(
+             labels.begin(), labels.end(), std::back_inserter(key_labels),
+             [](const auto &label) { return !label.first.empty(); });
+
+         // Extract the key names from the counter path, these key names form
+         // the main object section for their counters
+         string key_name_without_counter;
+         if (key_labels.empty()) {
+           size_t pos = counter_name_with_labels.rfind('.');
+           key_name_without_counter = counter_name_with_labels.substr(0, pos);
+           key_name = key_name_without_counter;  // key_name, osd
+         } else {
+           // key_name, osd_scrub_sh_repl
+           key_name = ceph::perf_counters::key_name(counter_name_with_labels);
+         }
+
+         /*
+            Construct a schema in the following format
+            {
+              "osd": [
+                {
+                  "labels": {},
+                  "counters":{
+                    "stat_byte": {
+                      "description": "",
+                      "nick": "",
+                      ...
+                    }
+                  }
+                }
+              ],
+              "osd_scrub_sh_repl":[
+                {
+                  "labels": {                         <---- 'label' section
+                    "level": "shallow",
+                    "pooltype": "replicated"
+                  },
+                  "counters":{                        <---- 'counters' section
+                    "successful_scrubs_elapsed":{     <---- 'sub counter' section
+                      "description": "",
+                      "nick": "",
+                      ...
+                    }
+                  }
+                }                                     <---- 'counter object' close
+              ]
+            }
+          */
+
+         if (prev_key_name != key_name) {
+           if (!prev_key_name.empty()) {
+             f.close_section();  // close 'counters'
+             f.close_section();  // close 'counter object' section
+           }
+           prev_key_name = key_name;
+           prev_key_labels = key_labels;
+           array_section.emplace(f, key_name);
+           dump_counter_with_labels(&f, key_labels, type);
+         } else if (
+             prev_key_name == key_name && prev_key_labels == key_labels) {
+           dump_sub_counter_information(&f, type);
+         } else if (
+             prev_key_name == key_name && prev_key_labels != key_labels) {
+           f.close_section();  // close previous 'counters' section
+           f.close_section();  // close previous counter object section
+           dump_counter_with_labels(&f, key_labels, type);
+         } else {
+           dout(4)
+               << fmt::format(
+                      "{} unable to create perf schema, not a valid condition",
+                      __func__)
+               << dendl;
+         }
+       }
+       f.close_section();  // close 'counters'
+       f.close_section();  // close 'counter object' section
+      });
+    }
+  } else {
+    dout(4) << fmt::format(
+                  "{}: No daemon state found for  {}.{}", __func__, svc_type,
+                  svc_id)
+           << dendl;
+  }
+  return f.get();
+}
+
 PyObject* ActivePyModules::get_rocksdb_version()
 {
   std::string version = std::to_string(ROCKSDB_MAJOR) + "." +

diff --git a/src/mgr/ActivePyModules.h b/src/mgr/ActivePyModules.h
index 172fc85694c595d1510fc134291944390847bd1a..4e6caf3ee1bce4e02a9fa52b73e16f39c0f8e3c1 100644 (file)
--- a/src/mgr/ActivePyModules.h
+++ b/src/mgr/ActivePyModules.h
@@ -102,6 +102,9 @@ public:
   PyObject *get_unlabeled_perf_schema_python(
       const std::string &svc_type,
       const std::string &svc_id);
+  PyObject *get_perf_schema_python(
+      const std::string &svc_type,
+      const std::string &svc_id);
   PyObject *get_rocksdb_version();
   PyObject *get_context();
   PyObject *get_osdmap();

diff --git a/src/mgr/BaseMgrModule.cc b/src/mgr/BaseMgrModule.cc
index ebdd6f4c26860d9ce625432c12982afb149f04d5..395cda35292301144a2dff3e3818791234c4dc08 100644 (file)
--- a/src/mgr/BaseMgrModule.cc
+++ b/src/mgr/BaseMgrModule.cc
@@ -680,6 +680,17 @@ get_unlabeled_perf_schema(BaseMgrModule *self, PyObject *args)
   return self->py_modules->get_unlabeled_perf_schema_python(type_str, svc_id);
 }
 
+static PyObject* get_perf_schema(BaseMgrModule *self, PyObject *args)
+{
+  char *type_str = nullptr;
+  char *svc_id = nullptr;
+  if (!PyArg_ParseTuple(args, "ss:get_perf_schema", &type_str, &svc_id)) {
+    return nullptr;
+  }
+
+  return self->py_modules->get_perf_schema_python(type_str, svc_id);
+}
+
 static PyObject*
 ceph_get_rocksdb_version(BaseMgrModule *self)
 {
@@ -1490,6 +1501,9 @@ PyMethodDef BaseMgrModule_methods[] = {
   {"_ceph_get_unlabeled_perf_schema", (PyCFunction)get_unlabeled_perf_schema, METH_VARARGS,
     "Get the unlabeled performance counter schema"},
 
+  {"_ceph_get_perf_schema", (PyCFunction)get_perf_schema, METH_VARARGS,
+    "Get the performance counter schema"},
+
   {"_ceph_get_rocksdb_version", (PyCFunction)ceph_get_rocksdb_version, METH_NOARGS,
     "Get the current RocksDB version number"},
 

diff --git a/src/pybind/mgr/ceph_module.pyi b/src/pybind/mgr/ceph_module.pyi
index 17969de19e6c313c1707f2e918fd811e6a346261..91419461faad2f41cb85499186419e40d373a250 100644 (file)
--- a/src/pybind/mgr/ceph_module.pyi
+++ b/src/pybind/mgr/ceph_module.pyi
@@ -73,6 +73,7 @@ class BaseMgrModule(object):
     def _ceph_get_server(self, hostname: Optional[str]) -> Union[ServerInfoT,
                                                                  List[ServerInfoT]]: ...
     def _ceph_get_unlabeled_perf_schema(self, svc_type: str, svc_name: str) -> Dict[str, Any]: ...
+    def _ceph_get_perf_schema(self, svc_type: str, svc_name: str) -> Dict[str, Any]: ...
     def _ceph_get_rocksdb_version(self) -> str: ...
     def _ceph_get_unlabeled_counter(self, svc_type: str, svc_name: str, path: str) -> Dict[str, List[Tuple[float, int]]]: ...
     def _ceph_get_latest_unlabeled_counter(self, svc_type, svc_name, path): ...

diff --git a/src/pybind/mgr/mgr_module.py b/src/pybind/mgr/mgr_module.py
index ca32bbf3bfe2d3d8c0195e1a6213d680fd07d574..990343c31b0b20eda8a12f0b626e17d66b434425 100644 (file)
--- a/src/pybind/mgr/mgr_module.py
+++ b/src/pybind/mgr/mgr_module.py
@@ -1632,7 +1632,7 @@ class MgrModule(ceph_module.BaseMgrModule, MgrModuleLoggingMixin):
         self, svc_type: str, svc_name: str
     ) -> Dict[str, Dict[str, Dict[str, Union[str, int]]]]:
         """
-        Called by the plugin to fetch perf counter schema info.
+        Called by the plugin to fetch unlabeled perf counter schema info.
         svc_name can be nullptr, as can svc_type, in which case
         they are wildcards
 
@@ -1641,6 +1641,22 @@ class MgrModule(ceph_module.BaseMgrModule, MgrModuleLoggingMixin):
         :return: list of dicts describing the counters requested
         """
         return self._ceph_get_unlabeled_perf_schema(svc_type, svc_name)
+    
+    @API.expose
+    def get_perf_schema(self,
+                        svc_type: str,
+                        svc_name: str) -> Dict[str,
+                                               Dict[str, Dict[str, Union[str, int]]]]:
+        """
+        Called by the plugin to fetch perf counter schema info.
+        svc_name can be nullptr, as can svc_type, in which case
+        they are wildcards
+
+        :param str svc_type:
+        :param str svc_name:
+        :return: list of dicts describing the counters requested
+        """
+        return self._ceph_get_perf_schema(svc_type, svc_name)
 
     def get_rocksdb_version(self) -> str:
         """