From: Naveen Naidu Date: Wed, 19 Mar 2025 13:45:00 +0000 (+0530) Subject: mgr: add new API to fetch labeled perf schema X-Git-Tag: v20.3.0~45^2~3 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=a4aebd83907a7d4f1ed5275b0adccf6d491bc296;p=ceph.git mgr: add new API to fetch labeled perf schema The manager clients such as telemetry fetches the performance counter schema using the 'get_unlabled_perf_schema' API. The information from these schemas are then used to fetch the latest values of these perf counters. The schema from the current API does not expose the information about labeled performance counters. The representation format of the schema using the current API is not capable of supporting labeled counters as it is. This is because of the presence of NULL delimiters which are used to segregate the key name and labels in the counter path. Because of these NULL delimiters, the labeled counters would only have the key name in the emitted perf schema. Hence a new API 'get_perf_schema' has been introduced to support labeled counters. The new schema format from this API looks like: ```json { "osd_scrub_sh_repl": [ { "labels": { "level": "shallow", "pooltype": "replicated" }, "counters": { "successful_scrubs_elapsed": { "description": "", "nick": "", ... } } } ] } ``` Two new subfields 'labels' and 'counters' have been added to each counter. This format is similar to what one would expect when they run the command `ceph tell osd.0 counter dump` Signed-off-by: Naveen Naidu --- diff --git a/doc/mgr/modules.rst b/doc/mgr/modules.rst index b94eeaf70981..b07f6ad335e8 100644 --- a/doc/mgr/modules.rst +++ b/doc/mgr/modules.rst @@ -508,6 +508,7 @@ function. This will result in a circular locking exception. .. automethod:: MgrModule.get_unlabeled_perf_schema .. automethod:: MgrModule.get_unlabeled_counter .. automethod:: MgrModule.get_latest_unlabeled_counter +.. automethod:: MgrModule.get_perf_schema .. automethod:: MgrModule.get_mgr_id .. automethod:: MgrModule.get_daemon_health_metrics diff --git a/src/mgr/ActivePyModules.cc b/src/mgr/ActivePyModules.cc index 33a82e53c4d9..c197692c8c95 100644 --- a/src/mgr/ActivePyModules.cc +++ b/src/mgr/ActivePyModules.cc @@ -995,6 +995,179 @@ PyObject* ActivePyModules::get_unlabeled_perf_schema_python( return f.get(); } +// Holds a list of label pairs for a counter, [(level, shallow), (pooltype, replicated)] +typedef std::vector> perf_counter_label_pairs; + +PyObject* ActivePyModules::get_perf_schema_python( + const std::string& svc_type, + const std::string& svc_id) +{ + without_gil_t no_gil; + std::lock_guard l(lock); + + DaemonStateCollection daemons; + + if (svc_type == "") { + daemons = daemon_state.get_all(); + } else if (svc_id.empty()) { + daemons = daemon_state.get_by_service(svc_type); + } else { + auto key = DaemonKey{svc_type, svc_id}; + // so that the below can be a loop in all cases + auto got = daemon_state.get(key); + if (got != nullptr) { + daemons[key] = got; + } + } + + auto f = with_gil(no_gil, [&] { return PyFormatter(); }); + + auto dump_sub_counter_information = [](PyFormatter *f, PerfCounterType type) { + // Labels can also have "." in them, eg (notice, client.4620): + // "mds_client_metrics-cephfs^@client^@client.4620^@rank^@0^@.avg_metadata_latency" + // Hence search for the last occurence of "." to get sub counter name + size_t pos = type.path.rfind('.'); + std::string sub_counter_name = type.path.substr(pos + 1, type.path.length()); + Formatter::ObjectSection counter_section(*f, sub_counter_name); + f->create_unique("description", type.description); + if (!type.nick.empty()) { + f->dump_string("nick", type.nick); + } + f->dump_unsigned("type", type.type); + f->dump_unsigned("priority", type.priority); + f->dump_unsigned("units", type.unit); + }; + + auto dump_counter_with_labels = [&dump_sub_counter_information]( + PyFormatter *f, auto key_labels, + auto type) { + f->open_object_section(""); // counter should be enclosed by array + + for (Formatter::ObjectSection labels_section{*f, "labels"}; + const auto &label : key_labels) { + f->dump_string(label.first, label.second); + } + + f->open_object_section("counters"); + dump_sub_counter_information(f, type); + }; + + + if (!daemons.empty()) { + for (auto &[key, state] : daemons) { + std::lock_guard l(state->lock); + with_gil(no_gil, [&, key = ceph::to_string(key), state = state] { + std::string_view key_name, prev_key_name; + perf_counter_label_pairs prev_key_labels; + Formatter::ObjectSection counter_section( + f, key.c_str()); // Main Object Section + std::optional array_section; + + for (const auto &[counter_name_with_labels, _] : + state->perf_counters.instances) { + /* + The path of the counter can either be: + - labeled counter path: "osd_scrub_sh_repl^@level^@shallow^@pooltype^@replicated^@.successful_scrubs_elapsed" + - unlabeled counter path: "osd.stat_bytes" + + For the above counters: + - key_names are: 'osd_scrub_sh_repl' and 'osd' + - counter names are: 'successful_scrubs_elapsed' and 'stat_bytes' + + */ + auto type = state->perf_counters.types[counter_name_with_labels]; + + // create a vector of labels i.e [(level, shallow), (pooltype, replicated)] + perf_counter_label_pairs key_labels; + auto labels = + ceph::perf_counters::key_labels(counter_name_with_labels); + std::copy_if( + labels.begin(), labels.end(), std::back_inserter(key_labels), + [](const auto &label) { return !label.first.empty(); }); + + // Extract the key names from the counter path, these key names form + // the main object section for their counters + string key_name_without_counter; + if (key_labels.empty()) { + size_t pos = counter_name_with_labels.rfind('.'); + key_name_without_counter = counter_name_with_labels.substr(0, pos); + key_name = key_name_without_counter; // key_name, osd + } else { + // key_name, osd_scrub_sh_repl + key_name = ceph::perf_counters::key_name(counter_name_with_labels); + } + + /* + Construct a schema in the following format + { + "osd": [ + { + "labels": {}, + "counters":{ + "stat_byte": { + "description": "", + "nick": "", + ... + } + } + } + ], + "osd_scrub_sh_repl":[ + { + "labels": { <---- 'label' section + "level": "shallow", + "pooltype": "replicated" + }, + "counters":{ <---- 'counters' section + "successful_scrubs_elapsed":{ <---- 'sub counter' section + "description": "", + "nick": "", + ... + } + } + } <---- 'counter object' close + ] + } + */ + + if (prev_key_name != key_name) { + if (!prev_key_name.empty()) { + f.close_section(); // close 'counters' + f.close_section(); // close 'counter object' section + } + prev_key_name = key_name; + prev_key_labels = key_labels; + array_section.emplace(f, key_name); + dump_counter_with_labels(&f, key_labels, type); + } else if ( + prev_key_name == key_name && prev_key_labels == key_labels) { + dump_sub_counter_information(&f, type); + } else if ( + prev_key_name == key_name && prev_key_labels != key_labels) { + f.close_section(); // close previous 'counters' section + f.close_section(); // close previous counter object section + dump_counter_with_labels(&f, key_labels, type); + } else { + dout(4) + << fmt::format( + "{} unable to create perf schema, not a valid condition", + __func__) + << dendl; + } + } + f.close_section(); // close 'counters' + f.close_section(); // close 'counter object' section + }); + } + } else { + dout(4) << fmt::format( + "{}: No daemon state found for {}.{}", __func__, svc_type, + svc_id) + << dendl; + } + return f.get(); +} + PyObject* ActivePyModules::get_rocksdb_version() { std::string version = std::to_string(ROCKSDB_MAJOR) + "." + diff --git a/src/mgr/ActivePyModules.h b/src/mgr/ActivePyModules.h index 172fc85694c5..4e6caf3ee1bc 100644 --- a/src/mgr/ActivePyModules.h +++ b/src/mgr/ActivePyModules.h @@ -102,6 +102,9 @@ public: PyObject *get_unlabeled_perf_schema_python( const std::string &svc_type, const std::string &svc_id); + PyObject *get_perf_schema_python( + const std::string &svc_type, + const std::string &svc_id); PyObject *get_rocksdb_version(); PyObject *get_context(); PyObject *get_osdmap(); diff --git a/src/mgr/BaseMgrModule.cc b/src/mgr/BaseMgrModule.cc index ebdd6f4c2686..395cda352923 100644 --- a/src/mgr/BaseMgrModule.cc +++ b/src/mgr/BaseMgrModule.cc @@ -680,6 +680,17 @@ get_unlabeled_perf_schema(BaseMgrModule *self, PyObject *args) return self->py_modules->get_unlabeled_perf_schema_python(type_str, svc_id); } +static PyObject* get_perf_schema(BaseMgrModule *self, PyObject *args) +{ + char *type_str = nullptr; + char *svc_id = nullptr; + if (!PyArg_ParseTuple(args, "ss:get_perf_schema", &type_str, &svc_id)) { + return nullptr; + } + + return self->py_modules->get_perf_schema_python(type_str, svc_id); +} + static PyObject* ceph_get_rocksdb_version(BaseMgrModule *self) { @@ -1490,6 +1501,9 @@ PyMethodDef BaseMgrModule_methods[] = { {"_ceph_get_unlabeled_perf_schema", (PyCFunction)get_unlabeled_perf_schema, METH_VARARGS, "Get the unlabeled performance counter schema"}, + {"_ceph_get_perf_schema", (PyCFunction)get_perf_schema, METH_VARARGS, + "Get the performance counter schema"}, + {"_ceph_get_rocksdb_version", (PyCFunction)ceph_get_rocksdb_version, METH_NOARGS, "Get the current RocksDB version number"}, diff --git a/src/pybind/mgr/ceph_module.pyi b/src/pybind/mgr/ceph_module.pyi index 17969de19e6c..91419461faad 100644 --- a/src/pybind/mgr/ceph_module.pyi +++ b/src/pybind/mgr/ceph_module.pyi @@ -73,6 +73,7 @@ class BaseMgrModule(object): def _ceph_get_server(self, hostname: Optional[str]) -> Union[ServerInfoT, List[ServerInfoT]]: ... def _ceph_get_unlabeled_perf_schema(self, svc_type: str, svc_name: str) -> Dict[str, Any]: ... + def _ceph_get_perf_schema(self, svc_type: str, svc_name: str) -> Dict[str, Any]: ... def _ceph_get_rocksdb_version(self) -> str: ... def _ceph_get_unlabeled_counter(self, svc_type: str, svc_name: str, path: str) -> Dict[str, List[Tuple[float, int]]]: ... def _ceph_get_latest_unlabeled_counter(self, svc_type, svc_name, path): ... diff --git a/src/pybind/mgr/mgr_module.py b/src/pybind/mgr/mgr_module.py index ca32bbf3bfe2..990343c31b0b 100644 --- a/src/pybind/mgr/mgr_module.py +++ b/src/pybind/mgr/mgr_module.py @@ -1632,7 +1632,7 @@ class MgrModule(ceph_module.BaseMgrModule, MgrModuleLoggingMixin): self, svc_type: str, svc_name: str ) -> Dict[str, Dict[str, Dict[str, Union[str, int]]]]: """ - Called by the plugin to fetch perf counter schema info. + Called by the plugin to fetch unlabeled perf counter schema info. svc_name can be nullptr, as can svc_type, in which case they are wildcards @@ -1641,6 +1641,22 @@ class MgrModule(ceph_module.BaseMgrModule, MgrModuleLoggingMixin): :return: list of dicts describing the counters requested """ return self._ceph_get_unlabeled_perf_schema(svc_type, svc_name) + + @API.expose + def get_perf_schema(self, + svc_type: str, + svc_name: str) -> Dict[str, + Dict[str, Dict[str, Union[str, int]]]]: + """ + Called by the plugin to fetch perf counter schema info. + svc_name can be nullptr, as can svc_type, in which case + they are wildcards + + :param str svc_type: + :param str svc_name: + :return: list of dicts describing the counters requested + """ + return self._ceph_get_perf_schema(svc_type, svc_name) def get_rocksdb_version(self) -> str: """