From: Jan Fajerski Date: Fri, 7 Sep 2018 08:26:31 +0000 (+0200) Subject: mgr: add get_latest_counter() to C++ -> Python interface. X-Git-Tag: v14.0.1~218^2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=b421142b1c6fb0a6cbd2f76d007ee07502a8d689;p=ceph-ci.git mgr: add get_latest_counter() to C++ -> Python interface. This allows Python land to only retrieve the latest perf counters, instead of of retrieving all from C++ land and pick out the latest. Signed-off-by: Jan Fajerski --- diff --git a/src/mgr/ActivePyModules.cc b/src/mgr/ActivePyModules.cc index 6bac2ec6e03..0ba80c79239 100644 --- a/src/mgr/ActivePyModules.cc +++ b/src/mgr/ActivePyModules.cc @@ -603,10 +603,11 @@ std::map ActivePyModules::get_services() const return result; } -PyObject* ActivePyModules::get_counter_python( +PyObject* ActivePyModules::with_perf_counters( + std::function fct, const std::string &svc_name, const std::string &svc_id, - const std::string &path) + const std::string &path) const { PyThreadState *tstate = PyEval_SaveThread(); Mutex::Locker l(lock); @@ -621,27 +622,10 @@ PyObject* ActivePyModules::get_counter_python( if (metadata->perf_counters.instances.count(path)) { auto counter_instance = metadata->perf_counters.instances.at(path); auto counter_type = metadata->perf_counters.types.at(path); - if (counter_type.type & PERFCOUNTER_LONGRUNAVG) { - const auto &avg_data = counter_instance.get_data_avg(); - for (const auto &datapoint : avg_data) { - f.open_array_section("datapoint"); - f.dump_unsigned("t", datapoint.t.sec()); - f.dump_unsigned("s", datapoint.s); - f.dump_unsigned("c", datapoint.c); - f.close_section(); - } - } else { - const auto &data = counter_instance.get_data(); - for (const auto &datapoint : data) { - f.open_array_section("datapoint"); - f.dump_unsigned("t", datapoint.t.sec()); - f.dump_unsigned("v", datapoint.v); - f.close_section(); - } - } + fct(counter_instance, counter_type, f); } else { dout(4) << "Missing counter: '" << path << "' (" - << svc_name << "." << svc_id << ")" << dendl; + << svc_name << "." << svc_id << ")" << dendl; dout(20) << "Paths are:" << dendl; for (const auto &i : metadata->perf_counters.instances) { dout(20) << i.first << dendl; @@ -649,12 +633,68 @@ PyObject* ActivePyModules::get_counter_python( } } else { dout(4) << "No daemon state for " - << svc_name << "." << svc_id << ")" << dendl; + << svc_name << "." << svc_id << ")" << dendl; } f.close_section(); return f.get(); } +PyObject* ActivePyModules::get_counter_python( + const std::string &svc_name, + const std::string &svc_id, + const std::string &path) +{ + auto extract_counters = []( + PerfCounterInstance& counter_instance, + PerfCounterType& counter_type, + PyFormatter& f) + { + if (counter_type.type & PERFCOUNTER_LONGRUNAVG) { + const auto &avg_data = counter_instance.get_data_avg(); + for (const auto &datapoint : avg_data) { + f.open_array_section("datapoint"); + f.dump_unsigned("t", datapoint.t.sec()); + f.dump_unsigned("s", datapoint.s); + f.dump_unsigned("c", datapoint.c); + f.close_section(); + } + } else { + const auto &data = counter_instance.get_data(); + for (const auto &datapoint : data) { + f.open_array_section("datapoint"); + f.dump_unsigned("t", datapoint.t.sec()); + f.dump_unsigned("v", datapoint.v); + f.close_section(); + } + } + }; + return with_perf_counters(extract_counters, svc_name, svc_id, path); +} + +PyObject* ActivePyModules::get_latest_counter_python( + const std::string &svc_name, + const std::string &svc_id, + const std::string &path) +{ + auto extract_latest_counters = []( + PerfCounterInstance& counter_instance, + PerfCounterType& counter_type, + PyFormatter& f) + { + if (counter_type.type & PERFCOUNTER_LONGRUNAVG) { + const auto &datapoint = counter_instance.get_latest_data_avg(); + f.dump_unsigned("t", datapoint.t.sec()); + f.dump_unsigned("s", datapoint.s); + f.dump_unsigned("c", datapoint.c); + } else { + const auto &datapoint = counter_instance.get_latest_data(); + f.dump_unsigned("t", datapoint.t.sec()); + f.dump_unsigned("v", datapoint.v); + } + }; + return with_perf_counters(extract_latest_counters, svc_name, svc_id, path); +} + PyObject* ActivePyModules::get_perf_schema_python( const std::string &svc_type, const std::string &svc_id) diff --git a/src/mgr/ActivePyModules.h b/src/mgr/ActivePyModules.h index 0b2ccd7ae88..91fc09697ec 100644 --- a/src/mgr/ActivePyModules.h +++ b/src/mgr/ActivePyModules.h @@ -18,6 +18,8 @@ #include "common/Finisher.h" #include "common/Mutex.h" +#include "PyFormatter.h" + #include "osdc/Objecter.h" #include "client/Client.h" #include "common/LogClient.h" @@ -71,11 +73,23 @@ public: const std::string &svc_type, const std::string &svc_id, const std::string &path); + PyObject *get_latest_counter_python( + const std::string &svc_type, + const std::string &svc_id, + const std::string &path); PyObject *get_perf_schema_python( const std::string &svc_type, const std::string &svc_id); PyObject *get_context(); PyObject *get_osdmap(); + PyObject *with_perf_counters( + std::function fct, + const std::string &svc_name, + const std::string &svc_id, + const std::string &path) const; bool get_store(const std::string &module_name, const std::string &key, std::string *val) const; diff --git a/src/mgr/BaseMgrModule.cc b/src/mgr/BaseMgrModule.cc index c52bf165d86..6cc3cda87a5 100644 --- a/src/mgr/BaseMgrModule.cc +++ b/src/mgr/BaseMgrModule.cc @@ -505,6 +505,20 @@ get_counter(BaseMgrModule *self, PyObject *args) svc_name, svc_id, counter_path); } +static PyObject* +get_latest_counter(BaseMgrModule *self, PyObject *args) +{ + char *svc_name = nullptr; + char *svc_id = nullptr; + char *counter_path = nullptr; + if (!PyArg_ParseTuple(args, "sss:get_counter", &svc_name, + &svc_id, &counter_path)) { + return nullptr; + } + return self->py_modules->get_latest_counter_python( + svc_name, svc_id, counter_path); +} + static PyObject* get_perf_schema(BaseMgrModule *self, PyObject *args) { @@ -641,6 +655,9 @@ PyMethodDef BaseMgrModule_methods[] = { {"_ceph_get_counter", (PyCFunction)get_counter, METH_VARARGS, "Get a performance counter"}, + {"_ceph_get_latest_counter", (PyCFunction)get_latest_counter, METH_VARARGS, + "Get the latest performance counter"}, + {"_ceph_get_perf_schema", (PyCFunction)get_perf_schema, METH_VARARGS, "Get the performance counter schema"}, diff --git a/src/mgr/DaemonState.h b/src/mgr/DaemonState.h index 0a083ac5617..5732178390a 100644 --- a/src/mgr/DaemonState.h +++ b/src/mgr/DaemonState.h @@ -74,10 +74,18 @@ class PerfCounterInstance { return buffer; } + const DataPoint& get_latest_data() const + { + return buffer.back(); + } const boost::circular_buffer & get_data_avg() const { return avg_buffer; } + const AvgDataPoint& get_latest_data_avg() const + { + return avg_buffer.back(); + } void push(utime_t t, uint64_t const &v); void push_avg(utime_t t, uint64_t const &s, uint64_t const &c); diff --git a/src/pybind/mgr/mgr_module.py b/src/pybind/mgr/mgr_module.py index fd427bd0740..d1ab2de7191 100644 --- a/src/pybind/mgr/mgr_module.py +++ b/src/pybind/mgr/mgr_module.py @@ -479,6 +479,20 @@ class MgrModule(ceph_module.BaseMgrModule): """ return self._ceph_get_counter(svc_type, svc_name, path) + def get_latest_counter(self, svc_type, svc_name, path): + """ + Called by the plugin to fetch only the newest performance counter data + pointfor a particular counter on a particular service. + + :param str svc_type: + :param str svc_name: + :param str path: a period-separated concatenation of the subsystem and the + counter name, for example "mds.inodes". + :return: A list of two-tuples of (timestamp, value) is returned. This may be + empty if no data is available. + """ + return self._ceph_get_latest_counter(svc_type, svc_name, path) + def list_servers(self): """ Like ``get_server``, but gives information about all servers (i.e. all @@ -721,19 +735,17 @@ class MgrModule(ceph_module.BaseMgrModule): """ return self._ceph_get_osdmap() - # TODO: improve C++->Python interface to return just - # the latest if that's all we want. def get_latest(self, daemon_type, daemon_name, counter): - data = self.get_counter(daemon_type, daemon_name, counter)[counter] + data = self.get_latest_counter(daemon_type, daemon_name, counter)[counter] if data: - return data[-1][1] + return data[1] else: return 0 def get_latest_avg(self, daemon_type, daemon_name, counter): - data = self.get_counter(daemon_type, daemon_name, counter)[counter] + data = self.get_latest_counter(daemon_type, daemon_name, counter)[counter] if data: - return (data[-1][1], data[-1][2]) + return (data[1], data[2]) else: return (0, 0) @@ -742,7 +754,7 @@ class MgrModule(ceph_module.BaseMgrModule): Return the perf counters currently known to this ceph-mgr instance, filtered by priority equal to or greater than `prio_limit`. - The result us a map of string to dict, associating services + The result is a map of string to dict, associating services (like "osd.123") with their counters. The counter dict for each service maps counter paths to a counter info structure, which is the information from