crimson/osd: implement basic reactor-utilization stats report to log

Signed-off-by: Yingxin Cheng <yingxin.cheng@intel.com>
(cherry picked from commit 99909d45a282fb9dc89c6e8c98f4b866e67b09cb)

diff --git a/src/common/options/crimson.yaml.in b/src/common/options/crimson.yaml.in
index c52c54d5250b65764e616df12e10fcf3f4112062..6938786d475cbcc311a184eaf8886c9dfed58b9a 100644 (file)
--- a/src/common/options/crimson.yaml.in
+++ b/src/common/options/crimson.yaml.in
@@ -31,6 +31,11 @@ options:
   desc: CPU cores on which alienstore threads will run in cpuset(7) format
   flags:
   - startup
+- name: crimson_osd_stat_interval
+  type: int
+  level: advanced
+  default: 0
+  desc: Report OSD status periodically in seconds, 0 to disable
 - name: seastore_segment_size
   type: size
   desc: Segment size to use for SegmentManager

diff --git a/src/crimson/common/utility.h b/src/crimson/common/utility.h
index 86b30815585c229f2e0bf900726b04ca6407f9f2..fae53cb6bd0addc47a6c19e4312db5a8510db719 100644 (file)
--- a/src/crimson/common/utility.h
+++ b/src/crimson/common/utility.h
@@ -5,6 +5,8 @@
 
 #include <type_traits>
 
+#include <seastar/core/metrics_api.hh>
+
 namespace _impl {
   template <class T> struct always_false : std::false_type {};
 };
@@ -36,3 +38,16 @@ auto apply_method_to_tuple(Obj &obj, Method method, ArgTuple &&tuple) {
     obj, method, std::forward<ArgTuple>(tuple),
     std::make_index_sequence<tuple_size>());
 }
+
+inline double get_reactor_utilization() {
+  auto &value_map = seastar::metrics::impl::get_value_map();
+  auto found = value_map.find("reactor_utilization");
+  assert(found != value_map.end());
+  auto &[full_name, metric_family] = *found;
+  std::ignore = full_name;
+  assert(metric_family.size() == 1);
+  const auto& [labels, metric] = *metric_family.begin();
+  std::ignore = labels;
+  auto value = (*metric)();
+  return value.d();
+}

diff --git a/src/crimson/osd/osd.cc b/src/crimson/osd/osd.cc
index 939fbc59beb8cd6f820831550944bfba9dfcd2f2..005e8538ed1787a28c410183920bb2ca0d610cca 100644 (file)
--- a/src/crimson/osd/osd.cc
+++ b/src/crimson/osd/osd.cc
@@ -387,6 +387,26 @@ seastar::future<> OSD::start()
         std::ref(osd_states));
     });
   }).then([this, FNAME] {
+    auto stats_seconds = local_conf().get_val<int64_t>("crimson_osd_stat_interval");
+    if (stats_seconds > 0) {
+      shard_stats.resize(seastar::smp::count);
+      stats_timer.set_callback([this, FNAME] {
+        std::ignore = shard_services.invoke_on_all(
+          [this](auto &local_service) {
+          auto stats = local_service.report_stats();
+          shard_stats[seastar::this_shard_id()] = stats;
+        }).then([this, FNAME] {
+          std::ostringstream oss;
+          for (const auto &stats : shard_stats) {
+            oss << int(stats.reactor_utilization);
+            oss << ",";
+          }
+          INFO("reactor_utilizations: {}", oss.str());
+        });
+      });
+      stats_timer.arm_periodic(std::chrono::seconds(stats_seconds));
+    }
+
     heartbeat.reset(new Heartbeat{
        whoami, get_shard_services(),
        *monc, *hb_front_msgr, *hb_back_msgr});
@@ -1320,6 +1340,7 @@ seastar::future<> OSD::restart()
 {
   beacon_timer.cancel();
   tick_timer.cancel();
+  stats_timer.cancel();
   return pg_shard_manager.set_up_epoch(
     0
   ).then([this] {

diff --git a/src/crimson/osd/osd.h b/src/crimson/osd/osd.h
index fa3b0293072b8ce96305874cc3a1a37c9554897a..7b0a08fc3b9a22a122e50f225cf0e7e59db2e3fc 100644 (file)
--- a/src/crimson/osd/osd.h
+++ b/src/crimson/osd/osd.h
@@ -128,6 +128,9 @@ class OSD final : public crimson::net::Dispatcher,
   std::unique_ptr<Heartbeat> heartbeat;
   seastar::timer<seastar::lowres_clock> tick_timer;
 
+  seastar::timer<seastar::lowres_clock> stats_timer;
+  std::vector<ShardServices::shard_stats_t> shard_stats;
+
   const char** get_tracked_conf_keys() const final;
   void handle_conf_change(const ConfigProxy& conf,
                           const std::set<std::string> &changed) final;

diff --git a/src/crimson/osd/shard_services.h b/src/crimson/osd/shard_services.h
index 57dff9d2ee3e59dd2836367122f2e9ed5349f0b6..e00f3441319ea18c4bd6812d0ff11a79e13f86e5 100644 (file)
--- a/src/crimson/osd/shard_services.h
+++ b/src/crimson/osd/shard_services.h
@@ -402,6 +402,13 @@ public:
     return local_state.store;
   }
 
+  struct shard_stats_t {
+    double reactor_utilization;
+  };
+  shard_stats_t report_stats() {
+    return {get_reactor_utilization()};
+  }
+
   auto remove_pg(spg_t pgid) {
     local_state.pg_map.remove_pg(pgid);
     return pg_to_shard_mapping.remove_pg_mapping(pgid);