crimson/seastore: add seastore operation latency metrics

Signed-off-by: chunmei-liu <chunmei.liu@intel.com>

diff --git a/src/crimson/os/seastore/seastore.cc b/src/crimson/os/seastore/seastore.cc
index 13a32dea08e0739af6befd279918fde720e95f30..4b8fcb7b1c2d040c23c06310777656612ebc2863 100644 (file)
--- a/src/crimson/os/seastore/seastore.cc
+++ b/src/crimson/os/seastore/seastore.cc
@@ -40,10 +40,46 @@ SeaStore::SeaStore(
     transaction_manager(std::move(tm)),
     collection_manager(std::move(cm)),
     onode_manager(std::move(om))
-{}
+{
+  register_metrics();
+}
 
 SeaStore::~SeaStore() = default;
 
+void SeaStore::register_metrics()
+{
+  namespace sm = seastar::metrics;
+  using op_type_t = SeaStore::op_type_t;
+  auto lat_label = sm::label("latency");
+  std::map<op_type_t, sm::label_instance> labels_by_op_type = {
+    {op_type_t::TRANSACTION,     lat_label("TRANSACTION")},
+    {op_type_t::READ,            lat_label("READ")},
+    {op_type_t::WRITE,           lat_label("WRITE")},
+    {op_type_t::GET_ATTR,        lat_label("GET_ATTR")},
+    {op_type_t::GET_ATTRS,       lat_label("GET_ATTRS")},
+    {op_type_t::STAT,            lat_label("STAT")},
+    {op_type_t::OMAP_GET_VALUES, lat_label("OMAP_GET_VALUES")},
+    {op_type_t::OMAP_LIST,       lat_label("OMAP_LIST")},
+  };
+
+  for (auto& [op_type, label] : labels_by_op_type) {
+    auto desc = fmt::format("latency of seastore operation (optype={})",
+                            op_type);
+    metrics.add_group(
+      "seastore",
+      {
+        sm::make_histogram(
+          "op_lat", [this, op_type] {
+          return get_latency(op_type);
+          },
+          sm::description(desc),
+          {label}
+        ),
+      }
+    );
+  }
+}
+
 seastar::future<> SeaStore::stop()
 {
   return seastar::now();
@@ -216,6 +252,7 @@ SeaStore::read_errorator::future<ceph::bufferlist> SeaStore::read(
     ch,
     oid,
     Transaction::src_t::READ,
+    op_type_t::READ,
     [=](auto &t, auto &onode) -> ObjectDataHandler::read_ret {
       size_t size = onode.get_layout().size;
 
@@ -261,6 +298,7 @@ SeaStore::get_attr_errorator::future<ceph::bufferlist> SeaStore::get_attr(
     c,
     oid,
     Transaction::src_t::READ,
+    op_type_t::GET_ATTR,
     [=](auto &t, auto& onode) -> _omap_get_value_ertr::future<ceph::bufferlist> {
       auto& layout = onode.get_layout();
       if (name == OI_ATTR && layout.oi_size) {
@@ -295,6 +333,7 @@ SeaStore::get_attrs_ertr::future<SeaStore::attrs_t> SeaStore::get_attrs(
     c,
     oid,
     Transaction::src_t::READ,
+    op_type_t::GET_ATTRS,
     [=](auto &t, auto& onode) {
       auto& layout = onode.get_layout();
       return _omap_list(layout.xattr_root, t, std::nullopt,
@@ -329,6 +368,7 @@ seastar::future<struct stat> SeaStore::stat(
     c,
     oid,
     Transaction::src_t::READ,
+    op_type_t::STAT,
     [=, &oid](auto &t, auto &onode) {
       struct stat st;
       auto &olayout = onode.get_layout();
@@ -366,6 +406,7 @@ SeaStore::omap_get_values(
     c,
     oid,
     Transaction::src_t::READ,
+    op_type_t::OMAP_GET_VALUES,
     [this, keys](auto &t, auto &onode) {
       omap_root_t omap_root = onode.get_layout().omap_root.get();
       return _omap_get_values(
@@ -483,6 +524,7 @@ SeaStore::omap_get_values_ret_t SeaStore::omap_list(
     c,
     oid,
     Transaction::src_t::READ,
+    op_type_t::OMAP_LIST,
     [this, config, &start](auto &t, auto &onode) {
       return _omap_list(
        onode.get_layout().omap_root,
@@ -633,6 +675,7 @@ seastar::future<> SeaStore::do_transaction(
     _ch,
     std::move(_t),
     Transaction::src_t::MUTATE,
+    op_type_t::TRANSACTION,
     [this](auto &ctx) {
       return onode_manager->get_or_create_onodes(
        *ctx.transaction, ctx.iter.get_objects()

diff --git a/src/crimson/os/seastore/seastore.h b/src/crimson/os/seastore/seastore.h
index c7583ee05779702f8d8f80f0bb831a33e685e550..6bf7a8e473bd9a71a893c3f81ab6508cfd96ad26 100644 (file)
--- a/src/crimson/os/seastore/seastore.h
+++ b/src/crimson/os/seastore/seastore.h
@@ -128,6 +128,17 @@ public:
   unsigned get_max_attr_name_length() const final {
     return 256;
   }
+  enum class op_type_t : uint8_t {
+    TRANSACTION = 0,
+    READ,
+    WRITE,
+    GET_ATTR,
+    GET_ATTRS,
+    STAT,
+    OMAP_GET_VALUES,
+    OMAP_LIST,
+    MAX
+  };
 
 private:
   struct internal_context_t {
@@ -146,6 +157,7 @@ private:
     std::vector<OnodeRef> onodes;
 
     ceph::os::Transaction::iterator iter;
+    std::chrono::steady_clock::time_point begin_timestamp = std::chrono::steady_clock::now();
 
     template <typename TM>
     void reset_preserve_handle(TM &tm) {
@@ -162,13 +174,14 @@ private:
     CollectionRef ch,
     ceph::os::Transaction &&t,
     Transaction::src_t src,
+    op_type_t op_type,
     F &&f) {
     return seastar::do_with(
       internal_context_t(
        ch, std::move(t),
        transaction_manager->create_transaction(src)),
       std::forward<F>(f),
-      [this](auto &ctx, auto &f) {
+      [this, op_type](auto &ctx, auto &f) {
        return ctx.transaction->get_handle().take_collection_lock(
          static_cast<SeastoreCollection&>(*(ctx.ch)).ordering_lock
        ).then([&, this] {
@@ -181,6 +194,9 @@ private:
              on_error(ctx.ext_transaction);
            })
          );
+       }).then([this, op_type, &ctx] {
+         add_latency_sample(op_type,
+             std::chrono::steady_clock::now() - ctx.begin_timestamp);
        });
       });
   }
@@ -190,7 +206,9 @@ private:
     CollectionRef ch,
     const ghobject_t &oid,
     Transaction::src_t src,
+    op_type_t op_type,
     F &&f) const {
+    auto begin_time = std::chrono::steady_clock::now();
     return seastar::do_with(
       oid,
       Ret{},
@@ -208,7 +226,9 @@ private:
          }).safe_then([&ret](auto _ret) {
            ret = _ret;
          });
-       }).safe_then([&ret] {
+       }).safe_then([&ret, op_type, &t, begin_time, this] {
+         const_cast<SeaStore*>(this)->add_latency_sample(op_type,
+                     std::chrono::steady_clock::now() - begin_time);
          return seastar::make_ready_future<Ret>(ret);
        });
       });
@@ -310,6 +330,26 @@ private:
     std::map<std::string, ceph::bufferlist>&& kvs);
 
   boost::intrusive_ptr<SeastoreCollection> _get_collection(const coll_t& cid);
+
+  static constexpr auto LAT_MAX = static_cast<std::size_t>(op_type_t::MAX);
+  struct {
+    std::array<seastar::metrics::histogram, LAT_MAX> op_lat;
+  } stats;
+
+  seastar::metrics::histogram& get_latency(
+      op_type_t op_type) {
+    assert(static_cast<std::size_t>(op_type) < stats.op_lat.size());
+    return stats.op_lat[static_cast<std::size_t>(op_type)];
+  }
+
+  void add_latency_sample(op_type_t op_type,
+       std::chrono::steady_clock::duration dur) {
+    seastar::metrics::histogram& lat = get_latency(op_type);
+    lat.sample_count++;
+    lat.sample_sum += std::chrono::duration_cast<std::chrono::milliseconds>(dur).count();
+  }
+  seastar::metrics::metric_group metrics;
+  void register_metrics();
 };
 
 std::unique_ptr<SeaStore> make_seastore(