rbd-mirror: create and export replication perf counters to mgr

Fixes: https://tracker.ceph.com/issues/21392
Signed-off-by: Mykola Golub <mgolub@suse.com>

diff --git a/src/common/options.cc b/src/common/options.cc
index e92ff6229e948f63eadae6f148bea52803852d78..bf2c15ded6f94bde417562d769053d4af695f7d8 100644 (file)
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -7089,6 +7089,15 @@ static std::vector<Option> get_rbd_mirror_options() {
     Option("rbd_mirror_image_policy_rebalance_timeout", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
     .set_default(0)
     .set_description("number of seconds policy should be idle before trigerring reshuffle (rebalance) of images"),
+
+    Option("rbd_mirror_perf_stats_prio", Option::TYPE_INT, Option::LEVEL_ADVANCED)
+    .set_default((int64_t)PerfCountersBuilder::PRIO_USEFUL)
+    .set_description("Priority level for mirror daemon replication perf counters")
+    .set_long_description("The daemon will send perf counter data to the "
+                          "manager daemon if the priority is not lower than "
+                          "mgr_stats_threshold.")
+    .set_min_max((int64_t)PerfCountersBuilder::PRIO_DEBUGONLY,
+                 (int64_t)PerfCountersBuilder::PRIO_CRITICAL + 1),
   });
 }
 

diff --git a/src/test/rbd_mirror/test_mock_ImageReplayer.cc b/src/test/rbd_mirror/test_mock_ImageReplayer.cc
index c7439d4bc6580322022a26c39c4f6440fe1ba618..e68455360e737b10a5724a42abd61e51b457bf85 100644 (file)
--- a/src/test/rbd_mirror/test_mock_ImageReplayer.cc
+++ b/src/test/rbd_mirror/test_mock_ImageReplayer.cc
@@ -533,7 +533,9 @@ public:
       .WillRepeatedly(Return(0));
   }
 
-  void expect_committed(::journal::MockJournaler &mock_journaler, int times) {
+  void expect_committed(::journal::MockReplayEntry &mock_replay_entry,
+                        ::journal::MockJournaler &mock_journaler, int times) {
+    EXPECT_CALL(mock_replay_entry, get_data()).Times(times);
     EXPECT_CALL(mock_journaler, committed(
                   MatcherCast<const ::journal::MockReplayEntryProxy&>(_)))
       .Times(times);
@@ -1033,7 +1035,7 @@ TEST_F(TestMockImageReplayer, Replay) {
   expect_get_or_send_update(mock_replay_status_formatter);
   expect_get_commit_tid_in_debug(mock_replay_entry);
   expect_get_tag_tid_in_debug(mock_local_journal);
-  expect_committed(mock_remote_journaler, 2);
+  expect_committed(mock_replay_entry, mock_remote_journaler, 2);
 
   InSequence seq;
   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,
@@ -1243,7 +1245,7 @@ TEST_F(TestMockImageReplayer, DelayedReplay) {
   expect_get_or_send_update(mock_replay_status_formatter);
   expect_get_commit_tid_in_debug(mock_replay_entry);
   expect_get_tag_tid_in_debug(mock_local_journal);
-  expect_committed(mock_remote_journaler, 1);
+  expect_committed(mock_replay_entry, mock_remote_journaler, 1);
 
   InSequence seq;
   expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id,

diff --git a/src/tools/rbd_mirror/ImageReplayer.cc b/src/tools/rbd_mirror/ImageReplayer.cc
index 3fd9b5803527c03af469599560d4b26b7bd543c2..c3538d8d02b3db4507cae36ee5617fb649d6330f 100644 (file)
--- a/src/tools/rbd_mirror/ImageReplayer.cc
+++ b/src/tools/rbd_mirror/ImageReplayer.cc
@@ -1084,7 +1084,7 @@ void ImageReplayer<I>::allocate_local_tag() {
          local_tag_data.predecessor.mirror_uuid ==
            librbd::Journal<>::LOCAL_MIRROR_UUID)) {
       dout(15) << "skipping stale demotion event" << dendl;
-      handle_process_entry_safe(m_replay_entry, 0);
+      handle_process_entry_safe(m_replay_entry, m_replay_start_time, 0);
       handle_replay_ready();
       return;
     } else {
@@ -1166,6 +1166,7 @@ void ImageReplayer<I>::handle_preprocess_entry_ready(int r) {
   dout(20) << "r=" << r << dendl;
   ceph_assert(r == 0);
 
+  m_replay_start_time = ceph_clock_now();
   if (!m_event_preprocessor->is_required(m_event_entry)) {
     process_entry();
     return;
@@ -1208,7 +1209,8 @@ void ImageReplayer<I>::process_entry() {
 
   Context *on_ready = create_context_callback<
     ImageReplayer, &ImageReplayer<I>::handle_process_entry_ready>(this);
-  Context *on_commit = new C_ReplayCommitted(this, std::move(m_replay_entry));
+  Context *on_commit = new C_ReplayCommitted(this, std::move(m_replay_entry),
+                                             m_replay_start_time);
 
   m_local_replay->process(m_event_entry, on_ready, on_commit);
 }
@@ -1236,7 +1238,8 @@ void ImageReplayer<I>::handle_process_entry_ready(int r) {
 }
 
 template <typename I>
-void ImageReplayer<I>::handle_process_entry_safe(const ReplayEntry& replay_entry,
+void ImageReplayer<I>::handle_process_entry_safe(const ReplayEntry &replay_entry,
+                                                 const utime_t &replay_start_time,
                                                  int r) {
   dout(20) << "commit_tid=" << replay_entry.get_commit_tid() << ", r=" << r
           << dendl;
@@ -1248,6 +1251,13 @@ void ImageReplayer<I>::handle_process_entry_safe(const ReplayEntry& replay_entry
     ceph_assert(m_remote_journaler != nullptr);
     m_remote_journaler->committed(replay_entry);
   }
+
+  m_perf_counters->inc(l_rbd_mirror_replay);
+  m_perf_counters->inc(l_rbd_mirror_replay_bytes,
+                       replay_entry.get_data().length());
+  m_perf_counters->tinc(l_rbd_mirror_replay_latency,
+                        ceph_clock_now() - replay_start_time);
+
   m_event_replay_tracker.finish_op();
 }
 
@@ -1785,12 +1795,27 @@ void ImageReplayer<I>::register_admin_socket_hook() {
       return;
     }
 
+    ceph_assert(m_perf_counters == nullptr);
+
     dout(15) << "registered asok hook: " << m_name << dendl;
     asok_hook = new ImageReplayerAdminSocketHook<I>(g_ceph_context, m_name,
                                                     this);
     int r = asok_hook->register_commands();
     if (r == 0) {
       m_asok_hook = asok_hook;
+
+      CephContext *cct = static_cast<CephContext *>(m_local->cct());
+      auto prio = cct->_conf.get_val<int64_t>("rbd_mirror_perf_stats_prio");
+      PerfCountersBuilder plb(g_ceph_context, "rbd_mirror_" + m_name,
+                              l_rbd_mirror_first, l_rbd_mirror_last);
+      plb.add_u64_counter(l_rbd_mirror_replay, "replay", "Replays", "r", prio);
+      plb.add_u64_counter(l_rbd_mirror_replay_bytes, "replay_bytes",
+                          "Replayed data", "rb", prio, unit_t(UNIT_BYTES));
+      plb.add_time_avg(l_rbd_mirror_replay_latency, "replay_latency",
+                       "Replay latency", "rl", prio);
+      m_perf_counters = plb.create_perf_counters();
+      g_ceph_context->get_perfcounters_collection()->add(m_perf_counters);
+
       return;
     }
     derr << "error registering admin socket commands" << dendl;
@@ -1803,11 +1828,17 @@ void ImageReplayer<I>::unregister_admin_socket_hook() {
   dout(15) << dendl;
 
   AdminSocketHook *asok_hook = nullptr;
+  PerfCounters *perf_counters = nullptr;
   {
     Mutex::Locker locker(m_lock);
     std::swap(asok_hook, m_asok_hook);
+    std::swap(perf_counters, m_perf_counters);
   }
   delete asok_hook;
+  if (perf_counters != nullptr) {
+    g_ceph_context->get_perfcounters_collection()->remove(perf_counters);
+    delete perf_counters;
+  }
 }
 
 template <typename I>

diff --git a/src/tools/rbd_mirror/ImageReplayer.h b/src/tools/rbd_mirror/ImageReplayer.h
index 8847b8e8522f8816f9d2f41367fece84c8721d8e..ec9544ef5c7d14ad6e61c9ff14d3add01bc1d1c6 100644 (file)
--- a/src/tools/rbd_mirror/ImageReplayer.h
+++ b/src/tools/rbd_mirror/ImageReplayer.h
@@ -29,6 +29,7 @@
 #include <vector>
 
 class AdminSocketHook;
+class PerfCounters;
 
 namespace journal {
 
@@ -318,6 +319,7 @@ private:
   bool m_manual_stop = false;
 
   AdminSocketHook *m_asok_hook = nullptr;
+  PerfCounters *m_perf_counters = nullptr;
 
   image_replayer::BootstrapRequest<ImageCtxT> *m_bootstrap_request = nullptr;
 
@@ -330,6 +332,7 @@ private:
   librbd::journal::MirrorPeerClientMeta m_client_meta;
 
   ReplayEntry m_replay_entry;
+  utime_t m_replay_start_time;
   bool m_replay_tag_valid = false;
   uint64_t m_replay_tag_tid = 0;
   cls::journal::Tag m_replay_tag;
@@ -349,13 +352,16 @@ private:
   struct C_ReplayCommitted : public Context {
     ImageReplayer *replayer;
     ReplayEntry replay_entry;
+    utime_t replay_start_time;
 
     C_ReplayCommitted(ImageReplayer *replayer,
-                      ReplayEntry &&replay_entry)
-      : replayer(replayer), replay_entry(std::move(replay_entry)) {
+                      ReplayEntry &&replay_entry,
+                      const utime_t &replay_start_time)
+      : replayer(replayer), replay_entry(std::move(replay_entry)),
+        replay_start_time(replay_start_time) {
     }
     void finish(int r) override {
-      replayer->handle_process_entry_safe(replay_entry, r);
+      replayer->handle_process_entry_safe(replay_entry, replay_start_time, r);
     }
   };
 
@@ -414,7 +420,8 @@ private:
 
   void process_entry();
   void handle_process_entry_ready(int r);
-  void handle_process_entry_safe(const ReplayEntry& replay_entry, int r);
+  void handle_process_entry_safe(const ReplayEntry& replay_entry,
+                                 const utime_t &m_replay_start_time, int r);
 
   void register_admin_socket_hook();
   void unregister_admin_socket_hook();

diff --git a/src/tools/rbd_mirror/Types.h b/src/tools/rbd_mirror/Types.h
index 23ec5e291ea509d6fb503bdfb5553e6851855b74..ed3b9d8a450b1963aac791b1bb937088724c47fb 100644 (file)
--- a/src/tools/rbd_mirror/Types.h
+++ b/src/tools/rbd_mirror/Types.h
@@ -16,6 +16,15 @@
 namespace rbd {
 namespace mirror {
 
+// Performance counters
+enum {
+  l_rbd_mirror_first = 27000,
+  l_rbd_mirror_replay,
+  l_rbd_mirror_replay_bytes,
+  l_rbd_mirror_replay_latency,
+  l_rbd_mirror_last,
+};
+
 typedef std::shared_ptr<librados::Rados> RadosRef;
 typedef std::shared_ptr<librados::IoCtx> IoCtxRef;
 typedef std::shared_ptr<librbd::Image> ImageRef;