rbd-mirror: add journal-based replay performance metrics

The mirror image status for replaying journal-based images now includes
bytes and entries per second in addition to an estimated number of seconds
until the image is fully synced.

Signed-off-by: Jason Dillaman <dillaman@redhat.com>
(cherry picked from commit c0dc96bf137dac6942e14800e173bfb9bda233eb)

diff --git a/src/test/rbd_mirror/image_replayer/journal/test_mock_Replayer.cc b/src/test/rbd_mirror/image_replayer/journal/test_mock_Replayer.cc
index de1513f192bd75342e49274b90e46bf73c794645..7e233b8d6f7334ef63b18d64a5c7fc730b1b812f 100644 (file)
--- a/src/test/rbd_mirror/image_replayer/journal/test_mock_Replayer.cc
+++ b/src/test/rbd_mirror/image_replayer/journal/test_mock_Replayer.cc
@@ -207,6 +207,7 @@ struct ReplayStatusFormatter<librbd::MockTestImageCtx> {
     s_instance = nullptr;
   }
 
+  MOCK_METHOD1(handle_entry_processed, void(uint64_t));
   MOCK_METHOD2(get_or_send_update, bool(std::string *description, Context *on_finish));
 };
 

diff --git a/src/tools/rbd_mirror/CMakeLists.txt b/src/tools/rbd_mirror/CMakeLists.txt
index edf4775fd96832c4d80b20298c94cf8dc800f2ca..9c1a71c57d1c56a55ee18ec9d9e60a3da80fc3a8 100644 (file)
--- a/src/tools/rbd_mirror/CMakeLists.txt
+++ b/src/tools/rbd_mirror/CMakeLists.txt
@@ -43,6 +43,7 @@ set(rbd_mirror_internal
   image_replayer/PrepareLocalImageRequest.cc
   image_replayer/PrepareRemoteImageRequest.cc
   image_replayer/StateBuilder.cc
+  image_replayer/TimeRollingMean.cc
   image_replayer/Utils.cc
   image_replayer/journal/CreateLocalImageRequest.cc
   image_replayer/journal/EventPreprocessor.cc

diff --git a/src/tools/rbd_mirror/image_replayer/TimeRollingMean.cc b/src/tools/rbd_mirror/image_replayer/TimeRollingMean.cc
new file mode 100644 (file)
index 0000000..5d9c9ac
--- /dev/null
+++ b/src/tools/rbd_mirror/image_replayer/TimeRollingMean.cc
@@ -0,0 +1,34 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "tools/rbd_mirror/image_replayer/TimeRollingMean.h"
+#include "common/Clock.h"
+
+namespace rbd {
+namespace mirror {
+namespace image_replayer {
+
+void TimeRollingMean::operator()(uint32_t value) {
+  auto time = ceph_clock_now();
+  if (m_last_time.is_zero()) {
+    m_last_time = time;
+  } else if (m_last_time.sec() < time.sec()) {
+    auto sec = m_last_time.sec();
+    while (sec++ < time.sec()) {
+      m_rolling_mean(m_sum);
+      m_sum = 0;
+    }
+
+    m_last_time = time;
+  }
+
+  m_sum += value;
+}
+
+double TimeRollingMean::get_average() const {
+  return boost::accumulators::rolling_mean(m_rolling_mean);
+}
+
+} // namespace image_replayer
+} // namespace mirror
+} // namespace rbd

diff --git a/src/tools/rbd_mirror/image_replayer/TimeRollingMean.h b/src/tools/rbd_mirror/image_replayer/TimeRollingMean.h
new file mode 100644 (file)
index 0000000..139ef89
--- /dev/null
+++ b/src/tools/rbd_mirror/image_replayer/TimeRollingMean.h
@@ -0,0 +1,40 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef RBD_MIRROR_IMAGE_REPLAYER_TIME_ROLLING_MEAN_H
+#define RBD_MIRROR_IMAGE_REPLAYER_TIME_ROLLING_MEAN_H
+
+#include "include/utime.h"
+#include <boost/accumulators/accumulators.hpp>
+#include <boost/accumulators/statistics/stats.hpp>
+#include <boost/accumulators/statistics/rolling_mean.hpp>
+
+namespace rbd {
+namespace mirror {
+namespace image_replayer {
+
+class TimeRollingMean {
+public:
+
+  void operator()(uint32_t value);
+
+  double get_average() const;
+
+private:
+  typedef boost::accumulators::accumulator_set<
+    uint64_t, boost::accumulators::stats<
+      boost::accumulators::tag::rolling_mean>> RollingMean;
+
+  utime_t m_last_time;
+  uint64_t m_sum = 0;
+
+  RollingMean m_rolling_mean{
+    boost::accumulators::tag::rolling_window::window_size = 30};
+
+};
+
+} // namespace image_replayer
+} // namespace mirror
+} // namespace rbd
+
+#endif // RBD_MIRROR_IMAGE_REPLAYER_TIME_ROLLING_MEAN_H

diff --git a/src/tools/rbd_mirror/image_replayer/journal/ReplayStatusFormatter.cc b/src/tools/rbd_mirror/image_replayer/journal/ReplayStatusFormatter.cc
index 8c01e8a72033c285d14eb3f6e254d969192f358a..216a2fa2caf96b0e9511bd5bdfa49dd284352b3e 100644 (file)
--- a/src/tools/rbd_mirror/image_replayer/journal/ReplayStatusFormatter.cc
+++ b/src/tools/rbd_mirror/image_replayer/journal/ReplayStatusFormatter.cc
@@ -27,6 +27,10 @@ using librbd::util::unique_lock_name;
 
 namespace {
 
+double round_to_two_places(double value) {
+  return abs(round(value * 100) / 100);
+}
+
 json_spirit::mObject to_json_object(
     const cls::journal::ObjectPosition& position) {
   json_spirit::mObject object;
@@ -48,6 +52,14 @@ ReplayStatusFormatter<I>::ReplayStatusFormatter(Journaler *journaler,
     m_lock(ceph::make_mutex(unique_lock_name("ReplayStatusFormatter::m_lock", this))) {
 }
 
+template <typename I>
+void ReplayStatusFormatter<I>::handle_entry_processed(uint32_t bytes) {
+  dout(20) << dendl;
+
+  m_bytes_per_second(bytes);
+  m_entries_per_second(1);
+}
+
 template <typename I>
 bool ReplayStatusFormatter<I>::get_or_send_update(std::string *description,
                                                  Context *on_finish) {
@@ -232,7 +244,6 @@ void ReplayStatusFormatter<I>::handle_update_tag_cache(uint64_t master_tag_tid,
 
 template <typename I>
 void ReplayStatusFormatter<I>::format(std::string *description) {
-
   dout(20) << "m_master_position=" << m_master_position
           << ", m_mirror_position=" << m_mirror_position
           << ", m_entries_behind_master=" << m_entries_behind_master << dendl;
@@ -242,7 +253,28 @@ void ReplayStatusFormatter<I>::format(std::string *description) {
   root_obj["non_primary_position"] = to_json_object(m_mirror_position);
   root_obj["entries_behind_primary"] = (
     m_entries_behind_master > 0 ? m_entries_behind_master : 0);
-  *description = json_spirit::write(root_obj);
+
+  m_bytes_per_second(0);
+  root_obj["bytes_per_second"] = round_to_two_places(
+    m_bytes_per_second.get_average());
+
+  m_entries_per_second(0);
+  auto entries_per_second = m_entries_per_second.get_average();
+  root_obj["entries_per_second"] = round_to_two_places(entries_per_second);
+
+  if (m_entries_behind_master > 0 && entries_per_second > 0) {
+    auto seconds_until_synced = round_to_two_places(
+      m_entries_behind_master / entries_per_second);
+    if (seconds_until_synced >= std::numeric_limits<uint64_t>::max()) {
+      seconds_until_synced = std::numeric_limits<uint64_t>::max();
+    }
+
+    root_obj["seconds_until_synced"] = static_cast<uint64_t>(
+      seconds_until_synced);
+  }
+
+  *description = json_spirit::write(
+    root_obj, json_spirit::remove_trailing_zeros);
 }
 
 } // namespace journal

diff --git a/src/tools/rbd_mirror/image_replayer/journal/ReplayStatusFormatter.h b/src/tools/rbd_mirror/image_replayer/journal/ReplayStatusFormatter.h
index d5d6317c9c96441f2780823595dd9f1f455bc8ff..5dbbfe10d17607a4ac010251dc303cd5e18aaba9 100644 (file)
--- a/src/tools/rbd_mirror/image_replayer/journal/ReplayStatusFormatter.h
+++ b/src/tools/rbd_mirror/image_replayer/journal/ReplayStatusFormatter.h
@@ -9,6 +9,7 @@
 #include "cls/journal/cls_journal_types.h"
 #include "librbd/journal/Types.h"
 #include "librbd/journal/TypeTraits.h"
+#include "tools/rbd_mirror/image_replayer/TimeRollingMean.h"
 
 namespace journal { class Journaler; }
 namespace librbd { class ImageCtx; }
@@ -34,6 +35,8 @@ public:
 
   ReplayStatusFormatter(Journaler *journaler, const std::string &mirror_uuid);
 
+  void handle_entry_processed(uint32_t bytes);
+
   bool get_or_send_update(std::string *description, Context *on_finish);
 
 private:
@@ -47,6 +50,9 @@ private:
   cls::journal::Tag m_tag;
   std::map<uint64_t, librbd::journal::TagData> m_tag_cache;
 
+  TimeRollingMean m_bytes_per_second;
+  TimeRollingMean m_entries_per_second;
+
   bool calculate_behind_master_or_send_update();
   void send_update_tag_cache(uint64_t master_tag_tid, uint64_t mirror_tag_tid);
   void handle_update_tag_cache(uint64_t master_tag_tid, uint64_t mirror_tag_tid,

diff --git a/src/tools/rbd_mirror/image_replayer/journal/Replayer.cc b/src/tools/rbd_mirror/image_replayer/journal/Replayer.cc
index 4b6778e145502c0bcb5329918a33b5142d168e25..cd7a04926109e4a2b212185d760eaf01e12e649d 100644 (file)
--- a/src/tools/rbd_mirror/image_replayer/journal/Replayer.cc
+++ b/src/tools/rbd_mirror/image_replayer/journal/Replayer.cc
@@ -1071,6 +1071,8 @@ void Replayer<I>::handle_process_entry_ready(int r) {
     }
   }
 
+  m_replay_status_formatter->handle_entry_processed(m_replay_bytes);
+
   if (update_status) {
     unregister_perf_counters();
     register_perf_counters();