From: Jason Dillaman <dillaman@redhat.com>
Date: Fri, 3 Apr 2020 17:11:27 +0000 (-0400)
Subject: rbd-mirror: add snapshot-based replay performance metrics
X-Git-Tag: wip-pdonnell-testing-20200918.022351~1584^2~1
X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=03aa081435abf12429fbe98b7ff1095a4b37d830;p=ceph-ci.git

rbd-mirror: add snapshot-based replay performance metrics

The mirror image status for replaying snapshot-based images now includes
bytes per second and per snapshot, in addition to an estimated number of
seconds until the image is fully synced.

Signed-off-by: Jason Dillaman <dillaman@redhat.com>
---

diff --git a/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.cc b/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.cc
index b92d8ae58fc..9844f27dd32 100644
--- a/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.cc
+++ b/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.cc
@@ -45,6 +45,10 @@ namespace snapshot {
 
 namespace {
 
+double round_to_two_places(double value) {
+  return abs(round(value * 100) / 100);
+}
+
 template<typename I>
 std::pair<uint64_t, librbd::SnapInfo*> get_newest_mirror_snapshot(
     I* image_ctx) {
@@ -105,6 +109,7 @@ struct Replayer<I>::DeepCopyHandler : public librbd::deep_copy::Handler {
   }
 
   void handle_read(uint64_t bytes_read) override {
+    replayer->handle_copy_image_read(bytes_read);
   }
 
   int update_progress(uint64_t object_number, uint64_t object_count) override {
@@ -272,6 +277,26 @@ bool Replayer<I>::get_replay_status(std::string* description,
         static_cast<float>(std::max<uint64_t>(1U, m_local_object_count)));
   }
 
+  m_bytes_per_second(0);
+  auto bytes_per_second = m_bytes_per_second.get_average();
+  root_obj["bytes_per_second"] = round_to_two_places(bytes_per_second);
+
+  auto bytes_per_snapshot = boost::accumulators::rolling_mean(
+    m_bytes_per_snapshot);
+  root_obj["bytes_per_snapshot"] = round_to_two_places(bytes_per_snapshot);
+
+  auto pending_bytes = bytes_per_snapshot * m_pending_snapshots;
+  if (bytes_per_second > 0 && m_pending_snapshots > 0) {
+    auto seconds_until_synced = round_to_two_places(
+      pending_bytes / bytes_per_second);
+    if (seconds_until_synced >= std::numeric_limits<uint64_t>::max()) {
+      seconds_until_synced = std::numeric_limits<uint64_t>::max();
+    }
+
+    root_obj["seconds_until_synced"] = static_cast<uint64_t>(
+      seconds_until_synced);
+  }
+
   *description = json_spirit::write(
     root_obj, json_spirit::remove_trailing_zeros);
 
@@ -474,6 +499,8 @@ void Replayer<I>::scan_remote_mirror_snapshots(
     std::unique_lock<ceph::mutex>* locker) {
   dout(10) << dendl;
 
+  m_pending_snapshots = 0;
+
   bool split_brain = false;
   bool remote_demoted = false;
   auto remote_image_ctx = m_state_builder->remote_image_ctx;
@@ -559,9 +586,14 @@ void Replayer<I>::scan_remote_mirror_snapshots(
       continue;
     }
 
+    // found candidate snapshot to sync
+    ++m_pending_snapshots;
+    if (m_remote_snap_id_end != CEPH_NOSNAP) {
+      continue;
+    }
+
     m_remote_snap_id_end = remote_snap_id;
     m_remote_mirror_snap_ns = *mirror_ns;
-    break;
   }
   image_locker.unlock();
 
@@ -791,6 +823,7 @@ void Replayer<I>::copy_image() {
            << m_local_mirror_snap_ns.last_copied_object_number << ", "
            << "snap_seqs=" << m_local_mirror_snap_ns.snap_seqs << dendl;
 
+  m_snapshot_bytes = 0;
   m_deep_copy_handler = new DeepCopyHandler(this);
   auto ctx = create_context_callback<
     Replayer<I>, &Replayer<I>::handle_copy_image>(this);
@@ -819,6 +852,12 @@ void Replayer<I>::handle_copy_image(int r) {
     return;
   }
 
+  {
+    std::unique_lock locker{m_lock};
+    m_bytes_per_snapshot(m_snapshot_bytes);
+    m_snapshot_bytes = 0;
+  }
+
   apply_image_state();
 }
 
@@ -836,6 +875,15 @@ void Replayer<I>::handle_copy_image_progress(uint64_t object_number,
   update_non_primary_snapshot(false);
 }
 
+template <typename I>
+void Replayer<I>::handle_copy_image_read(uint64_t bytes_read) {
+  dout(20) << "bytes_read=" << bytes_read << dendl;
+
+  std::unique_lock locker{m_lock};
+  m_bytes_per_second(bytes_read);
+  m_snapshot_bytes += bytes_read;
+}
+
 template <typename I>
 void Replayer<I>::apply_image_state() {
   dout(10) << dendl;
diff --git a/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.h b/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.h
index ba49d7cd799..4a96291d614 100644
--- a/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.h
+++ b/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.h
@@ -9,6 +9,10 @@
 #include "common/AsyncOpTracker.h"
 #include "cls/rbd/cls_rbd_types.h"
 #include "librbd/mirror/snapshot/Types.h"
+#include "tools/rbd_mirror/image_replayer/TimeRollingMean.h"
+#include <boost/accumulators/accumulators.hpp>
+#include <boost/accumulators/statistics/stats.hpp>
+#include <boost/accumulators/statistics/rolling_mean.hpp>
 #include <string>
 #include <type_traits>
 
@@ -222,6 +226,16 @@ private:
   librbd::mirror::snapshot::ImageState m_image_state;
   DeepCopyHandler* m_deep_copy_handler = nullptr;
 
+  TimeRollingMean m_bytes_per_second;
+
+  uint64_t m_snapshot_bytes = 0;
+  boost::accumulators::accumulator_set<
+    uint64_t, boost::accumulators::stats<
+      boost::accumulators::tag::rolling_mean>> m_bytes_per_snapshot{
+    boost::accumulators::tag::rolling_window::window_size = 2};
+
+  uint32_t m_pending_snapshots = 0;
+
   bool m_remote_image_updated = false;
   bool m_updating_sync_point = false;
   bool m_sync_in_progress = false;
@@ -257,6 +271,7 @@ private:
   void handle_copy_image(int r);
   void handle_copy_image_progress(uint64_t object_number,
                                   uint64_t object_count);
+  void handle_copy_image_read(uint64_t bytes_read);
 
   void apply_image_state();
   void handle_apply_image_state(int r);