From cac1c10e1e72c43ed1313af251f15bf74daaca6d Mon Sep 17 00:00:00 2001 From: Jason Dillaman Date: Wed, 1 Apr 2020 19:24:15 -0400 Subject: [PATCH] rbd-mirror: add journal-based replay performance metrics The mirror image status for replaying journal-based images now includes bytes and entries per second in addition to an estimated number of seconds until the image is fully synced. Signed-off-by: Jason Dillaman (cherry picked from commit c0dc96bf137dac6942e14800e173bfb9bda233eb) --- .../journal/test_mock_Replayer.cc | 1 + src/tools/rbd_mirror/CMakeLists.txt | 1 + .../image_replayer/TimeRollingMean.cc | 34 ++++++++++++++++ .../image_replayer/TimeRollingMean.h | 40 +++++++++++++++++++ .../journal/ReplayStatusFormatter.cc | 36 ++++++++++++++++- .../journal/ReplayStatusFormatter.h | 6 +++ .../image_replayer/journal/Replayer.cc | 2 + 7 files changed, 118 insertions(+), 2 deletions(-) create mode 100644 src/tools/rbd_mirror/image_replayer/TimeRollingMean.cc create mode 100644 src/tools/rbd_mirror/image_replayer/TimeRollingMean.h diff --git a/src/test/rbd_mirror/image_replayer/journal/test_mock_Replayer.cc b/src/test/rbd_mirror/image_replayer/journal/test_mock_Replayer.cc index de1513f192bd7..7e233b8d6f733 100644 --- a/src/test/rbd_mirror/image_replayer/journal/test_mock_Replayer.cc +++ b/src/test/rbd_mirror/image_replayer/journal/test_mock_Replayer.cc @@ -207,6 +207,7 @@ struct ReplayStatusFormatter { s_instance = nullptr; } + MOCK_METHOD1(handle_entry_processed, void(uint64_t)); MOCK_METHOD2(get_or_send_update, bool(std::string *description, Context *on_finish)); }; diff --git a/src/tools/rbd_mirror/CMakeLists.txt b/src/tools/rbd_mirror/CMakeLists.txt index edf4775fd9683..9c1a71c57d1c5 100644 --- a/src/tools/rbd_mirror/CMakeLists.txt +++ b/src/tools/rbd_mirror/CMakeLists.txt @@ -43,6 +43,7 @@ set(rbd_mirror_internal image_replayer/PrepareLocalImageRequest.cc image_replayer/PrepareRemoteImageRequest.cc image_replayer/StateBuilder.cc + image_replayer/TimeRollingMean.cc image_replayer/Utils.cc image_replayer/journal/CreateLocalImageRequest.cc image_replayer/journal/EventPreprocessor.cc diff --git a/src/tools/rbd_mirror/image_replayer/TimeRollingMean.cc b/src/tools/rbd_mirror/image_replayer/TimeRollingMean.cc new file mode 100644 index 0000000000000..5d9c9aca13a3f --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/TimeRollingMean.cc @@ -0,0 +1,34 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "tools/rbd_mirror/image_replayer/TimeRollingMean.h" +#include "common/Clock.h" + +namespace rbd { +namespace mirror { +namespace image_replayer { + +void TimeRollingMean::operator()(uint32_t value) { + auto time = ceph_clock_now(); + if (m_last_time.is_zero()) { + m_last_time = time; + } else if (m_last_time.sec() < time.sec()) { + auto sec = m_last_time.sec(); + while (sec++ < time.sec()) { + m_rolling_mean(m_sum); + m_sum = 0; + } + + m_last_time = time; + } + + m_sum += value; +} + +double TimeRollingMean::get_average() const { + return boost::accumulators::rolling_mean(m_rolling_mean); +} + +} // namespace image_replayer +} // namespace mirror +} // namespace rbd diff --git a/src/tools/rbd_mirror/image_replayer/TimeRollingMean.h b/src/tools/rbd_mirror/image_replayer/TimeRollingMean.h new file mode 100644 index 0000000000000..139ef893fd90b --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/TimeRollingMean.h @@ -0,0 +1,40 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef RBD_MIRROR_IMAGE_REPLAYER_TIME_ROLLING_MEAN_H +#define RBD_MIRROR_IMAGE_REPLAYER_TIME_ROLLING_MEAN_H + +#include "include/utime.h" +#include +#include +#include + +namespace rbd { +namespace mirror { +namespace image_replayer { + +class TimeRollingMean { +public: + + void operator()(uint32_t value); + + double get_average() const; + +private: + typedef boost::accumulators::accumulator_set< + uint64_t, boost::accumulators::stats< + boost::accumulators::tag::rolling_mean>> RollingMean; + + utime_t m_last_time; + uint64_t m_sum = 0; + + RollingMean m_rolling_mean{ + boost::accumulators::tag::rolling_window::window_size = 30}; + +}; + +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +#endif // RBD_MIRROR_IMAGE_REPLAYER_TIME_ROLLING_MEAN_H diff --git a/src/tools/rbd_mirror/image_replayer/journal/ReplayStatusFormatter.cc b/src/tools/rbd_mirror/image_replayer/journal/ReplayStatusFormatter.cc index 8c01e8a72033c..216a2fa2caf96 100644 --- a/src/tools/rbd_mirror/image_replayer/journal/ReplayStatusFormatter.cc +++ b/src/tools/rbd_mirror/image_replayer/journal/ReplayStatusFormatter.cc @@ -27,6 +27,10 @@ using librbd::util::unique_lock_name; namespace { +double round_to_two_places(double value) { + return abs(round(value * 100) / 100); +} + json_spirit::mObject to_json_object( const cls::journal::ObjectPosition& position) { json_spirit::mObject object; @@ -48,6 +52,14 @@ ReplayStatusFormatter::ReplayStatusFormatter(Journaler *journaler, m_lock(ceph::make_mutex(unique_lock_name("ReplayStatusFormatter::m_lock", this))) { } +template +void ReplayStatusFormatter::handle_entry_processed(uint32_t bytes) { + dout(20) << dendl; + + m_bytes_per_second(bytes); + m_entries_per_second(1); +} + template bool ReplayStatusFormatter::get_or_send_update(std::string *description, Context *on_finish) { @@ -232,7 +244,6 @@ void ReplayStatusFormatter::handle_update_tag_cache(uint64_t master_tag_tid, template void ReplayStatusFormatter::format(std::string *description) { - dout(20) << "m_master_position=" << m_master_position << ", m_mirror_position=" << m_mirror_position << ", m_entries_behind_master=" << m_entries_behind_master << dendl; @@ -242,7 +253,28 @@ void ReplayStatusFormatter::format(std::string *description) { root_obj["non_primary_position"] = to_json_object(m_mirror_position); root_obj["entries_behind_primary"] = ( m_entries_behind_master > 0 ? m_entries_behind_master : 0); - *description = json_spirit::write(root_obj); + + m_bytes_per_second(0); + root_obj["bytes_per_second"] = round_to_two_places( + m_bytes_per_second.get_average()); + + m_entries_per_second(0); + auto entries_per_second = m_entries_per_second.get_average(); + root_obj["entries_per_second"] = round_to_two_places(entries_per_second); + + if (m_entries_behind_master > 0 && entries_per_second > 0) { + auto seconds_until_synced = round_to_two_places( + m_entries_behind_master / entries_per_second); + if (seconds_until_synced >= std::numeric_limits::max()) { + seconds_until_synced = std::numeric_limits::max(); + } + + root_obj["seconds_until_synced"] = static_cast( + seconds_until_synced); + } + + *description = json_spirit::write( + root_obj, json_spirit::remove_trailing_zeros); } } // namespace journal diff --git a/src/tools/rbd_mirror/image_replayer/journal/ReplayStatusFormatter.h b/src/tools/rbd_mirror/image_replayer/journal/ReplayStatusFormatter.h index d5d6317c9c964..5dbbfe10d1760 100644 --- a/src/tools/rbd_mirror/image_replayer/journal/ReplayStatusFormatter.h +++ b/src/tools/rbd_mirror/image_replayer/journal/ReplayStatusFormatter.h @@ -9,6 +9,7 @@ #include "cls/journal/cls_journal_types.h" #include "librbd/journal/Types.h" #include "librbd/journal/TypeTraits.h" +#include "tools/rbd_mirror/image_replayer/TimeRollingMean.h" namespace journal { class Journaler; } namespace librbd { class ImageCtx; } @@ -34,6 +35,8 @@ public: ReplayStatusFormatter(Journaler *journaler, const std::string &mirror_uuid); + void handle_entry_processed(uint32_t bytes); + bool get_or_send_update(std::string *description, Context *on_finish); private: @@ -47,6 +50,9 @@ private: cls::journal::Tag m_tag; std::map m_tag_cache; + TimeRollingMean m_bytes_per_second; + TimeRollingMean m_entries_per_second; + bool calculate_behind_master_or_send_update(); void send_update_tag_cache(uint64_t master_tag_tid, uint64_t mirror_tag_tid); void handle_update_tag_cache(uint64_t master_tag_tid, uint64_t mirror_tag_tid, diff --git a/src/tools/rbd_mirror/image_replayer/journal/Replayer.cc b/src/tools/rbd_mirror/image_replayer/journal/Replayer.cc index 4b6778e145502..cd7a04926109e 100644 --- a/src/tools/rbd_mirror/image_replayer/journal/Replayer.cc +++ b/src/tools/rbd_mirror/image_replayer/journal/Replayer.cc @@ -1071,6 +1071,8 @@ void Replayer::handle_process_entry_ready(int r) { } } + m_replay_status_formatter->handle_entry_processed(m_replay_bytes); + if (update_status) { unregister_perf_counters(); register_perf_counters(); -- 2.39.5