From e191c7e99c00f7b6ae763c16417a3fa1132dc7d1 Mon Sep 17 00:00:00 2001 From: Jason Dillaman Date: Tue, 7 Jan 2020 19:32:19 -0500 Subject: [PATCH] rbd-mirror: helper abstraction layer for image sync points The initial implementation interfaces with the remote journal to store the sync points and snapshot sequence map in the client meta. Signed-off-by: Jason Dillaman --- src/tools/rbd_mirror/CMakeLists.txt | 1 + .../rbd_mirror/image_replayer/StateBuilder.cc | 13 +++ .../rbd_mirror/image_replayer/StateBuilder.h | 9 +- .../image_replayer/journal/StateBuilder.cc | 9 ++ .../image_replayer/journal/StateBuilder.h | 6 + .../journal/SyncPointHandler.cc | 109 ++++++++++++++++++ .../image_replayer/journal/SyncPointHandler.h | 55 +++++++++ src/tools/rbd_mirror/image_sync/Types.h | 74 ++++++++++++ 8 files changed, 275 insertions(+), 1 deletion(-) create mode 100644 src/tools/rbd_mirror/image_replayer/journal/SyncPointHandler.cc create mode 100644 src/tools/rbd_mirror/image_replayer/journal/SyncPointHandler.h create mode 100644 src/tools/rbd_mirror/image_sync/Types.h diff --git a/src/tools/rbd_mirror/CMakeLists.txt b/src/tools/rbd_mirror/CMakeLists.txt index b920d1d8d80..60dff58144a 100644 --- a/src/tools/rbd_mirror/CMakeLists.txt +++ b/src/tools/rbd_mirror/CMakeLists.txt @@ -48,6 +48,7 @@ set(rbd_mirror_internal image_replayer/journal/Replayer.cc image_replayer/journal/ReplayStatusFormatter.cc image_replayer/journal/StateBuilder.cc + image_replayer/journal/SyncPointHandler.cc image_sync/SyncPointCreateRequest.cc image_sync/SyncPointPruneRequest.cc pool_watcher/RefreshImagesRequest.cc diff --git a/src/tools/rbd_mirror/image_replayer/StateBuilder.cc b/src/tools/rbd_mirror/image_replayer/StateBuilder.cc index b740e0bd4d0..7c45648b87b 100644 --- a/src/tools/rbd_mirror/image_replayer/StateBuilder.cc +++ b/src/tools/rbd_mirror/image_replayer/StateBuilder.cc @@ -9,6 +9,7 @@ #include "journal/Journaler.h" #include "librbd/ImageCtx.h" #include "tools/rbd_mirror/image_replayer/CloseImageRequest.h" +#include "tools/rbd_mirror/image_sync/Types.h" #define dout_context g_ceph_context #define dout_subsys ceph_subsys_rbd_mirror @@ -30,6 +31,7 @@ StateBuilder::StateBuilder(const std::string& global_image_id) template StateBuilder::~StateBuilder() { ceph_assert(local_image_ctx == nullptr); + ceph_assert(m_sync_point_handler == nullptr); } template @@ -61,6 +63,17 @@ void StateBuilder::handle_close_local_image(int r, Context* on_finish) { on_finish->complete(r); } +template +void StateBuilder::destroy_sync_point_handler() { + if (m_sync_point_handler == nullptr) { + return; + } + + dout(15) << dendl; + m_sync_point_handler->destroy(); + m_sync_point_handler = nullptr; +} + } // namespace image_replayer } // namespace mirror } // namespace rbd diff --git a/src/tools/rbd_mirror/image_replayer/StateBuilder.h b/src/tools/rbd_mirror/image_replayer/StateBuilder.h index 81f26d4c563..b506fc24e75 100644 --- a/src/tools/rbd_mirror/image_replayer/StateBuilder.h +++ b/src/tools/rbd_mirror/image_replayer/StateBuilder.h @@ -11,6 +11,9 @@ namespace librbd { struct ImageCtx; } namespace rbd { namespace mirror { + +namespace image_sync { struct SyncPointHandler; } + namespace image_replayer { template @@ -34,6 +37,9 @@ public: virtual cls::rbd::MirrorImageMode get_mirror_image_mode() const = 0; + virtual image_sync::SyncPointHandler* create_sync_point_handler() = 0; + void destroy_sync_point_handler(); + std::string global_image_id; std::string local_image_id; @@ -43,12 +49,13 @@ public: std::string remote_image_id; protected: + image_sync::SyncPointHandler* m_sync_point_handler = nullptr; + StateBuilder(const std::string& global_image_id); void close_local_image(Context* on_finish); private: - void handle_close_local_image(int r, Context* on_finish); }; diff --git a/src/tools/rbd_mirror/image_replayer/journal/StateBuilder.cc b/src/tools/rbd_mirror/image_replayer/journal/StateBuilder.cc index db8b7ab5f8f..edab1a3cabc 100644 --- a/src/tools/rbd_mirror/image_replayer/journal/StateBuilder.cc +++ b/src/tools/rbd_mirror/image_replayer/journal/StateBuilder.cc @@ -9,6 +9,7 @@ #include "journal/Journaler.h" #include "librbd/ImageCtx.h" #include "librbd/Journal.h" +#include "tools/rbd_mirror/image_replayer/journal/SyncPointHandler.h" #define dout_context g_ceph_context #define dout_subsys ceph_subsys_rbd_mirror @@ -66,6 +67,14 @@ cls::rbd::MirrorImageMode StateBuilder::get_mirror_image_mode() const { return cls::rbd::MIRROR_IMAGE_MODE_JOURNAL; } +template +image_sync::SyncPointHandler* StateBuilder::create_sync_point_handler() { + dout(10) << dendl; + + this->m_sync_point_handler = SyncPointHandler::create(this); + return this->m_sync_point_handler; +} + template void StateBuilder::shut_down_remote_journaler(Context* on_finish) { if (remote_journaler == nullptr) { diff --git a/src/tools/rbd_mirror/image_replayer/journal/StateBuilder.h b/src/tools/rbd_mirror/image_replayer/journal/StateBuilder.h index 751c09bc1c8..5d9edba47ce 100644 --- a/src/tools/rbd_mirror/image_replayer/journal/StateBuilder.h +++ b/src/tools/rbd_mirror/image_replayer/journal/StateBuilder.h @@ -19,6 +19,8 @@ namespace mirror { namespace image_replayer { namespace journal { +template class SyncPointHandler; + template class StateBuilder : public image_replayer::StateBuilder { public: @@ -41,6 +43,8 @@ public: cls::rbd::MirrorImageMode get_mirror_image_mode() const override; + image_sync::SyncPointHandler* create_sync_point_handler() override; + std::string local_tag_owner; Journaler* remote_journaler = nullptr; @@ -48,6 +52,8 @@ public: cls::journal::CLIENT_STATE_CONNECTED; librbd::journal::MirrorPeerClientMeta remote_client_meta; + SyncPointHandler* sync_point_handler = nullptr; + private: void shut_down_remote_journaler(Context* on_finish); diff --git a/src/tools/rbd_mirror/image_replayer/journal/SyncPointHandler.cc b/src/tools/rbd_mirror/image_replayer/journal/SyncPointHandler.cc new file mode 100644 index 00000000000..66d13e555fe --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/journal/SyncPointHandler.cc @@ -0,0 +1,109 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "SyncPointHandler.h" +#include "StateBuilder.h" +#include "include/ceph_assert.h" +#include "include/Context.h" +#include "common/debug.h" +#include "common/errno.h" +#include "journal/Journaler.h" +#include "librbd/ImageCtx.h" + +#define dout_context g_ceph_context +#define dout_subsys ceph_subsys_rbd_mirror +#undef dout_prefix +#define dout_prefix *_dout << "rbd::mirror::image_replayer::journal::" \ + << "SyncPointHandler: " << this << " " \ + << __func__ << ": " + +namespace rbd { +namespace mirror { +namespace image_replayer { +namespace journal { + +template +SyncPointHandler::SyncPointHandler(StateBuilder* state_builder) + : m_state_builder(state_builder), + m_client_meta_copy(state_builder->remote_client_meta) { +} + +template +typename SyncPointHandler::SyncPoints +SyncPointHandler::get_sync_points() const { + SyncPoints sync_points; + for (auto& sync_point : m_client_meta_copy.sync_points) { + sync_points.emplace_back( + sync_point.snap_namespace, + sync_point.snap_name, + sync_point.from_snap_name, + sync_point.object_number); + } + return sync_points; +} + +template +librbd::SnapSeqs SyncPointHandler::get_snap_seqs() const { + return m_client_meta_copy.snap_seqs; +} + +template +void SyncPointHandler::update_sync_points( + const librbd::SnapSeqs& snap_seqs, const SyncPoints& sync_points, + bool sync_complete, Context* on_finish) { + dout(10) << dendl; + + if (sync_complete && sync_points.empty()) { + m_client_meta_copy.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + } + + m_client_meta_copy.snap_seqs = snap_seqs; + m_client_meta_copy.sync_points.clear(); + for (auto& sync_point : sync_points) { + m_client_meta_copy.sync_points.emplace_back( + sync_point.snap_namespace, + sync_point.snap_name, + sync_point.from_snap_name, + sync_point.object_number); + + if (sync_point.object_number) { + m_client_meta_copy.sync_object_count = std::max( + m_client_meta_copy.sync_object_count, *sync_point.object_number + 1); + } + } + + dout(20) << "client_meta=" << m_client_meta_copy << dendl; + bufferlist client_data_bl; + librbd::journal::ClientData client_data{m_client_meta_copy}; + encode(client_data, client_data_bl); + + auto ctx = new LambdaContext([this, on_finish](int r) { + handle_update_sync_points(r, on_finish); + }); + m_state_builder->remote_journaler->update_client(client_data_bl, ctx); +} + +template +void SyncPointHandler::handle_update_sync_points(int r, Context* on_finish) { + dout(10) << "r=" << r << dendl; + + if (r >= 0) { + m_state_builder->remote_client_meta.snap_seqs = + m_client_meta_copy.snap_seqs; + m_state_builder->remote_client_meta.sync_points = + m_client_meta_copy.sync_points; + } else { + derr << "failed to update remote journal client meta for image " + << m_state_builder->global_image_id << ": " << cpp_strerror(r) + << dendl; + } + + on_finish->complete(r); +} + +} // namespace journal +} // namespace image_sync +} // namespace mirror +} // namespace rbd + +template class rbd::mirror::image_replayer::journal::SyncPointHandler; diff --git a/src/tools/rbd_mirror/image_replayer/journal/SyncPointHandler.h b/src/tools/rbd_mirror/image_replayer/journal/SyncPointHandler.h new file mode 100644 index 00000000000..b4f492c19e4 --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/journal/SyncPointHandler.h @@ -0,0 +1,55 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef RBD_MIRROR_IMAGE_REPLAYER_JOURNAL_SYNC_POINT_HANDLER_H +#define RBD_MIRROR_IMAGE_REPLAYER_JOURNAL_SYNC_POINT_HANDLER_H + +#include "tools/rbd_mirror/image_sync/Types.h" +#include "librbd/journal/Types.h" + +struct Context; +namespace librbd { struct ImageCtx; } + +namespace rbd { +namespace mirror { +namespace image_replayer { +namespace journal { + +template class StateBuilder; + +template +class SyncPointHandler : public image_sync::SyncPointHandler { +public: + using SyncPoint = image_sync::SyncPoint; + using SyncPoints = image_sync::SyncPoints; + + static SyncPointHandler* create(StateBuilder* state_builder) { + return new SyncPointHandler(state_builder); + } + SyncPointHandler(StateBuilder* state_builder); + + SyncPoints get_sync_points() const override; + librbd::SnapSeqs get_snap_seqs() const override; + + void update_sync_points(const librbd::SnapSeqs& snap_seqs, + const SyncPoints& sync_points, + bool sync_complete, + Context* on_finish) override; + +private: + StateBuilder* m_state_builder; + + librbd::journal::MirrorPeerClientMeta m_client_meta_copy; + + void handle_update_sync_points(int r, Context* on_finish); + +}; + +} // namespace journal +} // namespace image_sync +} // namespace mirror +} // namespace rbd + +extern template class rbd::mirror::image_replayer::journal::SyncPointHandler; + +#endif // RBD_MIRROR_IMAGE_REPLAYER_JOURNAL_SYNC_POINT_HANDLER_H diff --git a/src/tools/rbd_mirror/image_sync/Types.h b/src/tools/rbd_mirror/image_sync/Types.h new file mode 100644 index 00000000000..d748dc93e8f --- /dev/null +++ b/src/tools/rbd_mirror/image_sync/Types.h @@ -0,0 +1,74 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef RBD_MIRROR_IMAGE_SYNC_TYPES_H +#define RBD_MIRROR_IMAGE_SYNC_TYPES_H + +#include "cls/rbd/cls_rbd_types.h" +#include "librbd/Types.h" +#include +#include +#include + +struct Context; + +namespace rbd { +namespace mirror { +namespace image_sync { + +struct SyncPoint { + typedef boost::optional ObjectNumber; + + SyncPoint() { + } + SyncPoint(const cls::rbd::SnapshotNamespace& snap_namespace, + const std::string& snap_name, + const std::string& from_snap_name, + const ObjectNumber& object_number) + : snap_namespace(snap_namespace), snap_name(snap_name), + from_snap_name(from_snap_name), object_number(object_number) { + } + + cls::rbd::SnapshotNamespace snap_namespace = + {cls::rbd::UserSnapshotNamespace{}}; + std::string snap_name; + std::string from_snap_name; + ObjectNumber object_number = boost::none; + + bool operator==(const SyncPoint& rhs) const { + return (snap_namespace == rhs.snap_namespace && + snap_name == rhs.snap_name && + from_snap_name == rhs.from_snap_name && + object_number == rhs.object_number); + } +}; + +typedef std::list SyncPoints; + +struct SyncPointHandler { +public: + SyncPointHandler(const SyncPointHandler&) = delete; + SyncPointHandler& operator=(const SyncPointHandler&) = delete; + + virtual ~SyncPointHandler() {} + virtual void destroy() { + delete this; + } + + virtual SyncPoints get_sync_points() const = 0; + virtual librbd::SnapSeqs get_snap_seqs() const = 0; + + virtual void update_sync_points(const librbd::SnapSeqs& snap_seq, + const SyncPoints& sync_points, + bool sync_complete, + Context* on_finish) = 0; + +protected: + SyncPointHandler() {} +}; + +} // namespace image_sync +} // namespace mirror +} // namespace rbd + +#endif // RBD_MIRROR_IMAGE_SYNC_TYPES_H -- 2.39.5