From c58de9e1487bf1f4fb4ff8477f30233e2f083d57 Mon Sep 17 00:00:00 2001 From: Mykola Golub Date: Sun, 17 Jan 2021 10:36:02 +0000 Subject: [PATCH] rbd-mirror: initial group replayer implementation Signed-off-by: Mykola Golub Signed-off-by: Prasanna Kumar Kalever --- .../test_mock_CreateLocalImageRequest.cc | 3 +- .../test_mock_CreateLocalImageRequest.cc | 3 +- .../test_mock_BootstrapRequest.cc | 9 +- .../test_mock_CreateImageRequest.cc | 2 +- src/test/rbd_mirror/test_ImageReplayer.cc | 9 +- src/test/rbd_mirror/test_ImageSync.cc | 7 +- .../rbd_mirror/test_mock_ImageReplayer.cc | 21 +- src/test/rbd_mirror/test_mock_ImageSync.cc | 2 +- .../rbd_mirror/test_mock_InstanceReplayer.cc | 5 +- src/tools/rbd_mirror/CMakeLists.txt | 1 + src/tools/rbd_mirror/GroupReplayer.cc | 558 ++++++- src/tools/rbd_mirror/GroupReplayer.h | 79 +- src/tools/rbd_mirror/ImageMap.cc | 20 +- src/tools/rbd_mirror/ImageReplayer.cc | 89 +- src/tools/rbd_mirror/ImageReplayer.h | 49 +- src/tools/rbd_mirror/ImageSync.cc | 14 +- src/tools/rbd_mirror/ImageSync.h | 11 +- src/tools/rbd_mirror/InstanceReplayer.cc | 22 +- src/tools/rbd_mirror/MirrorStatusUpdater.cc | 173 +- src/tools/rbd_mirror/MirrorStatusUpdater.h | 34 +- src/tools/rbd_mirror/NamespaceReplayer.cc | 1 - src/tools/rbd_mirror/PoolMetaCache.cc | 14 + src/tools/rbd_mirror/PoolMetaCache.h | 2 + src/tools/rbd_mirror/Types.cc | 14 +- src/tools/rbd_mirror/Types.h | 22 + .../group_replayer/BootstrapRequest.cc | 1406 +++++++++++++++++ .../group_replayer/BootstrapRequest.h | 272 ++++ .../image_replayer/BootstrapRequest.cc | 12 +- .../image_replayer/BootstrapRequest.h | 10 +- .../image_replayer/CreateImageRequest.cc | 9 +- .../image_replayer/CreateImageRequest.h | 14 +- .../rbd_mirror/image_replayer/StateBuilder.h | 2 + .../journal/CreateLocalImageRequest.cc | 2 +- .../journal/CreateLocalImageRequest.h | 12 +- .../image_replayer/journal/StateBuilder.cc | 5 +- .../image_replayer/journal/StateBuilder.h | 1 + .../snapshot/CreateLocalImageRequest.cc | 17 +- .../snapshot/CreateLocalImageRequest.h | 12 +- .../image_replayer/snapshot/StateBuilder.cc | 5 +- .../image_replayer/snapshot/StateBuilder.h | 1 + 40 files changed, 2808 insertions(+), 136 deletions(-) create mode 100644 src/tools/rbd_mirror/group_replayer/BootstrapRequest.cc create mode 100644 src/tools/rbd_mirror/group_replayer/BootstrapRequest.h diff --git a/src/test/rbd_mirror/image_replayer/journal/test_mock_CreateLocalImageRequest.cc b/src/test/rbd_mirror/image_replayer/journal/test_mock_CreateLocalImageRequest.cc index cc22671609d7d..7cf3bbf0cc303 100644 --- a/src/test/rbd_mirror/image_replayer/journal/test_mock_CreateLocalImageRequest.cc +++ b/src/test/rbd_mirror/image_replayer/journal/test_mock_CreateLocalImageRequest.cc @@ -64,6 +64,7 @@ struct CreateImageRequest { static CreateImageRequest* create(Threads* threads, librados::IoCtx &local_io_ctx, + GroupCtx *local_group_ctx, const std::string &global_image_id, const std::string &remote_mirror_uuid, const std::string &local_image_name, @@ -190,7 +191,7 @@ public: const std::string& global_image_id, Context* on_finish) { return new MockCreateLocalImageRequest( - &mock_threads, m_local_io_ctx, m_mock_remote_image_ctx, + &mock_threads, m_local_io_ctx, nullptr, m_mock_remote_image_ctx, global_image_id, nullptr, nullptr, &mock_state_builder, on_finish); } diff --git a/src/test/rbd_mirror/image_replayer/snapshot/test_mock_CreateLocalImageRequest.cc b/src/test/rbd_mirror/image_replayer/snapshot/test_mock_CreateLocalImageRequest.cc index 9a958c966cc49..0d14e3f412d53 100644 --- a/src/test/rbd_mirror/image_replayer/snapshot/test_mock_CreateLocalImageRequest.cc +++ b/src/test/rbd_mirror/image_replayer/snapshot/test_mock_CreateLocalImageRequest.cc @@ -65,6 +65,7 @@ struct CreateImageRequest { static CreateImageRequest* create(Threads* threads, librados::IoCtx &local_io_ctx, + GroupCtx *local_group_ctx, const std::string &global_image_id, const std::string &remote_mirror_uuid, const std::string &local_image_name, @@ -200,7 +201,7 @@ public: const std::string& global_image_id, Context* on_finish) { return new MockCreateLocalImageRequest( - &mock_threads, m_local_io_ctx, m_mock_remote_image_ctx, + &mock_threads, m_local_io_ctx, nullptr, m_mock_remote_image_ctx, global_image_id, &m_pool_meta_cache, nullptr, &mock_state_builder, on_finish); } diff --git a/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc b/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc index 0411370013355..03a3f2cc8dffe 100644 --- a/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc +++ b/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc @@ -54,6 +54,7 @@ struct ImageSync { static ImageSync* create( Threads* threads, + GroupCtx *local_group_ctx, librbd::MockTestImageCtx *local_image_ctx, librbd::MockTestImageCtx *remote_image_ctx, const std::string &local_mirror_uuid, @@ -238,9 +239,10 @@ struct StateBuilder { MOCK_CONST_METHOD0(replay_requires_remote_image, bool()); MOCK_METHOD1(close_remote_image, void(Context*)); - MOCK_METHOD6(create_local_image_request, + MOCK_METHOD7(create_local_image_request, BaseRequest*(Threads*, librados::IoCtx&, + GroupCtx*, const std::string&, PoolMetaCache*, ProgressContext*, @@ -419,8 +421,8 @@ public: void expect_create_local_image(MockStateBuilder& mock_state_builder, const std::string& local_image_id, int r) { EXPECT_CALL(mock_state_builder, - create_local_image_request(_, _, _, _, _, _)) - .WillOnce(WithArg<5>( + create_local_image_request(_, _, _, _, _, _, _)) + .WillOnce(WithArg<6>( Invoke([&mock_state_builder, local_image_id, r](Context* ctx) { if (r >= 0) { mock_state_builder.local_image_id = local_image_id; @@ -473,6 +475,7 @@ public: return new MockBootstrapRequest(mock_threads, m_local_io_ctx, m_remote_io_ctx, + nullptr, mock_instance_watcher, global_image_id, local_mirror_uuid, diff --git a/src/test/rbd_mirror/image_replayer/test_mock_CreateImageRequest.cc b/src/test/rbd_mirror/image_replayer/test_mock_CreateImageRequest.cc index ed2cf8f96e436..be20b4c4f797a 100644 --- a/src/test/rbd_mirror/image_replayer/test_mock_CreateImageRequest.cc +++ b/src/test/rbd_mirror/image_replayer/test_mock_CreateImageRequest.cc @@ -339,7 +339,7 @@ public: const std::string &local_image_id, librbd::MockTestImageCtx &mock_remote_image_ctx, Context *on_finish) { - return new MockCreateImageRequest(mock_threads, m_local_io_ctx, + return new MockCreateImageRequest(mock_threads, m_local_io_ctx, nullptr, global_image_id, remote_mirror_uuid, local_image_name, local_image_id, &mock_remote_image_ctx, diff --git a/src/test/rbd_mirror/test_ImageReplayer.cc b/src/test/rbd_mirror/test_ImageReplayer.cc index d0bbfb094789a..0321ee8142d1d 100644 --- a/src/test/rbd_mirror/test_ImageReplayer.cc +++ b/src/test/rbd_mirror/test_ImageReplayer.cc @@ -210,10 +210,11 @@ public: } void create_replayer() { - m_replayer = new ImageReplayer<>(m_local_ioctx, m_local_mirror_uuid, - m_global_image_id, m_threads.get(), - m_instance_watcher, m_local_status_updater, - nullptr, &m_pool_meta_cache); + m_replayer = new ImageReplayer<>(m_local_ioctx, nullptr, + m_local_mirror_uuid, m_global_image_id, + m_threads.get(), m_instance_watcher, + m_local_status_updater, nullptr, + &m_pool_meta_cache); m_replayer->add_peer({"peer uuid", m_remote_ioctx, {m_remote_mirror_uuid, "remote mirror peer uuid"}, nullptr}); diff --git a/src/test/rbd_mirror/test_ImageSync.cc b/src/test/rbd_mirror/test_ImageSync.cc index 9418496bce2d4..06cc0f65f3cfc 100644 --- a/src/test/rbd_mirror/test_ImageSync.cc +++ b/src/test/rbd_mirror/test_ImageSync.cc @@ -136,9 +136,10 @@ public: } ImageSync<> *create_request(Context *ctx) { - return new ImageSync<>(m_threads, m_local_image_ctx, m_remote_image_ctx, - "mirror-uuid", m_sync_point_handler, - m_instance_watcher, nullptr, ctx); + return new ImageSync<>(m_threads, nullptr, m_local_image_ctx, + m_remote_image_ctx, "mirror-uuid", + m_sync_point_handler, m_instance_watcher, nullptr, + ctx); } librbd::ImageCtx *m_remote_image_ctx; diff --git a/src/test/rbd_mirror/test_mock_ImageReplayer.cc b/src/test/rbd_mirror/test_mock_ImageReplayer.cc index 32a4f82b53b44..0a0f2245477c3 100644 --- a/src/test/rbd_mirror/test_mock_ImageReplayer.cc +++ b/src/test/rbd_mirror/test_mock_ImageReplayer.cc @@ -59,7 +59,7 @@ ImageDeleter* ImageDeleter:: template <> struct MirrorStatusUpdater { - MOCK_METHOD1(exists, bool(const std::string&)); + MOCK_METHOD1(mirror_image_exists, bool(const std::string&)); MOCK_METHOD3(set_mirror_image_status, void(const std::string&, const cls::rbd::MirrorImageSiteStatus&, bool)); @@ -67,6 +67,18 @@ struct MirrorStatusUpdater { Context*)); MOCK_METHOD3(remove_mirror_image_status, void(const std::string&, bool, Context*)); + MOCK_METHOD1(mirror_group_exists, bool(const std::string&)); + MOCK_METHOD3(set_mirror_group_status, + void(const std::string&, const cls::rbd::MirrorGroupSiteStatus&, + bool)); + MOCK_METHOD2(remove_mirror_group_status, void(const std::string&, Context*)); + MOCK_METHOD3(mirror_group_image_exists, + bool(const std::string&, int64_t, const std::string&)); + MOCK_METHOD5(set_mirror_group_image_status, + void(const std::string&, int64_t, const std::string&, + const cls::rbd::MirrorImageSiteStatus&, bool)); + MOCK_METHOD4(remove_mirror_group_image_status, + void(const std::string&, int64_t, const std::string&, Context*)); }; template <> @@ -105,6 +117,7 @@ struct BootstrapRequest { Threads* threads, librados::IoCtx &local_io_ctx, librados::IoCtx& remote_io_ctx, + rbd::mirror::GroupCtx *local_group_ctx, rbd::mirror::InstanceWatcher *instance_watcher, const std::string &global_image_id, const std::string &local_mirror_uuid, @@ -357,15 +370,15 @@ public: } void expect_mirror_image_status_exists(bool exists) { - EXPECT_CALL(m_local_status_updater, exists(_)) + EXPECT_CALL(m_local_status_updater, mirror_image_exists(_)) .WillOnce(Return(exists)); - EXPECT_CALL(m_remote_status_updater, exists(_)) + EXPECT_CALL(m_remote_status_updater, mirror_image_exists(_)) .WillOnce(Return(exists)); } void create_image_replayer(MockThreads &mock_threads) { m_image_replayer = new MockImageReplayer( - m_local_io_ctx, "local_mirror_uuid", "global image id", + m_local_io_ctx, nullptr, "local_mirror_uuid", "global image id", &mock_threads, &m_instance_watcher, &m_local_status_updater, nullptr, nullptr); m_image_replayer->add_peer({"peer_uuid", m_remote_io_ctx, diff --git a/src/test/rbd_mirror/test_mock_ImageSync.cc b/src/test/rbd_mirror/test_mock_ImageSync.cc index bd6a2907830e6..7104d171952ca 100644 --- a/src/test/rbd_mirror/test_mock_ImageSync.cc +++ b/src/test/rbd_mirror/test_mock_ImageSync.cc @@ -255,7 +255,7 @@ public: MockSyncPointHandler& mock_sync_point_handler, MockInstanceWatcher &mock_instance_watcher, Context *ctx) { - return new MockImageSync(&mock_threads, &mock_local_image_ctx, + return new MockImageSync(&mock_threads, nullptr, &mock_local_image_ctx, &mock_remote_image_ctx, "mirror-uuid", &mock_sync_point_handler, &mock_instance_watcher, nullptr, ctx); diff --git a/src/test/rbd_mirror/test_mock_InstanceReplayer.cc b/src/test/rbd_mirror/test_mock_InstanceReplayer.cc index 1de7fec8f1968..9c671dc050aee 100644 --- a/src/test/rbd_mirror/test_mock_InstanceReplayer.cc +++ b/src/test/rbd_mirror/test_mock_InstanceReplayer.cc @@ -99,6 +99,7 @@ struct GroupReplayer { MOCK_METHOD0(is_running, bool()); MOCK_METHOD0(is_stopped, bool()); MOCK_METHOD0(is_blocklisted, bool()); + MOCK_METHOD0(needs_restart, bool()); MOCK_CONST_METHOD0(is_finished, bool()); MOCK_METHOD1(set_finished, void(bool)); @@ -114,7 +115,9 @@ struct ImageReplayer { std::string global_image_id; static ImageReplayer *create( - librados::IoCtx &local_io_ctx, const std::string &local_mirror_uuid, + librados::IoCtx &local_io_ctx, + GroupCtx *local_group_ctx, + const std::string &local_mirror_uuid, const std::string &global_image_id, Threads *threads, InstanceWatcher *instance_watcher, diff --git a/src/tools/rbd_mirror/CMakeLists.txt b/src/tools/rbd_mirror/CMakeLists.txt index 1913f06af6d00..bb66fa70adf32 100644 --- a/src/tools/rbd_mirror/CMakeLists.txt +++ b/src/tools/rbd_mirror/CMakeLists.txt @@ -26,6 +26,7 @@ set(rbd_mirror_internal Threads.cc Throttler.cc Types.cc + group_replayer/BootstrapRequest.cc image_deleter/SnapshotPurgeRequest.cc image_deleter/TrashMoveRequest.cc image_deleter/TrashRemoveRequest.cc diff --git a/src/tools/rbd_mirror/GroupReplayer.cc b/src/tools/rbd_mirror/GroupReplayer.cc index be9540ce2d8ff..b53a807e56df0 100644 --- a/src/tools/rbd_mirror/GroupReplayer.cc +++ b/src/tools/rbd_mirror/GroupReplayer.cc @@ -7,6 +7,12 @@ #include "common/debug.h" #include "common/errno.h" #include "librbd/ImageCtx.h" +#include "librbd/Utils.h" +#include "librbd/asio/ContextWQ.h" +#include "tools/rbd_mirror/ImageReplayer.h" +#include "tools/rbd_mirror/MirrorStatusUpdater.h" +#include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/group_replayer/BootstrapRequest.h" #include "tools/rbd_mirror/image_replayer/Utils.h" #include "GroupReplayer.h" @@ -21,8 +27,21 @@ extern PerfCounters *g_perf_counters; namespace rbd { namespace mirror { +using librbd::util::create_context_callback; +using librbd::util::unique_lock_name; + namespace { +std::string calc_ind_mirror_snap_name(uint64_t pool_id, + const std::string &group_id, + const std::string &snap_id) +{ + std::stringstream ind_snap_name_stream; + ind_snap_name_stream << ".mirror." << std::hex << pool_id << "_" + << group_id << "_" << snap_id; + return ind_snap_name_stream.str(); +} + template class GroupReplayerAdminSocketCommand { public: @@ -161,29 +180,6 @@ private: } // anonymous namespace -template -std::ostream &operator<<(std::ostream &os, - const typename GroupReplayer::State &state) { - switch (state) { - case GroupReplayer::STATE_STARTING: - os << "Starting"; - break; - case GroupReplayer::STATE_REPLAYING: - os << "Replaying"; - break; - case GroupReplayer::STATE_STOPPING: - os << "Stopping"; - break; - case GroupReplayer::STATE_STOPPED: - os << "Stopped"; - break; - default: - os << "Unknown (" << static_cast(state) << ")"; - break; - } - return os; -} - template std::ostream &operator<<(std::ostream &os, const GroupReplayer &replayer) { std::string nspace = replayer.get_namespace(); @@ -211,8 +207,7 @@ GroupReplayer::GroupReplayer( m_cache_manager_handler(cache_manager_handler), m_pool_meta_cache(pool_meta_cache), m_local_group_name(global_group_id), - m_lock(ceph::make_mutex("rbd::mirror::GroupReplayer " + - stringify(local_io_ctx.get_id()) + " " + global_group_id)) { + m_lock(ceph::make_mutex(unique_lock_name("GroupReplayer::m_lock", this))) { // Register asok commands using a temporary "remote_pool_name/global_group_id" // name. When the group name becomes known on start the asok commands will be // re-registered using "remote_pool_name/remote_group_name" name. @@ -227,8 +222,25 @@ GroupReplayer::~GroupReplayer() { unregister_admin_socket_hook(); ceph_assert(m_on_start_finish == nullptr); ceph_assert(m_on_stop_finish == nullptr); + ceph_assert(m_bootstrap_request == nullptr); } +template +bool GroupReplayer::needs_restart() const { + std::lock_guard locker{m_lock}; + + if (m_state != STATE_REPLAYING) { + return false; + } + + for (auto &[_, image_replayer] : m_image_replayers) { + if (image_replayer->is_stopped()) { + return true; + } + } + + return false; +} template image_replayer::HealthState GroupReplayer::get_health_state() const { @@ -279,6 +291,8 @@ void GroupReplayer::start(Context *on_finish, bool manual, bool restart) { m_last_r = 0; m_state_desc.clear(); m_manual_stop = false; + ceph_assert(m_on_start_finish == nullptr); + std::swap(m_on_start_finish, on_finish); } } @@ -289,8 +303,7 @@ void GroupReplayer::start(Context *on_finish, bool manual, bool restart) { return; } - // TODO - on_finish->complete(0); + bootstrap_group(); } template @@ -298,6 +311,8 @@ void GroupReplayer::stop(Context *on_finish, bool manual, bool restart) { dout(10) << "on_finish=" << on_finish << ", manual=" << manual << ", restart=" << restart << dendl; + group_replayer::BootstrapRequest *bootstrap_request = nullptr; + bool shut_down_replay = false; bool running = true; { std::lock_guard locker{m_lock}; @@ -316,13 +331,30 @@ void GroupReplayer::stop(Context *on_finish, bool manual, bool restart) { if (!is_stopped_()) { if (m_state == STATE_STARTING) { dout(10) << "canceling start" << dendl; + if (m_bootstrap_request != nullptr) { + bootstrap_request = m_bootstrap_request; + bootstrap_request->get(); + } } else { dout(10) << "interrupting replay" << dendl; + shut_down_replay = true; + m_state = STATE_STOPPING; } + + ceph_assert(m_on_stop_finish == nullptr); + std::swap(m_on_stop_finish, on_finish); + m_stop_requested = true; + m_manual_stop = manual; } } } + if (bootstrap_request != nullptr) { + dout(10) << "canceling bootstrap" << dendl; + bootstrap_request->cancel(); + bootstrap_request->put(); + } + if (!running) { dout(20) << "not running" << dendl; if (on_finish) { @@ -331,8 +363,16 @@ void GroupReplayer::stop(Context *on_finish, bool manual, bool restart) { return; } - // TODO - on_finish->complete(0); + if (shut_down_replay) { + stop_image_replayers(); + } else if (on_finish != nullptr) { + // XXXMG: clean up + { + std::lock_guard locker{m_lock}; + m_stop_requested = false; + } + on_finish->complete(0); + } } template @@ -360,7 +400,10 @@ void GroupReplayer::flush() { } dout(10) << dendl; - // TODO + + for (auto &[_, image_replayer] : m_image_replayers) { + image_replayer->flush(); + } } template @@ -371,10 +414,251 @@ void GroupReplayer::print_status(Formatter *f) { f->open_object_section("group_replayer"); f->dump_string("name", m_group_spec); - f->dump_stream("state") << m_state; - f->close_section(); + auto state = m_state; + if (m_local_group_ctx.primary && state == STATE_REPLAYING) { // XXXMG + state = STATE_STOPPED; + } + f->dump_string("state", state_to_string(state)); + f->open_array_section("image_replayers"); + for (auto &[_, image_replayer] : m_image_replayers) { + image_replayer->print_status(f); + } + f->close_section(); // image_replayers + f->close_section(); // group_replayer +} + +template +void GroupReplayer::bootstrap_group() { + dout(10) << dendl; + + std::unique_lock locker{m_lock}; + if (m_peers.empty()) { + locker.unlock(); + + dout(5) << "no peer clusters" << dendl; + finish_start(-ENOENT, "no peer clusters"); + return; + } + + // TODO need to support multiple remote groups + ceph_assert(!m_peers.empty()); + m_remote_group_peer = *m_peers.begin(); + + if (finish_start_if_interrupted(m_lock)) { + return; + } + + ceph_assert(m_image_replayers.empty()); + + auto ctx = create_context_callback< + GroupReplayer, + &GroupReplayer::handle_bootstrap_group>(this); + auto request = group_replayer::BootstrapRequest::create( + m_threads, m_local_io_ctx, m_remote_group_peer.io_ctx, m_global_group_id, + m_local_mirror_uuid, m_instance_watcher, m_local_status_updater, + m_remote_group_peer.mirror_status_updater, m_cache_manager_handler, + m_pool_meta_cache, &m_local_group_ctx, &m_image_replayers, ctx); + + request->get(); + m_bootstrap_request = request; + locker.unlock(); + + set_mirror_group_status_update( + cls::rbd::MIRROR_GROUP_STATUS_STATE_STARTING_REPLAY, "bootstrapping"); + request->send(); +} + +template +void GroupReplayer::handle_bootstrap_group(int r) { + dout(10) << "r=" << r << dendl; + { + std::lock_guard locker{m_lock}; + m_bootstrap_request->put(); + m_bootstrap_request = nullptr; + } + + if (finish_start_if_interrupted()) { + return; + } else if (r == -ENOENT) { + finish_start(r, "group removed"); + return; + } else if (r == -EREMOTEIO) { + finish_start(r, "remote group is non-primary"); + return; + } else if (r == -EEXIST) { + finish_start(r, "split-brain detected"); + return; + } else if (r < 0) { + finish_start(r, "bootstrap failed"); + return; + } + + { + std::lock_guard locker{m_lock}; + m_local_group_name = m_local_group_ctx.name; + } + reregister_admin_socket_hook(); + + start_image_replayers(); +} + +template +void GroupReplayer::start_image_replayers() { + dout(10) << m_image_replayers.size() << dendl; + + set_mirror_group_status_update( + cls::rbd::MIRROR_GROUP_STATUS_STATE_STARTING_REPLAY, "starting replay"); + + auto ctx = create_context_callback< + GroupReplayer, &GroupReplayer::handle_start_image_replayers>(this); + C_Gather *gather_ctx = new C_Gather(g_ceph_context, ctx); + { + std::lock_guard locker{m_lock}; + for (auto &[_, image_replayer] : m_image_replayers) { + image_replayer->start(gather_ctx->new_sub(), false); + } + } + gather_ctx->activate(); +} + +template +void GroupReplayer::handle_start_image_replayers(int r) { + dout(10) << "r=" << r << dendl; + + if (finish_start_if_interrupted()) { + return; + } else if (r < 0) { + finish_start(r, ""); + return; + } + + finish_start(0, ""); +} + +template +void GroupReplayer::stop_image_replayers() { + dout(10) << m_image_replayers.size() << dendl; + + set_mirror_group_status_update( + cls::rbd::MIRROR_GROUP_STATUS_STATE_STOPPING_REPLAY, "stopping"); + + auto ctx = create_context_callback< + GroupReplayer, &GroupReplayer::handle_stop_image_replayers>(this); + C_Gather *gather_ctx = new C_Gather(g_ceph_context, ctx); + { + std::lock_guard locker{m_lock}; + for (auto &[_, image_replayer] : m_image_replayers) { + image_replayer->stop(gather_ctx->new_sub(), false); + } + } + gather_ctx->activate(); +} + +template +void GroupReplayer::handle_stop_image_replayers(int r) { + dout(10) << "r=" << r << dendl; + + Context *on_finish = nullptr; + { + std::lock_guard locker{m_lock}; + ceph_assert(m_state == STATE_STOPPING); + m_stop_requested = false; + m_state = STATE_STOPPED; + std::swap(on_finish, m_on_stop_finish); + + for (auto &[_, image_replayer] : m_image_replayers) { + delete image_replayer; + } + m_image_replayers.clear(); + } + + set_mirror_group_status_update( + cls::rbd::MIRROR_GROUP_STATUS_STATE_STOPPED, "stopped"); + + if (on_finish != nullptr) { + on_finish->complete(0); + } +} + +template +bool GroupReplayer::finish_start_if_interrupted() { + std::lock_guard locker{m_lock}; + + return finish_start_if_interrupted(m_lock); +} + +template +bool GroupReplayer::finish_start_if_interrupted(ceph::mutex &lock) { + ceph_assert(ceph_mutex_is_locked_by_me(m_lock)); + ceph_assert(m_state == STATE_STARTING); + if (!m_stop_requested) { + return false; + } + + finish_start(-ECANCELED, ""); + return true; +} + +template +void GroupReplayer::finish_start(int r, const std::string &desc) { + dout(10) << "r=" << r << ", desc=" << desc << dendl; + Context *ctx = new LambdaContext( + [this, r, desc](int _r) { + Context *on_finish = nullptr; + { + std::lock_guard locker{m_lock}; + ceph_assert(m_state == STATE_STARTING); + m_state = STATE_REPLAYING; + std::swap(m_on_start_finish, on_finish); + m_state_desc = desc; + if (r < 0) { + auto state = cls::rbd::MIRROR_GROUP_STATUS_STATE_STOPPED; + if (r == -ECANCELED) { + dout(10) << "start canceled" << dendl; + } else if (r == -ENOENT) { + dout(10) << "mirroring group removed" << dendl; + } else if (r == -EREMOTEIO) { + dout(10) << "mirroring group demoted" << dendl; + } else { + derr << "start failed: " << cpp_strerror(r) << dendl; + state = cls::rbd::MIRROR_GROUP_STATUS_STATE_ERROR; + } + on_finish = new LambdaContext( + [this, r, state, desc, on_finish](int) { + set_mirror_group_status_update(state, desc); + + if (r == -ENOENT) { + set_finished(true); + } + if (on_finish != nullptr) { + on_finish->complete(r); + } + }); + } + } + + if (r < 0) { + stop(on_finish, false, false); + return; + } + + if (m_local_group_ctx.primary) { // XXXMG + set_mirror_group_status_update( + cls::rbd::MIRROR_GROUP_STATUS_STATE_STOPPED, + "local group is primary"); + } else { + set_mirror_group_status_update( + cls::rbd::MIRROR_GROUP_STATUS_STATE_REPLAYING, "replaying"); + } + + if (on_finish != nullptr) { + on_finish->complete(0); + } + }); + m_threads->work_queue->queue(ctx, 0); } + template void GroupReplayer::register_admin_socket_hook() { GroupReplayerAdminSocketHook *asok_hook; @@ -433,6 +717,216 @@ void GroupReplayer::reregister_admin_socket_hook() { register_admin_socket_hook(); } +template +void GroupReplayer::set_mirror_group_status_update( + cls::rbd::MirrorGroupStatusState state, const std::string &desc) { + dout(20) << "state=" << state << ", description=" << desc << dendl; + + cls::rbd::MirrorGroupSiteStatus local_status; + local_status.state = state; + local_status.description = desc; + local_status.up = true; + + auto remote_status = local_status; + + { + std::unique_lock locker{m_lock}; + for (auto &[_, ir] : m_image_replayers) { + cls::rbd::MirrorImageSiteStatus mirror_image; + if (ir->is_running()) { + if (ir->is_replaying()) { + mirror_image.state = cls::rbd::MIRROR_IMAGE_STATUS_STATE_REPLAYING; + } else { + mirror_image.state = cls::rbd::MIRROR_IMAGE_STATUS_STATE_STARTING_REPLAY; + } + } else if (ir->is_stopped()) { + mirror_image.state = cls::rbd::MIRROR_IMAGE_STATUS_STATE_STOPPED; + } else { + mirror_image.state = cls::rbd::MIRROR_IMAGE_STATUS_STATE_STOPPING_REPLAY; + } + mirror_image.description = ir->get_state_description(); + + local_status.mirror_images[{ir->get_local_pool_id(), + ir->get_global_image_id()}] = mirror_image; + auto remote_pool_id = ir->get_remote_pool_id(); + if (remote_pool_id >= 0) { + remote_status.mirror_images[{remote_pool_id, + ir->get_global_image_id()}] = mirror_image; + } + } + } + + m_local_status_updater->set_mirror_group_status(m_global_group_id, + local_status, true); + if (m_remote_group_peer.mirror_status_updater != nullptr) { + m_remote_group_peer.mirror_status_updater->set_mirror_group_status( + m_global_group_id, remote_status, true); + } +} + +template +void GroupReplayer::create_mirror_snapshot_start( + const std::string &remote_group_snap_id, ImageReplayer *image_replayer, + int64_t *local_group_pool_id, std::string *local_group_id, + std::string *local_group_snap_id, Context *on_finish) { + dout(20) << remote_group_snap_id << " " << image_replayer << dendl; + + std::unique_lock locker{m_lock}; + ceph_assert(m_pending_snap_create == false); + + if (m_state != STATE_REPLAYING) { + derr << "not in replaying state" << dendl; + locker.unlock(); + on_finish->complete(-ESTALE); + return; + } + + if (m_remote_group_snap_id.empty()) { + ceph_assert(m_create_snap_requests.empty()); + m_remote_group_snap_id = remote_group_snap_id; + + // XXXMG: make the same name as the primary snapshot has + m_group_snap = {librbd::util::generate_uuid(m_local_io_ctx), + cls::rbd::MirrorGroupSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, + {}, m_remote_group_peer.uuid, m_remote_group_snap_id}, + calc_ind_mirror_snap_name( + m_remote_group_peer.io_ctx.get_id(), m_remote_group_id, + m_remote_group_snap_id), + cls::rbd::GROUP_SNAPSHOT_STATE_INCOMPLETE}; + } + + ceph_assert(m_create_snap_requests.count(image_replayer) == 0); + m_create_snap_requests[image_replayer] = on_finish; + + *local_group_pool_id = m_local_io_ctx.get_id(); + *local_group_id = m_local_group_ctx.group_id; + *local_group_snap_id = m_group_snap.id; + + if (m_remote_group_snap_id != remote_group_snap_id) { + dout(15) << "request snap_id " << remote_group_snap_id + << " does not match us " << m_remote_group_snap_id + << " -- restarting" << dendl; + auto create_snap_requests = m_create_snap_requests; + m_create_snap_requests.clear(); + locker.unlock(); + for (auto &[_, on_finish] : create_snap_requests) { + on_finish->complete(-EAGAIN); + } + return; + } + + // XXXMG: m_image_replayers.size() will not always work + if (m_create_snap_requests.size() < m_image_replayers.size()) { + return; + } + + m_pending_snap_create = true; + + for (auto &[io_ctx, image_replayer] : m_image_replayers) { + m_group_snap.snaps.emplace_back(io_ctx.get_id(), + image_replayer->get_local_image_id(), + CEPH_NOSNAP); + ceph_assert(!m_group_snap.snaps.back().image_id.empty()); + } + + dout(20) << m_group_snap.id << " " << m_group_snap.name << dendl; + + librados::ObjectWriteOperation op; + librbd::cls_client::group_snap_set(&op, m_group_snap); + auto comp = create_rados_callback< + GroupReplayer, + &GroupReplayer::handle_create_mirror_snapshot_start>(this); + + int r = m_local_io_ctx.aio_operate( + librbd::util::group_header_name(m_local_group_ctx.group_id), comp, &op); + ceph_assert(r == 0); + comp->release(); +} + +template +void GroupReplayer::handle_create_mirror_snapshot_start(int r) { + dout(20) << m_remote_group_snap_id << " r=" << r << dendl; + + std::unique_lock locker{m_lock}; + ceph_assert(m_pending_snap_create == true); + + auto create_snap_requests = m_create_snap_requests; + m_create_snap_requests.clear(); + locker.unlock(); + + for (auto &[_, on_finish] : create_snap_requests) { + on_finish->complete(r); + } +} + +template +void GroupReplayer::create_mirror_snapshot_finish( + const std::string &remote_group_snap_id, ImageReplayer *image_replayer, + uint64_t snap_id, Context *on_finish) { + dout(20) << remote_group_snap_id << " " << image_replayer << " snap_id=" + << snap_id << dendl; + + std::lock_guard locker{m_lock}; + ceph_assert(m_pending_snap_create == true); + ceph_assert(m_state == STATE_REPLAYING || m_state == STATE_STOPPING); + ceph_assert(m_remote_group_snap_id == remote_group_snap_id); + ceph_assert(m_create_snap_requests.count(image_replayer) == 0); + + m_create_snap_requests[image_replayer] = on_finish; + + auto pool = image_replayer->get_local_pool_id(); + auto image_id = image_replayer->get_local_image_id(); + auto it = std::find_if( + m_group_snap.snaps.begin(), m_group_snap.snaps.end(), + [&pool, &image_id](const cls::rbd::ImageSnapshotSpec &s) { + return pool == s.pool && image_id == s.image_id; + }); + ceph_assert(it != m_group_snap.snaps.end()); + it->snap_id = snap_id; + + if (m_create_snap_requests.size() < m_image_replayers.size()) { + return; + } + + m_group_snap.state = cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE; + + dout(20) << m_group_snap.id << " " << m_group_snap.name << dendl; + + librados::ObjectWriteOperation op; + librbd::cls_client::group_snap_set(&op, m_group_snap); + auto comp = create_rados_callback< + GroupReplayer, + &GroupReplayer::handle_create_mirror_snapshot_finish>(this); + + int r = m_local_io_ctx.aio_operate( + librbd::util::group_header_name(m_local_group_ctx.group_id), comp, &op); + ceph_assert(r == 0); + comp->release(); +} + +template +void GroupReplayer::handle_create_mirror_snapshot_finish(int r) { + dout(20) << m_remote_group_snap_id << " r=" << r << dendl; + + std::unique_lock locker{m_lock}; + ceph_assert(m_pending_snap_create == true); + m_pending_snap_create = false; + m_remote_group_snap_id.clear(); + bool stopping = (m_state == STATE_STOPPING); + auto create_snap_requests = m_create_snap_requests; + m_create_snap_requests.clear(); + locker.unlock(); + + for (auto &[_, on_finish] : create_snap_requests) { + on_finish->complete(r); + } + + if (stopping) { + stop_image_replayers(); + } +} + } // namespace mirror } // namespace rbd diff --git a/src/tools/rbd_mirror/GroupReplayer.h b/src/tools/rbd_mirror/GroupReplayer.h index 07d9802e9460f..572645d8e2d5a 100644 --- a/src/tools/rbd_mirror/GroupReplayer.h +++ b/src/tools/rbd_mirror/GroupReplayer.h @@ -4,12 +4,14 @@ #ifndef CEPH_RBD_MIRROR_GROUP_REPLAYER_H #define CEPH_RBD_MIRROR_GROUP_REPLAYER_H +#include "cls/rbd/cls_rbd_types.h" #include "common/ceph_mutex.h" #include "include/rados/librados.hpp" #include "tools/rbd_mirror/Types.h" #include "tools/rbd_mirror/image_replayer/Types.h" #include #include +#include class AdminSocketHook; @@ -19,11 +21,16 @@ namespace librbd { class ImageCtx; } namespace rbd { namespace mirror { +template class ImageReplayer; template struct InstanceWatcher; template struct MirrorStatusUpdater; struct PoolMetaCache; template struct Threads; +namespace group_replayer { + template class BootstrapRequest; +} + /** * Replays changes from a remote cluster for a single group. */ @@ -57,11 +64,23 @@ public: GroupReplayer(const GroupReplayer&) = delete; GroupReplayer& operator=(const GroupReplayer&) = delete; - bool is_stopped() { std::lock_guard l{m_lock}; return is_stopped_(); } - bool is_running() { std::lock_guard l{m_lock}; return is_running_(); } - bool is_replaying() { std::lock_guard l{m_lock}; return is_replaying_(); } + bool is_stopped() const { + std::lock_guard l{m_lock}; + return is_stopped_(); + } + bool is_running() const { + std::lock_guard l{m_lock}; + return is_running_(); + } + bool is_replaying() const { + std::lock_guard l{m_lock}; + return is_replaying_(); + } - std::string get_name() { std::lock_guard l{m_lock}; return m_group_spec; }; + std::string get_name() const { + std::lock_guard l{m_lock}; + return m_group_spec; + } void set_state_description(int r, const std::string &desc); // TODO temporary until policy handles release of group replayers @@ -79,6 +98,8 @@ public: return (m_last_r == -EBLOCKLISTED); } + bool needs_restart() const; + image_replayer::HealthState get_health_state() const; void add_peer(const Peer& peer); @@ -115,6 +136,18 @@ public: * * * | * * v (error) * + * BOOTSTRAP_GROUP * * * * * * * * * * * * * * * * * * * * + * | * + * v (error) * + * START_IMAGE_REPLAYERS * * * * * * * * * * * * * * * * * + * | + * v + * REPLAYING + * | + * v + * STOP_IMAGE_REPLAYERS + * | + * v * * * @endverbatim @@ -136,13 +169,15 @@ private: std::string m_global_group_id; Threads *m_threads; InstanceWatcher *m_instance_watcher; - MirrorStatusUpdater* m_local_status_updater; + MirrorStatusUpdater *m_local_status_updater; journal::CacheManagerHandler *m_cache_manager_handler; PoolMetaCache* m_pool_meta_cache; std::string m_local_group_name; std::string m_group_spec; + GroupCtx m_local_group_ctx; Peers m_peers; + Peer m_remote_group_peer; mutable ceph::mutex m_lock; State m_state = STATE_STOPPED; @@ -158,6 +193,24 @@ private: AdminSocketHook *m_asok_hook = nullptr; + group_replayer::BootstrapRequest *m_bootstrap_request = nullptr; + std::list *>> m_image_replayers; + + static std::string state_to_string(const State &state) { + switch (state) { + case STATE_STARTING: + return "Starting"; + case STATE_REPLAYING: + return "Replaying"; + case STATE_STOPPING: + return "Stopping"; + case STATE_STOPPED: + return "Stopped"; + default: + return "Unknown (" + stringify(static_cast(state)) + ")"; + } + } + bool is_stopped_() const { return m_state == STATE_STOPPED; } @@ -168,9 +221,25 @@ private: return (m_state == STATE_REPLAYING); } + void bootstrap_group(); + void handle_bootstrap_group(int r); + + void start_image_replayers(); + void handle_start_image_replayers(int r); + + bool finish_start_if_interrupted(); + bool finish_start_if_interrupted(ceph::mutex &lock); + void finish_start(int r, const std::string &desc); + + void stop_image_replayers(); + void handle_stop_image_replayers(int r); + void register_admin_socket_hook(); void unregister_admin_socket_hook(); void reregister_admin_socket_hook(); + + void set_mirror_group_status_update(cls::rbd::MirrorGroupStatusState state, + const std::string &desc); }; } // namespace mirror diff --git a/src/tools/rbd_mirror/ImageMap.cc b/src/tools/rbd_mirror/ImageMap.cc index f8d4381e37b47..36a7bd54e17ba 100644 --- a/src/tools/rbd_mirror/ImageMap.cc +++ b/src/tools/rbd_mirror/ImageMap.cc @@ -319,7 +319,11 @@ void ImageMap::notify_listener_acquire_release_images( m_threads->work_queue, new C_NotifyInstance(this, global_id, true))); } else if (update.entity.type == MIRROR_ENTITY_TYPE_GROUP) { - // TODO + m_listener.acquire_group( + update.entity.global_id, update.instance_id, + create_async_context_callback( + m_threads->work_queue, + new C_NotifyInstance(this, global_id, true))); } else { ceph_abort_msgf("invalid mirror entity type: %d", (int)update.entity.type); @@ -335,7 +339,11 @@ void ImageMap::notify_listener_acquire_release_images( m_threads->work_queue, new C_NotifyInstance(this, global_id, true))); } else if (update.entity.type == MIRROR_ENTITY_TYPE_GROUP) { - // TODO + m_listener.release_group( + update.entity.global_id, update.instance_id, + create_async_context_callback( + m_threads->work_queue, + new C_NotifyInstance(this, global_id, true))); } else { ceph_abort_msgf("invalid mirror entity type: %d", (int)update.entity.type); @@ -358,7 +366,11 @@ void ImageMap::notify_listener_remove_images(const std::string &mirror_uuid, m_threads->work_queue, new C_NotifyInstance(this, global_id, false))); } else if (update.entity.type == MIRROR_ENTITY_TYPE_GROUP) { - // TODO + m_listener.remove_group( + mirror_uuid, update.entity.global_id, update.instance_id, + create_async_context_callback( + m_threads->work_queue, + new C_NotifyInstance(this, global_id, false))); } else { ceph_abort_msgf("invalid mirror entity type: %d", (int)update.entity.type); @@ -442,7 +454,7 @@ void ImageMap::update_images_removed( } if (entity_mapped && peer_removed && !mirror_uuid.empty()) { - // peer image has been deleted + // peer entity has been deleted or local non-image entity needs restart to_remove.emplace_back(entity, info.instance_id); } diff --git a/src/tools/rbd_mirror/ImageReplayer.cc b/src/tools/rbd_mirror/ImageReplayer.cc index 97bf531278d3f..76e4e35d7c7c4 100644 --- a/src/tools/rbd_mirror/ImageReplayer.cc +++ b/src/tools/rbd_mirror/ImageReplayer.cc @@ -215,15 +215,16 @@ struct ImageReplayer::ReplayerListener template ImageReplayer::ImageReplayer( - librados::IoCtx &local_io_ctx, const std::string &local_mirror_uuid, + librados::IoCtx &local_io_ctx, GroupCtx *local_group_ctx, + const std::string &local_mirror_uuid, const std::string &global_image_id, Threads *threads, InstanceWatcher *instance_watcher, MirrorStatusUpdater* local_status_updater, journal::CacheManagerHandler *cache_manager_handler, PoolMetaCache* pool_meta_cache) : - m_local_io_ctx(local_io_ctx), m_local_mirror_uuid(local_mirror_uuid), - m_global_image_id(global_image_id), m_threads(threads), - m_instance_watcher(instance_watcher), + m_local_io_ctx(local_io_ctx), m_local_group_ctx(local_group_ctx), + m_local_mirror_uuid(local_mirror_uuid), m_global_image_id(global_image_id), + m_threads(threads), m_instance_watcher(instance_watcher), m_local_status_updater(local_status_updater), m_cache_manager_handler(cache_manager_handler), m_pool_meta_cache(pool_meta_cache), @@ -355,8 +356,8 @@ void ImageReplayer::bootstrap() { auto ctx = create_context_callback< ImageReplayer, &ImageReplayer::handle_bootstrap>(this); auto request = image_replayer::BootstrapRequest::create( - m_threads, m_local_io_ctx, m_remote_image_peer.io_ctx, m_instance_watcher, - m_global_image_id, m_local_mirror_uuid, + m_threads, m_local_io_ctx, m_remote_image_peer.io_ctx, m_local_group_ctx, + m_instance_watcher, m_global_image_id, m_local_mirror_uuid, m_remote_image_peer.remote_pool_meta, m_cache_manager_handler, m_pool_meta_cache, &m_progress_cxt, &m_state_builder, &m_resync_requested, ctx); @@ -883,11 +884,22 @@ void ImageReplayer::set_mirror_image_status_update( } dout(15) << "status=" << status << dendl; - m_local_status_updater->set_mirror_image_status(m_global_image_id, status, - force); - if (m_remote_image_peer.mirror_status_updater != nullptr) { - m_remote_image_peer.mirror_status_updater->set_mirror_image_status( - m_global_image_id, status, force); + if (m_local_group_ctx != nullptr) { + m_local_status_updater->set_mirror_group_image_status( + m_local_group_ctx->global_group_id, m_local_io_ctx.get_id(), + m_global_image_id, status, force); + if (m_remote_image_peer.mirror_status_updater != nullptr) { + m_remote_image_peer.mirror_status_updater->set_mirror_group_image_status( + m_local_group_ctx->global_group_id, m_remote_image_peer.io_ctx.get_id(), + m_global_image_id, status, force); + } + } else { + m_local_status_updater->set_mirror_image_status(m_global_image_id, status, + force); + if (m_remote_image_peer.mirror_status_updater != nullptr) { + m_remote_image_peer.mirror_status_updater->set_mirror_image_status( + m_global_image_id, status, force); + } } m_in_flight_op_tracker.finish_op(); @@ -998,6 +1010,57 @@ void ImageReplayer::handle_shut_down(int r) { return; } + if (m_local_group_ctx != nullptr) { + if (m_local_status_updater->mirror_group_image_exists( + m_local_group_ctx->global_group_id, m_local_io_ctx.get_id(), + m_global_image_id)) { + dout(15) << "removing local mirror group image status" << dendl; + auto ctx = new LambdaContext([this, r](int) { + handle_shut_down(r); + }); + m_local_status_updater->remove_mirror_group_image_status( + m_local_group_ctx->global_group_id, m_local_io_ctx.get_id(), + m_global_image_id, ctx); + return; + } + + if (m_remote_image_peer.mirror_status_updater != nullptr && + m_remote_image_peer.mirror_status_updater->mirror_group_image_exists( + m_local_group_ctx->global_group_id, + m_remote_image_peer.io_ctx.get_id(), m_global_image_id)) { + dout(15) << "removing remote mirror group image status" << dendl; + auto ctx = new LambdaContext([this, r](int) { + handle_shut_down(r); + }); + m_remote_image_peer.mirror_status_updater->remove_mirror_group_image_status( + m_local_group_ctx->global_group_id, + m_remote_image_peer.io_ctx.get_id(), m_global_image_id, ctx); + return; + } + } else { + if (m_local_status_updater->mirror_image_exists(m_global_image_id)) { + dout(15) << "removing local mirror image status" << dendl; + auto ctx = new LambdaContext([this, r](int) { + handle_shut_down(r); + }); + m_local_status_updater->remove_mirror_image_status(m_global_image_id, + true, ctx); + return; + } + + if (m_remote_image_peer.mirror_status_updater != nullptr && + m_remote_image_peer.mirror_status_updater->mirror_image_exists( + m_global_image_id)) { + dout(15) << "removing remote mirror image status" << dendl; + auto ctx = new LambdaContext([this, r](int) { + handle_shut_down(r); + }); + m_remote_image_peer.mirror_status_updater->remove_mirror_image_status( + m_global_image_id, true, ctx); + return; + } + } + if (m_state_builder != nullptr) { m_state_builder->destroy(); m_state_builder = nullptr; @@ -1157,7 +1220,7 @@ void ImageReplayer::remove_image_status(bool force, Context *on_finish) remove_image_status_remote(force, on_finish); }); - if (m_local_status_updater->exists(m_global_image_id)) { + if (m_local_status_updater->mirror_image_exists(m_global_image_id)) { dout(15) << "removing local mirror image status" << dendl; if (force) { m_local_status_updater->remove_mirror_image_status( @@ -1176,7 +1239,7 @@ template void ImageReplayer::remove_image_status_remote(bool force, Context *on_finish) { if (m_remote_image_peer.mirror_status_updater != nullptr && - m_remote_image_peer.mirror_status_updater->exists(m_global_image_id)) { + m_remote_image_peer.mirror_status_updater->mirror_image_exists(m_global_image_id)) { dout(15) << "removing remote mirror image status" << dendl; if (force) { m_remote_image_peer.mirror_status_updater->remove_mirror_image_status( diff --git a/src/tools/rbd_mirror/ImageReplayer.h b/src/tools/rbd_mirror/ImageReplayer.h index de76256798e3e..6818f617c629e 100644 --- a/src/tools/rbd_mirror/ImageReplayer.h +++ b/src/tools/rbd_mirror/ImageReplayer.h @@ -42,21 +42,23 @@ template class ImageReplayer { public: static ImageReplayer *create( - librados::IoCtx &local_io_ctx, const std::string &local_mirror_uuid, - const std::string &global_image_id, Threads *threads, - InstanceWatcher *instance_watcher, + librados::IoCtx &local_io_ctx, GroupCtx *local_group_ctx, + const std::string &local_mirror_uuid, const std::string &global_image_id, + Threads *threads, InstanceWatcher *instance_watcher, MirrorStatusUpdater* local_status_updater, journal::CacheManagerHandler *cache_manager_handler, PoolMetaCache* pool_meta_cache) { - return new ImageReplayer(local_io_ctx, local_mirror_uuid, global_image_id, - threads, instance_watcher, local_status_updater, - cache_manager_handler, pool_meta_cache); + return new ImageReplayer(local_io_ctx, local_group_ctx, local_mirror_uuid, + global_image_id, threads, instance_watcher, + local_status_updater, cache_manager_handler, + pool_meta_cache); } void destroy() { delete this; } ImageReplayer(librados::IoCtx &local_io_ctx, + GroupCtx *local_group_ctx, const std::string &local_mirror_uuid, const std::string &global_image_id, Threads *threads, @@ -68,11 +70,27 @@ public: ImageReplayer(const ImageReplayer&) = delete; ImageReplayer& operator=(const ImageReplayer&) = delete; - bool is_stopped() { std::lock_guard l{m_lock}; return is_stopped_(); } - bool is_running() { std::lock_guard l{m_lock}; return is_running_(); } - bool is_replaying() { std::lock_guard l{m_lock}; return is_replaying_(); } + inline bool is_stopped() const { + std::lock_guard l{m_lock}; + return is_stopped_(); + } + inline bool is_running() const { + std::lock_guard l{m_lock}; + return is_running_(); + } + inline bool is_replaying() const { + std::lock_guard l{m_lock}; + return is_replaying_(); + } - std::string get_name() { std::lock_guard l{m_lock}; return m_image_spec; }; + inline std::string get_name() const { + std::lock_guard l{m_lock}; + return m_image_spec; + }; + inline std::string get_state_description() const { + std::lock_guard l{m_lock}; + return m_state_desc; + } void set_state_description(int r, const std::string &desc); // TODO temporary until policy handles release of image replayers @@ -97,6 +115,16 @@ public: inline int64_t get_local_pool_id() const { return m_local_io_ctx.get_id(); } + inline int64_t get_remote_pool_id() const { + std::unique_lock locker{m_lock}; + if (m_remote_image_peer.io_ctx.is_valid()) { + return m_remote_image_peer.io_ctx.get_id(); + } + if (!m_peers.empty()) { + return m_peers.begin()->io_ctx.get_id(); + } + return -1; + } inline const std::string& get_global_image_id() const { return m_global_image_id; } @@ -178,6 +206,7 @@ private: }; librados::IoCtx &m_local_io_ctx; + GroupCtx *m_local_group_ctx; std::string m_local_mirror_uuid; std::string m_global_image_id; Threads *m_threads; diff --git a/src/tools/rbd_mirror/ImageSync.cc b/src/tools/rbd_mirror/ImageSync.cc index b0434bef6c5ac..fcd659131994e 100644 --- a/src/tools/rbd_mirror/ImageSync.cc +++ b/src/tools/rbd_mirror/ImageSync.cc @@ -15,6 +15,7 @@ #include "librbd/asio/ContextWQ.h" #include "librbd/deep_copy/Handler.h" #include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/Types.h" #include "tools/rbd_mirror/image_sync/SyncPointCreateRequest.h" #include "tools/rbd_mirror/image_sync/SyncPointPruneRequest.h" #include "tools/rbd_mirror/image_sync/Types.h" @@ -53,6 +54,7 @@ public: template ImageSync::ImageSync( Threads* threads, + GroupCtx *local_group_ctx, I *local_image_ctx, I *remote_image_ctx, const std::string &local_mirror_uuid, @@ -73,6 +75,12 @@ ImageSync::ImageSync( m_update_sync_point_interval( m_local_image_ctx->cct->_conf.template get_val( "rbd_mirror_sync_point_update_age")) { + if (local_group_ctx == nullptr) { + m_sync_id = m_local_image_ctx->id; + } else { + m_sync_id = stringify(m_local_image_ctx->md_ctx.get_id()) + ":" + + m_local_image_ctx->id; + } } template @@ -95,7 +103,7 @@ void ImageSync::cancel() { m_canceled = true; - if (m_instance_watcher->cancel_sync_request(m_local_image_ctx->id)) { + if (m_instance_watcher->cancel_sync_request(m_sync_id)) { return; } @@ -120,7 +128,7 @@ void ImageSync::send_notify_sync_request() { Context *ctx = create_async_context_callback( m_threads->work_queue, create_context_callback< ImageSync, &ImageSync::handle_notify_sync_request>(this)); - m_instance_watcher->notify_sync_request(m_local_image_ctx->id, ctx); + m_instance_watcher->notify_sync_request(m_sync_id, ctx); m_lock.unlock(); } @@ -461,7 +469,7 @@ template void ImageSync::finish(int r) { dout(20) << ": r=" << r << dendl; - m_instance_watcher->notify_sync_complete(m_local_image_ctx->id); + m_instance_watcher->notify_sync_complete(m_sync_id); CancelableRequest::finish(r); } diff --git a/src/tools/rbd_mirror/ImageSync.h b/src/tools/rbd_mirror/ImageSync.h index b3389ce18fec8..c56d376d9522c 100644 --- a/src/tools/rbd_mirror/ImageSync.h +++ b/src/tools/rbd_mirror/ImageSync.h @@ -21,6 +21,7 @@ namespace mirror { class ProgressContext; template class InstanceWatcher; template class Threads; +struct GroupCtx; namespace image_sync { struct SyncPointHandler; } @@ -29,6 +30,7 @@ class ImageSync : public CancelableRequest { public: static ImageSync* create( Threads* threads, + GroupCtx *local_group_ctx, ImageCtxT *local_image_ctx, ImageCtxT *remote_image_ctx, const std::string &local_mirror_uuid, @@ -36,13 +38,15 @@ public: InstanceWatcher *instance_watcher, ProgressContext *progress_ctx, Context *on_finish) { - return new ImageSync(threads, local_image_ctx, remote_image_ctx, - local_mirror_uuid, sync_point_handler, - instance_watcher, progress_ctx, on_finish); + return new ImageSync(threads, local_group_ctx, local_image_ctx, + remote_image_ctx, local_mirror_uuid, + sync_point_handler, instance_watcher, progress_ctx, + on_finish); } ImageSync( Threads* threads, + GroupCtx *local_group_ctx, ImageCtxT *local_image_ctx, ImageCtxT *remote_image_ctx, const std::string &local_mirror_uuid, @@ -100,6 +104,7 @@ private: ceph::mutex m_lock; bool m_canceled = false; + std::string m_sync_id; librbd::DeepCopyRequest *m_image_copy_request = nullptr; ImageCopyProgressHandler *m_image_copy_prog_handler = nullptr; diff --git a/src/tools/rbd_mirror/InstanceReplayer.cc b/src/tools/rbd_mirror/InstanceReplayer.cc index 11c73e9f1ab78..5600a890c8178 100644 --- a/src/tools/rbd_mirror/InstanceReplayer.cc +++ b/src/tools/rbd_mirror/InstanceReplayer.cc @@ -110,6 +110,7 @@ void InstanceReplayer::shut_down(Context *on_finish) { Context *ctx = new LambdaContext( [this] (int r) { + cancel_group_state_check_task(); cancel_image_state_check_task(); wait_for_ops(); }); @@ -133,6 +134,7 @@ void InstanceReplayer::release_all(Context *on_finish) { std::lock_guard locker{m_lock}; C_Gather *gather_ctx = new C_Gather(g_ceph_context, on_finish); + dout(10) << "group_replayers: { " << m_group_replayers << " }" << dendl; for (auto it = m_group_replayers.begin(); it != m_group_replayers.end(); it = m_group_replayers.erase(it)) { auto group_replayer = it->second; @@ -171,7 +173,7 @@ void InstanceReplayer::acquire_image(InstanceWatcher *instance_watcher, auto it = m_image_replayers.find(global_image_id); if (it == m_image_replayers.end()) { auto image_replayer = ImageReplayer::create( - m_local_io_ctx, m_local_mirror_uuid, global_image_id, + m_local_io_ctx, nullptr, m_local_mirror_uuid, global_image_id, m_threads, instance_watcher, m_local_status_updater, m_cache_manager_handler, m_pool_meta_cache); @@ -294,6 +296,7 @@ void InstanceReplayer::release_group(const std::string &global_group_id, std::lock_guard locker{m_lock}; ceph_assert(m_on_shut_down == nullptr); + dout(10) << "group_replayers: { " << m_group_replayers << " }" << dendl; auto it = m_group_replayers.find(global_group_id); if (it == m_group_replayers.end()) { dout(5) << global_group_id << ": not found" << dendl; @@ -601,10 +604,11 @@ void InstanceReplayer::handle_stop_image_replayers(int r) { } m_image_replayers.clear(); - ceph_assert(m_on_shut_down != nullptr); std::swap(on_finish, m_on_shut_down); } - on_finish->complete(r); + if (on_finish) { + on_finish->complete(r); + } } template @@ -646,10 +650,15 @@ void InstanceReplayer::schedule_image_state_check_task() { template void InstanceReplayer::start_group_replayer( GroupReplayer *group_replayer) { + dout(10) << dendl; ceph_assert(ceph_mutex_is_locked(m_lock)); std::string global_group_id = group_replayer->get_global_group_id(); if (!group_replayer->is_stopped()) { + if (group_replayer->needs_restart()) { + stop_group_replayer(group_replayer, new C_TrackedOp(m_async_op_tracker, + nullptr)); + } return; } else if (group_replayer->is_blocklisted()) { derr << "global_group_id=" << global_group_id << ": blocklisted detected " @@ -687,6 +696,7 @@ void InstanceReplayer::start_group_replayers(int r) { std::lock_guard locker{m_lock}; if (m_on_shut_down != nullptr) { + m_async_op_tracker.finish_op(); return; } @@ -790,10 +800,11 @@ void InstanceReplayer::handle_stop_group_replayers(int r) { } m_group_replayers.clear(); - ceph_assert(m_on_shut_down != nullptr); std::swap(on_finish, m_on_shut_down); } - on_finish->complete(r); + if (on_finish) { + on_finish->complete(r); + } } template @@ -849,6 +860,7 @@ void InstanceReplayer::handle_wait_for_ops(int r) { ceph_assert(r == 0); std::lock_guard locker{m_lock}; + stop_group_replayers(); stop_image_replayers(); } diff --git a/src/tools/rbd_mirror/MirrorStatusUpdater.cc b/src/tools/rbd_mirror/MirrorStatusUpdater.cc index 257cb1df27cf7..08db0dd5fa854 100644 --- a/src/tools/rbd_mirror/MirrorStatusUpdater.cc +++ b/src/tools/rbd_mirror/MirrorStatusUpdater.cc @@ -165,11 +165,14 @@ void MirrorStatusUpdater::finalize_shutdown(int r, Context* on_finish) { } template -bool MirrorStatusUpdater::exists(const std::string& global_image_id) { - dout(15) << "global_image_id=" << global_image_id << dendl; - +bool MirrorStatusUpdater::mirror_image_exists( + const std::string& global_image_id) const { std::unique_lock locker(m_lock); - return (m_global_image_status.count(global_image_id) > 0); + bool exists = (m_global_image_status.count(global_image_id) > 0); + + dout(15) << "global_image_id=" << global_image_id << ": " << exists << dendl; + + return exists; } template @@ -246,6 +249,127 @@ bool MirrorStatusUpdater::try_remove_mirror_image_status( return true; } +template +bool MirrorStatusUpdater::mirror_group_exists( + const std::string& global_group_id) const { + std::unique_lock locker(m_lock); + bool exists = (m_global_group_status.count(global_group_id) > 0); + + dout(15) << "global_group_id=" << global_group_id << ": " << exists << dendl; + + return exists; +} + +template +void MirrorStatusUpdater::set_mirror_group_status( + const std::string& global_group_id, + const cls::rbd::MirrorGroupSiteStatus& mirror_group_site_status, + bool immediate_update) { + dout(15) << "global_group_id=" << global_group_id << ", " + << "mirror_group_site_status=" << mirror_group_site_status << dendl; + + std::unique_lock locker(m_lock); + + m_global_group_status[global_group_id] = mirror_group_site_status; + if (immediate_update) { + m_update_global_group_ids.insert(global_group_id); + queue_update_task(std::move(locker)); + } +} + +template +void MirrorStatusUpdater::remove_mirror_group_status( + const std::string& global_group_id, Context* on_finish) { + if (try_remove_mirror_group_status(global_group_id, on_finish)) { + m_threads->work_queue->queue(on_finish, 0); + } +} + +template +bool MirrorStatusUpdater::try_remove_mirror_group_status( + const std::string& global_group_id, Context* on_finish) { + dout(15) << "global_group_id=" << global_group_id << dendl; + + std::unique_lock locker(m_lock); + if ((m_update_in_flight && + m_updating_global_group_ids.count(global_group_id) > 0) || + ((m_update_in_progress || m_update_requested) && + m_update_global_group_ids.count(global_group_id) > 0)) { + // if update is scheduled/in-progress, wait for it to complete + on_finish = new LambdaContext( + [this, global_group_id, on_finish](int r) { + if (try_remove_mirror_group_status(global_group_id, on_finish)) { + on_finish->complete(0); + } + }); + m_update_on_finish_ctxs.push_back(on_finish); + return false; + } + + m_global_group_status.erase(global_group_id); + return true; +} + +template +bool MirrorStatusUpdater::mirror_group_image_exists( + const std::string& global_group_id, int64_t image_pool_id, + const std::string& global_image_id) const { + std::unique_lock locker(m_lock); + auto it = m_global_group_status.find(global_group_id); + bool exists = (it != m_global_group_status.end() && + it->second.mirror_images.count({image_pool_id, + global_image_id}) > 0); + + dout(15) << "global_group_id=" << global_group_id + << ", image_pool_id=" << image_pool_id + << ", global_image_id=" << global_image_id + << ": " << exists << dendl; + return exists; +} + +template +void MirrorStatusUpdater::set_mirror_group_image_status( + const std::string& global_group_id, + int64_t image_pool_id, const std::string& global_image_id, + const cls::rbd::MirrorImageSiteStatus& mirror_image_site_status, + bool immediate_update) { + dout(15) << "global_group_id=" << global_group_id << ", " + << "image_pool_id=" << image_pool_id << ", " + << "global_image_id=" << global_image_id << ", " + << "mirror_image_site_status=" << mirror_image_site_status << dendl; + + std::unique_lock locker(m_lock); + + auto it = m_global_group_status.find(global_group_id); + ceph_assert(it != m_global_group_status.end()); + + it->second.mirror_images[{image_pool_id, global_image_id}] = + mirror_image_site_status; + if (immediate_update) { + m_update_global_group_ids.insert(global_group_id); + queue_update_task(std::move(locker)); + } +} + +template +void MirrorStatusUpdater::remove_mirror_group_image_status( + const std::string& global_group_id, int64_t image_pool_id, + const std::string& global_image_id, Context* on_finish) { + dout(15) << "global_group_id=" << global_group_id << ", " + << "image_pool_id=" << image_pool_id << ", " + << "global_image_id=" << global_image_id << dendl; + + std::unique_lock locker(m_lock); + + auto it = m_global_group_status.find(global_group_id); + ceph_assert(it != m_global_group_status.end()); + + it->second.mirror_images.erase({image_pool_id, global_image_id}); + m_update_global_group_ids.insert(global_group_id); + m_update_on_finish_ctxs.push_back(on_finish); + queue_update_task(std::move(locker)); +} + template void MirrorStatusUpdater::schedule_timer_task() { dout(10) << dendl; @@ -268,8 +392,11 @@ void MirrorStatusUpdater::handle_timer_task(int r) { schedule_timer_task(); std::unique_lock locker(m_lock); - for (auto& pair : m_global_image_status) { - m_update_global_image_ids.insert(pair.first); + for (auto &[global_image_id, _] : m_global_image_status) { + m_update_global_image_ids.insert(global_image_id); + } + for (auto&[global_group_id, _] : m_global_group_status) { + m_update_global_group_ids.insert(global_group_id); } queue_update_task(std::move(locker)); @@ -314,12 +441,16 @@ void MirrorStatusUpdater::update_task(int r) { std::swap(m_updating_global_image_ids, m_update_global_image_ids); auto updating_global_image_ids = m_updating_global_image_ids; auto global_image_status = m_global_image_status; + + std::swap(m_updating_global_group_ids, m_update_global_group_ids); + auto updating_global_group_ids = m_updating_global_group_ids; + auto global_group_status = m_global_group_status; locker.unlock(); Context* ctx = create_context_callback< MirrorStatusUpdater, &MirrorStatusUpdater::handle_update_task>(this); - if (updating_global_image_ids.empty()) { + if (updating_global_image_ids.empty() && updating_global_group_ids.empty()) { ctx->complete(0); return; } @@ -355,6 +486,33 @@ void MirrorStatusUpdater::update_task(int r) { aio_comp->release(); } + auto group_it = updating_global_group_ids.begin(); + while (group_it != updating_global_group_ids.end()) { + librados::ObjectWriteOperation op; + uint32_t op_count = 0; + + while (group_it != updating_global_group_ids.end() && + op_count < MAX_UPDATES_PER_OP) { + auto &global_group_id = *group_it; + ++group_it; + + auto status_it = global_group_status.find(global_group_id); + if (status_it == global_group_status.end()) { + continue; + } + + status_it->second.mirror_uuid = m_local_mirror_uuid; + librbd::cls_client::mirror_group_status_set(&op, global_group_id, + status_it->second); + ++op_count; + } + + auto aio_comp = create_rados_callback(gather->new_sub()); + int r = m_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op); + ceph_assert(r == 0); + aio_comp->release(); + } + gather->activate(); } @@ -378,6 +536,7 @@ void MirrorStatusUpdater::handle_update_task(int r) { m_update_in_flight = false; m_updating_global_image_ids.clear(); + m_updating_global_group_ids.clear(); if (m_update_requested) { m_update_requested = false; diff --git a/src/tools/rbd_mirror/MirrorStatusUpdater.h b/src/tools/rbd_mirror/MirrorStatusUpdater.h index 783b818fc56e8..ce1f9a1dd17a3 100644 --- a/src/tools/rbd_mirror/MirrorStatusUpdater.h +++ b/src/tools/rbd_mirror/MirrorStatusUpdater.h @@ -38,7 +38,7 @@ public: void init(Context* on_finish); void shut_down(Context* on_finish); - bool exists(const std::string& global_image_id); + bool mirror_image_exists(const std::string& global_image_id) const; void set_mirror_image_status( const std::string& global_image_id, const cls::rbd::MirrorImageSiteStatus& mirror_image_site_status, @@ -48,6 +48,26 @@ public: void remove_refresh_mirror_image_status(const std::string& global_image_id, Context* on_finish); + bool mirror_group_exists(const std::string& global_group_id) const; + void set_mirror_group_status( + const std::string& global_group_id, + const cls::rbd::MirrorGroupSiteStatus& mirror_group_site_status, + bool immediate_update); + void remove_mirror_group_status(const std::string& global_group_id, + Context* on_finish); + + bool mirror_group_image_exists(const std::string& global_group_id, + int64_t image_pool_id, + const std::string& global_image_id) const; + void set_mirror_group_image_status( + const std::string& global_group_id, + int64_t image_pool_id, const std::string& global_image_id, + const cls::rbd::MirrorImageSiteStatus& mirror_image_site_status, + bool immediate_update); + void remove_mirror_group_image_status( + const std::string& global_group_id, int64_t image_pool_id, + const std::string& global_image_id, Context* on_finish); + private: /** * @verbatim @@ -69,6 +89,9 @@ private: typedef std::set GlobalImageIds; typedef std::map GlobalImageStatus; + typedef std::set GlobalGroupIds; + typedef std::map + GlobalGroupStatus; librados::IoCtx m_io_ctx; Threads* m_threads; @@ -76,7 +99,7 @@ private: Context* m_timer_task = nullptr; - ceph::mutex m_lock; + mutable ceph::mutex m_lock; bool m_initialized = false; @@ -85,16 +108,23 @@ private: GlobalImageIds m_update_global_image_ids; GlobalImageStatus m_global_image_status; + GlobalGroupIds m_update_global_group_ids; + GlobalGroupStatus m_global_group_status; + bool m_update_in_progress = false; bool m_update_in_flight = false; bool m_update_requested = false; Contexts m_update_on_finish_ctxs; GlobalImageIds m_updating_global_image_ids; + GlobalImageIds m_updating_global_group_ids; bool try_remove_mirror_image_status(const std::string& global_image_id, bool queue_update, bool immediate_update, Context* on_finish); + bool try_remove_mirror_group_status(const std::string& global_image_id, + Context* on_finish); + void init_mirror_status_watcher(Context* on_finish); void handle_init_mirror_status_watcher(int r, Context* on_finish); diff --git a/src/tools/rbd_mirror/NamespaceReplayer.cc b/src/tools/rbd_mirror/NamespaceReplayer.cc index 545ee4d620b1d..8f7e09eea38c4 100644 --- a/src/tools/rbd_mirror/NamespaceReplayer.cc +++ b/src/tools/rbd_mirror/NamespaceReplayer.cc @@ -883,7 +883,6 @@ void NamespaceReplayer::handle_remove_group(const std::string &mirror_uuid, const std::string &global_group_id, const std::string &instance_id, Context* on_finish) { - ceph_assert(!mirror_uuid.empty()); dout(5) << "mirror_uuid=" << mirror_uuid << ", " << "global_group_id=" << global_group_id << ", " << "instance_id=" << instance_id << dendl; diff --git a/src/tools/rbd_mirror/PoolMetaCache.cc b/src/tools/rbd_mirror/PoolMetaCache.cc index 261802a55c534..6c54771d5fba7 100644 --- a/src/tools/rbd_mirror/PoolMetaCache.cc +++ b/src/tools/rbd_mirror/PoolMetaCache.cc @@ -15,6 +15,20 @@ namespace rbd { namespace mirror { +bool PoolMetaCache::local_pool_meta_exists(int64_t pool_id) const { + dout(15) << "pool_id=" << pool_id << dendl; + + std::unique_lock locker(m_lock); + return m_local_pool_metas.count(pool_id) > 0; +} + +bool PoolMetaCache::remote_pool_meta_exists(int64_t pool_id) const { + dout(15) << "pool_id=" << pool_id << dendl; + + std::unique_lock locker(m_lock); + return m_remote_pool_metas.count(pool_id) > 0; +} + int PoolMetaCache::get_local_pool_meta( int64_t pool_id, LocalPoolMeta* local_pool_meta) const { diff --git a/src/tools/rbd_mirror/PoolMetaCache.h b/src/tools/rbd_mirror/PoolMetaCache.h index f0440120fee06..53be6a2c46c88 100644 --- a/src/tools/rbd_mirror/PoolMetaCache.h +++ b/src/tools/rbd_mirror/PoolMetaCache.h @@ -20,6 +20,8 @@ public: PoolMetaCache(const PoolMetaCache&) = delete; PoolMetaCache& operator=(const PoolMetaCache&) = delete; + bool local_pool_meta_exists(int64_t pool_id) const; + bool remote_pool_meta_exists(int64_t pool_id) const; int get_local_pool_meta(int64_t pool_id, LocalPoolMeta* local_pool_meta) const; void set_local_pool_meta(int64_t pool_id, diff --git a/src/tools/rbd_mirror/Types.cc b/src/tools/rbd_mirror/Types.cc index 7853367c8bf32..558920d73c531 100644 --- a/src/tools/rbd_mirror/Types.cc +++ b/src/tools/rbd_mirror/Types.cc @@ -34,14 +34,20 @@ std::ostream& operator<<(std::ostream& os, std::ostream& operator<<(std::ostream& os, const RemotePoolMeta& remote_pool_meta) { - return os << "mirror_uuid=" << remote_pool_meta.mirror_uuid << ", " - "mirror_peer_uuid=" << remote_pool_meta.mirror_peer_uuid; + return os << "mirror_uuid=" << remote_pool_meta.mirror_uuid + << ", mirror_peer_uuid=" << remote_pool_meta.mirror_peer_uuid; } std::ostream& operator<<(std::ostream& os, const PeerSpec &peer) { return os << "uuid: " << peer.uuid - << " cluster: " << peer.cluster_name - << " client: " << peer.client_name; + << " cluster: " << peer.cluster_name + << " client: " << peer.client_name; +} + +std::ostream& operator<<(std::ostream& os, const GroupCtx &group_ctx) { + return os << "name: " << group_ctx.name + << ", group_id: " << group_ctx.group_id + << ", global_group_id: " << group_ctx.global_group_id; } } // namespace mirror diff --git a/src/tools/rbd_mirror/Types.h b/src/tools/rbd_mirror/Types.h index d2b4f57517a50..1a963f4160772 100644 --- a/src/tools/rbd_mirror/Types.h +++ b/src/tools/rbd_mirror/Types.h @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -173,6 +174,27 @@ struct PeerSpec { std::ostream& operator<<(std::ostream& os, const PeerSpec &peer); +struct GroupCtx { + std::string name; + std::string group_id; + std::string global_group_id; + bool primary = false; + mutable librados::IoCtx io_ctx; + + GroupCtx() { + } + + GroupCtx(const std::string &name, const std::string &group_id, + const std::string &global_group_id, bool primary, + librados::IoCtx &io_ctx_) + : name(name), group_id(group_id), global_group_id(global_group_id), + primary(primary) { + io_ctx.dup(io_ctx_); + } +}; + +std::ostream& operator<<(std::ostream& lhs, const GroupCtx &group_ctx); + } // namespace mirror } // namespace rbd diff --git a/src/tools/rbd_mirror/group_replayer/BootstrapRequest.cc b/src/tools/rbd_mirror/group_replayer/BootstrapRequest.cc new file mode 100644 index 0000000000000..83c6dad074f02 --- /dev/null +++ b/src/tools/rbd_mirror/group_replayer/BootstrapRequest.cc @@ -0,0 +1,1406 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "include/compat.h" +#include "BootstrapRequest.h" +#include "common/debug.h" +#include "common/dout.h" +#include "common/errno.h" +#include "cls/rbd/cls_rbd_client.h" +#include "librbd/internal.h" +#include "librbd/group/ListSnapshotsRequest.h" +#include "librbd/group/RemoveImageRequest.h" +#include "librbd/Utils.h" +#include "tools/rbd_mirror/ImageReplayer.h" +#include "tools/rbd_mirror/PoolMetaCache.h" +#include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/image_deleter/TrashMoveRequest.h" + +#define dout_context g_ceph_context +#define dout_subsys ceph_subsys_rbd_mirror +#undef dout_prefix +#define dout_prefix *_dout << "rbd::mirror::group_replayer::" \ + << "BootstrapRequest: " << this << " " \ + << __func__ << ": " + +namespace rbd { +namespace mirror { +namespace group_replayer { + +using librbd::util::create_context_callback; +using librbd::util::create_rados_callback; + +namespace { + +static const uint32_t MAX_RETURN = 1024; + +int get_last_mirror_snapshot_state( + const std::vector &snaps, + cls::rbd::MirrorSnapshotState *state) { + for (auto it = snaps.rbegin(); it != snaps.rend(); it++) { + auto ns = std::get_if( + &it->snapshot_namespace); + if (ns != nullptr) { + // XXXMG: check primary_mirror_uuid matches? + *state = ns->state; + return 0; + } + } + + return -ENOENT; +} + +} // anonymous namespace + +template +BootstrapRequest::BootstrapRequest( + Threads *threads, + librados::IoCtx &local_io_ctx, + librados::IoCtx &remote_io_ctx, + const std::string &global_group_id, + const std::string &local_mirror_uuid, + InstanceWatcher *instance_watcher, + MirrorStatusUpdater *local_status_updater, + MirrorStatusUpdater *remote_status_updater, + journal::CacheManagerHandler *cache_manager_handler, + PoolMetaCache *pool_meta_cache, + GroupCtx *local_group_ctx, + std::list *>> *image_replayers, + Context* on_finish) + : CancelableRequest("rbd::mirror::group_replayer::BootstrapRequest", + reinterpret_cast(local_io_ctx.cct()), + on_finish), + m_threads(threads), + m_local_io_ctx(local_io_ctx), + m_remote_io_ctx(remote_io_ctx), + m_global_group_id(global_group_id), + m_local_mirror_uuid(local_mirror_uuid), + m_instance_watcher(instance_watcher), + m_local_status_updater(local_status_updater), + m_remote_status_updater(remote_status_updater), + m_cache_manager_handler(cache_manager_handler), + m_pool_meta_cache(pool_meta_cache), + m_local_group_ctx(local_group_ctx), + m_image_replayers(image_replayers), + m_on_finish(on_finish) { + dout(10) << "global_group_id=" << m_global_group_id << dendl; +} + +template +void BootstrapRequest::send() { + get_remote_group_id(); +} + +template +void BootstrapRequest::cancel() { + dout(10) << dendl; + + m_canceled = true; +} + +template +std::string BootstrapRequest::prepare_non_primary_mirror_snap_name( + const std::string &global_group_id, + const std::string &snap_id) { + dout(5) << "global_group_id: " << global_group_id + << ", snap_id: " << snap_id << dendl; + std::stringstream ind_snap_name_stream; + ind_snap_name_stream << ".mirror.non-primary." + << global_group_id << "." << snap_id; + return ind_snap_name_stream.str(); +} + +template +bool BootstrapRequest::has_remote_image( + int64_t local_pool_id, const std::string &global_image_id) const { + + std::string pool_name; + int r = librados::Rados(m_local_io_ctx).pool_reverse_lookup(local_pool_id, + &pool_name); + if (r < 0) { + return false; + } + int64_t remote_pool_id = + librados::Rados(m_remote_io_ctx).pool_lookup(pool_name.c_str()); + if (remote_pool_id < 0) { + return false; + } + + return m_remote_images.count({remote_pool_id, global_image_id}) > 0; +} + +template +void BootstrapRequest::get_remote_group_id() { + dout(10) << dendl; + + librados::ObjectReadOperation op; + librbd::cls_client::mirror_group_get_group_id_start(&op, m_global_group_id); + m_out_bl.clear(); + auto comp = create_rados_callback< + BootstrapRequest, + &BootstrapRequest::handle_get_remote_group_id>(this); + + int r = m_remote_io_ctx.aio_operate(RBD_MIRRORING, comp, &op, &m_out_bl); + ceph_assert(r == 0); + comp->release(); +} + +template +void BootstrapRequest::handle_get_remote_group_id(int r) { + dout(10) << "r=" << r << dendl; + + if (m_canceled) { + finish(-ECANCELED); + return; + } + + if (r == -ENOENT) { + get_local_group_id(); + return; + } + + if (r == 0) { + auto iter = m_out_bl.cbegin(); + r = librbd::cls_client::mirror_group_get_group_id_finish( + &iter, &m_remote_group_id); + } + + if (r < 0) { + derr << "error getting remote group id: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + get_remote_group_name(); +} + +template +void BootstrapRequest::get_remote_group_name() { + dout(10) << dendl; + + librados::ObjectReadOperation op; + librbd::cls_client::dir_get_name_start(&op, m_remote_group_id); + m_out_bl.clear(); + auto comp = create_rados_callback< + BootstrapRequest, + &BootstrapRequest::handle_get_remote_group_name>(this); + + int r = m_remote_io_ctx.aio_operate(RBD_GROUP_DIRECTORY, comp, &op, + &m_out_bl); + ceph_assert(r == 0); + comp->release(); +} + +template +void BootstrapRequest::handle_get_remote_group_name(int r) { + dout(10) << "r=" << r << dendl; + + if (m_canceled) { + finish(-ECANCELED); + return; + } + + if (r == -ENOENT) { + get_local_group_id(); + return; + } + + if (r == 0) { + auto iter = m_out_bl.cbegin(); + r = librbd::cls_client::dir_get_name_finish(&iter, &m_group_name); + } + + if (r < 0) { + derr << "error getting remote group name: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + get_remote_mirror_group(); +} + +template +void BootstrapRequest::get_remote_mirror_group() { + dout(10) << dendl; + + librados::ObjectReadOperation op; + librbd::cls_client::mirror_group_get_start(&op, m_remote_group_id); + m_out_bl.clear(); + auto comp = create_rados_callback< + BootstrapRequest, + &BootstrapRequest::handle_get_remote_mirror_group>(this); + + int r = m_remote_io_ctx.aio_operate(RBD_MIRRORING, comp, &op, &m_out_bl); + ceph_assert(r == 0); + comp->release(); +} + +template +void BootstrapRequest::handle_get_remote_mirror_group(int r) { + dout(10) << "r=" << r << dendl; + + if (m_canceled) { + finish(-ECANCELED); + return; + } + + if (r == -ENOENT) { + m_remote_mirror_group.state = cls::rbd::MIRROR_GROUP_STATE_DISABLED; + get_local_group_id(); + return; + } + + if (r == 0) { + auto iter = m_out_bl.cbegin(); + r = librbd::cls_client::mirror_group_get_finish(&iter, + &m_remote_mirror_group); + } + + if (r < 0 && r != -ENOENT) { + derr << "error getting remote mirror group: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + if (m_remote_mirror_group.global_group_id != m_global_group_id) { + derr << "invalid global group id: " + << m_remote_mirror_group.global_group_id << dendl; + finish(-EINVAL); + return; + } + + list_remote_group_snapshots(); +} + +template +void BootstrapRequest::list_remote_group_snapshots() { + dout(10) << dendl; + + auto ctx = create_context_callback< + BootstrapRequest, + &BootstrapRequest::handle_list_remote_group_snapshots>(this); + + auto req = librbd::group::ListSnapshotsRequest::create(m_remote_io_ctx, + *m_remote_group_id, true, true, &m_remote_group_snaps, ctx); + req->send(); +} + +template +void BootstrapRequest::handle_list_remote_group_snapshots(int r) { + dout(10) << "r=" << r << dendl; + + if (m_canceled) { + finish(-ECANCELED); + return; + } + + if (r < 0) { + derr << "error listing remote mirror group snapshots: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + if (m_remote_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED) { + cls::rbd::MirrorSnapshotState state; + r = get_last_mirror_snapshot_state(m_remote_group_snaps, &state); + if (r == -ENOENT) { + derr << "failed to find remote mirror group snapshot" << dendl; + finish(-EINVAL); + return; + } + ceph_assert(r == 0); + m_remote_mirror_group_primary = (state == cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY); + } + + if (m_remote_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED && + m_remote_mirror_group_primary) { + list_remote_group(); + } else { + get_local_group_id(); + } +} + +template +void BootstrapRequest::list_remote_group() { + dout(10) << dendl; + + librados::ObjectReadOperation op; + cls::rbd::GroupImageSpec start_after; + if (!m_images.empty()) { + start_after = m_images.rbegin()->spec; + } + librbd::cls_client::group_image_list_start(&op, start_after, MAX_RETURN); + + auto comp = create_rados_callback< + BootstrapRequest, + &BootstrapRequest::handle_list_remote_group>(this); + m_out_bl.clear(); + int r = m_remote_io_ctx.aio_operate( + librbd::util::group_header_name(m_remote_group_id), comp, &op, &m_out_bl); + ceph_assert(r == 0); + comp->release(); +} + +template +void BootstrapRequest::handle_list_remote_group(int r) { + dout(10) << "r=" << r << dendl; + + if (m_canceled) { + finish(-ECANCELED); + return; + } + + std::vector images; + if (r == 0) { + auto iter = m_out_bl.cbegin(); + r = librbd::cls_client::group_image_list_finish(&iter, &images); + } + + if (r < 0 && r != -ENOENT) { + derr << "error listing remote group: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + m_images.insert(m_images.end(), images.begin(), images.end()); + + if (images.size() == MAX_RETURN) { + list_remote_group(); + return; + } + + get_remote_mirror_image(); +} + +template +void BootstrapRequest::get_remote_mirror_image() { + while (!m_images.empty() && + m_images.front().state != cls::rbd::GROUP_IMAGE_LINK_STATE_ATTACHED) { + dout(20) << "skip " << m_images.front().spec.pool_id << " " + << m_images.front().spec.image_id << dendl; + m_images.pop_front(); + } + + if (m_images.empty()) { + get_local_group_id(); + return; + } + + auto &spec = m_images.front().spec; + + dout(10) << spec.pool_id << " " << spec.image_id << dendl; + + if (!m_pool_meta_cache->remote_pool_meta_exists(spec.pool_id)) { + derr << "failed to find remote image pool in meta cache" << dendl; + finish(-ENOENT); + return; + } + + int r = librbd::util::create_ioctx(m_remote_io_ctx, "remote image pool", + spec.pool_id, {}, &m_image_io_ctx); + if (r < 0) { + derr << "failed to open remote image pool " << spec.pool_id << ": " + << cpp_strerror(r) << dendl; + finish(r); + return; + } + + librados::ObjectReadOperation op; + librbd::cls_client::mirror_image_get_start(&op, spec.image_id); + m_out_bl.clear(); + auto comp = create_rados_callback< + BootstrapRequest, + &BootstrapRequest::handle_get_remote_mirror_image>(this); + + r = m_image_io_ctx.aio_operate(RBD_MIRRORING, comp, &op, &m_out_bl); + ceph_assert(r == 0); + comp->release(); +} + +template +void BootstrapRequest::handle_get_remote_mirror_image(int r) { + dout(10) << "r=" << r << dendl; + + if (m_canceled) { + finish(-ECANCELED); + return; + } + + auto &spec = m_images.front().spec; + cls::rbd::MirrorImage mirror_image; + + if (r == 0) { + auto iter = m_out_bl.cbegin(); + r = librbd::cls_client::mirror_image_get_finish(&iter, &mirror_image); + } + + if (r < 0 && r != -ENOENT) { + derr << "error getting remote mirror image: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + m_remote_images.insert({spec.pool_id, mirror_image.global_image_id}); + + m_images.pop_front(); + + get_remote_mirror_image(); +} + +template +void BootstrapRequest::get_local_group_id() { + dout(10) << dendl; + + librados::ObjectReadOperation op; + librbd::cls_client::mirror_group_get_group_id_start(&op, m_global_group_id); + m_out_bl.clear(); + auto comp = create_rados_callback< + BootstrapRequest, + &BootstrapRequest::handle_get_local_group_id>(this); + + int r = m_local_io_ctx.aio_operate(RBD_MIRRORING, comp, &op, &m_out_bl); + ceph_assert(r == 0); + comp->release(); +} + +template +void BootstrapRequest::handle_get_local_group_id(int r) { + dout(10) << "r=" << r << dendl; + + if (m_canceled) { + finish(-ECANCELED); + return; + } + + if (r == -ENOENT && + m_remote_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED && + m_remote_mirror_group_primary) { + ceph_assert(!m_group_name.empty()); + get_local_group_id_by_name(); + return; + } + + if (r == 0) { + auto iter = m_out_bl.cbegin(); + r = librbd::cls_client::mirror_group_get_group_id_finish( + &iter, &m_local_group_id); + } + + if (r < 0) { + if (r != -ENOENT) { + derr << "error getting local group id: " << cpp_strerror(r) << dendl; + } else { + m_local_mirror_group.state = cls::rbd::MIRROR_GROUP_STATE_DISABLED; + r = 0; + } + finish(r); + return; + } + + get_local_group_name(); +} + +template +void BootstrapRequest::get_local_group_name() { + dout(10) << dendl; + + librados::ObjectReadOperation op; + librbd::cls_client::dir_get_name_start(&op, m_local_group_id); + m_out_bl.clear(); + auto comp = create_rados_callback< + BootstrapRequest, + &BootstrapRequest::handle_get_local_group_name>(this); + + int r = m_local_io_ctx.aio_operate(RBD_GROUP_DIRECTORY, comp, &op, + &m_out_bl); + ceph_assert(r == 0); + comp->release(); +} + +template +void BootstrapRequest::handle_get_local_group_name(int r) { + dout(10) << "r=" << r << dendl; + + if (m_canceled) { + finish(-ECANCELED); + return; + } + + std::string local_group_name; + + if (r == 0) { + auto iter = m_out_bl.cbegin(); + r = librbd::cls_client::dir_get_name_finish(&iter, &local_group_name); + } + + if (r < 0) { + derr << "error getting local group name: " << cpp_strerror(r) << dendl; + if (r == -ENOENT) { + r = -EEXIST; // split-brain + } + finish(r); + return; + } + + if (m_group_name.empty()) { + m_group_name = local_group_name; + } else if (m_group_name != local_group_name) { + // should never happen + derr << "local group name '" << local_group_name << "' does not match " + << "remote group name '" << m_group_name << "'" << dendl; + finish(-EEXIST); // split-brain + return; + } + + get_local_mirror_group(); +} + +template +void BootstrapRequest::get_local_group_id_by_name() { + dout(10) << dendl; + + librados::ObjectReadOperation op; + librbd::cls_client::dir_get_id_start(&op, m_group_name); + m_out_bl.clear(); + auto comp = create_rados_callback< + BootstrapRequest, + &BootstrapRequest::handle_get_local_group_id_by_name>(this); + + int r = m_local_io_ctx.aio_operate(RBD_GROUP_DIRECTORY, comp, &op, &m_out_bl); + ceph_assert(r == 0); + comp->release(); +} + +template +void BootstrapRequest::handle_get_local_group_id_by_name(int r) { + dout(10) << "r=" << r << dendl; + + if (m_canceled) { + finish(-ECANCELED); + return; + } + + if (r == -ENOENT) { + create_local_group_id(); + return; + } + + if (r == 0) { + auto iter = m_out_bl.cbegin(); + r = librbd::cls_client::dir_get_id_finish(&iter, &m_local_group_id); + } + + if (r < 0) { + derr << "error getting local group id: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + m_local_group_id_by_name = true; + get_local_mirror_group(); +} + +template +void BootstrapRequest::get_local_mirror_group() { + dout(10) << dendl; + + librados::ObjectReadOperation op; + librbd::cls_client::mirror_group_get_start(&op, m_local_group_id); + m_out_bl.clear(); + auto comp = create_rados_callback< + BootstrapRequest, + &BootstrapRequest::handle_get_local_mirror_group>(this); + + int r = m_local_io_ctx.aio_operate(RBD_MIRRORING, comp, &op, &m_out_bl); + ceph_assert(r == 0); + comp->release(); +} + +template +void BootstrapRequest::handle_get_local_mirror_group(int r) { + dout(10) << "r=" << r << dendl; + + if (m_canceled) { + finish(-ECANCELED); + return; + } + + if (r == 0) { + auto iter = m_out_bl.cbegin(); + r = librbd::cls_client::mirror_group_get_finish(&iter, + &m_local_mirror_group); + } + + if (r == -ENOENT) { + if (m_local_group_id_by_name) { + derr << "local group is not mirrored" << dendl; + finish(-EINVAL); + return; + } + if (m_remote_mirror_group.state != cls::rbd::MIRROR_GROUP_STATE_ENABLED || + !m_remote_mirror_group_primary) { + derr << "can't find primary for group: " << m_group_name << dendl; + finish(-EEXIST); // split-brain + return; + } + r = 0; + } + + if (r < 0) { + derr << "error getting local mirror group: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + dout(20) << m_local_mirror_group << dendl; + + list_local_group_snapshots(); +} + +template +void BootstrapRequest::list_local_group_snapshots() { + dout(10) << dendl; + + auto ctx = create_context_callback< + BootstrapRequest, + &BootstrapRequest::handle_list_local_group_snapshots>(this); + + auto req = librbd::group::ListSnapshotsRequest::create(m_local_io_ctx, + m_local_group_id, true, true, &m_local_group_snaps, ctx); + req->send(); +} + +template +void BootstrapRequest::handle_list_local_group_snapshots(int r) { + dout(10) << "r=" << r << dendl; + + if (m_canceled) { + finish(-ECANCELED); + return; + } + + if (r < 0) { + derr << "error listing local mirror group snapshots: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + if (m_local_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED) { + cls::rbd::MirrorSnapshotState state; + r = get_last_mirror_snapshot_state(m_local_group_snaps, &state); + if (r == -ENOENT) { + derr << "failed to find local mirror group snapshot" << dendl; + finish(-EINVAL); + return; + } + ceph_assert(r == 0); + m_local_mirror_group_primary = (state == cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY); + } + + if (m_remote_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED) { + if (m_remote_mirror_group_primary) { + if (m_local_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED && + m_local_mirror_group_primary) { + derr << "both remote and local groups are primary" << dendl; + finish(-EEXIST); // split-brain + return; + } + } else if (m_local_mirror_group.state != cls::rbd::MIRROR_GROUP_STATE_ENABLED || + !m_local_mirror_group_primary) { + derr << "both remote and local groups are not primary" << dendl; + finish(-EREMOTEIO); + return; + } + } else if (m_local_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED && + !m_local_mirror_group_primary) { + // trigger group removal + m_local_mirror_group.state = cls::rbd::MIRROR_GROUP_STATE_DISABLED; + } + + list_local_group(); +} + +template +void BootstrapRequest::list_local_group() { + dout(10) << dendl; + + librados::ObjectReadOperation op; + cls::rbd::GroupImageSpec start_after; + if (!m_images.empty()) { + start_after = m_images.rbegin()->spec; + } + librbd::cls_client::group_image_list_start(&op, start_after, MAX_RETURN); + auto comp = create_rados_callback< + BootstrapRequest, + &BootstrapRequest::handle_list_local_group>(this); + m_out_bl.clear(); + int r = m_local_io_ctx.aio_operate( + librbd::util::group_header_name(m_local_group_id), comp, &op, &m_out_bl); + ceph_assert(r == 0); + comp->release(); +} + +template +void BootstrapRequest::handle_list_local_group(int r) { + dout(10) << "r=" << r << dendl; + + if (m_canceled) { + finish(-ECANCELED); + return; + } + + std::vector images; + if (r == 0) { + auto iter = m_out_bl.cbegin(); + r = librbd::cls_client::group_image_list_finish(&iter, &images); + } + + if (r < 0 && r != -ENOENT) { + derr << "error listing local group: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + m_images.insert(m_images.end(), images.begin(), images.end()); + + if (images.size() == MAX_RETURN) { + list_local_group(); + return; + } + + get_local_mirror_image(); +} + +template +void BootstrapRequest::get_local_mirror_image() { + if (m_images.empty()) { + remove_local_image_from_group(); + return; + } + + auto &spec = m_images.front().spec; + + dout(10) << spec.pool_id << " " << spec.image_id << dendl; + + int r = librbd::util::create_ioctx(m_local_io_ctx, "local image pool", + spec.pool_id, {}, &m_image_io_ctx); + if (r < 0) { + derr << "failed to open local image pool " << spec.pool_id << ": " + << cpp_strerror(r) << dendl; + finish(r); + return; + } + + librados::ObjectReadOperation op; + librbd::cls_client::mirror_image_get_start(&op, spec.image_id); + m_out_bl.clear(); + auto comp = create_rados_callback< + BootstrapRequest, + &BootstrapRequest::handle_get_local_mirror_image>(this); + + r = m_image_io_ctx.aio_operate(RBD_MIRRORING, comp, &op, &m_out_bl); + ceph_assert(r == 0); + comp->release(); +} + +template +void BootstrapRequest::handle_get_local_mirror_image(int r) { + dout(10) << "r=" << r << dendl; + + if (m_canceled) { + finish(-ECANCELED); + return; + } + + auto &spec = m_images.front().spec; + cls::rbd::MirrorImage mirror_image; + + if (r == 0) { + auto iter = m_out_bl.cbegin(); + r = librbd::cls_client::mirror_image_get_finish(&iter, &mirror_image); + } + + if (r < 0) { + if (r != -ENOENT) { + derr << "error getting local mirror image: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + } else { + if (m_local_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED && + m_local_mirror_group_primary) { + dout(10) << "add primary to replayer queue: " << spec.pool_id << " " + << spec.image_id << " " << mirror_image.global_image_id + << dendl; + m_local_images.insert({spec.pool_id, mirror_image.global_image_id}); + } else if (m_remote_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED && + m_remote_mirror_group_primary && + m_local_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED && + m_local_mirror_group.global_group_id == m_global_group_id && + has_remote_image(spec.pool_id, mirror_image.global_image_id)) { + dout(10) << "add secondary to replayer queue: " << spec.pool_id << " " + << spec.image_id << " " << mirror_image.global_image_id + << dendl; + m_local_images.insert({spec.pool_id, mirror_image.global_image_id}); + } else { + dout(10) << "add to trash queue: " << spec.pool_id << " " + << spec.image_id << " " << mirror_image.global_image_id + << dendl; + m_local_trash_images[{spec.pool_id, mirror_image.global_image_id}] = + spec.image_id; + } + } + + m_images.pop_front(); + + get_local_mirror_image(); +} + +template +void BootstrapRequest::remove_local_image_from_group() { + if (m_local_trash_images.empty()) { + disable_local_mirror_group(); + return; + } + + auto &[pool_id, global_image_id] = m_local_trash_images.begin()->first; + auto &image_id = m_local_trash_images.begin()->second; + + dout(10) << "pool_id=" << pool_id << ", image_id=" << image_id << dendl; + + int r = librbd::util::create_ioctx(m_local_io_ctx, "local image pool", + pool_id, {}, &m_image_io_ctx); + if (r < 0) { + derr << "failed to open local image pool " << pool_id << ": " + << cpp_strerror(r) << dendl; + handle_remove_local_image_from_group(-ENOENT); + return; + } + + auto ctx = create_context_callback< + BootstrapRequest, + &BootstrapRequest::handle_remove_local_image_from_group>(this); + + auto req = librbd::group::RemoveImageRequest::create( + m_local_io_ctx, m_local_group_id, m_image_io_ctx, image_id, ctx); + req->send(); +} + +template +void BootstrapRequest::handle_remove_local_image_from_group(int r) { + dout(10) << "r=" << r << dendl; + + if (r < 0 && r != -ENOENT) { + derr << "error removing mirror image from group: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + move_local_image_to_trash(); +} + +template +void BootstrapRequest::move_local_image_to_trash() { + ceph_assert(!m_local_trash_images.empty()); + auto &[pool_id, global_image_id] = m_local_trash_images.begin()->first; + + dout(10) << "pool_id=" << pool_id << ", global_image_id=" << global_image_id + << dendl; + + int r = librbd::util::create_ioctx(m_local_io_ctx, "local image pool", + pool_id, {}, &m_image_io_ctx); + if (r < 0) { + derr << "failed to open local image pool " << pool_id << ": " + << cpp_strerror(r) << dendl; + handle_move_local_image_to_trash(-ENOENT); + return; + } + + auto ctx = create_context_callback< + BootstrapRequest, + &BootstrapRequest::handle_move_local_image_to_trash>(this); + + auto req = image_deleter::TrashMoveRequest::create( + m_image_io_ctx, global_image_id, false, m_threads->work_queue, ctx); + req->send(); +} + +template +void BootstrapRequest::handle_move_local_image_to_trash(int r) { + dout(10) << "r=" << r << dendl; + + if (m_canceled) { + finish(-ECANCELED); + return; + } + + if (r < 0 && r != -ENOENT) { + derr << "error moving mirror image to trash: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + m_local_trash_images.erase(m_local_trash_images.begin()); + + remove_local_image_from_group(); +} + +template +void BootstrapRequest::disable_local_mirror_group() { + if (m_local_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED && + m_local_mirror_group.global_group_id == m_global_group_id) { + finish(0); + return; + } + + dout(10) << dendl; + + librados::ObjectWriteOperation op; + m_local_mirror_group.state = cls::rbd::MIRROR_GROUP_STATE_DISABLING; + librbd::cls_client::mirror_group_set(&op, m_local_group_id, + m_local_mirror_group); + + auto comp = create_rados_callback< + BootstrapRequest, + &BootstrapRequest::handle_disable_local_mirror_group>(this); + + int r = m_local_io_ctx.aio_operate(RBD_MIRRORING, comp, &op); + ceph_assert(r == 0); + comp->release(); +} + +template +void BootstrapRequest::handle_disable_local_mirror_group(int r) { + dout(10) << "r=" << r << dendl; + + if (r < 0 && r != -ENOENT) { + derr << "error disabling local mirror group: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + remove_local_mirror_group(); +} + +template +void BootstrapRequest::remove_local_mirror_group() { + if (m_local_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED && + m_local_mirror_group.global_group_id == m_global_group_id) { + finish(0); + return; + } + + dout(10) << dendl; + + librados::ObjectWriteOperation op; + librbd::cls_client::mirror_group_remove(&op, m_local_group_id); + + auto comp = create_rados_callback< + BootstrapRequest, + &BootstrapRequest::handle_remove_local_mirror_group>(this); + + int r = m_local_io_ctx.aio_operate(RBD_MIRRORING, comp, &op); + ceph_assert(r == 0); + comp->release(); +} + +template +void BootstrapRequest::handle_remove_local_mirror_group(int r) { + dout(10) << "r=" << r << dendl; + + if (r < 0 && r != -ENOENT) { + derr << "error removing local mirror group: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + m_local_mirror_group.state = cls::rbd::MIRROR_GROUP_STATE_DISABLED; + + if (m_remote_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED && + m_remote_mirror_group_primary) { + create_local_mirror_group(); + } else { + remove_local_group(); + } +} + +template +void BootstrapRequest::remove_local_group() { + dout(10) << m_group_name << " " << m_local_group_id << dendl; + + ceph_assert(!m_local_group_id.empty()); + ceph_assert(!m_group_name.empty()); + + librados::ObjectWriteOperation op; + op.remove(); + auto comp = create_rados_callback< + BootstrapRequest, + &BootstrapRequest::handle_remove_local_group>(this); + + int r = m_local_io_ctx.aio_operate( + librbd::util::group_header_name(m_local_group_id), comp, &op); + ceph_assert(r == 0); + comp->release(); +} + +template +void BootstrapRequest::handle_remove_local_group(int r) { + dout(10) << "r=" << r << dendl; + + if (r < 0 && r != -ENOENT) { + derr << "error removing local group: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + remove_local_group_id(); +} + +template +void BootstrapRequest::remove_local_group_id() { + dout(10) << dendl; + + librados::ObjectWriteOperation op; + librbd::cls_client::group_dir_remove(&op, m_group_name, m_local_group_id); + + auto comp = create_rados_callback< + BootstrapRequest, + &BootstrapRequest::handle_remove_local_group_id>(this); + + int r = m_local_io_ctx.aio_operate(RBD_GROUP_DIRECTORY, comp, &op); + ceph_assert(r == 0); + comp->release(); +} + +template +void BootstrapRequest::handle_remove_local_group_id(int r) { + dout(10) << "r=" << r << dendl; + + if (r < 0 && r != -ENOENT) { + derr << "error removing local group id: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + finish(0); +} + + +template +void BootstrapRequest::create_local_group_id() { + dout(10) << dendl; + + m_local_group_id = librbd::util::generate_uuid(m_local_io_ctx); + + librados::ObjectWriteOperation op; + librbd::cls_client::group_dir_add(&op, m_group_name, m_local_group_id); + + auto comp = create_rados_callback< + BootstrapRequest, + &BootstrapRequest::handle_create_local_group_id>(this); + + int r = m_local_io_ctx.aio_operate(RBD_GROUP_DIRECTORY, comp, &op); + ceph_assert(r == 0); + comp->release(); +} + +template +void BootstrapRequest::handle_create_local_group_id(int r) { + dout(10) << "r=" << r << dendl; + + if (r < 0) { + derr << "error creating local group id: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + create_local_group(); +} + +template +void BootstrapRequest::create_local_group() { + dout(10) << dendl; + + librados::ObjectWriteOperation op; + op.create(true); + auto comp = create_rados_callback< + BootstrapRequest, + &BootstrapRequest::handle_create_local_group>(this); + + int r = m_local_io_ctx.aio_operate( + librbd::util::group_header_name(m_local_group_id), comp, &op); + ceph_assert(r == 0); + comp->release(); +} + +template +void BootstrapRequest::handle_create_local_group(int r) { + dout(10) << "r=" << r << dendl; + + if (r < 0) { + derr << "error creating local group: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + create_local_mirror_group(); +} + +template +void BootstrapRequest::create_local_mirror_group() { + dout(10) << dendl; + + ceph_assert( + m_remote_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED && + m_remote_mirror_group_primary); + + librados::ObjectWriteOperation op; + m_local_mirror_group = {m_global_group_id, + m_remote_mirror_group.mirror_image_mode, + cls::rbd::MIRROR_GROUP_STATE_ENABLED}; + librbd::cls_client::mirror_group_set(&op, m_local_group_id, + m_local_mirror_group); + auto comp = create_rados_callback< + BootstrapRequest, + &BootstrapRequest::handle_create_local_mirror_group>(this); + + int r = m_local_io_ctx.aio_operate(RBD_MIRRORING, comp, &op); + ceph_assert(r == 0); + comp->release(); +} + +template +void BootstrapRequest::handle_create_local_mirror_group(int r) { + dout(10) << "r=" << r << dendl; + + if (r < 0) { + derr << "error creating local mirror group: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + create_local_non_primary_group_snapshot(); +} + +template +void BootstrapRequest::create_local_non_primary_group_snapshot() { + dout(10) << dendl; + + RemotePoolMeta remote_pool_meta; + int r = m_pool_meta_cache->get_remote_pool_meta(m_remote_io_ctx.get_id(), + &remote_pool_meta); + if (r < 0 || remote_pool_meta.mirror_peer_uuid.empty()) { + derr << "failed to retrieve mirror peer uuid from remote image pool" + << dendl; + finish(r < 0 ? r : -EINVAL); + return; + } + + librados::ObjectWriteOperation op; + std::string group_snap_id = librbd::util::generate_image_id(m_local_io_ctx); + cls::rbd::GroupSnapshot group_snap{ + group_snap_id, + cls::rbd::MirrorGroupSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, + {}, remote_pool_meta.mirror_peer_uuid, {}}, + prepare_non_primary_mirror_snap_name(m_global_group_id, group_snap_id), + cls::rbd::GROUP_SNAPSHOT_STATE_INCOMPLETE}; + librbd::cls_client::group_snap_set(&op, group_snap); + group_snap.state = cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE; + librbd::cls_client::group_snap_set(&op, group_snap); + + auto comp = create_rados_callback< + BootstrapRequest, + &BootstrapRequest::handle_create_local_non_primary_group_snapshot>(this); + + r = m_local_io_ctx.aio_operate( + librbd::util::group_header_name(*m_local_group_id), comp, &op); + ceph_assert(r == 0); + comp->release(); +} + +template +void BootstrapRequest::handle_create_local_non_primary_group_snapshot(int r) { + dout(10) << "r=" << r << dendl; + + if (r < 0) { + derr << "error creating local non-primary group snapshot: " + << cpp_strerror(r) << dendl; + finish(r); + return; + } + + finish(0); +} + +template +void BootstrapRequest::finish(int r) { + dout(10) << "r=" << r << dendl; + + ceph_assert(r != -ENOENT); + if (r == 0) { + if (m_local_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_DISABLED) { + r = -ENOENT; + } else { + *m_local_group_ctx = {m_group_name, m_local_group_id, m_global_group_id, + m_local_mirror_group_primary, m_local_io_ctx}; + r = create_replayers(); + } + } + + m_on_finish->complete(r); +} + +template +int BootstrapRequest::create_replayers() { + dout(10) << dendl; + + int r = 0; + if (m_remote_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED && + m_remote_mirror_group_primary) { + for (auto &[remote_pool_id, global_image_id] : m_remote_images) { + m_image_replayers->emplace_back(librados::IoCtx(), nullptr); + auto &local_io_ctx = m_image_replayers->back().first; + auto &image_replayer = m_image_replayers->back().second; + + RemotePoolMeta remote_pool_meta; + r = m_pool_meta_cache->get_remote_pool_meta(remote_pool_id, + &remote_pool_meta); + if (r < 0 || remote_pool_meta.mirror_peer_uuid.empty()) { + derr << "failed to retrieve mirror peer uuid from remote image pool" + << dendl; + r = -ENOENT; + break; + } + + librados::IoCtx remote_io_ctx; + r = librbd::util::create_ioctx(m_remote_io_ctx, "remote image pool", + remote_pool_id, {}, &remote_io_ctx); + if (r < 0) { + derr << "failed to open remote image pool " << remote_pool_id << ": " + << cpp_strerror(r) << dendl; + if (r == -ENOENT) { + r = -EINVAL; + } + break; + } + + int64_t local_pool_id = librados::Rados(m_local_io_ctx).pool_lookup( + remote_io_ctx.get_pool_name().c_str()); + + LocalPoolMeta local_pool_meta; + r = m_pool_meta_cache->get_local_pool_meta(local_pool_id, + &local_pool_meta); + if (r < 0 || local_pool_meta.mirror_uuid.empty()) { + if (r == 0 || r == -ENOENT) { + r = -EINVAL; + } + derr << "failed to retrieve mirror uuid from local image pool" << dendl; + break; + } + + r = librbd::util::create_ioctx(m_local_io_ctx, "local image pool", + local_pool_id, {}, &local_io_ctx); + if (r < 0) { + derr << "failed to open local image pool " << local_pool_id << ": " + << cpp_strerror(r) << dendl; + if (r == -ENOENT) { + r = -EINVAL; + } + break; + } + + image_replayer = ImageReplayer::create( + local_io_ctx, m_local_group_ctx, local_pool_meta.mirror_uuid, + global_image_id, m_threads, m_instance_watcher, m_local_status_updater, + m_cache_manager_handler, m_pool_meta_cache); + + // TODO only a single peer is currently supported + image_replayer->add_peer({local_pool_meta.mirror_uuid, remote_io_ctx, + remote_pool_meta, m_remote_status_updater}); + } + } else if (m_local_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED && + m_local_mirror_group_primary) { + for (auto &[local_pool_id, global_image_id] : m_local_images) { + m_image_replayers->emplace_back(librados::IoCtx(), nullptr); + auto &local_io_ctx = m_image_replayers->back().first; + auto &image_replayer = m_image_replayers->back().second; + + LocalPoolMeta local_pool_meta; + r = m_pool_meta_cache->get_local_pool_meta(local_pool_id, + &local_pool_meta); + if (r < 0 || local_pool_meta.mirror_uuid.empty()) { + if (r == 0 || r == -ENOENT) { + r = -EINVAL; + } + derr << "failed to retrieve mirror uuid from local image pool" << dendl; + break; + } + + r = librbd::util::create_ioctx(m_local_io_ctx, "local image pool", + local_pool_id, {}, &local_io_ctx); + if (r < 0) { + derr << "failed to open local image pool " << local_pool_id << ": " + << cpp_strerror(r) << dendl; + if (r == -ENOENT) { + r = -EINVAL; + } + break; + } + + int64_t remote_pool_id = librados::Rados(m_remote_io_ctx).pool_lookup( + local_io_ctx.get_pool_name().c_str()); + + RemotePoolMeta remote_pool_meta; + r = m_pool_meta_cache->get_remote_pool_meta(remote_pool_id, + &remote_pool_meta); + if (r < 0 || remote_pool_meta.mirror_peer_uuid.empty()) { + derr << "failed to retrieve mirror peer uuid from remote image pool" + << dendl; + r = -ENOENT; + break; + } + + librados::IoCtx remote_io_ctx; + r = librbd::util::create_ioctx(m_remote_io_ctx, "remote image pool", + remote_pool_id, {}, &remote_io_ctx); + if (r < 0) { + derr << "failed to open remote image pool " << remote_pool_id << ": " + << cpp_strerror(r) << dendl; + if (r == -ENOENT) { + r = -EINVAL; + } + break; + } + + image_replayer = ImageReplayer::create( + local_io_ctx, m_local_group_ctx, local_pool_meta.mirror_uuid, + global_image_id, m_threads, m_instance_watcher, m_local_status_updater, + m_cache_manager_handler, m_pool_meta_cache); + + // TODO only a single peer is currently supported + image_replayer->add_peer({local_pool_meta.mirror_uuid, remote_io_ctx, + remote_pool_meta, m_remote_status_updater}); + } + } else { + ceph_abort(); + } + + if (r < 0) { + for (auto &[_, image_replayer] : *m_image_replayers) { + delete image_replayer; + } + m_image_replayers->clear(); + return r; + } + + return 0; +} + +} // namespace group_replayer +} // namespace mirror +} // namespace rbd + +template class rbd::mirror::group_replayer::BootstrapRequest; diff --git a/src/tools/rbd_mirror/group_replayer/BootstrapRequest.h b/src/tools/rbd_mirror/group_replayer/BootstrapRequest.h new file mode 100644 index 0000000000000..f99ccdce2c03d --- /dev/null +++ b/src/tools/rbd_mirror/group_replayer/BootstrapRequest.h @@ -0,0 +1,272 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef RBD_MIRROR_GROUP_REPLAYER_BOOTSTRAP_REQUEST_H +#define RBD_MIRROR_GROUP_REPLAYER_BOOTSTRAP_REQUEST_H + +#include "include/rados/librados.hpp" +#include "cls/rbd/cls_rbd_types.h" +#include "tools/rbd_mirror/CancelableRequest.h" + +#include +#include +#include +#include + + +class Context; + +namespace journal { struct CacheManagerHandler; } +namespace librbd { class ImageCtx; } + +namespace rbd { +namespace mirror { + +struct GroupCtx; +template struct ImageReplayer; +template struct InstanceWatcher; +template struct MirrorStatusUpdater; +struct PoolMetaCache; +template struct Threads; + +namespace group_replayer { + +template +class BootstrapRequest : public CancelableRequest { +public: + static BootstrapRequest *create( + Threads *threads, + librados::IoCtx &local_io_ctx, + librados::IoCtx &remote_io_ctx, + const std::string &global_group_id, + const std::string &local_mirror_uuid, + InstanceWatcher *instance_watcher, + MirrorStatusUpdater *local_status_updater, + MirrorStatusUpdater *remote_status_updater, + journal::CacheManagerHandler *cache_manager_handler, + PoolMetaCache *pool_meta_cache, + GroupCtx *local_group_ctx, + std::list *>> *image_replayers, + Context *on_finish) { + return new BootstrapRequest( + threads, local_io_ctx, remote_io_ctx, global_group_id, local_mirror_uuid, + instance_watcher, local_status_updater, remote_status_updater, + cache_manager_handler, pool_meta_cache, local_group_ctx, image_replayers, + on_finish); + } + + BootstrapRequest( + Threads *threads, + librados::IoCtx &local_io_ctx, + librados::IoCtx &remote_io_ctx, + const std::string &global_group_id, + const std::string &local_mirror_uuid, + InstanceWatcher *instance_watcher, + MirrorStatusUpdater *local_status_updater, + MirrorStatusUpdater *remote_status_updater, + journal::CacheManagerHandler *cache_manager_handler, + PoolMetaCache *pool_meta_cache, + GroupCtx *local_group_ctx, + std::list *>> *image_replayers, + Context* on_finish); + + void send() override; + void cancel() override; + std::string prepare_non_primary_mirror_snap_name( + const std::string &global_group_id, const std::string &snap_id); + +private: + /** + * @verbatim + * + * + * | + * v + * GET_REMOTE_GROUP_ID * * * * * * * * * * * + * | (noent) * + * v v + * GET_REMOTE_GROUP_NAME * * * * * * * * * * + * | (noent) * + * v v + * GET_REMOTE_MIRROR_GROUP * * * * * * * * * + * | (noent or not primary) * + * v v + * LIST_REMOTE_GROUP_SNAPSHOTS * * * * * * * + * | (noent) * + * v v + * LIST_REMOTE_GROUP * * * * * * * * * * * * + * | (repeat if neeeded) (noent) * + * v v + * GET_REMOTE_MIRROR_IMAGE * * * * * * * * * + * | (repeat for every image) (noent) * + * | v + * |/< * * * * * * * * * * * * * * * * * * + * v + * GET_LOCAL_GROUP_ID * * * * * * * * * * * * + * | (noent) * + * v * + * GET_LOCAL_GROUP_NAME * + * | * + * v v + * GET_LOCAL_MIRROR_GROUP <------------- GET_LOCAL_GROUP_ID_BY_NAME + * | * (noent) + * v (noent) * + * LIST_LOCAL_GROUP_SNAPSHOTS * * * * + * | * * + * v (noent) * * + * LIST_LOCAL_GROUP * * * * * * * * * + * | (repeat if neeeded) * * + * v * * + * GET_LOCAL_MIRROR_IMAGE * * + * | (repeat for every image) * * + * v * * + * REMOVE_LOCAL_IMAGE_FROM_GROUP * * + * | ^ * * + * v | * v + * MOVE_LOCAL_IMAGE_TO_TRASH -/ * CREATE_LOCAL_GROUP_ID + * | (repeat for every * | + * | stale image) * v + * |\----\ * * > CREATE_LOCAL_GROUP + * | | (if stale | + * | v or removing) | + * | DISABLE_LOCAL_MIRROR_GROUP | + * | | | + * | v v + * | REMOVE_LOCAL_MIRROR_GROUP ----> CREATE_LOCAL_MIRROR_GROUP + * | | (if stale) | + * | v v + * | REMOVE_LOCAL_GROUP CREATE_LOCAL_NON_PRIMARY_GROUP_SNAPSHOT + * | | (if removing) | + * | v | + * | REMOVE_LOCAL_GROUP_ID | + * | | (if removing) | + * v v | + * <-------------------------------/ + * + * @endverbatim + */ + + typedef std::pair GlobalImageId; + + Threads* m_threads; + librados::IoCtx &m_local_io_ctx; + librados::IoCtx &m_remote_io_ctx; + std::string m_global_group_id; + std::string m_local_mirror_uuid; + InstanceWatcher *m_instance_watcher; + MirrorStatusUpdater *m_local_status_updater; + MirrorStatusUpdater *m_remote_status_updater; + journal::CacheManagerHandler *m_cache_manager_handler; + PoolMetaCache *m_pool_meta_cache; + GroupCtx *m_local_group_ctx; + std::list *>> *m_image_replayers; + Context *m_on_finish; + + std::atomic m_canceled = false; + + std::string m_group_name; + std::string m_remote_group_id; + std::string m_local_group_id; + bool m_local_group_id_by_name = false; + cls::rbd::MirrorGroup m_remote_mirror_group; + cls::rbd::MirrorGroup m_local_mirror_group; + std::vector m_remote_group_snaps; + std::vector m_local_group_snaps; + bool m_remote_mirror_group_primary = false; + bool m_local_mirror_group_primary = false; + std::list m_images; + librados::IoCtx m_image_io_ctx; + + std::set m_remote_images; + std::set m_local_images; + std::map m_local_trash_images; + + bufferlist m_out_bl; + + bool has_remote_image(int64_t local_pool_id, + const std::string &global_image_id) const; + + void get_remote_group_id(); + void handle_get_remote_group_id(int r); + + void get_remote_group_name(); + void handle_get_remote_group_name(int r); + + void get_remote_mirror_group(); + void handle_get_remote_mirror_group(int r); + + void get_remote_mirror_image(); + void handle_get_remote_mirror_image(int r); + + void list_remote_group_snapshots(); + void handle_list_remote_group_snapshots(int r); + + void list_remote_group(); + void handle_list_remote_group(int r); + + void get_local_group_id(); + void handle_get_local_group_id(int r); + + void get_local_group_id_by_name(); + void handle_get_local_group_id_by_name(int r); + + void get_local_group_name(); + void handle_get_local_group_name(int r); + + void get_local_mirror_group(); + void handle_get_local_mirror_group(int r); + + void list_local_group_snapshots(); + void handle_list_local_group_snapshots(int r); + + void list_local_group(); + void handle_list_local_group(int r); + + void get_local_mirror_image(); + void handle_get_local_mirror_image(int r); + + void remove_local_image_from_group(); + void handle_remove_local_image_from_group(int r); + + void move_local_image_to_trash(); + void handle_move_local_image_to_trash(int r); + + void remove_local_mirror_image(); + void handle_remove_local_mirror_image(int r); + + void disable_local_mirror_group(); + void handle_disable_local_mirror_group(int r); + + void remove_local_mirror_group(); + void handle_remove_local_mirror_group(int r); + + void remove_local_group(); + void handle_remove_local_group(int r); + + void remove_local_group_id(); + void handle_remove_local_group_id(int r); + + void create_local_group_id(); + void handle_create_local_group_id(int r); + + void create_local_group(); + void handle_create_local_group(int r); + + void create_local_mirror_group(); + void handle_create_local_mirror_group(int r); + + void create_local_non_primary_group_snapshot(); + void handle_create_local_non_primary_group_snapshot(int r); + + void finish(int r); + + int create_replayers(); +}; + +} // namespace group_replayer +} // namespace mirror +} // namespace rbd + +extern template class rbd::mirror::group_replayer::BootstrapRequest; + +#endif // RBD_MIRROR_GROUP_REPLAYER_BOOTSTRAP_REQUEST_H diff --git a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc index bda5b5f9bd760..f24058325d915 100644 --- a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc +++ b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc @@ -47,6 +47,7 @@ BootstrapRequest::BootstrapRequest( Threads* threads, librados::IoCtx& local_io_ctx, librados::IoCtx& remote_io_ctx, + GroupCtx *local_group_ctx, InstanceWatcher* instance_watcher, const std::string& global_image_id, const std::string& local_mirror_uuid, @@ -63,6 +64,7 @@ BootstrapRequest::BootstrapRequest( m_threads(threads), m_local_io_ctx(local_io_ctx), m_remote_io_ctx(remote_io_ctx), + m_local_group_ctx(local_group_ctx), m_instance_watcher(instance_watcher), m_global_image_id(global_image_id), m_local_mirror_uuid(local_mirror_uuid), @@ -361,8 +363,8 @@ void BootstrapRequest::create_local_image() { BootstrapRequest, &BootstrapRequest::handle_create_local_image>(this); auto request = (*m_state_builder)->create_local_image_request( - m_threads, m_local_io_ctx, m_global_image_id, m_pool_meta_cache, - m_progress_ctx, ctx); + m_threads, m_local_io_ctx, m_local_group_ctx, m_global_image_id, + m_pool_meta_cache, m_progress_ctx, ctx); request->send(); } @@ -405,9 +407,9 @@ void BootstrapRequest::image_sync() { Context *ctx = create_context_callback< BootstrapRequest, &BootstrapRequest::handle_image_sync>(this); m_image_sync = ImageSync::create( - m_threads, state_builder->local_image_ctx, state_builder->remote_image_ctx, - m_local_mirror_uuid, sync_point_handler, m_instance_watcher, - m_progress_ctx, ctx); + m_threads, m_local_group_ctx, state_builder->local_image_ctx, + state_builder->remote_image_ctx, m_local_mirror_uuid, sync_point_handler, + m_instance_watcher, m_progress_ctx, ctx); m_image_sync->get(); locker.unlock(); diff --git a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h index f5bb8dd8a16cf..273a6077dedac 100644 --- a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h +++ b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h @@ -42,6 +42,7 @@ public: Threads* threads, librados::IoCtx& local_io_ctx, librados::IoCtx& remote_io_ctx, + GroupCtx *local_group_ctx, InstanceWatcher* instance_watcher, const std::string& global_image_id, const std::string& local_mirror_uuid, @@ -53,15 +54,17 @@ public: bool* do_resync, Context* on_finish) { return new BootstrapRequest( - threads, local_io_ctx, remote_io_ctx, instance_watcher, global_image_id, - local_mirror_uuid, remote_pool_meta, cache_manager_handler, - pool_meta_cache, progress_ctx, state_builder, do_resync, on_finish); + threads, local_io_ctx, remote_io_ctx, local_group_ctx, instance_watcher, + global_image_id, local_mirror_uuid, remote_pool_meta, + cache_manager_handler, pool_meta_cache, progress_ctx, state_builder, + do_resync, on_finish); } BootstrapRequest( Threads* threads, librados::IoCtx& local_io_ctx, librados::IoCtx& remote_io_ctx, + GroupCtx *local_group_ctx, InstanceWatcher* instance_watcher, const std::string& global_image_id, const std::string& local_mirror_uuid, @@ -124,6 +127,7 @@ private: Threads* m_threads; librados::IoCtx &m_local_io_ctx; librados::IoCtx &m_remote_io_ctx; + GroupCtx *m_local_group_ctx; InstanceWatcher *m_instance_watcher; std::string m_global_image_id; std::string m_local_mirror_uuid; diff --git a/src/tools/rbd_mirror/image_replayer/CreateImageRequest.cc b/src/tools/rbd_mirror/image_replayer/CreateImageRequest.cc index 96717b4edfd98..6c99cdbe1b09f 100644 --- a/src/tools/rbd_mirror/image_replayer/CreateImageRequest.cc +++ b/src/tools/rbd_mirror/image_replayer/CreateImageRequest.cc @@ -41,6 +41,7 @@ template CreateImageRequest::CreateImageRequest( Threads* threads, librados::IoCtx &local_io_ctx, + GroupCtx *local_group_ctx, const std::string &global_image_id, const std::string &remote_mirror_uuid, const std::string &local_image_name, @@ -50,7 +51,7 @@ CreateImageRequest::CreateImageRequest( cls::rbd::MirrorImageMode mirror_image_mode, Context *on_finish) : m_threads(threads), m_local_io_ctx(local_io_ctx), - m_global_image_id(global_image_id), + m_local_group_ctx(local_group_ctx), m_global_image_id(global_image_id), m_remote_mirror_uuid(remote_mirror_uuid), m_local_image_name(local_image_name), m_local_image_id(local_image_id), m_remote_image_ctx(remote_image_ctx), @@ -444,6 +445,12 @@ void CreateImageRequest::populate_image_options( } image_options->set(RBD_IMAGE_OPTION_CLONE_FORMAT, clone_format); } + + if (m_local_group_ctx != nullptr) { + image_options->set(RBD_IMAGE_OPTION_GROUP_NAME, m_local_group_ctx->name); + image_options->set(RBD_IMAGE_OPTION_GROUP_POOL, + m_local_group_ctx->io_ctx.get_pool_name()); + } } } // namespace image_replayer diff --git a/src/tools/rbd_mirror/image_replayer/CreateImageRequest.h b/src/tools/rbd_mirror/image_replayer/CreateImageRequest.h index 2ff7794e81aec..9963277cf5da6 100644 --- a/src/tools/rbd_mirror/image_replayer/CreateImageRequest.h +++ b/src/tools/rbd_mirror/image_replayer/CreateImageRequest.h @@ -19,6 +19,7 @@ namespace rbd { namespace mirror { class PoolMetaCache; +struct GroupCtx; template struct Threads; namespace image_replayer { @@ -29,6 +30,7 @@ public: static CreateImageRequest *create( Threads *threads, librados::IoCtx &local_io_ctx, + GroupCtx *local_group_ctx, const std::string &global_image_id, const std::string &remote_mirror_uuid, const std::string &local_image_name, @@ -37,15 +39,16 @@ public: PoolMetaCache* pool_meta_cache, cls::rbd::MirrorImageMode mirror_image_mode, Context *on_finish) { - return new CreateImageRequest(threads, local_io_ctx, global_image_id, - remote_mirror_uuid, local_image_name, - local_image_id, remote_image_ctx, - pool_meta_cache, mirror_image_mode, - on_finish); + return new CreateImageRequest(threads, local_io_ctx, local_group_ctx, + global_image_id, remote_mirror_uuid, + local_image_name, local_image_id, + remote_image_ctx, pool_meta_cache, + mirror_image_mode, on_finish); } CreateImageRequest( Threads *threads, librados::IoCtx &local_io_ctx, + GroupCtx *local_group_ctx, const std::string &global_image_id, const std::string &remote_mirror_uuid, const std::string &local_image_name, @@ -87,6 +90,7 @@ private: Threads *m_threads; librados::IoCtx &m_local_io_ctx; + GroupCtx *m_local_group_ctx; std::string m_global_image_id; std::string m_remote_mirror_uuid; std::string m_local_image_name; diff --git a/src/tools/rbd_mirror/image_replayer/StateBuilder.h b/src/tools/rbd_mirror/image_replayer/StateBuilder.h index 51cf8668c1eff..d6a83afb259b2 100644 --- a/src/tools/rbd_mirror/image_replayer/StateBuilder.h +++ b/src/tools/rbd_mirror/image_replayer/StateBuilder.h @@ -15,6 +15,7 @@ namespace rbd { namespace mirror { struct BaseRequest; +struct GroupCtx; template class InstanceWatcher; struct PoolMetaCache; struct ProgressContext; @@ -59,6 +60,7 @@ public: virtual BaseRequest* create_local_image_request( Threads* threads, librados::IoCtx& local_io_ctx, + GroupCtx *local_group_ctx, const std::string& global_image_id, PoolMetaCache* pool_meta_cache, ProgressContext* progress_ctx, diff --git a/src/tools/rbd_mirror/image_replayer/journal/CreateLocalImageRequest.cc b/src/tools/rbd_mirror/image_replayer/journal/CreateLocalImageRequest.cc index 087cf4f5fd470..5eeeeb132d105 100644 --- a/src/tools/rbd_mirror/image_replayer/journal/CreateLocalImageRequest.cc +++ b/src/tools/rbd_mirror/image_replayer/journal/CreateLocalImageRequest.cc @@ -117,7 +117,7 @@ void CreateLocalImageRequest::create_local_image() { CreateLocalImageRequest, &CreateLocalImageRequest::handle_create_local_image>(this); auto request = CreateImageRequest::create( - m_threads, m_local_io_ctx, m_global_image_id, + m_threads, m_local_io_ctx, m_local_group_ctx, m_global_image_id, m_state_builder->remote_mirror_uuid, image_name, m_state_builder->local_image_id, m_remote_image_ctx, m_pool_meta_cache, cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, ctx); diff --git a/src/tools/rbd_mirror/image_replayer/journal/CreateLocalImageRequest.h b/src/tools/rbd_mirror/image_replayer/journal/CreateLocalImageRequest.h index fc776ecc30117..ae2947bf5985d 100644 --- a/src/tools/rbd_mirror/image_replayer/journal/CreateLocalImageRequest.h +++ b/src/tools/rbd_mirror/image_replayer/journal/CreateLocalImageRequest.h @@ -17,6 +17,7 @@ namespace mirror { class PoolMetaCache; class ProgressContext; +struct GroupCtx; template struct Threads; namespace image_replayer { @@ -32,20 +33,23 @@ public: static CreateLocalImageRequest* create( Threads* threads, librados::IoCtx& local_io_ctx, + GroupCtx *local_group_ctx, ImageCtxT* remote_image_ctx, const std::string& global_image_id, PoolMetaCache* pool_meta_cache, ProgressContext* progress_ctx, StateBuilder* state_builder, Context* on_finish) { - return new CreateLocalImageRequest(threads, local_io_ctx, remote_image_ctx, - global_image_id, pool_meta_cache, - progress_ctx, state_builder, on_finish); + return new CreateLocalImageRequest(threads, local_io_ctx, local_group_ctx, + remote_image_ctx, global_image_id, + pool_meta_cache, progress_ctx, + state_builder, on_finish); } CreateLocalImageRequest( Threads* threads, librados::IoCtx& local_io_ctx, + GroupCtx *local_group_ctx, ImageCtxT* remote_image_ctx, const std::string& global_image_id, PoolMetaCache* pool_meta_cache, @@ -55,6 +59,7 @@ public: : BaseRequest(on_finish), m_threads(threads), m_local_io_ctx(local_io_ctx), + m_local_group_ctx(local_group_ctx), m_remote_image_ctx(remote_image_ctx), m_global_image_id(global_image_id), m_pool_meta_cache(pool_meta_cache), @@ -87,6 +92,7 @@ private: Threads* m_threads; librados::IoCtx& m_local_io_ctx; + GroupCtx *m_local_group_ctx; ImageCtxT* m_remote_image_ctx; std::string m_global_image_id; PoolMetaCache* m_pool_meta_cache; diff --git a/src/tools/rbd_mirror/image_replayer/journal/StateBuilder.cc b/src/tools/rbd_mirror/image_replayer/journal/StateBuilder.cc index 5f1fb0e2f09fc..0735c9b501bac 100644 --- a/src/tools/rbd_mirror/image_replayer/journal/StateBuilder.cc +++ b/src/tools/rbd_mirror/image_replayer/journal/StateBuilder.cc @@ -80,13 +80,14 @@ template BaseRequest* StateBuilder::create_local_image_request( Threads* threads, librados::IoCtx& local_io_ctx, + GroupCtx *local_group_ctx, const std::string& global_image_id, PoolMetaCache* pool_meta_cache, ProgressContext* progress_ctx, Context* on_finish) { return CreateLocalImageRequest::create( - threads, local_io_ctx, this->remote_image_ctx, this->global_image_id, - pool_meta_cache, progress_ctx, this, on_finish); + threads, local_io_ctx, local_group_ctx, this->remote_image_ctx, + this->global_image_id, pool_meta_cache, progress_ctx, this, on_finish); } template diff --git a/src/tools/rbd_mirror/image_replayer/journal/StateBuilder.h b/src/tools/rbd_mirror/image_replayer/journal/StateBuilder.h index 790d1390bec6a..bc56a030b1070 100644 --- a/src/tools/rbd_mirror/image_replayer/journal/StateBuilder.h +++ b/src/tools/rbd_mirror/image_replayer/journal/StateBuilder.h @@ -49,6 +49,7 @@ public: BaseRequest* create_local_image_request( Threads* threads, librados::IoCtx& local_io_ctx, + GroupCtx *local_group_ctx, const std::string& global_image_id, PoolMetaCache* pool_meta_cache, ProgressContext* progress_ctx, diff --git a/src/tools/rbd_mirror/image_replayer/snapshot/CreateLocalImageRequest.cc b/src/tools/rbd_mirror/image_replayer/snapshot/CreateLocalImageRequest.cc index c923395c9bf33..14eddf4815c8d 100644 --- a/src/tools/rbd_mirror/image_replayer/snapshot/CreateLocalImageRequest.cc +++ b/src/tools/rbd_mirror/image_replayer/snapshot/CreateLocalImageRequest.cc @@ -11,6 +11,7 @@ #include "librbd/ImageCtx.h" #include "librbd/Utils.h" #include "tools/rbd_mirror/ProgressContext.h" +#include "tools/rbd_mirror/Types.h" #include "tools/rbd_mirror/image_replayer/CreateImageRequest.h" #include "tools/rbd_mirror/image_replayer/snapshot/StateBuilder.h" @@ -46,8 +47,13 @@ void CreateLocalImageRequest::disable_mirror_image() { // need to send 'disabling' since the cls methods will fail if we aren't // in that state + cls::rbd::GroupSpec group_spec; + if (m_local_group_ctx != nullptr) { + group_spec = {m_local_group_ctx->group_id, + m_local_group_ctx->io_ctx.get_id()}; + } cls::rbd::MirrorImage mirror_image{ - cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT, m_global_image_id, + cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT, m_global_image_id, group_spec, cls::rbd::MIRROR_IMAGE_STATE_DISABLING}; librados::ObjectWriteOperation op; librbd::cls_client::mirror_image_set(&op, m_state_builder->local_image_id, @@ -115,10 +121,15 @@ void CreateLocalImageRequest::add_mirror_image() { dout(10) << "local_image_id=" << m_state_builder->local_image_id << dendl; update_progress("ADD_MIRROR_IMAGE"); + cls::rbd::GroupSpec group_spec; + if (m_local_group_ctx != nullptr) { + group_spec = {m_local_group_ctx->group_id, + m_local_group_ctx->io_ctx.get_id()}; + } // use 'creating' to track a partially constructed image. it will // be switched to 'enabled' once the image is fully created cls::rbd::MirrorImage mirror_image{ - cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT, m_global_image_id, + cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT, m_global_image_id, group_spec, cls::rbd::MIRROR_IMAGE_STATE_CREATING}; librados::ObjectWriteOperation op; librbd::cls_client::mirror_image_set(&op, m_state_builder->local_image_id, @@ -159,7 +170,7 @@ void CreateLocalImageRequest::create_local_image() { CreateLocalImageRequest, &CreateLocalImageRequest::handle_create_local_image>(this); auto request = CreateImageRequest::create( - m_threads, m_local_io_ctx, m_global_image_id, + m_threads, m_local_io_ctx, m_local_group_ctx, m_global_image_id, m_state_builder->remote_mirror_uuid, image_name, m_state_builder->local_image_id, m_remote_image_ctx, m_pool_meta_cache, cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT, ctx); diff --git a/src/tools/rbd_mirror/image_replayer/snapshot/CreateLocalImageRequest.h b/src/tools/rbd_mirror/image_replayer/snapshot/CreateLocalImageRequest.h index 3345154b49188..921d850528734 100644 --- a/src/tools/rbd_mirror/image_replayer/snapshot/CreateLocalImageRequest.h +++ b/src/tools/rbd_mirror/image_replayer/snapshot/CreateLocalImageRequest.h @@ -16,6 +16,7 @@ namespace mirror { class PoolMetaCache; class ProgressContext; +struct GroupCtx; template struct Threads; namespace image_replayer { @@ -31,20 +32,23 @@ public: static CreateLocalImageRequest* create( Threads* threads, librados::IoCtx& local_io_ctx, + GroupCtx *local_group_ctx, ImageCtxT* remote_image_ctx, const std::string& global_image_id, PoolMetaCache* pool_meta_cache, ProgressContext* progress_ctx, StateBuilder* state_builder, Context* on_finish) { - return new CreateLocalImageRequest(threads, local_io_ctx, remote_image_ctx, - global_image_id, pool_meta_cache, - progress_ctx, state_builder, on_finish); + return new CreateLocalImageRequest(threads, local_io_ctx, local_group_ctx, + remote_image_ctx, global_image_id, + pool_meta_cache, progress_ctx, + state_builder, on_finish); } CreateLocalImageRequest( Threads* threads, librados::IoCtx& local_io_ctx, + GroupCtx *local_group_ctx, ImageCtxT* remote_image_ctx, const std::string& global_image_id, PoolMetaCache* pool_meta_cache, @@ -54,6 +58,7 @@ public: : BaseRequest(on_finish), m_threads(threads), m_local_io_ctx(local_io_ctx), + m_local_group_ctx(local_group_ctx), m_remote_image_ctx(remote_image_ctx), m_global_image_id(global_image_id), m_pool_meta_cache(pool_meta_cache), @@ -89,6 +94,7 @@ private: Threads* m_threads; librados::IoCtx& m_local_io_ctx; + GroupCtx *m_local_group_ctx; ImageCtxT* m_remote_image_ctx; std::string m_global_image_id; PoolMetaCache* m_pool_meta_cache; diff --git a/src/tools/rbd_mirror/image_replayer/snapshot/StateBuilder.cc b/src/tools/rbd_mirror/image_replayer/snapshot/StateBuilder.cc index ca3e6918babc8..2d79a915e2c20 100644 --- a/src/tools/rbd_mirror/image_replayer/snapshot/StateBuilder.cc +++ b/src/tools/rbd_mirror/image_replayer/snapshot/StateBuilder.cc @@ -79,13 +79,14 @@ template BaseRequest* StateBuilder::create_local_image_request( Threads* threads, librados::IoCtx& local_io_ctx, + GroupCtx *local_group_ctx, const std::string& global_image_id, PoolMetaCache* pool_meta_cache, ProgressContext* progress_ctx, Context* on_finish) { return CreateLocalImageRequest::create( - threads, local_io_ctx, this->remote_image_ctx, global_image_id, - pool_meta_cache, progress_ctx, this, on_finish); + threads, local_io_ctx, local_group_ctx, this->remote_image_ctx, + global_image_id, pool_meta_cache, progress_ctx, this, on_finish); } template diff --git a/src/tools/rbd_mirror/image_replayer/snapshot/StateBuilder.h b/src/tools/rbd_mirror/image_replayer/snapshot/StateBuilder.h index a4ab82982c103..1cc5b9b1ec025 100644 --- a/src/tools/rbd_mirror/image_replayer/snapshot/StateBuilder.h +++ b/src/tools/rbd_mirror/image_replayer/snapshot/StateBuilder.h @@ -54,6 +54,7 @@ public: BaseRequest* create_local_image_request( Threads* threads, librados::IoCtx& local_io_ctx, + GroupCtx *local_group_ctx, const std::string& global_image_id, PoolMetaCache* pool_meta_cache, ProgressContext* progress_ctx, -- 2.39.5