From 2974ea609e23dc99e4a1954abb14dc8b52834410 Mon Sep 17 00:00:00 2001 From: Jason Dillaman Date: Mon, 27 Jan 2020 12:09:49 -0500 Subject: [PATCH] rbd-mirror: prepare remote image now uses get mirror info request This replaces the existing get mirror image call to provide the additional detail for promotion state. Signed-off-by: Jason Dillaman --- .../test_mock_PrepareRemoteImageRequest.cc | 121 ++++++++++++++---- .../PrepareRemoteImageRequest.cc | 38 +++--- .../PrepareRemoteImageRequest.h | 12 +- 3 files changed, 119 insertions(+), 52 deletions(-) diff --git a/src/test/rbd_mirror/image_replayer/test_mock_PrepareRemoteImageRequest.cc b/src/test/rbd_mirror/image_replayer/test_mock_PrepareRemoteImageRequest.cc index 49b22dac70f..ff65fef4c26 100644 --- a/src/test/rbd_mirror/image_replayer/test_mock_PrepareRemoteImageRequest.cc +++ b/src/test/rbd_mirror/image_replayer/test_mock_PrepareRemoteImageRequest.cc @@ -4,6 +4,7 @@ #include "test/rbd_mirror/test_mock_fixture.h" #include "cls/rbd/cls_rbd_types.h" #include "librbd/journal/TypeTraits.h" +#include "librbd/mirror/GetInfoRequest.h" #include "tools/rbd_mirror/Threads.h" #include "tools/rbd_mirror/image_replayer/GetMirrorImageIdRequest.h" #include "tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.h" @@ -33,6 +34,46 @@ struct TypeTraits { }; } // namespace journal + +namespace mirror { + +template<> +struct GetInfoRequest { + static GetInfoRequest* s_instance; + cls::rbd::MirrorImage *mirror_image; + PromotionState *promotion_state; + std::string *primary_mirror_uuid; + Context *on_finish = nullptr; + + static GetInfoRequest* create(librados::IoCtx& io_ctx, + ContextWQ* context_wq, + const std::string& image_id, + cls::rbd::MirrorImage *mirror_image, + PromotionState *promotion_state, + std::string* primary_mirror_uuid, + Context *on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->mirror_image = mirror_image; + s_instance->promotion_state = promotion_state; + s_instance->primary_mirror_uuid = primary_mirror_uuid; + s_instance->on_finish = on_finish; + return s_instance; + } + + GetInfoRequest() { + ceph_assert(s_instance == nullptr); + s_instance = this; + } + ~GetInfoRequest() { + s_instance = nullptr; + } + + MOCK_METHOD0(send, void()); +}; + +GetInfoRequest* GetInfoRequest::s_instance = nullptr; + +} // namespace mirror } // namespace librbd namespace rbd { @@ -138,6 +179,7 @@ public: typedef GetMirrorImageIdRequest MockGetMirrorImageIdRequest; typedef StateBuilder MockStateBuilder; typedef journal::StateBuilder MockJournalStateBuilder; + typedef librbd::mirror::GetInfoRequest MockGetMirrorInfoRequest; void expect_get_mirror_image_mode(MockStateBuilder& mock_state_builder, cls::rbd::MirrorImageMode mirror_image_mode) { @@ -154,21 +196,21 @@ public: })); } - void expect_get_mirror_image(librados::IoCtx &io_ctx, - cls::rbd::MirrorImageMode mode, int r) { - cls::rbd::MirrorImage mirror_image; - mirror_image.mode = mode; - - bufferlist bl; - encode(mirror_image, bl); - - EXPECT_CALL(get_mock_io_ctx(io_ctx), - exec(RBD_MIRRORING, _, StrEq("rbd"), - StrEq("mirror_image_get"), _, _, _)) - .WillOnce(DoAll(WithArg<5>(Invoke([bl](bufferlist *out_bl) { - *out_bl = bl; - })), - Return(r))); + void expect_get_mirror_info( + MockGetMirrorInfoRequest &mock_get_mirror_info_request, + const cls::rbd::MirrorImage &mirror_image, + librbd::mirror::PromotionState promotion_state, + const std::string& primary_mirror_uuid, int r) { + EXPECT_CALL(mock_get_mirror_info_request, send()) + .WillOnce(Invoke([this, &mock_get_mirror_info_request, mirror_image, + promotion_state, primary_mirror_uuid, r]() { + *mock_get_mirror_info_request.mirror_image = mirror_image; + *mock_get_mirror_info_request.promotion_state = promotion_state; + *mock_get_mirror_info_request.primary_mirror_uuid = + primary_mirror_uuid; + m_threads->work_queue->queue( + mock_get_mirror_info_request.on_finish, r); + })); } void expect_journaler_get_client(::journal::MockJournaler &mock_journaler, @@ -205,8 +247,13 @@ TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, Success) { expect_get_mirror_image_id(mock_get_mirror_image_id_request, "remote image id", 0); - expect_get_mirror_image(m_remote_io_ctx, cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, - 0); + MockGetMirrorInfoRequest mock_get_mirror_info_request; + expect_get_mirror_info(mock_get_mirror_info_request, + {cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_ENABLED}, + librbd::mirror::PROMOTION_STATE_PRIMARY, + "remote mirror uuid", 0); EXPECT_CALL(mock_remote_journaler, construct()); @@ -252,8 +299,14 @@ TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, SuccessNotRegistered) { expect_get_mirror_image_id(mock_get_mirror_image_id_request, "remote image id", 0); - expect_get_mirror_image(m_remote_io_ctx, cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, - 0); + MockGetMirrorInfoRequest mock_get_mirror_info_request; + expect_get_mirror_info(mock_get_mirror_info_request, + {cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_ENABLED}, + librbd::mirror::PROMOTION_STATE_PRIMARY, + "remote mirror uuid", 0); + MockJournalStateBuilder mock_journal_state_builder; expect_get_mirror_image_mode(mock_journal_state_builder, cls::rbd::MIRROR_IMAGE_MODE_JOURNAL); @@ -319,7 +372,7 @@ TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, MirrorImageIdError) { ASSERT_TRUE(mock_journal_state_builder.remote_journaler == nullptr); } -TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, MirrorModeError) { +TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, GetMirrorInfoError) { ::journal::MockJournaler mock_remote_journaler; MockThreads mock_threads(m_threads); @@ -328,8 +381,13 @@ TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, MirrorModeError) { expect_get_mirror_image_id(mock_get_mirror_image_id_request, "remote image id", 0); - expect_get_mirror_image(m_remote_io_ctx, cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, - -EINVAL); + MockGetMirrorInfoRequest mock_get_mirror_info_request; + expect_get_mirror_info(mock_get_mirror_info_request, + {cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_ENABLED}, + librbd::mirror::PROMOTION_STATE_PRIMARY, + "remote mirror uuid", -EINVAL); MockJournalStateBuilder mock_journal_state_builder; MockStateBuilder* mock_state_builder = nullptr; @@ -358,8 +416,13 @@ TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, GetClientError) { expect_get_mirror_image_id(mock_get_mirror_image_id_request, "remote image id", 0); - expect_get_mirror_image(m_remote_io_ctx, cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, - 0); + MockGetMirrorInfoRequest mock_get_mirror_info_request; + expect_get_mirror_info(mock_get_mirror_info_request, + {cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_ENABLED}, + librbd::mirror::PROMOTION_STATE_PRIMARY, + "remote mirror uuid", 0); EXPECT_CALL(mock_remote_journaler, construct()); @@ -394,8 +457,14 @@ TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, RegisterClientError) { expect_get_mirror_image_id(mock_get_mirror_image_id_request, "remote image id", 0); - expect_get_mirror_image(m_remote_io_ctx, cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, - 0); + MockGetMirrorInfoRequest mock_get_mirror_info_request; + expect_get_mirror_info(mock_get_mirror_info_request, + {cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_ENABLED}, + librbd::mirror::PROMOTION_STATE_PRIMARY, + "remote mirror uuid", 0); + MockJournalStateBuilder mock_journal_state_builder; expect_get_mirror_image_mode(mock_journal_state_builder, cls::rbd::MIRROR_IMAGE_MODE_JOURNAL); diff --git a/src/tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.cc b/src/tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.cc index 65ff88002c1..dfeb849c5b9 100644 --- a/src/tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.cc +++ b/src/tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.cc @@ -8,9 +8,10 @@ #include "common/errno.h" #include "common/WorkQueue.h" #include "journal/Journaler.h" +#include "journal/Settings.h" #include "librbd/ImageCtx.h" #include "librbd/Utils.h" -#include "librbd/journal/Types.h" +#include "librbd/mirror/GetInfoRequest.h" #include "tools/rbd_mirror/Threads.h" #include "tools/rbd_mirror/image_replayer/GetMirrorImageIdRequest.h" #include "tools/rbd_mirror/image_replayer/Utils.h" @@ -63,33 +64,26 @@ void PrepareRemoteImageRequest::handle_get_remote_image_id(int r) { return; } - get_mirror_image(); + get_mirror_info(); } template -void PrepareRemoteImageRequest::get_mirror_image() { +void PrepareRemoteImageRequest::get_mirror_info() { dout(10) << dendl; - librados::ObjectReadOperation op; - librbd::cls_client::mirror_image_get_start(&op, m_remote_image_id); - - auto aio_comp = create_rados_callback< + auto ctx = create_context_callback< PrepareRemoteImageRequest, - &PrepareRemoteImageRequest::handle_get_mirror_image>(this); - m_out_bl.clear(); - int r = m_remote_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op, &m_out_bl); - ceph_assert(r == 0); - aio_comp->release(); + &PrepareRemoteImageRequest::handle_get_mirror_info>(this); + auto req = librbd::mirror::GetInfoRequest::create( + m_remote_io_ctx, m_threads->work_queue, m_remote_image_id, + &m_mirror_image, &m_promotion_state, &m_primary_mirror_uuid, + ctx); + req->send(); } template -void PrepareRemoteImageRequest::handle_get_mirror_image(int r) { +void PrepareRemoteImageRequest::handle_get_mirror_info(int r) { dout(10) << "r=" << r << dendl; - cls::rbd::MirrorImage mirror_image; - if (r == 0) { - auto iter = m_out_bl.cbegin(); - r = librbd::cls_client::mirror_image_get_finish(&iter, &mirror_image); - } if (r == -ENOENT) { dout(10) << "image " << m_global_image_id << " not mirrored" << dendl; @@ -103,21 +97,21 @@ void PrepareRemoteImageRequest::handle_get_mirror_image(int r) { } if (*m_state_builder != nullptr && - (*m_state_builder)->get_mirror_image_mode() != mirror_image.mode) { + (*m_state_builder)->get_mirror_image_mode() != m_mirror_image.mode) { derr << "local and remote mirror image using different mirroring modes " << "for image " << m_global_image_id << ": split-brain" << dendl; finish(-EEXIST); return; } - switch (mirror_image.mode) { + switch (m_mirror_image.mode) { case cls::rbd::MIRROR_IMAGE_MODE_JOURNAL: get_client(); break; case cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT: // TODO default: - derr << "unsupported mirror image mode " << mirror_image.mode << " " + derr << "unsupported mirror image mode " << m_mirror_image.mode << " " << "for image " << m_global_image_id << dendl; finish(-EOPNOTSUPP); break; @@ -214,7 +208,7 @@ void PrepareRemoteImageRequest::finalize_journal_state_builder( journal::StateBuilder* state_builder = nullptr; if (*m_state_builder != nullptr) { // already verified that it's a matching builder in - // 'handle_get_mirror_image' + // 'handle_get_mirror_info' state_builder = dynamic_cast*>(*m_state_builder); ceph_assert(state_builder != nullptr); } else { diff --git a/src/tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.h b/src/tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.h index 59a28feddf7..397436bc747 100644 --- a/src/tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.h +++ b/src/tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.h @@ -7,9 +7,10 @@ #include "include/buffer_fwd.h" #include "include/rados/librados_fwd.hpp" #include "cls/journal/cls_journal_types.h" -#include "journal/Settings.h" +#include "cls/rbd/cls_rbd_types.h" #include "librbd/journal/Types.h" #include "librbd/journal/TypeTraits.h" +#include "librbd/mirror/Types.h" #include "tools/rbd_mirror/Types.h" #include @@ -87,7 +88,7 @@ private: * GET_REMOTE_IMAGE_ID * | * v - * GET_REMOTE_MIRROR_IMAGE + * GET_REMOTE_MIRROR_INFO * | * | (journal) * \-----------> GET_CLIENT @@ -113,6 +114,9 @@ private: bufferlist m_out_bl; std::string m_remote_image_id; + cls::rbd::MirrorImage m_mirror_image; + librbd::mirror::PromotionState m_promotion_state; + std::string m_primary_mirror_uuid; // journal-based mirroring Journaler *m_remote_journaler = nullptr; @@ -121,8 +125,8 @@ private: void get_remote_image_id(); void handle_get_remote_image_id(int r); - void get_mirror_image(); - void handle_get_mirror_image(int r); + void get_mirror_info(); + void handle_get_mirror_info(int r); void get_client(); void handle_get_client(int r); -- 2.39.5