From: Jason Dillaman Date: Tue, 25 Apr 2017 19:45:18 +0000 (-0400) Subject: rbd-mirror: new state machine for preparing local image X-Git-Tag: v12.0.3~177^2~1 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=68a10a3b8b7813e663abe8846f682ce565e02fdc;p=ceph.git rbd-mirror: new state machine for preparing local image This state machine will be invoked before the bootstrap state machine and will be responsible for detecting if the local image is already primary or if it needs to be deleted. Signed-off-by: Jason Dillaman --- diff --git a/src/test/librbd/mock/MockJournal.h b/src/test/librbd/mock/MockJournal.h index a31a80386e34..9f0a985cb7bf 100644 --- a/src/test/librbd/mock/MockJournal.h +++ b/src/test/librbd/mock/MockJournal.h @@ -9,6 +9,10 @@ #include "librbd/journal/Types.h" #include +struct Context; +struct ContextWQ; +namespace librados { class IoCtx; } + namespace librbd { struct ImageCtx; @@ -28,6 +32,14 @@ struct MockJournal { return get_instance()->is_tag_owner(is_tag_owner); } + static void get_tag_owner(librados::IoCtx &, + const std::string &global_image_id, + std::string *tag_owner, ContextWQ *work_queue, + Context *on_finish) { + get_instance()->get_tag_owner(global_image_id, tag_owner, + work_queue, on_finish); + } + MockJournal() { s_instance = this; } @@ -38,6 +50,10 @@ struct MockJournal { MOCK_METHOD1(wait_for_journal_ready, void(Context *)); + MOCK_METHOD4(get_tag_owner, void(const std::string &, + std::string *, ContextWQ *, + Context *)); + MOCK_CONST_METHOD0(is_tag_owner, bool()); MOCK_CONST_METHOD1(is_tag_owner, int(bool *)); MOCK_METHOD3(allocate_tag, void(const std::string &mirror_uuid, diff --git a/src/test/rbd_mirror/CMakeLists.txt b/src/test/rbd_mirror/CMakeLists.txt index 291b457bb607..ba5e13f3e1f7 100644 --- a/src/test/rbd_mirror/CMakeLists.txt +++ b/src/test/rbd_mirror/CMakeLists.txt @@ -26,6 +26,7 @@ add_executable(unittest_rbd_mirror image_replayer/test_mock_BootstrapRequest.cc image_replayer/test_mock_CreateImageRequest.cc image_replayer/test_mock_EventPreprocessor.cc + image_replayer/test_mock_PrepareLocalImageRequest.cc image_sync/test_mock_ImageCopyRequest.cc image_sync/test_mock_ObjectCopyRequest.cc image_sync/test_mock_SnapshotCopyRequest.cc diff --git a/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc b/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc index 4da8665799b4..b31ab74fb589 100644 --- a/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc +++ b/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc @@ -297,25 +297,6 @@ public: ASSERT_EQ(0, open_image(m_local_io_ctx, m_image_name, &m_local_image_ctx)); } - void expect_mirror_image_get_image_id(librados::IoCtx &io_ctx, - const std::string &global_image_id, - const std::string &image_id, int r) { - bufferlist in_bl; - ::encode(global_image_id, in_bl); - - bufferlist bl; - ::encode(image_id, bl); - - EXPECT_CALL(get_mock_io_ctx(io_ctx), - exec(RBD_MIRRORING, _, StrEq("rbd"), - StrEq("mirror_image_get_image_id"), ContentsEqual(in_bl), - _, _)) - .WillOnce(DoAll(WithArg<5>(Invoke([bl](bufferlist *out_bl) { - *out_bl = bl; - })), - Return(r))); - } - void expect_journaler_get_client(::journal::MockJournaler &mock_journaler, const std::string &client_id, cls::journal::Client &client, int r) { @@ -420,6 +401,7 @@ public: MockBootstrapRequest *create_request(MockImageSyncThrottler mock_image_sync_throttler, ::journal::MockJournaler &mock_journaler, + const std::string &local_image_id, const std::string &remote_image_id, const std::string &global_image_id, const std::string &local_mirror_uuid, @@ -428,7 +410,8 @@ public: return new MockBootstrapRequest(m_local_io_ctx, m_remote_io_ctx, mock_image_sync_throttler, - &m_local_test_image_ctx, "", + &m_local_test_image_ctx, + local_image_id, remote_image_id, global_image_id, m_threads->work_queue, @@ -453,11 +436,6 @@ TEST_F(TestMockImageReplayerBootstrapRequest, NonPrimaryRemoteSyncingState) { InSequence seq; - // look up local image by global image id - librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); - expect_mirror_image_get_image_id(m_local_io_ctx, "global image id", - mock_local_image_ctx.id, 0); - // lookup remote image tag class cls::journal::Client client; librbd::journal::ClientData client_data{ @@ -469,6 +447,7 @@ TEST_F(TestMockImageReplayerBootstrapRequest, NonPrimaryRemoteSyncingState) { client, 0); // lookup local peer in remote journal + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{ mock_local_image_ctx.id}; mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_SYNCING; @@ -499,9 +478,9 @@ TEST_F(TestMockImageReplayerBootstrapRequest, NonPrimaryRemoteSyncingState) { MockImageSyncThrottler mock_image_sync_throttler( new ImageSyncThrottler()); MockBootstrapRequest *request = create_request( - mock_image_sync_throttler, mock_journaler, mock_remote_image_ctx.id, - "global image id", "local mirror uuid", "remote mirror uuid", - &ctx); + mock_image_sync_throttler, mock_journaler, mock_local_image_ctx.id, + mock_remote_image_ctx.id, "global image id", "local mirror uuid", + "remote mirror uuid", &ctx); request->send(); ASSERT_EQ(-EREMOTEIO, ctx.wait()); } @@ -511,11 +490,6 @@ TEST_F(TestMockImageReplayerBootstrapRequest, RemoteDemotePromote) { InSequence seq; - // look up local image by global image id - librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); - expect_mirror_image_get_image_id(m_local_io_ctx, "global image id", - mock_local_image_ctx.id, 0); - // lookup remote image tag class cls::journal::Client client; librbd::journal::ClientData client_data{ @@ -527,6 +501,7 @@ TEST_F(TestMockImageReplayerBootstrapRequest, RemoteDemotePromote) { client, 0); // lookup local peer in remote journal + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{ mock_local_image_ctx.id}; mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; @@ -578,9 +553,9 @@ TEST_F(TestMockImageReplayerBootstrapRequest, RemoteDemotePromote) { MockImageSyncThrottler mock_image_sync_throttler( new ImageSyncThrottler()); MockBootstrapRequest *request = create_request( - mock_image_sync_throttler, mock_journaler, mock_remote_image_ctx.id, - "global image id", "local mirror uuid", "remote mirror uuid", - &ctx); + mock_image_sync_throttler, mock_journaler, mock_local_image_ctx.id, + mock_remote_image_ctx.id, "global image id", "local mirror uuid", + "remote mirror uuid", &ctx); request->send(); ASSERT_EQ(0, ctx.wait()); } @@ -590,11 +565,6 @@ TEST_F(TestMockImageReplayerBootstrapRequest, MultipleRemoteDemotePromotes) { InSequence seq; - // look up local image by global image id - librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); - expect_mirror_image_get_image_id(m_local_io_ctx, "global image id", - mock_local_image_ctx.id, 0); - // lookup remote image tag class cls::journal::Client client; librbd::journal::ClientData client_data{ @@ -606,6 +576,7 @@ TEST_F(TestMockImageReplayerBootstrapRequest, MultipleRemoteDemotePromotes) { client, 0); // lookup local peer in remote journal + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{ mock_local_image_ctx.id}; mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; @@ -667,9 +638,9 @@ TEST_F(TestMockImageReplayerBootstrapRequest, MultipleRemoteDemotePromotes) { MockImageSyncThrottler mock_image_sync_throttler( new ImageSyncThrottler()); MockBootstrapRequest *request = create_request( - mock_image_sync_throttler, mock_journaler, mock_remote_image_ctx.id, - "global image id", "local mirror uuid", "remote mirror uuid", - &ctx); + mock_image_sync_throttler, mock_journaler, mock_local_image_ctx.id, + mock_remote_image_ctx.id, "global image id", "local mirror uuid", + "remote mirror uuid", &ctx); request->send(); ASSERT_EQ(0, ctx.wait()); } @@ -679,11 +650,6 @@ TEST_F(TestMockImageReplayerBootstrapRequest, LocalDemoteRemotePromote) { InSequence seq; - // look up local image by global image id - librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); - expect_mirror_image_get_image_id(m_local_io_ctx, "global image id", - mock_local_image_ctx.id, 0); - // lookup remote image tag class cls::journal::Client client; librbd::journal::ClientData client_data{ @@ -695,6 +661,7 @@ TEST_F(TestMockImageReplayerBootstrapRequest, LocalDemoteRemotePromote) { client, 0); // lookup local peer in remote journal + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{ mock_local_image_ctx.id}; mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; @@ -744,9 +711,9 @@ TEST_F(TestMockImageReplayerBootstrapRequest, LocalDemoteRemotePromote) { MockImageSyncThrottler mock_image_sync_throttler( new ImageSyncThrottler()); MockBootstrapRequest *request = create_request( - mock_image_sync_throttler, mock_journaler, mock_remote_image_ctx.id, - "global image id", "local mirror uuid", "remote mirror uuid", - &ctx); + mock_image_sync_throttler, mock_journaler, mock_local_image_ctx.id, + mock_remote_image_ctx.id, "global image id", "local mirror uuid", + "remote mirror uuid", &ctx); request->send(); ASSERT_EQ(0, ctx.wait()); } @@ -756,11 +723,6 @@ TEST_F(TestMockImageReplayerBootstrapRequest, SplitBrainForcePromote) { InSequence seq; - // look up local image by global image id - librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); - expect_mirror_image_get_image_id(m_local_io_ctx, "global image id", - mock_local_image_ctx.id, 0); - // lookup remote image tag class cls::journal::Client client; librbd::journal::ClientData client_data{ @@ -772,6 +734,7 @@ TEST_F(TestMockImageReplayerBootstrapRequest, SplitBrainForcePromote) { client, 0); // lookup local peer in remote journal + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{ mock_local_image_ctx.id}; mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; @@ -820,9 +783,9 @@ TEST_F(TestMockImageReplayerBootstrapRequest, SplitBrainForcePromote) { MockImageSyncThrottler mock_image_sync_throttler( new ImageSyncThrottler()); MockBootstrapRequest *request = create_request( - mock_image_sync_throttler, mock_journaler, mock_remote_image_ctx.id, - "global image id", "local mirror uuid", "remote mirror uuid", - &ctx); + mock_image_sync_throttler, mock_journaler, mock_local_image_ctx.id, + mock_remote_image_ctx.id, "global image id", "local mirror uuid", + "remote mirror uuid", &ctx); request->send(); ASSERT_EQ(-EEXIST, ctx.wait()); ASSERT_EQ(NULL, m_local_test_image_ctx); @@ -833,11 +796,6 @@ TEST_F(TestMockImageReplayerBootstrapRequest, ResyncRequested) { InSequence seq; - // look up local image by global image id - librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); - expect_mirror_image_get_image_id(m_local_io_ctx, "global image id", - mock_local_image_ctx.id, 0); - // lookup remote image tag class cls::journal::Client client; librbd::journal::ClientData client_data{ @@ -849,6 +807,7 @@ TEST_F(TestMockImageReplayerBootstrapRequest, ResyncRequested) { client, 0); // lookup local peer in remote journal + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{ mock_local_image_ctx.id}; mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; @@ -884,9 +843,9 @@ TEST_F(TestMockImageReplayerBootstrapRequest, ResyncRequested) { MockImageSyncThrottler mock_image_sync_throttler( new ImageSyncThrottler()); MockBootstrapRequest *request = create_request( - mock_image_sync_throttler, mock_journaler, mock_remote_image_ctx.id, - "global image id", "local mirror uuid", "remote mirror uuid", - &ctx); + mock_image_sync_throttler, mock_journaler, mock_local_image_ctx.id, + mock_remote_image_ctx.id, "global image id", "local mirror uuid", + "remote mirror uuid", &ctx); m_do_resync = false; request->send(); ASSERT_EQ(0, ctx.wait()); diff --git a/src/test/rbd_mirror/image_replayer/test_mock_PrepareLocalImageRequest.cc b/src/test/rbd_mirror/image_replayer/test_mock_PrepareLocalImageRequest.cc new file mode 100644 index 000000000000..b79d4de9e18d --- /dev/null +++ b/src/test/rbd_mirror/image_replayer/test_mock_PrepareLocalImageRequest.cc @@ -0,0 +1,196 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/rbd_mirror/test_mock_fixture.h" +#include "cls/rbd/cls_rbd_types.h" +#include "librbd/journal/TypeTraits.h" +#include "tools/rbd_mirror/image_replayer/PrepareLocalImageRequest.h" +#include "test/journal/mock/MockJournaler.h" +#include "test/librados_test_stub/MockTestMemIoCtxImpl.h" +#include "test/librbd/mock/MockImageCtx.h" +#include "test/librbd/mock/MockJournal.h" + +namespace librbd { + +namespace { + +struct MockTestImageCtx : public librbd::MockImageCtx { + MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace +} // namespace librbd + +// template definitions +#include "tools/rbd_mirror/image_replayer/PrepareLocalImageRequest.cc" + +namespace rbd { +namespace mirror { +namespace image_replayer { + +using ::testing::_; +using ::testing::DoAll; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::Return; +using ::testing::StrEq; +using ::testing::WithArg; +using ::testing::WithArgs; + +class TestMockImageReplayerPrepareLocalImageRequest : public TestMockFixture { +public: + typedef PrepareLocalImageRequest MockPrepareLocalImageRequest; + + void expect_mirror_image_get_image_id(librados::IoCtx &io_ctx, + const std::string &image_id, int r) { + bufferlist bl; + ::encode(image_id, bl); + + EXPECT_CALL(get_mock_io_ctx(io_ctx), + exec(RBD_MIRRORING, _, StrEq("rbd"), StrEq("mirror_image_get_image_id"), _, _, _)) + .WillOnce(DoAll(WithArg<5>(Invoke([bl](bufferlist *out_bl) { + *out_bl = bl; + })), + Return(r))); + } + + void expect_mirror_image_get(librados::IoCtx &io_ctx, + cls::rbd::MirrorImageState state, + const std::string &global_id, int r) { + cls::rbd::MirrorImage mirror_image; + mirror_image.state = state; + mirror_image.global_image_id = global_id; + + bufferlist bl; + ::encode(mirror_image, bl); + + EXPECT_CALL(get_mock_io_ctx(io_ctx), + exec(RBD_MIRRORING, _, StrEq("rbd"), StrEq("mirror_image_get"), _, _, _)) + .WillOnce(DoAll(WithArg<5>(Invoke([bl](bufferlist *out_bl) { + *out_bl = bl; + })), + Return(r))); + } + + void expect_get_tag_owner(librbd::MockJournal &mock_journal, + const std::string &local_image_id, + const std::string &tag_owner, int r) { + EXPECT_CALL(mock_journal, get_tag_owner(local_image_id, _, _, _)) + .WillOnce(WithArgs<1, 3>(Invoke([tag_owner, r](std::string *owner, Context *on_finish) { + *owner = tag_owner; + on_finish->complete(r); + }))); + } + +}; + +TEST_F(TestMockImageReplayerPrepareLocalImageRequest, Success) { + InSequence seq; + expect_mirror_image_get_image_id(m_local_io_ctx, "local image id", 0); + expect_mirror_image_get(m_local_io_ctx, cls::rbd::MIRROR_IMAGE_STATE_ENABLED, + "global image id", 0); + + librbd::MockJournal mock_journal; + expect_get_tag_owner(mock_journal, "local image id", "remote mirror uuid", 0); + + std::string local_image_id; + std::string tag_owner; + C_SaferCond ctx; + auto req = MockPrepareLocalImageRequest::create(m_local_io_ctx, + "global image id", + &local_image_id, + &tag_owner, + m_threads->work_queue, + &ctx); + req->send(); + + ASSERT_EQ(0, ctx.wait()); + ASSERT_EQ(std::string("local image id"), local_image_id); + ASSERT_EQ(std::string("remote mirror uuid"), tag_owner); +} + +TEST_F(TestMockImageReplayerPrepareLocalImageRequest, MirrorImageIdDNE) { + InSequence seq; + expect_mirror_image_get_image_id(m_local_io_ctx, "", -ENOENT); + + std::string local_image_id; + std::string tag_owner; + C_SaferCond ctx; + auto req = MockPrepareLocalImageRequest::create(m_local_io_ctx, + "global image id", + &local_image_id, + &tag_owner, + m_threads->work_queue, + &ctx); + req->send(); + + ASSERT_EQ(-ENOENT, ctx.wait()); +} + +TEST_F(TestMockImageReplayerPrepareLocalImageRequest, MirrorImageIdError) { + InSequence seq; + expect_mirror_image_get_image_id(m_local_io_ctx, "", -EINVAL); + + std::string local_image_id; + std::string tag_owner; + C_SaferCond ctx; + auto req = MockPrepareLocalImageRequest::create(m_local_io_ctx, + "global image id", + &local_image_id, + &tag_owner, + m_threads->work_queue, + &ctx); + req->send(); + + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageReplayerPrepareLocalImageRequest, MirrorImageError) { + InSequence seq; + expect_mirror_image_get_image_id(m_local_io_ctx, "local image id", 0); + expect_mirror_image_get(m_local_io_ctx, cls::rbd::MIRROR_IMAGE_STATE_DISABLED, + "", -EINVAL); + + std::string local_image_id; + std::string tag_owner; + C_SaferCond ctx; + auto req = MockPrepareLocalImageRequest::create(m_local_io_ctx, + "global image id", + &local_image_id, + &tag_owner, + m_threads->work_queue, + &ctx); + req->send(); + + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageReplayerPrepareLocalImageRequest, TagOwnerError) { + InSequence seq; + expect_mirror_image_get_image_id(m_local_io_ctx, "local image id", 0); + expect_mirror_image_get(m_local_io_ctx, cls::rbd::MIRROR_IMAGE_STATE_ENABLED, + "global image id", 0); + + librbd::MockJournal mock_journal; + expect_get_tag_owner(mock_journal, "local image id", "remote mirror uuid", + -ENOENT); + + std::string local_image_id; + std::string tag_owner; + C_SaferCond ctx; + auto req = MockPrepareLocalImageRequest::create(m_local_io_ctx, + "global image id", + &local_image_id, + &tag_owner, + m_threads->work_queue, + &ctx); + req->send(); + + ASSERT_EQ(-ENOENT, ctx.wait()); +} + +} // namespace image_replayer +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/test_mock_ImageReplayer.cc b/src/test/rbd_mirror/test_mock_ImageReplayer.cc index 01a31c950a70..61dce0e2b293 100644 --- a/src/test/rbd_mirror/test_mock_ImageReplayer.cc +++ b/src/test/rbd_mirror/test_mock_ImageReplayer.cc @@ -8,6 +8,7 @@ #include "tools/rbd_mirror/image_replayer/BootstrapRequest.h" #include "tools/rbd_mirror/image_replayer/CloseImageRequest.h" #include "tools/rbd_mirror/image_replayer/EventPreprocessor.h" +#include "tools/rbd_mirror/image_replayer/PrepareLocalImageRequest.h" #include "tools/rbd_mirror/ImageSyncThrottler.h" #include "test/rbd_mirror/test_mock_fixture.h" #include "test/journal/mock/MockJournaler.h" @@ -90,6 +91,33 @@ using ::testing::Return; using ::testing::SetArgPointee; using ::testing::WithArg; +template<> +struct PrepareLocalImageRequest { + static PrepareLocalImageRequest* s_instance; + std::string *local_image_id = nullptr; + std::string *tag_owner = nullptr; + Context *on_finish = nullptr; + + static PrepareLocalImageRequest* create(librados::IoCtx &, + const std::string &global_image_id, + std::string *local_image_id, + std::string *tag_owner, + ContextWQ *work_queue, + Context *on_finish) { + assert(s_instance != nullptr); + s_instance->local_image_id = local_image_id; + s_instance->tag_owner = tag_owner; + s_instance->on_finish = on_finish; + return s_instance; + } + + PrepareLocalImageRequest() { + s_instance = this; + } + + MOCK_METHOD0(send, void()); +}; + template<> struct BootstrapRequest { static BootstrapRequest* s_instance; @@ -226,6 +254,7 @@ struct ReplayStatusFormatter { BootstrapRequest* BootstrapRequest::s_instance = nullptr; CloseImageRequest* CloseImageRequest::s_instance = nullptr; EventPreprocessor* EventPreprocessor::s_instance = nullptr; +PrepareLocalImageRequest* PrepareLocalImageRequest::s_instance = nullptr; ReplayStatusFormatter* ReplayStatusFormatter::s_instance = nullptr; } // namespace image_replayer @@ -244,6 +273,7 @@ public: typedef BootstrapRequest MockBootstrapRequest; typedef CloseImageRequest MockCloseImageRequest; typedef EventPreprocessor MockEventPreprocessor; + typedef PrepareLocalImageRequest MockPrepareLocalImageRequest; typedef ReplayStatusFormatter MockReplayStatusFormatter; typedef librbd::journal::Replay MockReplay; typedef ImageReplayer MockImageReplayer; @@ -294,6 +324,20 @@ public: Return(true))); } + void expect_send(MockPrepareLocalImageRequest &mock_request, + const std::string &local_image_id, + const std::string &tag_owner, + int r) { + EXPECT_CALL(mock_request, send()) + .WillOnce(Invoke([&mock_request, local_image_id, tag_owner, r]() { + if (r == 0) { + *mock_request.local_image_id = local_image_id; + *mock_request.tag_owner = tag_owner; + } + mock_request.on_finish->complete(r); + })); + } + void expect_send(MockBootstrapRequest &mock_bootstrap_request, librbd::MockTestImageCtx &mock_local_image_ctx, bool do_resync, int r) { @@ -437,6 +481,7 @@ TEST_F(TestMockImageReplayer, StartStop) { mock_local_image_ctx.journal = &mock_local_journal; journal::MockJournaler mock_remote_journaler; + MockPrepareLocalImageRequest mock_prepare_local_image_request; MockBootstrapRequest mock_bootstrap_request; MockReplay mock_local_replay; MockEventPreprocessor mock_event_preprocessor; @@ -446,6 +491,8 @@ TEST_F(TestMockImageReplayer, StartStop) { InSequence seq; EXPECT_CALL(mock_remote_journaler, construct()); + expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id, + "remote mirror uuid", 0); expect_send(mock_bootstrap_request, mock_local_image_ctx, false, 0); EXPECT_CALL(mock_local_journal, add_listener(_)); @@ -483,12 +530,83 @@ TEST_F(TestMockImageReplayer, StartStop) { ASSERT_EQ(0, stop_ctx.wait()); } +TEST_F(TestMockImageReplayer, LocalImagePrimary) { + create_local_image(); + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + + journal::MockJournaler mock_remote_journaler; + MockPrepareLocalImageRequest mock_prepare_local_image_request; + MockReplayStatusFormatter mock_replay_status_formatter; + + expect_get_or_send_update(mock_replay_status_formatter); + + InSequence seq; + EXPECT_CALL(mock_remote_journaler, construct()); + expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id, + "", 0); + + EXPECT_CALL(mock_remote_journaler, remove_listener(_)); + expect_shut_down(mock_remote_journaler, 0); + + C_SaferCond start_ctx; + m_image_replayer->start(&start_ctx); + ASSERT_EQ(0, start_ctx.wait()); +} + +TEST_F(TestMockImageReplayer, LocalImageDNE) { + create_local_image(); + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + + journal::MockJournaler mock_remote_journaler; + MockPrepareLocalImageRequest mock_prepare_local_image_request; + MockBootstrapRequest mock_bootstrap_request; + MockReplayStatusFormatter mock_replay_status_formatter; + + expect_get_or_send_update(mock_replay_status_formatter); + + InSequence seq; + EXPECT_CALL(mock_remote_journaler, construct()); + expect_send(mock_prepare_local_image_request, "", "", -ENOENT); + expect_send(mock_bootstrap_request, mock_local_image_ctx, false, -EREMOTEIO); + + EXPECT_CALL(mock_remote_journaler, remove_listener(_)); + expect_shut_down(mock_remote_journaler, 0); + + C_SaferCond start_ctx; + m_image_replayer->start(&start_ctx); + ASSERT_EQ(0, start_ctx.wait()); +} + +TEST_F(TestMockImageReplayer, PrepareLocalImageError) { + create_local_image(); + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + + journal::MockJournaler mock_remote_journaler; + MockPrepareLocalImageRequest mock_prepare_local_image_request; + MockReplayStatusFormatter mock_replay_status_formatter; + + expect_get_or_send_update(mock_replay_status_formatter); + + InSequence seq; + EXPECT_CALL(mock_remote_journaler, construct()); + expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id, + "remote mirror uuid", -EINVAL); + + EXPECT_CALL(mock_remote_journaler, remove_listener(_)); + expect_shut_down(mock_remote_journaler, 0); + + C_SaferCond start_ctx; + m_image_replayer->start(&start_ctx); + ASSERT_EQ(-EINVAL, start_ctx.wait()); +} + TEST_F(TestMockImageReplayer, BootstrapError) { create_local_image(); librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); journal::MockJournaler mock_remote_journaler; + MockPrepareLocalImageRequest mock_prepare_local_image_request; MockBootstrapRequest mock_bootstrap_request; MockReplayStatusFormatter mock_replay_status_formatter; @@ -496,6 +614,8 @@ TEST_F(TestMockImageReplayer, BootstrapError) { InSequence seq; EXPECT_CALL(mock_remote_journaler, construct()); + expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id, + "remote mirror uuid", 0); expect_send(mock_bootstrap_request, mock_local_image_ctx, false, -EINVAL); EXPECT_CALL(mock_remote_journaler, remove_listener(_)); @@ -516,6 +636,7 @@ TEST_F(TestMockImageReplayer, StartExternalReplayError) { mock_local_image_ctx.journal = &mock_local_journal; journal::MockJournaler mock_remote_journaler; + MockPrepareLocalImageRequest mock_prepare_local_image_request; MockBootstrapRequest mock_bootstrap_request; MockReplay mock_local_replay; MockEventPreprocessor mock_event_preprocessor; @@ -525,6 +646,8 @@ TEST_F(TestMockImageReplayer, StartExternalReplayError) { InSequence seq; EXPECT_CALL(mock_remote_journaler, construct()); + expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id, + "remote mirror uuid", 0); expect_send(mock_bootstrap_request, mock_local_image_ctx, false, 0); EXPECT_CALL(mock_local_journal, add_listener(_)); @@ -560,6 +683,7 @@ TEST_F(TestMockImageReplayer, StopError) { mock_local_image_ctx.journal = &mock_local_journal; journal::MockJournaler mock_remote_journaler; + MockPrepareLocalImageRequest mock_prepare_local_image_request; MockBootstrapRequest mock_bootstrap_request; MockReplay mock_local_replay; MockEventPreprocessor mock_event_preprocessor; @@ -569,6 +693,8 @@ TEST_F(TestMockImageReplayer, StopError) { InSequence seq; EXPECT_CALL(mock_remote_journaler, construct()); + expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id, + "remote mirror uuid", 0); expect_send(mock_bootstrap_request, mock_local_image_ctx, false, 0); EXPECT_CALL(mock_local_journal, add_listener(_)); @@ -616,6 +742,7 @@ TEST_F(TestMockImageReplayer, Replay) { mock_local_image_ctx.journal = &mock_local_journal; journal::MockJournaler mock_remote_journaler; + MockPrepareLocalImageRequest mock_prepare_local_image_request; MockBootstrapRequest mock_bootstrap_request; MockReplay mock_local_replay; MockEventPreprocessor mock_event_preprocessor; @@ -628,6 +755,8 @@ TEST_F(TestMockImageReplayer, Replay) { InSequence seq; EXPECT_CALL(mock_remote_journaler, construct()); + expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id, + "remote mirror uuid", 0); expect_send(mock_bootstrap_request, mock_local_image_ctx, false, 0); EXPECT_CALL(mock_local_journal, add_listener(_)); @@ -714,6 +843,7 @@ TEST_F(TestMockImageReplayer, DecodeError) { mock_local_image_ctx.journal = &mock_local_journal; journal::MockJournaler mock_remote_journaler; + MockPrepareLocalImageRequest mock_prepare_local_image_request; MockBootstrapRequest mock_bootstrap_request; MockReplay mock_local_replay; MockEventPreprocessor mock_event_preprocessor; @@ -725,6 +855,8 @@ TEST_F(TestMockImageReplayer, DecodeError) { InSequence seq; EXPECT_CALL(mock_remote_journaler, construct()); + expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id, + "remote mirror uuid", 0); expect_send(mock_bootstrap_request, mock_local_image_ctx, false, 0); EXPECT_CALL(mock_local_journal, add_listener(_)); @@ -802,6 +934,7 @@ TEST_F(TestMockImageReplayer, DelayedReplay) { mock_local_image_ctx.journal = &mock_local_journal; journal::MockJournaler mock_remote_journaler; + MockPrepareLocalImageRequest mock_prepare_local_image_request; MockBootstrapRequest mock_bootstrap_request; MockReplay mock_local_replay; MockEventPreprocessor mock_event_preprocessor; @@ -814,6 +947,8 @@ TEST_F(TestMockImageReplayer, DelayedReplay) { InSequence seq; EXPECT_CALL(mock_remote_journaler, construct()); + expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id, + "remote mirror uuid", 0); expect_send(mock_bootstrap_request, mock_local_image_ctx, false, 0); EXPECT_CALL(mock_local_journal, add_listener(_)); diff --git a/src/tools/rbd_mirror/CMakeLists.txt b/src/tools/rbd_mirror/CMakeLists.txt index 3fb0536c762a..1a9d0fb1aab4 100644 --- a/src/tools/rbd_mirror/CMakeLists.txt +++ b/src/tools/rbd_mirror/CMakeLists.txt @@ -25,6 +25,7 @@ set(rbd_mirror_internal image_replayer/IsPrimaryRequest.cc image_replayer/OpenImageRequest.cc image_replayer/OpenLocalImageRequest.cc + image_replayer/PrepareLocalImageRequest.cc image_replayer/ReplayStatusFormatter.cc image_sync/ImageCopyRequest.cc image_sync/ObjectCopyRequest.cc diff --git a/src/tools/rbd_mirror/ImageReplayer.cc b/src/tools/rbd_mirror/ImageReplayer.cc index f9a38c2fe3fb..bb56c6e95a57 100644 --- a/src/tools/rbd_mirror/ImageReplayer.cc +++ b/src/tools/rbd_mirror/ImageReplayer.cc @@ -26,6 +26,7 @@ #include "tools/rbd_mirror/image_replayer/BootstrapRequest.h" #include "tools/rbd_mirror/image_replayer/CloseImageRequest.h" #include "tools/rbd_mirror/image_replayer/EventPreprocessor.h" +#include "tools/rbd_mirror/image_replayer/PrepareLocalImageRequest.h" #include "tools/rbd_mirror/image_replayer/ReplayStatusFormatter.h" #define dout_context g_ceph_context @@ -417,6 +418,37 @@ void ImageReplayer::start(Context *on_finish, bool manual) m_remote_image.io_ctx, m_remote_image.image_id, m_local_mirror_uuid, settings); + prepare_local_image(); +} + +template +void ImageReplayer::prepare_local_image() { + dout(20) << dendl; + + Context *ctx = create_context_callback< + ImageReplayer, &ImageReplayer::handle_prepare_local_image>(this); + auto req = PrepareLocalImageRequest::create( + m_local_ioctx, m_global_image_id, &m_local_image_id, + &m_local_image_tag_owner, m_threads->work_queue, ctx); + req->send(); +} + +template +void ImageReplayer::handle_prepare_local_image(int r) { + dout(20) << "r=" << r << dendl; + + if (r == -ENOENT) { + dout(20) << "local image does not exist" << dendl; + } else if (r < 0) { + on_start_fail(r, "error preparing local image for replay"); + return; + } else if (m_local_image_tag_owner == librbd::Journal<>::LOCAL_MIRROR_UUID) { + dout(5) << "local image is primary" << dendl; + on_start_fail(0, "local image is primary"); + return; + } + + // local image doesn't exist or is non-primary bootstrap(); } @@ -459,10 +491,12 @@ void ImageReplayer::handle_bootstrap(int r) { } if (r == -EREMOTEIO) { + m_local_image_tag_owner = ""; dout(5) << "remote image is non-primary or local image is primary" << dendl; on_start_fail(0, "remote image is non-primary or local image is primary"); return; } else if (r == -EEXIST) { + m_local_image_tag_owner = ""; on_start_fail(r, "split-brain detected"); return; } else if (r < 0) { diff --git a/src/tools/rbd_mirror/ImageReplayer.h b/src/tools/rbd_mirror/ImageReplayer.h index 934df3e460a4..5fc8bee0ec68 100644 --- a/src/tools/rbd_mirror/ImageReplayer.h +++ b/src/tools/rbd_mirror/ImageReplayer.h @@ -143,6 +143,9 @@ protected: * * * | * * v (error) * + * PREPARE_LOCAL_IMAGE * * * * * * * * * * * * * * * * * * + * | * + * v (error) * * BOOTSTRAP_IMAGE * * * * * * * * * * * * * * * * * * * * * | * * v (error) * @@ -301,6 +304,7 @@ private: nullptr; librados::IoCtx m_local_ioctx; ImageCtxT *m_local_image_ctx = nullptr; + std::string m_local_image_tag_owner; decltype(ImageCtxT::journal) m_local_journal = nullptr; librbd::journal::Replay *m_local_replay = nullptr; @@ -385,6 +389,9 @@ private: void handle_shut_down(int r); void handle_remote_journal_metadata_updated(); + void prepare_local_image(); + void handle_prepare_local_image(int r); + void bootstrap(); void handle_bootstrap(int r); diff --git a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc index 7d7d5ed1a6c3..919bbbfd1104 100644 --- a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc +++ b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc @@ -79,7 +79,7 @@ template void BootstrapRequest::send() { *m_do_resync = false; - get_local_image_id(); + get_remote_tag_class(); } template @@ -92,45 +92,6 @@ void BootstrapRequest::cancel() { m_image_sync_throttler->cancel_sync(m_local_io_ctx, m_local_image_id); } -template -void BootstrapRequest::get_local_image_id() { - dout(20) << dendl; - - update_progress("GET_LOCAL_IMAGE_ID"); - - // attempt to cross-reference a local image by the global image id - librados::ObjectReadOperation op; - librbd::cls_client::mirror_image_get_image_id_start(&op, m_global_image_id); - - librados::AioCompletion *aio_comp = create_rados_callback< - BootstrapRequest, &BootstrapRequest::handle_get_local_image_id>( - this); - int r = m_local_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op, &m_out_bl); - assert(r == 0); - aio_comp->release(); -} - -template -void BootstrapRequest::handle_get_local_image_id(int r) { - dout(20) << ": r=" << r << dendl; - - if (r == 0) { - bufferlist::iterator iter = m_out_bl.begin(); - r = librbd::cls_client::mirror_image_get_image_id_finish( - &iter, &m_local_image_id); - } - - if (r == -ENOENT) { - dout(10) << ": image not registered locally" << dendl; - } else if (r < 0) { - derr << ": failed to retrieve local image id: " << cpp_strerror(r) << dendl; - finish(r); - return; - } - - get_remote_tag_class(); -} - template void BootstrapRequest::get_remote_tag_class() { dout(20) << dendl; diff --git a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h index e367b2ae4826..00636f03aea7 100644 --- a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h +++ b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h @@ -87,9 +87,6 @@ private: * * | * v - * GET_LOCAL_IMAGE_ID * * * * * * * * * * * * * * * * * - * | * - * v * * GET_REMOTE_TAG_CLASS * * * * * * * * * * * * * * * * * | * * v * @@ -175,9 +172,6 @@ private: bufferlist m_out_bl; - void get_local_image_id(); - void handle_get_local_image_id(int r); - void get_remote_tag_class(); void handle_get_remote_tag_class(int r); diff --git a/src/tools/rbd_mirror/image_replayer/PrepareLocalImageRequest.cc b/src/tools/rbd_mirror/image_replayer/PrepareLocalImageRequest.cc new file mode 100644 index 000000000000..b26ac05e9429 --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/PrepareLocalImageRequest.cc @@ -0,0 +1,160 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "tools/rbd_mirror/image_replayer/PrepareLocalImageRequest.h" +#include "include/rados/librados.hpp" +#include "cls/rbd/cls_rbd_client.h" +#include "common/errno.h" +#include "librbd/ImageCtx.h" +#include "librbd/Journal.h" +#include "librbd/Utils.h" +#include "tools/rbd_mirror/Threads.h" +#include + +#define dout_context g_ceph_context +#define dout_subsys ceph_subsys_rbd_mirror +#undef dout_prefix +#define dout_prefix *_dout << "rbd::mirror::image_replayer::" \ + << "PrepareLocalImageRequest: " << this << " " \ + << __func__ << ": " + +namespace rbd { +namespace mirror { +namespace image_replayer { + +using librbd::util::create_context_callback; +using librbd::util::create_rados_callback; + +template +void PrepareLocalImageRequest::send() { + dout(20) << dendl; + get_local_image_id(); +} + +template +void PrepareLocalImageRequest::get_local_image_id() { + dout(20) << dendl; + + // attempt to cross-reference a local image by the global image id + librados::ObjectReadOperation op; + librbd::cls_client::mirror_image_get_image_id_start(&op, m_global_image_id); + + m_out_bl.clear(); + librados::AioCompletion *aio_comp = create_rados_callback< + PrepareLocalImageRequest, + &PrepareLocalImageRequest::handle_get_local_image_id>( + this); + int r = m_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op, &m_out_bl); + assert(r == 0); + aio_comp->release(); +} + +template +void PrepareLocalImageRequest::handle_get_local_image_id(int r) { + if (r == 0) { + bufferlist::iterator iter = m_out_bl.begin(); + r = librbd::cls_client::mirror_image_get_image_id_finish( + &iter, m_local_image_id); + } + + dout(20) << "r=" << r << ", " + << "local_image_id=" << *m_local_image_id << dendl; + + if (r < 0) { + if (r == -ENOENT) { + dout(10) << "image not registered locally" << dendl; + } else { + derr << "failed to retrieve local image id: " << cpp_strerror(r) + << dendl; + } + finish(r); + return; + } + + get_mirror_state(); +} + +template +void PrepareLocalImageRequest::get_mirror_state() { + dout(20) << dendl; + + librados::ObjectReadOperation op; + librbd::cls_client::mirror_image_get_start(&op, *m_local_image_id); + + m_out_bl.clear(); + librados::AioCompletion *aio_comp = create_rados_callback< + PrepareLocalImageRequest, + &PrepareLocalImageRequest::handle_get_mirror_state>(this); + int r = m_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op, &m_out_bl); + assert(r == 0); + aio_comp->release(); +} + +template +void PrepareLocalImageRequest::handle_get_mirror_state(int r) { + dout(20) << ": r=" << r << dendl; + + cls::rbd::MirrorImage mirror_image; + if (r == 0) { + bufferlist::iterator iter = m_out_bl.begin(); + r = librbd::cls_client::mirror_image_get_finish(&iter, &mirror_image); + } + + if (r < 0) { + derr << "failed to retrieve image mirror state: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + // TODO save current mirror state to determine if we should + // delete a partially formed image + // (e.g. MIRROR_IMAGE_STATE_CREATING/DELETING) + + get_tag_owner(); +} + +template +void PrepareLocalImageRequest::get_tag_owner() { + // deduce the class type for the journal to support unit tests + using Journal = typename std::decay< + typename std::remove_pointer().journal)> + ::type>::type; + + dout(20) << dendl; + + Context *ctx = create_context_callback< + PrepareLocalImageRequest, + &PrepareLocalImageRequest::handle_get_tag_owner>(this); + Journal::get_tag_owner(m_io_ctx, *m_local_image_id, m_tag_owner, + m_work_queue, ctx); +} + +template +void PrepareLocalImageRequest::handle_get_tag_owner(int r) { + dout(20) << "r=" << r << ", " + << "tag_owner=" << *m_tag_owner << dendl; + + if (r < 0) { + derr << "failed to retrieve journal tag owner: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + finish(0); +} + +template +void PrepareLocalImageRequest::finish(int r) { + dout(20) << "r=" << r << dendl; + + m_on_finish->complete(r); + delete this; +} + +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +template class rbd::mirror::image_replayer::PrepareLocalImageRequest; diff --git a/src/tools/rbd_mirror/image_replayer/PrepareLocalImageRequest.h b/src/tools/rbd_mirror/image_replayer/PrepareLocalImageRequest.h new file mode 100644 index 000000000000..913bfd1c242c --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/PrepareLocalImageRequest.h @@ -0,0 +1,92 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef RBD_MIRROR_IMAGE_REPLAYER_PREPARE_LOCAL_IMAGE_REQUEST_H +#define RBD_MIRROR_IMAGE_REPLAYER_PREPARE_LOCAL_IMAGE_REQUEST_H + +#include "include/buffer.h" +#include + +namespace librados { struct IoCtx; } +namespace librbd { struct ImageCtx; } + +struct Context; +struct ContextWQ; + +namespace rbd { +namespace mirror { +namespace image_replayer { + +template +class PrepareLocalImageRequest { +public: + static PrepareLocalImageRequest *create(librados::IoCtx &io_ctx, + const std::string &global_image_id, + std::string *local_image_id, + std::string *tag_owner, + ContextWQ *work_queue, + Context *on_finish) { + return new PrepareLocalImageRequest(io_ctx, global_image_id, local_image_id, + tag_owner, work_queue, on_finish); + } + + PrepareLocalImageRequest(librados::IoCtx &io_ctx, + const std::string &global_image_id, + std::string *local_image_id, + std::string *tag_owner, + ContextWQ *work_queue, + Context *on_finish) + : m_io_ctx(io_ctx), m_global_image_id(global_image_id), + m_local_image_id(local_image_id), m_tag_owner(tag_owner), + m_work_queue(work_queue), m_on_finish(on_finish) { + } + + void send(); + +private: + /** + * @verbatim + * + * + * | + * v + * GET_LOCAL_IMAGE_ID + * | + * v + * GET_MIRROR_STATE + * | + * v + * + + * @endverbatim + */ + + librados::IoCtx &m_io_ctx; + std::string m_global_image_id; + std::string *m_local_image_id; + std::string *m_tag_owner; + ContextWQ *m_work_queue; + Context *m_on_finish; + + bufferlist m_out_bl; + + void get_local_image_id(); + void handle_get_local_image_id(int r); + + void get_mirror_state(); + void handle_get_mirror_state(int r); + + void get_tag_owner(); + void handle_get_tag_owner(int r); + + void finish(int r); + +}; + +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +extern template class rbd::mirror::image_replayer::PrepareLocalImageRequest; + +#endif // RBD_MIRROR_IMAGE_REPLAYER_PREPARE_LOCAL_IMAGE_REQUEST_H