From: Jason Dillaman Date: Tue, 9 Aug 2016 03:45:46 +0000 (-0400) Subject: rbd-mirror: demote/promote in same cluster results in split-brain X-Git-Tag: ses5-milestone5~100^2~2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=a6901ca1a065419426b3ad704e27e43ba8d591b8;p=ceph.git rbd-mirror: demote/promote in same cluster results in split-brain Fixes: http://tracker.ceph.com/issues/16855 Signed-off-by: Jason Dillaman --- diff --git a/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc b/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc index 7d2e37e0114..18d24f18115 100644 --- a/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc +++ b/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc @@ -5,19 +5,25 @@ #include "librbd/journal/TypeTraits.h" #include "tools/rbd_mirror/ImageSync.h" #include "tools/rbd_mirror/ImageSyncThrottler.h" +#include "tools/rbd_mirror/Threads.h" #include "tools/rbd_mirror/image_replayer/BootstrapRequest.h" #include "tools/rbd_mirror/image_replayer/CloseImageRequest.h" #include "tools/rbd_mirror/image_replayer/CreateImageRequest.h" #include "tools/rbd_mirror/image_replayer/OpenImageRequest.h" #include "tools/rbd_mirror/image_replayer/OpenLocalImageRequest.h" #include "test/journal/mock/MockJournaler.h" +#include "test/librados_test_stub/MockTestMemIoCtxImpl.h" #include "test/librbd/mock/MockImageCtx.h" +#include "test/librbd/mock/MockJournal.h" namespace librbd { namespace { struct MockTestImageCtx : public librbd::MockImageCtx { + MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } }; } // anonymous namespace @@ -90,13 +96,16 @@ namespace image_replayer { template<> struct CloseImageRequest { static CloseImageRequest* s_instance; + librbd::MockTestImageCtx **image_ctx = nullptr; Context *on_finish = nullptr; static CloseImageRequest* create(librbd::MockTestImageCtx **image_ctx, ContextWQ *work_queue, bool destroy_only, Context *on_finish) { assert(s_instance != nullptr); + s_instance->image_ctx = image_ctx; s_instance->on_finish = on_finish; + s_instance->construct(*image_ctx); return s_instance; } @@ -104,7 +113,11 @@ struct CloseImageRequest { assert(s_instance == nullptr); s_instance = this; } + ~CloseImageRequest() { + s_instance = nullptr; + } + MOCK_METHOD1(construct, void(librbd::MockTestImageCtx *image_ctx)); MOCK_METHOD0(send, void()); }; @@ -129,6 +142,9 @@ struct CreateImageRequest { assert(s_instance == nullptr); s_instance = this; } + ~CreateImageRequest() { + s_instance = nullptr; + } MOCK_METHOD0(send, void()); }; @@ -136,15 +152,18 @@ struct CreateImageRequest { template<> struct OpenImageRequest { static OpenImageRequest* s_instance; + librbd::MockTestImageCtx **image_ctx = nullptr; Context *on_finish = nullptr; static OpenImageRequest* create(librados::IoCtx &io_ctx, librbd::MockTestImageCtx **image_ctx, - const std::string &local_image_id, + const std::string &image_id, bool read_only, ContextWQ *work_queue, Context *on_finish) { assert(s_instance != nullptr); + s_instance->image_ctx = image_ctx; s_instance->on_finish = on_finish; + s_instance->construct(io_ctx, image_id); return s_instance; } @@ -152,13 +171,19 @@ struct OpenImageRequest { assert(s_instance == nullptr); s_instance = this; } + ~OpenImageRequest() { + s_instance = nullptr; + } + MOCK_METHOD2(construct, void(librados::IoCtx &io_ctx, + const std::string &image_id)); MOCK_METHOD0(send, void()); }; template<> struct OpenLocalImageRequest { static OpenLocalImageRequest* s_instance; + librbd::MockTestImageCtx **image_ctx = nullptr; Context *on_finish = nullptr; static OpenLocalImageRequest* create(librados::IoCtx &local_io_ctx, @@ -168,7 +193,9 @@ struct OpenLocalImageRequest { ContextWQ *work_queue, Context *on_finish) { assert(s_instance != nullptr); + s_instance->image_ctx = local_image_ctx; s_instance->on_finish = on_finish; + s_instance->construct(local_io_ctx, local_image_id); return s_instance; } @@ -176,7 +203,12 @@ struct OpenLocalImageRequest { assert(s_instance == nullptr); s_instance = this; } + ~OpenLocalImageRequest() { + s_instance = nullptr; + } + MOCK_METHOD2(construct, void(librados::IoCtx &io_ctx, + const std::string &image_id)); MOCK_METHOD0(send, void()); }; @@ -201,12 +233,487 @@ namespace rbd { namespace mirror { namespace image_replayer { +using ::testing::_; +using ::testing::DoAll; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::Return; +using ::testing::SetArgPointee; +using ::testing::StrEq; +using ::testing::WithArg; + +MATCHER_P(IsSameIoCtx, io_ctx, "") { + return &get_mock_io_ctx(arg) == &get_mock_io_ctx(*io_ctx); +} + class TestMockImageReplayerBootstrapRequest : public TestMockFixture { public: + typedef ImageSyncThrottlerRef MockImageSyncThrottler; typedef BootstrapRequest MockBootstrapRequest; + typedef CloseImageRequest MockCloseImageRequest; + typedef OpenImageRequest MockOpenImageRequest; + typedef OpenLocalImageRequest MockOpenLocalImageRequest; + typedef std::list Tags; + + void SetUp() { + TestMockFixture::SetUp(); + + librbd::RBD rbd; + ASSERT_EQ(0, create_image(rbd, m_remote_io_ctx, m_image_name, m_image_size)); + ASSERT_EQ(0, open_image(m_remote_io_ctx, m_image_name, &m_remote_image_ctx)); + } + + void create_local_image() { + librbd::RBD rbd; + ASSERT_EQ(0, create_image(rbd, m_local_io_ctx, m_image_name, m_image_size)); + ASSERT_EQ(0, open_image(m_local_io_ctx, m_image_name, &m_local_image_ctx)); + } + + void expect_mirror_image_get_image_id(librados::IoCtx &io_ctx, + const std::string &global_image_id, + const std::string &image_id, int r) { + bufferlist in_bl; + ::encode(global_image_id, in_bl); + + bufferlist bl; + ::encode(image_id, bl); + + EXPECT_CALL(get_mock_io_ctx(io_ctx), + exec(RBD_MIRRORING, _, StrEq("rbd"), + StrEq("mirror_image_get_image_id"), ContentsEqual(in_bl), + _, _)) + .WillOnce(DoAll(WithArg<5>(Invoke([bl](bufferlist *out_bl) { + *out_bl = bl; + })), + Return(r))); + } + + void expect_journaler_get_client(::journal::MockJournaler &mock_journaler, + const std::string &client_id, + cls::journal::Client &client, int r) { + EXPECT_CALL(mock_journaler, get_client(StrEq(client_id), _, _)) + .WillOnce(DoAll(WithArg<1>(Invoke([client](cls::journal::Client *out_client) { + *out_client = client; + })), + WithArg<2>(Invoke([this, r](Context *on_finish) { + m_threads->work_queue->queue(on_finish, r); + })))); + } + + void expect_journaler_get_tags(::journal::MockJournaler &mock_journaler, + uint64_t tag_class, const Tags& tags, + int r) { + EXPECT_CALL(mock_journaler, get_tags(tag_class, _, _)) + .WillOnce(DoAll(WithArg<1>(Invoke([tags](Tags *out_tags) { + *out_tags = tags; + })), + WithArg<2>(Invoke([this, r](Context *on_finish) { + m_threads->work_queue->queue(on_finish, r); + })))); + } + + void expect_open_image(MockOpenImageRequest &mock_open_image_request, + librados::IoCtx &io_ctx, const std::string &image_id, + librbd::MockTestImageCtx &mock_image_ctx, int r) { + EXPECT_CALL(mock_open_image_request, construct(IsSameIoCtx(&io_ctx), image_id)); + EXPECT_CALL(mock_open_image_request, send()) + .WillOnce(Invoke([this, &mock_open_image_request, &mock_image_ctx, r]() { + *mock_open_image_request.image_ctx = &mock_image_ctx; + m_threads->work_queue->queue(mock_open_image_request.on_finish, r); + })); + } + + void expect_open_local_image(MockOpenLocalImageRequest &mock_open_local_image_request, + librados::IoCtx &io_ctx, const std::string &image_id, + librbd::MockTestImageCtx &mock_image_ctx, int r) { + EXPECT_CALL(mock_open_local_image_request, + construct(IsSameIoCtx(&io_ctx), image_id)); + EXPECT_CALL(mock_open_local_image_request, send()) + .WillOnce(Invoke([this, &mock_open_local_image_request, &mock_image_ctx, r]() { + *mock_open_local_image_request.image_ctx = &mock_image_ctx; + m_threads->work_queue->queue(mock_open_local_image_request.on_finish, + r); + })); + } + + void expect_close_image(MockCloseImageRequest &mock_close_image_request, + librbd::MockTestImageCtx &mock_image_ctx, int r) { + EXPECT_CALL(mock_close_image_request, construct(&mock_image_ctx)); + EXPECT_CALL(mock_close_image_request, send()) + .WillOnce(Invoke([this, &mock_close_image_request, r]() { + *mock_close_image_request.image_ctx = nullptr; + m_threads->work_queue->queue(mock_close_image_request.on_finish, r); + })); + } + + void expect_journal_is_tag_owner(librbd::MockJournal &mock_journal, + bool is_owner, int r) { + EXPECT_CALL(mock_journal, is_tag_owner(_)) + .WillOnce(DoAll(SetArgPointee<0>(is_owner), + Return(r))); + } + + void expect_journal_get_tag_tid(librbd::MockJournal &mock_journal, + uint64_t tag_tid) { + EXPECT_CALL(mock_journal, get_tag_tid()).WillOnce(Return(tag_tid)); + } + + void expect_journal_get_tag_data(librbd::MockJournal &mock_journal, + const librbd::journal::TagData &tag_data) { + EXPECT_CALL(mock_journal, get_tag_data()).WillOnce(Return(tag_data)); + } + + bufferlist encode_tag_data(const librbd::journal::TagData &tag_data) { + bufferlist bl; + ::encode(tag_data, bl); + return bl; + } + + MockBootstrapRequest *create_request(MockImageSyncThrottler mock_image_sync_throttler, + ::journal::MockJournaler &mock_journaler, + const std::string &remote_image_id, + const std::string &global_image_id, + const std::string &local_mirror_uuid, + const std::string &remote_mirror_uuid, + Context *on_finish) { + return new MockBootstrapRequest(m_local_io_ctx, + m_remote_io_ctx, + mock_image_sync_throttler, + &m_local_test_image_ctx, + "local image name", + remote_image_id, + global_image_id, + m_threads->work_queue, + m_threads->timer, + &m_threads->timer_lock, + local_mirror_uuid, + remote_mirror_uuid, + &mock_journaler, + &m_mirror_peer_client_meta, + on_finish); + } + + librbd::ImageCtx *m_remote_image_ctx; + librbd::ImageCtx *m_local_image_ctx = nullptr; + librbd::MockTestImageCtx *m_local_test_image_ctx = nullptr; + librbd::journal::MirrorPeerClientMeta m_mirror_peer_client_meta; }; +TEST_F(TestMockImageReplayerBootstrapRequest, RemoteDemotePromote) { + create_local_image(); + + InSequence seq; + + // look up local image by global image id + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + expect_mirror_image_get_image_id(m_local_io_ctx, "global image id", + mock_local_image_ctx.id, 0); + + // lookup remote image tag class + cls::journal::Client client; + librbd::journal::ClientData client_data{ + librbd::journal::ImageClientMeta{123}}; + ::encode(client_data, client.data); + ::journal::MockJournaler mock_journaler; + expect_journaler_get_client(mock_journaler, + librbd::Journal<>::IMAGE_CLIENT_ID, + client, 0); + + // lookup local peer in remote journal + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{ + mock_local_image_ctx.id}; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + client_data.client_meta = mirror_peer_client_meta; + client.data.clear(); + ::encode(client_data, client.data); + expect_journaler_get_client(mock_journaler, "local mirror uuid", + client, 0); + + // open the remote image + librbd::MockJournal mock_journal; + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + MockOpenImageRequest mock_open_image_request; + expect_open_image(mock_open_image_request, m_remote_io_ctx, + mock_remote_image_ctx.id, mock_remote_image_ctx, 0); + expect_journal_is_tag_owner(mock_journal, true, 0); + + // open the local image + mock_local_image_ctx.journal = &mock_journal; + MockOpenLocalImageRequest mock_open_local_image_request; + expect_open_local_image(mock_open_local_image_request, m_local_io_ctx, + mock_local_image_ctx.id, mock_local_image_ctx, 0); + + // remote demotion / promotion event + Tags tags = { + {2, 123, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 1, 99})}, + {3, 123, encode_tag_data({librbd::Journal<>::ORPHAN_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 2, 1})}, + {4, 123, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::ORPHAN_MIRROR_UUID, + true, 2, 1})}, + {5, 123, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::ORPHAN_MIRROR_UUID, + true, 4, 369})} + }; + expect_journaler_get_tags(mock_journaler, 123, tags, 0); + expect_journal_get_tag_tid(mock_journal, 345); + expect_journal_get_tag_data(mock_journal, {"remote mirror uuid"}); + + MockCloseImageRequest mock_close_image_request; + expect_close_image(mock_close_image_request, mock_remote_image_ctx, 0); + + C_SaferCond ctx; + MockImageSyncThrottler mock_image_sync_throttler( + new ImageSyncThrottler()); + MockBootstrapRequest *request = create_request( + mock_image_sync_throttler, mock_journaler, mock_remote_image_ctx.id, + "global image id", "local mirror uuid", "remote mirror uuid", + &ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageReplayerBootstrapRequest, MultipleRemoteDemotePromotes) { + create_local_image(); + + InSequence seq; + + // look up local image by global image id + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + expect_mirror_image_get_image_id(m_local_io_ctx, "global image id", + mock_local_image_ctx.id, 0); + + // lookup remote image tag class + cls::journal::Client client; + librbd::journal::ClientData client_data{ + librbd::journal::ImageClientMeta{123}}; + ::encode(client_data, client.data); + ::journal::MockJournaler mock_journaler; + expect_journaler_get_client(mock_journaler, + librbd::Journal<>::IMAGE_CLIENT_ID, + client, 0); + + // lookup local peer in remote journal + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{ + mock_local_image_ctx.id}; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + client_data.client_meta = mirror_peer_client_meta; + client.data.clear(); + ::encode(client_data, client.data); + expect_journaler_get_client(mock_journaler, "local mirror uuid", + client, 0); + + // open the remote image + librbd::MockJournal mock_journal; + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + MockOpenImageRequest mock_open_image_request; + expect_open_image(mock_open_image_request, m_remote_io_ctx, + mock_remote_image_ctx.id, mock_remote_image_ctx, 0); + expect_journal_is_tag_owner(mock_journal, true, 0); + + // open the local image + mock_local_image_ctx.journal = &mock_journal; + MockOpenLocalImageRequest mock_open_local_image_request; + expect_open_local_image(mock_open_local_image_request, m_local_io_ctx, + mock_local_image_ctx.id, mock_local_image_ctx, 0); + + // remote demotion / promotion event + Tags tags = { + {2, 123, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 1, 99})}, + {3, 123, encode_tag_data({librbd::Journal<>::ORPHAN_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 2, 1})}, + {4, 123, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::ORPHAN_MIRROR_UUID, + true, 3, 1})}, + {5, 123, encode_tag_data({librbd::Journal<>::ORPHAN_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 4, 1})}, + {6, 123, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::ORPHAN_MIRROR_UUID, + true, 5, 1})}, + {7, 123, encode_tag_data({librbd::Journal<>::ORPHAN_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 6, 1})}, + {8, 123, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::ORPHAN_MIRROR_UUID, + true, 7, 1})} + }; + expect_journaler_get_tags(mock_journaler, 123, tags, 0); + expect_journal_get_tag_tid(mock_journal, 345); + expect_journal_get_tag_data(mock_journal, {librbd::Journal<>::ORPHAN_MIRROR_UUID, + "remote mirror uuid", true, 4, 1}); + + MockCloseImageRequest mock_close_image_request; + expect_close_image(mock_close_image_request, mock_remote_image_ctx, 0); + + C_SaferCond ctx; + MockImageSyncThrottler mock_image_sync_throttler( + new ImageSyncThrottler()); + MockBootstrapRequest *request = create_request( + mock_image_sync_throttler, mock_journaler, mock_remote_image_ctx.id, + "global image id", "local mirror uuid", "remote mirror uuid", + &ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageReplayerBootstrapRequest, LocalDemoteRemotePromote) { + create_local_image(); + + InSequence seq; + + // look up local image by global image id + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + expect_mirror_image_get_image_id(m_local_io_ctx, "global image id", + mock_local_image_ctx.id, 0); + + // lookup remote image tag class + cls::journal::Client client; + librbd::journal::ClientData client_data{ + librbd::journal::ImageClientMeta{123}}; + ::encode(client_data, client.data); + ::journal::MockJournaler mock_journaler; + expect_journaler_get_client(mock_journaler, + librbd::Journal<>::IMAGE_CLIENT_ID, + client, 0); + + // lookup local peer in remote journal + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{ + mock_local_image_ctx.id}; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + client_data.client_meta = mirror_peer_client_meta; + client.data.clear(); + ::encode(client_data, client.data); + expect_journaler_get_client(mock_journaler, "local mirror uuid", + client, 0); + + // open the remote image + librbd::MockJournal mock_journal; + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + MockOpenImageRequest mock_open_image_request; + expect_open_image(mock_open_image_request, m_remote_io_ctx, + mock_remote_image_ctx.id, mock_remote_image_ctx, 0); + expect_journal_is_tag_owner(mock_journal, true, 0); + + // open the local image + mock_local_image_ctx.journal = &mock_journal; + MockOpenLocalImageRequest mock_open_local_image_request; + expect_open_local_image(mock_open_local_image_request, m_local_io_ctx, + mock_local_image_ctx.id, mock_local_image_ctx, 0); + + // remote demotion / promotion event + Tags tags = { + {2, 123, encode_tag_data({"local mirror uuid", "local mirror uuid", + true, 344, 99})}, + {3, 123, encode_tag_data({librbd::Journal<>::ORPHAN_MIRROR_UUID, + "local mirror uuid", true, 345, 1})}, + {4, 123, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::ORPHAN_MIRROR_UUID, + true, 3, 1})} + }; + expect_journaler_get_tags(mock_journaler, 123, tags, 0); + expect_journal_get_tag_tid(mock_journal, 346); + expect_journal_get_tag_data(mock_journal, + {librbd::Journal<>::ORPHAN_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 345, 1}); + + MockCloseImageRequest mock_close_image_request; + expect_close_image(mock_close_image_request, mock_remote_image_ctx, 0); + + C_SaferCond ctx; + MockImageSyncThrottler mock_image_sync_throttler( + new ImageSyncThrottler()); + MockBootstrapRequest *request = create_request( + mock_image_sync_throttler, mock_journaler, mock_remote_image_ctx.id, + "global image id", "local mirror uuid", "remote mirror uuid", + &ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageReplayerBootstrapRequest, SplitBrainForcePromote) { + create_local_image(); + + InSequence seq; + + // look up local image by global image id + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + expect_mirror_image_get_image_id(m_local_io_ctx, "global image id", + mock_local_image_ctx.id, 0); + + // lookup remote image tag class + cls::journal::Client client; + librbd::journal::ClientData client_data{ + librbd::journal::ImageClientMeta{123}}; + ::encode(client_data, client.data); + ::journal::MockJournaler mock_journaler; + expect_journaler_get_client(mock_journaler, + librbd::Journal<>::IMAGE_CLIENT_ID, + client, 0); + + // lookup local peer in remote journal + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{ + mock_local_image_ctx.id}; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + client_data.client_meta = mirror_peer_client_meta; + client.data.clear(); + ::encode(client_data, client.data); + expect_journaler_get_client(mock_journaler, "local mirror uuid", + client, 0); + + // open the remote image + librbd::MockJournal mock_journal; + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + MockOpenImageRequest mock_open_image_request; + expect_open_image(mock_open_image_request, m_remote_io_ctx, + mock_remote_image_ctx.id, mock_remote_image_ctx, 0); + expect_journal_is_tag_owner(mock_journal, true, 0); + + // open the local image + mock_local_image_ctx.journal = &mock_journal; + MockOpenLocalImageRequest mock_open_local_image_request; + expect_open_local_image(mock_open_local_image_request, m_local_io_ctx, + mock_local_image_ctx.id, mock_local_image_ctx, 0); + + // remote demotion / promotion event + Tags tags = { + {2, 123, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 1, 99})}, + {3, 123, encode_tag_data({librbd::Journal<>::ORPHAN_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 2, 1})} + }; + expect_journaler_get_tags(mock_journaler, 123, tags, 0); + expect_journal_get_tag_tid(mock_journal, 345); + expect_journal_get_tag_data(mock_journal, {librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::ORPHAN_MIRROR_UUID, + true, 344, 0}); + + MockCloseImageRequest mock_close_image_request; + expect_close_image(mock_close_image_request, mock_local_image_ctx, 0); + expect_close_image(mock_close_image_request, mock_remote_image_ctx, 0); + + C_SaferCond ctx; + MockImageSyncThrottler mock_image_sync_throttler( + new ImageSyncThrottler()); + MockBootstrapRequest *request = create_request( + mock_image_sync_throttler, mock_journaler, mock_remote_image_ctx.id, + "global image id", "local mirror uuid", "remote mirror uuid", + &ctx); + request->send(); + ASSERT_EQ(-EEXIST, ctx.wait()); + ASSERT_EQ(NULL, m_local_test_image_ctx); +} + } // namespace image_replayer } // namespace mirror } // namespace rbd diff --git a/src/test/rbd_mirror/test_mock_fixture.h b/src/test/rbd_mirror/test_mock_fixture.h index 5ce1a748bd6..0952f9c3b74 100644 --- a/src/test/rbd_mirror/test_mock_fixture.h +++ b/src/test/rbd_mirror/test_mock_fixture.h @@ -23,6 +23,12 @@ ACTION_P(CompleteContext, r) { arg0->complete(r); } +MATCHER_P(ContentsEqual, bl, "") { + // TODO fix const-correctness of bufferlist + return const_cast(arg).contents_equal( + const_cast(bl)); +} + namespace rbd { namespace mirror { diff --git a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc index 2fffebe642a..1687ce5580d 100644 --- a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc +++ b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc @@ -442,9 +442,10 @@ void BootstrapRequest::get_remote_tags() { update_progress("GET_REMOTE_TAGS"); - if (m_created_local_image) { + if (m_created_local_image || + m_client_meta->state == librbd::journal::MIRROR_PEER_STATE_SYNCING) { // optimization -- no need to compare remote tags if we just created - // the image locally + // the image locally or sync was interrupted image_sync(); return; } @@ -474,32 +475,11 @@ void BootstrapRequest::handle_get_remote_tags(int r) { return; } - // decode the remote tags - librbd::journal::TagData remote_tag_data; - for (auto &tag : m_remote_tags) { - try { - bufferlist::iterator it = tag.data.begin(); - ::decode(remote_tag_data, it); - } catch (const buffer::error &err) { - derr << ": failed to decode remote tag " << tag.tid << ": " - << err.what() << dendl; - m_ret_val = -EBADMSG; - close_local_image(); - return; - } - - dout(10) << ": decoded remote tag " << tag.tid << ": " - << remote_tag_data << dendl; - if (remote_tag_data.mirror_uuid == librbd::Journal<>::ORPHAN_MIRROR_UUID && - remote_tag_data.predecessor.mirror_uuid == m_local_mirror_uuid) { - // remote tag is chained off a local tag demotion - break; - } - } - - // At this point, the local image was existing and non-primary and the remote - // image is primary. Attempt to link the local image's most recent tag - // to the remote image's tag chain. + // At this point, the local image was existing, non-primary, and replaying; + // and the remote image is primary. Attempt to link the local image's most + // recent tag to the remote image's tag chain. + uint64_t local_tag_tid; + librbd::journal::TagData local_tag_data; I *local_image_ctx = (*m_local_image_ctx); { RWLock::RLocker snap_locker(local_image_ctx->snap_lock); @@ -510,27 +490,107 @@ void BootstrapRequest::handle_get_remote_tags(int r) { return; } - uint64_t local_tag_tid = local_image_ctx->journal->get_tag_tid(); - librbd::journal::TagData local_tag_data = - local_image_ctx->journal->get_tag_data(); + local_tag_tid = local_image_ctx->journal->get_tag_tid(); + local_tag_data = local_image_ctx->journal->get_tag_data(); dout(20) << ": local tag " << local_tag_tid << ": " << local_tag_data << dendl; + } - if (local_tag_data.mirror_uuid == librbd::Journal<>::ORPHAN_MIRROR_UUID && - remote_tag_data.mirror_uuid == librbd::Journal<>::ORPHAN_MIRROR_UUID && - remote_tag_data.predecessor.mirror_uuid == m_local_mirror_uuid) { - dout(20) << ": local image was demoted" << dendl; - } else if (local_tag_data.mirror_uuid == m_remote_mirror_uuid && - m_client_meta->state == librbd::journal::MIRROR_PEER_STATE_REPLAYING) { - dout(20) << ": local image is in clean replay state" << dendl; - } else if (m_client_meta->state == librbd::journal::MIRROR_PEER_STATE_SYNCING) { - dout(20) << ": previous sync was canceled" << dendl; - } else { - derr << ": split-brain detected -- skipping image replay" << dendl; - m_ret_val = -EEXIST; + bool remote_tag_data_valid = false; + librbd::journal::TagData remote_tag_data; + boost::optional remote_orphan_tag_tid = + boost::make_optional(false, 0U); + bool reconnect_orphan = false; + + // decode the remote tags + for (auto &remote_tag : m_remote_tags) { + if (local_tag_data.predecessor.commit_valid && + local_tag_data.predecessor.mirror_uuid == m_remote_mirror_uuid && + local_tag_data.predecessor.tag_tid > remote_tag.tid) { + dout(20) << ": skipping processed predecessor remote tag " + << remote_tag.tid << dendl; + continue; + } + + try { + bufferlist::iterator it = remote_tag.data.begin(); + ::decode(remote_tag_data, it); + remote_tag_data_valid = true; + } catch (const buffer::error &err) { + derr << ": failed to decode remote tag " << remote_tag.tid << ": " + << err.what() << dendl; + m_ret_val = -EBADMSG; close_local_image(); return; } + + dout(10) << ": decoded remote tag " << remote_tag.tid << ": " + << remote_tag_data << dendl; + + if (!local_tag_data.predecessor.commit_valid) { + // newly synced local image (no predecessor) replays from the first tag + if (remote_tag_data.mirror_uuid != librbd::Journal<>::LOCAL_MIRROR_UUID) { + dout(20) << ": skipping non-primary remote tag" << dendl; + continue; + } + + dout(20) << ": using initial primary remote tag" << dendl; + break; + } + + if (local_tag_data.mirror_uuid == librbd::Journal<>::ORPHAN_MIRROR_UUID) { + // demotion last available local epoch + + if (remote_tag_data.mirror_uuid == local_tag_data.mirror_uuid && + remote_tag_data.predecessor.commit_valid && + remote_tag_data.predecessor.tag_tid == + local_tag_data.predecessor.tag_tid) { + // demotion matches remote epoch + + if (remote_tag_data.predecessor.mirror_uuid == m_local_mirror_uuid && + local_tag_data.predecessor.mirror_uuid == + librbd::Journal<>::LOCAL_MIRROR_UUID) { + // local demoted and remote has matching event + dout(20) << ": found matching local demotion tag" << dendl; + remote_orphan_tag_tid = remote_tag.tid; + continue; + } + + if (local_tag_data.predecessor.mirror_uuid == m_remote_mirror_uuid && + remote_tag_data.predecessor.mirror_uuid == + librbd::Journal<>::LOCAL_MIRROR_UUID) { + // remote demoted and local has matching event + dout(20) << ": found matching remote demotion tag" << dendl; + remote_orphan_tag_tid = remote_tag.tid; + continue; + } + } + + if (remote_tag_data.mirror_uuid == librbd::Journal<>::LOCAL_MIRROR_UUID && + remote_tag_data.predecessor.mirror_uuid == librbd::Journal<>::ORPHAN_MIRROR_UUID && + remote_tag_data.predecessor.commit_valid && remote_orphan_tag_tid && + remote_tag_data.predecessor.tag_tid == *remote_orphan_tag_tid) { + // remote promotion tag chained to remote/local demotion tag + dout(20) << ": found chained remote promotion tag" << dendl; + reconnect_orphan = true; + break; + } + + // promotion must follow demotion + remote_orphan_tag_tid = boost::none; + } + } + + if (remote_tag_data_valid && + local_tag_data.mirror_uuid == m_remote_mirror_uuid) { + dout(20) << ": local image is in clean replay state" << dendl; + } else if (reconnect_orphan) { + dout(20) << ": remote image was demoted/promoted" << dendl; + } else { + derr << ": split-brain detected -- skipping image replay" << dendl; + m_ret_val = -EEXIST; + close_local_image(); + return; } image_sync();