From: Jason Dillaman Date: Fri, 20 Dec 2019 20:09:26 +0000 (-0500) Subject: rbd-mirror: extract pre-journal replay handling to new state machine X-Git-Tag: v15.1.0~352^2~6 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=d711e6b769f78a408fe29bca7c9e6c42325255de;p=ceph.git rbd-mirror: extract pre-journal replay handling to new state machine The new state machine will check for resync/sync states and split-brain conditions. Signed-off-by: Jason Dillaman --- diff --git a/src/test/rbd_mirror/CMakeLists.txt b/src/test/rbd_mirror/CMakeLists.txt index 32d6c16f30d..ddbb1bcf9af 100644 --- a/src/test/rbd_mirror/CMakeLists.txt +++ b/src/test/rbd_mirror/CMakeLists.txt @@ -39,6 +39,7 @@ add_executable(unittest_rbd_mirror image_replayer/test_mock_PrepareLocalImageRequest.cc image_replayer/test_mock_PrepareRemoteImageRequest.cc image_replayer/journal/test_mock_CreateLocalImageRequest.cc + image_replayer/journal/test_mock_PrepareReplayRequest.cc image_replayer/journal/test_mock_EventPreprocessor.cc image_replayer/journal/test_mock_Replayer.cc image_sync/test_mock_SyncPointCreateRequest.cc diff --git a/src/test/rbd_mirror/image_replayer/journal/test_mock_PrepareReplayRequest.cc b/src/test/rbd_mirror/image_replayer/journal/test_mock_PrepareReplayRequest.cc new file mode 100644 index 00000000000..a2dbc5a69d2 --- /dev/null +++ b/src/test/rbd_mirror/image_replayer/journal/test_mock_PrepareReplayRequest.cc @@ -0,0 +1,768 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/rbd_mirror/test_mock_fixture.h" +#include "librbd/journal/Types.h" +#include "librbd/journal/TypeTraits.h" +#include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/image_replayer/journal/PrepareReplayRequest.h" +#include "test/journal/mock/MockJournaler.h" +#include "test/librbd/mock/MockImageCtx.h" +#include "test/librbd/mock/MockJournal.h" + +namespace librbd { + +namespace { + +struct MockTestImageCtx : public librbd::MockImageCtx { + MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace + +namespace journal { + +template <> +struct TypeTraits { + typedef ::journal::MockJournaler Journaler; +}; + +} // namespace journal +} // namespace rbd + +// template definitions +#include "tools/rbd_mirror/image_replayer/journal/PrepareReplayRequest.cc" + +namespace rbd { +namespace mirror { +namespace image_replayer { +namespace journal { + +using ::testing::_; +using ::testing::DoAll; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::Return; +using ::testing::SetArgPointee; +using ::testing::StrEq; +using ::testing::WithArg; + +class TestMockImageReplayerJournalPrepareReplayRequest : public TestMockFixture { +public: + typedef PrepareReplayRequest MockPrepareReplayRequest; + typedef std::list Tags; + + void SetUp() override { + TestMockFixture::SetUp(); + + librbd::RBD rbd; + ASSERT_EQ(0, create_image(rbd, m_local_io_ctx, m_image_name, m_image_size)); + ASSERT_EQ(0, open_image(m_local_io_ctx, m_image_name, &m_local_image_ctx)); + } + + void expect_journaler_get_client(::journal::MockJournaler &mock_journaler, + const std::string &client_id, + cls::journal::Client &client, int r) { + EXPECT_CALL(mock_journaler, get_client(StrEq(client_id), _, _)) + .WillOnce(DoAll(WithArg<1>(Invoke([client](cls::journal::Client *out_client) { + *out_client = client; + })), + WithArg<2>(Invoke([this, r](Context *on_finish) { + m_threads->work_queue->queue(on_finish, r); + })))); + } + + void expect_journaler_update_client(::journal::MockJournaler &mock_journaler, + const librbd::journal::ClientData &client_data, + int r) { + bufferlist bl; + encode(client_data, bl); + + EXPECT_CALL(mock_journaler, update_client(ContentsEqual(bl), _)) + .WillOnce(WithArg<1>(Invoke([this, r](Context *on_finish) { + m_threads->work_queue->queue(on_finish, r); + }))); + } + + void expect_journaler_get_tags(::journal::MockJournaler &mock_journaler, + uint64_t tag_class, const Tags& tags, + int r) { + EXPECT_CALL(mock_journaler, get_tags(tag_class, _, _)) + .WillOnce(DoAll(WithArg<1>(Invoke([tags](Tags *out_tags) { + *out_tags = tags; + })), + WithArg<2>(Invoke([this, r](Context *on_finish) { + m_threads->work_queue->queue(on_finish, r); + })))); + } + + void expect_journal_get_tag_tid(librbd::MockJournal &mock_journal, + uint64_t tag_tid) { + EXPECT_CALL(mock_journal, get_tag_tid()).WillOnce(Return(tag_tid)); + } + + void expect_journal_get_tag_data(librbd::MockJournal &mock_journal, + const librbd::journal::TagData &tag_data) { + EXPECT_CALL(mock_journal, get_tag_data()).WillOnce(Return(tag_data)); + } + + void expect_is_resync_requested(librbd::MockJournal &mock_journal, + bool do_resync, int r) { + EXPECT_CALL(mock_journal, is_resync_requested(_)) + .WillOnce(DoAll(SetArgPointee<0>(do_resync), + Return(r))); + } + + bufferlist encode_tag_data(const librbd::journal::TagData &tag_data) { + bufferlist bl; + encode(tag_data, bl); + return bl; + } + + MockPrepareReplayRequest* create_request( + librbd::MockTestImageCtx& mock_local_image_ctx, + ::journal::MockJournaler& mock_remote_journaler, + librbd::mirror::PromotionState remote_promotion_state, + const std::string& local_mirror_uuid, + const std::string& remote_mirror_uuid, + librbd::journal::MirrorPeerClientMeta* client_meta, + bool* resync_requested, bool* syncing, Context* on_finish) { + return new MockPrepareReplayRequest( + &mock_local_image_ctx, &mock_remote_journaler, remote_promotion_state, + local_mirror_uuid, remote_mirror_uuid, client_meta, nullptr, + resync_requested, syncing, on_finish); + } + + librbd::ImageCtx *m_local_image_ctx = nullptr; +}; + +TEST_F(TestMockImageReplayerJournalPrepareReplayRequest, Success) { + InSequence seq; + + librbd::MockJournal mock_journal; + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + mock_local_image_ctx.journal = &mock_journal; + + // check initial state + expect_is_resync_requested(mock_journal, false, 0); + expect_journal_get_tag_tid(mock_journal, 345); + expect_journal_get_tag_data(mock_journal, {"remote mirror uuid"}); + + // lookup remote image tag class + librbd::journal::ClientData client_data{ + librbd::journal::ImageClientMeta{123}}; + cls::journal::Client client; + encode(client_data, client.data); + ::journal::MockJournaler mock_remote_journaler; + expect_journaler_get_client(mock_remote_journaler, + librbd::Journal<>::IMAGE_CLIENT_ID, + client, 0); + + // single promotion event + Tags tags = { + {2, 123, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 344, 99})}, + }; + expect_journaler_get_tags(mock_remote_journaler, 123, tags, 0); + + C_SaferCond ctx; + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + bool resync_requested; + bool syncing; + auto request = create_request( + mock_local_image_ctx, mock_remote_journaler, + librbd::mirror::PROMOTION_STATE_PRIMARY, "local mirror uuid", + "remote mirror uuid", &mirror_peer_client_meta, &resync_requested, + &syncing, &ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); + ASSERT_FALSE(resync_requested); + ASSERT_FALSE(syncing); +} + +TEST_F(TestMockImageReplayerJournalPrepareReplayRequest, NoLocalJournal) { + InSequence seq; + + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + + C_SaferCond ctx; + ::journal::MockJournaler mock_remote_journaler; + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + bool resync_requested; + bool syncing; + auto request = create_request( + mock_local_image_ctx, mock_remote_journaler, + librbd::mirror::PROMOTION_STATE_PRIMARY, "local mirror uuid", + "remote mirror uuid", &mirror_peer_client_meta, &resync_requested, + &syncing, &ctx); + request->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageReplayerJournalPrepareReplayRequest, ResyncRequested) { + InSequence seq; + + librbd::MockJournal mock_journal; + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + mock_local_image_ctx.journal = &mock_journal; + + // check initial state + expect_is_resync_requested(mock_journal, true, 0); + expect_journal_get_tag_tid(mock_journal, 345); + expect_journal_get_tag_data(mock_journal, {"remote mirror uuid"}); + + C_SaferCond ctx; + ::journal::MockJournaler mock_remote_journaler; + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + bool resync_requested; + bool syncing; + auto request = create_request( + mock_local_image_ctx, mock_remote_journaler, + librbd::mirror::PROMOTION_STATE_PRIMARY, "local mirror uuid", + "remote mirror uuid", &mirror_peer_client_meta, &resync_requested, + &syncing, &ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); + ASSERT_TRUE(resync_requested); + ASSERT_FALSE(syncing); +} + +TEST_F(TestMockImageReplayerJournalPrepareReplayRequest, ResyncRequestedError) { + InSequence seq; + + librbd::MockJournal mock_journal; + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + mock_local_image_ctx.journal = &mock_journal; + + // check initial state + expect_is_resync_requested(mock_journal, false, -EINVAL); + + C_SaferCond ctx; + ::journal::MockJournaler mock_remote_journaler; + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + bool resync_requested; + bool syncing; + auto request = create_request( + mock_local_image_ctx, mock_remote_journaler, + librbd::mirror::PROMOTION_STATE_PRIMARY, "local mirror uuid", + "remote mirror uuid", &mirror_peer_client_meta, &resync_requested, + &syncing, &ctx); + request->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageReplayerJournalPrepareReplayRequest, UnlinkedRemoteNonPrimary) { + InSequence seq; + + librbd::MockJournal mock_journal; + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + mock_local_image_ctx.journal = &mock_journal; + + // check initial state + expect_is_resync_requested(mock_journal, false, 0); + expect_journal_get_tag_tid(mock_journal, 345); + expect_journal_get_tag_data(mock_journal, {"blah"}); + + C_SaferCond ctx; + ::journal::MockJournaler mock_remote_journaler; + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + bool resync_requested; + bool syncing; + auto request = create_request( + mock_local_image_ctx, mock_remote_journaler, + librbd::mirror::PROMOTION_STATE_NON_PRIMARY, "local mirror uuid", + "remote mirror uuid", &mirror_peer_client_meta, &resync_requested, + &syncing, &ctx); + request->send(); + ASSERT_EQ(-EREMOTEIO, ctx.wait()); +} + +TEST_F(TestMockImageReplayerJournalPrepareReplayRequest, Syncing) { + InSequence seq; + + librbd::MockJournal mock_journal; + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + mock_local_image_ctx.journal = &mock_journal; + + // check initial state + expect_is_resync_requested(mock_journal, false, 0); + expect_journal_get_tag_tid(mock_journal, 345); + expect_journal_get_tag_data(mock_journal, {"remote mirror uuid"}); + + C_SaferCond ctx; + ::journal::MockJournaler mock_remote_journaler; + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_SYNCING; + bool resync_requested; + bool syncing; + auto request = create_request( + mock_local_image_ctx, mock_remote_journaler, + librbd::mirror::PROMOTION_STATE_PRIMARY, "local mirror uuid", + "remote mirror uuid", &mirror_peer_client_meta, &resync_requested, + &syncing, &ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); + ASSERT_FALSE(resync_requested); + ASSERT_TRUE(syncing); +} + +TEST_F(TestMockImageReplayerJournalPrepareReplayRequest, GetRemoteTagClassError) { + InSequence seq; + + librbd::MockJournal mock_journal; + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + mock_local_image_ctx.journal = &mock_journal; + + // check initial state + expect_is_resync_requested(mock_journal, false, 0); + expect_journal_get_tag_tid(mock_journal, 345); + expect_journal_get_tag_data(mock_journal, {"remote mirror uuid"}); + + // lookup remote image tag class + librbd::journal::ClientData client_data{ + librbd::journal::ImageClientMeta{123}}; + cls::journal::Client client; + encode(client_data, client.data); + ::journal::MockJournaler mock_remote_journaler; + expect_journaler_get_client(mock_remote_journaler, + librbd::Journal<>::IMAGE_CLIENT_ID, + client, -EINVAL); + + C_SaferCond ctx; + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + bool resync_requested; + bool syncing; + auto request = create_request( + mock_local_image_ctx, mock_remote_journaler, + librbd::mirror::PROMOTION_STATE_PRIMARY, "local mirror uuid", + "remote mirror uuid", &mirror_peer_client_meta, &resync_requested, + &syncing, &ctx); + request->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageReplayerJournalPrepareReplayRequest, GetRemoteTagsError) { + InSequence seq; + + librbd::MockJournal mock_journal; + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + mock_local_image_ctx.journal = &mock_journal; + + // check initial state + expect_is_resync_requested(mock_journal, false, 0); + expect_journal_get_tag_tid(mock_journal, 345); + expect_journal_get_tag_data(mock_journal, {"remote mirror uuid"}); + + // lookup remote image tag class + librbd::journal::ClientData client_data{ + librbd::journal::ImageClientMeta{123}}; + cls::journal::Client client; + encode(client_data, client.data); + ::journal::MockJournaler mock_remote_journaler; + expect_journaler_get_client(mock_remote_journaler, + librbd::Journal<>::IMAGE_CLIENT_ID, + client, 0); + + // single promotion event + Tags tags = { + {2, 123, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 344, 99})}, + }; + expect_journaler_get_tags(mock_remote_journaler, 123, tags, -EINVAL); + + C_SaferCond ctx; + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + bool resync_requested; + bool syncing; + auto request = create_request( + mock_local_image_ctx, mock_remote_journaler, + librbd::mirror::PROMOTION_STATE_PRIMARY, "local mirror uuid", + "remote mirror uuid", &mirror_peer_client_meta, &resync_requested, + &syncing, &ctx); + request->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageReplayerJournalPrepareReplayRequest, LocalDemotedRemoteSyncingState) { + InSequence seq; + + librbd::MockJournal mock_journal; + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + mock_local_image_ctx.journal = &mock_journal; + + // check initial state + expect_is_resync_requested(mock_journal, false, 0); + expect_journal_get_tag_tid(mock_journal, 345); + expect_journal_get_tag_data(mock_journal, {librbd::Journal<>::ORPHAN_MIRROR_UUID, + "remote mirror uuid", true, 4, 1}); + + // update client state + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{ + mock_local_image_ctx.id}; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + librbd::journal::ClientData client_data; + client_data.client_meta = mirror_peer_client_meta; + ::journal::MockJournaler mock_remote_journaler; + expect_journaler_update_client(mock_remote_journaler, client_data, 0); + + // lookup remote image tag class + client_data = {librbd::journal::ImageClientMeta{123}}; + cls::journal::Client client; + encode(client_data, client.data); + expect_journaler_get_client(mock_remote_journaler, + librbd::Journal<>::IMAGE_CLIENT_ID, + client, 0); + + // remote demotion / promotion event + Tags tags = { + {2, 123, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 1, 99})}, + {3, 123, encode_tag_data({librbd::Journal<>::ORPHAN_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 2, 1})}, + {4, 123, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::ORPHAN_MIRROR_UUID, + true, 3, 1})}, + {5, 123, encode_tag_data({librbd::Journal<>::ORPHAN_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 4, 1})}, + {6, 123, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::ORPHAN_MIRROR_UUID, + true, 5, 1})}, + {7, 123, encode_tag_data({librbd::Journal<>::ORPHAN_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 6, 1})}, + {8, 123, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::ORPHAN_MIRROR_UUID, + true, 7, 1})} + }; + expect_journaler_get_tags(mock_remote_journaler, 123, tags, 0); + + C_SaferCond ctx; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_SYNCING; + bool resync_requested; + bool syncing; + auto request = create_request( + mock_local_image_ctx, mock_remote_journaler, + librbd::mirror::PROMOTION_STATE_PRIMARY, "local mirror uuid", + "remote mirror uuid", &mirror_peer_client_meta, &resync_requested, + &syncing, &ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); + ASSERT_FALSE(resync_requested); + ASSERT_FALSE(syncing); +} + +TEST_F(TestMockImageReplayerJournalPrepareReplayRequest, UpdateClientError) { + InSequence seq; + + librbd::MockJournal mock_journal; + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + mock_local_image_ctx.journal = &mock_journal; + + // check initial state + expect_is_resync_requested(mock_journal, false, 0); + expect_journal_get_tag_tid(mock_journal, 345); + expect_journal_get_tag_data(mock_journal, {"remote mirror uuid"}); + + // lookup remote image tag class + librbd::journal::ClientData client_data{ + librbd::journal::ImageClientMeta{123}}; + cls::journal::Client client; + encode(client_data, client.data); + ::journal::MockJournaler mock_remote_journaler; + expect_journaler_get_client(mock_remote_journaler, + librbd::Journal<>::IMAGE_CLIENT_ID, + client, 0); + + // single promotion event + Tags tags = { + {2, 123, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 344, 99})}, + }; + expect_journaler_get_tags(mock_remote_journaler, 123, tags, 0); + + C_SaferCond ctx; + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + bool resync_requested; + bool syncing; + auto request = create_request( + mock_local_image_ctx, mock_remote_journaler, + librbd::mirror::PROMOTION_STATE_PRIMARY, "local mirror uuid", + "remote mirror uuid", &mirror_peer_client_meta, &resync_requested, + &syncing, &ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); + ASSERT_FALSE(resync_requested); + ASSERT_FALSE(syncing); +} + +TEST_F(TestMockImageReplayerJournalPrepareReplayRequest, NonPrimaryRemoteNotTagOwner) { + InSequence seq; + + librbd::MockJournal mock_journal; + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + mock_local_image_ctx.journal = &mock_journal; + + // check initial state + expect_is_resync_requested(mock_journal, false, 0); + expect_journal_get_tag_tid(mock_journal, 345); + expect_journal_get_tag_data(mock_journal, {librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::ORPHAN_MIRROR_UUID, + true, 344, 0}); + + C_SaferCond ctx; + ::journal::MockJournaler mock_remote_journaler; + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + bool resync_requested; + bool syncing; + auto request = create_request( + mock_local_image_ctx, mock_remote_journaler, + librbd::mirror::PROMOTION_STATE_NON_PRIMARY, "local mirror uuid", + "remote mirror uuid", &mirror_peer_client_meta, &resync_requested, + &syncing, &ctx); + request->send(); + ASSERT_EQ(-EREMOTEIO, ctx.wait()); +} + +TEST_F(TestMockImageReplayerJournalPrepareReplayRequest, RemoteDemotePromote) { + InSequence seq; + + librbd::MockJournal mock_journal; + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + mock_local_image_ctx.journal = &mock_journal; + + // check initial state + expect_is_resync_requested(mock_journal, false, 0); + expect_journal_get_tag_tid(mock_journal, 345); + expect_journal_get_tag_data(mock_journal, {"remote mirror uuid"}); + + // lookup remote image tag class + librbd::journal::ClientData client_data{ + librbd::journal::ImageClientMeta{123}}; + cls::journal::Client client; + encode(client_data, client.data); + ::journal::MockJournaler mock_remote_journaler; + expect_journaler_get_client(mock_remote_journaler, + librbd::Journal<>::IMAGE_CLIENT_ID, + client, 0); + + // remote demotion / promotion event + Tags tags = { + {2, 123, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 1, 99})}, + {3, 123, encode_tag_data({librbd::Journal<>::ORPHAN_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 2, 1})}, + {4, 123, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::ORPHAN_MIRROR_UUID, + true, 2, 1})}, + {5, 123, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::ORPHAN_MIRROR_UUID, + true, 4, 369})} + }; + expect_journaler_get_tags(mock_remote_journaler, 123, tags, 0); + + C_SaferCond ctx; + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + bool resync_requested; + bool syncing; + auto request = create_request( + mock_local_image_ctx, mock_remote_journaler, + librbd::mirror::PROMOTION_STATE_PRIMARY, "local mirror uuid", + "remote mirror uuid", &mirror_peer_client_meta, &resync_requested, + &syncing, &ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); + ASSERT_FALSE(resync_requested); + ASSERT_FALSE(syncing); +} + +TEST_F(TestMockImageReplayerJournalPrepareReplayRequest, MultipleRemoteDemotePromotes) { + InSequence seq; + + librbd::MockJournal mock_journal; + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + mock_local_image_ctx.journal = &mock_journal; + + // check initial state + expect_is_resync_requested(mock_journal, false, 0); + expect_journal_get_tag_tid(mock_journal, 345); + expect_journal_get_tag_data(mock_journal, {librbd::Journal<>::ORPHAN_MIRROR_UUID, + "remote mirror uuid", true, 4, 1}); + + // lookup remote image tag class + librbd::journal::ClientData client_data{ + librbd::journal::ImageClientMeta{123}}; + cls::journal::Client client; + encode(client_data, client.data); + ::journal::MockJournaler mock_remote_journaler; + expect_journaler_get_client(mock_remote_journaler, + librbd::Journal<>::IMAGE_CLIENT_ID, + client, 0); + + // remote demotion / promotion event + Tags tags = { + {2, 123, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 1, 99})}, + {3, 123, encode_tag_data({librbd::Journal<>::ORPHAN_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 2, 1})}, + {4, 123, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::ORPHAN_MIRROR_UUID, + true, 3, 1})}, + {5, 123, encode_tag_data({librbd::Journal<>::ORPHAN_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 4, 1})}, + {6, 123, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::ORPHAN_MIRROR_UUID, + true, 5, 1})}, + {7, 123, encode_tag_data({librbd::Journal<>::ORPHAN_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 6, 1})}, + {8, 123, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::ORPHAN_MIRROR_UUID, + true, 7, 1})} + }; + expect_journaler_get_tags(mock_remote_journaler, 123, tags, 0); + + C_SaferCond ctx; + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + bool resync_requested; + bool syncing; + auto request = create_request( + mock_local_image_ctx, mock_remote_journaler, + librbd::mirror::PROMOTION_STATE_PRIMARY, "local mirror uuid", + "remote mirror uuid", &mirror_peer_client_meta, &resync_requested, + &syncing, &ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); + ASSERT_FALSE(resync_requested); + ASSERT_FALSE(syncing); +} + +TEST_F(TestMockImageReplayerJournalPrepareReplayRequest, LocalDemoteRemotePromote) { + InSequence seq; + + librbd::MockJournal mock_journal; + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + mock_local_image_ctx.journal = &mock_journal; + + // check initial state + expect_is_resync_requested(mock_journal, false, 0); + expect_journal_get_tag_tid(mock_journal, 346); + expect_journal_get_tag_data(mock_journal, + {librbd::Journal<>::ORPHAN_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 345, 1}); + + // lookup remote image tag class + librbd::journal::ClientData client_data{ + librbd::journal::ImageClientMeta{123}}; + cls::journal::Client client; + encode(client_data, client.data); + ::journal::MockJournaler mock_remote_journaler; + expect_journaler_get_client(mock_remote_journaler, + librbd::Journal<>::IMAGE_CLIENT_ID, + client, 0); + + // remote demotion / promotion event + Tags tags = { + {2, 123, encode_tag_data({"local mirror uuid", "local mirror uuid", + true, 344, 99})}, + {3, 123, encode_tag_data({librbd::Journal<>::ORPHAN_MIRROR_UUID, + "local mirror uuid", true, 345, 1})}, + {4, 123, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::ORPHAN_MIRROR_UUID, + true, 3, 1})} + }; + expect_journaler_get_tags(mock_remote_journaler, 123, tags, 0); + + C_SaferCond ctx; + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + bool resync_requested; + bool syncing; + auto request = create_request( + mock_local_image_ctx, mock_remote_journaler, + librbd::mirror::PROMOTION_STATE_PRIMARY, "local mirror uuid", + "remote mirror uuid", &mirror_peer_client_meta, &resync_requested, + &syncing, &ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); + ASSERT_FALSE(resync_requested); + ASSERT_FALSE(syncing); +} + +TEST_F(TestMockImageReplayerJournalPrepareReplayRequest, SplitBrainForcePromote) { + InSequence seq; + + librbd::MockJournal mock_journal; + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + mock_local_image_ctx.journal = &mock_journal; + + // check initial state + expect_is_resync_requested(mock_journal, false, 0); + expect_journal_get_tag_tid(mock_journal, 345); + expect_journal_get_tag_data(mock_journal, {librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::ORPHAN_MIRROR_UUID, + true, 344, 0}); + + // lookup remote image tag class + librbd::journal::ClientData client_data{ + librbd::journal::ImageClientMeta{123}}; + cls::journal::Client client; + encode(client_data, client.data); + ::journal::MockJournaler mock_remote_journaler; + expect_journaler_get_client(mock_remote_journaler, + librbd::Journal<>::IMAGE_CLIENT_ID, + client, 0); + + // remote demotion / promotion event + Tags tags = { + {2, 123, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 1, 99})}, + {3, 123, encode_tag_data({librbd::Journal<>::ORPHAN_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 2, 1})} + }; + expect_journaler_get_tags(mock_remote_journaler, 123, tags, 0); + + C_SaferCond ctx; + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + bool resync_requested; + bool syncing; + auto request = create_request( + mock_local_image_ctx, mock_remote_journaler, + librbd::mirror::PROMOTION_STATE_PRIMARY, "local mirror uuid", + "remote mirror uuid", &mirror_peer_client_meta, &resync_requested, + &syncing, &ctx); + request->send(); + ASSERT_EQ(-EEXIST, ctx.wait()); +} + +} // namespace journal +} // namespace image_replayer +} // namespace mirror +} // namespace rbd diff --git a/src/tools/rbd_mirror/CMakeLists.txt b/src/tools/rbd_mirror/CMakeLists.txt index 26e77eaa7a6..4d937d53f67 100644 --- a/src/tools/rbd_mirror/CMakeLists.txt +++ b/src/tools/rbd_mirror/CMakeLists.txt @@ -43,6 +43,7 @@ set(rbd_mirror_internal image_replayer/Utils.cc image_replayer/journal/CreateLocalImageRequest.cc image_replayer/journal/EventPreprocessor.cc + image_replayer/journal/PrepareReplayRequest.cc image_replayer/journal/Replayer.cc image_replayer/journal/ReplayStatusFormatter.cc image_sync/SyncPointCreateRequest.cc diff --git a/src/tools/rbd_mirror/image_replayer/journal/PrepareReplayRequest.cc b/src/tools/rbd_mirror/image_replayer/journal/PrepareReplayRequest.cc new file mode 100644 index 00000000000..c3c48b94c82 --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/journal/PrepareReplayRequest.cc @@ -0,0 +1,315 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "PrepareReplayRequest.h" +#include "common/debug.h" +#include "common/dout.h" +#include "common/errno.h" +#include "journal/Journaler.h" +#include "librbd/ImageCtx.h" +#include "librbd/Journal.h" +#include "librbd/Utils.h" +#include "tools/rbd_mirror/ProgressContext.h" + +#define dout_context g_ceph_context +#define dout_subsys ceph_subsys_rbd_mirror +#undef dout_prefix +#define dout_prefix *_dout << "rbd::mirror::image_replayer::journal::" \ + << "PrepareReplayRequest: " << this << " " \ + << __func__ << ": " + +namespace rbd { +namespace mirror { +namespace image_replayer { +namespace journal { + +using librbd::util::create_context_callback; + +template +void PrepareReplayRequest::send() { + *m_resync_requested = false; + *m_syncing = false; + + std::shared_lock image_locker(m_local_image_ctx->image_lock); + if (m_local_image_ctx->journal == nullptr) { + image_locker.unlock(); + + derr << "local image does not support journaling" << dendl; + finish(-EINVAL); + return; + } + + int r = m_local_image_ctx->journal->is_resync_requested(m_resync_requested); + if (r < 0) { + image_locker.unlock(); + + derr << "failed to check if a resync was requested" << dendl; + finish(r); + return; + } + + m_local_tag_tid = m_local_image_ctx->journal->get_tag_tid(); + m_local_tag_data = m_local_image_ctx->journal->get_tag_data(); + dout(10) << "local tag=" << m_local_tag_tid << ", " + << "local tag data=" << m_local_tag_data << dendl; + image_locker.unlock(); + + if (m_local_tag_data.mirror_uuid != m_remote_mirror_uuid && + m_remote_promotion_state != librbd::mirror::PROMOTION_STATE_PRIMARY) { + // if the local mirror is not linked to the (now) non-primary image, + // stop the replay. Otherwise, we ignore that the remote is non-primary + // so that we can replay the demotion + dout(5) << "remote image is not primary -- skipping image replay" + << dendl; + finish(-EREMOTEIO); + return; + } + + if (*m_resync_requested) { + finish(0); + return; + } else if (m_client_meta->state == + librbd::journal::MIRROR_PEER_STATE_SYNCING && + m_local_tag_data.mirror_uuid == m_remote_mirror_uuid) { + // if the initial sync hasn't completed, we cannot replay + *m_syncing = true; + finish(0); + return; + } + + update_client_state(); +} + +template +void PrepareReplayRequest::update_client_state() { + if (m_client_meta->state != librbd::journal::MIRROR_PEER_STATE_SYNCING || + m_local_tag_data.mirror_uuid == m_remote_mirror_uuid) { + get_remote_tag_class(); + return; + } + + // our local image is not primary, is flagged as syncing on the remote side, + // but is no longer tied to the remote -- this implies we were forced + // promoted and then demoted at some point + dout(15) << dendl; + update_progress("UPDATE_CLIENT_STATE"); + + librbd::journal::MirrorPeerClientMeta client_meta(*m_client_meta); + client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + + librbd::journal::ClientData client_data(client_meta); + bufferlist data_bl; + encode(client_data, data_bl); + + auto ctx = create_context_callback< + PrepareReplayRequest, + &PrepareReplayRequest::handle_update_client_state>(this); + m_remote_journaler->update_client(data_bl, ctx); +} + +template +void PrepareReplayRequest::handle_update_client_state(int r) { + dout(15) << "r=" << r << dendl; + if (r < 0) { + derr << "failed to update client: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + m_client_meta->state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + get_remote_tag_class(); +} + +template +void PrepareReplayRequest::get_remote_tag_class() { + dout(10) << dendl; + update_progress("GET_REMOTE_TAG_CLASS"); + + auto ctx = create_context_callback< + PrepareReplayRequest, + &PrepareReplayRequest::handle_get_remote_tag_class>(this); + m_remote_journaler->get_client(librbd::Journal<>::IMAGE_CLIENT_ID, &m_client, + ctx); +} + +template +void PrepareReplayRequest::handle_get_remote_tag_class(int r) { + dout(10) << "r=" << r << dendl; + + if (r < 0) { + derr << "failed to retrieve remote client: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + librbd::journal::ClientData client_data; + auto it = m_client.data.cbegin(); + try { + decode(client_data, it); + } catch (const buffer::error &err) { + derr << "failed to decode remote client meta data: " << err.what() + << dendl; + finish(-EBADMSG); + return; + } + + librbd::journal::ImageClientMeta *client_meta = + boost::get(&client_data.client_meta); + if (client_meta == nullptr) { + derr << "unknown remote client registration" << dendl; + finish(-EINVAL); + return; + } + + m_remote_tag_class = client_meta->tag_class; + dout(10) << "remote tag class=" << m_remote_tag_class << dendl; + + get_remote_tags(); +} + +template +void PrepareReplayRequest::get_remote_tags() { + dout(10) << dendl; + update_progress("GET_REMOTE_TAGS"); + + auto ctx = create_context_callback< + PrepareReplayRequest, + &PrepareReplayRequest::handle_get_remote_tags>(this); + m_remote_journaler->get_tags(m_remote_tag_class, &m_remote_tags, ctx); +} + +template +void PrepareReplayRequest::handle_get_remote_tags(int r) { + dout(10) << "r=" << r << dendl; + + if (r < 0) { + derr << "failed to retrieve remote tags: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + // At this point, the local image was existing, non-primary, and replaying; + // and the remote image is primary. Attempt to link the local image's most + // recent tag to the remote image's tag chain. + bool remote_tag_data_valid = false; + librbd::journal::TagData remote_tag_data; + boost::optional remote_orphan_tag_tid = + boost::make_optional(false, 0U); + bool reconnect_orphan = false; + + // decode the remote tags + for (auto &remote_tag : m_remote_tags) { + if (m_local_tag_data.predecessor.commit_valid && + m_local_tag_data.predecessor.mirror_uuid == m_remote_mirror_uuid && + m_local_tag_data.predecessor.tag_tid > remote_tag.tid) { + dout(10) << "skipping processed predecessor remote tag " + << remote_tag.tid << dendl; + continue; + } + + try { + auto it = remote_tag.data.cbegin(); + decode(remote_tag_data, it); + remote_tag_data_valid = true; + } catch (const buffer::error &err) { + derr << "failed to decode remote tag " << remote_tag.tid << ": " + << err.what() << dendl; + finish(-EBADMSG); + return; + } + + dout(10) << "decoded remote tag " << remote_tag.tid << ": " + << remote_tag_data << dendl; + + if (!m_local_tag_data.predecessor.commit_valid) { + // newly synced local image (no predecessor) replays from the first tag + if (remote_tag_data.mirror_uuid != librbd::Journal<>::LOCAL_MIRROR_UUID) { + dout(10) << "skipping non-primary remote tag" << dendl; + continue; + } + + dout(10) << "using initial primary remote tag" << dendl; + break; + } + + if (m_local_tag_data.mirror_uuid == librbd::Journal<>::ORPHAN_MIRROR_UUID) { + // demotion last available local epoch + + if (remote_tag_data.mirror_uuid == m_local_tag_data.mirror_uuid && + remote_tag_data.predecessor.commit_valid && + remote_tag_data.predecessor.tag_tid == + m_local_tag_data.predecessor.tag_tid) { + // demotion matches remote epoch + + if (remote_tag_data.predecessor.mirror_uuid == m_local_mirror_uuid && + m_local_tag_data.predecessor.mirror_uuid == + librbd::Journal<>::LOCAL_MIRROR_UUID) { + // local demoted and remote has matching event + dout(10) << "found matching local demotion tag" << dendl; + remote_orphan_tag_tid = remote_tag.tid; + continue; + } + + if (m_local_tag_data.predecessor.mirror_uuid == m_remote_mirror_uuid && + remote_tag_data.predecessor.mirror_uuid == + librbd::Journal<>::LOCAL_MIRROR_UUID) { + // remote demoted and local has matching event + dout(10) << "found matching remote demotion tag" << dendl; + remote_orphan_tag_tid = remote_tag.tid; + continue; + } + } + + if (remote_tag_data.mirror_uuid == librbd::Journal<>::LOCAL_MIRROR_UUID && + remote_tag_data.predecessor.mirror_uuid == + librbd::Journal<>::ORPHAN_MIRROR_UUID && + remote_tag_data.predecessor.commit_valid && remote_orphan_tag_tid && + remote_tag_data.predecessor.tag_tid == *remote_orphan_tag_tid) { + // remote promotion tag chained to remote/local demotion tag + dout(10) << "found chained remote promotion tag" << dendl; + reconnect_orphan = true; + break; + } + + // promotion must follow demotion + remote_orphan_tag_tid = boost::none; + } + } + + if (remote_tag_data_valid && + m_local_tag_data.mirror_uuid == m_remote_mirror_uuid) { + dout(10) << "local image is in clean replay state" << dendl; + } else if (reconnect_orphan) { + dout(10) << "remote image was demoted/promoted" << dendl; + } else { + derr << "split-brain detected -- skipping image replay" << dendl; + finish(-EEXIST); + return; + } + + finish(0); +} + +template +void PrepareReplayRequest::finish(int r) { + dout(10) << "r=" << r << dendl; + + m_on_finish->complete(r); + delete this; +} + +template +void PrepareReplayRequest::update_progress(const std::string &description) { + dout(10) << description << dendl; + + if (m_progress_ctx != nullptr) { + m_progress_ctx->update_progress(description); + } +} + +} // namespace journal +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +template class rbd::mirror::image_replayer::journal::PrepareReplayRequest; diff --git a/src/tools/rbd_mirror/image_replayer/journal/PrepareReplayRequest.h b/src/tools/rbd_mirror/image_replayer/journal/PrepareReplayRequest.h new file mode 100644 index 00000000000..091d7956b31 --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/journal/PrepareReplayRequest.h @@ -0,0 +1,124 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef RBD_MIRROR_IMAGE_REPLAYER_JOURNAL_PREPARE_REPLAY_REQUEST_H +#define RBD_MIRROR_IMAGE_REPLAYER_JOURNAL_PREPARE_REPLAY_REQUEST_H + +#include "include/int_types.h" +#include "cls/journal/cls_journal_types.h" +#include "librbd/journal/Types.h" +#include "librbd/journal/TypeTraits.h" +#include "librbd/mirror/Types.h" +#include +#include + +struct Context; +namespace librbd { struct ImageCtx; } + +namespace rbd { +namespace mirror { + +class ProgressContext; + +namespace image_replayer { +namespace journal { + +template +class PrepareReplayRequest { +public: + typedef librbd::journal::TypeTraits TypeTraits; + typedef typename TypeTraits::Journaler Journaler; + typedef librbd::journal::MirrorPeerClientMeta MirrorPeerClientMeta; + + static PrepareReplayRequest* create( + ImageCtxT* local_image_ctx, Journaler* remote_journaler, + librbd::mirror::PromotionState remote_promotion_state, + const std::string& local_mirror_uuid, + const std::string& remote_mirror_uuid, + MirrorPeerClientMeta* client_meta, ProgressContext* progress_ctx, + bool* resync_requested, bool* syncing, Context* on_finish) { + return new PrepareReplayRequest( + local_image_ctx, remote_journaler, remote_promotion_state, + local_mirror_uuid, remote_mirror_uuid, client_meta, progress_ctx, + resync_requested, syncing, on_finish); + } + + PrepareReplayRequest( + ImageCtxT* local_image_ctx, Journaler* remote_journaler, + librbd::mirror::PromotionState remote_promotion_state, + const std::string& local_mirror_uuid, + const std::string& remote_mirror_uuid, + MirrorPeerClientMeta* client_meta, ProgressContext* progress_ctx, + bool* resync_requested, bool* syncing, Context* on_finish) + : m_local_image_ctx(local_image_ctx), m_remote_journaler(remote_journaler), + m_remote_promotion_state(remote_promotion_state), + m_local_mirror_uuid(local_mirror_uuid), + m_remote_mirror_uuid(remote_mirror_uuid), m_client_meta(client_meta), + m_progress_ctx(progress_ctx), m_resync_requested(resync_requested), + m_syncing(syncing), m_on_finish(on_finish) { + } + + void send(); + +private: + /** + * @verbatim + * + * + * | + * v + * UPDATE_CLIENT_STATE + * | + * v + * GET_REMOTE_TAG_CLASS + * | + * v + * GET_REMOTE_TAGS + * | + * v + * + * + * @endverbatim + */ + typedef std::list Tags; + + ImageCtxT* m_local_image_ctx; + Journaler* m_remote_journaler; + librbd::mirror::PromotionState m_remote_promotion_state; + std::string m_local_mirror_uuid; + std::string m_remote_mirror_uuid; + MirrorPeerClientMeta* m_client_meta; + ProgressContext* m_progress_ctx; + bool* m_resync_requested; + bool* m_syncing; + Context* m_on_finish; + + uint64_t m_local_tag_tid = 0; + librbd::journal::TagData m_local_tag_data; + + uint64_t m_remote_tag_class = 0; + Tags m_remote_tags; + cls::journal::Client m_client; + + void update_client_state(); + void handle_update_client_state(int r); + + void get_remote_tag_class(); + void handle_get_remote_tag_class(int r); + + void get_remote_tags(); + void handle_get_remote_tags(int r); + + void finish(int r); + void update_progress(const std::string& description); + +}; + +} // namespace journal +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +extern template class rbd::mirror::image_replayer::journal::PrepareReplayRequest; + +#endif // RBD_MIRROR_IMAGE_REPLAYER_JOURNAL_PREPARE_REPLAY_REQUEST_H