From c0fdb2ca0f36d18aa5a42299d6aeec1744264bcb Mon Sep 17 00:00:00 2001 From: Jason Dillaman Date: Wed, 29 Jan 2020 14:42:14 -0500 Subject: [PATCH] rbd-mirror: replace journal-specific parent image validation When cloning a mirrored image, instead of looking at the parent image's journal state to tell if its still being synced, we can instead verify that it doesn't have an rbd-mirror image sync snapshots. If it doesn't, the image isn't being actively synced, but it might not have started either. However, the clone state machine will open the local parent image and attempt to find the specified snapshot. If it exists, the image sync is complete. Signed-off-by: Jason Dillaman --- .../test_mock_CreateLocalImageRequest.cc | 41 +--- .../test_mock_BootstrapRequest.cc | 7 +- .../test_mock_CreateImageRequest.cc | 224 +++--------------- .../image_replayer/BootstrapRequest.cc | 3 +- .../image_replayer/CreateImageRequest.cc | 161 ++++--------- .../image_replayer/CreateImageRequest.h | 66 ++---- .../rbd_mirror/image_replayer/StateBuilder.h | 2 + .../journal/CreateLocalImageRequest.cc | 50 +--- .../journal/CreateLocalImageRequest.h | 27 ++- .../image_replayer/journal/StateBuilder.cc | 3 +- .../image_replayer/journal/StateBuilder.h | 1 + .../image_replayer/snapshot/StateBuilder.cc | 1 + .../image_replayer/snapshot/StateBuilder.h | 1 + 13 files changed, 154 insertions(+), 433 deletions(-) diff --git a/src/test/rbd_mirror/image_replayer/journal/test_mock_CreateLocalImageRequest.cc b/src/test/rbd_mirror/image_replayer/journal/test_mock_CreateLocalImageRequest.cc index 08402c213de..cc22671609d 100644 --- a/src/test/rbd_mirror/image_replayer/journal/test_mock_CreateLocalImageRequest.cc +++ b/src/test/rbd_mirror/image_replayer/journal/test_mock_CreateLocalImageRequest.cc @@ -69,6 +69,7 @@ struct CreateImageRequest { const std::string &local_image_name, const std::string &local_image_id, librbd::MockTestImageCtx *remote_image_ctx, + PoolMetaCache* pool_meta_cache, cls::rbd::MirrorImageMode mirror_image_mode, Context *on_finish) { ceph_assert(s_instance != nullptr); @@ -173,6 +174,7 @@ public: m_threads->work_queue->queue(on_finish, r); }))); } + void expect_create_image(MockCreateImageRequest& mock_create_image_request, const std::string& image_id, int r) { EXPECT_CALL(mock_create_image_request, construct(image_id)); @@ -189,7 +191,8 @@ public: Context* on_finish) { return new MockCreateLocalImageRequest( &mock_threads, m_local_io_ctx, m_mock_remote_image_ctx, - global_image_id, nullptr, &mock_state_builder, on_finish); + global_image_id, nullptr, nullptr, &mock_state_builder, + on_finish); } librbd::ImageCtx *m_remote_image_ctx; @@ -316,42 +319,12 @@ TEST_F(TestMockImageReplayerJournalCreateLocalImageRequest, CreateImageDuplicate MockCreateImageRequest mock_create_image_request; expect_create_image(mock_create_image_request, "local image id", -EBADF); - // update image id - expect_journaler_update_client(mock_journaler, client_data, 0); - - // re-create the local image - expect_create_image(mock_create_image_request, "local image id", 0); - - C_SaferCond ctx; - MockThreads mock_threads; - MockStateBuilder mock_state_builder; - auto request = create_request( - mock_threads, mock_state_builder, "global image id", &ctx); - request->send(); - ASSERT_EQ(0, ctx.wait()); -} - -TEST_F(TestMockImageReplayerJournalCreateLocalImageRequest, UpdateClientImageError) { - InSequence seq; - // re-register the client - ::journal::MockJournaler mock_journaler; expect_journaler_unregister_client(mock_journaler, 0); - - librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; - librbd::util::s_image_id = "local image id"; - mirror_peer_client_meta.image_id = "local image id"; - mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_SYNCING; - librbd::journal::ClientData client_data; - client_data.client_meta = mirror_peer_client_meta; expect_journaler_register_client(mock_journaler, client_data, 0); - // create the missing local image - MockCreateImageRequest mock_create_image_request; - expect_create_image(mock_create_image_request, "local image id", -EBADF); - - // update image id - expect_journaler_update_client(mock_journaler, client_data, -EINVAL); + // re-create the local image + expect_create_image(mock_create_image_request, "local image id", 0); C_SaferCond ctx; MockThreads mock_threads; @@ -359,7 +332,7 @@ TEST_F(TestMockImageReplayerJournalCreateLocalImageRequest, UpdateClientImageErr auto request = create_request( mock_threads, mock_state_builder, "global image id", &ctx); request->send(); - ASSERT_EQ(-EINVAL, ctx.wait()); + ASSERT_EQ(0, ctx.wait()); } } // namespace journal diff --git a/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc b/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc index 3eb42cf08e1..a6114e7d4cc 100644 --- a/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc +++ b/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc @@ -236,10 +236,11 @@ struct StateBuilder { MOCK_METHOD1(close_remote_image, void(Context*)); - MOCK_METHOD5(create_local_image_request, + MOCK_METHOD6(create_local_image_request, BaseRequest*(Threads*, librados::IoCtx&, const std::string&, + PoolMetaCache*, ProgressContext*, Context*)); MOCK_METHOD5(create_prepare_replay_request, @@ -399,8 +400,8 @@ public: void expect_create_local_image(MockStateBuilder& mock_state_builder, const std::string& local_image_id, int r) { EXPECT_CALL(mock_state_builder, - create_local_image_request(_, _, _, _, _)) - .WillOnce(WithArg<4>( + create_local_image_request(_, _, _, _, _, _)) + .WillOnce(WithArg<5>( Invoke([this, &mock_state_builder, local_image_id, r](Context* ctx) { if (r >= 0) { mock_state_builder.local_image_id = local_image_id; diff --git a/src/test/rbd_mirror/image_replayer/test_mock_CreateImageRequest.cc b/src/test/rbd_mirror/image_replayer/test_mock_CreateImageRequest.cc index 417c08c9657..63419fe4fa0 100644 --- a/src/test/rbd_mirror/image_replayer/test_mock_CreateImageRequest.cc +++ b/src/test/rbd_mirror/image_replayer/test_mock_CreateImageRequest.cc @@ -5,10 +5,10 @@ #include "include/rbd/librbd.hpp" #include "librbd/ImageState.h" #include "librbd/Operations.h" -#include "test/journal/mock/MockJournaler.h" #include "test/librados_test_stub/MockTestMemIoCtxImpl.h" #include "test/librados_test_stub/MockTestMemRadosClient.h" #include "test/librbd/mock/MockImageCtx.h" +#include "tools/rbd_mirror/PoolMetaCache.h" #include "tools/rbd_mirror/image_replayer/CreateImageRequest.h" #include "tools/rbd_mirror/image_replayer/CloseImageRequest.h" #include "tools/rbd_mirror/image_replayer/OpenImageRequest.h" @@ -108,15 +108,6 @@ CloneRequest* CloneRequest::s_instance = nullptr; } // namespace image - -namespace journal { - -template <> -struct TypeTraits { - typedef ::journal::MockJournalerProxy Journaler; -}; - -} // namespace journal } // namespace librbd namespace rbd { @@ -337,42 +328,6 @@ public: })); } - void expect_mirror_uuid_get(librados::IoCtx &io_ctx, - const std::string &mirror_uuid, int r) { - bufferlist bl; - encode(mirror_uuid, bl); - - EXPECT_CALL(get_mock_io_ctx(io_ctx), - exec(RBD_MIRRORING, _, StrEq("rbd"), StrEq("mirror_uuid_get"), _, _, _)) - .WillOnce(DoAll(WithArg<5>(Invoke([bl](bufferlist *out_bl) { - *out_bl = bl; - })), - Return(r))); - } - - void expect_journaler_get_client(journal::MockJournaler& mock_journaler, - const std::string& client_id, - librbd::journal::MirrorPeerState state, - int r) { - EXPECT_CALL(mock_journaler, construct()); - - librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; - mirror_peer_client_meta.state = state; - - librbd::journal::ClientData client_data{mirror_peer_client_meta}; - - cls::journal::Client client; - encode(client_data, client.data); - - EXPECT_CALL(mock_journaler, get_client(StrEq(client_id), _, _)) - .WillOnce(DoAll(WithArg<1>(Invoke([client](cls::journal::Client *out_client) { - *out_client = client; - })), - WithArg<2>(Invoke([this, r](Context *on_finish) { - m_threads->work_queue->queue(on_finish, r); - })))); - } - MockCreateImageRequest *create_request(MockThreads* mock_threads, const std::string &global_image_id, const std::string &remote_mirror_uuid, @@ -384,10 +339,12 @@ public: global_image_id, remote_mirror_uuid, local_image_name, local_image_id, &mock_remote_image_ctx, + &m_pool_meta_cache, cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, on_finish); } + PoolMetaCache m_pool_meta_cache{g_ceph_context}; librbd::ImageCtx *m_remote_image_ctx; }; @@ -425,111 +382,7 @@ TEST_F(TestMockImageReplayerCreateImageRequest, CreateError) { ASSERT_EQ(-EINVAL, ctx.wait()); } -TEST_F(TestMockImageReplayerCreateImageRequest, Clone) { - librbd::RBD rbd; - librbd::ImageCtx *local_image_ctx; - ASSERT_EQ(0, create_image(rbd, m_local_io_ctx, m_image_name, m_image_size)); - ASSERT_EQ(0, open_image(m_local_io_ctx, m_image_name, &local_image_ctx)); - snap_create(local_image_ctx, "snap"); - - std::string clone_image_name = get_temp_image_name(); - ASSERT_EQ(0, clone_image(m_remote_image_ctx, "snap", clone_image_name)); - - librbd::ImageCtx *remote_clone_image_ctx; - ASSERT_EQ(0, open_image(m_remote_io_ctx, clone_image_name, - &remote_clone_image_ctx)); - - librbd::MockTestImageCtx mock_remote_parent_image_ctx(*m_remote_image_ctx); - librbd::MockTestImageCtx mock_remote_clone_image_ctx(*remote_clone_image_ctx); - MockCloneRequest mock_clone_request; - MockOpenImageRequest mock_open_image_request; - MockCloseImageRequest mock_close_image_request; - journal::MockJournaler mock_remote_journaler; - - InSequence seq; - expect_ioctx_create(m_remote_io_ctx); - expect_ioctx_create(m_local_io_ctx); - expect_mirror_uuid_get(m_local_io_ctx, "local parent uuid", 0); - expect_journaler_get_client( - mock_remote_journaler, "local parent uuid", - librbd::journal::MIRROR_PEER_STATE_REPLAYING, 0); - expect_get_parent_global_image_id(m_remote_io_ctx, "global uuid", 0); - expect_mirror_image_get_image_id(m_local_io_ctx, "local parent id", 0); - - expect_open_image(mock_open_image_request, m_remote_io_ctx, - m_remote_image_ctx->id, mock_remote_parent_image_ctx, 0); - expect_test_op_features(mock_remote_clone_image_ctx, false); - expect_clone_image(mock_clone_request, 0); - expect_close_image(mock_close_image_request, mock_remote_parent_image_ctx, 0); - - C_SaferCond ctx; - MockThreads mock_threads(m_threads); - MockCreateImageRequest *request = create_request(&mock_threads, "global uuid", - "remote uuid", "image name", - "101241a7c4c9", - mock_remote_clone_image_ctx, - &ctx); - request->send(); - ASSERT_EQ(0, ctx.wait()); -} - -TEST_F(TestMockImageReplayerCreateImageRequest, CloneParentMirrorUuidGetError) { - std::string clone_image_name = get_temp_image_name(); - ASSERT_EQ(0, clone_image(m_remote_image_ctx, "snap", clone_image_name)); - - librbd::ImageCtx *remote_clone_image_ctx; - ASSERT_EQ(0, open_image(m_remote_io_ctx, clone_image_name, - &remote_clone_image_ctx)); - - librbd::MockTestImageCtx mock_remote_clone_image_ctx(*remote_clone_image_ctx); - - InSequence seq; - expect_ioctx_create(m_remote_io_ctx); - expect_ioctx_create(m_local_io_ctx); - expect_mirror_uuid_get(m_local_io_ctx, "local parent uuid", -EPERM); - - C_SaferCond ctx; - MockThreads mock_threads(m_threads); - MockCreateImageRequest *request = create_request(&mock_threads, "global uuid", - "remote uuid", "image name", - "101241a7c4c9", - mock_remote_clone_image_ctx, - &ctx); - request->send(); - ASSERT_EQ(-EPERM, ctx.wait()); -} - -TEST_F(TestMockImageReplayerCreateImageRequest, CloneGetRemoteParentClientStateError) { - std::string clone_image_name = get_temp_image_name(); - ASSERT_EQ(0, clone_image(m_remote_image_ctx, "snap", clone_image_name)); - - librbd::ImageCtx *remote_clone_image_ctx; - ASSERT_EQ(0, open_image(m_remote_io_ctx, clone_image_name, - &remote_clone_image_ctx)); - - librbd::MockTestImageCtx mock_remote_clone_image_ctx(*remote_clone_image_ctx); - journal::MockJournaler mock_remote_journaler; - - InSequence seq; - expect_ioctx_create(m_remote_io_ctx); - expect_ioctx_create(m_local_io_ctx); - expect_mirror_uuid_get(m_local_io_ctx, "local parent uuid", 0); - expect_journaler_get_client( - mock_remote_journaler, "local parent uuid", - librbd::journal::MIRROR_PEER_STATE_REPLAYING, -EPERM); - - C_SaferCond ctx; - MockThreads mock_threads(m_threads); - MockCreateImageRequest *request = create_request(&mock_threads, "global uuid", - "remote uuid", "image name", - "101241a7c4c9", - mock_remote_clone_image_ctx, - &ctx); - request->send(); - ASSERT_EQ(-EPERM, ctx.wait()); -} - -TEST_F(TestMockImageReplayerCreateImageRequest, CloneGetRemoteParentClientStateSyncing) { +TEST_F(TestMockImageReplayerCreateImageRequest, CloneGetGlobalImageIdError) { std::string clone_image_name = get_temp_image_name(); ASSERT_EQ(0, clone_image(m_remote_image_ctx, "snap", clone_image_name)); @@ -538,15 +391,11 @@ TEST_F(TestMockImageReplayerCreateImageRequest, CloneGetRemoteParentClientStateS &remote_clone_image_ctx)); librbd::MockTestImageCtx mock_remote_clone_image_ctx(*remote_clone_image_ctx); - journal::MockJournaler mock_remote_journaler; InSequence seq; expect_ioctx_create(m_remote_io_ctx); expect_ioctx_create(m_local_io_ctx); - expect_mirror_uuid_get(m_local_io_ctx, "local parent uuid", 0); - expect_journaler_get_client( - mock_remote_journaler, "local parent uuid", - librbd::journal::MIRROR_PEER_STATE_SYNCING, 0); + expect_get_parent_global_image_id(m_remote_io_ctx, "global uuid", -ENOENT); C_SaferCond ctx; MockThreads mock_threads(m_threads); @@ -559,7 +408,7 @@ TEST_F(TestMockImageReplayerCreateImageRequest, CloneGetRemoteParentClientStateS ASSERT_EQ(-ENOENT, ctx.wait()); } -TEST_F(TestMockImageReplayerCreateImageRequest, CloneGetGlobalImageIdError) { +TEST_F(TestMockImageReplayerCreateImageRequest, CloneGetLocalParentImageIdError) { std::string clone_image_name = get_temp_image_name(); ASSERT_EQ(0, clone_image(m_remote_image_ctx, "snap", clone_image_name)); @@ -568,16 +417,12 @@ TEST_F(TestMockImageReplayerCreateImageRequest, CloneGetGlobalImageIdError) { &remote_clone_image_ctx)); librbd::MockTestImageCtx mock_remote_clone_image_ctx(*remote_clone_image_ctx); - journal::MockJournaler mock_remote_journaler; InSequence seq; expect_ioctx_create(m_remote_io_ctx); expect_ioctx_create(m_local_io_ctx); - expect_mirror_uuid_get(m_local_io_ctx, "local parent uuid", 0); - expect_journaler_get_client( - mock_remote_journaler, "local parent uuid", - librbd::journal::MIRROR_PEER_STATE_REPLAYING, 0); - expect_get_parent_global_image_id(m_remote_io_ctx, "global uuid", -ENOENT); + expect_get_parent_global_image_id(m_remote_io_ctx, "global uuid", 0); + expect_mirror_image_get_image_id(m_local_io_ctx, "local parent id", -ENOENT); C_SaferCond ctx; MockThreads mock_threads(m_threads); @@ -590,7 +435,7 @@ TEST_F(TestMockImageReplayerCreateImageRequest, CloneGetGlobalImageIdError) { ASSERT_EQ(-ENOENT, ctx.wait()); } -TEST_F(TestMockImageReplayerCreateImageRequest, CloneGetLocalParentImageIdError) { +TEST_F(TestMockImageReplayerCreateImageRequest, CloneOpenRemoteParentError) { std::string clone_image_name = get_temp_image_name(); ASSERT_EQ(0, clone_image(m_remote_image_ctx, "snap", clone_image_name)); @@ -598,18 +443,19 @@ TEST_F(TestMockImageReplayerCreateImageRequest, CloneGetLocalParentImageIdError) ASSERT_EQ(0, open_image(m_remote_io_ctx, clone_image_name, &remote_clone_image_ctx)); + librbd::MockTestImageCtx mock_remote_parent_image_ctx(*m_remote_image_ctx); librbd::MockTestImageCtx mock_remote_clone_image_ctx(*remote_clone_image_ctx); - journal::MockJournaler mock_remote_journaler; + MockOpenImageRequest mock_open_image_request; InSequence seq; expect_ioctx_create(m_remote_io_ctx); expect_ioctx_create(m_local_io_ctx); - expect_mirror_uuid_get(m_local_io_ctx, "local parent uuid", 0); - expect_journaler_get_client( - mock_remote_journaler, "local parent uuid", - librbd::journal::MIRROR_PEER_STATE_REPLAYING, 0); expect_get_parent_global_image_id(m_remote_io_ctx, "global uuid", 0); - expect_mirror_image_get_image_id(m_local_io_ctx, "local parent id", -ENOENT); + expect_mirror_image_get_image_id(m_local_io_ctx, "local parent id", 0); + + expect_open_image(mock_open_image_request, m_remote_io_ctx, + m_remote_image_ctx->id, mock_remote_parent_image_ctx, + -ENOENT); C_SaferCond ctx; MockThreads mock_threads(m_threads); @@ -622,7 +468,14 @@ TEST_F(TestMockImageReplayerCreateImageRequest, CloneGetLocalParentImageIdError) ASSERT_EQ(-ENOENT, ctx.wait()); } -TEST_F(TestMockImageReplayerCreateImageRequest, CloneOpenRemoteParentError) { +TEST_F(TestMockImageReplayerCreateImageRequest, CloneParentImageSyncing) { + librbd::RBD rbd; + librbd::ImageCtx *local_image_ctx; + ASSERT_EQ(0, create_image(rbd, m_local_io_ctx, m_image_name, m_image_size)); + ASSERT_EQ(0, open_image(m_local_io_ctx, m_image_name, &local_image_ctx)); + snap_create(local_image_ctx, "snap"); + snap_create(m_remote_image_ctx, ".rbd-mirror.local parent uuid.1234"); + std::string clone_image_name = get_temp_image_name(); ASSERT_EQ(0, clone_image(m_remote_image_ctx, "snap", clone_image_name)); @@ -630,24 +483,23 @@ TEST_F(TestMockImageReplayerCreateImageRequest, CloneOpenRemoteParentError) { ASSERT_EQ(0, open_image(m_remote_io_ctx, clone_image_name, &remote_clone_image_ctx)); + m_pool_meta_cache.set_local_pool_meta( + m_local_io_ctx.get_id(), {"local parent uuid"}); + librbd::MockTestImageCtx mock_remote_parent_image_ctx(*m_remote_image_ctx); librbd::MockTestImageCtx mock_remote_clone_image_ctx(*remote_clone_image_ctx); MockOpenImageRequest mock_open_image_request; - journal::MockJournaler mock_remote_journaler; + MockCloseImageRequest mock_close_image_request; InSequence seq; expect_ioctx_create(m_remote_io_ctx); expect_ioctx_create(m_local_io_ctx); - expect_mirror_uuid_get(m_local_io_ctx, "local parent uuid", 0); - expect_journaler_get_client( - mock_remote_journaler, "local parent uuid", - librbd::journal::MIRROR_PEER_STATE_REPLAYING, 0); expect_get_parent_global_image_id(m_remote_io_ctx, "global uuid", 0); expect_mirror_image_get_image_id(m_local_io_ctx, "local parent id", 0); expect_open_image(mock_open_image_request, m_remote_io_ctx, - m_remote_image_ctx->id, mock_remote_parent_image_ctx, - -ENOENT); + m_remote_image_ctx->id, mock_remote_parent_image_ctx, 0); + expect_close_image(mock_close_image_request, mock_remote_parent_image_ctx, 0); C_SaferCond ctx; MockThreads mock_threads(m_threads); @@ -674,20 +526,18 @@ TEST_F(TestMockImageReplayerCreateImageRequest, CloneError) { ASSERT_EQ(0, open_image(m_remote_io_ctx, clone_image_name, &remote_clone_image_ctx)); + m_pool_meta_cache.set_local_pool_meta( + m_local_io_ctx.get_id(), {"local parent uuid"}); + librbd::MockTestImageCtx mock_remote_parent_image_ctx(*m_remote_image_ctx); librbd::MockTestImageCtx mock_remote_clone_image_ctx(*remote_clone_image_ctx); MockCloneRequest mock_clone_request; MockOpenImageRequest mock_open_image_request; MockCloseImageRequest mock_close_image_request; - journal::MockJournaler mock_remote_journaler; InSequence seq; expect_ioctx_create(m_remote_io_ctx); expect_ioctx_create(m_local_io_ctx); - expect_mirror_uuid_get(m_local_io_ctx, "local parent uuid", 0); - expect_journaler_get_client( - mock_remote_journaler, "local parent uuid", - librbd::journal::MIRROR_PEER_STATE_REPLAYING, 0); expect_get_parent_global_image_id(m_remote_io_ctx, "global uuid", 0); expect_mirror_image_get_image_id(m_local_io_ctx, "local parent id", 0); @@ -722,20 +572,18 @@ TEST_F(TestMockImageReplayerCreateImageRequest, CloneRemoteParentCloseError) { ASSERT_EQ(0, open_image(m_remote_io_ctx, clone_image_name, &remote_clone_image_ctx)); + m_pool_meta_cache.set_local_pool_meta( + m_local_io_ctx.get_id(), {"local parent uuid"}); + librbd::MockTestImageCtx mock_remote_parent_image_ctx(*m_remote_image_ctx); librbd::MockTestImageCtx mock_remote_clone_image_ctx(*remote_clone_image_ctx); MockCloneRequest mock_clone_request; MockOpenImageRequest mock_open_image_request; MockCloseImageRequest mock_close_image_request; - journal::MockJournaler mock_remote_journaler; InSequence seq; expect_ioctx_create(m_remote_io_ctx); expect_ioctx_create(m_local_io_ctx); - expect_mirror_uuid_get(m_local_io_ctx, "local parent uuid", 0); - expect_journaler_get_client( - mock_remote_journaler, "local parent uuid", - librbd::journal::MIRROR_PEER_STATE_REPLAYING, 0); expect_get_parent_global_image_id(m_remote_io_ctx, "global uuid", 0); expect_mirror_image_get_image_id(m_local_io_ctx, "local parent id", 0); diff --git a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc index 7f338cc3e08..b41f71f6446 100644 --- a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc +++ b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc @@ -338,7 +338,8 @@ void BootstrapRequest::create_local_image() { BootstrapRequest, &BootstrapRequest::handle_create_local_image>(this); auto request = (*m_state_builder)->create_local_image_request( - m_threads, m_local_io_ctx, m_global_image_id, m_progress_ctx, ctx); + m_threads, m_local_io_ctx, m_global_image_id, m_pool_meta_cache, + m_progress_ctx, ctx); request->send(); } diff --git a/src/tools/rbd_mirror/image_replayer/CreateImageRequest.cc b/src/tools/rbd_mirror/image_replayer/CreateImageRequest.cc index 854f306f5fe..f1f9cc42bb4 100644 --- a/src/tools/rbd_mirror/image_replayer/CreateImageRequest.cc +++ b/src/tools/rbd_mirror/image_replayer/CreateImageRequest.cc @@ -8,17 +8,18 @@ #include "common/errno.h" #include "common/WorkQueue.h" #include "cls/rbd/cls_rbd_client.h" -#include "journal/Journaler.h" -#include "journal/Settings.h" #include "librbd/ImageCtx.h" #include "librbd/ImageState.h" #include "librbd/internal.h" #include "librbd/Utils.h" #include "librbd/image/CreateRequest.h" #include "librbd/image/CloneRequest.h" -#include "librbd/journal/Types.h" +#include "tools/rbd_mirror/PoolMetaCache.h" +#include "tools/rbd_mirror/Types.h" #include "tools/rbd_mirror/Threads.h" #include "tools/rbd_mirror/image_replayer/Utils.h" +#include "tools/rbd_mirror/image_sync/Utils.h" +#include #define dout_context g_ceph_context #define dout_subsys ceph_subsys_rbd_mirror @@ -43,6 +44,7 @@ CreateImageRequest::CreateImageRequest( const std::string &local_image_name, const std::string &local_image_id, I *remote_image_ctx, + PoolMetaCache* pool_meta_cache, cls::rbd::MirrorImageMode mirror_image_mode, Context *on_finish) : m_threads(threads), m_local_io_ctx(local_io_ctx), @@ -50,6 +52,7 @@ CreateImageRequest::CreateImageRequest( m_remote_mirror_uuid(remote_mirror_uuid), m_local_image_name(local_image_name), m_local_image_id(local_image_id), m_remote_image_ctx(remote_image_ctx), + m_pool_meta_cache(pool_meta_cache), m_mirror_image_mode(mirror_image_mode), m_on_finish(on_finish) { } @@ -64,7 +67,7 @@ void CreateImageRequest::send() { if (m_remote_parent_spec.pool_id == -1) { create_image(); } else { - get_local_parent_mirror_uuid(); + get_parent_global_image_id(); } } @@ -108,102 +111,6 @@ void CreateImageRequest::handle_create_image(int r) { finish(0); } -template -void CreateImageRequest::get_local_parent_mirror_uuid() { - dout(10) << dendl; - - librados::ObjectReadOperation op; - librbd::cls_client::mirror_uuid_get_start(&op); - - librados::AioCompletion *aio_comp = create_rados_callback< - CreateImageRequest, - &CreateImageRequest::handle_get_local_parent_mirror_uuid>(this); - m_out_bl.clear(); - int r = m_local_parent_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op, - &m_out_bl); - ceph_assert(r == 0); - aio_comp->release(); -} - -template -void CreateImageRequest::handle_get_local_parent_mirror_uuid(int r) { - if (r >= 0) { - auto it = m_out_bl.cbegin(); - r = librbd::cls_client::mirror_uuid_get_finish( - &it, &m_local_parent_mirror_uuid); - if (r >= 0 && m_local_parent_mirror_uuid.empty()) { - r = -ENOENT; - } - } - - dout(10) << "r=" << r << dendl; - if (r < 0) { - if (r == -ENOENT) { - dout(5) << "local parent mirror uuid missing" << dendl; - } else { - derr << "failed to retrieve local parent mirror uuid: " << cpp_strerror(r) - << dendl; - } - finish(r); - return; - } - - dout(15) << "local_parent_mirror_uuid=" << m_local_parent_mirror_uuid - << dendl; - get_remote_parent_client_state(); -} - -template -void CreateImageRequest::get_remote_parent_client_state() { - dout(10) << dendl; - - m_remote_journaler = new Journaler(m_threads->work_queue, m_threads->timer, - &m_threads->timer_lock, - m_remote_parent_io_ctx, - m_remote_parent_spec.image_id, - m_local_parent_mirror_uuid, {}, nullptr); - - Context *ctx = create_async_context_callback( - m_threads->work_queue, create_context_callback< - CreateImageRequest, - &CreateImageRequest::handle_get_remote_parent_client_state>(this)); - m_remote_journaler->get_client(m_local_parent_mirror_uuid, &m_client, ctx); -} - -template -void CreateImageRequest::handle_get_remote_parent_client_state(int r) { - dout(10) << "r=" << r << dendl; - - delete m_remote_journaler; - m_remote_journaler = nullptr; - - librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; - if (r == -ENOENT) { - dout(15) << "client not registered to parent image" << dendl; - finish(r); - return; - } else if (r < 0) { - derr << "failed to retrieve parent client: " << cpp_strerror(r) << dendl; - finish(r); - return; - } else if (!util::decode_client_meta(m_client, &mirror_peer_client_meta)) { - // require operator intervention since the data is corrupt - derr << "failed to decode parent client: " << cpp_strerror(r) << dendl; - finish(-EBADMSG); - return; - } else if (mirror_peer_client_meta.state != - librbd::journal::MIRROR_PEER_STATE_REPLAYING) { - // avoid possible race w/ incomplete parent image since the parent snapshot - // might be deleted if the sync restarts - dout(15) << "parent image still syncing" << dendl; - finish(-ENOENT); - return; - } - - get_parent_global_image_id(); -} - - template void CreateImageRequest::get_parent_global_image_id() { dout(10) << dendl; @@ -324,18 +231,51 @@ template void CreateImageRequest::clone_image() { dout(10) << dendl; + LocalPoolMeta local_parent_pool_meta; + int r = m_pool_meta_cache->get_local_pool_meta( + m_local_parent_io_ctx.get_id(), &local_parent_pool_meta); + if (r < 0) { + derr << "failed to retrieve local parent mirror uuid for pool " + << m_local_parent_io_ctx.get_id() << dendl; + m_ret_val = r; + close_remote_parent_image(); + return; + } + + // ensure no image sync snapshots for the local cluster exist in the + // remote image + bool found_parent_snap = false; + bool found_image_sync_snap = false; std::string snap_name; cls::rbd::SnapshotNamespace snap_namespace; { + auto snap_prefix = image_sync::util::get_snapshot_name_prefix( + local_parent_pool_meta.mirror_uuid); + std::shared_lock remote_image_locker(m_remote_parent_image_ctx->image_lock); - auto it = m_remote_parent_image_ctx->snap_info.find( - m_remote_parent_spec.snap_id); - if (it != m_remote_parent_image_ctx->snap_info.end()) { - snap_name = it->second.name; - snap_namespace = it->second.snap_namespace; + for (auto snap_info : m_remote_parent_image_ctx->snap_info) { + if (snap_info.first == m_remote_parent_spec.snap_id) { + found_parent_snap = true; + snap_name = snap_info.second.name; + snap_namespace = snap_info.second.snap_namespace; + } else if (boost::starts_with(snap_info.second.name, snap_prefix)) { + found_image_sync_snap = true; + } } } + if (!found_parent_snap) { + dout(15) << "remote parent image snapshot not found" << dendl; + m_ret_val = -ENOENT; + close_remote_parent_image(); + return; + } else if (found_image_sync_snap) { + dout(15) << "parent image not synced to local cluster" << dendl; + m_ret_val = -ENOENT; + close_remote_parent_image(); + return; + } + librbd::ImageOptions opts; populate_image_options(&opts); @@ -359,8 +299,7 @@ void CreateImageRequest::handle_clone_image(int r) { dout(10) << "r=" << r << dendl; if (r == -EBADF) { dout(5) << "image id " << m_local_image_id << " already in-use" << dendl; - finish(r); - return; + m_ret_val = r; } else if (r < 0) { derr << "failed to clone image " << m_parent_pool_name << "/" << m_remote_parent_spec.image_id << " to " @@ -436,9 +375,10 @@ int CreateImageRequest::validate_parent() { } // map remote parent pool to local parent pool - librados::Rados remote_rados(m_remote_image_ctx->md_ctx); - int r = remote_rados.ioctx_create2(m_remote_parent_spec.pool_id, - m_remote_parent_io_ctx); + int r = librbd::util::create_ioctx( + m_remote_image_ctx->md_ctx, "remote parent pool", + m_remote_parent_spec.pool_id, m_remote_parent_spec.pool_namespace, + &m_remote_parent_io_ctx); if (r < 0) { derr << "failed to open remote parent pool " << m_remote_parent_spec.pool_id << ": " << cpp_strerror(r) << dendl; @@ -455,6 +395,7 @@ int CreateImageRequest::validate_parent() { << cpp_strerror(r) << dendl; return r; } + m_local_parent_io_ctx.set_namespace(m_remote_parent_io_ctx.get_namespace()); return 0; } diff --git a/src/tools/rbd_mirror/image_replayer/CreateImageRequest.h b/src/tools/rbd_mirror/image_replayer/CreateImageRequest.h index c8564e6179b..eda12cdb3f9 100644 --- a/src/tools/rbd_mirror/image_replayer/CreateImageRequest.h +++ b/src/tools/rbd_mirror/image_replayer/CreateImageRequest.h @@ -7,21 +7,19 @@ #include "include/int_types.h" #include "include/types.h" #include "include/rados/librados.hpp" -#include "cls/journal/cls_journal_types.h" #include "cls/rbd/cls_rbd_types.h" #include "librbd/Types.h" -#include "librbd/journal/TypeTraits.h" #include class Context; class ContextWQ; -namespace journal { class Journaler; } namespace librbd { class ImageCtx; } namespace librbd { class ImageOptions; } namespace rbd { namespace mirror { +class PoolMetaCache; template struct Threads; namespace image_replayer { @@ -29,29 +27,34 @@ namespace image_replayer { template class CreateImageRequest { public: - static CreateImageRequest *create(Threads *threads, - librados::IoCtx &local_io_ctx, - const std::string &global_image_id, - const std::string &remote_mirror_uuid, - const std::string &local_image_name, - const std::string &local_image_id, - ImageCtxT *remote_image_ctx, - cls::rbd::MirrorImageMode mirror_image_mode, - Context *on_finish) { + static CreateImageRequest *create( + Threads *threads, + librados::IoCtx &local_io_ctx, + const std::string &global_image_id, + const std::string &remote_mirror_uuid, + const std::string &local_image_name, + const std::string &local_image_id, + ImageCtxT *remote_image_ctx, + PoolMetaCache* pool_meta_cache, + cls::rbd::MirrorImageMode mirror_image_mode, + Context *on_finish) { return new CreateImageRequest(threads, local_io_ctx, global_image_id, remote_mirror_uuid, local_image_name, local_image_id, remote_image_ctx, - mirror_image_mode, on_finish); + pool_meta_cache, mirror_image_mode, + on_finish); } - CreateImageRequest(Threads *threads, librados::IoCtx &local_io_ctx, - const std::string &global_image_id, - const std::string &remote_mirror_uuid, - const std::string &local_image_name, - const std::string &local_image_id, - ImageCtxT *remote_image_ctx, - cls::rbd::MirrorImageMode mirror_image_mode, - Context *on_finish); + CreateImageRequest( + Threads *threads, librados::IoCtx &local_io_ctx, + const std::string &global_image_id, + const std::string &remote_mirror_uuid, + const std::string &local_image_name, + const std::string &local_image_id, + ImageCtxT *remote_image_ctx, + PoolMetaCache* pool_meta_cache, + cls::rbd::MirrorImageMode mirror_image_mode, + Context *on_finish); void send(); @@ -65,13 +68,7 @@ private: * |\------------> CREATE_IMAGE ---------------------\ * (error) * | | * * | (clone) | * - * \-------------> GET_LOCAL_PARENT_MIRROR_UUID * * | * * * * - * | | * * - * v | * - * GET_REMOTE_PARENT_CLIENT_STATE * | * * * * - * | | * * - * v | * - * GET_PARENT_GLOBAL_IMAGE_ID * * * | * * * * + * \-------------> GET_PARENT_GLOBAL_IMAGE_ID * * * | * * * * * | | * * * v | * * GET_LOCAL_PARENT_IMAGE_ID * * * * | * * * * @@ -89,9 +86,6 @@ private: * @endverbatim */ - typedef librbd::journal::TypeTraits TypeTraits; - typedef typename TypeTraits::Journaler Journaler; - Threads *m_threads; librados::IoCtx &m_local_io_ctx; std::string m_global_image_id; @@ -99,12 +93,11 @@ private: std::string m_local_image_name; std::string m_local_image_id; ImageCtxT *m_remote_image_ctx; + PoolMetaCache* m_pool_meta_cache; cls::rbd::MirrorImageMode m_mirror_image_mode; Context *m_on_finish; librados::IoCtx m_remote_parent_io_ctx; - std::string m_local_parent_mirror_uuid; - Journaler *m_remote_journaler = nullptr; ImageCtxT *m_remote_parent_image_ctx = nullptr; cls::rbd::ParentImageSpec m_remote_parent_spec; @@ -114,18 +107,11 @@ private: bufferlist m_out_bl; std::string m_parent_global_image_id; std::string m_parent_pool_name; - cls::journal::Client m_client; int m_ret_val = 0; void create_image(); void handle_create_image(int r); - void get_local_parent_mirror_uuid(); - void handle_get_local_parent_mirror_uuid(int r); - - void get_remote_parent_client_state(); - void handle_get_remote_parent_client_state(int r); - void get_parent_global_image_id(); void handle_get_parent_global_image_id(int r); diff --git a/src/tools/rbd_mirror/image_replayer/StateBuilder.h b/src/tools/rbd_mirror/image_replayer/StateBuilder.h index a85cb3b779a..9386b01c99e 100644 --- a/src/tools/rbd_mirror/image_replayer/StateBuilder.h +++ b/src/tools/rbd_mirror/image_replayer/StateBuilder.h @@ -15,6 +15,7 @@ namespace rbd { namespace mirror { struct BaseRequest; +struct PoolMetaCache; struct ProgressContext; template class Threads; @@ -55,6 +56,7 @@ public: Threads* threads, librados::IoCtx& local_io_ctx, const std::string& global_image_id, + PoolMetaCache* pool_meta_cache, ProgressContext* progress_ctx, Context* on_finish) = 0; diff --git a/src/tools/rbd_mirror/image_replayer/journal/CreateLocalImageRequest.cc b/src/tools/rbd_mirror/image_replayer/journal/CreateLocalImageRequest.cc index fa8765faf1c..087cf4f5fd4 100644 --- a/src/tools/rbd_mirror/image_replayer/journal/CreateLocalImageRequest.cc +++ b/src/tools/rbd_mirror/image_replayer/journal/CreateLocalImageRequest.cc @@ -10,8 +10,11 @@ #include "librbd/ImageCtx.h" #include "librbd/Utils.h" #include "librbd/journal/Types.h" +#include "tools/rbd_mirror/PoolMetaCache.h" #include "tools/rbd_mirror/ProgressContext.h" +#include "tools/rbd_mirror/Threads.h" #include "tools/rbd_mirror/image_replayer/CreateImageRequest.h" +#include "tools/rbd_mirror/image_replayer/Utils.h" #include "tools/rbd_mirror/image_replayer/journal/StateBuilder.h" #define dout_context g_ceph_context @@ -26,11 +29,11 @@ namespace mirror { namespace image_replayer { namespace journal { +using librbd::util::create_async_context_callback; using librbd::util::create_context_callback; template void CreateLocalImageRequest::send() { - m_state_builder->local_image_id = ""; unregister_client(); } @@ -55,6 +58,7 @@ void CreateLocalImageRequest::handle_unregister_client(int r) { return; } + m_state_builder->local_image_id = ""; m_state_builder->remote_client_meta = {}; register_client(); } @@ -116,7 +120,7 @@ void CreateLocalImageRequest::create_local_image() { m_threads, m_local_io_ctx, m_global_image_id, m_state_builder->remote_mirror_uuid, image_name, m_state_builder->local_image_id, m_remote_image_ctx, - cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, ctx); + m_pool_meta_cache, cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, ctx); request->send(); } template @@ -126,8 +130,7 @@ void CreateLocalImageRequest::handle_create_local_image(int r) { if (r == -EBADF) { dout(5) << "image id " << m_state_builder->local_image_id << " " << "already in-use" << dendl; - m_state_builder->local_image_id = ""; - update_client_image(); + unregister_client(); return; } else if (r < 0) { if (r == -ENOENT) { @@ -142,45 +145,6 @@ void CreateLocalImageRequest::handle_create_local_image(int r) { finish(0); } -template -void CreateLocalImageRequest::update_client_image() { - ceph_assert(m_state_builder->local_image_id.empty()); - m_state_builder->local_image_id = - librbd::util::generate_image_id(m_local_io_ctx); - - dout(10) << "local_image_id=" << m_state_builder->local_image_id << dendl; - update_progress("UPDATE_CLIENT_IMAGE"); - - librbd::journal::MirrorPeerClientMeta client_meta{ - m_state_builder->local_image_id}; - client_meta.state = librbd::journal::MIRROR_PEER_STATE_SYNCING; - - librbd::journal::ClientData client_data(client_meta); - bufferlist data_bl; - encode(client_data, data_bl); - - auto ctx = create_context_callback< - CreateLocalImageRequest, - &CreateLocalImageRequest::handle_update_client_image>(this); - m_state_builder->remote_journaler->update_client(data_bl, ctx); -} - -template -void CreateLocalImageRequest::handle_update_client_image(int r) { - dout(10) << "r=" << r << dendl; - - if (r < 0) { - derr << "failed to update client: " << cpp_strerror(r) << dendl; - finish(r); - return; - } - - m_state_builder->remote_client_meta = {m_state_builder->local_image_id}; - m_state_builder->remote_client_meta.state = - librbd::journal::MIRROR_PEER_STATE_SYNCING; - create_local_image(); -} - template void CreateLocalImageRequest::update_progress( const std::string& description) { diff --git a/src/tools/rbd_mirror/image_replayer/journal/CreateLocalImageRequest.h b/src/tools/rbd_mirror/image_replayer/journal/CreateLocalImageRequest.h index 1ea19aca269..fc776ecc301 100644 --- a/src/tools/rbd_mirror/image_replayer/journal/CreateLocalImageRequest.h +++ b/src/tools/rbd_mirror/image_replayer/journal/CreateLocalImageRequest.h @@ -9,11 +9,13 @@ #include struct Context; +namespace journal { class Journaler; } namespace librbd { class ImageCtx; } namespace rbd { namespace mirror { +class PoolMetaCache; class ProgressContext; template struct Threads; @@ -32,12 +34,13 @@ public: librados::IoCtx& local_io_ctx, ImageCtxT* remote_image_ctx, const std::string& global_image_id, + PoolMetaCache* pool_meta_cache, ProgressContext* progress_ctx, StateBuilder* state_builder, Context* on_finish) { return new CreateLocalImageRequest(threads, local_io_ctx, remote_image_ctx, - global_image_id, progress_ctx, - state_builder, on_finish); + global_image_id, pool_meta_cache, + progress_ctx, state_builder, on_finish); } CreateLocalImageRequest( @@ -45,6 +48,7 @@ public: librados::IoCtx& local_io_ctx, ImageCtxT* remote_image_ctx, const std::string& global_image_id, + PoolMetaCache* pool_meta_cache, ProgressContext* progress_ctx, StateBuilder* state_builder, Context* on_finish) @@ -53,6 +57,7 @@ public: m_local_io_ctx(local_io_ctx), m_remote_image_ctx(remote_image_ctx), m_global_image_id(global_image_id), + m_pool_meta_cache(pool_meta_cache), m_progress_ctx(progress_ctx), m_state_builder(state_builder) { } @@ -66,14 +71,12 @@ private: * * | * v - * UNREGISTER_CLIENT - * | - * v - * REGISTER_CLIENT - * | - * | . . . . . . . . . UPDATE_CLIENT_IMAGE - * | . ^ - * v v (id exists) * + * UNREGISTER_CLIENT < * * * * * * * * + * | * + * v * + * REGISTER_CLIENT * + * | * + * v (id exists) * * CREATE_LOCAL_IMAGE * * * * * * * * * * | * v @@ -86,6 +89,7 @@ private: librados::IoCtx& m_local_io_ctx; ImageCtxT* m_remote_image_ctx; std::string m_global_image_id; + PoolMetaCache* m_pool_meta_cache; ProgressContext* m_progress_ctx; StateBuilder* m_state_builder; @@ -98,9 +102,6 @@ private: void create_local_image(); void handle_create_local_image(int r); - void update_client_image(); - void handle_update_client_image(int r); - void update_progress(const std::string& description); }; diff --git a/src/tools/rbd_mirror/image_replayer/journal/StateBuilder.cc b/src/tools/rbd_mirror/image_replayer/journal/StateBuilder.cc index 9056785475b..0bc641f7743 100644 --- a/src/tools/rbd_mirror/image_replayer/journal/StateBuilder.cc +++ b/src/tools/rbd_mirror/image_replayer/journal/StateBuilder.cc @@ -75,11 +75,12 @@ BaseRequest* StateBuilder::create_local_image_request( Threads* threads, librados::IoCtx& local_io_ctx, const std::string& global_image_id, + PoolMetaCache* pool_meta_cache, ProgressContext* progress_ctx, Context* on_finish) { return CreateLocalImageRequest::create( threads, local_io_ctx, this->remote_image_ctx, this->global_image_id, - progress_ctx, this, on_finish); + pool_meta_cache, progress_ctx, this, on_finish); } template diff --git a/src/tools/rbd_mirror/image_replayer/journal/StateBuilder.h b/src/tools/rbd_mirror/image_replayer/journal/StateBuilder.h index ee0743d3b90..8fed18bd0ab 100644 --- a/src/tools/rbd_mirror/image_replayer/journal/StateBuilder.h +++ b/src/tools/rbd_mirror/image_replayer/journal/StateBuilder.h @@ -46,6 +46,7 @@ public: Threads* threads, librados::IoCtx& local_io_ctx, const std::string& global_image_id, + PoolMetaCache* pool_meta_cache, ProgressContext* progress_ctx, Context* on_finish) override; diff --git a/src/tools/rbd_mirror/image_replayer/snapshot/StateBuilder.cc b/src/tools/rbd_mirror/image_replayer/snapshot/StateBuilder.cc index 604b62f9c74..648e38d2f6e 100644 --- a/src/tools/rbd_mirror/image_replayer/snapshot/StateBuilder.cc +++ b/src/tools/rbd_mirror/image_replayer/snapshot/StateBuilder.cc @@ -67,6 +67,7 @@ BaseRequest* StateBuilder::create_local_image_request( Threads* threads, librados::IoCtx& local_io_ctx, const std::string& global_image_id, + PoolMetaCache* pool_meta_cache, ProgressContext* progress_ctx, Context* on_finish) { // TODO diff --git a/src/tools/rbd_mirror/image_replayer/snapshot/StateBuilder.h b/src/tools/rbd_mirror/image_replayer/snapshot/StateBuilder.h index 7444ceb344d..d0604e4dfd6 100644 --- a/src/tools/rbd_mirror/image_replayer/snapshot/StateBuilder.h +++ b/src/tools/rbd_mirror/image_replayer/snapshot/StateBuilder.h @@ -41,6 +41,7 @@ public: Threads* threads, librados::IoCtx& local_io_ctx, const std::string& global_image_id, + PoolMetaCache* pool_meta_cache, ProgressContext* progress_ctx, Context* on_finish) override; -- 2.39.5