From: Jason Dillaman Date: Wed, 29 Jan 2020 19:42:14 +0000 (-0500) Subject: rbd-mirror: replace journal-specific parent image validation X-Git-Tag: v15.1.1~472^2~14 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=c0fdb2ca0f36d18aa5a42299d6aeec1744264bcb;p=ceph.git rbd-mirror: replace journal-specific parent image validation When cloning a mirrored image, instead of looking at the parent image's journal state to tell if its still being synced, we can instead verify that it doesn't have an rbd-mirror image sync snapshots. If it doesn't, the image isn't being actively synced, but it might not have started either. However, the clone state machine will open the local parent image and attempt to find the specified snapshot. If it exists, the image sync is complete. Signed-off-by: Jason Dillaman --- diff --git a/src/test/rbd_mirror/image_replayer/journal/test_mock_CreateLocalImageRequest.cc b/src/test/rbd_mirror/image_replayer/journal/test_mock_CreateLocalImageRequest.cc index 08402c213dec..cc22671609d7 100644 --- a/src/test/rbd_mirror/image_replayer/journal/test_mock_CreateLocalImageRequest.cc +++ b/src/test/rbd_mirror/image_replayer/journal/test_mock_CreateLocalImageRequest.cc @@ -69,6 +69,7 @@ struct CreateImageRequest { const std::string &local_image_name, const std::string &local_image_id, librbd::MockTestImageCtx *remote_image_ctx, + PoolMetaCache* pool_meta_cache, cls::rbd::MirrorImageMode mirror_image_mode, Context *on_finish) { ceph_assert(s_instance != nullptr); @@ -173,6 +174,7 @@ public: m_threads->work_queue->queue(on_finish, r); }))); } + void expect_create_image(MockCreateImageRequest& mock_create_image_request, const std::string& image_id, int r) { EXPECT_CALL(mock_create_image_request, construct(image_id)); @@ -189,7 +191,8 @@ public: Context* on_finish) { return new MockCreateLocalImageRequest( &mock_threads, m_local_io_ctx, m_mock_remote_image_ctx, - global_image_id, nullptr, &mock_state_builder, on_finish); + global_image_id, nullptr, nullptr, &mock_state_builder, + on_finish); } librbd::ImageCtx *m_remote_image_ctx; @@ -316,42 +319,12 @@ TEST_F(TestMockImageReplayerJournalCreateLocalImageRequest, CreateImageDuplicate MockCreateImageRequest mock_create_image_request; expect_create_image(mock_create_image_request, "local image id", -EBADF); - // update image id - expect_journaler_update_client(mock_journaler, client_data, 0); - - // re-create the local image - expect_create_image(mock_create_image_request, "local image id", 0); - - C_SaferCond ctx; - MockThreads mock_threads; - MockStateBuilder mock_state_builder; - auto request = create_request( - mock_threads, mock_state_builder, "global image id", &ctx); - request->send(); - ASSERT_EQ(0, ctx.wait()); -} - -TEST_F(TestMockImageReplayerJournalCreateLocalImageRequest, UpdateClientImageError) { - InSequence seq; - // re-register the client - ::journal::MockJournaler mock_journaler; expect_journaler_unregister_client(mock_journaler, 0); - - librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; - librbd::util::s_image_id = "local image id"; - mirror_peer_client_meta.image_id = "local image id"; - mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_SYNCING; - librbd::journal::ClientData client_data; - client_data.client_meta = mirror_peer_client_meta; expect_journaler_register_client(mock_journaler, client_data, 0); - // create the missing local image - MockCreateImageRequest mock_create_image_request; - expect_create_image(mock_create_image_request, "local image id", -EBADF); - - // update image id - expect_journaler_update_client(mock_journaler, client_data, -EINVAL); + // re-create the local image + expect_create_image(mock_create_image_request, "local image id", 0); C_SaferCond ctx; MockThreads mock_threads; @@ -359,7 +332,7 @@ TEST_F(TestMockImageReplayerJournalCreateLocalImageRequest, UpdateClientImageErr auto request = create_request( mock_threads, mock_state_builder, "global image id", &ctx); request->send(); - ASSERT_EQ(-EINVAL, ctx.wait()); + ASSERT_EQ(0, ctx.wait()); } } // namespace journal diff --git a/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc b/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc index 3eb42cf08e18..a6114e7d4cc5 100644 --- a/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc +++ b/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc @@ -236,10 +236,11 @@ struct StateBuilder { MOCK_METHOD1(close_remote_image, void(Context*)); - MOCK_METHOD5(create_local_image_request, + MOCK_METHOD6(create_local_image_request, BaseRequest*(Threads*, librados::IoCtx&, const std::string&, + PoolMetaCache*, ProgressContext*, Context*)); MOCK_METHOD5(create_prepare_replay_request, @@ -399,8 +400,8 @@ public: void expect_create_local_image(MockStateBuilder& mock_state_builder, const std::string& local_image_id, int r) { EXPECT_CALL(mock_state_builder, - create_local_image_request(_, _, _, _, _)) - .WillOnce(WithArg<4>( + create_local_image_request(_, _, _, _, _, _)) + .WillOnce(WithArg<5>( Invoke([this, &mock_state_builder, local_image_id, r](Context* ctx) { if (r >= 0) { mock_state_builder.local_image_id = local_image_id; diff --git a/src/test/rbd_mirror/image_replayer/test_mock_CreateImageRequest.cc b/src/test/rbd_mirror/image_replayer/test_mock_CreateImageRequest.cc index 417c08c96578..63419fe4fa0e 100644 --- a/src/test/rbd_mirror/image_replayer/test_mock_CreateImageRequest.cc +++ b/src/test/rbd_mirror/image_replayer/test_mock_CreateImageRequest.cc @@ -5,10 +5,10 @@ #include "include/rbd/librbd.hpp" #include "librbd/ImageState.h" #include "librbd/Operations.h" -#include "test/journal/mock/MockJournaler.h" #include "test/librados_test_stub/MockTestMemIoCtxImpl.h" #include "test/librados_test_stub/MockTestMemRadosClient.h" #include "test/librbd/mock/MockImageCtx.h" +#include "tools/rbd_mirror/PoolMetaCache.h" #include "tools/rbd_mirror/image_replayer/CreateImageRequest.h" #include "tools/rbd_mirror/image_replayer/CloseImageRequest.h" #include "tools/rbd_mirror/image_replayer/OpenImageRequest.h" @@ -108,15 +108,6 @@ CloneRequest* CloneRequest::s_instance = nullptr; } // namespace image - -namespace journal { - -template <> -struct TypeTraits { - typedef ::journal::MockJournalerProxy Journaler; -}; - -} // namespace journal } // namespace librbd namespace rbd { @@ -337,42 +328,6 @@ public: })); } - void expect_mirror_uuid_get(librados::IoCtx &io_ctx, - const std::string &mirror_uuid, int r) { - bufferlist bl; - encode(mirror_uuid, bl); - - EXPECT_CALL(get_mock_io_ctx(io_ctx), - exec(RBD_MIRRORING, _, StrEq("rbd"), StrEq("mirror_uuid_get"), _, _, _)) - .WillOnce(DoAll(WithArg<5>(Invoke([bl](bufferlist *out_bl) { - *out_bl = bl; - })), - Return(r))); - } - - void expect_journaler_get_client(journal::MockJournaler& mock_journaler, - const std::string& client_id, - librbd::journal::MirrorPeerState state, - int r) { - EXPECT_CALL(mock_journaler, construct()); - - librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; - mirror_peer_client_meta.state = state; - - librbd::journal::ClientData client_data{mirror_peer_client_meta}; - - cls::journal::Client client; - encode(client_data, client.data); - - EXPECT_CALL(mock_journaler, get_client(StrEq(client_id), _, _)) - .WillOnce(DoAll(WithArg<1>(Invoke([client](cls::journal::Client *out_client) { - *out_client = client; - })), - WithArg<2>(Invoke([this, r](Context *on_finish) { - m_threads->work_queue->queue(on_finish, r); - })))); - } - MockCreateImageRequest *create_request(MockThreads* mock_threads, const std::string &global_image_id, const std::string &remote_mirror_uuid, @@ -384,10 +339,12 @@ public: global_image_id, remote_mirror_uuid, local_image_name, local_image_id, &mock_remote_image_ctx, + &m_pool_meta_cache, cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, on_finish); } + PoolMetaCache m_pool_meta_cache{g_ceph_context}; librbd::ImageCtx *m_remote_image_ctx; }; @@ -425,111 +382,7 @@ TEST_F(TestMockImageReplayerCreateImageRequest, CreateError) { ASSERT_EQ(-EINVAL, ctx.wait()); } -TEST_F(TestMockImageReplayerCreateImageRequest, Clone) { - librbd::RBD rbd; - librbd::ImageCtx *local_image_ctx; - ASSERT_EQ(0, create_image(rbd, m_local_io_ctx, m_image_name, m_image_size)); - ASSERT_EQ(0, open_image(m_local_io_ctx, m_image_name, &local_image_ctx)); - snap_create(local_image_ctx, "snap"); - - std::string clone_image_name = get_temp_image_name(); - ASSERT_EQ(0, clone_image(m_remote_image_ctx, "snap", clone_image_name)); - - librbd::ImageCtx *remote_clone_image_ctx; - ASSERT_EQ(0, open_image(m_remote_io_ctx, clone_image_name, - &remote_clone_image_ctx)); - - librbd::MockTestImageCtx mock_remote_parent_image_ctx(*m_remote_image_ctx); - librbd::MockTestImageCtx mock_remote_clone_image_ctx(*remote_clone_image_ctx); - MockCloneRequest mock_clone_request; - MockOpenImageRequest mock_open_image_request; - MockCloseImageRequest mock_close_image_request; - journal::MockJournaler mock_remote_journaler; - - InSequence seq; - expect_ioctx_create(m_remote_io_ctx); - expect_ioctx_create(m_local_io_ctx); - expect_mirror_uuid_get(m_local_io_ctx, "local parent uuid", 0); - expect_journaler_get_client( - mock_remote_journaler, "local parent uuid", - librbd::journal::MIRROR_PEER_STATE_REPLAYING, 0); - expect_get_parent_global_image_id(m_remote_io_ctx, "global uuid", 0); - expect_mirror_image_get_image_id(m_local_io_ctx, "local parent id", 0); - - expect_open_image(mock_open_image_request, m_remote_io_ctx, - m_remote_image_ctx->id, mock_remote_parent_image_ctx, 0); - expect_test_op_features(mock_remote_clone_image_ctx, false); - expect_clone_image(mock_clone_request, 0); - expect_close_image(mock_close_image_request, mock_remote_parent_image_ctx, 0); - - C_SaferCond ctx; - MockThreads mock_threads(m_threads); - MockCreateImageRequest *request = create_request(&mock_threads, "global uuid", - "remote uuid", "image name", - "101241a7c4c9", - mock_remote_clone_image_ctx, - &ctx); - request->send(); - ASSERT_EQ(0, ctx.wait()); -} - -TEST_F(TestMockImageReplayerCreateImageRequest, CloneParentMirrorUuidGetError) { - std::string clone_image_name = get_temp_image_name(); - ASSERT_EQ(0, clone_image(m_remote_image_ctx, "snap", clone_image_name)); - - librbd::ImageCtx *remote_clone_image_ctx; - ASSERT_EQ(0, open_image(m_remote_io_ctx, clone_image_name, - &remote_clone_image_ctx)); - - librbd::MockTestImageCtx mock_remote_clone_image_ctx(*remote_clone_image_ctx); - - InSequence seq; - expect_ioctx_create(m_remote_io_ctx); - expect_ioctx_create(m_local_io_ctx); - expect_mirror_uuid_get(m_local_io_ctx, "local parent uuid", -EPERM); - - C_SaferCond ctx; - MockThreads mock_threads(m_threads); - MockCreateImageRequest *request = create_request(&mock_threads, "global uuid", - "remote uuid", "image name", - "101241a7c4c9", - mock_remote_clone_image_ctx, - &ctx); - request->send(); - ASSERT_EQ(-EPERM, ctx.wait()); -} - -TEST_F(TestMockImageReplayerCreateImageRequest, CloneGetRemoteParentClientStateError) { - std::string clone_image_name = get_temp_image_name(); - ASSERT_EQ(0, clone_image(m_remote_image_ctx, "snap", clone_image_name)); - - librbd::ImageCtx *remote_clone_image_ctx; - ASSERT_EQ(0, open_image(m_remote_io_ctx, clone_image_name, - &remote_clone_image_ctx)); - - librbd::MockTestImageCtx mock_remote_clone_image_ctx(*remote_clone_image_ctx); - journal::MockJournaler mock_remote_journaler; - - InSequence seq; - expect_ioctx_create(m_remote_io_ctx); - expect_ioctx_create(m_local_io_ctx); - expect_mirror_uuid_get(m_local_io_ctx, "local parent uuid", 0); - expect_journaler_get_client( - mock_remote_journaler, "local parent uuid", - librbd::journal::MIRROR_PEER_STATE_REPLAYING, -EPERM); - - C_SaferCond ctx; - MockThreads mock_threads(m_threads); - MockCreateImageRequest *request = create_request(&mock_threads, "global uuid", - "remote uuid", "image name", - "101241a7c4c9", - mock_remote_clone_image_ctx, - &ctx); - request->send(); - ASSERT_EQ(-EPERM, ctx.wait()); -} - -TEST_F(TestMockImageReplayerCreateImageRequest, CloneGetRemoteParentClientStateSyncing) { +TEST_F(TestMockImageReplayerCreateImageRequest, CloneGetGlobalImageIdError) { std::string clone_image_name = get_temp_image_name(); ASSERT_EQ(0, clone_image(m_remote_image_ctx, "snap", clone_image_name)); @@ -538,15 +391,11 @@ TEST_F(TestMockImageReplayerCreateImageRequest, CloneGetRemoteParentClientStateS &remote_clone_image_ctx)); librbd::MockTestImageCtx mock_remote_clone_image_ctx(*remote_clone_image_ctx); - journal::MockJournaler mock_remote_journaler; InSequence seq; expect_ioctx_create(m_remote_io_ctx); expect_ioctx_create(m_local_io_ctx); - expect_mirror_uuid_get(m_local_io_ctx, "local parent uuid", 0); - expect_journaler_get_client( - mock_remote_journaler, "local parent uuid", - librbd::journal::MIRROR_PEER_STATE_SYNCING, 0); + expect_get_parent_global_image_id(m_remote_io_ctx, "global uuid", -ENOENT); C_SaferCond ctx; MockThreads mock_threads(m_threads); @@ -559,7 +408,7 @@ TEST_F(TestMockImageReplayerCreateImageRequest, CloneGetRemoteParentClientStateS ASSERT_EQ(-ENOENT, ctx.wait()); } -TEST_F(TestMockImageReplayerCreateImageRequest, CloneGetGlobalImageIdError) { +TEST_F(TestMockImageReplayerCreateImageRequest, CloneGetLocalParentImageIdError) { std::string clone_image_name = get_temp_image_name(); ASSERT_EQ(0, clone_image(m_remote_image_ctx, "snap", clone_image_name)); @@ -568,16 +417,12 @@ TEST_F(TestMockImageReplayerCreateImageRequest, CloneGetGlobalImageIdError) { &remote_clone_image_ctx)); librbd::MockTestImageCtx mock_remote_clone_image_ctx(*remote_clone_image_ctx); - journal::MockJournaler mock_remote_journaler; InSequence seq; expect_ioctx_create(m_remote_io_ctx); expect_ioctx_create(m_local_io_ctx); - expect_mirror_uuid_get(m_local_io_ctx, "local parent uuid", 0); - expect_journaler_get_client( - mock_remote_journaler, "local parent uuid", - librbd::journal::MIRROR_PEER_STATE_REPLAYING, 0); - expect_get_parent_global_image_id(m_remote_io_ctx, "global uuid", -ENOENT); + expect_get_parent_global_image_id(m_remote_io_ctx, "global uuid", 0); + expect_mirror_image_get_image_id(m_local_io_ctx, "local parent id", -ENOENT); C_SaferCond ctx; MockThreads mock_threads(m_threads); @@ -590,7 +435,7 @@ TEST_F(TestMockImageReplayerCreateImageRequest, CloneGetGlobalImageIdError) { ASSERT_EQ(-ENOENT, ctx.wait()); } -TEST_F(TestMockImageReplayerCreateImageRequest, CloneGetLocalParentImageIdError) { +TEST_F(TestMockImageReplayerCreateImageRequest, CloneOpenRemoteParentError) { std::string clone_image_name = get_temp_image_name(); ASSERT_EQ(0, clone_image(m_remote_image_ctx, "snap", clone_image_name)); @@ -598,18 +443,19 @@ TEST_F(TestMockImageReplayerCreateImageRequest, CloneGetLocalParentImageIdError) ASSERT_EQ(0, open_image(m_remote_io_ctx, clone_image_name, &remote_clone_image_ctx)); + librbd::MockTestImageCtx mock_remote_parent_image_ctx(*m_remote_image_ctx); librbd::MockTestImageCtx mock_remote_clone_image_ctx(*remote_clone_image_ctx); - journal::MockJournaler mock_remote_journaler; + MockOpenImageRequest mock_open_image_request; InSequence seq; expect_ioctx_create(m_remote_io_ctx); expect_ioctx_create(m_local_io_ctx); - expect_mirror_uuid_get(m_local_io_ctx, "local parent uuid", 0); - expect_journaler_get_client( - mock_remote_journaler, "local parent uuid", - librbd::journal::MIRROR_PEER_STATE_REPLAYING, 0); expect_get_parent_global_image_id(m_remote_io_ctx, "global uuid", 0); - expect_mirror_image_get_image_id(m_local_io_ctx, "local parent id", -ENOENT); + expect_mirror_image_get_image_id(m_local_io_ctx, "local parent id", 0); + + expect_open_image(mock_open_image_request, m_remote_io_ctx, + m_remote_image_ctx->id, mock_remote_parent_image_ctx, + -ENOENT); C_SaferCond ctx; MockThreads mock_threads(m_threads); @@ -622,7 +468,14 @@ TEST_F(TestMockImageReplayerCreateImageRequest, CloneGetLocalParentImageIdError) ASSERT_EQ(-ENOENT, ctx.wait()); } -TEST_F(TestMockImageReplayerCreateImageRequest, CloneOpenRemoteParentError) { +TEST_F(TestMockImageReplayerCreateImageRequest, CloneParentImageSyncing) { + librbd::RBD rbd; + librbd::ImageCtx *local_image_ctx; + ASSERT_EQ(0, create_image(rbd, m_local_io_ctx, m_image_name, m_image_size)); + ASSERT_EQ(0, open_image(m_local_io_ctx, m_image_name, &local_image_ctx)); + snap_create(local_image_ctx, "snap"); + snap_create(m_remote_image_ctx, ".rbd-mirror.local parent uuid.1234"); + std::string clone_image_name = get_temp_image_name(); ASSERT_EQ(0, clone_image(m_remote_image_ctx, "snap", clone_image_name)); @@ -630,24 +483,23 @@ TEST_F(TestMockImageReplayerCreateImageRequest, CloneOpenRemoteParentError) { ASSERT_EQ(0, open_image(m_remote_io_ctx, clone_image_name, &remote_clone_image_ctx)); + m_pool_meta_cache.set_local_pool_meta( + m_local_io_ctx.get_id(), {"local parent uuid"}); + librbd::MockTestImageCtx mock_remote_parent_image_ctx(*m_remote_image_ctx); librbd::MockTestImageCtx mock_remote_clone_image_ctx(*remote_clone_image_ctx); MockOpenImageRequest mock_open_image_request; - journal::MockJournaler mock_remote_journaler; + MockCloseImageRequest mock_close_image_request; InSequence seq; expect_ioctx_create(m_remote_io_ctx); expect_ioctx_create(m_local_io_ctx); - expect_mirror_uuid_get(m_local_io_ctx, "local parent uuid", 0); - expect_journaler_get_client( - mock_remote_journaler, "local parent uuid", - librbd::journal::MIRROR_PEER_STATE_REPLAYING, 0); expect_get_parent_global_image_id(m_remote_io_ctx, "global uuid", 0); expect_mirror_image_get_image_id(m_local_io_ctx, "local parent id", 0); expect_open_image(mock_open_image_request, m_remote_io_ctx, - m_remote_image_ctx->id, mock_remote_parent_image_ctx, - -ENOENT); + m_remote_image_ctx->id, mock_remote_parent_image_ctx, 0); + expect_close_image(mock_close_image_request, mock_remote_parent_image_ctx, 0); C_SaferCond ctx; MockThreads mock_threads(m_threads); @@ -674,20 +526,18 @@ TEST_F(TestMockImageReplayerCreateImageRequest, CloneError) { ASSERT_EQ(0, open_image(m_remote_io_ctx, clone_image_name, &remote_clone_image_ctx)); + m_pool_meta_cache.set_local_pool_meta( + m_local_io_ctx.get_id(), {"local parent uuid"}); + librbd::MockTestImageCtx mock_remote_parent_image_ctx(*m_remote_image_ctx); librbd::MockTestImageCtx mock_remote_clone_image_ctx(*remote_clone_image_ctx); MockCloneRequest mock_clone_request; MockOpenImageRequest mock_open_image_request; MockCloseImageRequest mock_close_image_request; - journal::MockJournaler mock_remote_journaler; InSequence seq; expect_ioctx_create(m_remote_io_ctx); expect_ioctx_create(m_local_io_ctx); - expect_mirror_uuid_get(m_local_io_ctx, "local parent uuid", 0); - expect_journaler_get_client( - mock_remote_journaler, "local parent uuid", - librbd::journal::MIRROR_PEER_STATE_REPLAYING, 0); expect_get_parent_global_image_id(m_remote_io_ctx, "global uuid", 0); expect_mirror_image_get_image_id(m_local_io_ctx, "local parent id", 0); @@ -722,20 +572,18 @@ TEST_F(TestMockImageReplayerCreateImageRequest, CloneRemoteParentCloseError) { ASSERT_EQ(0, open_image(m_remote_io_ctx, clone_image_name, &remote_clone_image_ctx)); + m_pool_meta_cache.set_local_pool_meta( + m_local_io_ctx.get_id(), {"local parent uuid"}); + librbd::MockTestImageCtx mock_remote_parent_image_ctx(*m_remote_image_ctx); librbd::MockTestImageCtx mock_remote_clone_image_ctx(*remote_clone_image_ctx); MockCloneRequest mock_clone_request; MockOpenImageRequest mock_open_image_request; MockCloseImageRequest mock_close_image_request; - journal::MockJournaler mock_remote_journaler; InSequence seq; expect_ioctx_create(m_remote_io_ctx); expect_ioctx_create(m_local_io_ctx); - expect_mirror_uuid_get(m_local_io_ctx, "local parent uuid", 0); - expect_journaler_get_client( - mock_remote_journaler, "local parent uuid", - librbd::journal::MIRROR_PEER_STATE_REPLAYING, 0); expect_get_parent_global_image_id(m_remote_io_ctx, "global uuid", 0); expect_mirror_image_get_image_id(m_local_io_ctx, "local parent id", 0); diff --git a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc index 7f338cc3e087..b41f71f64469 100644 --- a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc +++ b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc @@ -338,7 +338,8 @@ void BootstrapRequest::create_local_image() { BootstrapRequest, &BootstrapRequest::handle_create_local_image>(this); auto request = (*m_state_builder)->create_local_image_request( - m_threads, m_local_io_ctx, m_global_image_id, m_progress_ctx, ctx); + m_threads, m_local_io_ctx, m_global_image_id, m_pool_meta_cache, + m_progress_ctx, ctx); request->send(); } diff --git a/src/tools/rbd_mirror/image_replayer/CreateImageRequest.cc b/src/tools/rbd_mirror/image_replayer/CreateImageRequest.cc index 854f306f5fee..f1f9cc42bb4a 100644 --- a/src/tools/rbd_mirror/image_replayer/CreateImageRequest.cc +++ b/src/tools/rbd_mirror/image_replayer/CreateImageRequest.cc @@ -8,17 +8,18 @@ #include "common/errno.h" #include "common/WorkQueue.h" #include "cls/rbd/cls_rbd_client.h" -#include "journal/Journaler.h" -#include "journal/Settings.h" #include "librbd/ImageCtx.h" #include "librbd/ImageState.h" #include "librbd/internal.h" #include "librbd/Utils.h" #include "librbd/image/CreateRequest.h" #include "librbd/image/CloneRequest.h" -#include "librbd/journal/Types.h" +#include "tools/rbd_mirror/PoolMetaCache.h" +#include "tools/rbd_mirror/Types.h" #include "tools/rbd_mirror/Threads.h" #include "tools/rbd_mirror/image_replayer/Utils.h" +#include "tools/rbd_mirror/image_sync/Utils.h" +#include #define dout_context g_ceph_context #define dout_subsys ceph_subsys_rbd_mirror @@ -43,6 +44,7 @@ CreateImageRequest::CreateImageRequest( const std::string &local_image_name, const std::string &local_image_id, I *remote_image_ctx, + PoolMetaCache* pool_meta_cache, cls::rbd::MirrorImageMode mirror_image_mode, Context *on_finish) : m_threads(threads), m_local_io_ctx(local_io_ctx), @@ -50,6 +52,7 @@ CreateImageRequest::CreateImageRequest( m_remote_mirror_uuid(remote_mirror_uuid), m_local_image_name(local_image_name), m_local_image_id(local_image_id), m_remote_image_ctx(remote_image_ctx), + m_pool_meta_cache(pool_meta_cache), m_mirror_image_mode(mirror_image_mode), m_on_finish(on_finish) { } @@ -64,7 +67,7 @@ void CreateImageRequest::send() { if (m_remote_parent_spec.pool_id == -1) { create_image(); } else { - get_local_parent_mirror_uuid(); + get_parent_global_image_id(); } } @@ -108,102 +111,6 @@ void CreateImageRequest::handle_create_image(int r) { finish(0); } -template -void CreateImageRequest::get_local_parent_mirror_uuid() { - dout(10) << dendl; - - librados::ObjectReadOperation op; - librbd::cls_client::mirror_uuid_get_start(&op); - - librados::AioCompletion *aio_comp = create_rados_callback< - CreateImageRequest, - &CreateImageRequest::handle_get_local_parent_mirror_uuid>(this); - m_out_bl.clear(); - int r = m_local_parent_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op, - &m_out_bl); - ceph_assert(r == 0); - aio_comp->release(); -} - -template -void CreateImageRequest::handle_get_local_parent_mirror_uuid(int r) { - if (r >= 0) { - auto it = m_out_bl.cbegin(); - r = librbd::cls_client::mirror_uuid_get_finish( - &it, &m_local_parent_mirror_uuid); - if (r >= 0 && m_local_parent_mirror_uuid.empty()) { - r = -ENOENT; - } - } - - dout(10) << "r=" << r << dendl; - if (r < 0) { - if (r == -ENOENT) { - dout(5) << "local parent mirror uuid missing" << dendl; - } else { - derr << "failed to retrieve local parent mirror uuid: " << cpp_strerror(r) - << dendl; - } - finish(r); - return; - } - - dout(15) << "local_parent_mirror_uuid=" << m_local_parent_mirror_uuid - << dendl; - get_remote_parent_client_state(); -} - -template -void CreateImageRequest::get_remote_parent_client_state() { - dout(10) << dendl; - - m_remote_journaler = new Journaler(m_threads->work_queue, m_threads->timer, - &m_threads->timer_lock, - m_remote_parent_io_ctx, - m_remote_parent_spec.image_id, - m_local_parent_mirror_uuid, {}, nullptr); - - Context *ctx = create_async_context_callback( - m_threads->work_queue, create_context_callback< - CreateImageRequest, - &CreateImageRequest::handle_get_remote_parent_client_state>(this)); - m_remote_journaler->get_client(m_local_parent_mirror_uuid, &m_client, ctx); -} - -template -void CreateImageRequest::handle_get_remote_parent_client_state(int r) { - dout(10) << "r=" << r << dendl; - - delete m_remote_journaler; - m_remote_journaler = nullptr; - - librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; - if (r == -ENOENT) { - dout(15) << "client not registered to parent image" << dendl; - finish(r); - return; - } else if (r < 0) { - derr << "failed to retrieve parent client: " << cpp_strerror(r) << dendl; - finish(r); - return; - } else if (!util::decode_client_meta(m_client, &mirror_peer_client_meta)) { - // require operator intervention since the data is corrupt - derr << "failed to decode parent client: " << cpp_strerror(r) << dendl; - finish(-EBADMSG); - return; - } else if (mirror_peer_client_meta.state != - librbd::journal::MIRROR_PEER_STATE_REPLAYING) { - // avoid possible race w/ incomplete parent image since the parent snapshot - // might be deleted if the sync restarts - dout(15) << "parent image still syncing" << dendl; - finish(-ENOENT); - return; - } - - get_parent_global_image_id(); -} - - template void CreateImageRequest::get_parent_global_image_id() { dout(10) << dendl; @@ -324,18 +231,51 @@ template void CreateImageRequest::clone_image() { dout(10) << dendl; + LocalPoolMeta local_parent_pool_meta; + int r = m_pool_meta_cache->get_local_pool_meta( + m_local_parent_io_ctx.get_id(), &local_parent_pool_meta); + if (r < 0) { + derr << "failed to retrieve local parent mirror uuid for pool " + << m_local_parent_io_ctx.get_id() << dendl; + m_ret_val = r; + close_remote_parent_image(); + return; + } + + // ensure no image sync snapshots for the local cluster exist in the + // remote image + bool found_parent_snap = false; + bool found_image_sync_snap = false; std::string snap_name; cls::rbd::SnapshotNamespace snap_namespace; { + auto snap_prefix = image_sync::util::get_snapshot_name_prefix( + local_parent_pool_meta.mirror_uuid); + std::shared_lock remote_image_locker(m_remote_parent_image_ctx->image_lock); - auto it = m_remote_parent_image_ctx->snap_info.find( - m_remote_parent_spec.snap_id); - if (it != m_remote_parent_image_ctx->snap_info.end()) { - snap_name = it->second.name; - snap_namespace = it->second.snap_namespace; + for (auto snap_info : m_remote_parent_image_ctx->snap_info) { + if (snap_info.first == m_remote_parent_spec.snap_id) { + found_parent_snap = true; + snap_name = snap_info.second.name; + snap_namespace = snap_info.second.snap_namespace; + } else if (boost::starts_with(snap_info.second.name, snap_prefix)) { + found_image_sync_snap = true; + } } } + if (!found_parent_snap) { + dout(15) << "remote parent image snapshot not found" << dendl; + m_ret_val = -ENOENT; + close_remote_parent_image(); + return; + } else if (found_image_sync_snap) { + dout(15) << "parent image not synced to local cluster" << dendl; + m_ret_val = -ENOENT; + close_remote_parent_image(); + return; + } + librbd::ImageOptions opts; populate_image_options(&opts); @@ -359,8 +299,7 @@ void CreateImageRequest::handle_clone_image(int r) { dout(10) << "r=" << r << dendl; if (r == -EBADF) { dout(5) << "image id " << m_local_image_id << " already in-use" << dendl; - finish(r); - return; + m_ret_val = r; } else if (r < 0) { derr << "failed to clone image " << m_parent_pool_name << "/" << m_remote_parent_spec.image_id << " to " @@ -436,9 +375,10 @@ int CreateImageRequest::validate_parent() { } // map remote parent pool to local parent pool - librados::Rados remote_rados(m_remote_image_ctx->md_ctx); - int r = remote_rados.ioctx_create2(m_remote_parent_spec.pool_id, - m_remote_parent_io_ctx); + int r = librbd::util::create_ioctx( + m_remote_image_ctx->md_ctx, "remote parent pool", + m_remote_parent_spec.pool_id, m_remote_parent_spec.pool_namespace, + &m_remote_parent_io_ctx); if (r < 0) { derr << "failed to open remote parent pool " << m_remote_parent_spec.pool_id << ": " << cpp_strerror(r) << dendl; @@ -455,6 +395,7 @@ int CreateImageRequest::validate_parent() { << cpp_strerror(r) << dendl; return r; } + m_local_parent_io_ctx.set_namespace(m_remote_parent_io_ctx.get_namespace()); return 0; } diff --git a/src/tools/rbd_mirror/image_replayer/CreateImageRequest.h b/src/tools/rbd_mirror/image_replayer/CreateImageRequest.h index c8564e6179b6..eda12cdb3f98 100644 --- a/src/tools/rbd_mirror/image_replayer/CreateImageRequest.h +++ b/src/tools/rbd_mirror/image_replayer/CreateImageRequest.h @@ -7,21 +7,19 @@ #include "include/int_types.h" #include "include/types.h" #include "include/rados/librados.hpp" -#include "cls/journal/cls_journal_types.h" #include "cls/rbd/cls_rbd_types.h" #include "librbd/Types.h" -#include "librbd/journal/TypeTraits.h" #include class Context; class ContextWQ; -namespace journal { class Journaler; } namespace librbd { class ImageCtx; } namespace librbd { class ImageOptions; } namespace rbd { namespace mirror { +class PoolMetaCache; template struct Threads; namespace image_replayer { @@ -29,29 +27,34 @@ namespace image_replayer { template class CreateImageRequest { public: - static CreateImageRequest *create(Threads *threads, - librados::IoCtx &local_io_ctx, - const std::string &global_image_id, - const std::string &remote_mirror_uuid, - const std::string &local_image_name, - const std::string &local_image_id, - ImageCtxT *remote_image_ctx, - cls::rbd::MirrorImageMode mirror_image_mode, - Context *on_finish) { + static CreateImageRequest *create( + Threads *threads, + librados::IoCtx &local_io_ctx, + const std::string &global_image_id, + const std::string &remote_mirror_uuid, + const std::string &local_image_name, + const std::string &local_image_id, + ImageCtxT *remote_image_ctx, + PoolMetaCache* pool_meta_cache, + cls::rbd::MirrorImageMode mirror_image_mode, + Context *on_finish) { return new CreateImageRequest(threads, local_io_ctx, global_image_id, remote_mirror_uuid, local_image_name, local_image_id, remote_image_ctx, - mirror_image_mode, on_finish); + pool_meta_cache, mirror_image_mode, + on_finish); } - CreateImageRequest(Threads *threads, librados::IoCtx &local_io_ctx, - const std::string &global_image_id, - const std::string &remote_mirror_uuid, - const std::string &local_image_name, - const std::string &local_image_id, - ImageCtxT *remote_image_ctx, - cls::rbd::MirrorImageMode mirror_image_mode, - Context *on_finish); + CreateImageRequest( + Threads *threads, librados::IoCtx &local_io_ctx, + const std::string &global_image_id, + const std::string &remote_mirror_uuid, + const std::string &local_image_name, + const std::string &local_image_id, + ImageCtxT *remote_image_ctx, + PoolMetaCache* pool_meta_cache, + cls::rbd::MirrorImageMode mirror_image_mode, + Context *on_finish); void send(); @@ -65,13 +68,7 @@ private: * |\------------> CREATE_IMAGE ---------------------\ * (error) * | | * * | (clone) | * - * \-------------> GET_LOCAL_PARENT_MIRROR_UUID * * | * * * * - * | | * * - * v | * - * GET_REMOTE_PARENT_CLIENT_STATE * | * * * * - * | | * * - * v | * - * GET_PARENT_GLOBAL_IMAGE_ID * * * | * * * * + * \-------------> GET_PARENT_GLOBAL_IMAGE_ID * * * | * * * * * | | * * * v | * * GET_LOCAL_PARENT_IMAGE_ID * * * * | * * * * @@ -89,9 +86,6 @@ private: * @endverbatim */ - typedef librbd::journal::TypeTraits TypeTraits; - typedef typename TypeTraits::Journaler Journaler; - Threads *m_threads; librados::IoCtx &m_local_io_ctx; std::string m_global_image_id; @@ -99,12 +93,11 @@ private: std::string m_local_image_name; std::string m_local_image_id; ImageCtxT *m_remote_image_ctx; + PoolMetaCache* m_pool_meta_cache; cls::rbd::MirrorImageMode m_mirror_image_mode; Context *m_on_finish; librados::IoCtx m_remote_parent_io_ctx; - std::string m_local_parent_mirror_uuid; - Journaler *m_remote_journaler = nullptr; ImageCtxT *m_remote_parent_image_ctx = nullptr; cls::rbd::ParentImageSpec m_remote_parent_spec; @@ -114,18 +107,11 @@ private: bufferlist m_out_bl; std::string m_parent_global_image_id; std::string m_parent_pool_name; - cls::journal::Client m_client; int m_ret_val = 0; void create_image(); void handle_create_image(int r); - void get_local_parent_mirror_uuid(); - void handle_get_local_parent_mirror_uuid(int r); - - void get_remote_parent_client_state(); - void handle_get_remote_parent_client_state(int r); - void get_parent_global_image_id(); void handle_get_parent_global_image_id(int r); diff --git a/src/tools/rbd_mirror/image_replayer/StateBuilder.h b/src/tools/rbd_mirror/image_replayer/StateBuilder.h index a85cb3b779a5..9386b01c99e5 100644 --- a/src/tools/rbd_mirror/image_replayer/StateBuilder.h +++ b/src/tools/rbd_mirror/image_replayer/StateBuilder.h @@ -15,6 +15,7 @@ namespace rbd { namespace mirror { struct BaseRequest; +struct PoolMetaCache; struct ProgressContext; template class Threads; @@ -55,6 +56,7 @@ public: Threads* threads, librados::IoCtx& local_io_ctx, const std::string& global_image_id, + PoolMetaCache* pool_meta_cache, ProgressContext* progress_ctx, Context* on_finish) = 0; diff --git a/src/tools/rbd_mirror/image_replayer/journal/CreateLocalImageRequest.cc b/src/tools/rbd_mirror/image_replayer/journal/CreateLocalImageRequest.cc index fa8765faf1c8..087cf4f5fd47 100644 --- a/src/tools/rbd_mirror/image_replayer/journal/CreateLocalImageRequest.cc +++ b/src/tools/rbd_mirror/image_replayer/journal/CreateLocalImageRequest.cc @@ -10,8 +10,11 @@ #include "librbd/ImageCtx.h" #include "librbd/Utils.h" #include "librbd/journal/Types.h" +#include "tools/rbd_mirror/PoolMetaCache.h" #include "tools/rbd_mirror/ProgressContext.h" +#include "tools/rbd_mirror/Threads.h" #include "tools/rbd_mirror/image_replayer/CreateImageRequest.h" +#include "tools/rbd_mirror/image_replayer/Utils.h" #include "tools/rbd_mirror/image_replayer/journal/StateBuilder.h" #define dout_context g_ceph_context @@ -26,11 +29,11 @@ namespace mirror { namespace image_replayer { namespace journal { +using librbd::util::create_async_context_callback; using librbd::util::create_context_callback; template void CreateLocalImageRequest::send() { - m_state_builder->local_image_id = ""; unregister_client(); } @@ -55,6 +58,7 @@ void CreateLocalImageRequest::handle_unregister_client(int r) { return; } + m_state_builder->local_image_id = ""; m_state_builder->remote_client_meta = {}; register_client(); } @@ -116,7 +120,7 @@ void CreateLocalImageRequest::create_local_image() { m_threads, m_local_io_ctx, m_global_image_id, m_state_builder->remote_mirror_uuid, image_name, m_state_builder->local_image_id, m_remote_image_ctx, - cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, ctx); + m_pool_meta_cache, cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, ctx); request->send(); } template @@ -126,8 +130,7 @@ void CreateLocalImageRequest::handle_create_local_image(int r) { if (r == -EBADF) { dout(5) << "image id " << m_state_builder->local_image_id << " " << "already in-use" << dendl; - m_state_builder->local_image_id = ""; - update_client_image(); + unregister_client(); return; } else if (r < 0) { if (r == -ENOENT) { @@ -142,45 +145,6 @@ void CreateLocalImageRequest::handle_create_local_image(int r) { finish(0); } -template -void CreateLocalImageRequest::update_client_image() { - ceph_assert(m_state_builder->local_image_id.empty()); - m_state_builder->local_image_id = - librbd::util::generate_image_id(m_local_io_ctx); - - dout(10) << "local_image_id=" << m_state_builder->local_image_id << dendl; - update_progress("UPDATE_CLIENT_IMAGE"); - - librbd::journal::MirrorPeerClientMeta client_meta{ - m_state_builder->local_image_id}; - client_meta.state = librbd::journal::MIRROR_PEER_STATE_SYNCING; - - librbd::journal::ClientData client_data(client_meta); - bufferlist data_bl; - encode(client_data, data_bl); - - auto ctx = create_context_callback< - CreateLocalImageRequest, - &CreateLocalImageRequest::handle_update_client_image>(this); - m_state_builder->remote_journaler->update_client(data_bl, ctx); -} - -template -void CreateLocalImageRequest::handle_update_client_image(int r) { - dout(10) << "r=" << r << dendl; - - if (r < 0) { - derr << "failed to update client: " << cpp_strerror(r) << dendl; - finish(r); - return; - } - - m_state_builder->remote_client_meta = {m_state_builder->local_image_id}; - m_state_builder->remote_client_meta.state = - librbd::journal::MIRROR_PEER_STATE_SYNCING; - create_local_image(); -} - template void CreateLocalImageRequest::update_progress( const std::string& description) { diff --git a/src/tools/rbd_mirror/image_replayer/journal/CreateLocalImageRequest.h b/src/tools/rbd_mirror/image_replayer/journal/CreateLocalImageRequest.h index 1ea19aca2698..fc776ecc3011 100644 --- a/src/tools/rbd_mirror/image_replayer/journal/CreateLocalImageRequest.h +++ b/src/tools/rbd_mirror/image_replayer/journal/CreateLocalImageRequest.h @@ -9,11 +9,13 @@ #include struct Context; +namespace journal { class Journaler; } namespace librbd { class ImageCtx; } namespace rbd { namespace mirror { +class PoolMetaCache; class ProgressContext; template struct Threads; @@ -32,12 +34,13 @@ public: librados::IoCtx& local_io_ctx, ImageCtxT* remote_image_ctx, const std::string& global_image_id, + PoolMetaCache* pool_meta_cache, ProgressContext* progress_ctx, StateBuilder* state_builder, Context* on_finish) { return new CreateLocalImageRequest(threads, local_io_ctx, remote_image_ctx, - global_image_id, progress_ctx, - state_builder, on_finish); + global_image_id, pool_meta_cache, + progress_ctx, state_builder, on_finish); } CreateLocalImageRequest( @@ -45,6 +48,7 @@ public: librados::IoCtx& local_io_ctx, ImageCtxT* remote_image_ctx, const std::string& global_image_id, + PoolMetaCache* pool_meta_cache, ProgressContext* progress_ctx, StateBuilder* state_builder, Context* on_finish) @@ -53,6 +57,7 @@ public: m_local_io_ctx(local_io_ctx), m_remote_image_ctx(remote_image_ctx), m_global_image_id(global_image_id), + m_pool_meta_cache(pool_meta_cache), m_progress_ctx(progress_ctx), m_state_builder(state_builder) { } @@ -66,14 +71,12 @@ private: * * | * v - * UNREGISTER_CLIENT - * | - * v - * REGISTER_CLIENT - * | - * | . . . . . . . . . UPDATE_CLIENT_IMAGE - * | . ^ - * v v (id exists) * + * UNREGISTER_CLIENT < * * * * * * * * + * | * + * v * + * REGISTER_CLIENT * + * | * + * v (id exists) * * CREATE_LOCAL_IMAGE * * * * * * * * * * | * v @@ -86,6 +89,7 @@ private: librados::IoCtx& m_local_io_ctx; ImageCtxT* m_remote_image_ctx; std::string m_global_image_id; + PoolMetaCache* m_pool_meta_cache; ProgressContext* m_progress_ctx; StateBuilder* m_state_builder; @@ -98,9 +102,6 @@ private: void create_local_image(); void handle_create_local_image(int r); - void update_client_image(); - void handle_update_client_image(int r); - void update_progress(const std::string& description); }; diff --git a/src/tools/rbd_mirror/image_replayer/journal/StateBuilder.cc b/src/tools/rbd_mirror/image_replayer/journal/StateBuilder.cc index 9056785475b5..0bc641f7743e 100644 --- a/src/tools/rbd_mirror/image_replayer/journal/StateBuilder.cc +++ b/src/tools/rbd_mirror/image_replayer/journal/StateBuilder.cc @@ -75,11 +75,12 @@ BaseRequest* StateBuilder::create_local_image_request( Threads* threads, librados::IoCtx& local_io_ctx, const std::string& global_image_id, + PoolMetaCache* pool_meta_cache, ProgressContext* progress_ctx, Context* on_finish) { return CreateLocalImageRequest::create( threads, local_io_ctx, this->remote_image_ctx, this->global_image_id, - progress_ctx, this, on_finish); + pool_meta_cache, progress_ctx, this, on_finish); } template diff --git a/src/tools/rbd_mirror/image_replayer/journal/StateBuilder.h b/src/tools/rbd_mirror/image_replayer/journal/StateBuilder.h index ee0743d3b909..8fed18bd0aba 100644 --- a/src/tools/rbd_mirror/image_replayer/journal/StateBuilder.h +++ b/src/tools/rbd_mirror/image_replayer/journal/StateBuilder.h @@ -46,6 +46,7 @@ public: Threads* threads, librados::IoCtx& local_io_ctx, const std::string& global_image_id, + PoolMetaCache* pool_meta_cache, ProgressContext* progress_ctx, Context* on_finish) override; diff --git a/src/tools/rbd_mirror/image_replayer/snapshot/StateBuilder.cc b/src/tools/rbd_mirror/image_replayer/snapshot/StateBuilder.cc index 604b62f9c74d..648e38d2f6e7 100644 --- a/src/tools/rbd_mirror/image_replayer/snapshot/StateBuilder.cc +++ b/src/tools/rbd_mirror/image_replayer/snapshot/StateBuilder.cc @@ -67,6 +67,7 @@ BaseRequest* StateBuilder::create_local_image_request( Threads* threads, librados::IoCtx& local_io_ctx, const std::string& global_image_id, + PoolMetaCache* pool_meta_cache, ProgressContext* progress_ctx, Context* on_finish) { // TODO diff --git a/src/tools/rbd_mirror/image_replayer/snapshot/StateBuilder.h b/src/tools/rbd_mirror/image_replayer/snapshot/StateBuilder.h index 7444ceb344d7..d0604e4dfd6d 100644 --- a/src/tools/rbd_mirror/image_replayer/snapshot/StateBuilder.h +++ b/src/tools/rbd_mirror/image_replayer/snapshot/StateBuilder.h @@ -41,6 +41,7 @@ public: Threads* threads, librados::IoCtx& local_io_ctx, const std::string& global_image_id, + PoolMetaCache* pool_meta_cache, ProgressContext* progress_ctx, Context* on_finish) override;