From f1eb1c1992b5b09d4aca390b80e81d1f9e59e0f1 Mon Sep 17 00:00:00 2001 From: Jason Dillaman Date: Wed, 12 Sep 2018 14:04:49 -0400 Subject: [PATCH] rbd-mirror: prevent creation of clones when parents are syncing This will prevent a possible race condition where a thrashing rbd-mirror daemon in mid-sync with a parent image would result in the deletion of all snapshots when it restarts the sync. Fixes: http://tracker.ceph.com/issues/24140 Signed-off-by: Jason Dillaman --- .../test_mock_BootstrapRequest.cc | 87 ++++--- .../test_mock_CreateImageRequest.cc | 238 ++++++++++++++++-- .../rbd_mirror/test_mock_ImageReplayer.cc | 4 +- src/tools/rbd_mirror/CMakeLists.txt | 1 + src/tools/rbd_mirror/ImageReplayer.cc | 5 +- .../image_replayer/BootstrapRequest.cc | 24 +- .../image_replayer/BootstrapRequest.h | 24 +- .../image_replayer/CreateImageRequest.cc | 176 ++++++++++--- .../image_replayer/CreateImageRequest.h | 36 ++- .../PrepareRemoteImageRequest.cc | 28 +-- .../PrepareRemoteImageRequest.h | 2 - src/tools/rbd_mirror/image_replayer/Utils.cc | 50 ++++ src/tools/rbd_mirror/image_replayer/Utils.h | 23 ++ 13 files changed, 545 insertions(+), 153 deletions(-) create mode 100644 src/tools/rbd_mirror/image_replayer/Utils.cc create mode 100644 src/tools/rbd_mirror/image_replayer/Utils.h diff --git a/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc b/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc index 01e9f77da2bc2..82112d24a434d 100644 --- a/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc +++ b/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc @@ -55,6 +55,18 @@ namespace mirror { class ProgressContext; +template <> +struct Threads { + Mutex &timer_lock; + SafeTimer *timer; + ContextWQ *work_queue; + + Threads(Threads *threads) + : timer_lock(threads->timer_lock), timer(threads->timer), + work_queue(threads->work_queue) { + } +}; + template<> struct ImageSync { static ImageSync* s_instance; @@ -128,8 +140,8 @@ struct CreateImageRequest { static CreateImageRequest* s_instance; Context *on_finish = nullptr; - static CreateImageRequest* create(librados::IoCtx &local_io_ctx, - ContextWQ *work_queue, + static CreateImageRequest* create(Threads* threads, + librados::IoCtx &local_io_ctx, const std::string &global_image_id, const std::string &remote_mirror_uuid, const std::string &local_image_name, @@ -277,6 +289,7 @@ MATCHER_P(IsSameIoCtx, io_ctx, "") { class TestMockImageReplayerBootstrapRequest : public TestMockFixture { public: + typedef Threads MockThreads; typedef BootstrapRequest MockBootstrapRequest; typedef CloseImageRequest MockCloseImageRequest; typedef CreateImageRequest MockCreateImageRequest; @@ -441,7 +454,8 @@ public: return bl; } - MockBootstrapRequest *create_request(MockInstanceWatcher *mock_instance_watcher, + MockBootstrapRequest *create_request(MockThreads* mock_threads, + MockInstanceWatcher *mock_instance_watcher, ::journal::MockJournaler &mock_journaler, const std::string &local_image_id, const std::string &remote_image_id, @@ -451,16 +465,13 @@ public: cls::journal::ClientState *client_state, librbd::journal::MirrorPeerClientMeta *mirror_peer_client_meta, Context *on_finish) { - return new MockBootstrapRequest(m_local_io_ctx, + return new MockBootstrapRequest(mock_threads, m_local_io_ctx, m_remote_io_ctx, mock_instance_watcher, &m_local_test_image_ctx, local_image_id, remote_image_id, global_image_id, - m_threads->work_queue, - m_threads->timer, - &m_threads->timer_lock, local_mirror_uuid, remote_mirror_uuid, &mock_journaler, @@ -512,13 +523,15 @@ TEST_F(TestMockImageReplayerBootstrapRequest, NonPrimaryRemoteSyncingState) { expect_close_image(mock_close_image_request, mock_remote_image_ctx, 0); C_SaferCond ctx; + MockThreads mock_threads(m_threads); MockInstanceWatcher mock_instance_watcher; cls::journal::ClientState client_state = cls::journal::CLIENT_STATE_CONNECTED; mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_SYNCING; MockBootstrapRequest *request = create_request( - &mock_instance_watcher, mock_journaler, mock_local_image_ctx.id, - mock_remote_image_ctx.id, "global image id", "local mirror uuid", - "remote mirror uuid", &client_state, &mirror_peer_client_meta, &ctx); + &mock_threads, &mock_instance_watcher, mock_journaler, + mock_local_image_ctx.id, mock_remote_image_ctx.id, "global image id", + "local mirror uuid", "remote mirror uuid", &client_state, + &mirror_peer_client_meta, &ctx); request->send(); ASSERT_EQ(-EREMOTEIO, ctx.wait()); } @@ -567,15 +580,17 @@ TEST_F(TestMockImageReplayerBootstrapRequest, NonPrimaryRemoteNotTagOwner) { expect_close_image(mock_close_image_request, mock_remote_image_ctx, 0); C_SaferCond ctx; + MockThreads mock_threads(m_threads); MockInstanceWatcher mock_instance_watcher; cls::journal::ClientState client_state = cls::journal::CLIENT_STATE_CONNECTED; librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{ mock_local_image_ctx.id}; mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; MockBootstrapRequest *request = create_request( - &mock_instance_watcher, mock_journaler, mock_local_image_ctx.id, - mock_remote_image_ctx.id, "global image id", "local mirror uuid", - "remote mirror uuid", &client_state, &mirror_peer_client_meta, &ctx); + &mock_threads, &mock_instance_watcher, mock_journaler, + mock_local_image_ctx.id, mock_remote_image_ctx.id, "global image id", + "local mirror uuid", "remote mirror uuid", &client_state, + &mirror_peer_client_meta, &ctx); request->send(); ASSERT_EQ(-EREMOTEIO, ctx.wait()); } @@ -638,15 +653,17 @@ TEST_F(TestMockImageReplayerBootstrapRequest, RemoteDemotePromote) { expect_close_image(mock_close_image_request, mock_remote_image_ctx, 0); C_SaferCond ctx; + MockThreads mock_threads(m_threads); MockInstanceWatcher mock_instance_watcher; cls::journal::ClientState client_state = cls::journal::CLIENT_STATE_CONNECTED; librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{ mock_local_image_ctx.id}; mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; MockBootstrapRequest *request = create_request( - &mock_instance_watcher, mock_journaler, mock_local_image_ctx.id, - mock_remote_image_ctx.id, "global image id", "local mirror uuid", - "remote mirror uuid", &client_state, &mirror_peer_client_meta, &ctx); + &mock_threads, &mock_instance_watcher, mock_journaler, + mock_local_image_ctx.id, mock_remote_image_ctx.id, "global image id", + "local mirror uuid", "remote mirror uuid", &client_state, + &mirror_peer_client_meta, &ctx); request->send(); ASSERT_EQ(0, ctx.wait()); } @@ -719,15 +736,17 @@ TEST_F(TestMockImageReplayerBootstrapRequest, MultipleRemoteDemotePromotes) { expect_close_image(mock_close_image_request, mock_remote_image_ctx, 0); C_SaferCond ctx; + MockThreads mock_threads(m_threads); MockInstanceWatcher mock_instance_watcher; cls::journal::ClientState client_state = cls::journal::CLIENT_STATE_CONNECTED; librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{ mock_local_image_ctx.id}; mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; MockBootstrapRequest *request = create_request( - &mock_instance_watcher, mock_journaler, mock_local_image_ctx.id, - mock_remote_image_ctx.id, "global image id", "local mirror uuid", - "remote mirror uuid", &client_state, &mirror_peer_client_meta, &ctx); + &mock_threads, &mock_instance_watcher, mock_journaler, + mock_local_image_ctx.id, mock_remote_image_ctx.id, "global image id", + "local mirror uuid", "remote mirror uuid", &client_state, + &mirror_peer_client_meta, &ctx); request->send(); ASSERT_EQ(0, ctx.wait()); } @@ -788,15 +807,17 @@ TEST_F(TestMockImageReplayerBootstrapRequest, LocalDemoteRemotePromote) { expect_close_image(mock_close_image_request, mock_remote_image_ctx, 0); C_SaferCond ctx; + MockThreads mock_threads(m_threads); MockInstanceWatcher mock_instance_watcher; cls::journal::ClientState client_state = cls::journal::CLIENT_STATE_CONNECTED; librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{ mock_local_image_ctx.id}; mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; MockBootstrapRequest *request = create_request( - &mock_instance_watcher, mock_journaler, mock_local_image_ctx.id, - mock_remote_image_ctx.id, "global image id", "local mirror uuid", - "remote mirror uuid", &client_state, &mirror_peer_client_meta, &ctx); + &mock_threads, &mock_instance_watcher, mock_journaler, + mock_local_image_ctx.id, mock_remote_image_ctx.id, "global image id", + "local mirror uuid", "remote mirror uuid", &client_state, + &mirror_peer_client_meta, &ctx); request->send(); ASSERT_EQ(0, ctx.wait()); } @@ -856,15 +877,17 @@ TEST_F(TestMockImageReplayerBootstrapRequest, SplitBrainForcePromote) { expect_close_image(mock_close_image_request, mock_remote_image_ctx, 0); C_SaferCond ctx; + MockThreads mock_threads(m_threads); MockInstanceWatcher mock_instance_watcher; cls::journal::ClientState client_state = cls::journal::CLIENT_STATE_CONNECTED; librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{ mock_local_image_ctx.id}; mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; MockBootstrapRequest *request = create_request( - &mock_instance_watcher, mock_journaler, mock_local_image_ctx.id, - mock_remote_image_ctx.id, "global image id", "local mirror uuid", - "remote mirror uuid", &client_state, &mirror_peer_client_meta, &ctx); + &mock_threads, &mock_instance_watcher, mock_journaler, + mock_local_image_ctx.id, mock_remote_image_ctx.id, "global image id", + "local mirror uuid", "remote mirror uuid", &client_state, + &mirror_peer_client_meta, &ctx); request->send(); ASSERT_EQ(-EEXIST, ctx.wait()); ASSERT_EQ(NULL, m_local_test_image_ctx); @@ -913,15 +936,17 @@ TEST_F(TestMockImageReplayerBootstrapRequest, ResyncRequested) { expect_close_image(mock_close_image_request, mock_remote_image_ctx, 0); C_SaferCond ctx; + MockThreads mock_threads(m_threads); MockInstanceWatcher mock_instance_watcher; cls::journal::ClientState client_state = cls::journal::CLIENT_STATE_CONNECTED; librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{ mock_local_image_ctx.id}; mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; MockBootstrapRequest *request = create_request( - &mock_instance_watcher, mock_journaler, mock_local_image_ctx.id, - mock_remote_image_ctx.id, "global image id", "local mirror uuid", - "remote mirror uuid", &client_state, &mirror_peer_client_meta, &ctx); + &mock_threads, &mock_instance_watcher, mock_journaler, + mock_local_image_ctx.id, mock_remote_image_ctx.id, "global image id", + "local mirror uuid", "remote mirror uuid", &client_state, + &mirror_peer_client_meta, &ctx); m_do_resync = false; request->send(); ASSERT_EQ(0, ctx.wait()); @@ -988,12 +1013,13 @@ TEST_F(TestMockImageReplayerBootstrapRequest, PrimaryRemote) { expect_close_image(mock_close_image_request, mock_remote_image_ctx, 0); C_SaferCond ctx; + MockThreads mock_threads(m_threads); MockInstanceWatcher mock_instance_watcher; cls::journal::ClientState client_state = cls::journal::CLIENT_STATE_CONNECTED; mirror_peer_client_meta.image_id = ""; mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; MockBootstrapRequest *request = create_request( - &mock_instance_watcher, mock_journaler, "", + &mock_threads, &mock_instance_watcher, mock_journaler, "", mock_remote_image_ctx.id, "global image id", "local mirror uuid", "remote mirror uuid", &client_state, &mirror_peer_client_meta, &ctx); request->send(); @@ -1074,12 +1100,13 @@ TEST_F(TestMockImageReplayerBootstrapRequest, PrimaryRemoteLocalDeleted) { expect_close_image(mock_close_image_request, mock_remote_image_ctx, 0); C_SaferCond ctx; + MockThreads mock_threads(m_threads); MockInstanceWatcher mock_instance_watcher; cls::journal::ClientState client_state = cls::journal::CLIENT_STATE_CONNECTED; mirror_peer_client_meta.image_id = "missing image id"; mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; MockBootstrapRequest *request = create_request( - &mock_instance_watcher, mock_journaler, "missing image id", + &mock_threads, &mock_instance_watcher, mock_journaler, "missing image id", mock_remote_image_ctx.id, "global image id", "local mirror uuid", "remote mirror uuid", &client_state, &mirror_peer_client_meta, &ctx); request->send(); diff --git a/src/test/rbd_mirror/image_replayer/test_mock_CreateImageRequest.cc b/src/test/rbd_mirror/image_replayer/test_mock_CreateImageRequest.cc index 643ce26d5bdc2..ace6db45ce340 100644 --- a/src/test/rbd_mirror/image_replayer/test_mock_CreateImageRequest.cc +++ b/src/test/rbd_mirror/image_replayer/test_mock_CreateImageRequest.cc @@ -5,6 +5,7 @@ #include "include/rbd/librbd.hpp" #include "librbd/ImageState.h" #include "librbd/Operations.h" +#include "test/journal/mock/MockJournaler.h" #include "test/librados_test_stub/MockTestMemIoCtxImpl.h" #include "test/librados_test_stub/MockTestMemRadosClient.h" #include "test/librbd/mock/MockImageCtx.h" @@ -103,10 +104,32 @@ CloneRequest* CloneRequest::s_instance = nullptr; } // namespace image + +namespace journal { + +template <> +struct TypeTraits { + typedef ::journal::MockJournalerProxy Journaler; +}; + +} // namespace journal } // namespace librbd namespace rbd { namespace mirror { + +template <> +struct Threads { + Mutex &timer_lock; + SafeTimer *timer; + ContextWQ *work_queue; + + Threads(Threads *threads) + : timer_lock(threads->timer_lock), timer(threads->timer), + work_queue(threads->work_queue) { + } +}; + namespace image_replayer { template<> @@ -195,6 +218,7 @@ MATCHER_P(IsSameIoCtx, io_ctx, "") { class TestMockImageReplayerCreateImageRequest : public TestMockFixture { public: + typedef Threads MockThreads; typedef librbd::image::CreateRequest MockCreateRequest; typedef librbd::image::CloneRequest MockCloneRequest; typedef CreateImageRequest MockCreateImageRequest; @@ -302,13 +326,50 @@ public: })); } - MockCreateImageRequest *create_request(const std::string &global_image_id, + void expect_mirror_uuid_get(librados::IoCtx &io_ctx, + const std::string &mirror_uuid, int r) { + bufferlist bl; + encode(mirror_uuid, bl); + + EXPECT_CALL(get_mock_io_ctx(io_ctx), + exec(RBD_MIRRORING, _, StrEq("rbd"), StrEq("mirror_uuid_get"), _, _, _)) + .WillOnce(DoAll(WithArg<5>(Invoke([bl](bufferlist *out_bl) { + *out_bl = bl; + })), + Return(r))); + } + + void expect_journaler_get_client(journal::MockJournaler& mock_journaler, + const std::string& client_id, + librbd::journal::MirrorPeerState state, + int r) { + EXPECT_CALL(mock_journaler, construct()); + + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; + mirror_peer_client_meta.state = state; + + librbd::journal::ClientData client_data{mirror_peer_client_meta}; + + cls::journal::Client client; + encode(client_data, client.data); + + EXPECT_CALL(mock_journaler, get_client(StrEq(client_id), _, _)) + .WillOnce(DoAll(WithArg<1>(Invoke([client](cls::journal::Client *out_client) { + *out_client = client; + })), + WithArg<2>(Invoke([this, r](Context *on_finish) { + m_threads->work_queue->queue(on_finish, r); + })))); + } + + MockCreateImageRequest *create_request(MockThreads* mock_threads, + const std::string &global_image_id, const std::string &remote_mirror_uuid, const std::string &local_image_name, const std::string &local_image_id, librbd::MockTestImageCtx &mock_remote_image_ctx, Context *on_finish) { - return new MockCreateImageRequest(m_local_io_ctx, m_threads->work_queue, + return new MockCreateImageRequest(mock_threads, m_local_io_ctx, global_image_id, remote_mirror_uuid, local_image_name, local_image_id, &mock_remote_image_ctx, on_finish); @@ -325,8 +386,10 @@ TEST_F(TestMockImageReplayerCreateImageRequest, Create) { expect_create_image(mock_create_request, m_local_io_ctx, 0); C_SaferCond ctx; - MockCreateImageRequest *request = create_request("global uuid", "remote uuid", - "image name", "101241a7c4c9", + MockThreads mock_threads(m_threads); + MockCreateImageRequest *request = create_request(&mock_threads, "global uuid", + "remote uuid", "image name", + "101241a7c4c9", mock_remote_image_ctx, &ctx); request->send(); ASSERT_EQ(0, ctx.wait()); @@ -340,8 +403,10 @@ TEST_F(TestMockImageReplayerCreateImageRequest, CreateError) { expect_create_image(mock_create_request, m_local_io_ctx, -EINVAL); C_SaferCond ctx; - MockCreateImageRequest *request = create_request("global uuid", "remote uuid", - "image name", "101241a7c4c9", + MockThreads mock_threads(m_threads); + MockCreateImageRequest *request = create_request(&mock_threads, "global uuid", + "remote uuid", "image name", + "101241a7c4c9", mock_remote_image_ctx, &ctx); request->send(); ASSERT_EQ(-EINVAL, ctx.wait()); @@ -366,10 +431,15 @@ TEST_F(TestMockImageReplayerCreateImageRequest, Clone) { MockCloneRequest mock_clone_request; MockOpenImageRequest mock_open_image_request; MockCloseImageRequest mock_close_image_request; + journal::MockJournaler mock_remote_journaler; InSequence seq; expect_ioctx_create(m_remote_io_ctx); expect_ioctx_create(m_local_io_ctx); + expect_mirror_uuid_get(m_local_io_ctx, "local parent uuid", 0); + expect_journaler_get_client( + mock_remote_journaler, "local parent uuid", + librbd::journal::MIRROR_PEER_STATE_REPLAYING, 0); expect_get_parent_global_image_id(m_remote_io_ctx, "global uuid", 0); expect_mirror_image_get_image_id(m_local_io_ctx, "local parent id", 0); @@ -379,14 +449,102 @@ TEST_F(TestMockImageReplayerCreateImageRequest, Clone) { expect_close_image(mock_close_image_request, mock_remote_parent_image_ctx, 0); C_SaferCond ctx; - MockCreateImageRequest *request = create_request("global uuid", "remote uuid", - "image name", "101241a7c4c9", + MockThreads mock_threads(m_threads); + MockCreateImageRequest *request = create_request(&mock_threads, "global uuid", + "remote uuid", "image name", + "101241a7c4c9", mock_remote_clone_image_ctx, &ctx); request->send(); ASSERT_EQ(0, ctx.wait()); } +TEST_F(TestMockImageReplayerCreateImageRequest, CloneParentMirrorUuidGetError) { + std::string clone_image_name = get_temp_image_name(); + ASSERT_EQ(0, clone_image(m_remote_image_ctx, "snap", clone_image_name)); + + librbd::ImageCtx *remote_clone_image_ctx; + ASSERT_EQ(0, open_image(m_remote_io_ctx, clone_image_name, + &remote_clone_image_ctx)); + + librbd::MockTestImageCtx mock_remote_clone_image_ctx(*remote_clone_image_ctx); + + InSequence seq; + expect_ioctx_create(m_remote_io_ctx); + expect_ioctx_create(m_local_io_ctx); + expect_mirror_uuid_get(m_local_io_ctx, "local parent uuid", -EPERM); + + C_SaferCond ctx; + MockThreads mock_threads(m_threads); + MockCreateImageRequest *request = create_request(&mock_threads, "global uuid", + "remote uuid", "image name", + "101241a7c4c9", + mock_remote_clone_image_ctx, + &ctx); + request->send(); + ASSERT_EQ(-EPERM, ctx.wait()); +} + +TEST_F(TestMockImageReplayerCreateImageRequest, CloneGetRemoteParentClientStateError) { + std::string clone_image_name = get_temp_image_name(); + ASSERT_EQ(0, clone_image(m_remote_image_ctx, "snap", clone_image_name)); + + librbd::ImageCtx *remote_clone_image_ctx; + ASSERT_EQ(0, open_image(m_remote_io_ctx, clone_image_name, + &remote_clone_image_ctx)); + + librbd::MockTestImageCtx mock_remote_clone_image_ctx(*remote_clone_image_ctx); + journal::MockJournaler mock_remote_journaler; + + InSequence seq; + expect_ioctx_create(m_remote_io_ctx); + expect_ioctx_create(m_local_io_ctx); + expect_mirror_uuid_get(m_local_io_ctx, "local parent uuid", 0); + expect_journaler_get_client( + mock_remote_journaler, "local parent uuid", + librbd::journal::MIRROR_PEER_STATE_REPLAYING, -EPERM); + + C_SaferCond ctx; + MockThreads mock_threads(m_threads); + MockCreateImageRequest *request = create_request(&mock_threads, "global uuid", + "remote uuid", "image name", + "101241a7c4c9", + mock_remote_clone_image_ctx, + &ctx); + request->send(); + ASSERT_EQ(-EPERM, ctx.wait()); +} + +TEST_F(TestMockImageReplayerCreateImageRequest, CloneGetRemoteParentClientStateSyncing) { + std::string clone_image_name = get_temp_image_name(); + ASSERT_EQ(0, clone_image(m_remote_image_ctx, "snap", clone_image_name)); + + librbd::ImageCtx *remote_clone_image_ctx; + ASSERT_EQ(0, open_image(m_remote_io_ctx, clone_image_name, + &remote_clone_image_ctx)); + + librbd::MockTestImageCtx mock_remote_clone_image_ctx(*remote_clone_image_ctx); + journal::MockJournaler mock_remote_journaler; + + InSequence seq; + expect_ioctx_create(m_remote_io_ctx); + expect_ioctx_create(m_local_io_ctx); + expect_mirror_uuid_get(m_local_io_ctx, "local parent uuid", 0); + expect_journaler_get_client( + mock_remote_journaler, "local parent uuid", + librbd::journal::MIRROR_PEER_STATE_SYNCING, 0); + + C_SaferCond ctx; + MockThreads mock_threads(m_threads); + MockCreateImageRequest *request = create_request(&mock_threads, "global uuid", + "remote uuid", "image name", + "101241a7c4c9", + mock_remote_clone_image_ctx, + &ctx); + request->send(); + ASSERT_EQ(-ENOENT, ctx.wait()); +} + TEST_F(TestMockImageReplayerCreateImageRequest, CloneGetGlobalImageIdError) { std::string clone_image_name = get_temp_image_name(); ASSERT_EQ(0, clone_image(m_remote_image_ctx, "snap", clone_image_name)); @@ -396,16 +554,22 @@ TEST_F(TestMockImageReplayerCreateImageRequest, CloneGetGlobalImageIdError) { &remote_clone_image_ctx)); librbd::MockTestImageCtx mock_remote_clone_image_ctx(*remote_clone_image_ctx); - MockCloneRequest mock_clone_request; + journal::MockJournaler mock_remote_journaler; InSequence seq; expect_ioctx_create(m_remote_io_ctx); expect_ioctx_create(m_local_io_ctx); + expect_mirror_uuid_get(m_local_io_ctx, "local parent uuid", 0); + expect_journaler_get_client( + mock_remote_journaler, "local parent uuid", + librbd::journal::MIRROR_PEER_STATE_REPLAYING, 0); expect_get_parent_global_image_id(m_remote_io_ctx, "global uuid", -ENOENT); C_SaferCond ctx; - MockCreateImageRequest *request = create_request("global uuid", "remote uuid", - "image name", "101241a7c4c9", + MockThreads mock_threads(m_threads); + MockCreateImageRequest *request = create_request(&mock_threads, "global uuid", + "remote uuid", "image name", + "101241a7c4c9", mock_remote_clone_image_ctx, &ctx); request->send(); @@ -421,17 +585,23 @@ TEST_F(TestMockImageReplayerCreateImageRequest, CloneGetLocalParentImageIdError) &remote_clone_image_ctx)); librbd::MockTestImageCtx mock_remote_clone_image_ctx(*remote_clone_image_ctx); - MockCloneRequest mock_clone_request; + journal::MockJournaler mock_remote_journaler; InSequence seq; expect_ioctx_create(m_remote_io_ctx); expect_ioctx_create(m_local_io_ctx); + expect_mirror_uuid_get(m_local_io_ctx, "local parent uuid", 0); + expect_journaler_get_client( + mock_remote_journaler, "local parent uuid", + librbd::journal::MIRROR_PEER_STATE_REPLAYING, 0); expect_get_parent_global_image_id(m_remote_io_ctx, "global uuid", 0); expect_mirror_image_get_image_id(m_local_io_ctx, "local parent id", -ENOENT); C_SaferCond ctx; - MockCreateImageRequest *request = create_request("global uuid", "remote uuid", - "image name", "101241a7c4c9", + MockThreads mock_threads(m_threads); + MockCreateImageRequest *request = create_request(&mock_threads, "global uuid", + "remote uuid", "image name", + "101241a7c4c9", mock_remote_clone_image_ctx, &ctx); request->send(); @@ -448,21 +618,28 @@ TEST_F(TestMockImageReplayerCreateImageRequest, CloneOpenRemoteParentError) { librbd::MockTestImageCtx mock_remote_parent_image_ctx(*m_remote_image_ctx); librbd::MockTestImageCtx mock_remote_clone_image_ctx(*remote_clone_image_ctx); - MockCloneRequest mock_clone_request; MockOpenImageRequest mock_open_image_request; + journal::MockJournaler mock_remote_journaler; InSequence seq; expect_ioctx_create(m_remote_io_ctx); expect_ioctx_create(m_local_io_ctx); + expect_mirror_uuid_get(m_local_io_ctx, "local parent uuid", 0); + expect_journaler_get_client( + mock_remote_journaler, "local parent uuid", + librbd::journal::MIRROR_PEER_STATE_REPLAYING, 0); expect_get_parent_global_image_id(m_remote_io_ctx, "global uuid", 0); expect_mirror_image_get_image_id(m_local_io_ctx, "local parent id", 0); expect_open_image(mock_open_image_request, m_remote_io_ctx, - m_remote_image_ctx->id, mock_remote_parent_image_ctx, -ENOENT); + m_remote_image_ctx->id, mock_remote_parent_image_ctx, + -ENOENT); C_SaferCond ctx; - MockCreateImageRequest *request = create_request("global uuid", "remote uuid", - "image name", "101241a7c4c9", + MockThreads mock_threads(m_threads); + MockCreateImageRequest *request = create_request(&mock_threads, "global uuid", + "remote uuid", "image name", + "101241a7c4c9", mock_remote_clone_image_ctx, &ctx); request->send(); @@ -488,10 +665,15 @@ TEST_F(TestMockImageReplayerCreateImageRequest, CloneError) { MockCloneRequest mock_clone_request; MockOpenImageRequest mock_open_image_request; MockCloseImageRequest mock_close_image_request; + journal::MockJournaler mock_remote_journaler; InSequence seq; expect_ioctx_create(m_remote_io_ctx); expect_ioctx_create(m_local_io_ctx); + expect_mirror_uuid_get(m_local_io_ctx, "local parent uuid", 0); + expect_journaler_get_client( + mock_remote_journaler, "local parent uuid", + librbd::journal::MIRROR_PEER_STATE_REPLAYING, 0); expect_get_parent_global_image_id(m_remote_io_ctx, "global uuid", 0); expect_mirror_image_get_image_id(m_local_io_ctx, "local parent id", 0); @@ -501,8 +683,10 @@ TEST_F(TestMockImageReplayerCreateImageRequest, CloneError) { expect_close_image(mock_close_image_request, mock_remote_parent_image_ctx, 0); C_SaferCond ctx; - MockCreateImageRequest *request = create_request("global uuid", "remote uuid", - "image name", "101241a7c4c9", + MockThreads mock_threads(m_threads); + MockCreateImageRequest *request = create_request(&mock_threads, "global uuid", + "remote uuid", "image name", + "101241a7c4c9", mock_remote_clone_image_ctx, &ctx); request->send(); @@ -528,21 +712,29 @@ TEST_F(TestMockImageReplayerCreateImageRequest, CloneRemoteParentCloseError) { MockCloneRequest mock_clone_request; MockOpenImageRequest mock_open_image_request; MockCloseImageRequest mock_close_image_request; + journal::MockJournaler mock_remote_journaler; InSequence seq; expect_ioctx_create(m_remote_io_ctx); expect_ioctx_create(m_local_io_ctx); + expect_mirror_uuid_get(m_local_io_ctx, "local parent uuid", 0); + expect_journaler_get_client( + mock_remote_journaler, "local parent uuid", + librbd::journal::MIRROR_PEER_STATE_REPLAYING, 0); expect_get_parent_global_image_id(m_remote_io_ctx, "global uuid", 0); expect_mirror_image_get_image_id(m_local_io_ctx, "local parent id", 0); expect_open_image(mock_open_image_request, m_remote_io_ctx, m_remote_image_ctx->id, mock_remote_parent_image_ctx, 0); expect_clone_image(mock_clone_request, 0); - expect_close_image(mock_close_image_request, mock_remote_parent_image_ctx, -EINVAL); + expect_close_image(mock_close_image_request, mock_remote_parent_image_ctx, + -EINVAL); C_SaferCond ctx; - MockCreateImageRequest *request = create_request("global uuid", "remote uuid", - "image name", "101241a7c4c9", + MockThreads mock_threads(m_threads); + MockCreateImageRequest *request = create_request(&mock_threads, "global uuid", + "remote uuid", "image name", + "101241a7c4c9", mock_remote_clone_image_ctx, &ctx); request->send(); diff --git a/src/test/rbd_mirror/test_mock_ImageReplayer.cc b/src/test/rbd_mirror/test_mock_ImageReplayer.cc index 0c60d98d4f6e8..85d22ec70a9f1 100644 --- a/src/test/rbd_mirror/test_mock_ImageReplayer.cc +++ b/src/test/rbd_mirror/test_mock_ImageReplayer.cc @@ -196,12 +196,12 @@ struct BootstrapRequest { bool *do_resync = nullptr; static BootstrapRequest* create( + Threads* threads, librados::IoCtx &local_io_ctx, librados::IoCtx &remote_io_ctx, rbd::mirror::InstanceWatcher *instance_watcher, librbd::MockTestImageCtx **local_image_ctx, const std::string &local_image_name, const std::string &remote_image_id, - const std::string &global_image_id, MockContextWQ *work_queue, - MockSafeTimer *timer, Mutex *timer_lock, + const std::string &global_image_id, const std::string &local_mirror_uuid, const std::string &remote_mirror_uuid, ::journal::MockJournalerProxy *journaler, diff --git a/src/tools/rbd_mirror/CMakeLists.txt b/src/tools/rbd_mirror/CMakeLists.txt index b9ecdcfbbb0c6..e3f8407fd4ea7 100644 --- a/src/tools/rbd_mirror/CMakeLists.txt +++ b/src/tools/rbd_mirror/CMakeLists.txt @@ -41,6 +41,7 @@ set(rbd_mirror_internal image_replayer/PrepareLocalImageRequest.cc image_replayer/PrepareRemoteImageRequest.cc image_replayer/ReplayStatusFormatter.cc + image_replayer/Utils.cc image_sync/SyncPointCreateRequest.cc image_sync/SyncPointPruneRequest.cc pool_watcher/RefreshImagesRequest.cc diff --git a/src/tools/rbd_mirror/ImageReplayer.cc b/src/tools/rbd_mirror/ImageReplayer.cc index 59b660a7366fc..bca0f7a650628 100644 --- a/src/tools/rbd_mirror/ImageReplayer.cc +++ b/src/tools/rbd_mirror/ImageReplayer.cc @@ -492,10 +492,9 @@ void ImageReplayer::bootstrap() { ImageReplayer, &ImageReplayer::handle_bootstrap>(this); BootstrapRequest *request = BootstrapRequest::create( - *m_local_ioctx, m_remote_image.io_ctx, m_instance_watcher, + m_threads, *m_local_ioctx, m_remote_image.io_ctx, m_instance_watcher, &m_local_image_ctx, m_local_image_id, m_remote_image.image_id, - m_global_image_id, m_threads->work_queue, m_threads->timer, - &m_threads->timer_lock, m_local_mirror_uuid, m_remote_image.mirror_uuid, + m_global_image_id, m_local_mirror_uuid, m_remote_image.mirror_uuid, m_remote_journaler, &m_client_state, &m_client_meta, ctx, &m_resync_requested, &m_progress_cxt); diff --git a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc index 149a72460e46d..5025adc48e9ea 100644 --- a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc +++ b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc @@ -22,6 +22,7 @@ #include "librbd/journal/Types.h" #include "tools/rbd_mirror/ProgressContext.h" #include "tools/rbd_mirror/ImageSync.h" +#include "tools/rbd_mirror/Threads.h" #define dout_context g_ceph_context #define dout_subsys ceph_subsys_rbd_mirror @@ -39,6 +40,7 @@ using librbd::util::unique_lock_name; template BootstrapRequest::BootstrapRequest( + Threads* threads, librados::IoCtx &local_io_ctx, librados::IoCtx &remote_io_ctx, InstanceWatcher *instance_watcher, @@ -46,8 +48,6 @@ BootstrapRequest::BootstrapRequest( const std::string &local_image_id, const std::string &remote_image_id, const std::string &global_image_id, - ContextWQ *work_queue, SafeTimer *timer, - Mutex *timer_lock, const std::string &local_mirror_uuid, const std::string &remote_mirror_uuid, Journaler *journaler, @@ -58,11 +58,10 @@ BootstrapRequest::BootstrapRequest( rbd::mirror::ProgressContext *progress_ctx) : BaseRequest("rbd::mirror::image_replayer::BootstrapRequest", reinterpret_cast(local_io_ctx.cct()), on_finish), - m_local_io_ctx(local_io_ctx), m_remote_io_ctx(remote_io_ctx), - m_instance_watcher(instance_watcher), m_local_image_ctx(local_image_ctx), - m_local_image_id(local_image_id), m_remote_image_id(remote_image_id), - m_global_image_id(global_image_id), m_work_queue(work_queue), - m_timer(timer), m_timer_lock(timer_lock), + m_threads(threads), m_local_io_ctx(local_io_ctx), + m_remote_io_ctx(remote_io_ctx), m_instance_watcher(instance_watcher), + m_local_image_ctx(local_image_ctx), m_local_image_id(local_image_id), + m_remote_image_id(remote_image_id), m_global_image_id(global_image_id), m_local_mirror_uuid(local_mirror_uuid), m_remote_mirror_uuid(remote_mirror_uuid), m_journaler(journaler), m_client_state(client_state), m_client_meta(client_meta), @@ -281,7 +280,7 @@ void BootstrapRequest::open_local_image() { BootstrapRequest, &BootstrapRequest::handle_open_local_image>( this); OpenLocalImageRequest *request = OpenLocalImageRequest::create( - m_local_io_ctx, m_local_image_ctx, m_local_image_id, m_work_queue, + m_local_io_ctx, m_local_image_ctx, m_local_image_id, m_threads->work_queue, ctx); request->send(); } @@ -488,7 +487,7 @@ void BootstrapRequest::create_local_image() { BootstrapRequest, &BootstrapRequest::handle_create_local_image>( this); CreateImageRequest *request = CreateImageRequest::create( - m_local_io_ctx, m_work_queue, m_global_image_id, m_remote_mirror_uuid, + m_threads, m_local_io_ctx, m_global_image_id, m_remote_mirror_uuid, image_name, m_local_image_id, m_remote_image_ctx, ctx); request->send(); } @@ -668,9 +667,10 @@ void BootstrapRequest::image_sync() { Context *ctx = create_context_callback< BootstrapRequest, &BootstrapRequest::handle_image_sync>(this); m_image_sync = ImageSync::create( - *m_local_image_ctx, m_remote_image_ctx, m_timer, m_timer_lock, - m_local_mirror_uuid, m_journaler, m_client_meta, m_work_queue, - m_instance_watcher, ctx, m_progress_ctx); + *m_local_image_ctx, m_remote_image_ctx, m_threads->timer, + &m_threads->timer_lock, m_local_mirror_uuid, m_journaler, + m_client_meta, m_threads->work_queue, m_instance_watcher, ctx, + m_progress_ctx); m_image_sync->get(); diff --git a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h index a28789b9f8fb8..441281b36b0eb 100644 --- a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h +++ b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h @@ -30,6 +30,7 @@ class ProgressContext; template class ImageSync; template class InstanceWatcher; +template struct Threads; namespace image_replayer { @@ -42,6 +43,7 @@ public: typedef rbd::mirror::ProgressContext ProgressContext; static BootstrapRequest* create( + Threads* threads, librados::IoCtx &local_io_ctx, librados::IoCtx &remote_io_ctx, InstanceWatcher *instance_watcher, @@ -49,8 +51,6 @@ public: const std::string &local_image_id, const std::string &remote_image_id, const std::string &global_image_id, - ContextWQ *work_queue, SafeTimer *timer, - Mutex *timer_lock, const std::string &local_mirror_uuid, const std::string &remote_mirror_uuid, Journaler *journaler, @@ -59,23 +59,23 @@ public: Context *on_finish, bool *do_resync, ProgressContext *progress_ctx = nullptr) { - return new BootstrapRequest(local_io_ctx, remote_io_ctx, + return new BootstrapRequest(threads, local_io_ctx, remote_io_ctx, instance_watcher, local_image_ctx, local_image_id, remote_image_id, - global_image_id, work_queue, timer, timer_lock, - local_mirror_uuid, remote_mirror_uuid, - journaler, client_state, client_meta, on_finish, - do_resync, progress_ctx); + global_image_id, local_mirror_uuid, + remote_mirror_uuid, journaler, client_state, + client_meta, on_finish, do_resync, + progress_ctx); } - BootstrapRequest(librados::IoCtx &local_io_ctx, + BootstrapRequest(Threads* threads, + librados::IoCtx &local_io_ctx, librados::IoCtx &remote_io_ctx, InstanceWatcher *instance_watcher, ImageCtxT **local_image_ctx, const std::string &local_image_id, const std::string &remote_image_id, - const std::string &global_image_id, ContextWQ *work_queue, - SafeTimer *timer, Mutex *timer_lock, + const std::string &global_image_id, const std::string &local_mirror_uuid, const std::string &remote_mirror_uuid, Journaler *journaler, cls::journal::ClientState *client_state, @@ -146,6 +146,7 @@ private: */ typedef std::list Tags; + Threads* m_threads; librados::IoCtx &m_local_io_ctx; librados::IoCtx &m_remote_io_ctx; InstanceWatcher *m_instance_watcher; @@ -153,9 +154,6 @@ private: std::string m_local_image_id; std::string m_remote_image_id; std::string m_global_image_id; - ContextWQ *m_work_queue; - SafeTimer *m_timer; - Mutex *m_timer_lock; std::string m_local_mirror_uuid; std::string m_remote_mirror_uuid; Journaler *m_journaler; diff --git a/src/tools/rbd_mirror/image_replayer/CreateImageRequest.cc b/src/tools/rbd_mirror/image_replayer/CreateImageRequest.cc index f57b196257b71..eb34a5e915f83 100644 --- a/src/tools/rbd_mirror/image_replayer/CreateImageRequest.cc +++ b/src/tools/rbd_mirror/image_replayer/CreateImageRequest.cc @@ -8,19 +8,25 @@ #include "common/errno.h" #include "common/WorkQueue.h" #include "cls/rbd/cls_rbd_client.h" +#include "journal/Journaler.h" +#include "journal/Settings.h" #include "librbd/ImageCtx.h" #include "librbd/ImageState.h" #include "librbd/internal.h" #include "librbd/Utils.h" #include "librbd/image/CreateRequest.h" #include "librbd/image/CloneRequest.h" +#include "librbd/journal/Types.h" +#include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/image_replayer/Utils.h" #define dout_context g_ceph_context #define dout_subsys ceph_subsys_rbd_mirror #undef dout_prefix #define dout_prefix *_dout << "rbd::mirror::image_replayer::CreateImageRequest: " \ - << this << " " << __func__ + << this << " " << __func__ << ": " +using librbd::util::create_async_context_callback; using librbd::util::create_context_callback; using librbd::util::create_rados_callback; @@ -29,15 +35,15 @@ namespace mirror { namespace image_replayer { template -CreateImageRequest::CreateImageRequest(librados::IoCtx &local_io_ctx, - ContextWQ *work_queue, +CreateImageRequest::CreateImageRequest(Threads* threads, + librados::IoCtx &local_io_ctx, const std::string &global_image_id, const std::string &remote_mirror_uuid, const std::string &local_image_name, const std::string &local_image_id, I *remote_image_ctx, Context *on_finish) - : m_local_io_ctx(local_io_ctx), m_work_queue(work_queue), + : m_threads(threads), m_local_io_ctx(local_io_ctx), m_global_image_id(global_image_id), m_remote_mirror_uuid(remote_mirror_uuid), m_local_image_name(local_image_name), m_local_image_id(local_image_id), @@ -55,23 +61,24 @@ void CreateImageRequest::send() { if (m_remote_parent_spec.pool_id == -1) { create_image(); } else { - get_parent_global_image_id(); + get_local_parent_mirror_uuid(); } } template void CreateImageRequest::create_image() { - dout(20) << dendl; + dout(10) << dendl; using klass = CreateImageRequest; - Context *ctx = create_context_callback(this); + Context *ctx = create_context_callback< + klass, &klass::handle_create_image>(this); RWLock::RLocker snap_locker(m_remote_image_ctx->snap_lock); librbd::ImageOptions image_options; populate_image_options(&image_options); - librbd::image::CreateRequest *req = librbd::image::CreateRequest::create( + auto req = librbd::image::CreateRequest::create( m_local_io_ctx, m_local_image_name, m_local_image_id, m_remote_image_ctx->size, image_options, m_global_image_id, m_remote_mirror_uuid, false, m_remote_image_ctx->op_work_queue, ctx); @@ -80,9 +87,9 @@ void CreateImageRequest::create_image() { template void CreateImageRequest::handle_create_image(int r) { - dout(20) << ": r=" << r << dendl; + dout(10) << "r=" << r << dendl; if (r < 0) { - derr << ": failed to create local image: " << cpp_strerror(r) << dendl; + derr << "failed to create local image: " << cpp_strerror(r) << dendl; finish(r); return; } @@ -90,12 +97,109 @@ void CreateImageRequest::handle_create_image(int r) { finish(0); } +template +void CreateImageRequest::get_local_parent_mirror_uuid() { + dout(10) << dendl; + + librados::ObjectReadOperation op; + librbd::cls_client::mirror_uuid_get_start(&op); + + librados::AioCompletion *aio_comp = create_rados_callback< + CreateImageRequest, + &CreateImageRequest::handle_get_local_parent_mirror_uuid>(this); + m_out_bl.clear(); + int r = m_local_parent_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op, + &m_out_bl); + ceph_assert(r == 0); + aio_comp->release(); +} + +template +void CreateImageRequest::handle_get_local_parent_mirror_uuid(int r) { + if (r >= 0) { + auto it = m_out_bl.cbegin(); + r = librbd::cls_client::mirror_uuid_get_finish( + &it, &m_local_parent_mirror_uuid); + if (r >= 0 && m_local_parent_mirror_uuid.empty()) { + r = -ENOENT; + } + } + + dout(10) << "r=" << r << dendl; + if (r < 0) { + if (r == -ENOENT) { + dout(5) << "local parent mirror uuid missing" << dendl; + } else { + derr << "failed to retrieve local parent mirror uuid: " << cpp_strerror(r) + << dendl; + } + finish(r); + return; + } + + dout(15) << "local_parent_mirror_uuid=" << m_local_parent_mirror_uuid + << dendl; + get_remote_parent_client_state(); +} + +template +void CreateImageRequest::get_remote_parent_client_state() { + dout(10) << dendl; + + m_remote_journaler = new Journaler(m_threads->work_queue, m_threads->timer, + &m_threads->timer_lock, + m_remote_parent_io_ctx, + m_remote_parent_spec.image_id, + m_local_parent_mirror_uuid, {}); + + Context *ctx = create_async_context_callback( + m_threads->work_queue, create_context_callback< + CreateImageRequest, + &CreateImageRequest::handle_get_remote_parent_client_state>(this)); + m_remote_journaler->get_client(m_local_parent_mirror_uuid, &m_client, ctx); +} + +template +void CreateImageRequest::handle_get_remote_parent_client_state(int r) { + dout(10) << "r=" << r << dendl; + + delete m_remote_journaler; + m_remote_journaler = nullptr; + + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; + if (r == -ENOENT) { + dout(15) << "client not registered to parent image" << dendl; + finish(r); + return; + } else if (r < 0) { + derr << "failed to retrieve parent client: " << cpp_strerror(r) << dendl; + finish(r); + return; + } else if (!util::decode_client_meta(m_client, &mirror_peer_client_meta)) { + // require operator intervention since the data is corrupt + derr << "failed to decode parent client: " << cpp_strerror(r) << dendl; + finish(-EBADMSG); + return; + } else if (mirror_peer_client_meta.state != + librbd::journal::MIRROR_PEER_STATE_REPLAYING) { + // avoid possible race w/ incomplete parent image since the parent snapshot + // might be deleted if the sync restarts + dout(15) << "parent image still syncing" << dendl; + finish(-ENOENT); + return; + } + + get_parent_global_image_id(); +} + + template void CreateImageRequest::get_parent_global_image_id() { - dout(20) << dendl; + dout(10) << dendl; librados::ObjectReadOperation op; - librbd::cls_client::mirror_image_get_start(&op, m_remote_parent_spec.image_id); + librbd::cls_client::mirror_image_get_start(&op, + m_remote_parent_spec.image_id); librados::AioCompletion *aio_comp = create_rados_callback< CreateImageRequest, @@ -109,25 +213,25 @@ void CreateImageRequest::get_parent_global_image_id() { template void CreateImageRequest::handle_get_parent_global_image_id(int r) { - dout(20) << ": r=" << r << dendl; + dout(10) << "r=" << r << dendl; if (r == 0) { cls::rbd::MirrorImage mirror_image; auto iter = m_out_bl.cbegin(); r = librbd::cls_client::mirror_image_get_finish(&iter, &mirror_image); if (r == 0) { m_parent_global_image_id = mirror_image.global_image_id; - dout(20) << ": parent_global_image_id=" << m_parent_global_image_id + dout(15) << "parent_global_image_id=" << m_parent_global_image_id << dendl; } } if (r == -ENOENT) { - dout(10) << ": parent image " << m_remote_parent_spec.image_id << " not mirrored" - << dendl; + dout(10) << "parent image " << m_remote_parent_spec.image_id + << " not mirrored" << dendl; finish(r); return; } else if (r < 0) { - derr << ": failed to retrieve global image id for parent image " + derr << "failed to retrieve global image id for parent image " << m_remote_parent_spec.image_id << ": " << cpp_strerror(r) << dendl; finish(r); return; @@ -138,7 +242,7 @@ void CreateImageRequest::handle_get_parent_global_image_id(int r) { template void CreateImageRequest::get_local_parent_image_id() { - dout(20) << dendl; + dout(10) << dendl; librados::ObjectReadOperation op; librbd::cls_client::mirror_image_get_image_id_start( @@ -156,7 +260,7 @@ void CreateImageRequest::get_local_parent_image_id() { template void CreateImageRequest::handle_get_local_parent_image_id(int r) { - dout(20) << ": r=" << r << dendl; + dout(10) << "r=" << r << dendl; if (r == 0) { auto iter = m_out_bl.cbegin(); @@ -165,12 +269,12 @@ void CreateImageRequest::handle_get_local_parent_image_id(int r) { } if (r == -ENOENT) { - dout(10) << ": parent image " << m_parent_global_image_id << " not " + dout(10) << "parent image " << m_parent_global_image_id << " not " << "registered locally" << dendl; finish(r); return; } else if (r < 0) { - derr << ": failed to retrieve local image id for parent image " + derr << "failed to retrieve local image id for parent image " << m_parent_global_image_id << ": " << cpp_strerror(r) << dendl; finish(r); return; @@ -181,7 +285,7 @@ void CreateImageRequest::handle_get_local_parent_image_id(int r) { template void CreateImageRequest::open_remote_parent_image() { - dout(20) << dendl; + dout(10) << dendl; Context *ctx = create_context_callback< CreateImageRequest, @@ -194,9 +298,9 @@ void CreateImageRequest::open_remote_parent_image() { template void CreateImageRequest::handle_open_remote_parent_image(int r) { - dout(20) << ": r=" << r << dendl; + dout(10) << "r=" << r << dendl; if (r < 0) { - derr << ": failed to open remote parent image " << m_parent_pool_name << "/" + derr << "failed to open remote parent image " << m_parent_pool_name << "/" << m_remote_parent_spec.image_id << dendl; finish(r); return; @@ -207,7 +311,7 @@ void CreateImageRequest::handle_open_remote_parent_image(int r) { template void CreateImageRequest::clone_image() { - dout(20) << dendl; + dout(10) << dendl; std::string snap_name; cls::rbd::SnapshotNamespace snap_namespace; @@ -238,9 +342,9 @@ void CreateImageRequest::clone_image() { template void CreateImageRequest::handle_clone_image(int r) { - dout(20) << ": r=" << r << dendl; + dout(10) << "r=" << r << dendl; if (r < 0) { - derr << ": failed to clone image " << m_parent_pool_name << "/" + derr << "failed to clone image " << m_parent_pool_name << "/" << m_remote_parent_spec.image_id << " to " << m_local_image_name << dendl; m_ret_val = r; @@ -251,7 +355,7 @@ void CreateImageRequest::handle_clone_image(int r) { template void CreateImageRequest::close_remote_parent_image() { - dout(20) << dendl; + dout(10) << dendl; Context *ctx = create_context_callback< CreateImageRequest, &CreateImageRequest::handle_close_remote_parent_image>(this); @@ -262,9 +366,9 @@ void CreateImageRequest::close_remote_parent_image() { template void CreateImageRequest::handle_close_remote_parent_image(int r) { - dout(20) << ": r=" << r << dendl; + dout(10) << "r=" << r << dendl; if (r < 0) { - derr << ": error encountered closing remote parent image: " + derr << "error encountered closing remote parent image: " << cpp_strerror(r) << dendl; } @@ -273,15 +377,15 @@ void CreateImageRequest::handle_close_remote_parent_image(int r) { template void CreateImageRequest::error(int r) { - dout(20) << ": r=" << r << dendl; + dout(10) << "r=" << r << dendl; - m_work_queue->queue(create_context_callback< + m_threads->work_queue->queue(create_context_callback< CreateImageRequest, &CreateImageRequest::finish>(this), r); } template void CreateImageRequest::finish(int r) { - dout(20) << ": r=" << r << dendl; + dout(10) << "r=" << r << dendl; m_on_finish->complete(r); delete this; } @@ -304,7 +408,7 @@ int CreateImageRequest::validate_parent() { } if (m_remote_parent_spec != parent_spec) { - derr << ": remote image parent spec mismatch" << dendl; + derr << "remote image parent spec mismatch" << dendl; return -EINVAL; } } @@ -318,7 +422,7 @@ int CreateImageRequest::validate_parent() { int r = remote_rados.ioctx_create2(m_remote_parent_spec.pool_id, m_remote_parent_io_ctx); if (r < 0) { - derr << ": failed to open remote parent pool " << m_remote_parent_spec.pool_id + derr << "failed to open remote parent pool " << m_remote_parent_spec.pool_id << ": " << cpp_strerror(r) << dendl; return r; } @@ -329,7 +433,7 @@ int CreateImageRequest::validate_parent() { r = local_rados.ioctx_create(m_parent_pool_name.c_str(), m_local_parent_io_ctx); if (r < 0) { - derr << ": failed to open local parent pool " << m_parent_pool_name << ": " + derr << "failed to open local parent pool " << m_parent_pool_name << ": " << cpp_strerror(r) << dendl; return r; } diff --git a/src/tools/rbd_mirror/image_replayer/CreateImageRequest.h b/src/tools/rbd_mirror/image_replayer/CreateImageRequest.h index 104a132f22b12..22c4aa0100b70 100644 --- a/src/tools/rbd_mirror/image_replayer/CreateImageRequest.h +++ b/src/tools/rbd_mirror/image_replayer/CreateImageRequest.h @@ -7,35 +7,41 @@ #include "include/int_types.h" #include "include/types.h" #include "include/rados/librados.hpp" +#include "cls/journal/cls_journal_types.h" #include "librbd/Types.h" +#include "librbd/journal/TypeTraits.h" #include class Context; class ContextWQ; +namespace journal { class Journaler; } namespace librbd { class ImageCtx; } namespace librbd { class ImageOptions; } namespace rbd { namespace mirror { + +template struct Threads; + namespace image_replayer { template class CreateImageRequest { public: - static CreateImageRequest *create(librados::IoCtx &local_io_ctx, - ContextWQ *work_queue, + static CreateImageRequest *create(Threads *threads, + librados::IoCtx &local_io_ctx, const std::string &global_image_id, const std::string &remote_mirror_uuid, const std::string &local_image_name, const std::string &local_image_id, ImageCtxT *remote_image_ctx, Context *on_finish) { - return new CreateImageRequest(local_io_ctx, work_queue, global_image_id, + return new CreateImageRequest(threads, local_io_ctx, global_image_id, remote_mirror_uuid, local_image_name, local_image_id, remote_image_ctx, on_finish); } - CreateImageRequest(librados::IoCtx &local_io_ctx, ContextWQ *work_queue, + CreateImageRequest(Threads *threads, librados::IoCtx &local_io_ctx, const std::string &global_image_id, const std::string &remote_mirror_uuid, const std::string &local_image_name, @@ -55,7 +61,13 @@ private: * |\------------> CREATE_IMAGE ---------------------\ * (error) * | | * * | (clone) | * - * \-------------> GET_PARENT_GLOBAL_IMAGE_ID * * * | * * * * + * \-------------> GET_LOCAL_PARENT_MIRROR_UUID * * | * * * * + * | | * * + * v | * + * GET_REMOTE_PARENT_CLIENT_STATE * | * * * * + * | | * * + * v | * + * GET_PARENT_GLOBAL_IMAGE_ID * * * | * * * * * | | * * * v | * * GET_LOCAL_PARENT_IMAGE_ID * * * * | * * * * @@ -73,8 +85,11 @@ private: * @endverbatim */ + typedef librbd::journal::TypeTraits TypeTraits; + typedef typename TypeTraits::Journaler Journaler; + + Threads *m_threads; librados::IoCtx &m_local_io_ctx; - ContextWQ *m_work_queue; std::string m_global_image_id; std::string m_remote_mirror_uuid; std::string m_local_image_name; @@ -83,6 +98,8 @@ private: Context *m_on_finish; librados::IoCtx m_remote_parent_io_ctx; + std::string m_local_parent_mirror_uuid; + Journaler *m_remote_journaler = nullptr; ImageCtxT *m_remote_parent_image_ctx = nullptr; librbd::ParentSpec m_remote_parent_spec; @@ -92,11 +109,18 @@ private: bufferlist m_out_bl; std::string m_parent_global_image_id; std::string m_parent_pool_name; + cls::journal::Client m_client; int m_ret_val = 0; void create_image(); void handle_create_image(int r); + void get_local_parent_mirror_uuid(); + void handle_get_local_parent_mirror_uuid(int r); + + void get_remote_parent_client_state(); + void handle_get_remote_parent_client_state(int r); + void get_parent_global_image_id(); void handle_get_parent_global_image_id(int r); diff --git a/src/tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.cc b/src/tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.cc index cc3ade36ab41c..3ebd5992d436b 100644 --- a/src/tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.cc +++ b/src/tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.cc @@ -14,6 +14,7 @@ #include "librbd/journal/Types.h" #include "tools/rbd_mirror/Threads.h" #include "tools/rbd_mirror/image_replayer/GetMirrorImageIdRequest.h" +#include "tools/rbd_mirror/image_replayer/Utils.h" #define dout_context g_ceph_context #define dout_subsys ceph_subsys_rbd_mirror @@ -134,7 +135,7 @@ void PrepareRemoteImageRequest::handle_get_client(int r) { } else if (r < 0) { derr << "failed to retrieve client: " << cpp_strerror(r) << dendl; finish(r); - } else if (!decode_client_meta()) { + } else if (!util::decode_client_meta(m_client, m_client_meta)) { // require operator intervention since the data is corrupt finish(-EBADMSG); } else { @@ -181,31 +182,6 @@ void PrepareRemoteImageRequest::handle_register_client(int r) { finish(0); } -template -bool PrepareRemoteImageRequest::decode_client_meta() { - dout(20) << dendl; - - librbd::journal::ClientData client_data; - auto it = m_client.data.cbegin(); - try { - decode(client_data, it); - } catch (const buffer::error &err) { - derr << "failed to decode client meta data: " << err.what() << dendl; - return false; - } - - librbd::journal::MirrorPeerClientMeta *client_meta = - boost::get(&client_data.client_meta); - if (client_meta == nullptr) { - derr << "unknown peer registration" << dendl; - return false; - } - - *m_client_meta = *client_meta; - dout(20) << "client found: client_meta=" << *m_client_meta << dendl; - return true; -} - template void PrepareRemoteImageRequest::finish(int r) { dout(20) << "r=" << r << dendl; diff --git a/src/tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.h b/src/tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.h index 8d8f553d5362e..5f278a8c65cf1 100644 --- a/src/tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.h +++ b/src/tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.h @@ -123,8 +123,6 @@ private: void handle_register_client(int r); void finish(int r); - - bool decode_client_meta(); }; } // namespace image_replayer diff --git a/src/tools/rbd_mirror/image_replayer/Utils.cc b/src/tools/rbd_mirror/image_replayer/Utils.cc new file mode 100644 index 0000000000000..eda0179f29d56 --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/Utils.cc @@ -0,0 +1,50 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "tools/rbd_mirror/image_replayer/Utils.h" +#include "common/debug.h" +#include "common/errno.h" +#include "cls/journal/cls_journal_types.h" +#include "librbd/journal/Types.h" + +#define dout_context g_ceph_context +#define dout_subsys ceph_subsys_rbd_mirror +#undef dout_prefix +#define dout_prefix *_dout << "rbd::mirror::image_replayer::util::" \ + << __func__ << ": " + +namespace rbd { +namespace mirror { +namespace image_replayer { +namespace util { + +bool decode_client_meta(const cls::journal::Client& client, + librbd::journal::MirrorPeerClientMeta* client_meta) { + dout(15) << dendl; + + librbd::journal::ClientData client_data; + auto it = client.data.cbegin(); + try { + decode(client_data, it); + } catch (const buffer::error &err) { + derr << "failed to decode client meta data: " << err.what() << dendl; + return false; + } + + auto local_client_meta = boost::get( + &client_data.client_meta); + if (local_client_meta == nullptr) { + derr << "unknown peer registration" << dendl; + return false; + } + + *client_meta = *local_client_meta; + dout(15) << "client found: client_meta=" << *client_meta << dendl; + return true; +} + +} // namespace util +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + diff --git a/src/tools/rbd_mirror/image_replayer/Utils.h b/src/tools/rbd_mirror/image_replayer/Utils.h new file mode 100644 index 0000000000000..d42146d150f47 --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/Utils.h @@ -0,0 +1,23 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef RBD_MIRROR_IMAGE_REPLAYER_UTILS_H +#define RBD_MIRROR_IMAGE_REPLAYER_UTILS_H + +namespace cls { namespace journal { struct Client; } } +namespace librbd { namespace journal { struct MirrorPeerClientMeta; } } + +namespace rbd { +namespace mirror { +namespace image_replayer { +namespace util { + +bool decode_client_meta(const cls::journal::Client& client, + librbd::journal::MirrorPeerClientMeta* client_meta); + +} // namespace util +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +#endif // RBD_MIRROR_IMAGE_REPLAYER_UTILS_H -- 2.39.5