From: Jason Dillaman Date: Thu, 11 Feb 2021 20:23:56 +0000 (-0500) Subject: rbd-mirror: update snapshot mirror image state after snapshot creation X-Git-Tag: v17.1.0~2893^2~2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=c9378ab8293e531a49970a01becd9762cf1f8f80;p=ceph.git rbd-mirror: update snapshot mirror image state after snapshot creation The non-primary mirror snapshot is what is used to link the non-primary to the primary image. If there is an interruption between creating the non-primary image and the creation of the first non-primary snapshot, the images will be considerered unlinked. A future commit will modify librbd to avoid setting the mirror image state to enabled for non-primary snapshot-based mirroring images. rbd-mirror will already automatically delete images in the CREATING state during the bootstrap phase. Signed-off-by: Jason Dillaman --- diff --git a/src/test/rbd_mirror/image_replayer/snapshot/test_mock_Replayer.cc b/src/test/rbd_mirror/image_replayer/snapshot/test_mock_Replayer.cc index ad89a29f384c..5567312f2acf 100644 --- a/src/test/rbd_mirror/image_replayer/snapshot/test_mock_Replayer.cc +++ b/src/test/rbd_mirror/image_replayer/snapshot/test_mock_Replayer.cc @@ -4,6 +4,7 @@ #include "test/rbd_mirror/test_mock_fixture.h" #include "librbd/deep_copy/ImageCopyRequest.h" #include "librbd/deep_copy/SnapshotCopyRequest.h" +#include "librbd/mirror/ImageStateUpdateRequest.h" #include "librbd/mirror/snapshot/CreateNonPrimaryRequest.h" #include "librbd/mirror/snapshot/GetImageStateRequest.h" #include "librbd/mirror/snapshot/ImageMeta.h" @@ -114,6 +115,34 @@ SnapshotCopyRequest* SnapshotCopyRequest::s_ } // namespace deep_copy namespace mirror { + +template <> +struct ImageStateUpdateRequest { + static ImageStateUpdateRequest* s_instance; + static ImageStateUpdateRequest* create( + librados::IoCtx& io_ctx, + const std::string& image_id, + cls::rbd::MirrorImageState mirror_image_state, + const cls::rbd::MirrorImage& mirror_image, + Context* on_finish) { + ceph_assert(s_instance != nullptr); + EXPECT_EQ(cls::rbd::MIRROR_IMAGE_STATE_ENABLED, + mirror_image_state); + EXPECT_EQ(cls::rbd::MirrorImage{}, mirror_image); + s_instance->on_finish = on_finish; + return s_instance; + } + + Context* on_finish = nullptr; + ImageStateUpdateRequest() { + s_instance = this; + } + + MOCK_METHOD0(send, void()); +}; + +ImageStateUpdateRequest* ImageStateUpdateRequest::s_instance = nullptr; + namespace snapshot { template <> @@ -364,6 +393,7 @@ public: typedef CloseImageRequest MockCloseImageRequest; typedef librbd::deep_copy::ImageCopyRequest MockImageCopyRequest; typedef librbd::deep_copy::SnapshotCopyRequest MockSnapshotCopyRequest; + typedef librbd::mirror::ImageStateUpdateRequest MockImageStateUpdateRequest; typedef librbd::mirror::snapshot::CreateNonPrimaryRequest MockCreateNonPrimaryRequest; typedef librbd::mirror::snapshot::GetImageStateRequest MockGetImageStateRequest; typedef librbd::mirror::snapshot::ImageMeta MockImageMeta; @@ -518,6 +548,14 @@ public: })); } + void expect_update_mirror_image_state(MockImageStateUpdateRequest& mock_image_state_update_request, + int r) { + EXPECT_CALL(mock_image_state_update_request, send()) + .WillOnce(Invoke([this, &req=mock_image_state_update_request, r]() { + m_threads->work_queue->queue(req.on_finish, r); + })); + } + void expect_notify_sync_request(MockInstanceWatcher& mock_instance_watcher, const std::string& image_id, int r) { EXPECT_CALL(mock_instance_watcher, notify_sync_request(image_id, _)) @@ -762,6 +800,8 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, SyncSnapshot) { expect_create_non_primary_request(mock_create_non_primary_request, false, "remote mirror uuid", 1, {{1, CEPH_NOSNAP}}, 11, 0); + MockImageStateUpdateRequest mock_image_state_update_request; + expect_update_mirror_image_state(mock_image_state_update_request, 0); expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0); MockImageCopyRequest mock_image_copy_request; expect_image_copy(mock_image_copy_request, 0, 1, 0, {}, @@ -1030,6 +1070,8 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, RemoteImageDemoted) { expect_create_non_primary_request(mock_create_non_primary_request, true, "remote mirror uuid", 1, {{1, CEPH_NOSNAP}}, 11, 0); + MockImageStateUpdateRequest mock_image_state_update_request; + expect_update_mirror_image_state(mock_image_state_update_request, 0); expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0); MockImageCopyRequest mock_image_copy_request; expect_image_copy(mock_image_copy_request, 0, 1, 0, {}, @@ -1659,6 +1701,74 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, CreateNonPrimarySnapshotError) { mock_remote_image_ctx)); } +TEST_F(TestMockImageReplayerSnapshotReplayer, UpdateMirrorImageStateError) { + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; + librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + + MockReplayerListener mock_replayer_listener; + expect_notification(mock_threads, mock_replayer_listener); + + InSequence seq; + + MockInstanceWatcher mock_instance_watcher; + MockImageMeta mock_image_meta; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_image_ctx, + mock_image_meta); + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; + m_pool_meta_cache.set_remote_pool_meta( + m_remote_io_ctx.get_id(), + {"remote mirror uuid", "remote mirror peer uuid"}); + + librbd::UpdateWatchCtx* update_watch_ctx = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx, + mock_replayer_listener, + mock_image_meta, + &update_watch_ctx)); + + // inject snapshot + mock_remote_image_ctx.snap_info = { + {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, "", + CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}}; + + // sync snap1 + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, false); + expect_is_refresh_required(mock_remote_image_ctx, false); + MockSnapshotCopyRequest mock_snapshot_copy_request; + expect_snapshot_copy(mock_snapshot_copy_request, 0, 1, 0, {{1, CEPH_NOSNAP}}, + 0); + MockGetImageStateRequest mock_get_image_state_request; + expect_get_image_state(mock_get_image_state_request, 1, 0); + MockCreateNonPrimaryRequest mock_create_non_primary_request; + expect_create_non_primary_request(mock_create_non_primary_request, + false, "remote mirror uuid", 1, + {{1, CEPH_NOSNAP}}, 11, 0); + MockImageStateUpdateRequest mock_image_state_update_request; + expect_update_mirror_image_state(mock_image_state_update_request, -EIO); + + // wake-up replayer + update_watch_ctx->handle_notify(); + + // wait for sync to complete and expect replay complete + ASSERT_EQ(0, wait_for_notification(1)); + ASSERT_FALSE(mock_replayer.is_replaying()); + ASSERT_EQ(-EIO, mock_replayer.get_error_code()); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx)); +} + TEST_F(TestMockImageReplayerSnapshotReplayer, RequestSyncError) { librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; @@ -1711,6 +1821,8 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, RequestSyncError) { expect_create_non_primary_request(mock_create_non_primary_request, false, "remote mirror uuid", 1, {{1, CEPH_NOSNAP}}, 11, 0); + MockImageStateUpdateRequest mock_image_state_update_request; + expect_update_mirror_image_state(mock_image_state_update_request, 0); expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, -ECANCELED); @@ -1725,7 +1837,6 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, RequestSyncError) { ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, mock_local_image_ctx, mock_remote_image_ctx)); - } TEST_F(TestMockImageReplayerSnapshotReplayer, CopyImageError) { @@ -1780,6 +1891,8 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, CopyImageError) { expect_create_non_primary_request(mock_create_non_primary_request, false, "remote mirror uuid", 1, {{1, CEPH_NOSNAP}}, 11, 0); + MockImageStateUpdateRequest mock_image_state_update_request; + expect_update_mirror_image_state(mock_image_state_update_request, 0); expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0); MockImageCopyRequest mock_image_copy_request; expect_image_copy(mock_image_copy_request, 0, 1, 0, {}, @@ -1851,6 +1964,8 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, UpdateNonPrimarySnapshotError) { expect_create_non_primary_request(mock_create_non_primary_request, false, "remote mirror uuid", 1, {{1, CEPH_NOSNAP}}, 11, 0); + MockImageStateUpdateRequest mock_image_state_update_request; + expect_update_mirror_image_state(mock_image_state_update_request, 0); expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0); MockImageCopyRequest mock_image_copy_request; expect_image_copy(mock_image_copy_request, 0, 1, 0, {}, @@ -2357,6 +2472,8 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, SkipImageSync) { expect_create_non_primary_request(mock_create_non_primary_request, false, "remote mirror uuid", 1, {{1, CEPH_NOSNAP}}, 11, 0); + MockImageStateUpdateRequest mock_image_state_update_request; + expect_update_mirror_image_state(mock_image_state_update_request, 0); MockApplyImageStateRequest mock_apply_state_request; expect_apply_image_state(mock_apply_state_request, 0); expect_mirror_image_snapshot_set_copy_progress( diff --git a/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.cc b/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.cc index 46be2ec8fe97..5be76aa0683c 100644 --- a/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.cc +++ b/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.cc @@ -16,6 +16,7 @@ #include "librbd/deep_copy/Handler.h" #include "librbd/deep_copy/ImageCopyRequest.h" #include "librbd/deep_copy/SnapshotCopyRequest.h" +#include "librbd/mirror/ImageStateUpdateRequest.h" #include "librbd/mirror/snapshot/CreateNonPrimaryRequest.h" #include "librbd/mirror/snapshot/GetImageStateRequest.h" #include "librbd/mirror/snapshot/ImageMeta.h" @@ -938,6 +939,40 @@ void Replayer::handle_create_non_primary_snapshot(int r) { dout(15) << "local_snap_id_end=" << m_local_snap_id_end << dendl; + update_mirror_image_state(); +} + +template +void Replayer::update_mirror_image_state() { + if (m_local_snap_id_start > 0) { + request_sync(); + return; + } + + // a newly created non-primary image has a local mirror state of CREATING + // until this point so that we could avoid preserving the image until + // the first non-primary snapshot linked the two images together. + dout(10) << dendl; + auto ctx = create_context_callback< + Replayer, &Replayer::handle_update_mirror_image_state>(this); + auto req = librbd::mirror::ImageStateUpdateRequest::create( + m_state_builder->local_image_ctx->md_ctx, + m_state_builder->local_image_ctx->id, + cls::rbd::MIRROR_IMAGE_STATE_ENABLED, {}, ctx); + req->send(); +} + +template +void Replayer::handle_update_mirror_image_state(int r) { + dout(10) << "r=" << r << dendl; + + if (r < 0) { + derr << "failed to update local mirror image state: " << cpp_strerror(r) + << dendl; + handle_replay_complete(r, "failed to update local mirror image state"); + return; + } + request_sync(); } diff --git a/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.h b/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.h index 3eab052a20d3..170d3c1d0cc8 100644 --- a/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.h +++ b/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.h @@ -130,6 +130,9 @@ private: * | v | | * | CREATE_NON_PRIMARY_SNAPSHOT | | * | | | | + * | v (skip if not needed)| | + * | UPDATE_MIRROR_IMAGE_STATE | | + * | | | | * | |/--------------------/ | * | | | * | v | @@ -269,6 +272,9 @@ private: void create_non_primary_snapshot(); void handle_create_non_primary_snapshot(int r); + void update_mirror_image_state(); + void handle_update_mirror_image_state(int r); + void request_sync(); void handle_request_sync(int r);