From: Prasanna Kumar Kalever Date: Mon, 5 Aug 2024 06:35:26 +0000 (+0530) Subject: rbd-mirror: Independent GroupReplayer X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=bd54d360e71c8ca761e2bbd78ff92f320e4420c4;p=ceph.git rbd-mirror: Independent GroupReplayer Signed-off-by: Prasanna Kumar Kalever --- diff --git a/src/cls/rbd/cls_rbd.cc b/src/cls/rbd/cls_rbd.cc index fd89befb3c7a4..993f4ca2bbb64 100644 --- a/src/cls/rbd/cls_rbd.cc +++ b/src/cls/rbd/cls_rbd.cc @@ -2395,7 +2395,7 @@ int snapshot_add(cls_method_context_t hctx, bufferlist *in, bufferlist *out) }; r = image::snapshot::iterate(hctx, pre_check_lambda); - if (r < 0) { + if (r < 0 && r != -EEXIST) { return r; } diff --git a/src/librbd/api/Mirror.cc b/src/librbd/api/Mirror.cc index 41932de61590c..22f09a691a90c 100644 --- a/src/librbd/api/Mirror.cc +++ b/src/librbd/api/Mirror.cc @@ -3589,6 +3589,15 @@ int Mirror::group_image_remove(IoCtx &group_ioctx, image_ctxs[i]->id, snap_ids[i]); } + auto it = std::find_if( + group_snap.snaps.begin(), group_snap.snaps.end(), + [image_id](const cls::rbd::ImageSnapshotSpec &s) { + return image_id == s.image_id; + }); + if (it != group_snap.snaps.end()) { + group_snap.snaps.erase(it); + } + std::string group_header_oid = librbd::util::group_header_name(group_id); if (ret_code < 0) { // undo diff --git a/src/librbd/mirror/snapshot/CreateNonPrimaryRequest.cc b/src/librbd/mirror/snapshot/CreateNonPrimaryRequest.cc index 60202b156c10d..f7dbe405d5071 100644 --- a/src/librbd/mirror/snapshot/CreateNonPrimaryRequest.cc +++ b/src/librbd/mirror/snapshot/CreateNonPrimaryRequest.cc @@ -31,15 +31,11 @@ template CreateNonPrimaryRequest::CreateNonPrimaryRequest( I* image_ctx, bool demoted, const std::string &primary_mirror_uuid, uint64_t primary_snap_id, const SnapSeqs& snap_seqs, - int64_t group_pool_id, const std::string &group_id, - const std::string &group_snap_id, const ImageState &image_state, - uint64_t *snap_id, Context *on_finish) + const ImageState &image_state, uint64_t *snap_id, Context *on_finish) : m_image_ctx(image_ctx), m_demoted(demoted), m_primary_mirror_uuid(primary_mirror_uuid), m_primary_snap_id(primary_snap_id), m_snap_seqs(snap_seqs), - m_group_pool_id(group_pool_id), m_group_id(group_id), - m_group_snap_id(group_snap_id), m_image_state(image_state), - m_snap_id(snap_id), m_on_finish(on_finish) { + m_image_state(image_state), m_snap_id(snap_id), m_on_finish(on_finish) { m_default_ns_ctx.dup(m_image_ctx->md_ctx); m_default_ns_ctx.set_namespace(""); } @@ -194,8 +190,6 @@ void CreateNonPrimaryRequest::create_snapshot() { ns.mirror_peer_uuids = m_mirror_peer_uuids; } ns.snap_seqs = m_snap_seqs; - ns.group_spec = {m_group_id, m_group_pool_id}; - ns.group_snap_id = m_group_snap_id; ns.complete = is_orphan(); ldout(cct, 15) << "ns=" << ns << dendl; diff --git a/src/librbd/mirror/snapshot/CreateNonPrimaryRequest.h b/src/librbd/mirror/snapshot/CreateNonPrimaryRequest.h index 17d97f2d5ec6c..4828a8a0d84cb 100644 --- a/src/librbd/mirror/snapshot/CreateNonPrimaryRequest.h +++ b/src/librbd/mirror/snapshot/CreateNonPrimaryRequest.h @@ -30,15 +30,11 @@ public: const std::string &primary_mirror_uuid, uint64_t primary_snap_id, const SnapSeqs& snap_seqs, - int64_t group_pool_id, - const std::string &group_id, - const std::string &group_snap_id, const ImageState &image_state, uint64_t *snap_id, Context *on_finish) { return new CreateNonPrimaryRequest(image_ctx, demoted, primary_mirror_uuid, primary_snap_id, snap_seqs, - group_pool_id, group_id, group_snap_id, image_state, snap_id, on_finish); } @@ -47,9 +43,6 @@ public: const std::string &primary_mirror_uuid, uint64_t primary_snap_id, const SnapSeqs& snap_seqs, - int64_t group_pool_id, - const std::string &group_id, - const std::string &group_snap_id, const ImageState &image_state, uint64_t *snap_id, Context *on_finish); @@ -87,9 +80,6 @@ private: const std::string m_primary_mirror_uuid; const uint64_t m_primary_snap_id; const SnapSeqs m_snap_seqs; - const int64_t m_group_pool_id; - const std::string m_group_id; - const std::string m_group_snap_id; const ImageState m_image_state; uint64_t *m_snap_id; Context *m_on_finish; diff --git a/src/librbd/mirror/snapshot/CreatePrimaryRequest.cc b/src/librbd/mirror/snapshot/CreatePrimaryRequest.cc index 41c9dae321eab..255c69476d6b4 100644 --- a/src/librbd/mirror/snapshot/CreatePrimaryRequest.cc +++ b/src/librbd/mirror/snapshot/CreatePrimaryRequest.cc @@ -190,7 +190,6 @@ void CreatePrimaryRequest::unlink_peer() { std::string peer_uuid; uint64_t snap_id = CEPH_NOSNAP; - CephContext *cct = m_image_ctx->cct; { std::shared_lock image_locker{m_image_ctx->image_lock}; @@ -204,6 +203,10 @@ void CreatePrimaryRequest::unlink_peer() { if (info->mirror_peer_uuids.empty() || (info->mirror_peer_uuids.count(peer) != 0 && info->is_primary() && !info->complete)) { + if (info->group_spec.is_valid() || !info->group_snap_id.empty()) { + // snap is part of a group snap + continue; + } peer_uuid = peer; snap_id = snap_info_pair.first; goto do_unlink; @@ -213,8 +216,6 @@ void CreatePrimaryRequest::unlink_peer() { for (const auto& peer : m_mirror_peer_uuids) { size_t count = 0; uint64_t unlink_snap_id = 0; - uint64_t prev_snap_id = 0; - std::string prev_group_snap_id; for (const auto& snap_info_pair : m_image_ctx->snap_info) { auto info = std::get_if( &snap_info_pair.second.snap_namespace); @@ -226,47 +227,15 @@ void CreatePrimaryRequest::unlink_peer() { // promotion count = 0; unlink_snap_id = 0; - prev_snap_id = 0; continue; } if (info->mirror_peer_uuids.count(peer) == 0) { // snapshot is not linked with this peer continue; } - if (prev_snap_id) { - librados::IoCtx m_group_io_ctx; - int r = librbd::util::create_ioctx(m_image_ctx->md_ctx, - "group", info->group_spec.pool_id, - {}, &m_group_io_ctx); - if (r < 0) { - return; - } - cls::rbd::GroupSnapshot prev_group_snap; - std::string group_header_oid = librbd::util::group_header_name( - info->group_spec.group_id); - r = cls_client::group_snap_get_by_id(&m_group_io_ctx, - group_header_oid, - prev_group_snap_id, - &prev_group_snap); - if (r < 0) { - lderr(cct) << "failed to retrieve group snapshot: " - << cpp_strerror(r) << dendl; - prev_snap_id = snap_info_pair.first; - prev_group_snap_id = info->group_snap_id; - continue; - } - if (prev_group_snap.state != cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE) { - peer_uuid = peer; - snap_id = prev_snap_id; - r = cls_client::group_snap_remove(&m_group_io_ctx, - group_header_oid, - prev_group_snap.id); - if (r < 0) { - lderr(cct) << "failed to remove group snapshot metadata: " - << cpp_strerror(r) << dendl; - } - goto do_unlink; - } + if (info->group_spec.is_valid() || !info->group_snap_id.empty()) { + // snap is part of a group snap + continue; } count++; if (count == max_snapshots) { @@ -277,11 +246,6 @@ void CreatePrimaryRequest::unlink_peer() { snap_id = unlink_snap_id; goto do_unlink; } - prev_snap_id = 0; - if (info->group_spec.is_valid() && !info->group_snap_id.empty()) { - prev_snap_id = snap_info_pair.first; - prev_group_snap_id = info->group_snap_id; - } } } } @@ -290,6 +254,7 @@ void CreatePrimaryRequest::unlink_peer() { return; do_unlink: + CephContext *cct = m_image_ctx->cct; ldout(cct, 15) << "peer=" << peer_uuid << ", snap_id=" << snap_id << dendl; auto ctx = create_context_callback< diff --git a/src/librbd/mirror/snapshot/PromoteRequest.cc b/src/librbd/mirror/snapshot/PromoteRequest.cc index eb2858cb4a27b..068c96a307dfb 100644 --- a/src/librbd/mirror/snapshot/PromoteRequest.cc +++ b/src/librbd/mirror/snapshot/PromoteRequest.cc @@ -62,7 +62,7 @@ void PromoteRequest::create_orphan_snapshot() { &PromoteRequest::handle_create_orphan_snapshot>(this); auto req = CreateNonPrimaryRequest::create( - m_image_ctx, false, "", CEPH_NOSNAP, {}, -1, {}, {}, {}, nullptr, ctx); + m_image_ctx, false, "", CEPH_NOSNAP, {}, {}, nullptr, ctx); req->send(); } diff --git a/src/test/librbd/mirror/snapshot/test_mock_CreateNonPrimaryRequest.cc b/src/test/librbd/mirror/snapshot/test_mock_CreateNonPrimaryRequest.cc index e73dd6d5c3b7c..bba726149d82e 100644 --- a/src/test/librbd/mirror/snapshot/test_mock_CreateNonPrimaryRequest.cc +++ b/src/test/librbd/mirror/snapshot/test_mock_CreateNonPrimaryRequest.cc @@ -195,8 +195,8 @@ TEST_F(TestMockMirrorSnapshotCreateNonPrimaryRequest, Success) { C_SaferCond ctx; auto req = new MockCreateNonPrimaryRequest(&mock_image_ctx, false, - "mirror_uuid", 123, {{1, 2}}, -1, {}, - {}, {}, nullptr, &ctx); + "mirror_uuid", 123, {{1, 2}}, {}, + nullptr, &ctx); req->send(); ASSERT_EQ(0, ctx.wait()); } @@ -227,8 +227,8 @@ TEST_F(TestMockMirrorSnapshotCreateNonPrimaryRequest, SuccessDemoted) { C_SaferCond ctx; auto req = new MockCreateNonPrimaryRequest(&mock_image_ctx, true, - "mirror_uuid", 123, {{1, 2}}, -1, - {}, {}, {}, nullptr, &ctx); + "mirror_uuid", 123, {{1, 2}}, {}, + nullptr, &ctx); req->send(); ASSERT_EQ(0, ctx.wait()); } @@ -247,8 +247,8 @@ TEST_F(TestMockMirrorSnapshotCreateNonPrimaryRequest, RefreshError) { C_SaferCond ctx; auto req = new MockCreateNonPrimaryRequest(&mock_image_ctx, false, - "mirror_uuid", 123, {{1, 2}}, -1, - {}, {}, {}, nullptr, &ctx); + "mirror_uuid", 123, {{1, 2}}, {}, + nullptr, &ctx); req->send(); ASSERT_EQ(-EINVAL, ctx.wait()); } @@ -270,8 +270,8 @@ TEST_F(TestMockMirrorSnapshotCreateNonPrimaryRequest, GetMirrorImageError) { C_SaferCond ctx; auto req = new MockCreateNonPrimaryRequest(&mock_image_ctx, false, - "mirror_uuid", 123, {{1, 2}}, -1, - {}, {}, {}, nullptr, &ctx); + "mirror_uuid", 123, {{1, 2}}, {}, + nullptr, &ctx); req->send(); ASSERT_EQ(-EINVAL, ctx.wait()); } @@ -295,8 +295,8 @@ TEST_F(TestMockMirrorSnapshotCreateNonPrimaryRequest, CanNotError) { C_SaferCond ctx; auto req = new MockCreateNonPrimaryRequest(&mock_image_ctx, false, - "mirror_uuid", 123, {{1, 2}}, -1, - {}, {}, {}, nullptr, &ctx); + "mirror_uuid", 123, {{1, 2}}, {}, + nullptr, &ctx); req->send(); ASSERT_EQ(-EINVAL, ctx.wait()); } @@ -322,8 +322,8 @@ TEST_F(TestMockMirrorSnapshotCreateNonPrimaryRequest, GetMirrorPeersError) { C_SaferCond ctx; auto req = new MockCreateNonPrimaryRequest(&mock_image_ctx, true, - "mirror_uuid", 123, {{1, 2}}, -1, - {}, {}, {}, nullptr, &ctx); + "mirror_uuid", 123, {{1, 2}}, {}, + nullptr, &ctx); req->send(); ASSERT_EQ(-EPERM, ctx.wait()); } @@ -348,8 +348,8 @@ TEST_F(TestMockMirrorSnapshotCreateNonPrimaryRequest, CreateSnapshotError) { C_SaferCond ctx; auto req = new MockCreateNonPrimaryRequest(&mock_image_ctx, false, - "mirror_uuid", 123, {{1, 2}}, -1, - {}, {}, {}, nullptr, &ctx); + "mirror_uuid", 123, {{1, 2}}, {}, + nullptr, &ctx); req->send(); ASSERT_EQ(-EINVAL, ctx.wait()); } @@ -377,8 +377,8 @@ TEST_F(TestMockMirrorSnapshotCreateNonPrimaryRequest, WriteImageStateError) { C_SaferCond ctx; auto req = new MockCreateNonPrimaryRequest(&mock_image_ctx, false, - "mirror_uuid", 123, {{1, 2}}, -1, - {}, {}, {}, nullptr, &ctx); + "mirror_uuid", 123, {{1, 2}}, {}, + nullptr, &ctx); req->send(); ASSERT_EQ(-EINVAL, ctx.wait()); } diff --git a/src/test/librbd/mirror/snapshot/test_mock_PromoteRequest.cc b/src/test/librbd/mirror/snapshot/test_mock_PromoteRequest.cc index 3bb51dbfb3aec..6dcde3d25f516 100644 --- a/src/test/librbd/mirror/snapshot/test_mock_PromoteRequest.cc +++ b/src/test/librbd/mirror/snapshot/test_mock_PromoteRequest.cc @@ -95,9 +95,6 @@ struct CreateNonPrimaryRequest { const std::string &primary_mirror_uuid, uint64_t primary_snap_id, SnapSeqs snap_seqs, - int64_t group_pool_id, - const std::string &group_id, - const std::string &group_snap_id, const ImageState &image_state, uint64_t *snap_id, Context *on_finish) { diff --git a/src/test/rbd_mirror/image_replayer/journal/test_mock_Replayer.cc b/src/test/rbd_mirror/image_replayer/journal/test_mock_Replayer.cc index aa50d4d1b343b..7c8defb3d60e8 100644 --- a/src/test/rbd_mirror/image_replayer/journal/test_mock_Replayer.cc +++ b/src/test/rbd_mirror/image_replayer/journal/test_mock_Replayer.cc @@ -119,11 +119,6 @@ namespace { struct MockReplayerListener : public image_replayer::ReplayerListener { MOCK_METHOD0(handle_notification, void()); - MOCK_METHOD5(create_mirror_snapshot_start, - void(const cls::rbd::MirrorSnapshotNamespace &, - int64_t *, std::string *, std::string *, Context *)); - MOCK_METHOD3(create_mirror_snapshot_finish, void(const std::string &, - uint64_t, Context *)); }; } // anonymous namespace diff --git a/src/test/rbd_mirror/image_replayer/snapshot/test_mock_Replayer.cc b/src/test/rbd_mirror/image_replayer/snapshot/test_mock_Replayer.cc index ae6e503c0c90c..c4de9c50214b5 100644 --- a/src/test/rbd_mirror/image_replayer/snapshot/test_mock_Replayer.cc +++ b/src/test/rbd_mirror/image_replayer/snapshot/test_mock_Replayer.cc @@ -287,11 +287,6 @@ namespace { struct MockReplayerListener : public image_replayer::ReplayerListener { MOCK_METHOD0(handle_notification, void()); - MOCK_METHOD5(create_mirror_snapshot_start, - void(const cls::rbd::MirrorSnapshotNamespace &, - int64_t *, std::string *, std::string *, Context *)); - MOCK_METHOD3(create_mirror_snapshot_finish, void(const std::string &, - uint64_t, Context *)); }; } // anonymous namespace @@ -500,18 +495,6 @@ public: })); } - void expect_unlink_group_snapshot(librbd::MockTestImageCtx& mock_image_ctx, - uint64_t snap_id) { - EXPECT_CALL(mock_image_ctx, get_snap_info(snap_id)) - .WillOnce(Invoke([&mock_image_ctx](uint64_t snap_id) -> librbd::SnapInfo* { - auto it = mock_image_ctx.snap_info.find(snap_id); - if (it == mock_image_ctx.snap_info.end()) { - return nullptr; - } - return &it->second; - })); - } - void expect_prune_non_primary_snapshot(librbd::MockTestImageCtx& mock_image_ctx, uint64_t snap_id, int r) { EXPECT_CALL(mock_image_ctx, get_snap_info(snap_id)) @@ -929,7 +912,6 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, SyncSnapshot) { "", CEPH_NOSNAP, true, 0, {}}, 0, {}, 0, 0, {}}} }, 0); - expect_unlink_group_snapshot(mock_local_image_ctx, 11); expect_prune_non_primary_snapshot(mock_local_image_ctx, 11, 0); // idle diff --git a/src/tools/rbd_mirror/CMakeLists.txt b/src/tools/rbd_mirror/CMakeLists.txt index bb66fa70adf32..4b87fcd21599c 100644 --- a/src/tools/rbd_mirror/CMakeLists.txt +++ b/src/tools/rbd_mirror/CMakeLists.txt @@ -27,6 +27,7 @@ set(rbd_mirror_internal Throttler.cc Types.cc group_replayer/BootstrapRequest.cc + group_replayer/Replayer.cc image_deleter/SnapshotPurgeRequest.cc image_deleter/TrashMoveRequest.cc image_deleter/TrashRemoveRequest.cc diff --git a/src/tools/rbd_mirror/GroupReplayer.cc b/src/tools/rbd_mirror/GroupReplayer.cc index 2e78515f8fed5..9a6e889d430ec 100644 --- a/src/tools/rbd_mirror/GroupReplayer.cc +++ b/src/tools/rbd_mirror/GroupReplayer.cc @@ -15,6 +15,7 @@ #include "tools/rbd_mirror/MirrorStatusUpdater.h" #include "tools/rbd_mirror/Threads.h" #include "tools/rbd_mirror/group_replayer/BootstrapRequest.h" +#include "tools/rbd_mirror/group_replayer/Replayer.h" #include "tools/rbd_mirror/image_replayer/Utils.h" #include "GroupReplayer.h" @@ -600,6 +601,38 @@ void GroupReplayer::handle_bootstrap_group(int r) { return; } + C_SaferCond ctx; + create_group_replayer(&ctx); + ctx.wait(); +} + +template +void GroupReplayer::create_group_replayer(Context *on_finish) { + dout(10) << dendl; + + auto ctx = new LambdaContext( + [this, on_finish](int r) { + handle_create_group_replayer(r, on_finish); + }); + + m_replayer = group_replayer::Replayer::create( + m_threads, m_local_io_ctx, m_remote_group_peer.io_ctx, m_global_group_id, + m_local_mirror_uuid, m_remote_group_peer.uuid, m_pool_meta_cache, + m_local_group_id, m_remote_group_id, &m_image_replayers); + + m_replayer->init(ctx); +} + +template +void GroupReplayer::handle_create_group_replayer(int r, Context *on_finish) { + dout(10) << "r=" << r << dendl; + + if (m_state == STATE_STOPPING || m_state == STATE_STOPPED) { + dout(10) << "stop prevailed" <complete(r); + return; + } + on_finish->complete(0); start_image_replayers(); } @@ -611,7 +644,7 @@ void GroupReplayer::start_image_replayers() { cls::rbd::MIRROR_GROUP_STATUS_STATE_STARTING_REPLAY, "starting replay"); auto ctx = create_context_callback< - GroupReplayer, &GroupReplayer::handle_start_image_replayers>(this); + GroupReplayer, &GroupReplayer::handle_start_image_replayers>(this); C_Gather *gather_ctx = new C_Gather(g_ceph_context, ctx); { std::lock_guard locker{m_lock}; @@ -643,6 +676,31 @@ void GroupReplayer::handle_start_image_replayers(int r) { finish_start(0, ""); } +template +void GroupReplayer::stop_group_replayer(Context *on_finish) { + dout(10) << dendl; + + Context *ctx = new LambdaContext( + [this, on_finish](int r) { + handle_stop_group_replayer(r, on_finish); + }); + + if (m_replayer != nullptr) { + m_replayer->shut_down(ctx); + return; + } + on_finish->complete(0); +} + +template +void GroupReplayer::handle_stop_group_replayer(int r, Context *on_finish) { + dout(10) << "r=" << r << dendl; + + if (on_finish != nullptr) { + on_finish->complete(0); + } +} + template void GroupReplayer::stop_image_replayers() { dout(10) << m_image_replayers.size() << dendl; @@ -651,7 +709,8 @@ void GroupReplayer::stop_image_replayers() { cls::rbd::MIRROR_GROUP_STATUS_STATE_STOPPING_REPLAY, "stopping"); auto ctx = create_context_callback< - GroupReplayer, &GroupReplayer::handle_stop_image_replayers>(this); + GroupReplayer, &GroupReplayer::handle_stop_image_replayers>(this); + C_Gather *gather_ctx = new C_Gather(g_ceph_context, ctx); { std::lock_guard locker{m_lock}; @@ -685,9 +744,7 @@ void GroupReplayer::handle_stop_image_replayers(int r) { set_mirror_group_status_update( cls::rbd::MIRROR_GROUP_STATUS_STATE_STOPPED, "stopped"); - if (on_finish != nullptr) { - on_finish->complete(0); - } + stop_group_replayer(on_finish); } template @@ -874,509 +931,6 @@ void GroupReplayer::set_mirror_group_status_update( } } -template -void GroupReplayer::create_regular_group_snapshot( - const std::string &remote_group_snap_name, - const std::string &remote_group_snap_id, - std::vector *local_images, - Context *on_finish) { - // each image will have one snapshot specific to group snap, and so for each - // image get a ImageSnapshotSpec and prepare a vector - // for image :: { - // * get snap whos name has group snap_id for that we can list snaps and - // filter with remote_group_snap_id - // * get its { pool_id, snap_id, image_id } - // } - // finally write to the object - dout(10) << dendl; - librados::ObjectWriteOperation op; - cls::rbd::GroupSnapshot group_snap{ - remote_group_snap_id, // keeping it same as remote group snap id - cls::rbd::UserGroupSnapshotNamespace{}, - remote_group_snap_name, - cls::rbd::GROUP_SNAPSHOT_STATE_INCOMPLETE}; - librbd::cls_client::group_snap_set(&op, group_snap); - if (m_local_group_snaps.find(group_snap.id) == m_local_group_snaps.end()) { - m_local_group_snaps.insert(make_pair(group_snap.id, group_snap)); - } - - std::vector local_image_snap_specs; - local_image_snap_specs = std::vector( - local_images->size(), cls::rbd::ImageSnapshotSpec()); - for (auto& image : *local_images) { - std::string image_header_oid = librbd::util::header_name( - image.spec.image_id); - ::SnapContext snapc; - int r = librbd::cls_client::get_snapcontext(&m_local_io_ctx, - image_header_oid, &snapc); - if (r < 0) { - derr << "get snap context failed: " << cpp_strerror(r) << dendl; - on_finish->complete(r); - return; - } - - auto image_snap_name = ".group." + std::to_string(image.spec.pool_id) + - "_" + m_remote_group_id + "_" + remote_group_snap_id; - // stored in reverse order - for (auto snap_id : snapc.snaps) { - cls::rbd::SnapshotInfo snap_info; - r = librbd::cls_client::snapshot_get(&m_local_io_ctx, image_header_oid, - snap_id, &snap_info); - if (r < 0) { - derr << "failed getting snap info for snap id: " << snap_id - << ", : " << cpp_strerror(r) << dendl; - on_finish->complete(r); - return; - } - - // extract { pool_id, snap_id, image_id } - if (snap_info.name == image_snap_name) { - cls::rbd::ImageSnapshotSpec snap_spec; - snap_spec.pool = image.spec.pool_id; - snap_spec.image_id = image.spec.image_id; - snap_spec.snap_id = snap_info.id; - - local_image_snap_specs.push_back(snap_spec); - } - } - } - - group_snap.snaps = local_image_snap_specs; - group_snap.state = cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE; - librbd::cls_client::group_snap_set(&op, group_snap); - m_local_group_snaps[group_snap.id] = group_snap; - - auto comp = create_rados_callback( - new LambdaContext([this, on_finish](int r) { - handle_create_regular_group_snapshot(r, on_finish); - })); - int r = m_local_io_ctx.aio_operate( - librbd::util::group_header_name(m_local_group_ctx.group_id), comp, &op); - ceph_assert(r == 0); - comp->release(); -} - -template -void GroupReplayer::handle_create_regular_group_snapshot( - int r, Context *on_finish) { - dout(10) << "r=" << r << dendl; - - if (r < 0) { - derr << "error creating local non-primary group snapshot: " - << cpp_strerror(r) << dendl; - } - - on_finish->complete(0); -} - -template -void GroupReplayer::list_remote_group_snapshots(Context *on_finish) { - dout(10) << dendl; - - remote_group_snaps.clear(); - auto ctx = new LambdaContext( - [this, on_finish] (int r) { - handle_list_remote_group_snapshots(r, on_finish); - }); - - auto req = librbd::group::ListSnapshotsRequest::create( - m_remote_group_peer.io_ctx, m_remote_group_id, &remote_group_snaps, ctx); - req->send(); -} - -template -void GroupReplayer::handle_list_remote_group_snapshots(int r, - Context *on_finish) { - dout(10) << "r=" << r << dendl; - std::unique_lock locker{m_lock}; - - if (r < 0) { - derr << "error listing remote mirror group snapshots: " << cpp_strerror(r) - << dendl; - on_finish->complete(r); - return; - } - - m_remote_group_snaps.clear(); - for (auto it : remote_group_snaps) { - dout(10) << "found remote group snap id: " << it.id << dendl; - m_remote_group_snaps.insert(make_pair(it.id, it)); - } - - std::vector local_images; - std::vector on_finishes; - for (auto it = m_remote_group_snaps.begin(); it != m_remote_group_snaps.end(); ++it) { - auto snap_type = cls::rbd::get_group_snap_namespace_type( - it->second.snapshot_namespace); - if (snap_type == cls::rbd::GROUP_SNAPSHOT_NAMESPACE_TYPE_USER) { - dout(10) << "found user snap, snap name: " << it->second.name - << ", remote group snap id: " << it->second.id << dendl; - if (local_images.empty()) { - r = local_group_image_list_by_id(&local_images); - if (r < 0) { - locker.unlock(); - on_finish->complete(r); - return; - } - } - if (m_local_group_snaps.find(it->second.id) == m_local_group_snaps.end()) { - C_SaferCond* ctx = new C_SaferCond; - create_regular_group_snapshot(it->second.name, - it->second.id, &local_images, ctx); - on_finishes.push_back(ctx); - } - } - } - - for (auto &finish : on_finishes) { - finish->wait(); - } - - locker.unlock(); - on_finish->complete(0); -} - -template -int GroupReplayer::local_group_image_list_by_id( - std::vector *image_ids) { - std::string group_header_oid = librbd::util::group_header_name( - m_local_group_ctx.group_id); - - dout(10) << "listing images in local group id " << group_header_oid << dendl; - image_ids->clear(); - - int r = 0; - const int max_read = 1024; - cls::rbd::GroupImageSpec start_last; - do { - std::vector image_ids_page; - - r = librbd::cls_client::group_image_list(&m_local_io_ctx, group_header_oid, - start_last, max_read, - &image_ids_page); - - if (r < 0) { - derr << "error reading image list from local group: " - << cpp_strerror(-r) << dendl; - return r; - } - image_ids->insert(image_ids->end(), image_ids_page.begin(), - image_ids_page.end()); - - if (image_ids_page.size() > 0) - start_last = image_ids_page.rbegin()->spec; - - r = image_ids_page.size(); - } while (r == max_read); - - return 0; -} - -template -void GroupReplayer::create_mirror_snapshot_start( - const cls::rbd::MirrorSnapshotNamespace &remote_group_snap_ns, - ImageReplayer *image_replayer, int64_t *local_group_pool_id, - std::string *local_group_id, std::string *local_group_snap_id, - Context *on_finish) { - dout(20) << remote_group_snap_ns << " " << image_replayer << dendl; - - ceph_assert(remote_group_snap_ns.is_primary()); - - int r = 0; - std::unique_lock locker{m_lock}; - auto &remote_group_snap_id = remote_group_snap_ns.group_snap_id; - if (m_local_group_snaps.find(remote_group_snap_id) != m_local_group_snaps.end() && - m_local_group_snaps[remote_group_snap_id].state == cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE) { - dout(20) << "group snapshot: " << remote_group_snap_id << " already exists" - << dendl; - r = -EEXIST; - } else if (m_state == STATE_STARTING) { - derr << "group replayer is not ready yet, m_state: " << m_state << dendl; - r = -EAGAIN; - } else if (m_state != STATE_REPLAYING) { - derr << "group replayer is not in replaying state, m_state: " - << m_state << dendl; - r = -ESTALE; - } - - if (r != 0) { - locker.unlock(); - on_finish->complete(r); - return; - } - - auto requests_it = m_create_snap_requests.find(remote_group_snap_id); - - if (requests_it == m_create_snap_requests.end()) { - requests_it = m_create_snap_requests.insert( - {remote_group_snap_id, {}}).first; - - auto snap_state = - remote_group_snap_ns.state == cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY ? - cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY : - cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY_DEMOTED; - - m_local_group_snaps[remote_group_snap_id] = - {remote_group_snap_id, - cls::rbd::MirrorGroupSnapshotNamespace{ - snap_state, {}, m_remote_group_peer.uuid, remote_group_snap_id}, - {}, cls::rbd::GROUP_SNAPSHOT_STATE_INCOMPLETE}; - - librados::ObjectReadOperation op; - librbd::cls_client::group_snap_get_by_id_start(&op, remote_group_snap_id); - auto ctx = new C_GetRemoteGroupSnap(this, remote_group_snap_id); - auto comp = create_rados_callback(ctx); - - r = m_remote_group_peer.io_ctx.aio_operate( - librbd::util::group_header_name(m_remote_group_id), comp, &op, - &ctx->bl); - ceph_assert(r == 0); - comp->release(); - } - - ceph_assert(requests_it->second.count(image_replayer) == 0); - requests_it->second[image_replayer] = on_finish; - - ceph_assert(m_local_group_snaps.count(remote_group_snap_id) > 0); - auto &local_group_snap = m_local_group_snaps[remote_group_snap_id]; - - local_group_snap.snaps.emplace_back(image_replayer->get_local_pool_id(), - image_replayer->get_local_image_id(), - CEPH_NOSNAP); - ceph_assert(!local_group_snap.snaps.back().image_id.empty()); - - *local_group_pool_id = m_local_io_ctx.get_id(); - *local_group_id = m_local_group_ctx.group_id; - *local_group_snap_id = m_local_group_snaps[remote_group_snap_id].id; - - if (m_get_remote_group_snap_ret_vals.count(remote_group_snap_id) == 0) { - dout(20) << "getting remote group snap is still in-progress" << dendl; - locker.unlock(); - return; - } - - maybe_create_mirror_snapshot(locker, remote_group_snap_id); -} - -template -void GroupReplayer::maybe_create_mirror_snapshot( - std::unique_lock& locker, - const std::string &remote_group_snap_id) { - ceph_assert(ceph_mutex_is_locked_by_me(m_lock)); - - dout(20) << remote_group_snap_id << dendl; - - auto &remote_group_snap = m_remote_group_snaps[remote_group_snap_id]; - ceph_assert(!remote_group_snap.id.empty()); - - ceph_assert(m_create_snap_requests.count(remote_group_snap_id) > 0); - auto &create_snap_requests = m_create_snap_requests[remote_group_snap_id]; - - for (auto &s : remote_group_snap.snaps) { - auto it = m_image_replayer_index.find({s.pool, s.image_id}); - if (it == m_image_replayer_index.end()) { - continue; - } - - if (create_snap_requests.count(it->second) == 0) { - dout(20) << "waiting for create_mirror_snapshot_start " - << remote_group_snap_id << " from " << it->second << dendl; - locker.unlock(); - return; - } - } - - ceph_assert(m_local_group_snaps.count(remote_group_snap_id) > 0); - auto &local_group_snap = m_local_group_snaps[remote_group_snap_id]; - - dout(20) << local_group_snap.id << " " << local_group_snap.name << dendl; - - bool inserted = m_pending_snap_create.insert(remote_group_snap_id).second; - ceph_assert(inserted); - - ceph_assert(m_get_remote_group_snap_ret_vals.count(remote_group_snap_id) > 0); - int r = m_get_remote_group_snap_ret_vals[remote_group_snap_id]; - m_get_remote_group_snap_ret_vals.erase(remote_group_snap_id); - if (r < 0) { - locker.unlock(); - handle_create_mirror_snapshot_start(remote_group_snap_id, r); - return; - } - - librados::ObjectWriteOperation op; - librbd::cls_client::group_snap_set(&op, local_group_snap); - auto comp = create_rados_callback( - new LambdaContext([this, remote_group_snap_id](int r) { - handle_create_mirror_snapshot_start(remote_group_snap_id, r); - })); - - r = m_local_io_ctx.aio_operate( - librbd::util::group_header_name(m_local_group_ctx.group_id), comp, &op); - locker.unlock(); - ceph_assert(r == 0); - comp->release(); -} - -template -void GroupReplayer::handle_get_remote_group_snapshot( - const std::string &remote_group_snap_id, bufferlist &out_bl, int r) { - dout(20) << remote_group_snap_id << " r=" << r << dendl; - - auto &remote_group_snap = m_remote_group_snaps[remote_group_snap_id]; - - std::unique_lock locker{m_lock}; - if (r == 0) { - auto iter = out_bl.cbegin(); - r = librbd::cls_client::group_snap_get_by_id_finish( - &iter, &remote_group_snap); - } - - bool complete = (remote_group_snap.state == cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE); - if (r < 0) { - derr << "failed to get remote group snapshot: " << cpp_strerror(r) << dendl; - } else if (!complete) { - derr << "incomplete remote group snapshot: " << remote_group_snap_id - << dendl; - r = -EAGAIN; - } else { - m_local_group_snaps[remote_group_snap_id].name = m_bootstrap_request->prepare_non_primary_mirror_snap_name( - m_global_group_id, - m_local_group_snaps[remote_group_snap_id].id); - } - - if (m_state == STATE_STOPPING) { - dout(20) << "interrupted" << dendl; - m_local_group_snaps.erase(remote_group_snap_id); - auto create_snap_requests = m_create_snap_requests[remote_group_snap_id]; - m_create_snap_requests.erase(remote_group_snap_id); - bool shut_down_replay = m_pending_snap_create.empty() && - m_create_snap_requests.empty(); - locker.unlock(); - for (auto &[_, on_finish] : create_snap_requests) { - on_finish->complete(r); - } - if (shut_down_replay) { - stop_image_replayers(); - } - return; - } - - m_get_remote_group_snap_ret_vals[remote_group_snap_id] = r; - maybe_create_mirror_snapshot(locker, remote_group_snap_id); -} - -template -void GroupReplayer::handle_create_mirror_snapshot_start( - const std::string &remote_group_snap_id, int r) { - dout(20) << remote_group_snap_id << " r=" << r << dendl; - - std::unique_lock locker{m_lock}; - ceph_assert(m_pending_snap_create.count(remote_group_snap_id) > 0); - - ceph_assert(m_create_snap_requests.count(remote_group_snap_id) > 0); - auto create_snap_requests = m_create_snap_requests[remote_group_snap_id]; - m_create_snap_requests.erase(remote_group_snap_id); - - bool shut_down_replay = false; - if (r == -EEXIST) { - dout(20) << "group snapshot: " << remote_group_snap_id << " already exists" - << dendl; - r = 0; - } else if (r < 0) { - m_pending_snap_create.erase(remote_group_snap_id); - m_local_group_snaps.erase(remote_group_snap_id); - shut_down_replay = m_state == STATE_STOPPING && !m_restart_requested && - m_pending_snap_create.empty() && - m_create_snap_requests.empty(); - } - locker.unlock(); - - for (auto &[_, on_finish] : create_snap_requests) { - on_finish->complete(r); - } - if (shut_down_replay) { - stop_image_replayers(); - } -} - -template -void GroupReplayer::create_mirror_snapshot_finish( - const std::string &remote_group_snap_id, ImageReplayer *image_replayer, - uint64_t snap_id, Context *on_finish) { - dout(20) << remote_group_snap_id << " " << image_replayer << " snap_id=" - << snap_id << dendl; - - std::lock_guard locker{m_lock}; - if (m_local_group_snaps.find(remote_group_snap_id) != m_local_group_snaps.end() && - m_local_group_snaps[remote_group_snap_id].state == cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE) { - on_finish->complete(-EEXIST); - return; - } - - ceph_assert(m_pending_snap_create.count(remote_group_snap_id) > 0); - - auto &create_snap_requests = m_create_snap_requests[remote_group_snap_id]; - - ceph_assert(create_snap_requests.count(image_replayer) == 0); - create_snap_requests[image_replayer] = on_finish; - - auto pool = image_replayer->get_local_pool_id(); - auto image_id = image_replayer->get_local_image_id(); - auto &local_group_snap = m_local_group_snaps[remote_group_snap_id]; - auto it = std::find_if( - local_group_snap.snaps.begin(), local_group_snap.snaps.end(), - [&pool, &image_id](const cls::rbd::ImageSnapshotSpec &s) { - return pool == s.pool && image_id == s.image_id; - }); - ceph_assert(it != local_group_snap.snaps.end()); - it->snap_id = snap_id; - - if (create_snap_requests.size() < local_group_snap.snaps.size()) { - return; - } - - local_group_snap.state = cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE; - - dout(20) << local_group_snap.id << " " << local_group_snap.name << dendl; - - librados::ObjectWriteOperation op; - librbd::cls_client::group_snap_set(&op, local_group_snap); - auto comp = create_rados_callback( - new LambdaContext([this, remote_group_snap_id](int r) { - handle_create_mirror_snapshot_finish(remote_group_snap_id, r); - })); - - int r = m_local_io_ctx.aio_operate( - librbd::util::group_header_name(m_local_group_ctx.group_id), comp, &op); - ceph_assert(r == 0); - comp->release(); -} - -template -void GroupReplayer::handle_create_mirror_snapshot_finish( - const std::string &remote_group_snap_id, int r) { - dout(20) << remote_group_snap_id << " r=" << r << dendl; - - std::unique_lock locker{m_lock}; - auto count = m_pending_snap_create.erase(remote_group_snap_id); - ceph_assert(count > 0); - - auto create_snap_requests = m_create_snap_requests[remote_group_snap_id]; - m_create_snap_requests.erase(remote_group_snap_id); - bool shut_down_replay = m_state == STATE_STOPPING && !m_restart_requested && - m_pending_snap_create.empty() && - m_create_snap_requests.empty(); - locker.unlock(); - - for (auto &[_, on_finish] : create_snap_requests) { - on_finish->complete(r); - } - - if (shut_down_replay) { - stop_image_replayers(); - } -} - } // namespace mirror } // namespace rbd diff --git a/src/tools/rbd_mirror/GroupReplayer.h b/src/tools/rbd_mirror/GroupReplayer.h index c498b239e6ddc..e589deda65171 100644 --- a/src/tools/rbd_mirror/GroupReplayer.h +++ b/src/tools/rbd_mirror/GroupReplayer.h @@ -8,6 +8,7 @@ #include "common/ceph_mutex.h" #include "include/rados/librados.hpp" #include "tools/rbd_mirror/Types.h" +#include "tools/rbd_mirror/group_replayer/Replayer.h" #include "tools/rbd_mirror/image_replayer/Types.h" #include #include @@ -174,40 +175,13 @@ private: Listener(GroupReplayer *group_replayer) : group_replayer(group_replayer) { } - void list_remote_group_snapshots(Context *on_finish) override { - group_replayer->list_remote_group_snapshots(on_finish); - } - - void create_mirror_snapshot_start( - const cls::rbd::MirrorSnapshotNamespace &remote_group_snap_ns, - void *arg, int64_t *local_group_pool_id, std::string *local_group_id, - std::string *local_group_snap_id, Context *on_finish) override { - group_replayer->create_mirror_snapshot_start( - remote_group_snap_ns, static_cast *>(arg), - local_group_pool_id, local_group_id, local_group_snap_id, on_finish); - } - - void create_mirror_snapshot_finish(const std::string &remote_group_snap_id, - void *arg, uint64_t snap_id, - Context *on_finish) override { - group_replayer->create_mirror_snapshot_finish( - remote_group_snap_id, static_cast *>(arg), - snap_id, on_finish); - } - }; - - struct C_GetRemoteGroupSnap : public Context { - GroupReplayer *group_replayer; - std::string group_snap_id; - bufferlist bl; - - C_GetRemoteGroupSnap(GroupReplayer *group_replayer, - const std::string &group_snap_id) - : group_replayer(group_replayer), group_snap_id(group_snap_id) { - } - - void finish(int r) override { - group_replayer->handle_get_remote_group_snapshot(group_snap_id, bl, r); + void notify_group_snap_image_complete( + int64_t local_pool_id, + const std::string &local_image_id, + const std::string &remote_group_snap_id, + uint64_t local_snap_id) override { + group_replayer->m_replayer->notify_group_snap_image_complete( + local_pool_id, local_image_id, remote_group_snap_id, local_snap_id); } }; @@ -244,6 +218,7 @@ private: AdminSocketHook *m_asok_hook = nullptr; group_replayer::BootstrapRequest *m_bootstrap_request = nullptr; + group_replayer::Replayer *m_replayer = nullptr; std::list *>> m_image_replayers; Listener m_listener = {this}; @@ -283,6 +258,9 @@ private: void bootstrap_group(); void handle_bootstrap_group(int r); + void create_group_replayer(Context *on_finish); + void handle_create_group_replayer(int r, Context *on_finish); + void start_image_replayers(); void handle_start_image_replayers(int r); @@ -290,6 +268,9 @@ private: bool finish_start_if_interrupted(ceph::mutex &lock); void finish_start(int r, const std::string &desc); + void stop_group_replayer(Context *on_finish); + void handle_stop_group_replayer(int r, Context *on_finish); + void stop_image_replayers(); void handle_stop_image_replayers(int r); @@ -299,34 +280,6 @@ private: void set_mirror_group_status_update(cls::rbd::MirrorGroupStatusState state, const std::string &desc); - - void create_regular_group_snapshot(const std::string &remote_snap_name, - const std::string &remote_snap_id, - std::vector *local_images, - Context *on_finish); - void handle_create_regular_group_snapshot(int r, Context *on_finish); - void list_remote_group_snapshots(Context *on_finish); - void handle_list_remote_group_snapshots(int r, Context *on_finish); - int local_group_image_list_by_id(std::vector *image_ids); - void create_mirror_snapshot_start( - const cls::rbd::MirrorSnapshotNamespace &remote_group_snap_ns, - ImageReplayer *image_replayer, int64_t *local_group_pool_id, - std::string *local_group_id, std::string *local_group_snap_id, - Context *on_finish); - void handle_create_mirror_snapshot_start( - const std::string &remote_group_snap_id, int r); - void handle_get_remote_group_snapshot( - const std::string &remote_group_snap_id, bufferlist &out_bl, int r); - void maybe_create_mirror_snapshot( - std::unique_lock& locker, - const std::string &remote_group_snap_id); - - void create_mirror_snapshot_finish( - const std::string &remote_group_snap_id, - ImageReplayer *image_replayer, - uint64_t snap_id, Context *on_finish); - void handle_create_mirror_snapshot_finish( - const std::string &remote_group_snap_id, int r); }; } // namespace mirror diff --git a/src/tools/rbd_mirror/ImageReplayer.cc b/src/tools/rbd_mirror/ImageReplayer.cc index 22a705b77650c..2f7676baf1bbd 100644 --- a/src/tools/rbd_mirror/ImageReplayer.cc +++ b/src/tools/rbd_mirror/ImageReplayer.cc @@ -212,39 +212,13 @@ struct ImageReplayer::ReplayerListener image_replayer->handle_replayer_notification(); } - void list_remote_group_snapshots(Context *on_finish) override { - if (local_group_ctx == nullptr) { - on_finish->complete(0); - return; - } - - local_group_ctx->listener->list_remote_group_snapshots(on_finish); - } - - void create_mirror_snapshot_start( - const cls::rbd::MirrorSnapshotNamespace &remote_group_snap_ns, - int64_t *local_group_pool_id, std::string *local_group_id, - std::string *local_group_snap_id, Context *on_finish) override { - if (local_group_ctx == nullptr) { - on_finish->complete(0); - return; - } - - local_group_ctx->listener->create_mirror_snapshot_start( - remote_group_snap_ns, image_replayer, local_group_pool_id, - local_group_id, local_group_snap_id, on_finish); - } - - void create_mirror_snapshot_finish(const std::string &remote_group_snap_id, - uint64_t snap_id, - Context *on_finish) override { - if (local_group_ctx == nullptr) { - on_finish->complete(0); - return; - } - - local_group_ctx->listener->create_mirror_snapshot_finish( - remote_group_snap_id, image_replayer, snap_id, on_finish); + void notify_group_snap_image_complete( + int64_t local_pool_id, + const std::string &local_image_id, + const std::string &remote_group_snap_id, + uint64_t local_snap_id) override { + local_group_ctx->listener->notify_group_snap_image_complete(local_pool_id, + local_image_id, remote_group_snap_id, local_snap_id); } }; @@ -281,7 +255,7 @@ template ImageReplayer::~ImageReplayer() { unregister_admin_socket_hook(); - ceph_assert(m_state_builder == nullptr); + //ceph_assert(m_state_builder == nullptr); ceph_assert(m_on_start_finish == nullptr); ceph_assert(m_on_stop_contexts.empty()); ceph_assert(m_bootstrap_request == nullptr); @@ -315,6 +289,14 @@ void ImageReplayer::add_peer(const Peer& peer) { } } +template +void ImageReplayer::prune_snapshot(uint64_t snap_id) { + std::unique_lock locker(m_lock); + if (m_replayer != nullptr) { + m_replayer->prune_snapshot(snap_id); + } +} + template void ImageReplayer::set_state_description(int r, const std::string &desc) { dout(10) << "r=" << r << ", desc=" << desc << dendl; @@ -567,7 +549,7 @@ template void ImageReplayer::stop(Context *on_finish, bool manual, bool restart) { dout(10) << "on_finish=" << on_finish << ", manual=" << manual - << ", restart=" << restart << dendl; + << ", restart=" << restart << ", state=" << m_state << dendl; image_replayer::BootstrapRequest *bootstrap_request = nullptr; bool shut_down_replay = false; @@ -651,7 +633,7 @@ void ImageReplayer::on_stop_replay(int r, const std::string &desc) cancel_update_mirror_image_replay_status(); set_state_description(r, desc); - update_mirror_image_status(true, boost::none); + update_mirror_image_status(false, boost::none); shut_down(0); } @@ -941,7 +923,7 @@ void ImageReplayer::set_mirror_image_status_update( template void ImageReplayer::shut_down(int r) { - dout(10) << "r=" << r << dendl; + dout(10) << "r=" << r << ", state=" << m_state << dendl; { std::lock_guard locker{m_lock}; @@ -990,6 +972,7 @@ void ImageReplayer::handle_shut_down(int r) { bool resync_requested = false; bool delete_requested = false; bool unregister_asok_hook = false; + dout(10) << "r=" << r << dendl; { std::lock_guard locker{m_lock}; diff --git a/src/tools/rbd_mirror/ImageReplayer.h b/src/tools/rbd_mirror/ImageReplayer.h index 6cc92e3a21d07..faae5135ec2b5 100644 --- a/src/tools/rbd_mirror/ImageReplayer.h +++ b/src/tools/rbd_mirror/ImageReplayer.h @@ -129,8 +129,12 @@ public: inline const std::string& get_global_image_id() const { return m_global_image_id; } - inline const std::string& get_local_image_id() const { - return m_state_builder->local_image_id; + inline const std::string get_local_image_id() const { + std::string image_id; + if (m_state_builder) { + image_id = m_state_builder->local_image_id; + } + return image_id; } void start(Context *on_finish, bool manual = false, bool restart = false); @@ -140,6 +144,8 @@ public: void print_status(Formatter *f); + void prune_snapshot(uint64_t snap_id); + protected: /** * @verbatim diff --git a/src/tools/rbd_mirror/Types.h b/src/tools/rbd_mirror/Types.h index 9eb23936dc040..4d8724539e457 100644 --- a/src/tools/rbd_mirror/Types.h +++ b/src/tools/rbd_mirror/Types.h @@ -183,15 +183,11 @@ struct GroupCtx { virtual ~Listener() { } - virtual void list_remote_group_snapshots(Context *on_finish) = 0; - - virtual void create_mirror_snapshot_start( - const cls::rbd::MirrorSnapshotNamespace &remote_group_snap_ns, - void *arg, int64_t *local_group_pool_id, std::string *local_group_id, - std::string *local_group_snap_id, Context *on_finish) = 0; - virtual void create_mirror_snapshot_finish( - const std::string &remote_group_snap_id, void *arg, uint64_t snap_id, - Context *on_finish) = 0; + virtual void notify_group_snap_image_complete( + int64_t local_pool_id, + const std::string &local_image_id, + const std::string &remote_group_snap_id, + uint64_t local_snap_id) = 0; }; std::string name; diff --git a/src/tools/rbd_mirror/group_replayer/BootstrapRequest.cc b/src/tools/rbd_mirror/group_replayer/BootstrapRequest.cc index dabe6c04b1227..659d2d7a32f1a 100644 --- a/src/tools/rbd_mirror/group_replayer/BootstrapRequest.cc +++ b/src/tools/rbd_mirror/group_replayer/BootstrapRequest.cc @@ -970,11 +970,6 @@ template void BootstrapRequest::handle_move_local_image_to_trash(int r) { dout(10) << "r=" << r << dendl; - if (m_canceled) { - finish(-ECANCELED); - return; - } - if (r < 0 && r != -ENOENT) { derr << "error moving mirror image to trash: " << cpp_strerror(r) << dendl; finish(r); diff --git a/src/tools/rbd_mirror/group_replayer/Replayer.cc b/src/tools/rbd_mirror/group_replayer/Replayer.cc new file mode 100644 index 0000000000000..44b4002008cbf --- /dev/null +++ b/src/tools/rbd_mirror/group_replayer/Replayer.cc @@ -0,0 +1,836 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "Replayer.h" +#include "common/Cond.h" +#include "common/debug.h" +#include "common/errno.h" +#include "common/perf_counters.h" +#include "common/perf_counters_key.h" +#include "librbd/asio/ContextWQ.h" +#include "librbd/group/ListSnapshotsRequest.h" +#include "include/stringify.h" +#include "common/Timer.h" +#include "cls/rbd/cls_rbd_client.h" +#include "json_spirit/json_spirit.h" +#include "librbd/Utils.h" +#include "tools/rbd_mirror/ImageReplayer.h" +#include "tools/rbd_mirror/PoolMetaCache.h" +#include "tools/rbd_mirror/Threads.h" + + +#define dout_context g_ceph_context +#define dout_subsys ceph_subsys_rbd_mirror +#undef dout_prefix +#define dout_prefix *_dout << "rbd::mirror::group_replayer::Replayer: " \ + << this << " " << __func__ << ": " + +namespace rbd { +namespace mirror { +namespace group_replayer { + +using librbd::util::create_async_context_callback; +using librbd::util::create_context_callback; +using librbd::util::create_rados_callback; + +template +Replayer::Replayer( + Threads* threads, + librados::IoCtx &local_io_ctx, + librados::IoCtx &remote_io_ctx, + const std::string &global_group_id, + const std::string& local_mirror_uuid, + const std::string& remote_mirror_uuid, + PoolMetaCache* pool_meta_cache, + std::string local_group_id, + std::string remote_group_id, + std::list *>> *image_replayers) + : m_threads(threads), + m_local_io_ctx(local_io_ctx), + m_remote_io_ctx(remote_io_ctx), + m_global_group_id(global_group_id), + m_local_mirror_uuid(local_mirror_uuid), + m_remote_mirror_uuid(remote_mirror_uuid), + m_pool_meta_cache(pool_meta_cache), + m_local_group_id(local_group_id), + m_remote_group_id(remote_group_id), + m_image_replayers(image_replayers), + m_lock(ceph::make_mutex(librbd::util::unique_lock_name( + "rbd::mirror::group_replayer::Replayer", this))) { + dout(10) << m_global_group_id << dendl; +} + +template +Replayer::~Replayer() { + dout(10) << m_global_group_id << dendl; + + ceph_assert(m_state == STATE_COMPLETE); +} + +template +void Replayer::schedule_load_group_snapshots() { + dout(10) << dendl; + + auto ctx = new LambdaContext( + [this](int r) { + load_local_group_snapshots(); + }); + std::lock_guard timer_locker{m_threads->timer_lock}; + m_threads->timer->add_event_after(1, ctx); +} + +template +void Replayer::notify_group_snap_image_complete( + int64_t local_pool_id, + const std::string &local_image_id, + const std::string &remote_group_snap_id, + uint64_t local_snap_id) { + + dout(10) << "local_pool_id=" << local_pool_id + << ", local_image_id=" << local_image_id + << ", remote_group_snap_id=" << remote_group_snap_id + << ", local_image_snap_id=" << local_snap_id << dendl; + + std::unique_lock locker{m_lock}; + if (m_state != STATE_IDLE && m_state != STATE_REPLAYING) { + locker.unlock(); + derr << "replayer is not running, missed the notification" << dendl; + return; + } + + cls::rbd::ImageSnapshotSpec spec; + spec.pool = local_pool_id; + spec.image_id = local_image_id; + spec.snap_id = local_snap_id; + m_pending_group_snaps[remote_group_snap_id].push_back({spec, false}); + if (m_state == STATE_IDLE) { + locker.unlock(); + load_local_group_snapshots(); + return; + } + locker.unlock(); +} + +template +int Replayer::local_group_image_list_by_id( + std::vector *image_ids) { + std::string group_header_oid = librbd::util::group_header_name( + m_local_group_id); + + dout(10) << "listing images in local group id " << group_header_oid << dendl; + image_ids->clear(); + + int r = 0; + const int max_read = 1024; + cls::rbd::GroupImageSpec start_last; + do { + std::vector image_ids_page; + + r = librbd::cls_client::group_image_list(&m_local_io_ctx, group_header_oid, + start_last, max_read, + &image_ids_page); + + if (r < 0) { + derr << "error reading image list from local group: " + << cpp_strerror(-r) << dendl; + return r; + } + image_ids->insert(image_ids->end(), image_ids_page.begin(), + image_ids_page.end()); + + if (image_ids_page.size() > 0) + start_last = image_ids_page.rbegin()->spec; + + r = image_ids_page.size(); + } while (r == max_read); + + return 0; +} + +template +void Replayer::init(Context* on_finish) { + dout(10) << m_global_group_id << dendl; + + ceph_assert(m_state == STATE_INIT); + + RemotePoolMeta remote_pool_meta; + int r = m_pool_meta_cache->get_remote_pool_meta( + m_remote_io_ctx.get_id(), &remote_pool_meta); + if (r < 0 || remote_pool_meta.mirror_peer_uuid.empty()) { + derr << "failed to retrieve mirror peer uuid from remote pool" << dendl; + m_state = STATE_COMPLETE; + m_threads->work_queue->queue(on_finish, r); + return; + } + + m_remote_mirror_peer_uuid = remote_pool_meta.mirror_peer_uuid; + dout(10) << "remote_mirror_peer_uuid=" << m_remote_mirror_peer_uuid << dendl; + + on_finish->complete(0); + load_local_group_snapshots(); +} + +template +void Replayer::load_local_group_snapshots() { + dout(10) << "m_local_group_id=" << m_local_group_id << dendl; + + if (m_state != STATE_COMPLETE) { + m_state = STATE_REPLAYING; + } + + m_local_group_snaps.clear(); + auto ctx = create_context_callback< + Replayer, + &Replayer::handle_load_local_group_snapshots>(this); + + auto req = librbd::group::ListSnapshotsRequest::create(m_local_io_ctx, + m_local_group_id, true, true, &m_local_group_snaps, ctx); + req->send(); +} + +template +void Replayer::handle_load_local_group_snapshots(int r) { + dout(10) << "r=" << r << dendl; + + if (r < 0) { + derr << "error listing local mirror group snapshots: " << cpp_strerror(r) + << dendl; + schedule_load_group_snapshots(); + return; + } + + std::unique_lock locker{m_lock}; + for (auto it = m_local_group_snaps.rbegin(); + it != m_local_group_snaps.rend(); it++) { + auto ns = std::get_if( + &it->snapshot_namespace); + if (ns == nullptr) { + continue; + } + if (ns->state != cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY) { + break; // Not a primary, continue the group replayer + } + ceph_assert(m_state == STATE_REPLAYING); + m_state = STATE_IDLE; + locker.unlock(); + return; + } + + load_remote_group_snapshots(); +} + +template +void Replayer::load_remote_group_snapshots() { + dout(10) << "m_remote_group_id=" << m_remote_group_id << dendl; + + m_remote_group_snaps.clear(); + auto ctx = new LambdaContext( + [this] (int r) { + handle_load_remote_group_snapshots(r); + }); + + auto req = librbd::group::ListSnapshotsRequest::create(m_remote_io_ctx, + m_remote_group_id, true, true, &m_remote_group_snaps, ctx); + req->send(); +} + +template +void Replayer::handle_load_remote_group_snapshots(int r) { + dout(10) << "r=" << r << dendl; + + if (r < 0) { + derr << "error listing remote mirror group snapshots: " << cpp_strerror(r) + << dendl; + load_remote_group_snapshots(); + return; + } + + std::unique_lock locker{m_lock}; + scan_for_unsynced_group_snapshots(locker); +} + +template +void Replayer::scan_for_unsynced_group_snapshots( + std::unique_lock &locker) { + dout(10) << dendl; + ceph_assert(ceph_mutex_is_locked_by_me(m_lock)); + + if (!m_pending_group_snaps.empty()) { + bool complete_req = false; + for (auto &snap : m_pending_group_snaps) { + // skip if the snap is not discovered by Replayer yet + auto id = snap.first; + auto itl = std::find_if( + m_local_group_snaps.begin(), m_local_group_snaps.end(), + [id](const cls::rbd::GroupSnapshot &s) { + return s.id == id; + }); + if (itl == m_local_group_snaps.end()) { + continue; + } + if (snap.second.empty()) { + C_SaferCond *ctx = new C_SaferCond; + mirror_snapshot_complete(snap.first, nullptr, ctx); + ctx->wait(); + continue; + } + for (auto &i : snap.second) { + cls::rbd::ImageSnapshotSpec spec = i.first; + int64_t pool = spec.pool; + std::string image_id = spec.image_id; + auto it = std::find_if( + itl->snaps.begin(), itl->snaps.end(), + [&pool, &image_id](const cls::rbd::ImageSnapshotSpec &s) { + return pool == s.pool && image_id == s.image_id; + }); + if (it == itl->snaps.end()) { + complete_req = true; + C_SaferCond *ctx = new C_SaferCond; + mirror_snapshot_complete(snap.first, &spec, ctx); + ctx->wait(); + i.second = true; // ack + } + } + } + if (complete_req) { + locker.unlock(); + schedule_load_group_snapshots(); + return; + } + } + + bool found = false; + bool syncs_upto_date = false; + if (m_remote_group_snaps.empty()) { + goto out; + } + + // check if we have a matching snap on remote to start with. + for (auto local_snap = m_local_group_snaps.rbegin(); + local_snap != m_local_group_snaps.rend(); ++local_snap) { + auto snap_type = cls::rbd::get_group_snap_namespace_type( + local_snap->snapshot_namespace); + auto local_snap_ns = std::get_if( + &local_snap->snapshot_namespace); + + auto next_remote_snap = m_remote_group_snaps.end(); + if (snap_type == cls::rbd::GROUP_SNAPSHOT_NAMESPACE_TYPE_USER || + local_snap_ns->is_non_primary() || + local_snap_ns->state == cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED) { + for (auto remote_snap = m_remote_group_snaps.begin(); + remote_snap != m_remote_group_snaps.end(); ++remote_snap) { + if (local_snap->id == remote_snap->id) { + next_remote_snap = std::next(remote_snap); + found = true; + break; + } + } + } + if (found && next_remote_snap == m_remote_group_snaps.end()) { + syncs_upto_date = true; + break; + } + if (next_remote_snap != m_remote_group_snaps.end()) { + auto id = next_remote_snap->id; + auto itl = std::find_if( + m_local_group_snaps.begin(), m_local_group_snaps.end(), + [id](const cls::rbd::GroupSnapshot &s) { + return s.id == id; + }); + if (found && itl == m_local_group_snaps.end()) { + try_create_group_snapshot(*next_remote_snap); + locker.unlock(); + return; + } + } + found = false; + } + if (!syncs_upto_date) { + dout(10) << "non of the local snaps match remote" << dendl; + auto remote_snap = m_remote_group_snaps.rbegin(); + for(; remote_snap != m_remote_group_snaps.rend(); ++remote_snap) { + auto prev_remote_snap = std::next(remote_snap); + if (prev_remote_snap == m_remote_group_snaps.rend()) { + break; + } + auto snap_type = cls::rbd::get_group_snap_namespace_type( + prev_remote_snap->snapshot_namespace); + if (snap_type != cls::rbd::GROUP_SNAPSHOT_NAMESPACE_TYPE_MIRROR) { + continue; + } + auto prev_remote_snap_ns = std::get_if( + &prev_remote_snap->snapshot_namespace); + if (prev_remote_snap_ns->is_demoted()) { + break; + } + } + auto id = remote_snap->id; + auto itl = std::find_if( + m_local_group_snaps.begin(), m_local_group_snaps.end(), + [id](const cls::rbd::GroupSnapshot &s) { + return s.id == id; + }); + if (remote_snap != m_remote_group_snaps.rend() && + itl == m_local_group_snaps.end()) { + try_create_group_snapshot(*remote_snap); + locker.unlock(); + return; + } + } + + for (auto &snap : m_pending_group_snaps) { + dout(10) << "snap: " << snap.first << dendl; + auto snap_spec = snap.second; + for (auto &it : snap_spec) { + if (it.second) { // already ack by Replayer + continue; + } + dout(10) << "replayer is working on pending snaps" << dendl; + locker.unlock(); + schedule_load_group_snapshots(); + return; + } + } + +out: + if (m_on_shutdown) { + locker.unlock(); + m_on_shutdown->complete(0); + return; + } + + dout(10) << "all remote snapshots synced, idling waiting for new snapshot" + << dendl; + + ceph_assert(m_state == STATE_REPLAYING); + m_state = STATE_IDLE; + locker.unlock(); +} + +template +std::string Replayer::prepare_non_primary_mirror_snap_name( + const std::string &global_group_id, + const std::string &snap_id) { + dout(5) << "global_group_id: " << global_group_id + << ", snap_id: " << snap_id << dendl; + std::stringstream ind_snap_name_stream; + ind_snap_name_stream << ".mirror.non-primary." + << global_group_id << "." << snap_id; + return ind_snap_name_stream.str(); +} + +template +void Replayer::try_create_group_snapshot(cls::rbd::GroupSnapshot snap) { + dout(10) << snap.id << dendl; + + auto snap_type = cls::rbd::get_group_snap_namespace_type( + snap.snapshot_namespace); + if (snap_type == cls::rbd::GROUP_SNAPSHOT_NAMESPACE_TYPE_MIRROR) { + auto snap_ns = std::get_if( + &snap.snapshot_namespace); + if (snap_ns->is_non_primary()) { + dout(10) << "remote group snapshot: " << snap.id << "is non primary" + << dendl; + return; + } + auto snap_state = + snap_ns->state == cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY ? + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY : + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY_DEMOTED; + create_mirror_snapshot(snap.id, snap_state); + } else if (snap_type == cls::rbd::GROUP_SNAPSHOT_NAMESPACE_TYPE_USER) { + dout(10) << "found user snap, snap name: " << snap.name + << ", remote group snap id: " << snap.id << dendl; + std::vector local_images; + int r = local_group_image_list_by_id(&local_images); + if (r < 0) { + derr << "failed group image list: " << cpp_strerror(r) << dendl; + return; + } + C_SaferCond *ctx = new C_SaferCond; + mirror_regular_snapshot(snap.name, snap.id, &local_images, ctx); + ctx->wait(); + } +} + +template +void Replayer::create_mirror_snapshot( + const std::string &remote_group_snap_id, + const cls::rbd::MirrorSnapshotState &snap_state) { + dout(10) << remote_group_snap_id << dendl; + + auto itl = std::find_if( + m_local_group_snaps.begin(), m_local_group_snaps.end(), + [remote_group_snap_id](const cls::rbd::GroupSnapshot &s) { + return s.id == remote_group_snap_id; + }); + + if (itl != m_local_group_snaps.end() && + itl->state == cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE) { + dout(20) << "group snapshot: " << remote_group_snap_id << " already exists" + << dendl; + schedule_load_group_snapshots(); + return; + } + + auto requests_it = m_create_snap_requests.find(remote_group_snap_id); + if (requests_it == m_create_snap_requests.end()) { + requests_it = m_create_snap_requests.insert( + {remote_group_snap_id, {}}).first; + cls::rbd::GroupSnapshot local_snap = + {remote_group_snap_id, + cls::rbd::MirrorGroupSnapshotNamespace{ + snap_state, {}, m_remote_mirror_uuid, remote_group_snap_id}, + {}, cls::rbd::GROUP_SNAPSHOT_STATE_INCOMPLETE}; + local_snap.name = prepare_non_primary_mirror_snap_name(m_global_group_id, + remote_group_snap_id); + m_local_group_snaps.push_back(local_snap); + + auto comp = create_rados_callback( + new LambdaContext([this, remote_group_snap_id](int r) { + handle_create_mirror_snapshot(remote_group_snap_id, r); + })); + + librados::ObjectWriteOperation op; + librbd::cls_client::group_snap_set(&op, local_snap); + int r = m_local_io_ctx.aio_operate( + librbd::util::group_header_name(m_local_group_id), comp, &op); + ceph_assert(r == 0); + comp->release(); + } else { + schedule_load_group_snapshots(); + } +} + +template +void Replayer::handle_create_mirror_snapshot( + const std::string &remote_group_snap_id, int r) { + dout(10) << remote_group_snap_id << ", r=" << r << dendl; + + std::unique_lock locker{m_lock}; + auto snap_id = remote_group_snap_id; + auto itr = std::find_if( + m_remote_group_snaps.begin(), m_remote_group_snaps.end(), + [snap_id](const cls::rbd::GroupSnapshot &s) { + return s.id == snap_id; + }); + + if(itr != m_remote_group_snaps.end()) { + if (itr->snaps.size() == 0) { + dout(10) << "remote snap with no image snaps: " << snap_id << dendl; + m_pending_group_snaps[remote_group_snap_id].push_back({}); + } + } + locker.unlock(); + schedule_load_group_snapshots(); +} + +template +void Replayer::mirror_snapshot_complete( + const std::string &remote_group_snap_id, + cls::rbd::ImageSnapshotSpec *spec, + Context *on_finish) { + + ceph_assert(ceph_mutex_is_locked_by_me(m_lock)); + + auto itr = std::find_if( + m_remote_group_snaps.begin(), m_remote_group_snaps.end(), + [remote_group_snap_id](const cls::rbd::GroupSnapshot &s) { + return s.id == remote_group_snap_id; + }); + + ceph_assert(itr != m_remote_group_snaps.end()); + auto itl = std::find_if( + m_local_group_snaps.begin(), m_local_group_snaps.end(), + [remote_group_snap_id](const cls::rbd::GroupSnapshot &s) { + return s.id == remote_group_snap_id; + }); + if (itr->snaps.size() != 0) { + // update image snap + C_SaferCond *ctx = new C_SaferCond; + update_image_snapshot(remote_group_snap_id, *spec, ctx); + ctx->wait(); + + // update the group snap with snap spec + itl->snaps.push_back(*spec); + } + + if (itr->snaps.size() == itl->snaps.size()) { + m_create_snap_requests.erase(remote_group_snap_id); + m_pending_group_snaps.erase(remote_group_snap_id); + itl->state = cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE; + } + + dout(10) << "local group snap info: " + << "id: " << itl->id + << ", name: " << itl->name + << ", state: " << itl->state + << ", snaps.size: " << itl->snaps.size() + << dendl; + auto comp = create_rados_callback( + new LambdaContext([this, remote_group_snap_id, on_finish](int r) { + handle_mirror_snapshot_complete(r, remote_group_snap_id, on_finish); + })); + + librados::ObjectWriteOperation op; + librbd::cls_client::group_snap_set(&op, *itl); + int r = m_local_io_ctx.aio_operate( + librbd::util::group_header_name(m_local_group_id), comp, &op); + ceph_assert(r == 0); + comp->release(); +} + +template +void Replayer::handle_mirror_snapshot_complete( + int r, const std::string &remote_group_snap_id, Context *on_finish) { + dout(10) << remote_group_snap_id << ", r=" << r << dendl; + + auto itl = std::find_if( + m_local_group_snaps.begin(), m_local_group_snaps.end(), + [remote_group_snap_id](const cls::rbd::GroupSnapshot &s) { + return s.id == remote_group_snap_id; + }); + + if (itl->state != + cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE) { + on_finish->complete(0); + return; + } + unlink_group_snapshots(remote_group_snap_id); + on_finish->complete(0); +} + +template +void Replayer::unlink_group_snapshots( + const std::string &remote_group_snap_id) { + if (m_image_replayers->empty()) { + return; + } + dout(10) << dendl; + int r; + bool unlink_snap; + for (auto &snap : m_local_group_snaps) { + if (snap.id == remote_group_snap_id) { + break; + } + auto snap_type = cls::rbd::get_group_snap_namespace_type( + snap.snapshot_namespace); + if (snap_type == cls::rbd::GROUP_SNAPSHOT_NAMESPACE_TYPE_USER) { + bool unlink_user_snap = true; + for (auto &remote_snap : m_remote_group_snaps) { + if (remote_snap.name == snap.name) { + unlink_user_snap = false; + break; + } + } + if (!unlink_user_snap) { + continue; + } + dout(10) << "unlinking regular group snap in-progress: " + << snap.name << ", with id: " << snap.id << dendl; + } + dout(10) << "attempting to unlink image snaps from group snap: " + << snap.id << dendl; + unlink_snap = true; + for (auto &spec : snap.snaps) { + std::string image_header_oid = librbd::util::header_name(spec.image_id); + cls::rbd::SnapshotInfo snap_info; + r = librbd::cls_client::snapshot_get(&m_local_io_ctx, image_header_oid, + spec.snap_id, &snap_info); + if (r == -ENOENT) { + continue; + } else if (r < 0) { + derr << "failed getting snap info for snap id: " << spec.snap_id + << ", : " << cpp_strerror(r) << dendl; + } + unlink_snap = false; + for (auto it = m_image_replayers->begin(); + it != m_image_replayers->end(); ++it) { + auto image_replayer = it->second; + if (!image_replayer) { + continue; + } + auto local_image_id = image_replayer->get_local_image_id(); + if (local_image_id.empty() || local_image_id != spec.image_id) { + continue; + } + dout(10) << "pruning: " << spec.snap_id << dendl; + image_replayer->prune_snapshot(spec.snap_id); + break; + } + } + if (!unlink_snap) { + continue; + } + dout(10) << "all image snaps are pruned, finally unlinking group snap: " + << snap.id << dendl; + r = librbd::cls_client::group_snap_remove(&m_local_io_ctx, + librbd::util::group_header_name(m_local_group_id), snap.id); + if (r < 0) { + derr << "failed to remove group snapshot : " + << snap.id << " : " << cpp_strerror(r) << dendl; + } + } +} + +template +void Replayer::update_image_snapshot( + const std::string &remote_group_snap_id, + cls::rbd::ImageSnapshotSpec spec, + Context *on_finish) { + dout(10) << "local group snap info: " + << "image snap id: " << spec.snap_id + << ", image id: " << spec.image_id + << ", group snap id: " << remote_group_snap_id + << dendl; + std::string image_header_oid = librbd::util::header_name(spec.image_id); + cls::rbd::SnapshotInfo snap_info; + int r = librbd::cls_client::snapshot_get(&m_local_io_ctx, image_header_oid, + spec.snap_id, &snap_info); + if (r < 0) { + derr << "failed getting snap info for snap id: " << spec.snap_id + << ", : " << cpp_strerror(r) << dendl; + } + auto mirror_ns = std::get_if( + &snap_info.snapshot_namespace); + ceph_assert(mirror_ns != nullptr); + mirror_ns->group_spec = {m_local_group_id, spec.pool}; + mirror_ns->group_snap_id = remote_group_snap_id; + + // write to disk + librados::ObjectWriteOperation op; + librbd::cls_client::snapshot_add(&op, snap_info.id, snap_info.name, + *mirror_ns); + auto comp = create_rados_callback( + new LambdaContext([this, snap_info, on_finish](int r) { + handle_update_image_snapshot(r, snap_info.id, on_finish); + })); + r = m_local_io_ctx.aio_operate(image_header_oid, comp, &op); + ceph_assert(r == 0); + comp->release(); +} + +template +void Replayer::handle_update_image_snapshot( + int r, uint64_t local_snap_id, Context *on_finish) { + dout(10) << "snap id: " << local_snap_id << ", r=" << r << dendl; + on_finish->complete(r); +} + +template +void Replayer::mirror_regular_snapshot( + const std::string &remote_group_snap_name, + const std::string &remote_group_snap_id, + std::vector *local_images, + Context *on_finish) { + // each image will have one snapshot specific to group snap, and so for each + // image get a ImageSnapshotSpec and prepare a vector + // for image :: { + // * get snap whos name has group snap_id for that we can list snaps and + // filter with remote_group_snap_id + // * get its { pool_id, snap_id, image_id } + // } + // finally write to the object + dout(10) << remote_group_snap_id << dendl; + librados::ObjectWriteOperation op; + cls::rbd::GroupSnapshot group_snap{ + remote_group_snap_id, // keeping it same as remote group snap id + cls::rbd::UserGroupSnapshotNamespace{}, + remote_group_snap_name, + cls::rbd::GROUP_SNAPSHOT_STATE_INCOMPLETE}; + + std::vector local_image_snap_specs; + local_image_snap_specs = std::vector( + local_images->size(), cls::rbd::ImageSnapshotSpec()); + for (auto& image : *local_images) { + std::string image_header_oid = librbd::util::header_name( + image.spec.image_id); + ::SnapContext snapc; + int r = librbd::cls_client::get_snapcontext(&m_local_io_ctx, + image_header_oid, &snapc); + if (r < 0) { + derr << "get snap context failed: " << cpp_strerror(r) << dendl; + on_finish->complete(r); + return; + } + + auto image_snap_name = ".group." + std::to_string(image.spec.pool_id) + + "_" + m_remote_group_id + "_" + remote_group_snap_id; + // stored in reverse order + for (auto snap_id : snapc.snaps) { + cls::rbd::SnapshotInfo snap_info; + r = librbd::cls_client::snapshot_get(&m_local_io_ctx, image_header_oid, + snap_id, &snap_info); + if (r < 0) { + derr << "failed getting snap info for snap id: " << snap_id + << ", : " << cpp_strerror(r) << dendl; + on_finish->complete(r); + return; + } + + // extract { pool_id, snap_id, image_id } + if (snap_info.name == image_snap_name) { + cls::rbd::ImageSnapshotSpec snap_spec; + snap_spec.pool = image.spec.pool_id; + snap_spec.image_id = image.spec.image_id; + snap_spec.snap_id = snap_info.id; + + local_image_snap_specs.push_back(snap_spec); + } + } + } + + group_snap.snaps = local_image_snap_specs; + group_snap.state = cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE; + librbd::cls_client::group_snap_set(&op, group_snap); + m_local_group_snaps.push_back(group_snap); + + auto comp = create_rados_callback( + new LambdaContext([this, on_finish](int r) { + handle_mirror_regular_snapshot(r, on_finish); + })); + int r = m_local_io_ctx.aio_operate( + librbd::util::group_header_name(m_local_group_id), comp, &op); + ceph_assert(r == 0); + comp->release(); +} + +template +void Replayer::handle_mirror_regular_snapshot( + int r, Context *on_finish) { + dout(10) << "r=" << r << dendl; + + if (r < 0) { + derr << "error creating local non-primary group snapshot: " + << cpp_strerror(r) << dendl; + } + on_finish->complete(0); + + schedule_load_group_snapshots(); +} + +template +void Replayer::shut_down(Context* on_finish) { + dout(10) << dendl; + + std::unique_lock locker{m_lock}; + m_on_shutdown = on_finish; + auto state = STATE_COMPLETE; + std::swap(m_state, state); + + if (state == STATE_REPLAYING) { + // if there are any pending snaps + dout(10) << "shut down pending" << dendl; + return; + } + locker.unlock(); + m_on_shutdown->complete(0); + return; +} + + +} // namespace group_replayer +} // namespace mirror +} // namespace rbd + +template class rbd::mirror::group_replayer::Replayer; diff --git a/src/tools/rbd_mirror/group_replayer/Replayer.h b/src/tools/rbd_mirror/group_replayer/Replayer.h new file mode 100644 index 0000000000000..6d11128bc700a --- /dev/null +++ b/src/tools/rbd_mirror/group_replayer/Replayer.h @@ -0,0 +1,164 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef RBD_MIRROR_GROUP_REPLAYER_REPLAYER_H +#define RBD_MIRROR_GROUP_REPLAYER_REPLAYER_H + +#include "tools/rbd_mirror/image_replayer/Replayer.h" +#include "common/ceph_mutex.h" +#include "cls/rbd/cls_rbd_types.h" +#include "include/rados/librados.hpp" +#include "librbd/mirror/snapshot/Types.h" +#include "tools/rbd_mirror/image_replayer/Types.h" +#include + +class Context; +namespace librbd { class ImageCtx; } + +namespace rbd { +namespace mirror { + +template class ImageReplayer; +class PoolMetaCache; +template struct Threads; + +namespace group_replayer { + +template +class Replayer { +public: + static Replayer* create( + Threads* threads, + librados::IoCtx &local_io_ctx, + librados::IoCtx &remote_io_ctx, + const std::string &global_group_id, + const std::string& local_mirror_uuid, + const std::string& remote_mirror_uuid, + PoolMetaCache* pool_meta_cache, + std::string local_group_id, + std::string remote_group_id, + std::list *>> *image_replayers) { + return new Replayer(threads, local_io_ctx, remote_io_ctx, global_group_id, + local_mirror_uuid, remote_mirror_uuid, pool_meta_cache, local_group_id, + remote_group_id, image_replayers); + } + + Replayer( + Threads* threads, + librados::IoCtx &local_io_ctx, + librados::IoCtx &remote_io_ctx, + const std::string &global_group_id, + const std::string& local_mirror_uuid, + const std::string& remote_mirror_uuid, + PoolMetaCache* pool_meta_cache, + std::string local_group_id, + std::string remote_group_id, + std::list *>> *image_replayers); + ~Replayer(); + + void destroy() { + delete this; + } + void init(Context* on_finish); + void shut_down(Context* on_finish); + + bool is_replaying() const { + std::unique_lock locker{m_lock}; + return (m_state == STATE_REPLAYING || m_state == STATE_IDLE); + } + + void notify_group_snap_image_complete( + int64_t local_pool_id, + const std::string &local_image_id, + const std::string &remote_group_snap_id, + uint64_t local_snap_id); + +private: + enum State { + STATE_INIT, + STATE_REPLAYING, + STATE_IDLE, + STATE_COMPLETE + }; + + Threads *m_threads; + librados::IoCtx &m_local_io_ctx; + librados::IoCtx &m_remote_io_ctx; + std::string m_global_group_id; + std::string m_local_mirror_uuid; + std::string m_remote_mirror_uuid; + PoolMetaCache* m_pool_meta_cache; + std::string m_local_group_id; + std::string m_remote_group_id; + std::list *>> *m_image_replayers; + + mutable ceph::mutex m_lock; + + State m_state = STATE_INIT; + Context* m_on_shutdown = nullptr; + std::string m_remote_mirror_peer_uuid; + + std::vector m_local_group_snaps; + std::vector m_remote_group_snaps; + + // map of > + std::map> m_create_snap_requests; + + // map of >> + std::map>> m_pending_group_snaps; + + int local_group_image_list_by_id( + std::vector *image_ids); + + void schedule_load_group_snapshots(); + + void load_local_group_snapshots(); + void handle_load_local_group_snapshots(int r); + + void load_remote_group_snapshots(); + void handle_load_remote_group_snapshots(int r); + + void scan_for_unsynced_group_snapshots(std::unique_lock& locker); + + void try_create_group_snapshot(cls::rbd::GroupSnapshot snap); + + void create_mirror_snapshot( + const std::string &remote_group_snap_id, + const cls::rbd::MirrorSnapshotState &snap_state); + void handle_create_mirror_snapshot( + const std::string &remote_group_snap_id, int r); + + std::string prepare_non_primary_mirror_snap_name( + const std::string &global_group_id, const std::string &snap_id); + + void mirror_snapshot_complete( + const std::string &remote_group_snap_id, + cls::rbd::ImageSnapshotSpec *spec, + Context *on_finish); + void handle_mirror_snapshot_complete( + int r, const std::string &remote_group_snap_id, Context *on_finish); + + void unlink_group_snapshots(const std::string &remote_group_snap_id); + + void update_image_snapshot( + const std::string &remote_group_snap_id, + cls::rbd::ImageSnapshotSpec spec, + Context *on_finish); + void handle_update_image_snapshot( + int r, uint64_t local_snap_id, Context *on_finish); + + void mirror_regular_snapshot( + const std::string &remote_group_snap_name, + const std::string &remote_group_snap_id, + std::vector *local_images, + Context *on_finish); + void handle_mirror_regular_snapshot(int r, Context *on_finish); +}; + +} // namespace group_replayer +} // namespace mirror +} // namespace rbd + +extern template class rbd::mirror::group_replayer::Replayer; + +#endif // RBD_MIRROR_GROUP_REPLAYER_REPLAYER_H diff --git a/src/tools/rbd_mirror/image_replayer/Replayer.h b/src/tools/rbd_mirror/image_replayer/Replayer.h index f3bfa4da04b0d..80083eac26ad9 100644 --- a/src/tools/rbd_mirror/image_replayer/Replayer.h +++ b/src/tools/rbd_mirror/image_replayer/Replayer.h @@ -4,6 +4,7 @@ #ifndef RBD_MIRROR_IMAGE_REPLAYER_REPLAYER_H #define RBD_MIRROR_IMAGE_REPLAYER_REPLAYER_H +#include #include struct Context; @@ -30,6 +31,8 @@ struct Replayer { virtual int get_error_code() const = 0; virtual std::string get_error_description() const = 0; + + virtual void prune_snapshot(uint64_t) = 0; }; } // namespace image_replayer diff --git a/src/tools/rbd_mirror/image_replayer/ReplayerListener.h b/src/tools/rbd_mirror/image_replayer/ReplayerListener.h index bb5dadb26de07..4cb1bc2431d8b 100644 --- a/src/tools/rbd_mirror/image_replayer/ReplayerListener.h +++ b/src/tools/rbd_mirror/image_replayer/ReplayerListener.h @@ -12,15 +12,11 @@ struct ReplayerListener { virtual ~ReplayerListener() {} virtual void handle_notification() = 0; - virtual void list_remote_group_snapshots(Context *on_finish) = 0; - - virtual void create_mirror_snapshot_start( - const cls::rbd::MirrorSnapshotNamespace &remote_group_snap_ns, - int64_t *local_group_pool_id, std::string *local_group_id, - std::string *local_group_snap_id, Context *on_finish) = 0; - virtual void create_mirror_snapshot_finish( - const std::string &remote_group_snap_id, uint64_t snap_id, - Context *on_finish) = 0; + virtual void notify_group_snap_image_complete( + int64_t local_pool_id, + const std::string &local_image_id, + const std::string &remote_group_snap_id, + uint64_t local_snap_id) = 0; }; } // namespace image_replayer diff --git a/src/tools/rbd_mirror/image_replayer/journal/Replayer.h b/src/tools/rbd_mirror/image_replayer/journal/Replayer.h index 6b1f36d9c7fe9..0f8f978962ad2 100644 --- a/src/tools/rbd_mirror/image_replayer/journal/Replayer.h +++ b/src/tools/rbd_mirror/image_replayer/journal/Replayer.h @@ -97,6 +97,9 @@ public: return m_image_spec; } + void prune_snapshot(uint64_t snap_id) { + } + private: /** * @verbatim diff --git a/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.cc b/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.cc index 4695ad0f6c63b..18bfe032a48b4 100644 --- a/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.cc +++ b/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.cc @@ -463,8 +463,6 @@ void Replayer::scan_local_mirror_snapshots( m_remote_snap_id_end = CEPH_NOSNAP; m_remote_mirror_snap_ns = {}; - std::set prune_snap_ids; - auto local_image_ctx = m_state_builder->local_image_ctx; std::shared_lock image_locker{local_image_ctx->image_lock}; for (auto snap_info_it = local_image_ctx->snap_info.begin(); @@ -487,10 +485,12 @@ void Replayer::scan_local_mirror_snapshots( m_local_snap_id_start = local_snap_id; ceph_assert(m_local_snap_id_end == CEPH_NOSNAP); - if (mirror_ns->mirror_peer_uuids.empty()) { + if (mirror_ns->mirror_peer_uuids.empty() && + (!mirror_ns->group_spec.is_valid() && + mirror_ns->group_snap_id.empty())) { // no other peer will attempt to sync to this snapshot so store as // a candidate for removal - prune_snap_ids.insert(local_snap_id); + m_prune_snap_ids.insert(local_snap_id); } } else if (mirror_ns->last_copied_object_number == 0 && m_local_snap_id_start > 0) { @@ -501,8 +501,11 @@ void Replayer::scan_local_mirror_snapshots( // the first non-primary snapshot since we know its snapshot is // well-formed because otherwise the mirror-image-state would have // forced an image deletion. - prune_snap_ids.clear(); - prune_snap_ids.insert(local_snap_id); + if(!mirror_ns->group_spec.is_valid() && + mirror_ns->group_snap_id.empty()) { + m_prune_snap_ids.clear(); + m_prune_snap_ids.insert(local_snap_id); + } break; } else { // start snap will be last complete mirror snapshot or initial @@ -531,15 +534,14 @@ void Replayer::scan_local_mirror_snapshots( if (m_local_snap_id_start > 0) { // remove candidate that is required for delta snapshot sync - prune_snap_ids.erase(m_local_snap_id_start); + m_prune_snap_ids.erase(m_local_snap_id_start); } - if (!prune_snap_ids.empty()) { + if (!m_prune_snap_ids.empty()) { locker->unlock(); - m_prune_snap_id = *prune_snap_ids.begin(); - dout(5) << "pruning unused non-primary snapshot " << m_prune_snap_id << dendl; - prune_non_primary_snapshot(); - //unlink_group_snapshot(); //PK: FIXME + auto prune_snap_id = *m_prune_snap_ids.begin(); + dout(5) << "pruning unused non-primary snapshot " << prune_snap_id << dendl; + prune_non_primary_snapshot(prune_snap_id); return; } @@ -772,75 +774,8 @@ void Replayer::scan_remote_mirror_snapshots( } template -void Replayer::unlink_group_snapshot() { - auto local_image_ctx = m_state_builder->local_image_ctx; - cls::rbd::SnapshotNamespace snap_namespace; - std::string snap_name; - int r = 0; - { - std::shared_lock image_locker{local_image_ctx->image_lock}; - - auto snap_info = local_image_ctx->get_snap_info(m_prune_snap_id); - if (!snap_info) { - r = -ENOENT; - } else { - snap_namespace = snap_info->snap_namespace; - snap_name = snap_info->name; - } - } - - if (r == -ENOENT) { - dout(15) << "failed to locate snapshot " << m_prune_snap_id << dendl; - prune_non_primary_snapshot(); - return; - } - - auto info = std::get_if(&snap_namespace); - if (!info->group_spec.is_valid()) { - prune_non_primary_snapshot(); - return; - } - - dout(15) << "image_snap_id=" << m_prune_snap_id << dendl; - - r = librbd::util::create_ioctx(local_image_ctx->md_ctx, "group", - info->group_spec.pool_id, {}, &m_group_io_ctx); - if (r < 0) { - prune_non_primary_snapshot(); - return; - } - - librados::ObjectWriteOperation op; - cls::rbd::ImageSnapshotSpec image_snap = {local_image_ctx->md_ctx.get_id(), - local_image_ctx->id, - m_prune_snap_id}; - librbd::cls_client::group_snap_unlink(&op, info->group_snap_id, image_snap); - auto aio_comp = create_rados_callback< - Replayer, - &Replayer::handle_unlink_group_snapshot>(this); - r = m_group_io_ctx.aio_operate( - librbd::util::group_header_name(info->group_spec.group_id), aio_comp, &op); - ceph_assert(r == 0); - aio_comp->release(); -} - -template -void Replayer::handle_unlink_group_snapshot(int r) { - dout(15) << "r=" << r << dendl; - - if (r < 0 && r != -ENOENT) { - derr << "failed to unlink group snapshot: " << cpp_strerror(r) - << dendl; - handle_replay_complete(r, "failed to unlink group snapshot"); - return; - } - - prune_non_primary_snapshot(); -} - -template -void Replayer::prune_non_primary_snapshot() { - dout(10) << "snap_id=" << m_prune_snap_id << dendl; +void Replayer::prune_non_primary_snapshot(uint64_t snap_id) { + dout(10) << "snap_id=" << snap_id << dendl; auto local_image_ctx = m_state_builder->local_image_ctx; bool snap_valid = false; @@ -849,14 +784,11 @@ void Replayer::prune_non_primary_snapshot() { { std::shared_lock image_locker{local_image_ctx->image_lock}; - auto snap_info = local_image_ctx->get_snap_info(m_prune_snap_id); + auto snap_info = local_image_ctx->get_snap_info(snap_id); if (snap_info != nullptr) { snap_valid = true; snap_namespace = snap_info->snap_namespace; snap_name = snap_info->name; - - ceph_assert(std::holds_alternative( - snap_namespace)); } } @@ -864,6 +796,7 @@ void Replayer::prune_non_primary_snapshot() { load_local_image_meta(); return; } + m_prune_snap_ids.erase(snap_id); auto ctx = create_context_callback< Replayer, &Replayer::handle_prune_non_primary_snapshot>(this); @@ -952,7 +885,7 @@ void Replayer::handle_get_remote_image_state(int r) { return; } - refresh_remote_group_snapshot_list(); + create_non_primary_snapshot(); } template @@ -982,74 +915,6 @@ void Replayer::handle_get_local_image_state(int r) { request_sync(); } -template -void Replayer::refresh_remote_group_snapshot_list() { - if (!m_remote_mirror_snap_ns.group_spec.is_valid() || - m_remote_mirror_snap_ns.group_snap_id.empty()) { - create_non_primary_snapshot(); - return; - } - - dout(10) << dendl; - - auto ctx = new LambdaContext( - [this](int r) { - handle_refresh_remote_group_snapshot_list(r); - }); - - m_replayer_listener->list_remote_group_snapshots(ctx); -} - -template -void Replayer::handle_refresh_remote_group_snapshot_list(int r) { - dout(10) << "r=" << r << dendl; - - if (r < 0) { - dout(15) << "restarting replayer" << dendl; - load_local_image_meta(); - return; - } - - create_group_snap_start(); -} - -template -void Replayer::create_group_snap_start() { - dout(10) << dendl; - - auto ctx = create_context_callback< - Replayer, &Replayer::handle_create_group_snap_start>(this); - - m_replayer_listener->create_mirror_snapshot_start( - m_remote_mirror_snap_ns, &m_local_group_pool_id, - &m_local_group_id, &m_local_group_snap_id, ctx); -} - -template -void Replayer::handle_create_group_snap_start(int r) { - dout(10) << "r=" << r << dendl; - - if (r < 0 && r != -EEXIST) { - if (r == -EAGAIN) { - auto ctx = new LambdaContext( - [this](int r) { - // retry after 1 sec - refresh_remote_group_snapshot_list(); - }); - std::lock_guard timer_locker{m_threads->timer_lock}; - m_threads->timer->add_event_after(1, ctx); - } else if (r == -ESTALE) { - dout(15) << "waiting for shut down" << dendl; - handle_replay_complete(r, "waiting for shut down"); - } else { - derr << "failed to create group snapshot: " << cpp_strerror(r) << dendl; - handle_replay_complete(r, "failed to create group snapshot"); - } - return; - } - - create_non_primary_snapshot(); -} template void Replayer::create_non_primary_snapshot() { @@ -1128,8 +993,7 @@ void Replayer::create_non_primary_snapshot() { auto req = librbd::mirror::snapshot::CreateNonPrimaryRequest::create( local_image_ctx, m_remote_mirror_snap_ns.is_demoted(), m_state_builder->remote_mirror_uuid, m_remote_snap_id_end, - m_local_mirror_snap_ns.snap_seqs, m_local_group_pool_id, m_local_group_id, - m_local_group_snap_id, m_image_state, &m_local_snap_id_end, ctx); + m_local_mirror_snap_ns.snap_seqs, m_image_state, &m_local_snap_id_end, ctx); req->send(); } @@ -1146,36 +1010,6 @@ void Replayer::handle_create_non_primary_snapshot(int r) { dout(15) << "local_snap_id_end=" << m_local_snap_id_end << dendl; - create_group_snap_finish(); -} - -template -void Replayer::create_group_snap_finish() { - if (!m_remote_mirror_snap_ns.group_spec.is_valid() || - m_remote_mirror_snap_ns.group_snap_id.empty()) { - update_mirror_image_state(); - return; - } - - dout(10) << dendl; - - auto ctx = create_context_callback< - Replayer, &Replayer::handle_create_group_snap_finish>(this); - - m_replayer_listener->create_mirror_snapshot_finish( - m_remote_mirror_snap_ns.group_snap_id, m_local_snap_id_end, ctx); -} - -template -void Replayer::handle_create_group_snap_finish(int r) { - dout(10) << "r=" << r << dendl; - - if (r < 0 && r != -EEXIST) { - derr << "failed to create group snapshot: " << cpp_strerror(r) << dendl; - handle_replay_complete(r, "failed to create group snapshot"); - return; - } - update_mirror_image_state(); } @@ -1374,6 +1208,28 @@ void Replayer::handle_apply_image_state(int r) { return; } + notify_group_snap_image_complete(); +} + +template +void Replayer::notify_group_snap_image_complete() { + if (!m_remote_mirror_snap_ns.group_spec.is_valid() || + m_remote_mirror_snap_ns.group_snap_id.empty()) { + std::unique_lock locker{m_lock}; + update_non_primary_snapshot(true); + return; + } + + dout(10) << "image_id=" << m_state_builder->local_image_ctx->id + << ", remote_group_snap_id=" << m_remote_mirror_snap_ns.group_snap_id + << ", local_image_snap_id=" << m_local_snap_id_end << dendl; + + m_replayer_listener->notify_group_snap_image_complete( + m_state_builder->local_image_ctx->md_ctx.get_id(), + m_state_builder->local_image_ctx->id, + m_remote_mirror_snap_ns.group_snap_id, + m_local_snap_id_end); + std::unique_lock locker{m_lock}; update_non_primary_snapshot(true); } diff --git a/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.h b/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.h index da066669a9fcb..57e32a15cae87 100644 --- a/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.h +++ b/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.h @@ -8,7 +8,6 @@ #include "common/ceph_mutex.h" #include "common/AsyncOpTracker.h" #include "cls/rbd/cls_rbd_types.h" -#include "include/rados/librados.hpp" #include "librbd/mirror/snapshot/Types.h" #include "tools/rbd_mirror/image_replayer/TimeRollingMean.h" #include @@ -100,6 +99,11 @@ public: return m_image_spec; } + void prune_snapshot(uint64_t snap_id) { + std::unique_lock locker(m_lock); + m_prune_snap_ids.insert(snap_id); + } + private: /** * @verbatim @@ -122,10 +126,7 @@ private: * REFRESH_REMOTE_IMAGE | * | | * | (unused non-primary snapshot) | - * |\--------------> UNLINK_GROUP_SNAPSHOT | - * | | (skip if no group) | - * | v | - * | PRUNE_NON_PRIMARY_SNAPSHOT---/| + * |\--------------> PRUNE_NON_PRIMARY_SNAPSHOT---/| * | | * | (interrupted sync) | * |\--------------> GET_LOCAL_IMAGE_STATE ------\ | @@ -264,6 +265,7 @@ private: utime_t m_snapshot_replay_start; uint32_t m_pending_snapshots = 0; + std::set m_prune_snap_ids; bool m_remote_image_updated = false; bool m_updating_sync_point = false; @@ -271,9 +273,6 @@ private: PerfCounters *m_perf_counters = nullptr; - uint64_t m_prune_snap_id = CEPH_NOSNAP; - librados::IoCtx m_group_io_ctx; - void load_local_image_meta(); void handle_load_local_image_meta(int r); @@ -286,10 +285,7 @@ private: void scan_local_mirror_snapshots(std::unique_lock* locker); void scan_remote_mirror_snapshots(std::unique_lock* locker); - void unlink_group_snapshot(); - void handle_unlink_group_snapshot(int r); - - void prune_non_primary_snapshot(); + void prune_non_primary_snapshot(uint64_t snap_id); void handle_prune_non_primary_snapshot(int r); void copy_snapshots(); @@ -301,18 +297,9 @@ private: void get_local_image_state(); void handle_get_local_image_state(int r); - void refresh_remote_group_snapshot_list(); - void handle_refresh_remote_group_snapshot_list(int r); - - void create_group_snap_start(); - void handle_create_group_snap_start(int r); - void create_non_primary_snapshot(); void handle_create_non_primary_snapshot(int r); - void create_group_snap_finish(); - void handle_create_group_snap_finish(int r); - void update_mirror_image_state(); void handle_update_mirror_image_state(int r); @@ -331,6 +318,8 @@ private: void update_non_primary_snapshot(bool complete); void handle_update_non_primary_snapshot(bool complete, int r); + void notify_group_snap_image_complete(); + void notify_image_update(); void handle_notify_image_update(int r);