From aa7c58058a29bff28c287000c5da73b75f01ec00 Mon Sep 17 00:00:00 2001 From: Prasanna Kumar Kalever Date: Sun, 21 Jul 2024 21:22:12 +0530 Subject: [PATCH] rbd-mirror: remove group snaps on primary at snapshot creation Signed-off-by: Prasanna Kumar Kalever --- src/librbd/CMakeLists.txt | 1 + src/librbd/api/Mirror.cc | 36 ++++ src/librbd/group/UnlinkPeerGroupRequest.cc | 171 ++++++++++++++++++ src/librbd/group/UnlinkPeerGroupRequest.h | 61 +++++++ .../mirror/snapshot/UnlinkPeerRequest.cc | 53 +----- .../mirror/snapshot/UnlinkPeerRequest.h | 6 - .../rbd_mirror/group_replayer/Replayer.cc | 27 ++- 7 files changed, 296 insertions(+), 59 deletions(-) create mode 100644 src/librbd/group/UnlinkPeerGroupRequest.cc create mode 100644 src/librbd/group/UnlinkPeerGroupRequest.h diff --git a/src/librbd/CMakeLists.txt b/src/librbd/CMakeLists.txt index b787b72bd4263..62c234b733fda 100644 --- a/src/librbd/CMakeLists.txt +++ b/src/librbd/CMakeLists.txt @@ -80,6 +80,7 @@ set(librbd_internal_srcs group/ListSnapshotsRequest.cc group/AddImageRequest.cc group/RemoveImageRequest.cc + group/UnlinkPeerGroupRequest.cc image/AttachChildRequest.cc image/AttachParentRequest.cc image/CloneRequest.cc diff --git a/src/librbd/api/Mirror.cc b/src/librbd/api/Mirror.cc index 22f09a691a90c..61b68582766e4 100644 --- a/src/librbd/api/Mirror.cc +++ b/src/librbd/api/Mirror.cc @@ -22,6 +22,7 @@ #include "librbd/api/Namespace.h" #include "librbd/api/Utils.h" #include "librbd/group/ListSnapshotsRequest.h" +#include "librbd/group/UnlinkPeerGroupRequest.h" #include "librbd/mirror/DemoteRequest.h" #include "librbd/mirror/DisableRequest.h" #include "librbd/mirror/EnableRequest.h" @@ -3035,6 +3036,13 @@ int Mirror::group_promote(IoCtx& group_ioctx, const char *group_name, util::notify_unquiesce(image_ctxs, quiesce_requests); } + if (!ret_code) { + C_SaferCond cond; + auto req = group::UnlinkPeerGroupRequest::create( + group_ioctx, group_id, &image_ctxs, &cond); + req->send(); + cond.wait(); + } close_images(&image_ctxs); return ret_code; @@ -3191,6 +3199,13 @@ int Mirror::group_demote(IoCtx& group_ioctx, util::notify_unquiesce(image_ctxs, quiesce_requests); } + if (!ret_code) { + C_SaferCond cond; + auto req = group::UnlinkPeerGroupRequest::create( + group_ioctx, group_id, &image_ctxs, &cond); + req->send(); + cond.wait(); + } close_images(&image_ctxs); return ret_code; @@ -3387,6 +3402,13 @@ int Mirror::group_snapshot_create(IoCtx& group_ioctx, const char *group_name, util::notify_unquiesce(image_ctxs, quiesce_requests); } + if (!ret_code) { + C_SaferCond cond; + auto req = group::UnlinkPeerGroupRequest::create( + group_ioctx, group_id, &image_ctxs, &cond); + req->send(); + cond.wait(); + } close_images(&image_ctxs); return ret_code; @@ -3495,6 +3517,13 @@ int Mirror::group_image_add(IoCtx &group_ioctx, util::notify_unquiesce(image_ctxs, quiesce_requests); } + if (!ret_code) { + C_SaferCond cond; + auto req = group::UnlinkPeerGroupRequest::create( + group_ioctx, group_id, &image_ctxs, &cond); + req->send(); + cond.wait(); + } close_images(&image_ctxs); if (!ret_code) { @@ -3627,6 +3656,13 @@ int Mirror::group_image_remove(IoCtx &group_ioctx, util::notify_unquiesce(image_ctxs, quiesce_requests); } + if (!ret_code) { + C_SaferCond cond; + auto req = group::UnlinkPeerGroupRequest::create( + group_ioctx, group_id, &image_ctxs, &cond); + req->send(); + cond.wait(); + } close_images(&image_ctxs); if (!ret_code) { diff --git a/src/librbd/group/UnlinkPeerGroupRequest.cc b/src/librbd/group/UnlinkPeerGroupRequest.cc new file mode 100644 index 0000000000000..c04a319308e34 --- /dev/null +++ b/src/librbd/group/UnlinkPeerGroupRequest.cc @@ -0,0 +1,171 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "include/ceph_assert.h" +#include "common/Cond.h" +#include "common/dout.h" +#include "common/errno.h" +#include "common/ceph_context.h" +#include "cls/rbd/cls_rbd_client.h" +#include "librbd/Operations.h" +#include "librbd/Utils.h" +#include "librbd/api/Utils.h" +#include "librbd/api/Group.h" +#include "librbd/group/ListSnapshotsRequest.h" +#include "librbd/group/UnlinkPeerGroupRequest.h" +#include "librbd/mirror/snapshot/UnlinkPeerRequest.h" + +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "librbd::group::UnlinkPeerGroupRequest: " << this \ + << " " << __func__ << ": " + +namespace librbd { +namespace group { + +using util::create_rados_callback; +using util::create_context_callback; + +template +void UnlinkPeerGroupRequest::send() { + CephContext *cct = (CephContext *)m_group_io_ctx.cct(); + ldout(cct, 10) << dendl; + unlink_peer(); +} + +template +void UnlinkPeerGroupRequest::unlink_peer() { + CephContext *cct = (CephContext *)m_group_io_ctx.cct(); + ldout(cct, 10) << dendl; + + std::vector snaps; + C_SaferCond cond; + auto req = group::ListSnapshotsRequest<>::create(m_group_io_ctx, m_group_id, + true, true, &snaps, &cond); + req->send(); + cond.wait(); + uint64_t count = 0; + auto unlink_snap = snaps.end(); + auto unlink_unsynced_snap = snaps.end(); + bool unlink_unsynced = false; + for (auto it = snaps.begin(); it != snaps.end(); it++) { + auto ns = std::get_if( + &it->snapshot_namespace); + if (ns != nullptr) { + if (ns->state != cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY) { + continue; + } + count++; + if (count == 3) { + unlink_unsynced_snap = it; + } + ceph_assert(count <= 5); + + if (ns->mirror_peer_uuids.empty()) { + auto next_snap = std::next(it); + if (next_snap != snaps.end()) { + unlink_snap = it; + break; + } + } + } + // TODO: fix the hardcoded max_snaps value + if (count == 5) { + unlink_unsynced = true; + } + } + + if (unlink_snap != snaps.end()) { + remove_group_snapshot(*unlink_snap); + } else if (unlink_unsynced && unlink_unsynced_snap != snaps.end()) { + remove_group_snapshot(*unlink_unsynced_snap); + } else { + finish(0); + } +} + +template +void UnlinkPeerGroupRequest::remove_group_snapshot( + cls::rbd::GroupSnapshot group_snap) { + CephContext *cct = (CephContext *)m_group_io_ctx.cct(); + ldout(cct, 10) << dendl; + + for (auto &snap : group_snap.snaps) { + if (snap.snap_id == CEPH_NOSNAP) { + continue; + } + ImageCtx *ictx = nullptr; + for (size_t i = 0; i < m_image_ctxs->size(); ++i) { + ictx = (*m_image_ctxs)[i]; + if (ictx->id != snap.image_id) { + ictx = nullptr; + } else { + break; + } + } + if (!ictx) { + continue; + } + ldout(cct, 10) << "removing individual snapshot: " + << snap.snap_id << ", from image id:" << snap.image_id << dendl; + remove_image_snapshot(ictx, snap.snap_id); + } + + int r = cls_client::group_snap_remove(&m_group_io_ctx, + librbd::util::group_header_name(m_group_id), group_snap.id); + if (r < 0) { + lderr(cct) << "failed to remove group snapshot metadata: " + << cpp_strerror(r) << dendl; + } + + unlink_peer(); +} + +template +void UnlinkPeerGroupRequest::remove_image_snapshot( + ImageCtx *image_ctx, uint64_t snap_id) { + CephContext *cct = (CephContext *)m_group_io_ctx.cct(); + ldout(cct, 10) << snap_id << dendl; + + image_ctx->image_lock.lock_shared(); + int r = -ENOENT; + cls::rbd::SnapshotNamespace snap_namespace; + std::string snap_name; + for (auto snap_it = image_ctx->snap_info.find(snap_id); + snap_it != image_ctx->snap_info.end(); ++snap_it) { + if (snap_it->first == snap_id) { + r = 0; + snap_namespace = snap_it->second.snap_namespace; + snap_name = snap_it->second.name; + } + } + + if (r == -ENOENT) { + ldout(cct, 10) << "missing snapshot: snap_id=" << snap_id << dendl; + image_ctx->image_lock.unlock_shared(); + return; + } + + auto mirror_ns = std::get_if( + &snap_namespace); + if (mirror_ns == nullptr) { + lderr(cct) << "not mirror snapshot (snap_id=" << snap_id << ")" << dendl; + image_ctx->image_lock.unlock_shared(); + return; + } + image_ctx->image_lock.unlock_shared(); + image_ctx->operations->snap_remove(snap_namespace, snap_name.c_str()); +} + +template +void UnlinkPeerGroupRequest::finish(int r) { + CephContext *cct = (CephContext *)m_group_io_ctx.cct(); + ldout(cct, 10) << "r=" << r << dendl; + + m_on_finish->complete(r); +} + +} // namespace group +} // namespace librbd + +template class librbd::group::UnlinkPeerGroupRequest; diff --git a/src/librbd/group/UnlinkPeerGroupRequest.h b/src/librbd/group/UnlinkPeerGroupRequest.h new file mode 100644 index 0000000000000..b211a9ef7b652 --- /dev/null +++ b/src/librbd/group/UnlinkPeerGroupRequest.h @@ -0,0 +1,61 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_UNLINK_PEER_GROUP_REQUST_H +#define CEPH_LIBRBD_UNLINK_PEER_GROUP_REQUST_H + +#include "include/int_types.h" +#include "include/types.h" +#include "include/rados/librados.hpp" +#include "cls/rbd/cls_rbd_types.h" + +#include +#include + +class Context; + +namespace librbd { + +struct ImageCtx; + +namespace group { + +template +class UnlinkPeerGroupRequest { +public: + static UnlinkPeerGroupRequest *create( + librados::IoCtx &group_io_ctx, const std::string &group_id, + std::vector *image_ctxs, + Context *on_finish) { + return new UnlinkPeerGroupRequest(group_io_ctx, group_id, + image_ctxs, on_finish); + } + + UnlinkPeerGroupRequest(librados::IoCtx &group_io_ctx, + const std::string &group_id, + std::vector *image_ctxs, + Context *on_finish) + : m_group_io_ctx(group_io_ctx), m_group_id(group_id), + m_image_ctxs(image_ctxs), m_on_finish(on_finish) { + } + + void send(); + +private: + librados::IoCtx &m_group_io_ctx; + const std::string m_group_id; + std::vector *m_image_ctxs; + Context *m_on_finish; + + void unlink_peer(); + void remove_group_snapshot(cls::rbd::GroupSnapshot group_snap); + void remove_image_snapshot(ImageCtx *image_ctx, uint64_t snap_id); + void finish(int r); +}; + +} // namespace group +} // namespace librbd + +extern template class librbd::group::UnlinkPeerGroupRequest; + +#endif // CEPH_LIBRBD_UNLINK_PEER_GROUP_REQUST_H diff --git a/src/librbd/mirror/snapshot/UnlinkPeerRequest.cc b/src/librbd/mirror/snapshot/UnlinkPeerRequest.cc index adb431f0f1320..35313f6277981 100644 --- a/src/librbd/mirror/snapshot/UnlinkPeerRequest.cc +++ b/src/librbd/mirror/snapshot/UnlinkPeerRequest.cc @@ -103,7 +103,7 @@ void UnlinkPeerRequest::unlink_peer() { have_newer_mirror_snapshot) { if (m_allow_remove) { m_image_ctx->image_lock.unlock_shared(); - unlink_group_snapshot(snap_namespace, snap_name); + remove_snapshot(snap_namespace, snap_name); return; } else { ldout(cct, 15) << "skipping removal of snapshot: snap_id=" << m_snap_id @@ -187,57 +187,6 @@ void UnlinkPeerRequest::handle_notify_update(int r) { refresh_image(); } -template -void UnlinkPeerRequest::unlink_group_snapshot( - const cls::rbd::SnapshotNamespace& snap_namespace, - const std::string& snap_name) { - CephContext *cct = m_image_ctx->cct; - ldout(cct, 15) << "snap_name: " << snap_name << dendl; - - auto info = std::get_if(&snap_namespace); - if (!info->group_spec.is_valid()) { - remove_snapshot(snap_namespace, snap_name); - return; - } - - int r = util::create_ioctx(m_image_ctx->md_ctx, "group", info->group_spec.pool_id, - {}, &m_group_io_ctx); - if (r < 0) { - remove_snapshot(snap_namespace, snap_name); - return; - } - - librados::ObjectWriteOperation op; - cls::rbd::ImageSnapshotSpec image_snap = {m_image_ctx->md_ctx.get_id(), - m_image_ctx->id, m_snap_id}; - librbd::cls_client::group_snap_unlink(&op, info->group_snap_id, image_snap); - auto ctx = new LambdaContext([this, snap_namespace, snap_name](int r) { - handle_unlink_group_snapshot(snap_namespace, snap_name, r); - }); - auto aio_comp = create_rados_callback(ctx); - r = m_group_io_ctx.aio_operate( - util::group_header_name(info->group_spec.group_id), aio_comp, &op); - ceph_assert(r == 0); - aio_comp->release(); -} - -template -void UnlinkPeerRequest::handle_unlink_group_snapshot( - const cls::rbd::SnapshotNamespace& snap_namespace, - const std::string& snap_name, int r) { - CephContext *cct = m_image_ctx->cct; - ldout(cct, 15) << "r=" << r << dendl; - - if (r < 0 && r != -ENOENT) { - lderr(cct) << "failed to unlink group snapshot: " << cpp_strerror(r) - << dendl; - finish(r); - return; - } - - remove_snapshot(snap_namespace, snap_name); -} - template void UnlinkPeerRequest::remove_snapshot( const cls::rbd::SnapshotNamespace& snap_namespace, diff --git a/src/librbd/mirror/snapshot/UnlinkPeerRequest.h b/src/librbd/mirror/snapshot/UnlinkPeerRequest.h index 8690efadc3020..53178dabe7dc4 100644 --- a/src/librbd/mirror/snapshot/UnlinkPeerRequest.h +++ b/src/librbd/mirror/snapshot/UnlinkPeerRequest.h @@ -84,12 +84,6 @@ private: void notify_update(); void handle_notify_update(int r); - void unlink_group_snapshot(const cls::rbd::SnapshotNamespace& snap_namespace, - const std::string& snap_name); - void handle_unlink_group_snapshot( - const cls::rbd::SnapshotNamespace& snap_namespace, - const std::string& snap_name, int r); - void remove_snapshot(const cls::rbd::SnapshotNamespace& snap_namespace, const std::string& snap_name); void handle_remove_snapshot(int r); diff --git a/src/tools/rbd_mirror/group_replayer/Replayer.cc b/src/tools/rbd_mirror/group_replayer/Replayer.cc index 44b4002008cbf..d903fdff834af 100644 --- a/src/tools/rbd_mirror/group_replayer/Replayer.cc +++ b/src/tools/rbd_mirror/group_replayer/Replayer.cc @@ -596,7 +596,32 @@ void Replayer::handle_mirror_snapshot_complete( on_finish->complete(0); return; } - unlink_group_snapshots(remote_group_snap_id); + + // remove mirror_peer_uuids from remote snap + auto itr = std::find_if( + m_remote_group_snaps.begin(), m_remote_group_snaps.end(), + [remote_group_snap_id](const cls::rbd::GroupSnapshot &s) { + return s.id == remote_group_snap_id; + }); + + ceph_assert(itr != m_remote_group_snaps.end()); + auto rns = std::get_if( + &itr->snapshot_namespace); + if (rns != nullptr) { + rns->mirror_peer_uuids.clear(); + auto comp = create_rados_callback( + new LambdaContext([this, remote_group_snap_id](int r) { + unlink_group_snapshots(remote_group_snap_id); + })); + + librados::ObjectWriteOperation op; + librbd::cls_client::group_snap_set(&op, *itr); + int r = m_remote_io_ctx.aio_operate( + librbd::util::group_header_name(m_remote_group_id), comp, &op); + ceph_assert(r == 0); + comp->release(); + } + on_finish->complete(0); } -- 2.39.5