From c1f2c94dd8b8bbe34ba2be4c205f718ab02eeb64 Mon Sep 17 00:00:00 2001 From: Ramana Raja Date: Sun, 12 Jan 2025 16:01:23 -0500 Subject: [PATCH] librbd/mirror/snapshot: add async create mirror group snapshot Signed-off-by: Ramana Raja --- src/librbd/CMakeLists.txt | 1 + src/librbd/api/Mirror.cc | 136 +---- src/librbd/api/Mirror.h | 7 +- .../snapshot/GroupCreatePrimaryRequest.cc | 576 ++++++++++++++++++ .../snapshot/GroupCreatePrimaryRequest.h | 139 +++++ 5 files changed, 744 insertions(+), 115 deletions(-) create mode 100644 src/librbd/mirror/snapshot/GroupCreatePrimaryRequest.cc create mode 100644 src/librbd/mirror/snapshot/GroupCreatePrimaryRequest.h diff --git a/src/librbd/CMakeLists.txt b/src/librbd/CMakeLists.txt index 50c0e6c7405..2200cf5e115 100644 --- a/src/librbd/CMakeLists.txt +++ b/src/librbd/CMakeLists.txt @@ -161,6 +161,7 @@ set(librbd_internal_srcs mirror/snapshot/CreatePrimaryRequest.cc mirror/snapshot/DemoteRequest.cc mirror/snapshot/GetImageStateRequest.cc + mirror/snapshot/GroupCreatePrimaryRequest.cc mirror/snapshot/GroupGetInfoRequest.cc mirror/snapshot/GroupPrepareImagesRequest.cc mirror/snapshot/ImageMeta.cc diff --git a/src/librbd/api/Mirror.cc b/src/librbd/api/Mirror.cc index eb456f98e09..675d332ae4d 100644 --- a/src/librbd/api/Mirror.cc +++ b/src/librbd/api/Mirror.cc @@ -33,6 +33,7 @@ #include "librbd/mirror/Types.h" #include "librbd/MirroringWatcher.h" #include "librbd/mirror/snapshot/CreatePrimaryRequest.h" +#include "librbd/mirror/snapshot/GroupCreatePrimaryRequest.h" #include "librbd/mirror/snapshot/GroupGetInfoRequest.h" #include "librbd/mirror/snapshot/ImageMeta.h" #include "librbd/mirror/snapshot/UnlinkPeerRequest.h" @@ -3428,129 +3429,36 @@ int Mirror::group_resync(IoCtx& group_ioctx, const char *group_name) { } template -int Mirror::group_snapshot_create(IoCtx& group_ioctx, const char *group_name, - uint32_t flags, std::string *snap_id) { +void Mirror::group_snapshot_create(IoCtx& group_ioctx, + const std::string& group_name, + uint32_t flags, std::string *snap_id, + Context *on_finish) { CephContext *cct = (CephContext *)group_ioctx.cct(); - ldout(cct, 20) << "io_ctx=" << &group_ioctx + ldout(cct, 20) << "group io_ctx=" << &group_ioctx << ", group_name=" << group_name << ", flags=" << flags << dendl; - std::string group_id; - int r = cls_client::dir_get_id(&group_ioctx, RBD_GROUP_DIRECTORY, - group_name, &group_id); - if (r < 0) { - lderr(cct) << "error getting the group id: " << cpp_strerror(r) << dendl; - return r; - } - - cls::rbd::MirrorGroup mirror_group; - r = cls_client::mirror_group_get(&group_ioctx, group_id, &mirror_group); - if (r == -ENOENT) { - ldout(cct, 10) << "mirroring for group " << group_name - << " disabled" << dendl; - return -EINVAL; - } else if (r < 0) { - lderr(cct) << "failed to retrieve mirror group metadata: " - << cpp_strerror(r) << dendl; - return r; - } else if (mirror_group.mirror_image_mode != - cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT) { - auto mode = static_cast( - mirror_group.mirror_image_mode); - lderr(cct) << "cannot create snapshot, mirror mode is set to: " - << mode << dendl; - return -EOPNOTSUPP; - } - - cls::rbd::MirrorSnapshotState state; - r = get_last_mirror_snapshot_state(group_ioctx, group_id, &state); - if (r == -ENOENT) { - state = cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY; - r = 0; - } - if (r < 0) { - return r; - } - - if (state != cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY) { - lderr(cct) << "group " << group_name << " is not primary" << dendl; - return -EINVAL; - } - - std::string group_snap_id = librbd::util::generate_image_id(group_ioctx); - cls::rbd::GroupSnapshot group_snap{ - group_snap_id, - cls::rbd::GroupSnapshotNamespaceMirror{}, - prepare_primary_mirror_snap_name(cct, mirror_group.global_group_id, - group_snap_id), - cls::rbd::GROUP_SNAPSHOT_STATE_INCOMPLETE}; - - std::vector quiesce_requests; - std::vector image_ctxs; - r = prepare_group_images(group_ioctx, group_id, &image_ctxs, - &group_snap, quiesce_requests, - cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, - flags); - if (r != 0) { - return r; - } - - int ret_code = 0; - std::vector snap_ids(image_ctxs.size(), CEPH_NOSNAP); - std::vector on_finishes(image_ctxs.size(), nullptr); - - for (size_t i = 0; i < image_ctxs.size(); i++) { - C_SaferCond* on_finish = new C_SaferCond; - image_snapshot_create(image_ctxs[i], RBD_SNAP_CREATE_SKIP_QUIESCE, - group_snap_id, &snap_ids[i], on_finish); - on_finishes[i] = on_finish; - } - - for (size_t i = 0; i < image_ctxs.size(); i++) { - r = 0; - if (on_finishes[i]) { - r = on_finishes[i]->wait(); - delete on_finishes[i]; - } - if (r < 0) { - if (ret_code == 0) { - ret_code = r; - } - } else { - group_snap.snaps[i].snap_id = snap_ids[i]; - } - } - - std::string group_header_oid = librbd::util::group_header_name(group_id); - if (ret_code != 0) { - // undo - ldout(cct, 20) << "undoing group create snapshot: " << ret_code << dendl; - remove_interim_snapshots(group_ioctx, group_header_oid, &image_ctxs, &group_snap); - } else{ - group_snap.state = cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE; - r = cls_client::group_snap_set(&group_ioctx, group_header_oid, group_snap); - if (r < 0) { - lderr(cct) << "failed to update group snapshot metadata: " - << cpp_strerror(r) << dendl; - } + auto req = mirror::snapshot::GroupCreatePrimaryRequest::create( + group_ioctx, group_name, flags, snap_id, on_finish); + req->send(); +} - *snap_id = group_snap.id; - } +template +int Mirror::group_snapshot_create(IoCtx& group_ioctx, + const std::string& group_name, + uint32_t flags, std::string *snap_id) { + CephContext *cct = (CephContext *)group_ioctx.cct(); - if (!quiesce_requests.empty()) { - util::notify_unquiesce(image_ctxs, quiesce_requests); - } + C_SaferCond ctx; + group_snapshot_create(group_ioctx, group_name, flags, snap_id, &ctx); - if (!ret_code) { - C_SaferCond cond; - auto req = group::UnlinkPeerGroupRequest::create( - group_ioctx, group_id, &image_ctxs, &cond); - req->send(); - cond.wait(); + int r = ctx.wait(); + if (r < 0) { + lderr(cct) << "failed to create mirror snapshot for group '" << group_name + << "': " << cpp_strerror(r) << dendl; } - close_images(&image_ctxs); - return ret_code; + return r; } template diff --git a/src/librbd/api/Mirror.h b/src/librbd/api/Mirror.h index 7a006905f25..6881f77bb7f 100644 --- a/src/librbd/api/Mirror.h +++ b/src/librbd/api/Mirror.h @@ -150,7 +150,12 @@ struct Mirror { static int group_demote(IoCtx &group_ioctx, const char *group_name, uint32_t flags); static int group_resync(IoCtx &group_ioctx, const char *group_name); - static int group_snapshot_create(IoCtx& group_ioctx, const char *group_name, + static void group_snapshot_create(IoCtx& group_ioctx, + const std::string& group_name, + uint32_t flags, std::string *snap_id, + Context *on_finish); + static int group_snapshot_create(IoCtx& group_ioctx, + const std::string& group_name, uint32_t flags, std::string *snap_id); static int group_image_add(IoCtx &group_ioctx, const std::string &group_id, diff --git a/src/librbd/mirror/snapshot/GroupCreatePrimaryRequest.cc b/src/librbd/mirror/snapshot/GroupCreatePrimaryRequest.cc new file mode 100644 index 00000000000..b778ed127cf --- /dev/null +++ b/src/librbd/mirror/snapshot/GroupCreatePrimaryRequest.cc @@ -0,0 +1,576 @@ +// -*- mode:c++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/mirror/snapshot/GroupCreatePrimaryRequest.h" +#include "include/ceph_assert.h" +#include "common/dout.h" +#include "include/Context.h" +#include "common/Cond.h" +#include "common/errno.h" +#include "librbd/api/Group.h" +#include "librbd/internal.h" +#include "common/ceph_context.h" +#include "cls/rbd/cls_rbd_client.h" +#include "librbd/ImageCtx.h" +#include "librbd/Operations.h" +#include "librbd/ImageState.h" +#include "librbd/Utils.h" +#include "librbd/group/UnlinkPeerGroupRequest.h" +#include "librbd/group/ListSnapshotsRequest.h" +#include "librbd/mirror/GetInfoRequest.h" +#include "librbd/mirror/snapshot/CreatePrimaryRequest.h" +#include "librbd/mirror/snapshot/GroupPrepareImagesRequest.h" +#include "librbd/mirror/Types.h" + +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "librbd::mirror::snapshot::GroupCreatePrimaryRequest: " \ + << " " << __func__ << ": " + +namespace librbd { +namespace mirror { +namespace snapshot { + +using librbd::util::create_rados_callback; + + +std::string prepare_primary_mirror_snap_name(CephContext *cct, + const std::string &global_group_id, + const std::string &snap_id) { + ldout(cct, 10) << "global_group_id: " << global_group_id + << ", snap_id: " << snap_id << dendl; + + std::stringstream ind_snap_name_stream; + ind_snap_name_stream << ".mirror.primary." + << global_group_id << "." << snap_id; + return ind_snap_name_stream.str(); +} + + +template +struct C_ImageSnapshotCreate2 : public Context { + I *ictx; + uint64_t snap_create_flags; + int64_t group_pool_id; + std::string group_id; + std::string group_snap_id; + uint64_t *snap_id; + Context *on_finish; + + cls::rbd::MirrorImage mirror_image; + mirror::PromotionState promotion_state; + std::string primary_mirror_uuid; + + C_ImageSnapshotCreate2(I *ictx, uint64_t snap_create_flags, + int64_t group_pool_id, + const std::string &group_id, + const std::string &group_snap_id, + uint64_t *snap_id, + Context *on_finish) + : ictx(ictx), snap_create_flags(snap_create_flags), + group_pool_id(group_pool_id), group_id(group_id), + group_snap_id(group_snap_id), snap_id(snap_id), + on_finish(on_finish) { + } + + void finish(int r) override { + if (r < 0 && r != -ENOENT) { + on_finish->complete(r); + return; + } + + if (mirror_image.mode != cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT || + mirror_image.state != cls::rbd::MIRROR_IMAGE_STATE_ENABLED) { + lderr(ictx->cct) << "snapshot based mirroring is not enabled" << dendl; + on_finish->complete(-EINVAL); + return; + } + + auto req = mirror::snapshot::CreatePrimaryRequest::create( + ictx, mirror_image.global_image_id, CEPH_NOSNAP, snap_create_flags, 0U, + group_pool_id, group_id, group_snap_id, snap_id, on_finish); + req->send(); + } +}; + + +template +void image_snapshot_create2(I *ictx, uint32_t flags, + const std::string &group_snap_id, + uint64_t *snap_id, Context *on_finish) { + CephContext *cct = ictx->cct; + ldout(cct, 10) << "ictx=" << ictx << dendl; + + uint64_t snap_create_flags = 0; + int r = librbd::util::snap_create_flags_api_to_internal(cct, flags, + &snap_create_flags); + if (r < 0) { + on_finish->complete(r); + return; + } + + auto on_refresh = new LambdaContext( + [ictx, snap_create_flags, group_snap_id, snap_id, on_finish](int r) { + if (r < 0) { + lderr(ictx->cct) << "refresh failed: " << cpp_strerror(r) << dendl; + on_finish->complete(r); + return; + } + + auto ctx = new C_ImageSnapshotCreate2(ictx, snap_create_flags, + ictx->group_spec.pool_id, + ictx->group_spec.group_id, + group_snap_id, snap_id, + on_finish); + auto req = mirror::GetInfoRequest::create(*ictx, &ctx->mirror_image, + &ctx->promotion_state, + &ctx->primary_mirror_uuid, + ctx); + req->send(); + }); + + if (ictx->state->is_refresh_required()) { + ictx->state->refresh(on_refresh); + } else { + on_refresh->complete(0); + } +} + + +template +void GroupCreatePrimaryRequest::send() { + get_group_id(); +} + +template +void GroupCreatePrimaryRequest::get_group_id() { + ldout(m_cct, 10) << dendl; + + librados::ObjectReadOperation op; + cls_client::dir_get_id_start(&op, m_group_name); + + auto comp = create_rados_callback< + GroupCreatePrimaryRequest, + &GroupCreatePrimaryRequest::handle_get_group_id>(this); + + m_outbl.clear(); + int r = m_group_ioctx.aio_operate(RBD_GROUP_DIRECTORY, comp, &op, &m_outbl); + ceph_assert(r == 0); + comp->release(); +} + +template +void GroupCreatePrimaryRequest::handle_get_group_id(int r) { + ldout(m_cct, 10) << "r=" << r << dendl; + + if (r < 0) { + lderr(m_cct) << "failed to get ID of group '" << m_group_name + << "': " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + auto it = m_outbl.cbegin(); + r = cls_client::dir_get_id_finish(&it, &m_group_id); + if (r < 0) { + lderr(m_cct) << "failed to get ID of group '" << m_group_name + << "': " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + get_mirror_group(); +} + +template +void GroupCreatePrimaryRequest::get_mirror_group() { + ldout(m_cct, 10) << dendl; + + librados::ObjectReadOperation op; + cls_client::mirror_group_get_start(&op, m_group_id); + + auto comp = create_rados_callback< + GroupCreatePrimaryRequest, + &GroupCreatePrimaryRequest::handle_get_mirror_group>(this); + + m_outbl.clear(); + int r = m_group_ioctx.aio_operate(RBD_MIRRORING, comp, &op, &m_outbl); + ceph_assert(r == 0); + comp->release(); +} + +template +void GroupCreatePrimaryRequest::handle_get_mirror_group(int r) { + ldout(m_cct, 10) << "r=" << r << dendl; + + if (r == -ENOENT) { + ldout(m_cct, 10) << "mirroring for group '" << m_group_name + << "' disabled" << dendl; + finish(-EINVAL); + return; + } else if (r < 0) { + lderr(m_cct) << "failed to retrieve mirror group metadata for group '" + << m_group_name << "': " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + auto it = m_outbl.cbegin(); + r = cls_client::mirror_group_get_finish(&it, &m_mirror_group); + if (r < 0) { + lderr(m_cct) << "failed to retrieve mirror group metadata for group '" + << m_group_name << "': " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + if (m_mirror_group.mirror_image_mode != + cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT) { + auto mode = static_cast( + m_mirror_group.mirror_image_mode); + lderr(m_cct) << "cannot create snapshot, mirror mode is set to: " + << mode << dendl; + finish(-EOPNOTSUPP); + return; + } + + get_last_mirror_snapshot_state(); +} + +template +void GroupCreatePrimaryRequest::get_last_mirror_snapshot_state() { + ldout(m_cct, 10) << dendl; + + auto ctx = util::create_context_callback< + GroupCreatePrimaryRequest, + &GroupCreatePrimaryRequest::handle_get_last_mirror_snapshot_state>( + this); + + auto req = group::ListSnapshotsRequest::create( + m_group_ioctx, m_group_id, true, true, &m_existing_group_snaps, ctx); + + req->send(); +} + +template +void GroupCreatePrimaryRequest::handle_get_last_mirror_snapshot_state( + int r) { + ldout(m_cct, 10) << dendl; + + if (r < 0) { + lderr(m_cct) << "failed to list group snapshots of group '" << m_group_name + << "': " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + cls::rbd::MirrorSnapshotState state = + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY; + for (auto it = m_existing_group_snaps.rbegin(); + it != m_existing_group_snaps.rend(); it++) { + auto ns = std::get_if( + &it->snapshot_namespace); + if (ns != nullptr) { + // XXXMG: check primary_mirror_uuid matches? + ldout(m_cct, 10) << "the state of existing group snap is: " << ns->state + << dendl; + state = ns->state; + break; + } + } + + if (state != cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY) { + lderr(m_cct) << "group " << m_group_name << " is not primary" << dendl; + finish(-EINVAL); + return; + } + + generate_group_snap(); +} + +template +void GroupCreatePrimaryRequest::generate_group_snap() { + ldout(m_cct, 10) << dendl; + + m_group_snap.id = librbd::util::generate_image_id(m_group_ioctx); + m_group_snap.snapshot_namespace = cls::rbd::GroupSnapshotNamespaceMirror{}; + m_group_snap.name = prepare_primary_mirror_snap_name( + m_cct, m_mirror_group.global_group_id, m_group_snap.id); + + prepare_group_images(); +} + +template +void GroupCreatePrimaryRequest::prepare_group_images() { + ldout(m_cct, 10) << dendl; + + auto ctx = util::create_context_callback< + GroupCreatePrimaryRequest, + &GroupCreatePrimaryRequest::handle_prepare_group_images>(this); + + auto req = mirror::snapshot::GroupPrepareImagesRequest::create( + m_group_ioctx, m_group_id, cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, + m_flags, &m_group_snap, &m_image_ctxs, &m_quiesce_requests, ctx); + + req->send(); +} + +template +void GroupCreatePrimaryRequest::handle_prepare_group_images(int r) { + ldout(m_cct, 10) << "r=" << r << dendl; + + if (r < 0) { + lderr(m_cct) << "failed to prepare group images '" << m_group_name + << "': " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + create_image_snaps(); +} + +template +void GroupCreatePrimaryRequest::create_image_snaps() { + ldout(m_cct, 10) << "group name '" << m_group_name << "' group ID '" + << m_group_id + << "' group snap ID '" << m_group_snap.id << dendl; + + auto ctx = librbd::util::create_context_callback< + GroupCreatePrimaryRequest, + &GroupCreatePrimaryRequest::handle_create_image_snaps>(this); + auto gather_ctx = new C_Gather(m_cct, ctx); + + m_image_snap_ids.resize(m_image_ctxs.size(), CEPH_NOSNAP); + + for (size_t i = 0; i < m_image_ctxs.size(); i++) { + image_snapshot_create2(m_image_ctxs[i], RBD_SNAP_CREATE_SKIP_QUIESCE, + m_group_snap.id, &m_image_snap_ids[i], + gather_ctx->new_sub()); + } + + gather_ctx->activate(); +} + +template +void GroupCreatePrimaryRequest::handle_create_image_snaps(int r) { + ldout(m_cct, 10) << "r=" << r << dendl; + + std::string group_header_oid = librbd::util::group_header_name(m_group_id); + + if (r < 0) { + lderr(m_cct) << "failed to create image snaps: " + << cpp_strerror(r) << dendl; + + if (m_ret_code == 0) { + m_ret_code = r; + } + + ldout(m_cct, 10) << "undoing group create snapshot: " << r << dendl; + remove_incomplete_group_snap(); + return; + } else { + for (size_t i = 0; i < m_image_ctxs.size(); i++) { + m_group_snap.snaps[i].snap_id = m_image_snap_ids[i]; + } + + m_group_snap.state = cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE; + r = cls_client::group_snap_set(&m_group_ioctx, group_header_oid, + m_group_snap); + if (r < 0) { + lderr(m_cct) << "failed to update group snapshot metadata: " + << cpp_strerror(r) << dendl; + if (m_ret_code == 0) { + m_ret_code = r; + } + + ldout(m_cct, 10) << "undoing group create snapshot: " << r << dendl; + remove_incomplete_group_snap(); + return; + } + + *m_snap_id = m_group_snap.id; + } + + if (!m_quiesce_requests.empty()) { + notify_unquiesce(); + return; + } + + if (m_ret_code == 0) { + unlink_peer_group(); + return; + } + + close_images(); +} + +template +void GroupCreatePrimaryRequest::remove_incomplete_group_snap() { + ldout(m_cct, 10) << dendl; + + auto ctx = librbd::util::create_context_callback< + GroupCreatePrimaryRequest, + &GroupCreatePrimaryRequest::handle_remove_incomplete_group_snap>(this); + auto gather_ctx = new C_Gather(m_cct, ctx); + + + for (size_t i = 0; i < m_image_ctxs.size(); ++i) { + if (m_group_snap.snaps[i].snap_id == CEPH_NOSNAP) { + continue; + } + + librbd::ImageCtx *ictx = m_image_ctxs[i]; + + std::shared_lock image_locker{ictx->image_lock}; + auto info = ictx->get_snap_info( + m_group_snap.snaps[i].snap_id); + ceph_assert(info != nullptr); + image_locker.unlock(); + + ldout(m_cct, 10) << "removing individual snapshot: " + << info->name << dendl; + + ictx->operations->snap_remove(info->snap_namespace, + info->name, + gather_ctx->new_sub()); + } + + gather_ctx->activate(); +} + +template +void GroupCreatePrimaryRequest::handle_remove_incomplete_group_snap(int r) { + ldout(m_cct, 10) << "r=" << r << dendl; + + // if previous attempts to remove this snapshot failed then the + // image's snapshot may not exist + if (r < 0 && r != -ENOENT) { + lderr(m_cct) << "failed cleaning up group member image snapshots: " + << cpp_strerror(r) << dendl; + } + + if (r == 0) { + r = cls_client::group_snap_remove( + &m_group_ioctx, + librbd::util::group_header_name(m_group_id), + m_group_snap.id); + + if (r < 0) { + lderr(m_cct) << "failed to remove group snapshot metadata: " + << cpp_strerror(r) << dendl; + } + } + + if (!m_quiesce_requests.empty()) { + notify_unquiesce(); + return; + } + + close_images(); +} + +template +void GroupCreatePrimaryRequest::notify_unquiesce() { + ldout(m_cct, 10) << dendl; + + ceph_assert(m_quiesce_requests.size() == m_image_ctxs.size()); + + auto ctx = librbd::util::create_context_callback< + GroupCreatePrimaryRequest, + &GroupCreatePrimaryRequest::handle_notify_unquiesce>(this); + auto gather_ctx = new C_Gather(m_cct, ctx); + + int image_count = m_image_ctxs.size(); + for (int i = 0; i < image_count; ++i) { + auto ictx = m_image_ctxs[i]; + ictx->image_watcher->notify_unquiesce(m_quiesce_requests[i], + gather_ctx->new_sub()); + } + + gather_ctx->activate(); +} + +template +void GroupCreatePrimaryRequest::handle_notify_unquiesce(int r) { + ldout(m_cct, 10) << "r=" << r << dendl; + + if (r < 0) { + lderr(m_cct) << "failed to notify the unquiesce requests: " + << cpp_strerror(r) << dendl; + } + + if (m_ret_code == 0) { + unlink_peer_group(); + return; + } + + close_images(); +} + +template +void GroupCreatePrimaryRequest::unlink_peer_group() { + ldout(m_cct, 10) << dendl; + + auto ctx = librbd::util::create_context_callback< + GroupCreatePrimaryRequest, + &GroupCreatePrimaryRequest::handle_unlink_peer_group>(this); + + auto req = group::UnlinkPeerGroupRequest::create( + m_group_ioctx, m_group_id, &m_image_ctxs, ctx); + + req->send(); +} + +template +void GroupCreatePrimaryRequest::handle_unlink_peer_group(int r) { + ldout(m_cct, 10) << "r=" << r << dendl; + + if (r < 0) { + lderr(m_cct) << "failed to unlink peer group: " << cpp_strerror(r) + << dendl; + } + + close_images(); +} + +template +void GroupCreatePrimaryRequest::close_images() { + ldout(m_cct, 10) << dendl; + + auto ctx = librbd::util::create_context_callback< + GroupCreatePrimaryRequest, + &GroupCreatePrimaryRequest::handle_close_images>(this); + auto gather_ctx = new C_Gather(m_cct, ctx); + + for (auto ictx: m_image_ctxs) { + ictx->state->close(gather_ctx->new_sub()); + } + + gather_ctx->activate(); +} + +template +void GroupCreatePrimaryRequest::handle_close_images(int r) { + ldout(m_cct, 10) << "r=" << r << dendl; + + if (r < 0) { + lderr(m_cct) << "failed to close images: " << cpp_strerror(r) << dendl; + } + + m_image_ctxs.clear(); + finish(m_ret_code); +} + +template +void GroupCreatePrimaryRequest::finish(int r) { + ldout(m_cct, 10) << "r=" << r << dendl; + + m_on_finish->complete(r); + delete this; +} + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +template class librbd::mirror::snapshot::GroupCreatePrimaryRequest; diff --git a/src/librbd/mirror/snapshot/GroupCreatePrimaryRequest.h b/src/librbd/mirror/snapshot/GroupCreatePrimaryRequest.h new file mode 100644 index 00000000000..d8567a0c063 --- /dev/null +++ b/src/librbd/mirror/snapshot/GroupCreatePrimaryRequest.h @@ -0,0 +1,139 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_SNAPSHOT_GROUP_CREATE_PRIMARY_REQUEST_H +#define CEPH_LIBRBD_MIRROR_SNAPSHOT_GROUP_CREATE_PRIMARY_REQUEST_H + +#include "include/buffer.h" +#include "include/rados/librados.hpp" +#include "librbd/ImageWatcher.h" +#include "cls/rbd/cls_rbd_types.h" +#include "librbd/internal.h" +#include "librbd/mirror/snapshot/Types.h" + +#include +#include + +struct Context; + +namespace librbd { + +struct ImageCtx; + +namespace mirror { +namespace snapshot { + +template +class GroupCreatePrimaryRequest { +public: + static GroupCreatePrimaryRequest *create(librados::IoCtx& group_ioctx, + const std::string& group_name, + uint32_t flags, std::string *snap_id, + Context *on_finish) { + return new GroupCreatePrimaryRequest(group_ioctx, group_name, flags, + snap_id, on_finish); + } + + GroupCreatePrimaryRequest(librados::IoCtx& group_ioctx, + const std::string& group_name, + uint32_t flags, std::string *snap_id, + Context *on_finish) + : m_group_ioctx(group_ioctx), m_group_name(group_name), m_flags(flags), + m_snap_id(snap_id), m_on_finish(on_finish) { + m_cct = (CephContext *)group_ioctx.cct(); + } + + void send(); + +private: + // TODO: Complete the diagram + /** + * @verbatim + * + * + * | + * v + * GET GROUP ID + * | + * v + * GET LAST MIRROR SNAPSHOT STATE + * | + * v + * PREPARE GROUP IMAGES + * | + * | (on error) + * CREATE IMAGE SNAPS . . . . . . . REMOVE INCOMPLETE GROUP SNAP + * | . + * v . + * NOTIFY UNQUIESCE < . . . . . . . . . . + * | + * v + * UNLINK PEER GROUP + * | + * v + * CLOSE IMAGES + * | + * v + * + * + * @endverbatim + */ + + librados::IoCtx m_group_ioctx; + const std::string m_group_name; + const uint32_t m_flags; + std::string *m_snap_id; + Context *m_on_finish; + + CephContext *m_cct; + + bufferlist m_outbl; + std::string m_group_id; + cls::rbd::MirrorGroup m_mirror_group; + std::vector m_existing_group_snaps; + cls::rbd::GroupSnapshot m_group_snap; + std::vector m_image_ctxs; + std::vector m_quiesce_requests; + std::vector m_image_snap_ids; + int m_ret_code=0; + + void get_group_id(); + void handle_get_group_id(int r); + + void get_mirror_group(); + void handle_get_mirror_group(int r); + + void get_last_mirror_snapshot_state(); + void handle_get_last_mirror_snapshot_state(int r); + + void generate_group_snap(); + + void prepare_group_images(); + void handle_prepare_group_images(int r); + + void create_image_snaps(); + void handle_create_image_snaps(int r); + + void notify_unquiesce(); + void handle_notify_unquiesce(int r); + + void unlink_peer_group(); + void handle_unlink_peer_group(int r); + + void close_images(); + void handle_close_images(int r); + + void finish(int r); + + // cleanup + void remove_incomplete_group_snap(); + void handle_remove_incomplete_group_snap(int r); +}; + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +extern template class librbd::mirror::snapshot::GroupCreatePrimaryRequest; + +#endif // CEPH_LIBRBD_MIRROR_SNAPSHOT_GROUP_CREATE_PRIMARY_REQUEST_H -- 2.39.5