From b9a5ae8a415a68cc4cb0a2701a89920a114e0896 Mon Sep 17 00:00:00 2001 From: Ramana Raja Date: Fri, 24 Jan 2025 17:29:51 -0500 Subject: [PATCH] librbd: add async state machine for "mirror_group_get_info" Signed-off-by: Ramana Raja --- src/librbd/CMakeLists.txt | 1 + src/librbd/api/Mirror.cc | 54 ++---- src/librbd/api/Mirror.h | 4 + .../mirror/snapshot/GroupGetInfoRequest.cc | 172 ++++++++++++++++++ .../mirror/snapshot/GroupGetInfoRequest.h | 92 ++++++++++ 5 files changed, 289 insertions(+), 34 deletions(-) create mode 100644 src/librbd/mirror/snapshot/GroupGetInfoRequest.cc create mode 100644 src/librbd/mirror/snapshot/GroupGetInfoRequest.h diff --git a/src/librbd/CMakeLists.txt b/src/librbd/CMakeLists.txt index 62c234b733fda..f41f77388ea3a 100644 --- a/src/librbd/CMakeLists.txt +++ b/src/librbd/CMakeLists.txt @@ -161,6 +161,7 @@ set(librbd_internal_srcs mirror/snapshot/CreatePrimaryRequest.cc mirror/snapshot/DemoteRequest.cc mirror/snapshot/GetImageStateRequest.cc + mirror/snapshot/GroupGetInfoRequest.cc mirror/snapshot/ImageMeta.cc mirror/snapshot/PromoteRequest.cc mirror/snapshot/RemoveImageStateRequest.cc diff --git a/src/librbd/api/Mirror.cc b/src/librbd/api/Mirror.cc index 48ef7419e749f..eb456f98e09cc 100644 --- a/src/librbd/api/Mirror.cc +++ b/src/librbd/api/Mirror.cc @@ -33,6 +33,7 @@ #include "librbd/mirror/Types.h" #include "librbd/MirroringWatcher.h" #include "librbd/mirror/snapshot/CreatePrimaryRequest.h" +#include "librbd/mirror/snapshot/GroupGetInfoRequest.h" #include "librbd/mirror/snapshot/ImageMeta.h" #include "librbd/mirror/snapshot/UnlinkPeerRequest.h" #include "librbd/mirror/snapshot/Utils.h" @@ -4057,6 +4058,19 @@ int Mirror::group_info_list(librados::IoCtx& io_ctx, return 0; } +template +void Mirror::group_get_info(librados::IoCtx& io_ctx, + const std::string& group_name, + mirror_group_info_t *mirror_group_info, + Context *on_finish) { + CephContext *cct = reinterpret_cast(io_ctx.cct()); + ldout(cct, 20) << "group_name=" << group_name << dendl; + + auto req = mirror::snapshot::GroupGetInfoRequest::create( + io_ctx, group_name, mirror_group_info, on_finish); + req->send(); +} + template int Mirror::group_get_info(librados::IoCtx& io_ctx, const std::string &group_name, @@ -4064,43 +4078,15 @@ int Mirror::group_get_info(librados::IoCtx& io_ctx, CephContext *cct((CephContext *)io_ctx.cct()); ldout(cct, 20) << "group_name=" << group_name << dendl; - std::string group_id; - int r = cls_client::dir_get_id(&io_ctx, RBD_GROUP_DIRECTORY, group_name, - &group_id); - if (r < 0) { - ldout(cct, 20) << "error getting id of group " << group_name << ": " - << cpp_strerror(r) << dendl; - return r; - } - - cls::rbd::MirrorGroup mirror_group; - r = cls_client::mirror_group_get(&io_ctx, group_id, &mirror_group); + C_SaferCond ctx; + group_get_info(io_ctx, group_name, mirror_group_info, &ctx); + int r = ctx.wait(); if (r < 0) { - ldout(cct, 20) << "failed to get mirror group info: " << cpp_strerror(r) - << dendl; - return r; - } - - cls::rbd::MirrorSnapshotState promotion_state; - r = get_last_mirror_snapshot_state(io_ctx, group_id, &promotion_state); - if (r == -ENOENT) { - promotion_state = cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY; - } else if (r < 0) { - lderr(cct) << "failed to get last mirror snapshot state: " - << cpp_strerror(r) << dendl; - return r; + lderr(cct) << "failed to get mirror info of group '" << group_name + << "': " << cpp_strerror(r) << dendl; } - - mirror_group_info->global_id = mirror_group.global_group_id; - mirror_group_info->mirror_image_mode = - static_cast(mirror_group.mirror_image_mode); - mirror_group_info->state = - static_cast(mirror_group.state); - mirror_group_info->primary = - (promotion_state == cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY); - - return 0; + return r; } template diff --git a/src/librbd/api/Mirror.h b/src/librbd/api/Mirror.h index de8be202746c3..7a006905f25f0 100644 --- a/src/librbd/api/Mirror.h +++ b/src/librbd/api/Mirror.h @@ -175,6 +175,10 @@ struct Mirror { size_t max, std::map *entries); + static void group_get_info(librados::IoCtx& io_ctx, + const std::string &group_name, + mirror_group_info_t *mirror_group_info, + Context *on_finish); static int group_get_info(librados::IoCtx& io_ctx, const std::string &group_name, mirror_group_info_t *mirror_group_info); diff --git a/src/librbd/mirror/snapshot/GroupGetInfoRequest.cc b/src/librbd/mirror/snapshot/GroupGetInfoRequest.cc new file mode 100644 index 0000000000000..87889e6c65cc8 --- /dev/null +++ b/src/librbd/mirror/snapshot/GroupGetInfoRequest.cc @@ -0,0 +1,172 @@ +// -*- mode:c++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/mirror/snapshot/GroupGetInfoRequest.h" +#include "include/ceph_assert.h" +#include "common/dout.h" +#include "common/errno.h" +#include "common/ceph_context.h" +#include "cls/rbd/cls_rbd_client.h" +#include "librbd/ImageCtx.h" +#include "librbd/Utils.h" +#include "librbd/group/ListSnapshotsRequest.h" + +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "librbd::mirror::snapshot::GroupGetInfoRequest: " \ + << this << " " << __func__ << ": " + +namespace librbd { +namespace mirror { +namespace snapshot { + +using librbd::util::create_rados_callback; + +template +void GroupGetInfoRequest::send() { + get_id(); +} + +template +void GroupGetInfoRequest::get_id() { + auto cct = reinterpret_cast(m_group_ioctx.cct()); + ldout(cct, 10) << dendl; + + librados::ObjectReadOperation op; + cls_client::dir_get_id_start(&op, m_group_name); + + auto comp = create_rados_callback< + GroupGetInfoRequest, &GroupGetInfoRequest::handle_get_id>(this); + + m_outbl.clear(); + int r = m_group_ioctx.aio_operate(RBD_GROUP_DIRECTORY, comp, &op, &m_outbl); + ceph_assert(r == 0); + comp->release(); +} + +template +void GroupGetInfoRequest::handle_get_id(int r) { + auto cct = reinterpret_cast(m_group_ioctx.cct()); + ldout(cct, 10) << "r=" << r << dendl; + if (r < 0) { + lderr(cct) << "failed to get ID of group '" << m_group_name + << "': " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + auto it = m_outbl.cbegin(); + r = cls_client::dir_get_id_finish(&it, &m_group_id); + if (r < 0) { + lderr(cct) << "failed to get ID of group '" << m_group_name + << "': " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + get_info(); +} + +template +void GroupGetInfoRequest::get_info() { + auto cct = reinterpret_cast(m_group_ioctx.cct()); + ldout(cct, 10) << dendl; + + librados::ObjectReadOperation op; + cls_client::mirror_group_get_start(&op, m_group_id); + + auto comp = create_rados_callback< + GroupGetInfoRequest, &GroupGetInfoRequest::handle_get_info>(this); + + m_outbl.clear(); + int r = m_group_ioctx.aio_operate(RBD_MIRRORING, comp, &op, &m_outbl); + ceph_assert(r == 0); + comp->release(); +} + +template +void GroupGetInfoRequest::handle_get_info(int r) { + auto cct = reinterpret_cast(m_group_ioctx.cct()); + ldout(cct, 10) << "r=" << r << dendl; + if (r < 0) { + lderr(cct) << "failed to get mirror info of group '" << m_group_name + << "': " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + auto it = m_outbl.cbegin(); + cls::rbd::MirrorGroup mirror_group; + r = cls_client::mirror_group_get_finish(&it, &mirror_group); + if (r < 0) { + lderr(cct) << "failed to get mirror info of group '" << m_group_name + << "': " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + m_mirror_group_info->global_id = mirror_group.global_group_id; + m_mirror_group_info->mirror_image_mode = + static_cast(mirror_group.mirror_image_mode); + m_mirror_group_info->state = + static_cast(mirror_group.state); + m_mirror_group_info->primary = false; + + get_last_mirror_snapshot_state(); +} + +template +void GroupGetInfoRequest::get_last_mirror_snapshot_state() { + auto cct = reinterpret_cast(m_group_ioctx.cct()); + ldout(cct, 10) << dendl; + + auto ctx = util::create_context_callback< + GroupGetInfoRequest, + &GroupGetInfoRequest::handle_get_last_mirror_snapshot_state>(this); + + auto req = group::ListSnapshotsRequest::create( + m_group_ioctx, m_group_id, true, true, &m_group_snaps, ctx); + + req->send(); +} + +template +void GroupGetInfoRequest::handle_get_last_mirror_snapshot_state(int r) { + auto cct = reinterpret_cast(m_group_ioctx.cct()); + ldout(cct, 10) << dendl; + + if (r < 0) { + lderr(cct) << "failed to list group snapshots of group '" << m_group_name + << "': " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + for (auto it = m_group_snaps.rbegin(); it != m_group_snaps.rend(); it++) { + auto ns = std::get_if( + &it->snapshot_namespace); + if (ns != nullptr) { + // XXXMG: check primary_mirror_uuid matches? + m_mirror_group_info->primary = + (ns->state == cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY); + break; + } + } + + finish(0); +} + +template +void GroupGetInfoRequest::finish(int r) { + auto cct = reinterpret_cast(m_group_ioctx.cct()); + ldout(cct, 10) << "r=" << r << dendl; + + m_on_finish->complete(r); + delete this; +} + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +template class librbd::mirror::snapshot::GroupGetInfoRequest; diff --git a/src/librbd/mirror/snapshot/GroupGetInfoRequest.h b/src/librbd/mirror/snapshot/GroupGetInfoRequest.h new file mode 100644 index 0000000000000..cddf6c90c4341 --- /dev/null +++ b/src/librbd/mirror/snapshot/GroupGetInfoRequest.h @@ -0,0 +1,92 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_SNAPSHOT_GROUP_GET_INFO_REQUEST_H +#define CEPH_LIBRBD_MIRROR_SNAPSHOT_GROUP_GET_INFO_REQUEST_H + +#include "include/buffer.h" +#include "include/types.h" +#include "include/rados/librados.hpp" +#include "include/rbd/librbd.hpp" +#include "librbd/ImageCtx.h" +#include "cls/rbd/cls_rbd_types.h" + +struct Context; + +namespace librbd { + +struct ImageCtx; + +namespace mirror { +namespace snapshot { + +template +class GroupGetInfoRequest { +public: + static GroupGetInfoRequest *create(librados::IoCtx& group_ioctx, + const std::string& group_name, + mirror_group_info_t *mirror_group_info, + Context *on_finish) { + return new GroupGetInfoRequest(group_ioctx, group_name, mirror_group_info, + on_finish); + } + + GroupGetInfoRequest(librados::IoCtx& group_ioctx, + const std::string& group_name, + mirror_group_info_t *mirror_group_info, + Context *on_finish) + : m_group_ioctx(group_ioctx), m_group_name(group_name), + m_mirror_group_info(mirror_group_info), m_on_finish(on_finish) { + } + + void send(); + +private: + /** + * @verbatim + * + * + * | + * v + * GET ID + * | + * v + * GET INFO + * | + * v + * GET LAST MIRROR SNAPSHOT STATE + * | + * v + * + * + * @endverbatim + */ + + librados::IoCtx m_group_ioctx; + const std::string m_group_name; + std::string m_group_id; + mirror_group_info_t *m_mirror_group_info; + std::vector m_group_snaps; + Context *m_on_finish; + + bufferlist m_outbl; + + void get_id(); + void handle_get_id(int r); + + void get_info(); + void handle_get_info(int r); + + void get_last_mirror_snapshot_state(); + void handle_get_last_mirror_snapshot_state(int r); + + void finish(int r); +}; + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +extern template class librbd::mirror::snapshot::GroupGetInfoRequest; + +#endif // CEPH_LIBRBD_MIRROR_SNAPSHOT_GROUP_GET_INFO_REQUEST_H -- 2.39.5