From 4f737999b7455e71019e8147af6ce22f2a03433c Mon Sep 17 00:00:00 2001 From: Prasanna Kumar Kalever Date: Tue, 13 Jan 2026 11:58:39 +0530 Subject: [PATCH] rbd-mirror: implement semi-dynamic groups with support for adding images this allows images to be added to an already enabled mirror group without disabling and re-enabling the group. As a result, it avoids the overhead of re-mirroring the entire group over the network. Signed-off-by: Prasanna Kumar Kalever --- src/librbd/CMakeLists.txt | 1 + src/librbd/api/Group.cc | 11 +- src/librbd/api/Mirror.cc | 45 + src/librbd/api/Mirror.h | 2 + src/librbd/mirror/GroupAddImageRequest.cc | 803 ++++++++++++++++++ src/librbd/mirror/GroupAddImageRequest.h | 182 ++++ .../group_replayer/BootstrapRequest.cc | 46 +- 7 files changed, 1080 insertions(+), 10 deletions(-) create mode 100644 src/librbd/mirror/GroupAddImageRequest.cc create mode 100644 src/librbd/mirror/GroupAddImageRequest.h diff --git a/src/librbd/CMakeLists.txt b/src/librbd/CMakeLists.txt index 4c57d31145c..b9afd2cd755 100644 --- a/src/librbd/CMakeLists.txt +++ b/src/librbd/CMakeLists.txt @@ -154,6 +154,7 @@ set(librbd_internal_srcs mirror/GetStatusRequest.cc mirror/GetUuidRequest.cc mirror/GroupEnableRequest.cc + mirror/GroupAddImageRequest.cc mirror/GroupGetInfoRequest.cc mirror/ImageRemoveRequest.cc mirror/ImageStateUpdateRequest.cc diff --git a/src/librbd/api/Group.cc b/src/librbd/api/Group.cc index af5e519dc2b..e351f5ce743 100644 --- a/src/librbd/api/Group.cc +++ b/src/librbd/api/Group.cc @@ -601,10 +601,6 @@ int Group::image_add(librados::IoCtx& group_ioctx, const char *group_name, << cpp_strerror(r) << dendl; return r; } else if (r == 0) { - if (mirror_group.state != cls::rbd::MIRROR_GROUP_STATE_DISABLED) { - lderr(cct) << "cannot add image to mirror enabled group" << dendl; - return -EINVAL; - } if (promotion_state != mirror::PROMOTION_STATE_PRIMARY) { lderr(cct) << "group is not primary, cannot add image" << dendl; return -EINVAL; @@ -669,6 +665,13 @@ int Group::image_add(librados::IoCtx& group_ioctx, const char *group_name, } ImageWatcher<>::notify_header_update(image_ioctx, image_header_oid); + r = Mirror::group_image_add(group_ioctx, group_id, image_ioctx, image_id); + if (r < 0) { + lderr(cct) << "error add image to mirror group: " + << cpp_strerror(r) << dendl; + return r; + } + r = cls_client::group_image_set(&group_ioctx, group_header_oid, attached_st); if (r < 0) { diff --git a/src/librbd/api/Mirror.cc b/src/librbd/api/Mirror.cc index e7f41de1199..90793a8568a 100644 --- a/src/librbd/api/Mirror.cc +++ b/src/librbd/api/Mirror.cc @@ -29,6 +29,7 @@ #include "librbd/mirror/GetStatusRequest.h" #include "librbd/mirror/GetUuidRequest.h" #include "librbd/mirror/GroupEnableRequest.h" +#include "librbd/mirror/GroupAddImageRequest.h" #include "librbd/mirror/GroupGetInfoRequest.h" #include "librbd/mirror/PromoteRequest.h" #include "librbd/mirror/Types.h" @@ -3027,6 +3028,50 @@ int create_orphan_group_snapshot(IoCtx& group_ioctx, return 0; } +template +int Mirror::group_image_add(IoCtx &group_ioctx, + const std::string &group_id, + IoCtx &image_ioctx, + const std::string &image_id) { + CephContext *cct = (CephContext *)group_ioctx.cct(); + ldout(cct, 20) << "group io_ctx=" << &group_ioctx << ", group_id=" << group_id + << ", image io_ctx=" << &image_ioctx << ", image_id=" + << image_id << dendl; + + cls::rbd::MirrorGroup mirror_group; + int r = cls_client::mirror_group_get(&group_ioctx, group_id, &mirror_group); + if (r == -ENOENT) { + ldout(cct, 10) << "group is not enabled for mirroring" << dendl; + return 0; + } else if (r < 0) { + lderr(cct) << "failed to retrieve mirror group metadata: " + << cpp_strerror(r) << dendl; + return r; + } + + uint64_t internal_flags; + r = librbd::util::snap_create_flags_api_to_internal( + cct, librbd::util::get_default_snap_create_flags(group_ioctx), + &internal_flags); + if (r < 0) { + lderr(cct) << "error getting flags: " << cpp_strerror(r) << dendl; + return r; + } + + C_SaferCond cond; + auto req = mirror::GroupAddImageRequest<>::create( + group_ioctx, group_id, image_id, internal_flags, + static_cast(RBD_MIRROR_IMAGE_MODE_SNAPSHOT), &cond); + req->send(); + r = cond.wait(); + if (r < 0) { + lderr(cct) << "failed to add image to group: " + << cpp_strerror(r) << dendl; + return r; + } + return 0; +} + template int Mirror::group_promote(IoCtx& group_ioctx, const char *group_name, bool force) { diff --git a/src/librbd/api/Mirror.h b/src/librbd/api/Mirror.h index 8edc989a828..65ab4f04900 100644 --- a/src/librbd/api/Mirror.h +++ b/src/librbd/api/Mirror.h @@ -146,6 +146,8 @@ struct Mirror { mirror_image_mode_t group_image_mode); static int group_disable(IoCtx &group_ioctx, const char *group_name, bool force); + static int group_image_add(IoCtx &group_ioctx, const std::string &group_id, + IoCtx &image_ioctx, const std::string &image_id); static int group_promote(IoCtx &group_ioctx, const char *group_name, bool force); static int group_demote(IoCtx &group_ioctx, const char *group_name); diff --git a/src/librbd/mirror/GroupAddImageRequest.cc b/src/librbd/mirror/GroupAddImageRequest.cc new file mode 100644 index 00000000000..7e3ebc4849e --- /dev/null +++ b/src/librbd/mirror/GroupAddImageRequest.cc @@ -0,0 +1,803 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/mirror/GroupAddImageRequest.h" +#include "common/Cond.h" +#include "common/dout.h" +#include "common/errno.h" +#include "cls/rbd/cls_rbd_client.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "librbd/MirroringWatcher.h" +#include "librbd/Utils.h" +#include "librbd/mirror/ImageStateUpdateRequest.h" +#include "librbd/mirror/ImageRemoveRequest.h" +#include "librbd/mirror/snapshot/GroupImageCreatePrimaryRequest.h" +#include "librbd/mirror/snapshot/RemoveGroupSnapshotRequest.h" + +#include + +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "librbd::mirror::GroupAddImageRequest: " \ + << this << " " << __func__ << ": " + +namespace librbd { +namespace mirror { +namespace { + +const uint32_t MAX_RETURN = 1024; + +} // anonymous namespace + + +using util::create_context_callback; +using util::create_rados_callback; + + +template +GroupAddImageRequest::GroupAddImageRequest(librados::IoCtx &io_ctx, + const std::string &group_id, + const std::string &image_id, + uint64_t group_snap_create_flags, + cls::rbd::MirrorImageMode mode, + Context *on_finish) + : m_group_ioctx(io_ctx), m_group_id(group_id), m_image_id(image_id), + m_group_snap_create_flags(group_snap_create_flags), m_mode(mode), + m_on_finish(on_finish), m_cct(reinterpret_cast(io_ctx.cct())) { +} + +template +void GroupAddImageRequest::send() { + get_mirror_group(); +} + +template +void GroupAddImageRequest::get_mirror_group() { + ldout(m_cct, 10) << dendl; + + librados::ObjectReadOperation op; + cls_client::mirror_group_get_start(&op, m_group_id); + + using klass = GroupAddImageRequest; + librados::AioCompletion *comp = + create_rados_callback(this); + + m_out_bls.resize(1); + int r = m_group_ioctx.aio_operate(RBD_MIRRORING, comp, &op, &m_out_bls[0]); + ceph_assert(r == 0); + comp->release(); +} + +template +void GroupAddImageRequest::handle_get_mirror_group(int r) { + ldout(m_cct, 10) << "r=" << r << dendl; + + if (r == 0) { + auto iter = m_out_bls[0].cbegin(); + r = cls_client::mirror_group_get_finish(&iter, &m_mirror_group); + } + + m_out_bls[0].clear(); + + if (r == 0) { + if (m_mirror_group.mirror_image_mode != m_mode) { + lderr(m_cct) << "invalid group mirroring mode" << dendl; + r = -EINVAL; + } else if (m_mirror_group.state != cls::rbd::MIRROR_GROUP_STATE_ENABLED) { + ldout(m_cct, 10) << "mirroring on group is not enabled: " + << m_mirror_group.state << dendl; + r = -EINVAL; + } + if (r != 0) { + finish(r); + return; + } + } else if (r < 0) { + if (r == -EOPNOTSUPP) { + lderr(m_cct) << "mirroring on group is not supported by OSD" << dendl; + finish(r); + return; + } else if (r != -ENOENT) { + lderr(m_cct) << "failed to retrieve mirror group metadata: " + << cpp_strerror(r) << dendl; + finish(r); + return; + } + } + + list_group_images(); +} + +template +void GroupAddImageRequest::list_group_images() { + ldout(m_cct, 10) << dendl; + + librados::ObjectReadOperation op; + cls_client::group_image_list_start(&op, m_start_after, MAX_RETURN); + + auto comp = create_rados_callback< + GroupAddImageRequest, + &GroupAddImageRequest::handle_list_group_images>(this); + + int r = m_group_ioctx.aio_operate( + librbd::util::group_header_name(m_group_id), comp, &op, &m_out_bls[0]); + ceph_assert(r == 0); + comp->release(); +} + +template +void GroupAddImageRequest::handle_list_group_images(int r) { + ldout(m_cct, 10) << "r=" << r << dendl; + + std::vector images; + if (r == 0) { + auto iter = m_out_bls[0].cbegin(); + r = cls_client::group_image_list_finish(&iter, &images); + } + + m_out_bls[0].clear(); + + if (r < 0) { + lderr(m_cct) << "error listing images in group: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + auto image_count = images.size(); + m_images.insert(m_images.end(), images.begin(), images.end()); + if (image_count == MAX_RETURN) { + m_start_after = images.rbegin()->spec; + list_group_images(); + return; + } + + check_mirror_image_disabled(); +} + +template +void GroupAddImageRequest::check_mirror_image_disabled() { + ldout(m_cct, 10) << dendl; + + auto ctx = create_context_callback< + GroupAddImageRequest, + &GroupAddImageRequest::handle_check_mirror_image_disabled>(this); + auto gather_ctx = new C_Gather(m_cct, ctx); + + m_out_bls[0].clear(); + for (size_t i = 0; i < m_images.size(); i++) { + if (m_images[i].spec.image_id != m_image_id) { + continue; + } + librados::ObjectReadOperation op; + cls_client::mirror_image_get_start(&op, m_images[i].spec.image_id); + + auto on_mirror_image_get = new LambdaContext( + [this, i, new_sub_ctx=gather_ctx->new_sub()](int r) { + if (r == 0) { + auto iter = m_out_bls[0].cbegin(); + r = cls_client::mirror_image_get_finish(&iter, &m_mirror_images); + } + + if (r == -ENOENT) { + // image is disabled for mirroring as required + r = 0; + } else if (r == 0) { + lderr(m_cct) << "image_id=" << m_images[i].spec.image_id + << " is not disabled for mirroring" << dendl; + r = -EINVAL; + } else { + lderr(m_cct) << "failed to get mirror image info for image_id=" + << m_images[i].spec.image_id << dendl; + } + + new_sub_ctx->complete(r); + }); + + auto comp = create_rados_callback(on_mirror_image_get); + + int r = m_group_ioctx.aio_operate(RBD_MIRRORING, comp, &op, + &m_out_bls[0]); + ceph_assert(r == 0); + comp->release(); + } + + gather_ctx->activate(); +} + +template +void GroupAddImageRequest::handle_check_mirror_image_disabled(int r) { + ldout(m_cct, 10) << "r=" << r << dendl; + + m_out_bls[0].clear(); + + if (r < 0) { + lderr(m_cct) << "images not disabled for mirroring: " + << cpp_strerror(r) << dendl; + finish(r); + return; + } + + open_images(); +} + +template +void GroupAddImageRequest::open_images() { + ldout(m_cct, 10) << dendl; + + auto ctx = create_context_callback< + GroupAddImageRequest, &GroupAddImageRequest::handle_open_images>(this); + auto gather_ctx = new C_Gather(m_cct, ctx); + int r = 0; + std::vector ioctxs; + + for (const auto& image: m_images) { + librados::IoCtx image_io_ctx; + r = librbd::util::create_ioctx(m_group_ioctx, "image", + image.spec.pool_id, {}, + &image_io_ctx); + if (r < 0) { + finish(r); + return; + } + + ioctxs.push_back(std::move(image_io_ctx)); + } + + for (size_t i = 0; i < m_images.size(); i++) { + m_image_ctxs.push_back( + new ImageCtx("", m_images[i].spec.image_id.c_str(), nullptr, ioctxs[i], + false)); + + auto on_open = new LambdaContext( + [this, i, new_sub_ctx=gather_ctx->new_sub()](int r) { + // If asynchronous ImageState::open() fails, ImageState together with + // ImageCtx is destroyed. Simply NULL-out the respective image_ctxs[i] + // pointer to record that it's no longer valid. + if (r < 0) { + m_image_ctxs[i] = nullptr; + } + new_sub_ctx->complete(r); + }); + + // Open parent as well to check if the image is a clone + m_image_ctxs[i]->state->open(0, on_open); + } + + gather_ctx->activate(); +} + +template +void GroupAddImageRequest::handle_open_images(int r) { + ldout(m_cct, 10) << "r=" << r << dendl; + + if (r < 0) { + lderr(m_cct) << "failed to open group images: " << cpp_strerror(r) + << dendl; + m_ret_val = r; + + close_images(); + return; + } + + validate_images(); +} + +template +void GroupAddImageRequest::validate_images() { + ldout(m_cct, 10) << dendl; + + for (auto &image_ctx : m_image_ctxs) { + std::shared_lock image_locker{image_ctx->image_lock}; + if (image_ctx->parent) { + lderr(m_cct) << "cannot enable mirroring: cloned images are not " + << "supported" << dendl; + m_ret_val = -EINVAL; + + close_images(); + return; + } + } + + // FIXME: once the support for mirroring cloned images is added, need to + // check that the parents are enabled for mirroring + + // FIXME: for now images must belong to the same pool as the group + auto group_pool_id = m_group_ioctx.get_id(); + for (auto &image_ctx : m_image_ctxs) { + std::shared_lock image_locker{image_ctx->image_lock}; + if (image_ctx->md_ctx.get_id() != group_pool_id) { + lderr(m_cct) << "cannot add image to group for mirroring: image in a different pool" + << dendl; + m_ret_val = -EINVAL; + + close_images(); + return; + } + } + + get_mirror_images(); +} + +template +void GroupAddImageRequest::get_mirror_images() { + ldout(m_cct, 10) << dendl; + + if (m_images.empty()) { + get_mirror_peer_list(); + return; + } + + auto &spec = m_images.front().spec; + + ldout(m_cct, 10) << "pool_id: " << spec.pool_id + << ", image_id " << spec.image_id << dendl; + + // currently the group and its images must belong to the same pool. + + librados::ObjectReadOperation op; + librbd::cls_client::mirror_image_get_start(&op, spec.image_id); + m_out_bls[0].clear(); + auto comp = create_rados_callback< + GroupAddImageRequest, + &GroupAddImageRequest::handle_get_mirror_images>(this); + + int r = m_group_ioctx.aio_operate(RBD_MIRRORING, comp, &op, &m_out_bls[0]); + ceph_assert(r == 0); + comp->release(); +} + +template +void GroupAddImageRequest::handle_get_mirror_images(int r) { + ldout(m_cct, 10) << "r=" << r << dendl; + + auto &spec = m_images.front().spec; + cls::rbd::MirrorImage mirror_image; + + if (r == 0) { + auto iter = m_out_bls[0].cbegin(); + r = librbd::cls_client::mirror_image_get_finish(&iter, &mirror_image); + } + + if (r < 0 && r != -ENOENT) { + lderr(m_cct) << "error getting local mirror image: " + << cpp_strerror(r) << dendl; + finish(r); + return; + } else if (r == 0) { + m_local_images.emplace_back(spec.pool_id, mirror_image.global_image_id); + } + m_images.erase(m_images.begin()); + get_mirror_images(); +} + +template +void GroupAddImageRequest::get_mirror_peer_list() { + ldout(m_cct, 10) << dendl; + + m_default_ns_ioctx.dup(m_group_ioctx); + m_default_ns_ioctx.set_namespace(""); + + librados::ObjectReadOperation op; + cls_client::mirror_peer_list_start(&op); + + auto comp = create_rados_callback< + GroupAddImageRequest, + &GroupAddImageRequest::handle_get_mirror_peer_list>(this); + + int r = m_default_ns_ioctx.aio_operate(RBD_MIRRORING, comp, &op, + &m_out_bls[0]); + ceph_assert(r == 0); + comp->release(); +} + +template +void GroupAddImageRequest::handle_get_mirror_peer_list(int r) { + ldout(m_cct, 10) << "r=" << r << dendl; + + std::vector peers; + if (r == 0) { + auto it = m_out_bls[0].cbegin(); + r = cls_client::mirror_peer_list_finish(&it, &peers); + } + + m_out_bls[0].clear(); + + if (r < 0) { + lderr(m_cct) << "failed to retrieve mirror peers: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + for (auto &peer : peers) { + if (peer.mirror_peer_direction == cls::rbd::MIRROR_PEER_DIRECTION_RX) { + continue; + } + m_mirror_peer_uuids.insert(peer.uuid); + } + + if (m_mirror_peer_uuids.empty()) { + lderr(m_cct) << "no mirror tx peers configured for the pool" << dendl; + finish(-EINVAL); + return; + } + + create_primary_group_snapshot(); +} + +template +void GroupAddImageRequest::create_primary_group_snapshot() { + ldout(m_cct, 10) << dendl; + + m_group_snap.id = librbd::util::generate_image_id(m_group_ioctx); + + auto snap_name = ".mirror.primary." + m_mirror_group.global_group_id + + "." + m_group_snap.id; + m_group_snap.name = snap_name; + + librados::Rados rados(m_group_ioctx); + int8_t require_osd_release; + int r = rados.get_min_compatible_osd(&require_osd_release); + if (r < 0) { + lderr(m_cct) << "failed to retrieve min OSD release: " << cpp_strerror(r) + << dendl; + return; + } + + auto complete = cls::rbd::get_mirror_group_snapshot_complete_initial( + require_osd_release); + m_group_snap.snapshot_namespace = cls::rbd::GroupSnapshotNamespaceMirror{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, m_mirror_peer_uuids, {}, {}, + complete}; + + for (auto image_ctx: m_image_ctxs) { + m_group_snap.snaps.emplace_back(image_ctx->md_ctx.get_id(), image_ctx->id, + CEPH_NOSNAP); + } + + librados::ObjectWriteOperation op; + cls_client::group_snap_set(&op, m_group_snap); + + auto aio_comp = create_rados_callback< + GroupAddImageRequest, + &GroupAddImageRequest::handle_create_primary_group_snapshot>(this); + r = m_group_ioctx.aio_operate(librbd::util::group_header_name(m_group_id), + aio_comp, &op); + ceph_assert(r == 0); + aio_comp->release(); +} + +template +void GroupAddImageRequest::handle_create_primary_group_snapshot(int r) { + ldout(m_cct, 10) << "r=" << r << dendl; + + if (r < 0) { + lderr(m_cct) << "failed to create group snapshot: " + << cpp_strerror(r) << dendl; + m_ret_val = r; + + remove_primary_group_snapshot(); + return; + } + + create_primary_image_snapshots(); +} + +template +void GroupAddImageRequest::create_primary_image_snapshots() { + ldout(m_cct, 10) << dendl; + + auto num_images = m_image_ctxs.size(); + m_global_image_ids.resize(num_images); + + for (size_t i = 0; i < num_images; i++) { + if (i < m_local_images.size() && + !m_local_images[i].second.empty()) { + // existing image already in the group + m_global_image_ids[i] = m_local_images[i].second; + } else { + // new image being added to the group, hence generate new global id + uuid_d uuid_gen; + uuid_gen.generate_random(); + m_mirror_images.type = cls::rbd::MIRROR_IMAGE_TYPE_GROUP; + m_mirror_images.mode = m_mode; + m_mirror_images.global_image_id = uuid_gen.to_string(); + + m_global_image_ids[i] = m_mirror_images.global_image_id; + } + } + + auto ctx = librbd::util::create_context_callback< + GroupAddImageRequest, + &GroupAddImageRequest::handle_create_primary_image_snapshots>(this); + + m_snap_ids.resize(num_images, CEPH_NOSNAP); + + auto req = snapshot::GroupImageCreatePrimaryRequest::create( + m_cct, m_image_ctxs, m_global_image_ids, m_group_snap_create_flags, + snapshot::CREATE_PRIMARY_FLAG_IGNORE_EMPTY_PEERS, m_group_snap.id, + &m_snap_ids, ctx); + + req->send(); +} + +template +void GroupAddImageRequest::handle_create_primary_image_snapshots(int r) { + ldout(m_cct, 10) << "r=" << r << dendl; + + if (r < 0) { + lderr(m_cct) << "failed to create primary mirror image snapshots: " + << cpp_strerror(r) << dendl; + m_ret_val = r; + + for (size_t i = 0; i < m_image_ctxs.size(); i++) { + m_group_snap.snaps[i].snap_id = m_snap_ids[i]; + } + + remove_primary_group_snapshot(); + return; + } + + update_primary_group_snapshot(); +} + +template +void GroupAddImageRequest::update_primary_group_snapshot() { + ldout(m_cct, 10) << dendl; + + for (size_t i = 0; i < m_image_ctxs.size(); i++) { + m_group_snap.snaps[i].snap_id = m_snap_ids[i]; + } + + m_group_snap.state = cls::rbd::GROUP_SNAPSHOT_STATE_CREATED; + cls::rbd::set_mirror_group_snapshot_complete(m_group_snap); + + librados::ObjectWriteOperation op; + cls_client::group_snap_set(&op, m_group_snap); + + auto aio_comp = create_rados_callback< + GroupAddImageRequest, + &GroupAddImageRequest::handle_update_primary_group_snapshot>(this); + int r = m_group_ioctx.aio_operate(librbd::util::group_header_name(m_group_id), + aio_comp, &op); + ceph_assert(r == 0); + aio_comp->release(); +} + +template +void GroupAddImageRequest::handle_update_primary_group_snapshot(int r) { + ldout(m_cct, 10) << "r=" << r << dendl; + + if (r < 0) { + lderr(m_cct) << "failed to create group snapshot: " + << cpp_strerror(r) << dendl; + m_ret_val = r; + + remove_primary_group_snapshot(); + return; + } + + set_mirror_image_enabled(); +} + +template +void GroupAddImageRequest::set_mirror_image_enabled() { + ldout(m_cct, 10) << dendl; + + auto ctx = create_context_callback< + GroupAddImageRequest, + &GroupAddImageRequest::handle_set_mirror_image_enabled>(this); + + auto gather_ctx = new C_Gather(m_cct, ctx); + + for (size_t i = 0; i < m_image_ctxs.size(); i++) { + auto ictx = m_image_ctxs[i]; + if (ictx->id != m_image_id) { + continue; + } + auto req = ImageStateUpdateRequest::create( + ictx->md_ctx, ictx->id, cls::rbd::MIRROR_IMAGE_STATE_ENABLED, + m_mirror_images, gather_ctx->new_sub()); + + req->send(); + } + + gather_ctx->activate(); +} + +template +void GroupAddImageRequest::handle_set_mirror_image_enabled(int r) { + ldout(m_cct, 10) << "r=" << r << dendl; + + m_need_to_cleanup_mirror_image = true; + + if (r < 0) { + lderr(m_cct) << "failed to enabled mirror images: " << cpp_strerror(r) + << dendl; + m_ret_val = r; + + disable_mirror_image(); + return; + } + + notify_mirroring_watcher(); +} + +template +void GroupAddImageRequest::notify_mirroring_watcher() { + ldout(m_cct, 10) << dendl; + + auto ctx = util::create_context_callback< + GroupAddImageRequest, + &GroupAddImageRequest::handle_notify_mirroring_watcher>(this); + + MirroringWatcher::notify_group_updated( + m_group_ioctx, cls::rbd::MIRROR_GROUP_STATE_ENABLED, m_group_id, + m_mirror_group.global_group_id, m_image_ctxs.size(), ctx); +} + +template +void GroupAddImageRequest::handle_notify_mirroring_watcher(int r) { + ldout(m_cct, 10) << "r=" << r << dendl; + + if (r < 0) { + lderr(m_cct) << "failed to notify mirror group update: " << cpp_strerror(r) + << dendl; + m_ret_val = r; + } + + close_images(); +} + +template +void GroupAddImageRequest::close_images() { + ldout(m_cct, 10) << dendl; + + auto ctx = create_context_callback< + GroupAddImageRequest, + &GroupAddImageRequest::handle_close_images>(this); + + auto gather_ctx = new C_Gather(m_cct, ctx); + + for (auto ictx: m_image_ctxs) { + if (ictx != nullptr) { + ictx->state->close(gather_ctx->new_sub()); + } + } + + gather_ctx->activate(); +} + +template +void GroupAddImageRequest::handle_close_images(int r) { + ldout(m_cct, 10) << "r=" << r << dendl; + + if (r < 0) { + lderr(m_cct) << "failed to close images: " << cpp_strerror(r) << dendl; + if (m_ret_val == 0) { + m_ret_val = r; + } + } + + finish(m_ret_val); +} + +template +void GroupAddImageRequest::disable_mirror_image() { + ldout(m_cct, 10) << dendl; + + auto ctx = create_context_callback< + GroupAddImageRequest, + &GroupAddImageRequest::handle_disable_mirror_image>(this); + + auto gather_ctx = new C_Gather(m_cct, ctx); + for (size_t i = 0; i < m_image_ctxs.size(); i++) { + if (m_image_ctxs[i]->id != m_image_id) { + continue; + } + if (m_mirror_images.state != cls::rbd::MIRROR_IMAGE_STATE_DISABLED) { + auto req = ImageStateUpdateRequest::create( + m_image_ctxs[i]->md_ctx, m_image_ctxs[i]->id, + cls::rbd::MIRROR_IMAGE_STATE_DISABLING, m_mirror_images, + gather_ctx->new_sub()); + req->send(); + } + } + gather_ctx->activate(); +} + +template +void GroupAddImageRequest::handle_disable_mirror_image(int r) { + ldout(m_cct, 10) << "r=" << dendl; + + if (r < 0) { + lderr(m_cct) << "failed to disable mirror images: " << cpp_strerror(r) + << dendl; + close_images(); + return; + } + + remove_primary_group_snapshot(); +} + +template +void GroupAddImageRequest::remove_primary_group_snapshot() { + ldout(m_cct, 10) << dendl; + + auto ctx = create_context_callback< + GroupAddImageRequest, + &GroupAddImageRequest::handle_remove_primary_group_snapshot>(this); + + auto req = snapshot::RemoveGroupSnapshotRequest::create(m_group_ioctx, + m_group_id, &m_group_snap, &m_image_ctxs, ctx); + + req->send(); +} + +template +void GroupAddImageRequest::handle_remove_primary_group_snapshot(int r) { + ldout(m_cct, 10) << "r=" << r << dendl; + + if (r < 0) { + lderr(m_cct) << "failed to remove mirror group snapshot: " + << cpp_strerror(r) << dendl; + close_images(); + return; + } + + if (m_need_to_cleanup_mirror_image) { + remove_mirror_image(); + } else { + close_images(); + } +} + +template +void GroupAddImageRequest::remove_mirror_image() { + ldout(m_cct, 10) << dendl; + + auto ctx = create_context_callback< + GroupAddImageRequest, + &GroupAddImageRequest::handle_remove_mirror_image>(this); + + auto gather_ctx = new C_Gather(m_cct, ctx); + for (size_t i = 0; i < m_image_ctxs.size(); i++) { + if (m_image_ctxs[i]->id != m_image_id) { + continue; + } + auto req = ImageRemoveRequest::create( + m_image_ctxs[i]->md_ctx, m_global_image_ids[i], m_image_ctxs[i]->id, + gather_ctx->new_sub()); + req->send(); + } + gather_ctx->activate(); +} + +template +void GroupAddImageRequest::handle_remove_mirror_image(int r) { + ldout(m_cct, 10) << "r=" << r << dendl; + + if (r < 0) { + lderr(m_cct) << "failed to remove mirror images: " << cpp_strerror(r) + << dendl; + close_images(); + return; + } + + close_images(); +} + +template +void GroupAddImageRequest::finish(int r) { + ldout(m_cct, 10) << "r=" << r << dendl; + + m_on_finish->complete(r); + delete this; +} + +} // namespace mirror +} // namespace librbd + +template class librbd::mirror::GroupAddImageRequest; diff --git a/src/librbd/mirror/GroupAddImageRequest.h b/src/librbd/mirror/GroupAddImageRequest.h new file mode 100644 index 00000000000..d497384e1b4 --- /dev/null +++ b/src/librbd/mirror/GroupAddImageRequest.h @@ -0,0 +1,182 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_GROUP_ADD_IMAGE_REQUEST_H +#define CEPH_LIBRBD_MIRROR_GROUP_ADD_IMAGE_REQUEST_H + +#include "include/buffer.h" +#include "include/rados/librados_fwd.hpp" +#include "include/rbd/librbd.hpp" +#include "cls/rbd/cls_rbd_types.h" +#include "librbd/ImageCtx.h" +#include "librbd/mirror/Types.h" +#include +#include + +class Context; + +namespace librbd { + +namespace asio { struct ContextWQ; } + +namespace mirror { + +template +class GroupAddImageRequest { +public: + static GroupAddImageRequest *create(librados::IoCtx &group_io_ctx, + const std::string &group_id, + const std::string &image_id, + uint64_t group_snap_create_flags, + cls::rbd::MirrorImageMode mode, + Context *on_finish) { + return new GroupAddImageRequest(group_io_ctx, group_id, image_id, + group_snap_create_flags, mode, on_finish); + } + + void send(); + +private: + /** + * @verbatim + * + * + * | + * v (on error) + * GET_MIRROR_GROUP * * * * * * * * * * * + * | * + * v * + * LIST_GROUP_IMAGES * * * * * * * * * * * + * | * + * v (skip if not needed) * + * CHECK_MIRROR_IMAGE_DISABLED * * * * * * + * | * + * v (skip if not needed) * + * OPEN_IMAGES * * * * * * * * * * * * * + * | * + * v (skip if not needed) * + * VALIDATE_IMAGES * * * * * * * * * * * + * | * + * v * + * GET_MIRROR_IMAGES * * * * * * * * * * + * | * + * v * + * GET_MIRROR_PEERS * * * * * * * * * * * + * | * + * v (incomplete) * + * CREATE_PRIMARY_GROUP_SNAP * * * * * * * + * | * + * v (skip if not needed) * + * CREATE_PRIMARY_IMAGE_SNAPS * + * | * + * v (complete) * + * UPDATE_PRIMARY_GROUP_SNAP * * * * * * * + * | * + * v (skip if not needed) * + * SET_MIRROR_IMAGE_ENABLED * * * * * * * + * | * + * v (if required) + * NOTIFY_MIRRORING_WATCHER cleanup + * | * + * v (skip if not needed) * + * CLOSE_IMAGES * * * * * * * * * * * * * + * | + * v + * + * + * @endverbatim + */ + + GroupAddImageRequest(librados::IoCtx &io_ctx, const std::string &group_id, + const std::string &image_id, + uint64_t group_snap_create_flags, + cls::rbd::MirrorImageMode mode, Context *on_finish); + + typedef std::pair GlobalImageId; + + librados::IoCtx &m_group_ioctx; + const std::string m_group_id; + const std::string m_image_id; + uint64_t m_group_snap_create_flags; + const cls::rbd::MirrorImageMode m_mode; + Context *m_on_finish; + + CephContext *m_cct = nullptr; + std::vector m_out_bls; + cls::rbd::MirrorGroup m_mirror_group; + + int m_ret_val = 0; + librados::IoCtx m_default_ns_ioctx; + + std::vector m_image_ctxs; + cls::rbd::MirrorImage m_mirror_images; + + std::set m_mirror_peer_uuids; + cls::rbd::GroupImageSpec m_start_after; + std::vector m_images; + std::vector m_local_images; + + cls::rbd::GroupSnapshot m_group_snap; + std::vector m_snap_ids; + + std::vector m_global_image_ids; + + bool m_need_to_cleanup_mirror_image = false; + + void get_mirror_group(); + void handle_get_mirror_group(int r); + + void list_group_images(); + void handle_list_group_images(int r); + + void check_mirror_image_disabled(); + void handle_check_mirror_image_disabled(int r); + + void open_images(); + void handle_open_images(int r); + + void validate_images(); + + void get_mirror_images(); + void handle_get_mirror_images(int r); + + void get_mirror_peer_list(); + void handle_get_mirror_peer_list(int r); + + void create_primary_group_snapshot(); + void handle_create_primary_group_snapshot(int r); + + void create_primary_image_snapshots(); + void handle_create_primary_image_snapshots(int r); + + void update_primary_group_snapshot(); + void handle_update_primary_group_snapshot(int r); + + void set_mirror_image_enabled(); + void handle_set_mirror_image_enabled(int r); + + void notify_mirroring_watcher(); + void handle_notify_mirroring_watcher(int r); + + void close_images(); + void handle_close_images(int r); + + // cleanup + void disable_mirror_image(); + void handle_disable_mirror_image(int r); + + void remove_primary_group_snapshot(); + void handle_remove_primary_group_snapshot(int r); + + void remove_mirror_image(); + void handle_remove_mirror_image(int r); + + void finish(int r); +}; + +} // namespace mirror +} // namespace librbd + +extern template class librbd::mirror::GroupAddImageRequest; + +#endif // CEPH_LIBRBD_MIRROR_GROUP_ADD_IMAGE_REQUEST_H diff --git a/src/tools/rbd_mirror/group_replayer/BootstrapRequest.cc b/src/tools/rbd_mirror/group_replayer/BootstrapRequest.cc index e9833868bf7..ff625066938 100644 --- a/src/tools/rbd_mirror/group_replayer/BootstrapRequest.cc +++ b/src/tools/rbd_mirror/group_replayer/BootstrapRequest.cc @@ -284,20 +284,34 @@ template int BootstrapRequest::create_replayers() { dout(10) << dendl; - //TODO: check that the images have not changed - if (!m_image_replayers->empty()) { - dout(10) << "image replayers already exist."<< dendl; - return 0; - } - auto state_builder = *m_state_builder; int r = 0; if ((*m_state_builder)->is_local_primary()) { + //TODO: check that the images have not changed + if (m_image_replayers->size() == (*m_state_builder)->local_images.size()) { + dout(10) << "image replayers already exist."<< dendl; + return 0; + } // The ImageReplayers are required to run even when the group is primary in // order to update the image status for the mirror pool status to be healthy. for (auto &[global_image_id, p] : (*m_state_builder)->local_images) { auto &local_pool_id = p.first; + bool is_image_replayer_exist = false; + for (auto &[_, image_replayer] : *m_image_replayers) { + auto exising_replayer_global_image_id = + image_replayer->get_global_image_id(); + if (exising_replayer_global_image_id == global_image_id) { + is_image_replayer_exist = true; + break; + } + } + + if (is_image_replayer_exist) { + dout(10) << "image replayer for global image id: " << global_image_id + << "already exists" << dendl; + continue; + } m_image_replayers->emplace_back(librados::IoCtx(), nullptr); auto &local_io_ctx = m_image_replayers->back().first; @@ -360,7 +374,27 @@ int BootstrapRequest::create_replayers() { remote_pool_meta, m_remote_status_updater}); } } else if (!state_builder->remote_group_id.empty()) { + //TODO: check that the images have not changed + if (m_image_replayers->size() == (*m_state_builder)->remote_images.size()) { + dout(10) << "image replayers already exist."<< dendl; + return 0; + } for (auto &[remote_pool_id, global_image_id] : (*m_state_builder)->remote_images) { + bool is_image_replayer_exist = false; + for (auto &[_, image_replayer] : *m_image_replayers) { + auto exising_replayer_global_image_id = + image_replayer->get_global_image_id(); + if (exising_replayer_global_image_id == global_image_id) { + is_image_replayer_exist = true; + break; + } + } + + if (is_image_replayer_exist) { + dout(10) << "image replayer for global image id: " << global_image_id + << "already exists" << dendl; + continue; + } m_image_replayers->emplace_back(librados::IoCtx(), nullptr); auto &local_io_ctx = m_image_replayers->back().first; -- 2.47.3