group_image_add "${primary_cluster}" "${pool}/${group}" "${pool}/child_image"
# command fails with the following message now
- # 2025-01-30T16:34:25.359+0000 7fc1a79bfb40 -1 librbd::api::Mirror: image_enable: mirroring is not enabled for the parent
- # 2025-01-30T16:34:25.359+0000 7fc1a79bfb40 -1 librbd::api::Mirror: group_enable: failed enabling image: child_image: (22) Invalid argument
- expect_failure "failed enabling image" rbd --cluster=${primary_cluster} mirror group enable ${pool}/${group}
+ # 2025-07-29T11:09:54.666-0400 7f06696006c0 -1 librbd::mirror::GroupEnableRequest: 0x5622625a6240 validate_images: cannot enable mirroring: cloned images are not supported
+ # 2025-07-29T11:09:54.673-0400 7f0671958d00 -1 librbd::api::Mirror: group_enable: failed to mirror enable group: (22) Invalid argument
+ expect_failure "cloned images are not supported" rbd --cluster=${primary_cluster} mirror group enable ${pool}/${group}
# tidy up
group_remove "${primary_cluster}" "${pool}/${group}"
group_image_add "${primary_cluster}" "${pool0}/${group}" "${pool1}/${image_prefix}1"
# command fails with the following message now
- # 2025-06-12T17:33:25.241+0530 7fe40ccfbd00 -1 librbd::api::Mirror: prepare_group_images: cannot enable mirroring: image is in a different pool
- expect_failure "cannot enable mirroring: image is in a different pool" rbd --cluster=${primary_cluster} mirror group enable ${pool0}/${group}
+ # 2025-07-29T11:16:24.690-0400 7ff66b4006c0 -1 librbd::mirror::GroupEnableRequest: 0x55c172e13ec0 validate_images: cannot enable mirroring: image in a different pool
+ # 2025-07-29T11:16:24.693-0400 7ff6778b3d00 -1 librbd::api::Mirror: group_enable: failed to mirror enable group: (22) Invalid argument
+ expect_failure "image in a different pool" rbd --cluster=${primary_cluster} mirror group enable ${pool0}/${group}
# tidy up
group_remove "${primary_cluster}" "${pool0}/${group}"
mirror/GetInfoRequest.cc
mirror/GetStatusRequest.cc
mirror/GetUuidRequest.cc
+ mirror/GroupEnableRequest.cc
mirror/GroupGetInfoRequest.cc
mirror/ImageRemoveRequest.cc
mirror/ImageStateUpdateRequest.cc
mirror/snapshot/DemoteRequest.cc
mirror/snapshot/GetImageStateRequest.cc
mirror/snapshot/GroupCreatePrimaryRequest.cc
+ mirror/snapshot/GroupImageCreatePrimaryRequest.cc
mirror/snapshot/GroupUnlinkPeerRequest.cc
mirror/snapshot/ImageMeta.cc
mirror/snapshot/PromoteRequest.cc
#include "librbd/mirror/GetInfoRequest.h"
#include "librbd/mirror/GetStatusRequest.h"
#include "librbd/mirror/GetUuidRequest.h"
+#include "librbd/mirror/GroupEnableRequest.h"
#include "librbd/mirror/GroupGetInfoRequest.h"
#include "librbd/mirror/PromoteRequest.h"
#include "librbd/mirror/Types.h"
return -EINVAL;
}
- cls::rbd::MirrorGroup mirror_group;
- r = cls_client::mirror_group_get(&group_ioctx, group_id, &mirror_group);
- if (r == 0) {
- auto mode = static_cast<rbd_mirror_image_mode_t>(
- mirror_group.mirror_image_mode);
- if (mode != mirror_image_mode) {
- lderr(cct) << "invalid group mirroring mode" << dendl;
- r = -EINVAL;
- } else if (mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLING) {
- ldout(cct, 10) << "mirroring on group is in-progress of enabling"
- << dendl;
- r = -EINVAL;
- } else if (mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED) {
- ldout(cct, 10) << "mirroring on group is already enabled" << dendl;
- } else if (mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_DISABLING) {
- lderr(cct) << "mirroring on group is currently disabling" << dendl;
- r = -EINVAL;
- } else {
- lderr(cct) << "mirroring on group is in unexpected state: "
- << mirror_group.state << dendl;
- r = -EINVAL;
- }
- return r;
- } else if (r < 0) {
- if (r == -EOPNOTSUPP) {
- lderr(cct) << "mirroring on group is not supported by OSD" << dendl;
- return r;
- } else if (r != -ENOENT) {
- lderr(cct) << "failed to retrieve mirror group metadata: "
- << cpp_strerror(r) << dendl;
- return r;
- }
- }
-
- uuid_d uuid_gen;
- uuid_gen.generate_random();
-
- mirror_group = {uuid_gen.to_string(),
- static_cast<cls::rbd::MirrorImageMode>(mirror_image_mode),
- cls::rbd::MIRROR_GROUP_STATE_ENABLING};
-
- r = cls_client::mirror_group_set(&group_ioctx, group_id, mirror_group);
+ C_SaferCond cond;
+ auto req = mirror::GroupEnableRequest<>::create(
+ group_ioctx, group_id, internal_flags,
+ static_cast<cls::rbd::MirrorImageMode>(mirror_image_mode), &cond);
+ req->send();
+ r = cond.wait();
if (r < 0) {
- lderr(cct) << "failed to set mirroring group metadata: "
+ lderr(cct) << "failed to mirror enable group: "
<< cpp_strerror(r) << dendl;
return r;
}
-
- std::string group_snap_id = librbd::util::generate_image_id(group_ioctx);
- cls::rbd::GroupSnapshot group_snap{
- group_snap_id,
- cls::rbd::GroupSnapshotNamespaceMirror{},
- prepare_primary_mirror_snap_name(cct, uuid_gen.to_string(),
- group_snap_id),
- cls::rbd::GROUP_SNAPSHOT_STATE_INCOMPLETE};
-
- std::vector<uint64_t> quiesce_requests;
- std::vector<I *> image_ctxs;
- std::set<std::string> mirror_peer_uuids;
- int ret_code = 0;
- r = prepare_group_images(group_ioctx, group_id, mirror_group.state,
- &image_ctxs, &group_snap, quiesce_requests,
- cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY,
- &mirror_peer_uuids,
- internal_flags);
- if (r != 0) {
- ret_code = r;
- }
-
- auto image_count = image_ctxs.size();
- std::string group_header_oid = librbd::util::group_header_name(group_id);
- std::vector<uint64_t> snap_ids(image_ctxs.size(), CEPH_NOSNAP);
- if (ret_code) {
- goto cleanup;
- }
-
- for (size_t i = 0; i < image_ctxs.size(); i++) {
- r = image_enable(image_ctxs[i], group_snap_id, mirror_image_mode, false,
- &snap_ids[i]);
- group_snap.snaps[i].snap_id = snap_ids[i];
- if (r < 0) {
- lderr(cct) << "failed enabling image: "
- << image_ctxs[i]->name << ": " << cpp_strerror(r) << dendl;
- if (ret_code == 0) {
- ret_code = r;
- break;
- }
- }
- }
-
- if (!ret_code) {
- group_snap.state = cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE;
- r = cls_client::group_snap_set(&group_ioctx, group_header_oid, group_snap);
- if (r < 0) {
- lderr(cct) << "failed to update group snapshot metadata: "
- << cpp_strerror(r) << dendl;
- ret_code = r;
- goto cleanup;
- }
-
- mirror_group.state = cls::rbd::MIRROR_GROUP_STATE_ENABLED;
- r = cls_client::mirror_group_set(&group_ioctx, group_id, mirror_group);
- if (r < 0) {
- lderr(cct) << "failed to update mirroring group metadata: "
- << cpp_strerror(r) << dendl;
- ret_code = r;
- }
- }
-
-cleanup:
- if (ret_code) {
- // undo
- ldout(cct, 20) << "undoing group enable: " << ret_code << dendl;
- remove_group_snap(group_ioctx, group_id, &group_snap, &image_ctxs);
- // MirrorImage need to be removed, hence calling image disable is mandatory
- r = 0;
- for (size_t i = 0; i < image_ctxs.size(); i++) {
- if (snap_ids[i] == CEPH_NOSNAP) {
- continue;
- }
- r = image_disable(image_ctxs[i], false, true);
- if (r < 0) {
- lderr(cct) << "failed to disable mirroring on image: "
- << image_ctxs[i]->name << cpp_strerror(r) << dendl;
- break;
- }
- }
-
- if (r == 0) {
- mirror_group.state = cls::rbd::MIRROR_GROUP_STATE_DISABLING;
- r = cls_client::mirror_group_set(&group_ioctx, group_id, mirror_group);
- if (r < 0) {
- ret_code = r;
- lderr(cct) << "failed to update mirroring group metadata: "
- << cpp_strerror(r) << dendl;
- }
- }
-
- if (r == 0) {
- r = cls_client::mirror_group_remove(&group_ioctx, group_id);
- if (r < 0 && r != -ENOENT) {
- ret_code = r;
- lderr(cct) << "failed to remove mirroring group metadata: "
- << cpp_strerror(r) << dendl;
- }
- }
- }
-
- if (!quiesce_requests.empty()) {
- util::notify_unquiesce(image_ctxs, quiesce_requests);
- }
-
- close_images(&image_ctxs);
-
- if (!ret_code) {
- r = MirroringWatcher<I>::notify_group_updated(
- group_ioctx, cls::rbd::MIRROR_GROUP_STATE_ENABLED, group_id,
- mirror_group.global_group_id, image_count);
- if (r < 0) {
- lderr(cct) << "failed to notify mirroring group=" << group_name
- << " updated: " << cpp_strerror(r) << dendl;
- // not fatal
- }
- }
-
- return ret_code;
+ return 0;
}
template <typename I>
int Mirror<I>::group_get_info(librados::IoCtx& io_ctx,
const std::string &group_name,
mirror_group_info_t *mirror_group_info) {
- CephContext *cct((CephContext *)io_ctx.cct());
- ldout(cct, 20) << "group_name=" << group_name << dendl;
-
C_SaferCond ctx;
group_get_info(io_ctx, group_name, mirror_group_info, &ctx);
int r = ctx.wait();
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "librbd/mirror/GroupEnableRequest.h"
+#include "common/Cond.h"
+#include "common/dout.h"
+#include "common/errno.h"
+#include "cls/rbd/cls_rbd_client.h"
+#include "librbd/ImageCtx.h"
+#include "librbd/ImageState.h"
+#include "librbd/MirroringWatcher.h"
+#include "librbd/Utils.h"
+#include "librbd/mirror/ImageStateUpdateRequest.h"
+#include "librbd/mirror/ImageRemoveRequest.h"
+#include "librbd/mirror/snapshot/GroupImageCreatePrimaryRequest.h"
+#include "librbd/mirror/snapshot/RemoveGroupSnapshotRequest.h"
+
+#include <shared_mutex>
+
+#define dout_subsys ceph_subsys_rbd
+#undef dout_prefix
+#define dout_prefix *_dout << "librbd::mirror::GroupEnableRequest: " \
+ << this << " " << __func__ << ": "
+
+namespace librbd {
+namespace mirror {
+namespace {
+
+const uint32_t MAX_RETURN = 1024;
+
+} // anonymous namespace
+
+
+using util::create_context_callback;
+using util::create_rados_callback;
+
+
+template <typename I>
+GroupEnableRequest<I>::GroupEnableRequest(librados::IoCtx &io_ctx,
+ const std::string &group_id,
+ uint64_t group_snap_create_flags,
+ cls::rbd::MirrorImageMode mode,
+ Context *on_finish)
+ : m_group_ioctx(io_ctx), m_group_id(group_id),
+ m_group_snap_create_flags(group_snap_create_flags), m_mode(mode),
+ m_on_finish(on_finish), m_cct(reinterpret_cast<CephContext*>(io_ctx.cct())) {
+}
+
+template <typename I>
+void GroupEnableRequest<I>::send() {
+ get_mirror_group();
+}
+
+template <typename I>
+void GroupEnableRequest<I>::get_mirror_group() {
+ ldout(m_cct, 10) << dendl;
+
+ librados::ObjectReadOperation op;
+ cls_client::mirror_group_get_start(&op, m_group_id);
+
+ using klass = GroupEnableRequest<I>;
+ librados::AioCompletion *comp =
+ create_rados_callback<klass, &klass::handle_get_mirror_group>(this);
+
+ m_out_bls.resize(1);
+ int r = m_group_ioctx.aio_operate(RBD_MIRRORING, comp, &op, &m_out_bls[0]);
+ ceph_assert(r == 0);
+ comp->release();
+}
+
+template <typename I>
+void GroupEnableRequest<I>::handle_get_mirror_group(int r) {
+ ldout(m_cct, 10) << "r=" << r << dendl;
+
+ if (r == 0) {
+ auto iter = m_out_bls[0].cbegin();
+ r = cls_client::mirror_group_get_finish(&iter, &m_mirror_group);
+ }
+
+ m_out_bls[0].clear();
+
+ if (r == 0) {
+ if (m_mirror_group.mirror_image_mode != m_mode) {
+ lderr(m_cct) << "invalid group mirroring mode" << dendl;
+ r = -EINVAL;
+ } else if (m_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLING) {
+ lderr(m_cct) << "mirroring on group is currently enabling"
+ << dendl;
+ r = -EINVAL;
+ } else if (m_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED) {
+ ldout(m_cct, 10) << "mirroring on group is already enabled" << dendl;
+ } else if (
+ m_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_DISABLING) {
+ lderr(m_cct) << "mirroring on group is currently disabling" << dendl;
+ r = -EINVAL;
+ } else {
+ lderr(m_cct) << "mirroring on group is in unexpected state: "
+ << m_mirror_group.state << dendl;
+ r = -EINVAL;
+ }
+ finish(r);
+ return;
+ } else if (r < 0) {
+ if (r == -EOPNOTSUPP) {
+ lderr(m_cct) << "mirroring on group is not supported by OSD" << dendl;
+ finish(r);
+ return;
+ } else if (r != -ENOENT) {
+ lderr(m_cct) << "failed to retrieve mirror group metadata: "
+ << cpp_strerror(r) << dendl;
+ finish(r);
+ return;
+ }
+ }
+
+ uuid_d uuid_gen;
+ uuid_gen.generate_random();
+ m_mirror_group.global_group_id = uuid_gen.to_string();
+ m_mirror_group.mirror_image_mode = m_mode;
+
+ get_mirror_peer_list();
+}
+
+template <typename I>
+void GroupEnableRequest<I>::get_mirror_peer_list() {
+ ldout(m_cct, 10) << dendl;
+
+ m_default_ns_ioctx.dup(m_group_ioctx);
+ m_default_ns_ioctx.set_namespace("");
+
+ librados::ObjectReadOperation op;
+ cls_client::mirror_peer_list_start(&op);
+
+ auto comp = create_rados_callback<
+ GroupEnableRequest<I>,
+ &GroupEnableRequest<I>::handle_get_mirror_peer_list>(this);
+
+ int r = m_default_ns_ioctx.aio_operate(RBD_MIRRORING, comp, &op,
+ &m_out_bls[0]);
+ ceph_assert(r == 0);
+ comp->release();
+}
+
+template <typename I>
+void GroupEnableRequest<I>::handle_get_mirror_peer_list(int r) {
+ ldout(m_cct, 10) << "r=" << r << dendl;
+
+ std::vector<cls::rbd::MirrorPeer> peers;
+ if (r == 0) {
+ auto it = m_out_bls[0].cbegin();
+ r = cls_client::mirror_peer_list_finish(&it, &peers);
+ }
+
+ m_out_bls[0].clear();
+
+ if (r < 0) {
+ lderr(m_cct) << "failed to retrieve mirror peers: " << cpp_strerror(r)
+ << dendl;
+ finish(r);
+ return;
+ }
+
+ for (auto &peer : peers) {
+ if (peer.mirror_peer_direction == cls::rbd::MIRROR_PEER_DIRECTION_RX) {
+ continue;
+ }
+ m_mirror_peer_uuids.insert(peer.uuid);
+ }
+
+ if (m_mirror_peer_uuids.empty()) {
+ lderr(m_cct) << "no mirror tx peers configured for the pool" << dendl;
+ finish(-EINVAL);
+ return;
+ }
+
+ list_group_images();
+}
+
+template <typename I>
+void GroupEnableRequest<I>::list_group_images() {
+ ldout(m_cct, 10) << dendl;
+
+ librados::ObjectReadOperation op;
+ cls_client::group_image_list_start(&op, m_start_after, MAX_RETURN);
+
+ auto comp = create_rados_callback<
+ GroupEnableRequest<I>,
+ &GroupEnableRequest<I>::handle_list_group_images>(this);
+
+ int r = m_group_ioctx.aio_operate(
+ librbd::util::group_header_name(m_group_id), comp, &op, &m_out_bls[0]);
+ ceph_assert(r == 0);
+ comp->release();
+}
+
+template <typename I>
+void GroupEnableRequest<I>::handle_list_group_images(int r) {
+ ldout(m_cct, 10) << "r=" << r << dendl;
+
+ std::vector<cls::rbd::GroupImageStatus> images;
+ if (r == 0) {
+ auto iter = m_out_bls[0].cbegin();
+ r = cls_client::group_image_list_finish(&iter, &images);
+ }
+
+ m_out_bls[0].clear();
+
+ if (r < 0) {
+ lderr(m_cct) << "error listing images in group: " << cpp_strerror(r)
+ << dendl;
+ finish(r);
+ return;
+ }
+
+ auto image_count = images.size();
+ m_images.insert(m_images.end(), images.begin(), images.end());
+ if (image_count == MAX_RETURN) {
+ m_start_after = images.rbegin()->spec;
+ list_group_images();
+ return;
+ }
+
+ if (m_images.empty()) {
+ set_mirror_group_enabling();
+ } else {
+ check_mirror_images_disabled();
+ }
+}
+
+template <typename I>
+void GroupEnableRequest<I>::check_mirror_images_disabled() {
+ ldout(m_cct, 10) << dendl;
+
+ auto ctx = create_context_callback<
+ GroupEnableRequest<I>,
+ &GroupEnableRequest<I>::handle_check_mirror_images_disabled>(this);
+ auto gather_ctx = new C_Gather(m_cct, ctx);
+
+ m_mirror_images.resize(m_images.size());
+ m_out_bls.resize(m_images.size());
+ for (size_t i = 0; i < m_images.size(); i++) {
+ librados::ObjectReadOperation op;
+ cls_client::mirror_image_get_start(&op, m_images[i].spec.image_id);
+
+ auto on_mirror_image_get = new LambdaContext(
+ [this, i, new_sub_ctx=gather_ctx->new_sub()](int r) {
+ if (r == 0) {
+ auto iter = m_out_bls[i].cbegin();
+ r = cls_client::mirror_image_get_finish(&iter, &m_mirror_images[i]);
+ }
+
+ if (r == -ENOENT) {
+ // image is disabled for mirroring as required
+ r = 0;
+ } else if (r == 0) {
+ lderr(m_cct) << "image_id=" << m_images[i].spec.image_id
+ << " is not disabled for mirroring" << dendl;
+ r = -EINVAL;
+ } else {
+ lderr(m_cct) << "failed to get mirror image info for image_id="
+ << m_images[i].spec.image_id << dendl;
+ }
+
+ new_sub_ctx->complete(r);
+ });
+
+ auto comp = create_rados_callback(on_mirror_image_get);
+
+ int r = m_group_ioctx.aio_operate(RBD_MIRRORING, comp, &op,
+ &m_out_bls[i]);
+ ceph_assert(r == 0);
+ comp->release();
+ }
+
+ gather_ctx->activate();
+}
+
+template <typename I>
+void GroupEnableRequest<I>::handle_check_mirror_images_disabled(int r) {
+ ldout(m_cct, 10) << "r=" << r << dendl;
+
+ m_out_bls.clear();
+
+ if (r < 0) {
+ lderr(m_cct) << "images not disabled for mirroring: "
+ << cpp_strerror(r) << dendl;
+ finish(r);
+ return;
+ }
+
+ open_images();
+}
+
+template <typename I>
+void GroupEnableRequest<I>::open_images() {
+ ldout(m_cct, 10) << dendl;
+
+ auto ctx = create_context_callback<
+ GroupEnableRequest<I>, &GroupEnableRequest<I>::handle_open_images>(this);
+ auto gather_ctx = new C_Gather(m_cct, ctx);
+ int r = 0;
+ std::vector<librados::IoCtx> ioctxs;
+
+ for (const auto& image: m_images) {
+ librados::IoCtx image_io_ctx;
+ r = librbd::util::create_ioctx(m_group_ioctx, "image",
+ image.spec.pool_id, {},
+ &image_io_ctx);
+ if (r < 0) {
+ finish(r);
+ return;
+ }
+
+ ioctxs.push_back(std::move(image_io_ctx));
+ }
+
+ for (size_t i = 0; i < m_images.size(); i++) {
+ m_image_ctxs.push_back(
+ new ImageCtx("", m_images[i].spec.image_id.c_str(), nullptr, ioctxs[i],
+ false));
+
+ auto on_open = new LambdaContext(
+ [this, i, new_sub_ctx=gather_ctx->new_sub()](int r) {
+ // If asynchronous ImageState::open() fails, ImageState together with
+ // ImageCtx is destroyed. Simply NULL-out the respective image_ctxs[i]
+ // pointer to record that it's no longer valid.
+ if (r < 0) {
+ m_image_ctxs[i] = nullptr;
+ }
+ new_sub_ctx->complete(r);
+ });
+
+ // Open parent as well to check if the image is a clone
+ m_image_ctxs[i]->state->open(0, on_open);
+ }
+
+ gather_ctx->activate();
+}
+
+template <typename I>
+void GroupEnableRequest<I>::handle_open_images(int r) {
+ ldout(m_cct, 10) << "r=" << r << dendl;
+
+ if (r < 0) {
+ lderr(m_cct) << "failed to open group images: " << cpp_strerror(r)
+ << dendl;
+ m_ret_val = r;
+
+ close_images();
+ return;
+ }
+
+ validate_images();
+}
+
+template <typename I>
+void GroupEnableRequest<I>::validate_images() {
+ ldout(m_cct, 10) << dendl;
+
+ for (auto &image_ctx : m_image_ctxs) {
+ std::shared_lock image_locker{image_ctx->image_lock};
+ if (image_ctx->parent) {
+ lderr(m_cct) << "cannot enable mirroring: cloned images are not "
+ << "supported" << dendl;
+ m_ret_val = -EINVAL;
+
+ close_images();
+ return;
+ }
+ }
+
+ // FIXME: Once the support for mirroring cloned images is added, need to
+ // check that the parents are enabled for mirroring
+
+ // FIXME: For now images must belong to the same pool as the group
+ auto group_pool_id = m_group_ioctx.get_id();
+ for (auto &image_ctx : m_image_ctxs) {
+ std::shared_lock image_locker{image_ctx->image_lock};
+ if (image_ctx->md_ctx.get_id() != group_pool_id) {
+ lderr(m_cct) << "cannot enable mirroring: image in a different pool"
+ << dendl;
+ m_ret_val = -EINVAL;
+
+ close_images();
+ return;
+ }
+ }
+
+ set_mirror_group_enabling();
+}
+
+template <typename I>
+void GroupEnableRequest<I>::set_mirror_group_enabling() {
+ ldout(m_cct, 10) << dendl;
+
+ m_mirror_group.state = cls::rbd::MIRROR_GROUP_STATE_ENABLING;
+
+ librados::ObjectWriteOperation op;
+ cls_client::mirror_group_set(&op, m_group_id, m_mirror_group);
+ auto aio_comp = create_rados_callback<
+ GroupEnableRequest<I>,
+ &GroupEnableRequest<I>::handle_set_mirror_group_enabling>(this);
+ int r = m_group_ioctx.aio_operate(RBD_MIRRORING, aio_comp, &op);
+ ceph_assert(r == 0);
+ aio_comp->release();
+}
+
+template <typename I>
+void GroupEnableRequest<I>::handle_set_mirror_group_enabling(int r) {
+ ldout(m_cct, 10) << "r=" << r << dendl;
+
+ if (r < 0) {
+ lderr(m_cct) << "failed to set mirror group as enabling: "
+ << cpp_strerror(r) << dendl;
+ m_ret_val = r;
+
+ close_images();
+ return;
+ }
+
+ create_primary_group_snapshot();
+}
+
+template <typename I>
+void GroupEnableRequest<I>::create_primary_group_snapshot() {
+ ldout(m_cct, 10) << dendl;
+
+ m_group_snap.id = librbd::util::generate_image_id(m_group_ioctx);
+
+ auto snap_name = ".mirror.primary." + m_mirror_group.global_group_id
+ + "." + m_group_snap.id;
+ m_group_snap.name = snap_name;
+
+ cls::rbd::MirrorSnapshotState state = cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY;
+
+ // Create incomplete group snap
+ m_group_snap.snapshot_namespace = cls::rbd::GroupSnapshotNamespaceMirror{
+ state, m_mirror_peer_uuids, {}, {}};
+
+ for (auto image_ctx: m_image_ctxs) {
+ m_group_snap.snaps.emplace_back(image_ctx->md_ctx.get_id(), image_ctx->id,
+ CEPH_NOSNAP);
+ }
+
+ librados::ObjectWriteOperation op;
+ cls_client::group_snap_set(&op, m_group_snap);
+
+ auto aio_comp = create_rados_callback<
+ GroupEnableRequest<I>,
+ &GroupEnableRequest<I>::handle_create_primary_group_snapshot>(this);
+ int r = m_group_ioctx.aio_operate(librbd::util::group_header_name(m_group_id),
+ aio_comp, &op);
+ ceph_assert(r == 0);
+ aio_comp->release();
+}
+
+template <typename I>
+void GroupEnableRequest<I>::handle_create_primary_group_snapshot(int r) {
+ ldout(m_cct, 10) << "r=" << r << dendl;
+
+ m_need_to_cleanup_group_snapshot = true;
+
+ if (r < 0) {
+ lderr(m_cct) << "failed to create group snapshot: "
+ << cpp_strerror(r) << dendl;
+ m_ret_val = r;
+
+ disable_mirror_group();
+ return;
+ }
+
+ if (m_image_ctxs.empty()) {
+ update_primary_group_snapshot();
+ } else {
+ create_primary_image_snapshots();
+ }
+}
+
+template <typename I>
+void GroupEnableRequest<I>::create_primary_image_snapshots() {
+ ldout(m_cct, 10) << dendl;
+
+ auto num_images = m_image_ctxs.size();
+ m_global_image_ids.resize(num_images);
+
+ uuid_d uuid_gen;
+ for (size_t i = 0; i < num_images; i++) {
+ uuid_gen.generate_random();
+ m_global_image_ids[i] = uuid_gen.to_string();
+ m_mirror_images[i].global_image_id = m_global_image_ids[i];
+ }
+
+ auto ctx = librbd::util::create_context_callback<
+ GroupEnableRequest<I>,
+ &GroupEnableRequest<I>::handle_create_primary_image_snapshots>(this);
+
+ m_snap_ids.resize(num_images, CEPH_NOSNAP);
+
+ // quiescing and requesting exclusive locks of images
+ auto req = snapshot::GroupImageCreatePrimaryRequest<I>::create(
+ m_cct, m_image_ctxs, m_global_image_ids, m_group_snap_create_flags,
+ snapshot::CREATE_PRIMARY_FLAG_IGNORE_EMPTY_PEERS, m_group_snap.id,
+ &m_snap_ids, ctx);
+ req->send();
+}
+
+template <typename I>
+void GroupEnableRequest<I>::handle_create_primary_image_snapshots(int r) {
+ ldout(m_cct, 10) << "r=" << r << dendl;
+
+ if (r < 0) {
+ lderr(m_cct) << "failed to create primary mirror image snapshots: "
+ << cpp_strerror(r) << dendl;
+ m_ret_val = r;
+
+ for (size_t i = 0; i < m_image_ctxs.size(); i++) {
+ m_group_snap.snaps[i].snap_id = m_snap_ids[i];
+ }
+
+ disable_mirror_group();
+ return;
+ }
+
+ update_primary_group_snapshot();
+}
+
+template <typename I>
+void GroupEnableRequest<I>::update_primary_group_snapshot() {
+ ldout(m_cct, 10) << dendl;
+
+ for (size_t i = 0; i < m_image_ctxs.size(); i++) {
+ m_group_snap.snaps[i].snap_id = m_snap_ids[i];
+ }
+
+ m_group_snap.state = cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE;
+ librados::ObjectWriteOperation op;
+ cls_client::group_snap_set(&op, m_group_snap);
+
+ auto aio_comp = create_rados_callback<
+ GroupEnableRequest<I>,
+ &GroupEnableRequest<I>::handle_update_primary_group_snapshot>(this);
+ int r = m_group_ioctx.aio_operate(librbd::util::group_header_name(m_group_id),
+ aio_comp, &op);
+ ceph_assert(r == 0);
+ aio_comp->release();
+}
+
+template <typename I>
+void GroupEnableRequest<I>::handle_update_primary_group_snapshot(int r) {
+ ldout(m_cct, 10) << "r=" << r << dendl;
+
+ if (r < 0) {
+ lderr(m_cct) << "failed to create group snapshot: "
+ << cpp_strerror(r) << dendl;
+ m_ret_val = r;
+
+ disable_mirror_group();
+ return;
+ }
+
+ if (m_image_ctxs.empty()) {
+ set_mirror_group_enabled();
+ } else {
+ set_mirror_images_enabled();
+ }
+}
+
+template <typename I>
+void GroupEnableRequest<I>::set_mirror_images_enabled() {
+ ldout(m_cct, 10) << dendl;
+
+ auto ctx = create_context_callback<
+ GroupEnableRequest<I>,
+ &GroupEnableRequest<I>::handle_set_mirror_images_enabled>(this);
+
+ auto gather_ctx = new C_Gather(m_cct, ctx);
+
+ for (size_t i = 0; i < m_image_ctxs.size(); i++) {
+ auto ictx = m_image_ctxs[i];
+
+ m_mirror_images[i].type = cls::rbd::MIRROR_IMAGE_TYPE_GROUP;
+ m_mirror_images[i].mode = m_mode;
+ m_mirror_images[i].global_image_id = m_global_image_ids[i];
+
+ auto req = ImageStateUpdateRequest<I>::create(
+ ictx->md_ctx, ictx->id, cls::rbd::MIRROR_IMAGE_STATE_ENABLED,
+ m_mirror_images[i], gather_ctx->new_sub());
+
+ req->send();
+ }
+
+ gather_ctx->activate();
+}
+
+template <typename I>
+void GroupEnableRequest<I>::handle_set_mirror_images_enabled(int r) {
+ ldout(m_cct, 10) << "r=" << r << dendl;
+
+ m_need_to_cleanup_mirror_images = true;
+
+ if (r < 0) {
+ lderr(m_cct) << "failed to enabled mirror images: " << cpp_strerror(r)
+ << dendl;
+ m_ret_val = r;
+
+ disable_mirror_group();
+ return;
+ }
+
+ set_mirror_group_enabled();
+}
+
+template <typename I>
+void GroupEnableRequest<I>::set_mirror_group_enabled() {
+ ldout(m_cct, 10) << dendl;
+
+ m_mirror_group.state = cls::rbd::MIRROR_GROUP_STATE_ENABLED;
+
+ librados::ObjectWriteOperation op;
+ cls_client::mirror_group_set(&op, m_group_id, m_mirror_group);
+ auto aio_comp = create_rados_callback<
+ GroupEnableRequest<I>,
+ &GroupEnableRequest<I>::handle_set_mirror_group_enabled>(this);
+ int r = m_group_ioctx.aio_operate(RBD_MIRRORING, aio_comp, &op);
+ ceph_assert(r == 0);
+ aio_comp->release();
+}
+
+template <typename I>
+void GroupEnableRequest<I>::handle_set_mirror_group_enabled(int r) {
+ ldout(m_cct, 10) << "r=" << r << dendl;
+
+ if (r < 0) {
+ lderr(m_cct) << "failed to set mirror group as enabled: "
+ << cpp_strerror(r) << dendl;
+ m_ret_val = r;
+
+ disable_mirror_group();
+ return;
+ }
+
+ notify_mirroring_watcher();
+}
+
+template <typename I>
+void GroupEnableRequest<I>::notify_mirroring_watcher() {
+ ldout(m_cct, 10) << dendl;
+
+ auto ctx = util::create_context_callback<
+ GroupEnableRequest<I>,
+ &GroupEnableRequest<I>::handle_notify_mirroring_watcher>(this);
+
+ MirroringWatcher<I>::notify_group_updated(
+ m_group_ioctx, cls::rbd::MIRROR_GROUP_STATE_ENABLED, m_group_id,
+ m_mirror_group.global_group_id, m_image_ctxs.size(), ctx);
+}
+
+template <typename I>
+void GroupEnableRequest<I>::handle_notify_mirroring_watcher(int r) {
+ ldout(m_cct, 10) << "r=" << r << dendl;
+
+ if (r < 0) {
+ lderr(m_cct) << "failed to notify mirror group update: " << cpp_strerror(r)
+ << dendl;
+ m_ret_val = r;
+ }
+
+ close_images();
+}
+
+template <typename I>
+void GroupEnableRequest<I>::close_images() {
+ ldout(m_cct, 10) << dendl;
+
+ auto ctx = create_context_callback<
+ GroupEnableRequest<I>, &GroupEnableRequest<I>::handle_close_images>(this);
+
+ auto gather_ctx = new C_Gather(m_cct, ctx);
+
+ for (auto ictx: m_image_ctxs) {
+ if (ictx != nullptr) {
+ ictx->state->close(gather_ctx->new_sub());
+ }
+ }
+
+ gather_ctx->activate();
+}
+
+template <typename I>
+void GroupEnableRequest<I>::handle_close_images(int r) {
+ ldout(m_cct, 10) << "r=" << r << dendl;
+
+ if (r < 0) {
+ lderr(m_cct) << "failed to close images: " << cpp_strerror(r) << dendl;
+ if (m_ret_val == 0) {
+ m_ret_val = r;
+ }
+ }
+
+ finish(m_ret_val);
+}
+
+template <typename I>
+void GroupEnableRequest<I>::disable_mirror_group() {
+ ldout(m_cct, 10) << dendl;
+
+ librados::ObjectWriteOperation op;
+ m_mirror_group.state = cls::rbd::MIRROR_GROUP_STATE_DISABLING;
+
+ cls_client::mirror_group_set(&op, m_group_id, m_mirror_group);
+ auto aio_comp = create_rados_callback<
+ GroupEnableRequest<I>,
+ &GroupEnableRequest<I>::handle_disable_mirror_group>(this);
+ int r = m_group_ioctx.aio_operate(RBD_MIRRORING, aio_comp, &op);
+ ceph_assert(r == 0);
+ aio_comp->release();
+}
+
+template <typename I>
+void GroupEnableRequest<I>::handle_disable_mirror_group(int r) {
+ ldout(m_cct, 10) << "r=" << r << dendl;
+
+ if (r < 0) {
+ lderr(m_cct) << "failed to disable mirror group: " << cpp_strerror(r)
+ << dendl;
+ close_images();
+ return;
+ }
+
+ if (m_need_to_cleanup_mirror_images) {
+ get_mirror_images_for_cleanup();
+ } else if (m_need_to_cleanup_group_snapshot) {
+ remove_primary_group_snapshot();
+ } else {
+ remove_mirror_group();
+ }
+}
+
+template <typename I>
+void GroupEnableRequest<I>::get_mirror_images_for_cleanup() {
+ ldout(m_cct, 10) << dendl;
+
+ m_mirror_images.clear();
+ m_mirror_images.resize(m_images.size());
+ m_out_bls.resize(m_images.size());
+
+ auto ctx = create_context_callback<
+ GroupEnableRequest<I>,
+ &GroupEnableRequest<I>::handle_get_mirror_images_for_cleanup>(this);
+ auto gather_ctx = new C_Gather(m_cct, ctx);
+
+ for (size_t i = 0; i < m_images.size(); i++) {
+ librados::ObjectReadOperation op;
+ cls_client::mirror_image_get_start(&op, m_images[i].spec.image_id);
+
+ auto on_mirror_image_get = new LambdaContext(
+ [this, i, new_sub_ctx=gather_ctx->new_sub()](int r) {
+ if (r == 0) {
+ auto iter = m_out_bls[i].cbegin();
+ r = cls_client::mirror_image_get_finish(&iter, &m_mirror_images[i]);
+ }
+
+ if (r == -ENOENT) {
+ r = 0;
+ m_mirror_images[i].state = cls::rbd::MIRROR_IMAGE_STATE_DISABLED;
+ } else if (r < 0) {
+ lderr(m_cct) << "failed to get mirror image info for image_id="
+ << m_images[i].spec.image_id << dendl;
+ }
+
+ new_sub_ctx->complete(r);
+ });
+
+ auto comp = create_rados_callback(on_mirror_image_get);
+
+ int r = m_group_ioctx.aio_operate(RBD_MIRRORING, comp, &op, &m_out_bls[i]);
+ ceph_assert(r == 0);
+ comp->release();
+ }
+
+ gather_ctx->activate();
+}
+
+template <typename I>
+void GroupEnableRequest<I>::handle_get_mirror_images_for_cleanup(int r) {
+ ldout(m_cct, 10) << "r=" << r << dendl;
+
+ m_out_bls.clear();
+
+ if (r < 0) {
+ ldout(m_cct, 10) << "failed to get mirror image info for cleanup" << dendl;
+ close_images();
+ return;
+ }
+
+ disable_mirror_images();
+}
+
+template <typename I>
+void GroupEnableRequest<I>::disable_mirror_images() {
+ ldout(m_cct, 10) << dendl;
+
+ auto ctx = create_context_callback<
+ GroupEnableRequest<I>,
+ &GroupEnableRequest<I>::handle_disable_mirror_images>(this);
+
+ auto gather_ctx = new C_Gather(m_cct, ctx);
+ for (size_t i = 0; i < m_images.size(); i++) {
+ if (m_mirror_images[i].state != cls::rbd::MIRROR_IMAGE_STATE_DISABLED) {
+ auto req = ImageStateUpdateRequest<I>::create(
+ m_image_ctxs[i]->md_ctx, m_image_ctxs[i]->id,
+ cls::rbd::MIRROR_IMAGE_STATE_DISABLING, m_mirror_images[i],
+ gather_ctx->new_sub());
+ req->send();
+ }
+ }
+ gather_ctx->activate();
+}
+
+template <typename I>
+void GroupEnableRequest<I>::handle_disable_mirror_images(int r) {
+ ldout(m_cct, 10) << "r=" << dendl;
+
+ if (r < 0) {
+ lderr(m_cct) << "failed to disable mirror images: " << cpp_strerror(r)
+ << dendl;
+ close_images();
+ return;
+ }
+
+ remove_primary_group_snapshot();
+}
+
+template <typename I>
+void GroupEnableRequest<I>::remove_primary_group_snapshot() {
+ ldout(m_cct, 10) << dendl;
+
+ auto ctx = create_context_callback<
+ GroupEnableRequest<I>,
+ &GroupEnableRequest<I>::handle_remove_primary_group_snapshot>(this);
+
+ auto req = snapshot::RemoveGroupSnapshotRequest<I>::create(m_group_ioctx,
+ m_group_id, &m_group_snap, &m_image_ctxs, ctx);
+
+ req->send();
+}
+
+template <typename I>
+void GroupEnableRequest<I>::handle_remove_primary_group_snapshot(int r) {
+ ldout(m_cct, 10) << "r=" << r << dendl;
+
+ if (r < 0) {
+ lderr(m_cct) << "failed to remove mirror group snapshot: " << cpp_strerror(r)
+ << dendl;
+ close_images();
+ return;
+ }
+
+ if (m_need_to_cleanup_mirror_images) {
+ remove_mirror_images();
+ } else {
+ remove_mirror_group();
+ }
+}
+
+template <typename I>
+void GroupEnableRequest<I>::remove_mirror_images() {
+ ldout(m_cct, 10) << dendl;
+
+ auto ctx = create_context_callback<
+ GroupEnableRequest<I>,
+ &GroupEnableRequest<I>::handle_remove_mirror_images>(this);
+
+ auto gather_ctx = new C_Gather(m_cct, ctx);
+ for (size_t i = 0; i < m_images.size(); i++) {
+ auto req = ImageRemoveRequest<I>::create(
+ m_image_ctxs[i]->md_ctx, m_global_image_ids[i], m_image_ctxs[i]->id,
+ gather_ctx->new_sub());
+ req->send();
+ }
+ gather_ctx->activate();
+}
+
+template <typename I>
+void GroupEnableRequest<I>::handle_remove_mirror_images(int r) {
+ ldout(m_cct, 10) << "r=" << r << dendl;
+
+ if (r < 0) {
+ lderr(m_cct) << "failed to remove mirror images: " << cpp_strerror(r)
+ << dendl;
+ close_images();
+ return;
+ }
+
+ remove_mirror_group();
+}
+
+template <typename I>
+void GroupEnableRequest<I>::remove_mirror_group() {
+ ldout(m_cct, 10) << dendl;
+
+ librados::ObjectWriteOperation op;
+ cls_client::mirror_group_remove(&op, m_group_id);
+
+ auto comp = create_rados_callback<
+ GroupEnableRequest<I>,
+ &GroupEnableRequest<I>::handle_remove_mirror_group>(this);
+ int r = m_group_ioctx.aio_operate(RBD_MIRRORING, comp, &op);
+ ceph_assert(r == 0);
+ comp->release();
+
+}
+
+template <typename I>
+void GroupEnableRequest<I>::handle_remove_mirror_group(int r) {
+ ldout(m_cct, 10) << "r=" << r << dendl;
+
+ if (r < 0) {
+ lderr(m_cct) << "failed to remove mirror group: " << cpp_strerror(r)
+ << dendl;
+ }
+
+ close_images();
+}
+
+template <typename I>
+void GroupEnableRequest<I>::finish(int r) {
+ ldout(m_cct, 10) << "r=" << r << dendl;
+
+ m_on_finish->complete(r);
+ delete this;
+}
+
+} // namespace mirror
+} // namespace librbd
+
+template class librbd::mirror::GroupEnableRequest<librbd::ImageCtx>;
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CEPH_LIBRBD_MIRROR_GROUP_ENABLE_REQUEST_H
+#define CEPH_LIBRBD_MIRROR_GROUP_ENABLE_REQUEST_H
+
+#include "include/buffer_fwd.h"
+#include "include/rados/librados_fwd.hpp"
+#include "include/rbd/librbd.hpp"
+#include "cls/rbd/cls_rbd_types.h"
+#include "librbd/ImageCtx.h"
+#include "librbd/mirror/Types.h"
+#include <map>
+#include <string>
+
+class Context;
+
+namespace librbd {
+
+namespace asio { struct ContextWQ; }
+
+namespace mirror {
+
+template <typename ImageCtxT = ImageCtx>
+class GroupEnableRequest {
+public:
+ static GroupEnableRequest *create(librados::IoCtx &group_io_ctx,
+ const std::string &group_id,
+ uint64_t group_snap_create_flags,
+ cls::rbd::MirrorImageMode mode,
+ Context *on_finish) {
+ return new GroupEnableRequest(group_io_ctx, group_id,
+ group_snap_create_flags, mode, on_finish);
+ }
+
+ void send();
+
+private:
+ /**
+ * @verbatim
+ *
+ * <start>
+ * |
+ * v (on error)
+ * GET_MIRROR_GROUP * * * * * * * * * * *
+ * | *
+ * v *
+ * GET_MIRROR_PEERS * * * * * * * * * * *
+ * | *
+ * v *
+ * LIST_GROUP_IMAGES * * * * * * * * * * *
+ * | *
+ * v (skip if not needed) *
+ * CHECK_MIRROR_IMAGES_DISABLED * * * * *
+ * | *
+ * v (skip if not needed) *
+ * OPEN_IMAGES * * * * * * * * * * * * *
+ * | *
+ * v (skip if not needed) *
+ * VALIDATE_IMAGES * * * * * * * * * * *
+ * | *
+ * v *
+ * SET_MIRROR_GROUP_ENABLING * * * * * * *
+ * | *
+ * v (incomplete) *
+ * CREATE_PRIMARY_GROUP_SNAP * * * * * * *
+ * | *
+ * v (skip if not needed) *
+ * CREATE_PRIMARY_IMAGE_SNAPS *
+ * | *
+ * v (complete) *
+ * UPDATE_PRIMARY_GROUP_SNAP * * * * * * *
+ * | *
+ * v (skip if not needed) *
+ * SET_MIRROR_IMAGES_ENABLED * * * * * * *
+ * | *
+ * v *
+ * SET_MIRROR_GROUP_ENABLED * * * * * * *
+ * | *
+ * v (if required)
+ * NOTIFY_MIRRORING_WATCHER cleanup
+ * | *
+ * v (skip if not needed) *
+ * CLOSE_IMAGE < * * * * * * * * * * * * *
+ * |
+ * v
+ * <finish>
+ *
+ * @endverbatim
+ */
+
+ GroupEnableRequest(librados::IoCtx &io_ctx, const std::string &group_id,
+ uint64_t group_snap_create_flags,
+ cls::rbd::MirrorImageMode mode, Context *on_finish);
+
+ librados::IoCtx &m_group_ioctx;
+ const std::string m_group_id;
+ uint64_t m_group_snap_create_flags;
+ const cls::rbd::MirrorImageMode m_mode;
+ Context *m_on_finish;
+
+ CephContext *m_cct = nullptr;
+ std::vector<bufferlist> m_out_bls;
+ cls::rbd::MirrorGroup m_mirror_group;
+
+ int m_ret_val = 0;
+ librados::IoCtx m_default_ns_ioctx;
+
+ std::vector<ImageCtxT *> m_image_ctxs;
+ std::vector<cls::rbd::MirrorImage> m_mirror_images;
+
+ std::set<std::string> m_mirror_peer_uuids;
+ cls::rbd::GroupImageSpec m_start_after;
+ std::vector<cls::rbd::GroupImageStatus> m_images;
+
+ cls::rbd::GroupSnapshot m_group_snap;
+ std::vector<uint64_t> m_snap_ids;
+ std::vector<std::string> m_global_image_ids;
+
+ bool m_need_to_cleanup_group_snapshot = false;
+ bool m_need_to_cleanup_mirror_images = false;
+
+ void get_mirror_group();
+ void handle_get_mirror_group(int r);
+
+ void get_mirror_peer_list();
+ void handle_get_mirror_peer_list(int r);
+
+ void list_group_images();
+ void handle_list_group_images(int r);
+
+ void check_mirror_images_disabled();
+ void handle_check_mirror_images_disabled(int r);
+
+ void open_images();
+ void handle_open_images(int r);
+
+ void validate_images();
+
+ void create_primary_group_snapshot();
+ void handle_create_primary_group_snapshot(int r);
+
+ void set_mirror_group_enabling();
+ void handle_set_mirror_group_enabling(int r);
+
+ void create_primary_image_snapshots();
+ void handle_create_primary_image_snapshots(int r);
+
+ void update_primary_group_snapshot();
+ void handle_update_primary_group_snapshot(int r);
+
+ void set_mirror_images_enabled();
+ void handle_set_mirror_images_enabled(int r);
+
+ void set_mirror_group_enabled();
+ void handle_set_mirror_group_enabled(int r);
+
+ void notify_mirroring_watcher();
+ void handle_notify_mirroring_watcher(int r);
+
+ void close_images();
+ void handle_close_images(int r);
+
+ // Cleanup
+ void disable_mirror_group();
+ void handle_disable_mirror_group(int r);
+
+ void get_mirror_images_for_cleanup();
+ void handle_get_mirror_images_for_cleanup(int r);
+
+ void disable_mirror_images();
+ void handle_disable_mirror_images(int r);
+
+ void remove_primary_group_snapshot();
+ void handle_remove_primary_group_snapshot(int r);
+
+ void remove_mirror_images();
+ void handle_remove_mirror_images(int r);
+
+ void remove_mirror_group();
+ void handle_remove_mirror_group(int r);
+
+ void finish(int r);
+};
+
+} // namespace mirror
+} // namespace librbd
+
+extern template class librbd::mirror::GroupEnableRequest<librbd::ImageCtx>;
+
+#endif // CEPH_LIBRBD_MIRROR_GROUP_ENABLE_REQUEST_H
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "librbd/mirror/snapshot/GroupImageCreatePrimaryRequest.h"
+#include "common/dout.h"
+#include "common/errno.h"
+#include "cls/rbd/cls_rbd_client.h"
+#include "librbd/ExclusiveLock.h"
+#include "librbd/ImageCtx.h"
+#include "librbd/ImageState.h"
+#include "librbd/ImageWatcher.h"
+#include "librbd/Operations.h"
+#include "librbd/Utils.h"
+#include "librbd/mirror/snapshot/Utils.h"
+
+#include <shared_mutex> // for std::shared_lock
+
+#define dout_subsys ceph_subsys_rbd
+
+#undef dout_prefix
+#define dout_prefix *_dout << "librbd::mirror::snapshot::GroupImageCreatePrimaryRequest: " \
+ << this << " " << __func__ << ": "
+
+namespace librbd {
+namespace mirror {
+namespace snapshot {
+
+using librbd::util::create_context_callback;
+using librbd::util::create_rados_callback;
+using librbd::util::snap_create_flags_api_to_internal;
+using librbd::util::get_default_snap_create_flags;
+
+template <typename I>
+GroupImageCreatePrimaryRequest<I>::GroupImageCreatePrimaryRequest(
+ CephContext* cct, const std::vector<I *> &image_ctxs,
+ const std::vector<std::string> &global_image_ids,
+ uint64_t group_snap_create_flags, uint32_t flags,
+ const std::string &group_snap_id, std::vector<uint64_t> *snap_ids,
+ Context *on_finish)
+ : m_cct(cct), m_image_ctxs(image_ctxs), m_global_image_ids(global_image_ids),
+ m_group_snap_create_flags(group_snap_create_flags), m_flags(flags),
+ m_group_snap_id(group_snap_id), m_snap_ids(snap_ids),
+ m_on_finish(on_finish) {
+ ceph_assert(!m_image_ctxs.empty());
+ ceph_assert(!m_group_snap_id.empty());
+ ceph_assert(m_global_image_ids.size() == m_image_ctxs.size());
+ ceph_assert((*m_snap_ids).size() == m_image_ctxs.size());
+}
+
+template <typename I>
+void GroupImageCreatePrimaryRequest<I>::send() {
+ ldout(m_cct, 15) << dendl;
+
+ size_t i = 0;
+ for (; i < m_image_ctxs.size(); i++) {
+ if (!util::can_create_primary_snapshot(
+ m_image_ctxs[i],
+ ((m_flags & CREATE_PRIMARY_FLAG_DEMOTED) != 0),
+ ((m_flags & CREATE_PRIMARY_FLAG_FORCE) != 0), nullptr, nullptr)) {
+ lderr(m_cct) << "cannot create primary snapshot for "
+ << m_image_ctxs[i]->id << dendl;
+ finish(-EINVAL);
+ return;
+ }
+ }
+
+ m_snap_names.resize(m_image_ctxs.size());
+
+ for (i = 0; i < m_image_ctxs.size(); i++) {
+ std::stringstream ss;
+ ss << ".mirror.primary." << m_global_image_ids[i] << "."
+ << m_image_ctxs[i]->group_spec.pool_id << "_"
+ << m_image_ctxs[i]->group_spec.group_id << "_"
+ << m_group_snap_id;
+ m_snap_names[i] = ss.str();
+ }
+
+ get_mirror_peers();
+}
+
+template <typename I>
+void GroupImageCreatePrimaryRequest<I>::get_mirror_peers() {
+ ldout(m_cct, 15) << dendl;
+
+ auto ctx = create_context_callback<
+ GroupImageCreatePrimaryRequest<I>,
+ &GroupImageCreatePrimaryRequest<I>::handle_get_mirror_peers>(this);
+ auto gather_ctx = new C_Gather(m_cct, ctx);
+
+ m_default_ns_ctxs.resize(m_image_ctxs.size());
+ m_mirror_peers_uuids.resize(m_image_ctxs.size());
+ m_out_bls.resize(m_image_ctxs.size());
+
+ for (size_t i = 0; i < m_image_ctxs.size(); i++) {
+ m_default_ns_ctxs[i].dup(m_image_ctxs[i]->md_ctx);
+ m_default_ns_ctxs[i].set_namespace("");
+
+ librados::ObjectReadOperation op;
+ cls_client::mirror_peer_list_start(&op);
+
+ auto on_mirror_peer_list = new LambdaContext(
+ [this, i, new_sub_ctx = gather_ctx->new_sub()](int r) {
+ std::vector<cls::rbd::MirrorPeer> peers;
+ if (r == 0) {
+ auto iter = m_out_bls[i].cbegin();
+ r = cls_client::mirror_peer_list_finish(&iter, &peers);
+ }
+
+ if (r < 0) {
+ lderr(m_image_ctxs[i]->cct) << "failed to retrieve mirror peers: "
+ << cpp_strerror(r) << dendl;
+ } else {
+ for (auto &peer : peers) {
+ if (peer.mirror_peer_direction ==
+ cls::rbd::MIRROR_PEER_DIRECTION_RX) {
+ continue;
+ }
+ m_mirror_peers_uuids[i].insert(peer.uuid);
+ }
+
+ if (m_mirror_peers_uuids[i].empty() &&
+ ((m_flags & CREATE_PRIMARY_FLAG_IGNORE_EMPTY_PEERS) == 0)) {
+ lderr(m_image_ctxs[i]->cct) << "no mirror tx peers configured "
+ << "for the pool" << dendl;
+ r = -EINVAL;
+ }
+ }
+
+ new_sub_ctx->complete(r);
+ });
+
+ auto comp = create_rados_callback(on_mirror_peer_list);
+
+ int r = m_default_ns_ctxs[i].aio_operate(RBD_MIRRORING, comp, &op,
+ &m_out_bls[i]);
+ ceph_assert(r == 0);
+ comp->release();
+ }
+
+ gather_ctx->activate();
+}
+
+template <typename I>
+void GroupImageCreatePrimaryRequest<I>::handle_get_mirror_peers(int r) {
+ ldout(m_cct, 15) << "r=" << r << dendl;
+
+ m_default_ns_ctxs.clear();
+ m_out_bls.clear();
+
+ if (r < 0) {
+ lderr(m_cct) << "failed to retrieve mirror peers for images: "
+ << cpp_strerror(r) << dendl;
+ finish(r);
+ return;
+ }
+
+ if ((m_group_snap_create_flags &
+ SNAP_CREATE_FLAG_SKIP_NOTIFY_QUIESCE) != 0) {
+ acquire_exclusive_locks();
+ return;
+ }
+
+ notify_quiesce();
+}
+
+template <typename I>
+void GroupImageCreatePrimaryRequest<I>::notify_quiesce() {
+ ldout(m_cct, 15) << dendl;
+
+ auto ctx = create_context_callback<
+ GroupImageCreatePrimaryRequest<I>,
+ &GroupImageCreatePrimaryRequest<I>::handle_notify_quiesce>(this);
+ auto gather_ctx = new C_Gather(m_cct, ctx);
+
+ m_quiesce_requests.resize(m_image_ctxs.size());
+
+ for (size_t i = 0; i < m_image_ctxs.size(); ++i) {
+ auto ictx = m_image_ctxs[i];
+ ictx->image_watcher->notify_quiesce(&(m_quiesce_requests)[i], m_prog_ctx,
+ gather_ctx->new_sub());
+ }
+
+ gather_ctx->activate();
+}
+
+template <typename I>
+void GroupImageCreatePrimaryRequest<I>::handle_notify_quiesce(int r) {
+ ldout(m_cct, 15) << "r=" << r << dendl;
+
+ if (r < 0 &&
+ (m_group_snap_create_flags & SNAP_CREATE_FLAG_IGNORE_NOTIFY_QUIESCE_ERROR) == 0) {
+ m_ret_code = r;
+ notify_unquiesce();
+ return;
+ }
+
+ acquire_exclusive_locks();
+}
+
+template <typename I>
+void GroupImageCreatePrimaryRequest<I>::acquire_exclusive_locks() {
+ ldout(m_cct, 15) << dendl;
+
+ m_release_locks = true;
+
+ auto ctx = librbd::util::create_context_callback<
+ GroupImageCreatePrimaryRequest<I>,
+ &GroupImageCreatePrimaryRequest<I>::handle_acquire_exclusive_locks>(this);
+ auto gather_ctx = new C_Gather(m_cct, ctx);
+
+ for (auto ictx: m_image_ctxs) {
+ std::shared_lock owner_lock{ictx->owner_lock};
+ if (ictx->exclusive_lock != nullptr) {
+ ictx->exclusive_lock->block_requests(-EBUSY);
+ ictx->exclusive_lock->acquire_lock(gather_ctx->new_sub());
+ }
+ }
+
+ gather_ctx->activate();
+}
+
+template <typename I>
+void GroupImageCreatePrimaryRequest<I>::handle_acquire_exclusive_locks(int r) {
+ ldout(m_cct, 15) << "r=" << r << dendl;
+
+ if (r < 0) {
+ lderr(m_cct) << "failed to acquire image exclusive locks: "
+ << cpp_strerror(r) << dendl;
+ m_ret_code = r;
+ // release locks in case some of the lock acquisitions succeeded
+ release_exclusive_locks();
+ return;
+ }
+
+ create_snapshots();
+}
+
+template <typename I>
+void GroupImageCreatePrimaryRequest<I>::create_snapshots() {
+ ldout(m_cct, 15) << dendl;
+
+ auto ctx = create_context_callback<
+ GroupImageCreatePrimaryRequest<I>,
+ &GroupImageCreatePrimaryRequest<I>::handle_create_snapshots>(this);
+
+ auto gather_ctx = new C_Gather(m_cct, ctx);
+
+ for (size_t i = 0; i < m_image_ctxs.size(); i++) {
+ cls::rbd::MirrorSnapshotNamespace ns{
+ ((m_flags & CREATE_PRIMARY_FLAG_DEMOTED) != 0 ?
+ cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED :
+ cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY),
+ m_mirror_peers_uuids[i], "", CEPH_NOSNAP};
+ ns.group_spec = m_image_ctxs[i]->group_spec;
+ ns.group_snap_id = m_group_snap_id;
+
+ ldout(m_cct, 15) << "creating snapshot: image_id=" << m_image_ctxs[i]->id
+ << ", snap_name=" << m_snap_names[i]
+ << ", snap_ns=" << ns << dendl;
+
+ uint64_t snap_create_flags;
+ int r = snap_create_flags_api_to_internal(
+ m_cct, get_default_snap_create_flags(m_image_ctxs[i]),
+ &snap_create_flags);
+ ceph_assert(r == 0);
+
+ m_image_ctxs[i]->operations->snap_create(ns, m_snap_names[i],
+ snap_create_flags, m_prog_ctx,
+ gather_ctx->new_sub());
+ }
+
+ gather_ctx->activate();
+}
+
+template <typename I>
+void GroupImageCreatePrimaryRequest<I>::handle_create_snapshots(int r) {
+ ldout(m_cct, 15) << "r=" << r << dendl;
+
+ if (r < 0) {
+ lderr(m_cct) << "failed to create image snapshots: " << cpp_strerror(r)
+ << dendl;
+ m_ret_code = r;
+ // Refresh the images anyway so we can return any available snap_ids
+ }
+
+ refresh_images();
+}
+
+template <typename I>
+void GroupImageCreatePrimaryRequest<I>::refresh_images() {
+ // Refresh is required to retrieve the snapshot id (if snapshot
+ // created via remote RPC) and complete flag (regardless)
+ ldout(m_cct, 15) << dendl;
+
+ auto ctx = create_context_callback<
+ GroupImageCreatePrimaryRequest<I>,
+ &GroupImageCreatePrimaryRequest<I>::handle_refresh_images>(this);
+
+ auto gather_ctx = new C_Gather(m_cct, ctx);
+ for (size_t i = 0; i < m_image_ctxs.size(); i++) {
+ auto on_refresh = new LambdaContext(
+ [this, i, new_sub_ctx = gather_ctx->new_sub()](int r) {
+ auto cct = m_image_ctxs[i]->cct;
+ if (r < 0) {
+ lderr(cct) << "failed to refresh image: " << cpp_strerror(r)
+ << dendl;
+ } else {
+ ldout(cct, 15) << "snap_name=" << m_snap_names[i] << dendl;
+
+ std::shared_lock image_locker{m_image_ctxs[i]->image_lock};
+
+ auto snap_id = m_image_ctxs[i]->get_snap_id(
+ cls::rbd::MirrorSnapshotNamespace{}, m_snap_names[i]);
+ (*m_snap_ids)[i] = snap_id;
+ ldout(cct, 15) << "image_id: " << m_image_ctxs[i]->id
+ << ", snap_id=" << snap_id << dendl;
+ }
+
+ new_sub_ctx->complete(r);
+ });
+
+ m_image_ctxs[i]->state->refresh(on_refresh);
+ }
+ gather_ctx->activate();
+}
+
+template <typename I>
+void GroupImageCreatePrimaryRequest<I>::handle_refresh_images(int r) {
+ ldout(m_cct, 15) << "r=" << r << dendl;
+
+ if (r < 0) {
+ lderr(m_cct) << "failed to refresh images: " << cpp_strerror(r) << dendl;
+ if (m_ret_code == 0) {
+ m_ret_code = r;
+ }
+ }
+
+ if (m_release_locks) {
+ release_exclusive_locks();
+ } else {
+ notify_unquiesce();
+ }
+}
+
+template <typename I>
+void GroupImageCreatePrimaryRequest<I>::release_exclusive_locks() {
+ ldout(m_cct, 15) << dendl;
+
+ auto ctx = librbd::util::create_context_callback<
+ GroupImageCreatePrimaryRequest<I>,
+ &GroupImageCreatePrimaryRequest<I>::handle_release_exclusive_locks>(this);
+ auto gather_ctx = new C_Gather(m_cct, ctx);
+
+ for (auto ictx: m_image_ctxs) {
+ std::shared_lock owner_lock{ictx->owner_lock};
+ if (ictx->exclusive_lock != nullptr) {
+ ictx->exclusive_lock->release_lock(gather_ctx->new_sub());
+ }
+ }
+
+ gather_ctx->activate();
+}
+
+template <typename I>
+void GroupImageCreatePrimaryRequest<I>::handle_release_exclusive_locks(int r) {
+ ldout(m_cct, 15) << "r=" << r << dendl;
+
+ if (r < 0) {
+ lderr(m_cct) << "failed to release exclusive locks for images: "
+ << cpp_strerror(r) << dendl;
+ if (m_ret_code == 0) {
+ m_ret_code = r;
+ }
+ }
+
+ notify_unquiesce();
+}
+
+template <typename I>
+void GroupImageCreatePrimaryRequest<I>::notify_unquiesce() {
+ if (m_quiesce_requests.empty()) {
+ finish(m_ret_code);
+ return;
+ }
+
+ ldout(m_cct, 15) << dendl;
+
+ ceph_assert(m_quiesce_requests.size() == m_image_ctxs.size());
+
+ auto ctx = librbd::util::create_context_callback<
+ GroupImageCreatePrimaryRequest<I>,
+ &GroupImageCreatePrimaryRequest<I>::handle_notify_unquiesce>(this);
+ auto gather_ctx = new C_Gather(m_cct, ctx);
+
+ for (size_t i = 0; i < m_image_ctxs.size(); ++i) {
+ auto ictx = m_image_ctxs[i];
+ ictx->image_watcher->notify_unquiesce(m_quiesce_requests[i],
+ gather_ctx->new_sub());
+ }
+
+ gather_ctx->activate();
+}
+
+template <typename I>
+void GroupImageCreatePrimaryRequest<I>::handle_notify_unquiesce(int r) {
+ ldout(m_cct, 15) << "r=" << r << dendl;
+
+ if (r < 0) {
+ lderr(m_cct) << "failed to unquiesce requests: "
+ << cpp_strerror(r) << dendl;
+ if (m_ret_code == 0) {
+ m_ret_code = r;
+ }
+ }
+
+ finish(m_ret_code);
+}
+
+template <typename I>
+void GroupImageCreatePrimaryRequest<I>::finish(int r) {
+ ldout(m_cct, 15) << "r=" << r << dendl;
+
+ m_on_finish->complete(r);
+ delete this;
+}
+
+} // namespace snapshot
+} // namespace mirror
+} // namespace librbd
+
+template class librbd::mirror::snapshot::GroupImageCreatePrimaryRequest<librbd::ImageCtx>;
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CEPH_LIBRBD_MIRROR_SNAPSHOT_GROUP_IMAGE_CREATE_PRIMARY_REQUEST_H
+#define CEPH_LIBRBD_MIRROR_SNAPSHOT_GROUP_IMAGE_CREATE_PRIMARY_REQUEST_H
+
+#include "include/buffer.h"
+#include "include/rados/librados.hpp"
+#include "cls/rbd/cls_rbd_types.h"
+#include "librbd/internal.h"
+#include "librbd/mirror/snapshot/Types.h"
+
+#include <string>
+#include <set>
+
+struct Context;
+
+namespace librbd {
+
+struct ImageCtx;
+
+namespace mirror {
+namespace snapshot {
+
+template <typename ImageCtxT = librbd::ImageCtx>
+class GroupImageCreatePrimaryRequest {
+public:
+ static GroupImageCreatePrimaryRequest *create(
+ CephContext* cct, const std::vector<ImageCtxT *> &image_ctxs,
+ const std::vector<std::string> &global_image_ids,
+ uint64_t group_snap_create_flags, uint32_t flags,
+ const std::string &group_snap_id, std::vector<uint64_t> *snap_ids,
+ Context *on_finish) {
+ return new GroupImageCreatePrimaryRequest(
+ cct, image_ctxs, global_image_ids, group_snap_create_flags, flags,
+ group_snap_id, snap_ids, on_finish);
+ }
+
+ GroupImageCreatePrimaryRequest(
+ CephContext* cct, const std::vector<ImageCtxT *> &image_ctxs,
+ const std::vector<std::string> &global_image_ids,
+ uint64_t group_snap_create_flags, uint32_t flags,
+ const std::string &group_snap_id, std::vector<uint64_t> *snap_ids,
+ Context *on_finish);
+
+ void send();
+
+private:
+ /**
+ * @verbatim
+ *
+ * <start>
+ * |
+ * v (on-error)
+ * GET_MIRROR_PEERS . . . . . . > . . . . . . . .
+ * | .
+ * v (on-error) .
+ * NOTIFY_QUIESCE . . . . . . > . . . . .
+ * | . .
+ * v (on-error) . .
+ * ACQUIRE_EXCLUSIVE_LOCKS . . . . .
+ * | . . .
+ * v . . .
+ * CREATE_SNAPSHOTS . . .
+ * | v . .
+ * v . v .
+ * REFRESH_IMAGES . . .
+ * | . . .
+ * v . . .
+ * RELEASE_EXCLUSIVE_LOCKS . . . . .
+ * | . .
+ * v . .
+ * NOTIFY_UNQUIESCE . . . . < . . . . . .
+ * | .
+ * v .
+ * <finish> . . . . . . . . . . < . . . . . . . .
+ *
+ * @endverbatim
+ */
+
+ CephContext *m_cct;
+ const std::vector<ImageCtxT *> &m_image_ctxs;
+ const std::vector<std::string> &m_global_image_ids;
+ uint64_t m_group_snap_create_flags;
+ const uint32_t m_flags;
+ const std::string m_group_snap_id;
+ std::vector<uint64_t> *m_snap_ids;
+ Context *m_on_finish;
+
+ std::vector<std::set<std::string>> m_mirror_peers_uuids;
+ std::vector<std::string> m_snap_names;
+ std::vector<librados::IoCtx> m_default_ns_ctxs;
+ std::vector<bufferlist> m_out_bls;
+
+ std::vector<uint64_t> m_quiesce_requests;
+ bool m_release_locks = false;
+ int m_ret_code = 0;
+
+ NoOpProgressContext m_prog_ctx;
+
+ void get_mirror_peers();
+ void handle_get_mirror_peers(int r);
+
+ void create_snapshots();
+ void handle_create_snapshots(int r);
+
+ void refresh_images();
+ void handle_refresh_images(int r);
+
+ void notify_quiesce();
+ void handle_notify_quiesce(int r);
+
+ void notify_unquiesce();
+ void handle_notify_unquiesce(int r);
+
+ void acquire_exclusive_locks();
+ void handle_acquire_exclusive_locks(int r);
+
+ void release_exclusive_locks();
+ void handle_release_exclusive_locks(int r);
+
+ void finish(int r);
+};
+
+} // namespace snapshot
+} // namespace mirror
+} // namespace librbd
+
+extern template class librbd::mirror::snapshot::GroupImageCreatePrimaryRequest<librbd::ImageCtx>;
+
+#endif // CEPH_LIBRBD_MIRROR_SNAPSHOT_GROUP_IMAGE_CREATE_PRIMARY_REQUEST_H
projects / ceph-ci.git / commitdiff