From 2035ee2a25ff082f9658852a09b4055ff217b2dd Mon Sep 17 00:00:00 2001 From: Prasanna Kumar Kalever Date: Mon, 26 Feb 2024 14:23:28 +0530 Subject: [PATCH] rbd-mirror: add undo code, exclusive locking and quiescing * add essential logic to undo partially succeeded API's like, group promote, group demote, group enable, group disable, group image add and group image remove * add exclusive locking and quiescing with-in all the required group API's * adress code duplication and optimization with in the group API's Signed-off-by: Prasanna Kumar Kalever --- src/include/rbd/librbd.h | 20 +- src/include/rbd/librbd.hpp | 18 +- src/librbd/api/Group.cc | 35 +- src/librbd/api/Group.h | 17 +- src/librbd/api/Migration.cc | 4 +- src/librbd/api/Mirror.cc | 1140 +++++++++++++------------ src/librbd/api/Mirror.h | 14 +- src/librbd/librbd.cc | 66 +- src/pybind/rbd/c_rbd.pxd | 6 +- src/pybind/rbd/mock_rbd.pxi | 6 +- src/pybind/rbd/rbd.pyx | 16 +- src/test/librbd/test_Groups.cc | 42 +- src/test/librbd/test_Migration.cc | 8 +- src/tools/rbd/action/Group.cc | 20 +- src/tools/rbd/action/MirrorGroup.cc | 29 +- src/tools/rbd_mirror/GroupReplayer.cc | 4 +- 16 files changed, 821 insertions(+), 624 deletions(-) diff --git a/src/include/rbd/librbd.h b/src/include/rbd/librbd.h index 1fbdc132e9478..ce797652940fe 100644 --- a/src/include/rbd/librbd.h +++ b/src/include/rbd/librbd.h @@ -1568,15 +1568,18 @@ CEPH_RBD_API void rbd_config_image_list_cleanup(rbd_config_option_t *options, CEPH_RBD_API int rbd_group_image_add(rados_ioctx_t group_p, const char *group_name, rados_ioctx_t image_p, - const char *image_name); + const char *image_name, + uint32_t flags); CEPH_RBD_API int rbd_group_image_remove(rados_ioctx_t group_p, const char *group_name, rados_ioctx_t image_p, - const char *image_name); + const char *image_name, + uint32_t flags); CEPH_RBD_API int rbd_group_image_remove_by_id(rados_ioctx_t group_p, const char *group_name, rados_ioctx_t image_p, - const char *image_id); + const char *image_id, + uint32_t flags); CEPH_RBD_API int rbd_group_image_list(rados_ioctx_t group_p, const char *group_name, rbd_group_image_info_t *images, @@ -1633,12 +1636,17 @@ CEPH_RBD_API int rbd_group_snap_rollback_with_progress(rados_ioctx_t group_p, CEPH_RBD_API int rbd_mirror_group_list(rados_ioctx_t p, char *names, size_t *size); CEPH_RBD_API int rbd_mirror_group_enable(rados_ioctx_t p, const char *name, - rbd_mirror_image_mode_t mirror_image_mode); + rbd_mirror_image_mode_t mirror_image_mode, + uint32_t flags); CEPH_RBD_API int rbd_mirror_group_disable(rados_ioctx_t p, const char *name, bool force); -CEPH_RBD_API int rbd_mirror_group_promote(rados_ioctx_t p, const char *name, +CEPH_RBD_API int rbd_mirror_group_promote(rados_ioctx_t p, + const char *name, + uint32_t flags, bool force); -CEPH_RBD_API int rbd_mirror_group_demote(rados_ioctx_t p, const char *name); +CEPH_RBD_API int rbd_mirror_group_demote(rados_ioctx_t p, + const char *name, + uint32_t flags); CEPH_RBD_API int rbd_mirror_group_resync(rados_ioctx_t p, const char *name); CEPH_RBD_API int rbd_mirror_group_create_snapshot(rados_ioctx_t p, const char *name, diff --git a/src/include/rbd/librbd.hpp b/src/include/rbd/librbd.hpp index 6aea81e01e548..b58e9a0c810c1 100644 --- a/src/include/rbd/librbd.hpp +++ b/src/include/rbd/librbd.hpp @@ -487,11 +487,14 @@ public: const char *dest_group_name); int group_image_add(IoCtx& io_ctx, const char *group_name, - IoCtx& image_io_ctx, const char *image_name); + IoCtx& image_io_ctx, const char *image_name, + uint32_t flags); int group_image_remove(IoCtx& io_ctx, const char *group_name, - IoCtx& image_io_ctx, const char *image_name); + IoCtx& image_io_ctx, const char *image_name, + uint32_t flags); int group_image_remove_by_id(IoCtx& io_ctx, const char *group_name, - IoCtx& image_io_ctx, const char *image_id); + IoCtx& image_io_ctx, const char *image_id, + uint32_t flags); int group_image_list(IoCtx& io_ctx, const char *group_name, std::vector *images, size_t group_image_info_size); @@ -521,10 +524,13 @@ public: // RBD group mirroring support functions int mirror_group_list(IoCtx& io_ctx, std::vector *names); int mirror_group_enable(IoCtx& io_ctx, const char *group_name, - mirror_image_mode_t mirror_image_mode); + mirror_image_mode_t mirror_image_mode, + uint32_t flags); int mirror_group_disable(IoCtx& io_ctx, const char *group_name, bool force); - int mirror_group_promote(IoCtx& io_ctx, const char *group_name, bool force); - int mirror_group_demote(IoCtx& io_ctx, const char *group_name); + int mirror_group_promote(IoCtx& io_ctx, const char *group_name, + uint32_t flags, bool force); + int mirror_group_demote(IoCtx& io_ctx, const char *group_name, + uint32_t flags); int mirror_group_resync(IoCtx& io_ctx, const char *group_name); int mirror_group_create_snapshot(IoCtx& io_ctx, const char *group_name, uint32_t flags, std::string *snap_id); diff --git a/src/librbd/api/Group.cc b/src/librbd/api/Group.cc index f18cd5c6d1b92..83531e963dd1e 100644 --- a/src/librbd/api/Group.cc +++ b/src/librbd/api/Group.cc @@ -278,7 +278,8 @@ template int Group::image_remove_by_id(librados::IoCtx& group_ioctx, const char *group_name, librados::IoCtx& image_ioctx, - const char *image_id) + const char *image_id, + uint32_t flags) { CephContext *cct = (CephContext *)group_ioctx.cct(); ldout(cct, 20) << "io_ctx=" << &group_ioctx @@ -295,9 +296,10 @@ int Group::image_remove_by_id(librados::IoCtx& group_ioctx, } ldout(cct, 20) << "removing image from group name " << group_name - << " group id " << group_id << dendl; + << " group id " << group_id << dendl; - r = Group::group_image_remove(group_ioctx, group_id, image_ioctx, image_id); + r = Group::group_image_remove(group_ioctx, group_id, image_ioctx, image_id, + false, flags); return r; } @@ -389,7 +391,7 @@ int Group::remove(librados::IoCtx& io_ctx, const char *group_name) } r = Group::group_image_remove(io_ctx, group_id, image_ioctx, - image.spec.image_id); + image.spec.image_id, false, 0); if (r < 0 && r != -ENOENT) { lderr(cct) << "error removing image from a group" << dendl; return r; @@ -481,7 +483,8 @@ int Group::get_id(IoCtx& io_ctx, const char *group_name, template int Group::image_add(librados::IoCtx& group_ioctx, const char *group_name, - librados::IoCtx& image_ioctx, const char *image_name) + librados::IoCtx& image_ioctx, const char *image_name, + uint32_t flags) { CephContext *cct = (CephContext *)group_ioctx.cct(); ldout(cct, 20) << "io_ctx=" << &group_ioctx @@ -550,7 +553,8 @@ int Group::image_add(librados::IoCtx& group_ioctx, const char *group_name, } ImageWatcher<>::notify_header_update(image_ioctx, image_header_oid); - r = Mirror::group_image_add(group_ioctx, group_id, image_ioctx, image_id); + r = Mirror::group_image_add(group_ioctx, group_id, + image_ioctx, image_id, flags); if (r < 0) { return r; } @@ -568,7 +572,8 @@ int Group::image_add(librados::IoCtx& group_ioctx, const char *group_name, template int Group::image_remove(librados::IoCtx& group_ioctx, const char *group_name, - librados::IoCtx& image_ioctx, const char *image_name) + librados::IoCtx& image_ioctx, const char *image_name, + uint32_t flags) { CephContext *cct = (CephContext *)group_ioctx.cct(); ldout(cct, 20) << "io_ctx=" << &group_ioctx @@ -601,7 +606,8 @@ int Group::image_remove(librados::IoCtx& group_ioctx, const char *group_name, return r; } - r = Group::group_image_remove(group_ioctx, group_id, image_ioctx, image_id); + r = Group::group_image_remove(group_ioctx, group_id, image_ioctx, image_id, + false, flags); return r; } @@ -1222,7 +1228,8 @@ int Group::group_image_list_by_id(librados::IoCtx& group_ioctx, template int Group::group_image_remove(librados::IoCtx& group_ioctx, string group_id, - librados::IoCtx& image_ioctx, string image_id) { + librados::IoCtx& image_ioctx, string image_id, + bool resync, uint32_t flags) { CephContext *cct = (CephContext *)group_ioctx.cct(); string group_header_oid = librbd::util::group_header_name(group_id); @@ -1248,6 +1255,16 @@ int Group::group_image_remove(librados::IoCtx& group_ioctx, string group_id, return r; } + if (!resync) { + r = Mirror::group_image_remove(group_ioctx, group_id, image_ioctx, + image_id, flags); + if (r < 0) { + lderr(cct) << "couldn't remove image from group" + << cpp_strerror(-r) << dendl; + return r; + } + } + r = cls_client::image_group_remove(&image_ioctx, image_header_oid, group_spec); if ((r < 0) && (r != -ENOENT)) { diff --git a/src/librbd/api/Group.h b/src/librbd/api/Group.h index a73d082c8bef8..301b1dfdc406a 100644 --- a/src/librbd/api/Group.h +++ b/src/librbd/api/Group.h @@ -30,13 +30,16 @@ struct Group { const char *dest_group_name); static int image_add(librados::IoCtx& group_ioctx, const char *group_name, - librados::IoCtx& image_ioctx, const char *image_name); + librados::IoCtx& image_ioctx, const char *image_name, + uint32_t flags); static int image_remove(librados::IoCtx& group_ioctx, const char *group_name, - librados::IoCtx& image_ioctx, const char *image_name); + librados::IoCtx& image_ioctx, const char *image_name, + uint32_t flags); static int image_remove_by_id(librados::IoCtx& group_ioctx, const char *group_name, librados::IoCtx& image_ioctx, - const char *image_id); + const char *image_id, + uint32_t flags); static int image_list(librados::IoCtx& group_ioctx, const char *group_name, std::vector *images); @@ -62,8 +65,12 @@ struct Group { static int group_image_list_by_id(librados::IoCtx& group_ioctx, const std::string &group_id, std::vector *images); - static int group_image_remove(librados::IoCtx& group_ioctx, std::string group_id, - librados::IoCtx& image_ioctx, std::string image_id); + static int group_image_remove(librados::IoCtx& group_ioctx, + std::string group_id, + librados::IoCtx& image_ioctx, + std::string image_id, + bool resync, + uint32_t flags); }; diff --git a/src/librbd/api/Migration.cc b/src/librbd/api/Migration.cc index 4ae95f692fb5d..9fcba78d503a2 100644 --- a/src/librbd/api/Migration.cc +++ b/src/librbd/api/Migration.cc @@ -1647,7 +1647,7 @@ int Migration::remove_group(I *image_ctx, group_info_t *group_info) { r = librbd::api::Group::image_remove_by_id(group_ioctx, group_info->name.c_str(), image_ctx->md_ctx, - image_ctx->id.c_str()); + image_ctx->id.c_str(), 0); if (r < 0) { lderr(m_cct) << "failed to remove image from group: " << cpp_strerror(r) << dendl; @@ -1674,7 +1674,7 @@ int Migration::add_group(I *image_ctx, group_info_t &group_info) { r = librbd::api::Group::image_add(group_ioctx, group_info.name.c_str(), image_ctx->md_ctx, - image_ctx->name.c_str()); + image_ctx->name.c_str(), 0); if (r < 0) { lderr(m_cct) << "failed to add image to group: " << cpp_strerror(r) << dendl; diff --git a/src/librbd/api/Mirror.cc b/src/librbd/api/Mirror.cc index c40a059146707..4fcec826b1868 100644 --- a/src/librbd/api/Mirror.cc +++ b/src/librbd/api/Mirror.cc @@ -16,6 +16,7 @@ #include "librbd/MirroringWatcher.h" #include "librbd/Operations.h" #include "librbd/Utils.h" +#include "librbd/ExclusiveLock.h" #include "librbd/api/Group.h" #include "librbd/api/Image.h" #include "librbd/api/Namespace.h" @@ -516,7 +517,7 @@ std::string prepare_primary_mirror_snap_name(CephContext *cct, std::stringstream ind_snap_name_stream; ind_snap_name_stream << ".mirror.primary." << global_group_id << "." << snap_id; - return ind_snap_name_stream.str(); + return ind_snap_name_stream.str(); } int get_last_mirror_snapshot_state(librados::IoCtx &group_ioctx, @@ -677,7 +678,7 @@ int Mirror::image_disable(I *ictx, bool force) { return -EINVAL; } - // is mirroring enabled for the image? + // is mirroring enabled for the image? cls::rbd::MirrorImage mirror_image_internal; r = cls_client::mirror_image_get(&ictx->md_ctx, ictx->id, &mirror_image_internal); @@ -2339,9 +2340,212 @@ void Mirror::image_snapshot_create(I *ictx, uint32_t flags, } } +template +int prepare_group_images(IoCtx& group_ioctx, + std::string group_id, + std::vector *image_ctxs, + cls::rbd::GroupSnapshot *group_snap, + std::vector &quiesce_requests, + cls::rbd::MirrorSnapshotState state, + uint32_t flags) { + CephContext *cct = (CephContext *)group_ioctx.cct(); + ldout(cct, 20) << dendl; + + uint64_t internal_flags = 0; + int r = librbd::util::snap_create_flags_api_to_internal(cct, flags, + &internal_flags); + if (r < 0) { + return r; + } + + auto ns = group_ioctx.get_namespace(); + group_ioctx.set_namespace(""); + std::vector peers; + r = cls_client::mirror_peer_list(&group_ioctx, &peers); + if (r < 0) { + lderr(cct) << "error reading mirror peers: " << cpp_strerror(r) << dendl; + return r; + } + group_ioctx.set_namespace(ns); + + std::set mirror_peer_uuids; + for (auto &peer : peers) { + if (peer.mirror_peer_direction == cls::rbd::MIRROR_PEER_DIRECTION_RX) { + continue; + } + mirror_peer_uuids.insert(peer.uuid); + } + + if (mirror_peer_uuids.empty()) { + lderr(cct) << "no mirror tx peers configured for the pool" << dendl; + return -EINVAL; + } + r = open_group_images(group_ioctx, group_id, image_ctxs); + if (r < 0) { + return r; + } + + group_snap->snapshot_namespace = cls::rbd::MirrorGroupSnapshotNamespace{ + state, mirror_peer_uuids, {}, {}}; + + for (auto image_ctx: *image_ctxs) { + group_snap->snaps.emplace_back(image_ctx->md_ctx.get_id(), image_ctx->id, + CEPH_NOSNAP); + } + + int ret_code = 0; + std::vector on_finishes(image_ctxs->size(), nullptr); + std::string group_header_oid = librbd::util::group_header_name(group_id); + r = cls_client::group_snap_set(&group_ioctx, group_header_oid, *group_snap); + if (r < 0) { + lderr(cct) << "failed to set group snapshot metadata: " << cpp_strerror(r) + << dendl; + ret_code = r; + goto remove_record; + } + + if ((internal_flags & SNAP_CREATE_FLAG_SKIP_NOTIFY_QUIESCE) == 0) { + NoOpProgressContext prog_ctx; + r = util::notify_quiesce(*image_ctxs, prog_ctx, &quiesce_requests); + if (r < 0 && + (internal_flags & SNAP_CREATE_FLAG_IGNORE_NOTIFY_QUIESCE_ERROR) == 0) { + ret_code = r; + goto remove_record; + } + } + ldout(cct, 20) << "Requesting exclusive locks for images" << dendl; + + C_SaferCond* on_finish; + for (auto ictx: *image_ctxs) { + std::shared_lock owner_lock{ictx->owner_lock}; + if (ictx->exclusive_lock != nullptr) { + ictx->exclusive_lock->block_requests(-EBUSY); + } + } + for (size_t i = 0; i < image_ctxs->size(); ++i) { + ImageCtx *ictx = (*image_ctxs)[i]; + std::shared_lock owner_lock{ictx->owner_lock}; + + on_finish = new C_SaferCond; + if (ictx->exclusive_lock != nullptr) { + ictx->exclusive_lock->acquire_lock(on_finish); + on_finishes[i] = on_finish; + } + } + + for (size_t i = 0; i < image_ctxs->size(); ++i) { + r = 0; + ImageCtx *ictx = (*image_ctxs)[i]; + if (ictx->exclusive_lock != nullptr) { + r = on_finishes[i]->wait(); + } + delete on_finishes[i]; + if (r < 0) { + if (ret_code == 0) { + ret_code = r; + } + } + } + +remove_record: + if (ret_code < 0) { + r = cls_client::group_snap_remove(&group_ioctx, group_header_oid, + group_snap->id); + if (r < 0) { + lderr(cct) << "failed to remove group snapshot metadata: " + << cpp_strerror(r) << dendl; + } + + if (!quiesce_requests.empty()) { + util::notify_unquiesce(*image_ctxs, quiesce_requests); + } + + close_images(image_ctxs); + } + + return ret_code; +} + +template +void remove_interim_snapshots(IoCtx& group_ioctx, + std::string group_header_oid, + std::vector *image_ctxs, + cls::rbd::GroupSnapshot *group_snap) { + CephContext *cct = (CephContext *)group_ioctx.cct(); + ldout(cct, 20) << dendl; + + int r = cls_client::group_snap_remove(&group_ioctx, group_header_oid, + group_snap->id); + if (r < 0) { + lderr(cct) << "failed to remove group snapshot metadata: " + << cpp_strerror(r) << dendl; + } + + std::vector on_finishes(image_ctxs->size(), nullptr); + for (size_t i = 0; i < image_ctxs->size(); ++i) { + if (group_snap->snaps[i].snap_id == CEPH_NOSNAP) { + continue; + } + ldout(cct, 20) << "removing individual snapshot: " + << group_snap->snaps[i].snap_id << dendl; + + ImageCtx *ictx = (*image_ctxs)[i]; + C_SaferCond* on_finish = new C_SaferCond; + ictx->operations->snap_remove(ictx->snap_namespace, + ictx->snap_name.c_str(), + on_finish); + on_finishes[i] = on_finish; + } + + for (int i = 0, n = image_ctxs->size(); i < n; ++i) { + if (!on_finishes[i]) { + continue; + } + r = on_finishes[i]->wait(); + delete on_finishes[i]; + // if previous attempts to remove this snapshot failed then the image's snapshot may not exist + if (r < 0 && r != -ENOENT) { + lderr(cct) << "failed cleaning up image snapshot: " + << cpp_strerror(r) << dendl; + // just report error, but don't abort the process + } + } +} + +template +int are_images_mirror_enabled(std::vector *image_ctxs) { + std::vector mirror_images(image_ctxs->size()); + std::vector on_finishes(image_ctxs->size()); + int ret_code = 0; + ImageCtx *ictx; + for (size_t i = 0; i < image_ctxs->size(); i++) { + ictx = (*image_ctxs)[i]; + librbd::api::Mirror<>::image_get_info(ictx, + &mirror_images[i], &on_finishes[i]); + } + + for (size_t i = 0; i < image_ctxs->size(); i++) { + int r = on_finishes[i].wait(); + if (r < 0) { + if (ret_code == 0) { + ret_code = r; + } + } else if (mirror_images[i].state == RBD_MIRROR_IMAGE_ENABLED) { + ictx = (*image_ctxs)[i]; + lderr(ictx->cct) << "image " << ictx->name + << " has mirroring enabled" << dendl; + ret_code = -EINVAL; + break; + } + } + + return ret_code; +} + template int Mirror::group_list(IoCtx& io_ctx, std::vector *names) { CephContext *cct = reinterpret_cast(io_ctx.cct()); + ldout(cct, 20) << dendl; std::set group_ids; std::string last_read = ""; @@ -2385,7 +2589,8 @@ int Mirror::group_list(IoCtx& io_ctx, std::vector *names) { template int Mirror::group_enable(IoCtx& group_ioctx, const char *group_name, - mirror_image_mode_t mirror_image_mode) { + mirror_image_mode_t mirror_image_mode, + uint32_t flags) { CephContext *cct = (CephContext *)group_ioctx.cct(); ldout(cct, 20) << "io_ctx=" << &group_ioctx << ", group_name=" << group_name @@ -2449,151 +2654,113 @@ int Mirror::group_enable(IoCtx& group_ioctx, const char *group_name, return -EINVAL; } - // XXXMG: remove code duplication - auto ns = group_ioctx.get_namespace(); - group_ioctx.set_namespace(""); - std::vector peers; - r = cls_client::mirror_peer_list(&group_ioctx, &peers); - if (r < 0) { - lderr(cct) << "error reading mirror peers: " << cpp_strerror(r) << dendl; - return r; - } - group_ioctx.set_namespace(ns); - - std::set mirror_peer_uuids; - for (auto &peer : peers) { - ldout(cct, 0) << "peer: " << peer << dendl; - if (peer.mirror_peer_direction == cls::rbd::MIRROR_PEER_DIRECTION_RX) { - continue; - } - mirror_peer_uuids.insert(peer.uuid); - } - - if (mirror_peer_uuids.empty()) { - lderr(cct) << "no mirror tx peers configured for the pool" << dendl; - return -EINVAL; - } - - std::vector image_ctxs; - r = open_group_images(group_ioctx, group_id, &image_ctxs); - if (r < 0) { - return r; - } - - std::vector mirror_images(image_ctxs.size()); - std::vector on_finishes(image_ctxs.size()); - - for (size_t i = 0; i < on_finishes.size(); i++) { - image_get_info(image_ctxs[i], &mirror_images[i], &on_finishes[i]); - } - - int ret_code = 0; - for (size_t i = 0; i < on_finishes.size(); i++) { - r = on_finishes[i].wait(); - if (r < 0) { - if (ret_code != 0) { - ret_code = r; - } - } else if (mirror_images[i].state == RBD_MIRROR_IMAGE_ENABLED) { - lderr(cct) << "image " << image_ctxs[i]->name << " has mirroring enabled" - << dendl; - ret_code = -EINVAL; - } - } - - if (ret_code != 0) { - close_images(&image_ctxs); - return ret_code; - } - + std::string group_snap_id = librbd::util::generate_image_id(group_ioctx); uuid_d uuid_gen; uuid_gen.generate_random(); - mirror_group = {uuid_gen.to_string(), - static_cast(mirror_image_mode), - cls::rbd::MIRROR_GROUP_STATE_ENABLING}; - r = cls_client::mirror_group_set(&group_ioctx, group_id, mirror_group); - if (r < 0) { - lderr(cct) << "failed to set mirroring group metadata: " - << cpp_strerror(r) << dendl; - close_images(&image_ctxs); - return r; - } - - std::string group_snap_id = librbd::util::generate_image_id(group_ioctx); cls::rbd::GroupSnapshot group_snap{ group_snap_id, - cls::rbd::MirrorGroupSnapshotNamespace{ - cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, - mirror_peer_uuids, {}, {}}, + cls::rbd::MirrorGroupSnapshotNamespace{}, prepare_primary_mirror_snap_name(cct, uuid_gen.to_string(), group_snap_id), cls::rbd::GROUP_SNAPSHOT_STATE_INCOMPLETE}; - for (auto image_ctx: image_ctxs) { - group_snap.snaps.emplace_back(image_ctx->md_ctx.get_id(), image_ctx->id, - CEPH_NOSNAP); + std::vector quiesce_requests; + std::vector image_ctxs; + r = prepare_group_images(group_ioctx, group_id, &image_ctxs, + &group_snap, quiesce_requests, + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, + flags); + if (r != 0) { + return r; } + auto image_count = image_ctxs.size(); std::string group_header_oid = librbd::util::group_header_name(group_id); - r = cls_client::group_snap_set(&group_ioctx, group_header_oid, group_snap); - if (r < 0) { - lderr(cct) << "failed to set group snapshot metadata: " << cpp_strerror(r) - << dendl; - close_images(&image_ctxs); - return r; + std::vector snap_ids(image_ctxs.size(), CEPH_NOSNAP); + int ret_code = are_images_mirror_enabled(&image_ctxs); + if (!ret_code) { + mirror_group = {uuid_gen.to_string(), + static_cast(mirror_image_mode), + cls::rbd::MIRROR_GROUP_STATE_ENABLING}; + r = cls_client::mirror_group_set(&group_ioctx, group_id, mirror_group); + if (r < 0) { + lderr(cct) << "failed to set mirroring group metadata: " + << cpp_strerror(r) << dendl; + ret_code = r; + } } - std::vector snap_ids(image_ctxs.size()); + if (ret_code) { + remove_interim_snapshots(group_ioctx, group_header_oid, + &image_ctxs, &group_snap); + goto cleanup; + } for (size_t i = 0; i < image_ctxs.size(); i++) { - r = image_enable(image_ctxs[i], group_ioctx.get_id(), group_id, - group_snap_id, mirror_image_mode, false, &snap_ids[i]); + r = image_enable(image_ctxs[i], group_snap_id, mirror_image_mode, false, + &snap_ids[i]); if (r < 0) { - break; + lderr(cct) << "failed enabling image: " + << image_ctxs[i]->name << ": " << cpp_strerror(r) << dendl; + if (ret_code == 0) { + ret_code = r; + break; + } } + group_snap.snaps[i].snap_id = snap_ids[i]; } - auto image_count = image_ctxs.size(); - - close_images(&image_ctxs); - - if (r < 0) { - int rr = cls_client::group_snap_remove(&group_ioctx, group_header_oid, - group_snap.id); - if (rr < 0) { - lderr(cct) << "failed to remove group snapshot metadata: " - << cpp_strerror(rr) << dendl; + if (ret_code) { + // undo + ldout(cct, 20) << "undoing group enable: " << ret_code << dendl; + remove_interim_snapshots(group_ioctx, group_header_oid, + &image_ctxs, &group_snap); + for (size_t i = 0; i < image_ctxs.size(); i++) { + if (snap_ids[i] == CEPH_NOSNAP) { + continue; + } + r = image_disable(image_ctxs[i], false); + if (r < 0) { + lderr(cct) << "failed to disable mirroring on image: " + << image_ctxs[i]->name << cpp_strerror(r) << dendl; + } + } + } else { + mirror_group.state = cls::rbd::MIRROR_GROUP_STATE_ENABLED; + r = cls_client::mirror_group_set(&group_ioctx, group_id, mirror_group); + if (r < 0) { + lderr(cct) << "failed to update mirroring group metadata: " + << cpp_strerror(r) << dendl; + ret_code = r; } - // TODO: try to remove the created image snapshots - return r; + group_snap.state = cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE; + r = cls_client::group_snap_set(&group_ioctx, group_header_oid, group_snap); + if (r < 0) { + lderr(cct) << "failed to update group snapshot metadata: " + << cpp_strerror(r) << dendl; + } } - group_snap.state = cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE; - r = cls_client::group_snap_set(&group_ioctx, group_header_oid, group_snap); - if (r < 0) { - lderr(cct) << "failed to update group snapshot metadata: " - << cpp_strerror(r) << dendl; +cleanup: + if (!quiesce_requests.empty()) { + util::notify_unquiesce(image_ctxs, quiesce_requests); } - mirror_group.state = cls::rbd::MIRROR_GROUP_STATE_ENABLED; - r = cls_client::mirror_group_set(&group_ioctx, group_id, mirror_group); - if (r < 0) { - lderr(cct) << "failed to update mirroring group metadata: " - << cpp_strerror(r) << dendl; - return r; - } + close_images(&image_ctxs); - r = MirroringWatcher::notify_group_updated( - group_ioctx, cls::rbd::MIRROR_GROUP_STATE_ENABLED, group_id, - mirror_group.global_group_id, image_count); - if (r < 0) { - lderr(cct) << "failed to notify mirroring group=" << group_name - << " updated: " << cpp_strerror(r) << dendl; - // not fatal + if (!ret_code) { + r = MirroringWatcher::notify_group_updated( + group_ioctx, cls::rbd::MIRROR_GROUP_STATE_ENABLED, group_id, + mirror_group.global_group_id, image_count); + if (r < 0) { + lderr(cct) << "failed to notify mirroring group=" << group_name + << " updated: " << cpp_strerror(r) << dendl; + // not fatal + } } - return 0; + return ret_code; } template @@ -2668,21 +2835,17 @@ int Mirror::group_disable(IoCtx& group_ioctx, const char *group_name, return r; } - r = MirroringWatcher::notify_group_updated( - group_ioctx, cls::rbd::MIRROR_GROUP_STATE_DISABLED, group_id, - mirror_group.global_group_id, image_ctxs.size()); - if (r < 0) { - lderr(cct) << "failed to notify mirroring group=" << group_name - << " updated: " << cpp_strerror(r) << dendl; - // not fatal - } - + int ret_code = 0; for (auto image_ctx : image_ctxs) { ldout(cct, 10) << "attempting to disable image with id " << image_ctx->id << ": " << cpp_strerror(r) << dendl; r = image_disable(image_ctx, force); if (r < 0) { - break; + lderr(cct) << "failed to disable mirroring on image: " << image_ctx->name + << cpp_strerror(r) << dendl; + if (ret_code == 0) { + ret_code = r; + } } } @@ -2690,6 +2853,8 @@ int Mirror::group_disable(IoCtx& group_ioctx, const char *group_name, close_images(&image_ctxs); + // undo an image disable might not be of our interest. If needed, user must + // issue the same command again. if (r < 0) { lderr(cct) << "failed to disable one or more images: " << cpp_strerror(r) << dendl; @@ -2736,7 +2901,6 @@ int Mirror::group_disable(IoCtx& group_ioctx, const char *group_name, if (r < 0) { lderr(cct) << "failed to notify mirroring group=" << group_name << " updated: " << cpp_strerror(r) << dendl; - // not fatal } @@ -2745,7 +2909,7 @@ int Mirror::group_disable(IoCtx& group_ioctx, const char *group_name, template int Mirror::group_promote(IoCtx& group_ioctx, const char *group_name, - bool force) { + uint32_t flags, bool force) { CephContext *cct = (CephContext *)group_ioctx.cct(); ldout(cct, 20) << "io_ctx=" << &group_ioctx << ", group_name=" << group_name @@ -2796,72 +2960,44 @@ int Mirror::group_promote(IoCtx& group_ioctx, const char *group_name, return -EBUSY; } - std::vector peers; - r = cls_client::mirror_peer_list(&group_ioctx, &peers); - if (r < 0) { - lderr(cct) << "error reading mirror peers: " << cpp_strerror(r) << dendl; - return r; - } - - std::set mirror_peer_uuids; - for (auto &peer : peers) { - if (peer.mirror_peer_direction == cls::rbd::MIRROR_PEER_DIRECTION_RX) { - continue; - } - mirror_peer_uuids.insert(peer.uuid); - } - - if (mirror_peer_uuids.empty()) { - lderr(cct) << "no mirror tx peers configured for the pool" << dendl; - return -EINVAL; - } - - std::vector image_ctxs; - r = open_group_images(group_ioctx, group_id, &image_ctxs); - if (r < 0) { - return r; - } - std::string group_snap_id = librbd::util::generate_image_id(group_ioctx); - cls::rbd::GroupSnapshot group_snap{ group_snap_id, - cls::rbd::MirrorGroupSnapshotNamespace{ - cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, - mirror_peer_uuids, {}, {}}, + cls::rbd::MirrorGroupSnapshotNamespace{}, prepare_primary_mirror_snap_name(cct, mirror_group.global_group_id, group_snap_id), cls::rbd::GROUP_SNAPSHOT_STATE_INCOMPLETE}; - for (auto image_ctx: image_ctxs) { - group_snap.snaps.emplace_back(image_ctx->md_ctx.get_id(), image_ctx->id, - CEPH_NOSNAP); - } - - std::string group_header_oid = librbd::util::group_header_name(group_id); - r = cls_client::group_snap_set(&group_ioctx, group_header_oid, group_snap); - if (r < 0) { - lderr(cct) << "failed to set group snapshot metadata: " << cpp_strerror(r) - << dendl; - close_images(&image_ctxs); + std::vector quiesce_requests; + std::vector image_ctxs; + r = prepare_group_images(group_ioctx, group_id, &image_ctxs, + &group_snap, quiesce_requests, + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, + flags); + if (r != 0) { return r; } - // XXXMG: Requesting exclusive locks for images? - // XXXMG: notify quiesce? - - std::vector snap_ids(image_ctxs.size()); - std::vector on_finishes(image_ctxs.size()); + int ret_code = 0; + std::vector snap_ids(image_ctxs.size(), CEPH_NOSNAP); + std::vector on_finishes(image_ctxs.size(), nullptr); + C_SaferCond* on_finish; - for (size_t i = 0; i < on_finishes.size(); i++) { - image_promote(image_ctxs[i], group_ioctx.get_id(), group_id, group_snap_id, - force, &snap_ids[i], &on_finishes[i]); + for (size_t i = 0; i < image_ctxs.size(); i++) { + on_finish = new C_SaferCond; + image_promote(image_ctxs[i], group_snap_id, force, &snap_ids[i], on_finish); + on_finishes[i] = on_finish; } - int ret_code = 0; - for (size_t i = 0; i < on_finishes.size(); i++) { - r = on_finishes[i].wait(); + for (size_t i = 0; i < image_ctxs.size(); i++) { + r = 0; + if (on_finishes[i]) { + r = on_finishes[i]->wait(); + delete on_finishes[i]; + } if (r < 0) { + lderr(cct) << "failed promoting image: " << image_ctxs[i]->name << ": " + << cpp_strerror(r) << dendl; if (ret_code == 0) { ret_code = r; } @@ -2870,21 +3006,59 @@ int Mirror::group_promote(IoCtx& group_ioctx, const char *group_name, } } + std::string group_header_oid = librbd::util::group_header_name(group_id); if (ret_code < 0) { - r = cls_client::group_snap_remove(&group_ioctx, group_header_oid, - group_snap.id); - if (r < 0) { - lderr(cct) << "failed to remove group snapshot metadata: " - << cpp_strerror(r) << dendl; + // undo + ldout(cct, 20) << "undoing group promote: " << ret_code << dendl; + remove_interim_snapshots(group_ioctx, group_header_oid, &image_ctxs, &group_snap); + std::fill(snap_ids.begin(), snap_ids.end(), CEPH_NOSNAP); + group_snap.snaps.clear(); + if (!quiesce_requests.empty()) { + util::notify_unquiesce(image_ctxs, quiesce_requests); } - } else { - group_snap.state = cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE; - r = cls_client::group_snap_set(&group_ioctx, group_header_oid, group_snap); - if (r < 0) { - lderr(cct) << "failed to update group snapshot metadata: " - << cpp_strerror(r) << dendl; - ret_code = r; + close_images(&image_ctxs); + + r = prepare_group_images(group_ioctx, group_id, &image_ctxs, + &group_snap, quiesce_requests, + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED, + flags); + if (r != 0) { + return r; } + for (size_t i = 0; i < image_ctxs.size(); ++i) { + ImageCtx *ictx = image_ctxs[i]; + ldout(cct, 20) << "demoting individual image: " + << ictx->name.c_str() << dendl; + on_finish = new C_SaferCond; + image_demote(ictx, group_snap_id, &snap_ids[i], on_finish); + on_finishes[i] = on_finish; + } + + for (int i = 0, n = image_ctxs.size(); i < n; ++i) { + if (!on_finishes[i]) { + continue; + } + r = on_finishes[i]->wait(); + delete on_finishes[i]; + if (r < 0) { + lderr(cct) << "failed demoting image: " << image_ctxs[i]->name << ": " + << cpp_strerror(r) << dendl; + // just report error, but don't abort the process + } else{ + group_snap.snaps[i].snap_id = snap_ids[i]; + } + } + } + + group_snap.state = cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE; + r = cls_client::group_snap_set(&group_ioctx, group_header_oid, group_snap); + if (r < 0) { + lderr(cct) << "failed to update group snapshot metadata: " + << cpp_strerror(r) << dendl; + } + + if (!quiesce_requests.empty()) { + util::notify_unquiesce(image_ctxs, quiesce_requests); } close_images(&image_ctxs); @@ -2893,14 +3067,16 @@ int Mirror::group_promote(IoCtx& group_ioctx, const char *group_name, } template -int Mirror::group_demote(IoCtx& group_ioctx, const char *group_name) { +int Mirror::group_demote(IoCtx& group_ioctx, + const char *group_name, + uint32_t flags) { CephContext *cct = (CephContext *)group_ioctx.cct(); ldout(cct, 20) << "io_ctx=" << &group_ioctx << ", group_name=" << group_name << dendl; std::string group_id; int r = cls_client::dir_get_id(&group_ioctx, RBD_GROUP_DIRECTORY, - group_name, &group_id); + group_name, &group_id); if (r < 0) { lderr(cct) << "error getting the group id: " << cpp_strerror(r) << dendl; return r; @@ -2939,72 +3115,44 @@ int Mirror::group_demote(IoCtx& group_ioctx, const char *group_name) { return -EINVAL; } - std::vector peers; - r = cls_client::mirror_peer_list(&group_ioctx, &peers); - if (r < 0) { - lderr(cct) << "error reading mirror peers: " << cpp_strerror(r) << dendl; - return r; - } - - std::set mirror_peer_uuids; - for (auto &peer : peers) { - if (peer.mirror_peer_direction == cls::rbd::MIRROR_PEER_DIRECTION_RX) { - continue; - } - mirror_peer_uuids.insert(peer.uuid); - } - - if (mirror_peer_uuids.empty()) { - lderr(cct) << "no mirror tx peers configured for the pool" << dendl; - return -EINVAL; - } - - std::vector image_ctxs; - r = open_group_images(group_ioctx, group_id, &image_ctxs); - if (r < 0) { - return r; - } - std::string group_snap_id = librbd::util::generate_image_id(group_ioctx); - cls::rbd::GroupSnapshot group_snap{ group_snap_id, - cls::rbd::MirrorGroupSnapshotNamespace{ - cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED, - mirror_peer_uuids, {}, {}}, + cls::rbd::MirrorGroupSnapshotNamespace{}, prepare_primary_mirror_snap_name(cct, mirror_group.global_group_id, group_snap_id), cls::rbd::GROUP_SNAPSHOT_STATE_INCOMPLETE}; - for (auto image_ctx: image_ctxs) { - group_snap.snaps.emplace_back(image_ctx->md_ctx.get_id(), image_ctx->id, - CEPH_NOSNAP); - } - - std::string group_header_oid = librbd::util::group_header_name(group_id); - r = cls_client::group_snap_set(&group_ioctx, group_header_oid, group_snap); - if (r < 0) { - lderr(cct) << "failed to set group snapshot metadata: " << cpp_strerror(r) - << dendl; - close_images(&image_ctxs); + std::vector quiesce_requests; + std::vector image_ctxs; + r = prepare_group_images(group_ioctx, group_id, &image_ctxs, + &group_snap, quiesce_requests, + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED, + flags); + if (r != 0) { return r; } - // XXXMG: Requesting exclusive locks for images? - // XXXMG: notify quiesce? - - std::vector snap_ids(image_ctxs.size()); - std::vector on_finishes(image_ctxs.size()); + int ret_code = 0; + std::vector snap_ids(image_ctxs.size(), CEPH_NOSNAP); + std::vector on_finishes(image_ctxs.size(), nullptr); + C_SaferCond* on_finish; - for (size_t i = 0; i < on_finishes.size(); i++) { - image_demote(image_ctxs[i], group_ioctx.get_id(), group_id, group_snap_id, - &snap_ids[i], &on_finishes[i]); + for (size_t i = 0; i < image_ctxs.size(); i++) { + C_SaferCond* on_finish = new C_SaferCond; + image_demote(image_ctxs[i], group_snap_id, &snap_ids[i], on_finish); + on_finishes[i] = on_finish; } - int ret_code = 0; - for (size_t i = 0; i < on_finishes.size(); i++) { - r = on_finishes[i].wait(); + for (size_t i = 0; i < image_ctxs.size(); i++) { + r = 0; + if (on_finishes[i]) { + r = on_finishes[i]->wait(); + delete on_finishes[i]; + } if (r < 0) { + lderr(cct) << "failed demoting image: " << image_ctxs[i]->name << ": " + << cpp_strerror(r) << dendl; if (ret_code == 0) { ret_code = r; } @@ -3013,23 +3161,62 @@ int Mirror::group_demote(IoCtx& group_ioctx, const char *group_name) { } } + std::string group_header_oid = librbd::util::group_header_name(group_id); if (ret_code < 0) { - r = cls_client::group_snap_remove(&group_ioctx, group_header_oid, - group_snap.id); - if (r < 0) { - lderr(cct) << "failed to remove group snapshot metadata: " - << cpp_strerror(r) << dendl; + // undo + ldout(cct, 20) << "undoing group demote: " << ret_code << dendl; + remove_interim_snapshots(group_ioctx, group_header_oid, &image_ctxs, &group_snap); + std::fill(snap_ids.begin(), snap_ids.end(), CEPH_NOSNAP); + group_snap.snaps.clear(); + if (!quiesce_requests.empty()) { + util::notify_unquiesce(image_ctxs, quiesce_requests); } - } else { - group_snap.state = cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE; - r = cls_client::group_snap_set(&group_ioctx, group_header_oid, group_snap); - if (r < 0) { - lderr(cct) << "failed to update group snapshot metadata: " - << cpp_strerror(r) << dendl; - ret_code = r; + close_images(&image_ctxs); + + r = prepare_group_images(group_ioctx, group_id, &image_ctxs, + &group_snap, quiesce_requests, + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, + flags); + if (r != 0) { + return r; + } + for (size_t i = 0; i < image_ctxs.size(); ++i) { + ImageCtx *ictx = image_ctxs[i]; + ldout(cct, 20) << "promoting individual images: " + << ictx->name.c_str() << dendl; + + on_finish = new C_SaferCond; + image_promote(ictx, group_snap_id, false, &snap_ids[i], on_finish); // No force? + on_finishes[i] = on_finish; + } + + for (int i = 0, n = image_ctxs.size(); i < n; ++i) { + if (!on_finishes[i]) { + continue; + } + r = on_finishes[i]->wait(); + delete on_finishes[i]; + if (r < 0) { + lderr(cct) << "failed promoting image: " << image_ctxs[i]->name + << ": " << cpp_strerror(r) << dendl; + // just report error, but don't abort the process + } else { + group_snap.snaps[i].snap_id = snap_ids[i]; + } } } + group_snap.state = cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE; + r = cls_client::group_snap_set(&group_ioctx, group_header_oid, group_snap); + if (r < 0) { + lderr(cct) << "failed to update group snapshot metadata: " + << cpp_strerror(r) << dendl; + } + + if (!quiesce_requests.empty()) { + util::notify_unquiesce(image_ctxs, quiesce_requests); + } + close_images(&image_ctxs); return ret_code; @@ -3096,44 +3283,14 @@ int Mirror::group_snapshot_create(IoCtx& group_ioctx, const char *group_name, << ", group_name=" << group_name << ", flags=" << flags << dendl; - uint64_t internal_flags = 0; - int r = librbd::util::snap_create_flags_api_to_internal(cct, flags, - &internal_flags); - if (r < 0) { - return r; - } - std::string group_id; - r = cls_client::dir_get_id(&group_ioctx, RBD_GROUP_DIRECTORY, group_name, - &group_id); + int r = cls_client::dir_get_id(&group_ioctx, RBD_GROUP_DIRECTORY, + group_name, &group_id); if (r < 0) { lderr(cct) << "error getting the group id: " << cpp_strerror(r) << dendl; return r; } - auto ns = group_ioctx.get_namespace(); - group_ioctx.set_namespace(""); - std::vector peers; - r = cls_client::mirror_peer_list(&group_ioctx, &peers); - if (r < 0) { - lderr(cct) << "error reading mirror peers: " << cpp_strerror(r) << dendl; - return r; - } - group_ioctx.set_namespace(ns); - - std::set mirror_peer_uuids; - for (auto &peer : peers) { - if (peer.mirror_peer_direction == cls::rbd::MIRROR_PEER_DIRECTION_RX) { - continue; - } - mirror_peer_uuids.insert(peer.uuid); - } - - if (mirror_peer_uuids.empty()) { - lderr(cct) << "no mirror tx peers configured for the pool" << dendl; - return -EINVAL; - } - cls::rbd::MirrorGroup mirror_group; r = cls_client::mirror_group_get(&group_ioctx, group_id, &mirror_group); if (r == -ENOENT) { @@ -3168,62 +3325,41 @@ int Mirror::group_snapshot_create(IoCtx& group_ioctx, const char *group_name, return -EINVAL; } - std::vector image_ctxs; - r = open_group_images(group_ioctx, group_id, &image_ctxs); - if (r < 0) { - return r; - } - std::string group_snap_id = librbd::util::generate_image_id(group_ioctx); - cls::rbd::GroupSnapshot group_snap{ group_snap_id, - cls::rbd::MirrorGroupSnapshotNamespace{ - cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, - mirror_peer_uuids, {}, {}}, + cls::rbd::MirrorGroupSnapshotNamespace{}, prepare_primary_mirror_snap_name(cct, mirror_group.global_group_id, group_snap_id), cls::rbd::GROUP_SNAPSHOT_STATE_INCOMPLETE}; - for (auto image_ctx: image_ctxs) { - group_snap.snaps.emplace_back(image_ctx->md_ctx.get_id(), image_ctx->id, - CEPH_NOSNAP); - } - - std::string group_header_oid = librbd::util::group_header_name(group_id); - r = cls_client::group_snap_set(&group_ioctx, group_header_oid, group_snap); - if (r < 0) { - lderr(cct) << "failed to set group snapshot metadata: " << cpp_strerror(r) - << dendl; - close_images(&image_ctxs); - return r; - } - - // XXXMG: Requesting exclusive locks for images? - std::vector quiesce_requests; - if ((internal_flags & SNAP_CREATE_FLAG_SKIP_NOTIFY_QUIESCE) == 0) { - NoOpProgressContext prog_ctx; - r = util::notify_quiesce(image_ctxs, prog_ctx, &quiesce_requests); - if (r < 0 && - (internal_flags & SNAP_CREATE_FLAG_IGNORE_NOTIFY_QUIESCE_ERROR) == 0) { - close_images(&image_ctxs); - return r; - } + std::vector image_ctxs; + r = prepare_group_images(group_ioctx, group_id, &image_ctxs, + &group_snap, quiesce_requests, + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, + flags); + if (r != 0) { + return r; } - std::vector snap_ids(image_ctxs.size()); - std::vector on_finishes(image_ctxs.size()); + int ret_code = 0; + std::vector snap_ids(image_ctxs.size(), CEPH_NOSNAP); + std::vector on_finishes(image_ctxs.size(), nullptr); - for (size_t i = 0; i < on_finishes.size(); i++) { + for (size_t i = 0; i < image_ctxs.size(); i++) { + C_SaferCond* on_finish = new C_SaferCond; image_snapshot_create(image_ctxs[i], RBD_SNAP_CREATE_SKIP_QUIESCE, - group_ioctx.get_id(), group_id, group_snap_id, - &snap_ids[i], &on_finishes[i]); + group_snap_id, &snap_ids[i], on_finish); + on_finishes[i] = on_finish; } - int ret_code = 0; - for (size_t i = 0; i < on_finishes.size(); i++) { - r = on_finishes[i].wait(); + for (size_t i = 0; i < image_ctxs.size(); i++) { + r = 0; + if (on_finishes[i]) { + r = on_finishes[i]->wait(); + delete on_finishes[i]; + } if (r < 0) { if (ret_code == 0) { ret_code = r; @@ -3233,45 +3369,37 @@ int Mirror::group_snapshot_create(IoCtx& group_ioctx, const char *group_name, } } - if (ret_code < 0) { - r = cls_client::group_snap_remove(&group_ioctx, group_header_oid, - group_snap.id); - if (r < 0) { - lderr(cct) << "failed to remove group snapshot metadata: " - << cpp_strerror(r) << dendl; - } - } else { + std::string group_header_oid = librbd::util::group_header_name(group_id); + if (ret_code != 0) { + // undo + ldout(cct, 20) << "undoing group create snapshot: " << ret_code << dendl; + remove_interim_snapshots(group_ioctx, group_header_oid, &image_ctxs, &group_snap); + } else{ group_snap.state = cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE; r = cls_client::group_snap_set(&group_ioctx, group_header_oid, group_snap); if (r < 0) { lderr(cct) << "failed to update group snapshot metadata: " << cpp_strerror(r) << dendl; - ret_code = r; } + + *snap_id = group_snap.id; } if (!quiesce_requests.empty()) { util::notify_unquiesce(image_ctxs, quiesce_requests); } - // TODO: try to remove created snapshots in a failure case - close_images(&image_ctxs); - if (ret_code < 0) { - return ret_code; - } - - *snap_id = group_snap.id; - - return 0; + return ret_code; } template int Mirror::group_image_add(IoCtx &group_ioctx, const std::string &group_id, IoCtx &image_ioctx, - const std::string &image_id) { + const std::string &image_id, + uint32_t flags) { CephContext *cct = (CephContext *)group_ioctx.cct(); ldout(cct, 20) << "group io_ctx=" << &group_ioctx << ", group_id=" << group_id << ", image io_ctx=" << &image_ioctx << ", image_id=" @@ -3283,85 +3411,58 @@ int Mirror::group_image_add(IoCtx &group_ioctx, return 0; } - // XXXMG: remove code duplication - - auto ns = group_ioctx.get_namespace(); - group_ioctx.set_namespace(""); - std::vector peers; - r = cls_client::mirror_peer_list(&group_ioctx, &peers); - if (r < 0) { - lderr(cct) << "error reading mirror peers: " << cpp_strerror(r) << dendl; - return r; - } - group_ioctx.set_namespace(ns); - - std::set mirror_peer_uuids; - for (auto &peer : peers) { - if (peer.mirror_peer_direction == cls::rbd::MIRROR_PEER_DIRECTION_RX) { - continue; - } - mirror_peer_uuids.insert(peer.uuid); - } - - if (mirror_peer_uuids.empty()) { - lderr(cct) << "no mirror tx peers configured for the pool" << dendl; - return -EINVAL; - } - - std::vector image_ctxs; - r = open_group_images(group_ioctx, group_id, &image_ctxs); - if (r < 0) { - return r; - } - std::string group_snap_id = librbd::util::generate_image_id(group_ioctx); - cls::rbd::GroupSnapshot group_snap{ group_snap_id, - cls::rbd::MirrorGroupSnapshotNamespace{ - cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, - mirror_peer_uuids, {}, {}}, + cls::rbd::MirrorGroupSnapshotNamespace{}, prepare_primary_mirror_snap_name(cct, mirror_info.global_group_id, group_snap_id), cls::rbd::GROUP_SNAPSHOT_STATE_INCOMPLETE}; - for (auto image_ctx: image_ctxs) { - group_snap.snaps.emplace_back(image_ctx->md_ctx.get_id(), image_ctx->id, - CEPH_NOSNAP); - } - - std::string group_header_oid = librbd::util::group_header_name(group_id); - r = cls_client::group_snap_set(&group_ioctx, group_header_oid, group_snap); - if (r < 0) { - lderr(cct) << "failed to set group snapshot metadata: " << cpp_strerror(r) - << dendl; - close_images(&image_ctxs); + std::vector quiesce_requests; + std::vector image_ctxs; + int r = prepare_group_images(group_ioctx, group_id, &image_ctxs, + &group_snap, quiesce_requests, + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, + flags); + if (r != 0) { return r; } - // XXXMG: notify quiesce? - - std::vector snap_ids(image_ctxs.size()); - std::vector on_finishes(image_ctxs.size()); + int ret_code = 0; + auto image_count = image_ctxs.size(); + std::vector snap_ids(image_ctxs.size(), CEPH_NOSNAP); + std::vector on_finishes(image_ctxs.size(), nullptr); + C_SaferCond* on_finish; + int image_index = -1; auto mode = static_cast( mirror_info.mirror_image_mode); for (size_t i = 0; i < image_ctxs.size(); i++) { if (image_ctxs[i]->md_ctx.get_id() == image_ioctx.get_id() && image_ctxs[i]->id == image_id) { - r = image_enable(image_ctxs[i], group_ioctx.get_id(), group_id, - group_snap_id, mode, false, &snap_ids[i]); - on_finishes[i].complete(r); + r = image_enable(image_ctxs[i], group_snap_id, mode, false, &snap_ids[i]); + if (r < 0) { + lderr(cct) << "failed to enable mirroring on image: " + << image_ctxs[i]->name << cpp_strerror(r) << dendl; + ret_code = r; + } else { + image_index = i; + } } else { + on_finish = new C_SaferCond; image_snapshot_create(image_ctxs[i], RBD_SNAP_CREATE_SKIP_QUIESCE, - group_ioctx.get_id(), group_id, group_snap_id, - &snap_ids[i], &on_finishes[i]); + group_snap_id, &snap_ids[i], on_finish); + on_finishes[i] = on_finish; } } - int ret_code = 0; - for (size_t i = 0; i < on_finishes.size(); i++) { - r = on_finishes[i].wait(); + for (size_t i = 0; i < image_ctxs.size(); i++) { + r = 0; + if (on_finishes[i]) { + r = on_finishes[i]->wait(); + delete on_finishes[i]; + } if (r < 0) { if (ret_code == 0) { ret_code = r; @@ -3371,47 +3472,53 @@ int Mirror::group_image_add(IoCtx &group_ioctx, } } - auto image_count = image_ctxs.size(); - - close_images(&image_ctxs); - + std::string group_header_oid = librbd::util::group_header_name(group_id); if (ret_code < 0) { - r = cls_client::group_snap_remove(&group_ioctx, group_header_oid, - group_snap.id); + // undo + ldout(cct, 20) << "undoing group add image: " << ret_code << dendl; + if (image_index >= 0) { + r = image_disable(image_ctxs[image_index], false); + if (r < 0) { + lderr(cct) << "failed to disable mirroring on image: " + << image_ctxs[image_index]->name << cpp_strerror(r) << dendl; + } + } + remove_interim_snapshots(group_ioctx, group_header_oid, &image_ctxs, &group_snap); + } else{ + group_snap.state = cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE; + r = cls_client::group_snap_set(&group_ioctx, group_header_oid, group_snap); if (r < 0) { - lderr(cct) << "failed to remove group snapshot metadata: " + lderr(cct) << "failed to update group snapshot metadata: " << cpp_strerror(r) << dendl; } - - // TODO: try to remove the created image snapshots - return ret_code; } - group_snap.state = cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE; - r = cls_client::group_snap_set(&group_ioctx, group_header_oid, group_snap); - if (r < 0) { - lderr(cct) << "failed to update group snapshot metadata: " - << cpp_strerror(r) << dendl; - return r; + if (!quiesce_requests.empty()) { + util::notify_unquiesce(image_ctxs, quiesce_requests); } - r = MirroringWatcher::notify_group_updated( - group_ioctx, cls::rbd::MIRROR_GROUP_STATE_ENABLED, group_id, - mirror_info.global_group_id, image_count); - if (r < 0) { - lderr(cct) << "failed to notify mirroring group_id=" << group_id - << " updated: " << cpp_strerror(r) << dendl; - // not fatal + close_images(&image_ctxs); + + if (!ret_code) { + r = MirroringWatcher::notify_group_updated( + group_ioctx, cls::rbd::MIRROR_GROUP_STATE_ENABLED, group_id, + mirror_info.global_group_id, image_count); + if (r < 0) { + lderr(cct) << "failed to notify mirroring group_id=" << group_id + << " updated: " << cpp_strerror(r) << dendl; + // not fatal + } } - return 0; + return ret_code; } template int Mirror::group_image_remove(IoCtx &group_ioctx, const std::string &group_id, IoCtx &image_ioctx, - const std::string &image_id) { + const std::string &image_id, + uint32_t flags) { CephContext *cct = (CephContext *)group_ioctx.cct(); ldout(cct, 20) << "group io_ctx=" << &group_ioctx << ", group_id=" << group_id << ", image io_ctx=" << &image_ioctx << ", image_id=" @@ -3423,135 +3530,110 @@ int Mirror::group_image_remove(IoCtx &group_ioctx, return 0; } - // XXXMG: remove code duplication - - auto ns = group_ioctx.get_namespace(); - group_ioctx.set_namespace(""); - std::vector peers; - r = cls_client::mirror_peer_list(&group_ioctx, &peers); - if (r < 0) { - lderr(cct) << "error reading mirror peers: " << cpp_strerror(r) << dendl; - return r; - } - group_ioctx.set_namespace(ns); - - std::set mirror_peer_uuids; - for (auto &peer : peers) { - if (peer.mirror_peer_direction == cls::rbd::MIRROR_PEER_DIRECTION_RX) { - continue; - } - mirror_peer_uuids.insert(peer.uuid); - } - - if (mirror_peer_uuids.empty()) { - lderr(cct) << "no mirror tx peers configured for the pool" << dendl; - return -EINVAL; - } - - std::vector image_ctxs; - r = open_group_images(group_ioctx, group_id, &image_ctxs); - if (r < 0) { - return r; - } - std::string group_snap_id = librbd::util::generate_image_id(group_ioctx); - cls::rbd::GroupSnapshot group_snap{ group_snap_id, - cls::rbd::MirrorGroupSnapshotNamespace{ - cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, - mirror_peer_uuids, {}, {}}, + cls::rbd::MirrorGroupSnapshotNamespace{}, prepare_primary_mirror_snap_name(cct, mirror_info.global_group_id, group_snap_id), cls::rbd::GROUP_SNAPSHOT_STATE_INCOMPLETE}; - for (auto image_ctx: image_ctxs) { - if (image_ctx->md_ctx.get_id() == image_ioctx.get_id() && - image_ctx->id == image_id) { - continue; - } - group_snap.snaps.emplace_back(image_ctx->md_ctx.get_id(), image_ctx->id, - CEPH_NOSNAP); - } - - std::string group_header_oid = librbd::util::group_header_name(group_id); - r = cls_client::group_snap_set(&group_ioctx, group_header_oid, group_snap); - if (r < 0) { - lderr(cct) << "failed to set group snapshot metadata: " << cpp_strerror(r) - << dendl; - close_images(&image_ctxs); + std::vector quiesce_requests; + std::vector image_ctxs; + int r = prepare_group_images(group_ioctx, group_id, &image_ctxs, + &group_snap, quiesce_requests, + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, + flags); + if (r != 0) { return r; } - // XXXMG: notify quiesce? - - std::vector snap_ids(image_ctxs.size()); - std::vector on_finishes(image_ctxs.size()); + int ret_code = 0; + auto image_count = image_ctxs.size(); + std::vector snap_ids(image_ctxs.size(), CEPH_NOSNAP); + std::vector on_finishes(image_ctxs.size(), nullptr); + int image_index = -1; for (size_t i = 0; i < image_ctxs.size(); i++) { if (image_ctxs[i]->md_ctx.get_id() == image_ioctx.get_id() && image_ctxs[i]->id == image_id) { - r = image_disable(image_ctxs[i], true); - snap_ids[i] = CEPH_NOSNAP; - on_finishes[i].complete(r); + r = image_disable(image_ctxs[i], false); + if (r < 0) { + lderr(cct) << "failed to disable mirroring on image: " + << image_ctxs[i]->name << cpp_strerror(r) << dendl; + ret_code = r; + } else { + image_index = i; + } } else { + C_SaferCond* on_finish = new C_SaferCond; image_snapshot_create(image_ctxs[i], RBD_SNAP_CREATE_SKIP_QUIESCE, - group_ioctx.get_id(), group_id, group_snap_id, - &snap_ids[i], &on_finishes[i]); + group_snap_id, &snap_ids[i], on_finish); + on_finishes[i] = on_finish; } } - int ret_code = 0; - group_snap.snaps.clear(); - for (size_t i = 0; i < on_finishes.size(); i++) { - r = on_finishes[i].wait(); + for (size_t i = 0; i < image_ctxs.size(); i++) { + if (image_ctxs[i]->md_ctx.get_id() == image_ioctx.get_id() && + image_ctxs[i]->id == image_id) { + continue; + } + if (on_finishes[i]) { + r = on_finishes[i]->wait(); + delete on_finishes[i]; + } if (r < 0) { if (ret_code == 0) { ret_code = r; } } - if (image_ctxs[i]->md_ctx.get_id() == image_ioctx.get_id() && - image_ctxs[i]->id == image_id) { - continue; - } group_snap.snaps.emplace_back(image_ctxs[i]->md_ctx.get_id(), image_ctxs[i]->id, snap_ids[i]); } - auto image_count = image_ctxs.size(); - - close_images(&image_ctxs); - + std::string group_header_oid = librbd::util::group_header_name(group_id); if (ret_code < 0) { - r = cls_client::group_snap_remove(&group_ioctx, group_header_oid, - group_snap.id); + // undo + ldout(cct, 20) << "undoing group image remove: " << ret_code << dendl; + + if (image_index >= 0) { + auto mode = static_cast( + mirror_info.mirror_image_mode); + r = image_enable(image_ctxs[image_index], group_snap_id, mode, false, + &snap_ids[image_index]); + if (r < 0) { + lderr(cct) << "failed to enable mirroring on image: " + << image_ctxs[image_index]->name << cpp_strerror(r) << dendl; + } + } + remove_interim_snapshots(group_ioctx, group_header_oid, &image_ctxs, &group_snap); + } else { + group_snap.state = cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE; + r = cls_client::group_snap_set(&group_ioctx, group_header_oid, group_snap); if (r < 0) { - lderr(cct) << "failed to remove group snapshot metadata: " + lderr(cct) << "failed to update group snapshot metadata: " << cpp_strerror(r) << dendl; } - - // TODO: try to remove the created image snapshots - return ret_code; } - group_snap.state = cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE; - r = cls_client::group_snap_set(&group_ioctx, group_header_oid, group_snap); - if (r < 0) { - lderr(cct) << "failed to update group snapshot metadata: " - << cpp_strerror(r) << dendl; - return r; + if (!quiesce_requests.empty()) { + util::notify_unquiesce(image_ctxs, quiesce_requests); } - r = MirroringWatcher::notify_group_updated( - group_ioctx, cls::rbd::MIRROR_GROUP_STATE_ENABLED, group_id, - mirror_info.global_group_id, image_count); - if (r < 0) { - lderr(cct) << "failed to notify mirroring group_id=" << group_id - << " updated: " << cpp_strerror(r) << dendl; - // not fatal + close_images(&image_ctxs); + + if (!ret_code) { + r = MirroringWatcher::notify_group_updated( + group_ioctx, cls::rbd::MIRROR_GROUP_STATE_ENABLED, group_id, + mirror_info.global_group_id, image_count - 1); + if (r < 0) { + lderr(cct) << "failed to notify mirroring group_id=" << group_id + << " updated: " << cpp_strerror(r) << dendl; + // not fatal + } } - return 0; + return ret_code; } template diff --git a/src/librbd/api/Mirror.h b/src/librbd/api/Mirror.h index 66fcb2b010395..de8be202746c3 100644 --- a/src/librbd/api/Mirror.h +++ b/src/librbd/api/Mirror.h @@ -141,20 +141,24 @@ struct Mirror { static int group_list(IoCtx &io_ctx, std::vector *names); static int group_enable(IoCtx &group_ioctx, const char *group_name, - mirror_image_mode_t group_image_mode); + mirror_image_mode_t group_image_mode, + uint32_t flags); static int group_disable(IoCtx &group_ioctx, const char *group_name, bool force); static int group_promote(IoCtx &group_ioctx, const char *group_name, - bool force); - static int group_demote(IoCtx &group_ioctx, const char *group_name); + uint32_t flags, bool force); + static int group_demote(IoCtx &group_ioctx, const char *group_name, + uint32_t flags); static int group_resync(IoCtx &group_ioctx, const char *group_name); static int group_snapshot_create(IoCtx& group_ioctx, const char *group_name, uint32_t flags, std::string *snap_id); static int group_image_add(IoCtx &group_ioctx, const std::string &group_id, - IoCtx &image_ioctx, const std::string &image_id); + IoCtx &image_ioctx, const std::string &image_id, + uint32_t flags); static int group_image_remove(IoCtx &group_ioctx, const std::string &group_id, - IoCtx &image_ioctx, const std::string &image_id); + IoCtx &image_ioctx, const std::string &image_id, + uint32_t flags); static int group_status_list(librados::IoCtx& io_ctx, const std::string &start_id, size_t max, diff --git a/src/librbd/librbd.cc b/src/librbd/librbd.cc index 57b23410df56c..300e2dd6425a1 100644 --- a/src/librbd/librbd.cc +++ b/src/librbd/librbd.cc @@ -1459,7 +1459,8 @@ namespace librbd { } int RBD::group_image_add(IoCtx& group_ioctx, const char *group_name, - IoCtx& image_ioctx, const char *image_name) + IoCtx& image_ioctx, const char *image_name, + uint32_t flags) { TracepointProvider::initialize(get_cct(group_ioctx)); tracepoint(librbd, group_image_add_enter, @@ -1468,13 +1469,15 @@ namespace librbd { image_ioctx.get_pool_name().c_str(), image_ioctx.get_id(), image_name); int r = librbd::api::Group<>::image_add(group_ioctx, group_name, - image_ioctx, image_name); + image_ioctx, image_name, + flags); tracepoint(librbd, group_image_add_exit, r); return r; } int RBD::group_image_remove(IoCtx& group_ioctx, const char *group_name, - IoCtx& image_ioctx, const char *image_name) + IoCtx& image_ioctx, const char *image_name, + uint32_t flags) { TracepointProvider::initialize(get_cct(group_ioctx)); tracepoint(librbd, group_image_remove_enter, @@ -1483,13 +1486,15 @@ namespace librbd { image_ioctx.get_pool_name().c_str(), image_ioctx.get_id(), image_name); int r = librbd::api::Group<>::image_remove(group_ioctx, group_name, - image_ioctx, image_name); + image_ioctx, image_name, + flags); tracepoint(librbd, group_image_remove_exit, r); return r; } int RBD::group_image_remove_by_id(IoCtx& group_ioctx, const char *group_name, - IoCtx& image_ioctx, const char *image_id) + IoCtx& image_ioctx, const char *image_id, + uint32_t flags) { TracepointProvider::initialize(get_cct(group_ioctx)); tracepoint(librbd, group_image_remove_by_id_enter, @@ -1498,7 +1503,8 @@ namespace librbd { image_ioctx.get_pool_name().c_str(), image_ioctx.get_id(), image_id); int r = librbd::api::Group<>::image_remove_by_id(group_ioctx, group_name, - image_ioctx, image_id); + image_ioctx, image_id, + flags); tracepoint(librbd, group_image_remove_by_id_exit, r); return r; } @@ -1649,9 +1655,10 @@ namespace librbd { } int RBD::mirror_group_enable(IoCtx& group_ioctx, const char *group_name, - mirror_image_mode_t mirror_image_mode) { + mirror_image_mode_t mirror_image_mode, + uint32_t flags) { return librbd::api::Mirror<>::group_enable(group_ioctx, group_name, - mirror_image_mode); + mirror_image_mode, flags); } int RBD::mirror_group_disable(IoCtx& group_ioctx, const char *group_name, @@ -1660,12 +1667,14 @@ namespace librbd { } int RBD::mirror_group_promote(IoCtx& group_ioctx, const char *group_name, - bool force) { - return librbd::api::Mirror<>::group_promote(group_ioctx, group_name, force); + uint32_t flags, bool force) { + return librbd::api::Mirror<>::group_promote(group_ioctx, group_name, + flags, force); } - int RBD::mirror_group_demote(IoCtx& group_ioctx, const char *group_name) { - return librbd::api::Mirror<>::group_demote(group_ioctx, group_name); + int RBD::mirror_group_demote(IoCtx& group_ioctx, const char *group_name, + uint32_t flags) { + return librbd::api::Mirror<>::group_demote(group_ioctx, group_name, flags); } int RBD::mirror_group_resync(IoCtx& group_ioctx, const char *group_name) { @@ -7457,7 +7466,8 @@ extern "C" int rbd_group_get_id(rados_ioctx_t p, extern "C" int rbd_group_image_add(rados_ioctx_t group_p, const char *group_name, rados_ioctx_t image_p, - const char *image_name) + const char *image_name, + uint32_t flags) { librados::IoCtx group_ioctx; librados::IoCtx image_ioctx; @@ -7471,7 +7481,7 @@ extern "C" int rbd_group_image_add(rados_ioctx_t group_p, image_ioctx.get_id(), image_name); int r = librbd::api::Group<>::image_add(group_ioctx, group_name, image_ioctx, - image_name); + image_name, flags); tracepoint(librbd, group_image_add_exit, r); return r; @@ -7480,7 +7490,8 @@ extern "C" int rbd_group_image_add(rados_ioctx_t group_p, extern "C" int rbd_group_image_remove(rados_ioctx_t group_p, const char *group_name, rados_ioctx_t image_p, - const char *image_name) + const char *image_name, + uint32_t flags) { librados::IoCtx group_ioctx; librados::IoCtx image_ioctx; @@ -7494,7 +7505,7 @@ extern "C" int rbd_group_image_remove(rados_ioctx_t group_p, image_ioctx.get_id(), image_name); int r = librbd::api::Group<>::image_remove(group_ioctx, group_name, - image_ioctx, image_name); + image_ioctx, image_name, flags); tracepoint(librbd, group_image_remove_exit, r); return r; @@ -7503,7 +7514,8 @@ extern "C" int rbd_group_image_remove(rados_ioctx_t group_p, extern "C" int rbd_group_image_remove_by_id(rados_ioctx_t group_p, const char *group_name, rados_ioctx_t image_p, - const char *image_id) + const char *image_id, + uint32_t flags) { librados::IoCtx group_ioctx; librados::IoCtx image_ioctx; @@ -7519,7 +7531,8 @@ extern "C" int rbd_group_image_remove_by_id(rados_ioctx_t group_p, image_ioctx.get_id(), image_id); int r = librbd::api::Group<>::image_remove_by_id(group_ioctx, group_name, - image_ioctx, image_id); + image_ioctx, image_id, + flags); tracepoint(librbd, group_image_remove_by_id_exit, r); return r; @@ -7887,13 +7900,14 @@ extern "C" int rbd_mirror_group_list(rados_ioctx_t p, char *names, extern "C" int rbd_mirror_group_enable(rados_ioctx_t group_p, const char *group_name, - rbd_mirror_image_mode_t mirror_image_mode) + rbd_mirror_image_mode_t mirror_image_mode, + uint32_t flags) { librados::IoCtx group_ioctx; librados::IoCtx::from_rados_ioctx_t(group_p, group_ioctx); return librbd::api::Mirror<>::group_enable(group_ioctx, group_name, - mirror_image_mode); + mirror_image_mode, flags); } extern "C" int rbd_mirror_group_disable(rados_ioctx_t group_p, @@ -7906,21 +7920,25 @@ extern "C" int rbd_mirror_group_disable(rados_ioctx_t group_p, } extern "C" int rbd_mirror_group_promote(rados_ioctx_t group_p, - const char *group_name, bool force) + const char *group_name, + uint32_t flags, + bool force) { librados::IoCtx group_ioctx; librados::IoCtx::from_rados_ioctx_t(group_p, group_ioctx); - return librbd::api::Mirror<>::group_promote(group_ioctx, group_name, force); + return librbd::api::Mirror<>::group_promote(group_ioctx, group_name, + flags, force); } extern "C" int rbd_mirror_group_demote(rados_ioctx_t group_p, - const char *group_name) + const char *group_name, + uint32_t flags) { librados::IoCtx group_ioctx; librados::IoCtx::from_rados_ioctx_t(group_p, group_ioctx); - return librbd::api::Mirror<>::group_demote(group_ioctx, group_name); + return librbd::api::Mirror<>::group_demote(group_ioctx, group_name, flags); } extern "C" int rbd_mirror_group_resync(rados_ioctx_t group_p, diff --git a/src/pybind/rbd/c_rbd.pxd b/src/pybind/rbd/c_rbd.pxd index a5a61087c9e2e..80bc1f0e506a0 100644 --- a/src/pybind/rbd/c_rbd.pxd +++ b/src/pybind/rbd/c_rbd.pxd @@ -707,9 +707,11 @@ cdef extern from "rbd/librbd.h" nogil: void rbd_group_info_cleanup(rbd_group_info_t *group_info, size_t group_info_size) int rbd_group_image_add(rados_ioctx_t group_p, const char *group_name, - rados_ioctx_t image_p, const char *image_name) + rados_ioctx_t image_p, const char *image_name, + uint32_t flags) int rbd_group_image_remove(rados_ioctx_t group_p, const char *group_name, - rados_ioctx_t image_p, const char *image_name) + rados_ioctx_t image_p, const char *image_name, + uint32_t flags) int rbd_group_image_list(rados_ioctx_t group_p, const char *group_name, diff --git a/src/pybind/rbd/mock_rbd.pxi b/src/pybind/rbd/mock_rbd.pxi index 2f2871b738db4..6bf9b67e0ff96 100644 --- a/src/pybind/rbd/mock_rbd.pxi +++ b/src/pybind/rbd/mock_rbd.pxi @@ -894,10 +894,12 @@ cdef nogil: size_t group_info_size): pass int rbd_group_image_add(rados_ioctx_t group_p, const char *group_name, - rados_ioctx_t image_p, const char *image_name): + rados_ioctx_t image_p, const char *image_name, + uint32_t flags): pass int rbd_group_image_remove(rados_ioctx_t group_p, const char *group_name, - rados_ioctx_t image_p, const char *image_name): + rados_ioctx_t image_p, const char *image_name, + uint32_t flags): pass int rbd_group_image_list(rados_ioctx_t group_p, const char *group_name, diff --git a/src/pybind/rbd/rbd.pyx b/src/pybind/rbd/rbd.pyx index 2185f62e2f179..127f29b9b71b5 100644 --- a/src/pybind/rbd/rbd.pyx +++ b/src/pybind/rbd/rbd.pyx @@ -2740,7 +2740,7 @@ cdef class Group(object): finally: free(id) - def add_image(self, image_ioctx, image_name): + def add_image(self, image_ioctx, image_name, flags=0): """ Add an image to a group. @@ -2748,6 +2748,8 @@ cdef class Group(object): :type ioctx: :class:`rados.Ioctx` :param name: the name of the image to add :type name: str + :param flags: quiesce hook flags + :type flags: int :raises: :class:`ObjectNotFound` :raises: :class:`ObjectExists` @@ -2758,12 +2760,14 @@ cdef class Group(object): cdef: rados_ioctx_t _image_ioctx = convert_ioctx(image_ioctx) char *_image_name = image_name + uint32_t _flags = flags with nogil: - ret = rbd_group_image_add(self._ioctx, self._name, _image_ioctx, _image_name) + ret = rbd_group_image_add(self._ioctx, self._name, _image_ioctx, + _image_name, _flags) if ret != 0: raise make_ex(ret, 'error adding image to group', group_errno_to_exception) - def remove_image(self, image_ioctx, image_name): + def remove_image(self, image_ioctx, image_name, flags=0): """ Remove an image from a group. @@ -2771,6 +2775,8 @@ cdef class Group(object): :type ioctx: :class:`rados.Ioctx` :param name: the name of the image to remove :type name: str + :param flags: quiesce hook flags + :type flags: int :raises: :class:`ObjectNotFound` :raises: :class:`InvalidArgument` @@ -2780,8 +2786,10 @@ cdef class Group(object): cdef: rados_ioctx_t _image_ioctx = convert_ioctx(image_ioctx) char *_image_name = image_name + uint32_t _flags = flags with nogil: - ret = rbd_group_image_remove(self._ioctx, self._name, _image_ioctx, _image_name) + ret = rbd_group_image_remove(self._ioctx, self._name, + _image_ioctx, _image_name, _flags) if ret != 0: raise make_ex(ret, 'error removing image from group', group_errno_to_exception) diff --git a/src/test/librbd/test_Groups.cc b/src/test/librbd/test_Groups.cc index 91fa09783312b..950664589d143 100644 --- a/src/test/librbd/test_Groups.cc +++ b/src/test/librbd/test_Groups.cc @@ -142,7 +142,7 @@ TEST_F(TestGroup, add_image) rbd_group_info_cleanup(&group_info, sizeof(group_info)); ASSERT_EQ(0, rbd_group_image_add(ioctx, group_name, ioctx, - m_image_name.c_str())); + m_image_name.c_str(), 0)); ASSERT_EQ(-ERANGE, rbd_get_group(image, &group_info, 0)); ASSERT_EQ(0, rbd_get_group(image, &group_info, sizeof(group_info))); @@ -175,7 +175,7 @@ TEST_F(TestGroup, add_image) sizeof(rbd_group_image_info_t), num_images)); ASSERT_EQ(0, rbd_group_image_remove(ioctx, group_name, ioctx, - m_image_name.c_str())); + m_image_name.c_str(), 0)); ASSERT_EQ(0, rbd_get_features(image, &features)); ASSERT_TRUE((features & RBD_FEATURE_OPERATIONS) == 0ULL); @@ -218,7 +218,7 @@ TEST_F(TestGroup, add_imagePP) ASSERT_EQ(RBD_GROUP_INVALID_POOL, group_info.pool); ASSERT_EQ(0, rbd.group_image_add(ioctx, group_name, ioctx, - m_image_name.c_str())); + m_image_name.c_str(), 0)); ASSERT_EQ(-ERANGE, image.get_group(&group_info, 0)); ASSERT_EQ(0, image.get_group(&group_info, sizeof(group_info))); @@ -240,7 +240,7 @@ TEST_F(TestGroup, add_imagePP) ASSERT_EQ(ioctx.get_id(), images[0].pool); ASSERT_EQ(0, rbd.group_image_remove(ioctx, group_name, ioctx, - m_image_name.c_str())); + m_image_name.c_str(), 0)); ASSERT_EQ(0, image.features(&features)); ASSERT_TRUE((features & RBD_FEATURE_OPERATIONS) == 0ULL); @@ -286,7 +286,7 @@ TEST_F(TestGroup, add_snapshot) ASSERT_EQ(0, rbd_group_create(ioctx, group_name)); ASSERT_EQ(0, rbd_group_image_add(ioctx, group_name, ioctx, - m_image_name.c_str())); + m_image_name.c_str(), 0)); struct Watcher { static void quiesce_cb(void *arg) { @@ -422,7 +422,7 @@ TEST_F(TestGroup, add_snapshotPP) ASSERT_EQ(0, rbd.group_create(ioctx, group_name)); ASSERT_EQ(0, rbd.group_image_add(ioctx, group_name, ioctx, - m_image_name.c_str())); + m_image_name.c_str(), 0)); librbd::Image image; ASSERT_EQ(0, rbd.open(ioctx, image, m_image_name.c_str(), NULL)); @@ -508,6 +508,8 @@ TEST_F(TestGroup, snap_get_info) const char *gp_name = "gp_snapgetinfo"; ASSERT_EQ(0, rbd_group_create(ioctx2, gp_name)); + ASSERT_EQ(0, rbd_group_image_add(ioctx2, gp_name, ioctx, + m_image_name.c_str(), 0)); const char *gp_snap_name = "snap_snapshot"; ASSERT_EQ(0, rbd_group_snap_create(ioctx2, gp_name, gp_snap_name)); @@ -530,7 +532,7 @@ TEST_F(TestGroup, snap_get_info) ASSERT_EQ(0, rbd_group_snap_remove(ioctx2, gp_name, gp_snap_name)); ASSERT_EQ(0, rbd_group_image_add(ioctx2, gp_name, ioctx, - m_image_name.c_str())); + m_image_name.c_str(), 0)); ASSERT_EQ(0, rbd_group_snap_create(ioctx2, gp_name, gp_snap_name)); ASSERT_EQ(0, rbd_group_snap_get_info(ioctx2, gp_name, gp_snap_name, @@ -562,6 +564,8 @@ TEST_F(TestGroup, snap_get_infoPP) const char *gp_name = "gp_snapgetinfoPP"; ASSERT_EQ(0, m_rbd.group_create(ioctx2, gp_name)); + ASSERT_EQ(0, m_rbd.group_image_add(ioctx2, gp_name, m_ioctx, + m_image_name.c_str(), 0)); const char *gp_snap_name = "snap_snapshot"; ASSERT_EQ(0, m_rbd.group_snap_create(ioctx2, gp_name, gp_snap_name)); @@ -583,7 +587,7 @@ TEST_F(TestGroup, snap_get_infoPP) ASSERT_EQ(0, m_rbd.group_snap_remove(ioctx2, gp_name, gp_snap_name)); ASSERT_EQ(0, m_rbd.group_image_add(ioctx2, gp_name, m_ioctx, - m_image_name.c_str())); + m_image_name.c_str(), 0)); ASSERT_EQ(0, m_rbd.group_snap_create(ioctx2, gp_name, gp_snap_name)); ASSERT_EQ(0, m_rbd.group_snap_get_info(ioctx2, gp_name, gp_snap_name, @@ -636,15 +640,15 @@ TEST_F(TestGroup, snap_list2) ASSERT_EQ(0, rbd_group_snap_create(ioctx, gp_name, gp_snap_names[0])); ASSERT_EQ(0, rbd_group_image_add(ioctx, gp_name, ioctx, - m_image_name.c_str())); + m_image_name.c_str(), 0)); ASSERT_EQ(0, rbd_group_snap_create(ioctx, gp_name, gp_snap_names[1])); ASSERT_EQ(0, rbd_group_image_add(ioctx, gp_name, ioctx2, - image_name2.c_str())); + image_name2.c_str(), 0)); ASSERT_EQ(0, rbd_group_snap_create(ioctx, gp_name, gp_snap_names[2])); ASSERT_EQ(0, rbd_group_image_remove(ioctx, gp_name, ioctx, - m_image_name.c_str())); + m_image_name.c_str(), 0)); ASSERT_EQ(0, rbd_group_snap_create(ioctx, gp_name, gp_snap_names[3])); num_snaps = 3U; @@ -728,15 +732,15 @@ TEST_F(TestGroup, snap_list2PP) ASSERT_EQ(0, m_rbd.group_snap_create(m_ioctx, gp_name, gp_snap_names[0])); ASSERT_EQ(0, m_rbd.group_image_add(m_ioctx, gp_name, m_ioctx, - m_image_name.c_str())); + m_image_name.c_str(), 0)); ASSERT_EQ(0, m_rbd.group_snap_create(m_ioctx, gp_name, gp_snap_names[1])); ASSERT_EQ(0, m_rbd.group_image_add(m_ioctx, gp_name, ioctx2, - image_name2.c_str())); + image_name2.c_str(), 0)); ASSERT_EQ(0, m_rbd.group_snap_create(m_ioctx, gp_name, gp_snap_names[2])); - ASSERT_EQ(0, m_rbd.group_image_remove(m_ioctx, gp_name, - m_ioctx, m_image_name.c_str())); + ASSERT_EQ(0, m_rbd.group_image_remove(m_ioctx, gp_name, m_ioctx, + m_image_name.c_str(), 0)); ASSERT_EQ(0, m_rbd.group_snap_create(m_ioctx, gp_name, gp_snap_names[3])); ASSERT_EQ(0, m_rbd.group_snap_list2(m_ioctx, gp_name, &gp_snaps)); @@ -893,7 +897,7 @@ TEST_F(TestGroup, mirrorPP) const char *group_name = "snap_group"; ASSERT_EQ(0, m_rbd.group_create(m_ioctx, group_name)); ASSERT_EQ(0, m_rbd.group_image_add(m_ioctx, group_name, m_ioctx, - m_image_name.c_str())); + m_image_name.c_str(), 0)); std::vector group_snaps; ASSERT_EQ(0, m_rbd.group_snap_list(m_ioctx, group_name, &group_snaps, @@ -909,7 +913,7 @@ TEST_F(TestGroup, mirrorPP) ASSERT_EQ(0U, group_snaps.size()); ASSERT_EQ(0, m_rbd.mirror_group_enable(m_ioctx, group_name, - RBD_MIRROR_IMAGE_MODE_SNAPSHOT)); + RBD_MIRROR_IMAGE_MODE_SNAPSHOT, 0)); librbd::Image image; ASSERT_EQ(0, m_rbd.open(m_ioctx, image, m_image_name.c_str())); @@ -938,7 +942,7 @@ TEST_F(TestGroup, mirrorPP) sizeof(librbd::group_snap_info_t))); ASSERT_EQ(2U, group_snaps.size()); - ASSERT_EQ(0, m_rbd.mirror_group_demote(m_ioctx, group_name)); + ASSERT_EQ(0, m_rbd.mirror_group_demote(m_ioctx, group_name, 0)); librbd::mirror_image_info_t mirror_info; ASSERT_EQ(0, image.mirror_image_get_info(&mirror_info, sizeof(mirror_info))); @@ -946,7 +950,7 @@ TEST_F(TestGroup, mirrorPP) ASSERT_EQ(0, m_rbd.mirror_group_resync(m_ioctx, group_name)); - ASSERT_EQ(0, m_rbd.mirror_group_promote(m_ioctx, group_name, false)); + ASSERT_EQ(0, m_rbd.mirror_group_promote(m_ioctx, group_name, 0, false)); ASSERT_EQ(0, image.mirror_image_get_info(&mirror_info, sizeof(mirror_info))); ASSERT_TRUE(mirror_info.primary); diff --git a/src/test/librbd/test_Migration.cc b/src/test/librbd/test_Migration.cc index e094d2e7e5265..b370923e1062c 100644 --- a/src/test/librbd/test_Migration.cc +++ b/src/test/librbd/test_Migration.cc @@ -743,7 +743,7 @@ TEST_F(TestMigration, Group) ASSERT_EQ(0, librbd::api::Group<>::create(m_ioctx, "123")); ASSERT_EQ(0, librbd::api::Group<>::image_add(m_ioctx, "123", m_ioctx, - m_image_name.c_str())); + m_image_name.c_str(), 0)); librbd::group_info_t info; ASSERT_EQ(0, librbd::api::Group<>::image_get_group(m_ictx, &info)); @@ -755,7 +755,7 @@ TEST_F(TestMigration, Group) ASSERT_EQ(info.name, "123"); ASSERT_EQ(0, librbd::api::Group<>::image_remove(m_ioctx, "123", m_ioctx, - name.c_str())); + name.c_str(), 0)); ASSERT_EQ(0, librbd::api::Group<>::remove(m_ioctx, "123")); } @@ -765,7 +765,7 @@ TEST_F(TestMigration, GroupAbort) ASSERT_EQ(0, librbd::api::Group<>::create(m_ioctx, "123")); ASSERT_EQ(0, librbd::api::Group<>::image_add(m_ioctx, "123", m_ioctx, - m_image_name.c_str())); + m_image_name.c_str(), 0)); librbd::group_info_t info; ASSERT_EQ(0, librbd::api::Group<>::image_get_group(m_ictx, &info)); @@ -783,7 +783,7 @@ TEST_F(TestMigration, GroupAbort) ASSERT_EQ(info.name, "123"); ASSERT_EQ(0, librbd::api::Group<>::image_remove(m_ioctx, "123", m_ioctx, - m_image_name.c_str())); + m_image_name.c_str(), 0)); ASSERT_EQ(0, librbd::api::Group<>::remove(m_ioctx, "123")); } diff --git a/src/tools/rbd/action/Group.cc b/src/tools/rbd/action/Group.cc index 100bdc194968d..de38d3faf954f 100644 --- a/src/tools/rbd/action/Group.cc +++ b/src/tools/rbd/action/Group.cc @@ -363,6 +363,12 @@ int execute_add(const po::variables_map &vm, return -EINVAL; } + uint32_t flags; + r = utils::get_snap_create_flags(vm, &flags); + if (r < 0) { + return r; + } + librados::Rados rados; librados::IoCtx cg_io_ctx; r = utils::init(group_pool_name, group_namespace_name, &rados, &cg_io_ctx); @@ -378,7 +384,7 @@ int execute_add(const po::variables_map &vm, librbd::RBD rbd; r = rbd.group_image_add(cg_io_ctx, group_name.c_str(), - image_io_ctx, image_name.c_str()); + image_io_ctx, image_name.c_str(), flags); if (r < 0) { std::cerr << "rbd: add image error: " << cpp_strerror(r) << std::endl; return r; @@ -422,6 +428,12 @@ int execute_remove_image(const po::variables_map &vm, return r; } + uint32_t flags; + r = utils::get_snap_create_flags(vm, &flags); + if (r < 0) { + return r; + } + if (group_namespace_name != image_namespace_name) { std::cerr << "rbd: group and image namespace must match." << std::endl; return -EINVAL; @@ -447,10 +459,10 @@ int execute_remove_image(const po::variables_map &vm, librbd::RBD rbd; if (image_id.empty()) { r = rbd.group_image_remove(cg_io_ctx, group_name.c_str(), - image_io_ctx, image_name.c_str()); + image_io_ctx, image_name.c_str(), flags); } else { r = rbd.group_image_remove_by_id(cg_io_ctx, group_name.c_str(), - image_io_ctx, image_id.c_str()); + image_io_ctx, image_id.c_str(), flags); } if (r < 0) { std::cerr << "rbd: remove image error: " << cpp_strerror(r) << std::endl; @@ -964,6 +976,7 @@ void get_add_arguments(po::options_description *positional, add_prefixed_pool_option(options, "image"); add_prefixed_namespace_option(options, "image"); at::add_image_option(options, at::ARGUMENT_MODIFIER_NONE); + at::add_snap_create_options(options); } void get_remove_image_arguments(po::options_description *positional, @@ -987,6 +1000,7 @@ void get_remove_image_arguments(po::options_description *positional, at::add_image_option(options, at::ARGUMENT_MODIFIER_NONE); at::add_image_id_option(options); + at::add_snap_create_options(options); } void get_list_images_arguments(po::options_description *positional, diff --git a/src/tools/rbd/action/MirrorGroup.cc b/src/tools/rbd/action/MirrorGroup.cc index 10a7c456c749f..7a14b8b7c95fb 100644 --- a/src/tools/rbd/action/MirrorGroup.cc +++ b/src/tools/rbd/action/MirrorGroup.cc @@ -90,6 +90,7 @@ int validate_mirroring_enabled(librados::IoCtx io_ctx, void get_arguments(po::options_description *positional, po::options_description *options) { add_group_spec_options(positional, options); + at::add_snap_create_options(options); } void get_arguments_enable(po::options_description *positional, @@ -97,6 +98,7 @@ void get_arguments_enable(po::options_description *positional, add_group_spec_options(positional, options); positional->add_options() ("mode", "mirror group mode [default: snapshot]"); + at::add_snap_create_options(options); } void get_arguments_disable(po::options_description *positional, @@ -121,6 +123,14 @@ int execute_enable_disable(const po::variables_map &vm, bool enable, return r; } + uint32_t flags; + if (enable) { + r = utils::get_snap_create_flags(vm, &flags); + if (r < 0) { + return r; + } + } + librados::Rados rados; librados::IoCtx io_ctx; @@ -145,7 +155,7 @@ int execute_enable_disable(const po::variables_map &vm, bool enable, std::cerr << "rbd: invalid mode name: " << mode_arg << std::endl; return -EINVAL; } - r = rbd.mirror_group_enable(io_ctx, group_name.c_str(), mode); + r = rbd.mirror_group_enable(io_ctx, group_name.c_str(), mode, flags); } else { r = rbd.mirror_group_disable(io_ctx, group_name.c_str(), force); } @@ -173,6 +183,7 @@ void get_arguments_promote(po::options_description *positional, options->add_options() ("force", po::bool_switch(), "promote even if not cleanly demoted by remote cluster"); add_group_spec_options(positional, options); + at::add_snap_create_options(options); } int execute_promote(const po::variables_map &vm, @@ -190,6 +201,12 @@ int execute_promote(const po::variables_map &vm, return r; } + uint32_t flags; + r = utils::get_snap_create_flags(vm, &flags); + if (r < 0) { + return r; + } + bool force = vm["force"].as(); librados::Rados rados; @@ -207,7 +224,7 @@ int execute_promote(const po::variables_map &vm, librbd::RBD rbd; - r = rbd.mirror_group_promote(io_ctx, group_name.c_str(), force); + r = rbd.mirror_group_promote(io_ctx, group_name.c_str(), flags, force); if (r < 0) { std::cerr << "rbd: error promoting group to primary" << std::endl; return r; @@ -232,6 +249,12 @@ int execute_demote(const po::variables_map &vm, return r; } + uint32_t flags; + r = utils::get_snap_create_flags(vm, &flags); + if (r < 0) { + return r; + } + librados::Rados rados; librados::IoCtx io_ctx; @@ -247,7 +270,7 @@ int execute_demote(const po::variables_map &vm, librbd::RBD rbd; - r = rbd.mirror_group_demote(io_ctx, group_name.c_str()); + r = rbd.mirror_group_demote(io_ctx, group_name.c_str(), flags); if (r < 0) { std::cerr << "rbd: error demoting group to non-primary" << std::endl; return r; diff --git a/src/tools/rbd_mirror/GroupReplayer.cc b/src/tools/rbd_mirror/GroupReplayer.cc index a8857e7e3f201..a68a5a96ca0b3 100644 --- a/src/tools/rbd_mirror/GroupReplayer.cc +++ b/src/tools/rbd_mirror/GroupReplayer.cc @@ -981,7 +981,9 @@ void GroupReplayer::handle_get_remote_group_snapshot( << dendl; r = -EAGAIN; } else { - m_local_group_snaps[remote_group_snap_id].name = remote_group_snap.name; + m_local_group_snaps[remote_group_snap_id].name = m_bootstrap_request->prepare_non_primary_mirror_snap_name( + m_global_group_id, + m_local_group_snaps[remote_group_snap_id].id); } if (m_state == STATE_STOPPING) { -- 2.39.5