From 352c68f39ffcf493e0b7cba75b32b80437b9f40f Mon Sep 17 00:00:00 2001 From: Mykola Golub Date: Sat, 28 Nov 2020 10:15:26 +0000 Subject: [PATCH] librbd: add mirror group API Signed-off-by: Mykola Golub Signed-off-by: Prasanna Kumar Kalever --- src/include/rbd/librbd.h | 93 + src/include/rbd/librbd.hpp | 57 + src/librbd/Journal.cc | 5 +- src/librbd/Journal.h | 5 +- src/librbd/MirroringWatcher.cc | 11 +- src/librbd/MirroringWatcher.h | 7 +- src/librbd/api/Group.cc | 511 ++--- src/librbd/api/Group.h | 9 + src/librbd/api/Mirror.cc | 1781 ++++++++++++++++- src/librbd/api/Mirror.h | 59 + src/librbd/api/Utils.cc | 202 ++ src/librbd/api/Utils.h | 13 + src/librbd/librbd.cc | 517 ++++- src/librbd/mirror/DemoteRequest.cc | 3 +- src/librbd/mirror/DemoteRequest.h | 21 +- src/librbd/mirror/EnableRequest.cc | 11 +- src/librbd/mirror/EnableRequest.h | 32 +- src/librbd/mirror/ImageStateUpdateRequest.cc | 93 +- src/librbd/mirror/ImageStateUpdateRequest.h | 15 - src/librbd/mirror/PromoteRequest.cc | 3 +- src/librbd/mirror/PromoteRequest.h | 24 +- .../mirror/snapshot/CreatePrimaryRequest.cc | 10 +- .../mirror/snapshot/CreatePrimaryRequest.h | 31 +- src/librbd/mirror/snapshot/DemoteRequest.cc | 3 +- src/librbd/mirror/snapshot/DemoteRequest.h | 22 +- src/librbd/mirror/snapshot/PromoteRequest.cc | 3 +- src/librbd/mirror/snapshot/PromoteRequest.h | 22 +- src/librbd/mirroring_watcher/Types.cc | 5 +- src/librbd/mirroring_watcher/Types.h | 5 +- .../test_mock_CreatePrimaryRequest.cc | 57 +- .../snapshot/test_mock_PromoteRequest.cc | 14 +- src/test/librbd/test_Groups.cc | 74 + src/test/librbd/test_MirroringWatcher.cc | 5 +- src/test/rbd_mirror/test_mock_PoolWatcher.cc | 3 +- src/tools/rbd_mirror/PoolWatcher.cc | 10 +- src/tools/rbd_mirror/PoolWatcher.h | 2 +- 36 files changed, 3174 insertions(+), 564 deletions(-) diff --git a/src/include/rbd/librbd.h b/src/include/rbd/librbd.h index b347b03df9e28..f68e1c654b84c 100644 --- a/src/include/rbd/librbd.h +++ b/src/include/rbd/librbd.h @@ -68,6 +68,7 @@ extern "C" { #define RBD_FLAG_FAST_DIFF_INVALID (1<<1) #define RBD_MIRROR_IMAGE_STATUS_LOCAL_MIRROR_UUID "" +#define RBD_MIRROR_GROUP_STATUS_LOCAL_MIRROR_UUID "" typedef void *rbd_image_t; typedef void *rbd_image_options_t; @@ -233,6 +234,48 @@ typedef struct { rbd_mirror_image_site_status_t *site_statuses; } rbd_mirror_image_global_status_t; +typedef enum { + RBD_MIRROR_GROUP_DISABLING = 0, + RBD_MIRROR_GROUP_ENABLING = 1, + RBD_MIRROR_GROUP_ENABLED = 2, + RBD_MIRROR_GROUP_DISABLED = 3 +} rbd_mirror_group_state_t; + +typedef struct { + char *global_id; + rbd_mirror_image_mode_t mirror_image_mode; + rbd_mirror_group_state_t state; + bool primary; +} rbd_mirror_group_info_t; + +typedef enum { + MIRROR_GROUP_STATUS_STATE_UNKNOWN = 0, + MIRROR_GROUP_STATUS_STATE_ERROR = 1, + MIRROR_GROUP_STATUS_STATE_STARTING_REPLAY = 2, + MIRROR_GROUP_STATUS_STATE_REPLAYING = 3, + MIRROR_GROUP_STATUS_STATE_STOPPING_REPLAY = 4, + MIRROR_GROUP_STATUS_STATE_STOPPED = 5, +} rbd_mirror_group_status_state_t; + +typedef struct { + char *mirror_uuid; + rbd_mirror_group_status_state_t state; + char *description; + uint32_t mirror_image_count; + int64_t *mirror_image_pool_ids; + char **mirror_image_global_ids; + rbd_mirror_image_site_status_t *mirror_images; + time_t last_update; + bool up; +} rbd_mirror_group_site_status_t; + +typedef struct { + char *name; + rbd_mirror_group_info_t info; + uint32_t site_statuses_count; + rbd_mirror_group_site_status_t *site_statuses; +} rbd_mirror_group_global_status_t; + typedef enum { RBD_GROUP_IMAGE_STATE_ATTACHED, RBD_GROUP_IMAGE_STATE_INCOMPLETE @@ -706,6 +749,30 @@ CEPH_RBD_API void rbd_mirror_image_info_list_cleanup( char **image_ids, rbd_mirror_image_info_t *info_entries, size_t num_entries); +CEPH_RBD_API int rbd_mirror_group_global_status_list( + rados_ioctx_t io_ctx, const char *start_id, size_t max, char **group_ids, + rbd_mirror_group_global_status_t *groups, size_t *len); +CEPH_RBD_API void rbd_mirror_group_global_status_list_cleanup( + char **group_ids, rbd_mirror_group_global_status_t *groups, size_t len); +CEPH_RBD_API int rbd_mirror_group_status_summary( + rados_ioctx_t io_ctx, rbd_mirror_group_status_state_t *states, int *counts, + size_t *maxlen); +CEPH_RBD_API int rbd_mirror_group_instance_id_list(rados_ioctx_t io_ctx, + const char *start_id, + size_t max, char **group_ids, + char **instance_ids, + size_t *len); +CEPH_RBD_API void rbd_mirror_group_instance_id_list_cleanup(char **group_ids, + char **instance_ids, + size_t len); +CEPH_RBD_API int rbd_mirror_group_info_list( + rados_ioctx_t io_ctx, rbd_mirror_image_mode_t *mode_filter, + const char *start_id, size_t max, char **group_ids, + rbd_mirror_group_info_t *info_entries, size_t *num_entries); +CEPH_RBD_API void rbd_mirror_group_info_list_cleanup( + char **group_ids, rbd_mirror_group_info_t *info_entries, + size_t num_entries); + /* pool metadata */ CEPH_RBD_API int rbd_pool_metadata_get(rados_ioctx_t io_ctx, const char *key, char *value, size_t *val_len); @@ -1560,6 +1627,32 @@ CEPH_RBD_API int rbd_group_snap_rollback_with_progress(rados_ioctx_t group_p, librbd_progress_fn_t cb, void *cbdata); +// RBD group mirroring support functions +CEPH_RBD_API int rbd_mirror_group_list(rados_ioctx_t p, char *names, + size_t *size); +CEPH_RBD_API int rbd_mirror_group_enable(rados_ioctx_t p, const char *name, + rbd_mirror_image_mode_t mirror_image_mode); +CEPH_RBD_API int rbd_mirror_group_disable(rados_ioctx_t p, const char *name, + bool force); +CEPH_RBD_API int rbd_mirror_group_promote(rados_ioctx_t p, const char *name, + bool force); +CEPH_RBD_API int rbd_mirror_group_demote(rados_ioctx_t p, const char *name); +CEPH_RBD_API int rbd_mirror_group_resync(rados_ioctx_t p, const char *name); +CEPH_RBD_API int rbd_mirror_group_create_snapshot(rados_ioctx_t p, + const char *name, + uint32_t flags, + char **snap_id); +CEPH_RBD_API int rbd_mirror_group_get_info( + rados_ioctx_t p, const char *name, + rbd_mirror_group_info_t *mirror_group_info, size_t info_size); +CEPH_RBD_API void rbd_mirror_group_get_info_cleanup( + rbd_mirror_group_info_t *mirror_group_info); +CEPH_RBD_API int rbd_mirror_group_get_global_status( + rados_ioctx_t p, const char *name, + rbd_mirror_group_global_status_t *mirror_group_status, size_t status_size); +CEPH_RBD_API void rbd_mirror_group_global_status_cleanup( + rbd_mirror_group_global_status_t *mirror_group_status); + CEPH_RBD_API int rbd_namespace_create(rados_ioctx_t io, const char *namespace_name); CEPH_RBD_API int rbd_namespace_remove(rados_ioctx_t io, diff --git a/src/include/rbd/librbd.hpp b/src/include/rbd/librbd.hpp index 0ac315a76f488..6aea81e01e548 100644 --- a/src/include/rbd/librbd.hpp +++ b/src/include/rbd/librbd.hpp @@ -146,6 +146,33 @@ namespace librbd { std::vector site_statuses; } mirror_image_global_status_t; + typedef rbd_mirror_group_state_t mirror_group_state_t; + + typedef struct { + std::string global_id; + mirror_image_mode_t mirror_image_mode; + mirror_group_state_t state; + bool primary; + } mirror_group_info_t; + + typedef rbd_mirror_group_status_state_t mirror_group_status_state_t; + + typedef struct { + std::string mirror_uuid; + mirror_group_status_state_t state; + std::string description; + std::map, + mirror_image_site_status_t> mirror_images; + time_t last_update; + bool up; + } mirror_group_site_status_t; + + typedef struct { + std::string name; + mirror_group_info_t info; + std::vector site_statuses; + } mirror_group_global_status_t; + typedef rbd_group_image_state_t group_image_state_t; typedef struct { @@ -407,6 +434,17 @@ public: std::map> *entries); + int mirror_group_info_list(IoCtx& io_ctx, mirror_image_mode_t *mode_filter, + const std::string &start_id, size_t max, + std::map *entries); + int mirror_group_global_status_list( + IoCtx& io_ctx, const std::string &start_id, size_t max, + std::map *groups); + int mirror_group_status_summary(IoCtx& io_ctx, + std::map *states); + int mirror_group_instance_id_list(IoCtx& io_ctx, const std::string &start_id, + size_t max, std::map *sevice_ids); + /// mirror_peer_ commands are deprecated to mirror_peer_site_ equivalents int mirror_peer_add(IoCtx& io_ctx, std::string *uuid, const std::string &cluster_name, @@ -480,6 +518,25 @@ public: const char *snap_name, ProgressContext& pctx); + // RBD group mirroring support functions + int mirror_group_list(IoCtx& io_ctx, std::vector *names); + int mirror_group_enable(IoCtx& io_ctx, const char *group_name, + mirror_image_mode_t mirror_image_mode); + int mirror_group_disable(IoCtx& io_ctx, const char *group_name, bool force); + int mirror_group_promote(IoCtx& io_ctx, const char *group_name, bool force); + int mirror_group_demote(IoCtx& io_ctx, const char *group_name); + int mirror_group_resync(IoCtx& io_ctx, const char *group_name); + int mirror_group_create_snapshot(IoCtx& io_ctx, const char *group_name, + uint32_t flags, std::string *snap_id); + int mirror_group_get_info(IoCtx& io_ctx, const char *group_name, + mirror_group_info_t *mirror_group_info, + size_t info_size); + int mirror_group_get_status(IoCtx& io_ctx, const char *group_name, + mirror_group_global_status_t *mirror_group_status, + size_t status_size); + int mirror_group_get_instance_id(IoCtx& io_ctx, const char *group_name, + std::string *instance_id); + int namespace_create(IoCtx& ioctx, const char *namespace_name); int namespace_remove(IoCtx& ioctx, const char *namespace_name); int namespace_list(IoCtx& io_ctx, std::vector* namespace_names); diff --git a/src/librbd/Journal.cc b/src/librbd/Journal.cc index ab65007d9ad8b..fba65b7705565 100644 --- a/src/librbd/Journal.cc +++ b/src/librbd/Journal.cc @@ -451,7 +451,8 @@ void Journal::is_tag_owner(I *image_ctx, bool *owner, } template -void Journal::is_tag_owner(librados::IoCtx& io_ctx, std::string& image_id, +void Journal::is_tag_owner(librados::IoCtx& io_ctx, + const std::string& image_id, bool *is_tag_owner, asio::ContextWQ *op_work_queue, Context *on_finish) { @@ -466,7 +467,7 @@ void Journal::is_tag_owner(librados::IoCtx& io_ctx, std::string& image_id, } template -void Journal::get_tag_owner(IoCtx& io_ctx, std::string& image_id, +void Journal::get_tag_owner(IoCtx& io_ctx, const std::string& image_id, std::string *mirror_uuid, asio::ContextWQ *op_work_queue, Context *on_finish) { diff --git a/src/librbd/Journal.h b/src/librbd/Journal.h index 5327adac7192b..97ed1db0fcbd2 100644 --- a/src/librbd/Journal.h +++ b/src/librbd/Journal.h @@ -102,10 +102,11 @@ public: static void is_tag_owner(ImageCtxT *image_ctx, bool *is_tag_owner, Context *on_finish); - static void is_tag_owner(librados::IoCtx& io_ctx, std::string& image_id, + static void is_tag_owner(librados::IoCtx& io_ctx, const std::string& image_id, bool *is_tag_owner, asio::ContextWQ *op_work_queue, Context *on_finish); - static void get_tag_owner(librados::IoCtx& io_ctx, std::string& image_id, + static void get_tag_owner(librados::IoCtx& io_ctx, + const std::string& image_id, std::string *mirror_uuid, asio::ContextWQ *op_work_queue, Context *on_finish); static int request_resync(ImageCtxT *image_ctx); diff --git a/src/librbd/MirroringWatcher.cc b/src/librbd/MirroringWatcher.cc index 9abf29f105f24..d7ff11fa9fba0 100644 --- a/src/librbd/MirroringWatcher.cc +++ b/src/librbd/MirroringWatcher.cc @@ -95,10 +95,11 @@ void MirroringWatcher::notify_image_updated( template int MirroringWatcher::notify_group_updated( librados::IoCtx &io_ctx, cls::rbd::MirrorGroupState mirror_group_state, - const std::string &group_id, const std::string &global_group_id) { + const std::string &group_id, const std::string &global_group_id, + size_t image_count) { C_SaferCond ctx; notify_group_updated(io_ctx, mirror_group_state, group_id, global_group_id, - &ctx); + image_count, &ctx); return ctx.wait(); } @@ -106,7 +107,7 @@ template void MirroringWatcher::notify_group_updated( librados::IoCtx &io_ctx, cls::rbd::MirrorGroupState mirror_group_state, const std::string &group_id, const std::string &global_group_id, - Context *on_finish) { + size_t image_count, Context *on_finish) { CephContext *cct = reinterpret_cast(io_ctx.cct()); ldout(cct, 20) << "pool_name: " << io_ctx.get_pool_name() @@ -116,7 +117,7 @@ void MirroringWatcher::notify_group_updated( bufferlist bl; encode(NotifyMessage{GroupUpdatedPayload{ - mirror_group_state, group_id, global_group_id}}, bl); + mirror_group_state, group_id, global_group_id, image_count}}, bl); librados::AioCompletion *comp = create_rados_callback(on_finish); int r = io_ctx.aio_notify(RBD_MIRRORING, comp, bl, NOTIFY_TIMEOUT_MS, @@ -174,7 +175,7 @@ bool MirroringWatcher::handle_payload(const GroupUpdatedPayload &payload, CephContext *cct = this->m_cct; ldout(cct, 20) << "group state updated" << dendl; handle_group_updated(payload.mirror_group_state, payload.group_id, - payload.global_group_id); + payload.global_group_id, payload.image_count); return true; } diff --git a/src/librbd/MirroringWatcher.h b/src/librbd/MirroringWatcher.h index c4e8dbc2a25dd..b75fb37cb6141 100644 --- a/src/librbd/MirroringWatcher.h +++ b/src/librbd/MirroringWatcher.h @@ -45,11 +45,13 @@ public: static int notify_group_updated(librados::IoCtx &io_ctx, cls::rbd::MirrorGroupState mirror_group_state, const std::string &group_id, - const std::string &global_group_id); + const std::string &global_group_id, + size_t image_count); static void notify_group_updated(librados::IoCtx &io_ctx, cls::rbd::MirrorGroupState mirror_group_state, const std::string &group_id, const std::string &global_group_id, + size_t image_count, Context *on_finish); virtual void handle_mode_updated(cls::rbd::MirrorMode mirror_mode) = 0; @@ -58,7 +60,8 @@ public: const std::string &global_image_id) = 0; virtual void handle_group_updated(cls::rbd::MirrorGroupState state, const std::string &group_id, - const std::string &global_group_id) = 0; + const std::string &global_group_id, + size_t image_count) = 0; private: bool handle_payload(const mirroring_watcher::ModeUpdatedPayload &payload, diff --git a/src/librbd/api/Group.cc b/src/librbd/api/Group.cc index c63377e5a2586..f18cd5c6d1b92 100644 --- a/src/librbd/api/Group.cc +++ b/src/librbd/api/Group.cc @@ -4,13 +4,16 @@ #include "common/Cond.h" #include "common/errno.h" -#include "librbd/ExclusiveLock.h" #include "librbd/api/Group.h" +#include "librbd/ExclusiveLock.h" #include "librbd/ImageCtx.h" #include "librbd/ImageState.h" #include "librbd/ImageWatcher.h" +#include "librbd/MirroringWatcher.h" #include "librbd/Operations.h" #include "librbd/Utils.h" +#include "librbd/api/Mirror.h" +#include "librbd/api/Utils.h" #include "librbd/group/ListSnapshotsRequest.h" #include "librbd/internal.h" #include "librbd/io/AioCompletion.h" @@ -80,9 +83,9 @@ std::string calc_ind_image_snap_name(uint64_t pool_id, return ind_snap_name_stream.str(); } -int group_image_list(librados::IoCtx& group_ioctx, const char *group_name, - std::vector *image_ids) -{ +int group_image_list(librados::IoCtx& group_ioctx, + const std::string &group_name, + std::vector *images) { CephContext *cct = (CephContext *)group_ioctx.cct(); string group_id; @@ -90,198 +93,11 @@ int group_image_list(librados::IoCtx& group_ioctx, const char *group_name, int r = cls_client::dir_get_id(&group_ioctx, RBD_GROUP_DIRECTORY, group_name, &group_id); if (r < 0) { - lderr(cct) << "error reading group id object: " - << cpp_strerror(r) - << dendl; + lderr(cct) << "error getting the group id: " << cpp_strerror(r) << dendl; return r; } - string group_header_oid = util::group_header_name(group_id); - - ldout(cct, 20) << "listing images in group name " - << group_name << " group id " << group_header_oid << dendl; - image_ids->clear(); - - const int max_read = 1024; - cls::rbd::GroupImageSpec start_last; - do { - std::vector image_ids_page; - - r = cls_client::group_image_list(&group_ioctx, group_header_oid, - start_last, max_read, &image_ids_page); - - if (r < 0) { - lderr(cct) << "error reading image list from group: " - << cpp_strerror(-r) << dendl; - return r; - } - image_ids->insert(image_ids->end(), - image_ids_page.begin(), image_ids_page.end()); - - if (image_ids_page.size() > 0) - start_last = image_ids_page.rbegin()->spec; - - r = image_ids_page.size(); - } while (r == max_read); - - return 0; -} - -int group_image_remove(librados::IoCtx& group_ioctx, string group_id, - librados::IoCtx& image_ioctx, string image_id) -{ - CephContext *cct = (CephContext *)group_ioctx.cct(); - - string group_header_oid = util::group_header_name(group_id); - - string image_header_oid = util::header_name(image_id); - - ldout(cct, 20) << "removing image " << image_id - << " image id " << image_header_oid << dendl; - - cls::rbd::GroupSpec group_spec(group_id, group_ioctx.get_id()); - - cls::rbd::GroupImageStatus incomplete_st(image_id, image_ioctx.get_id(), - cls::rbd::GROUP_IMAGE_LINK_STATE_INCOMPLETE); - - cls::rbd::GroupImageSpec spec(image_id, image_ioctx.get_id()); - - int r = cls_client::group_image_set(&group_ioctx, group_header_oid, - incomplete_st); - - if (r < 0) { - lderr(cct) << "couldn't put image into removing state: " - << cpp_strerror(-r) << dendl; - return r; - } - - r = cls_client::image_group_remove(&image_ioctx, image_header_oid, - group_spec); - if ((r < 0) && (r != -ENOENT)) { - lderr(cct) << "couldn't remove group reference from image" - << cpp_strerror(-r) << dendl; - return r; - } else if (r >= 0) { - ImageWatcher<>::notify_header_update(image_ioctx, image_header_oid); - } - - r = cls_client::group_image_remove(&group_ioctx, group_header_oid, spec); - if (r < 0) { - lderr(cct) << "couldn't remove image from group" - << cpp_strerror(-r) << dendl; - return r; - } - - return 0; -} - -int group_snap_remove_by_record(librados::IoCtx& group_ioctx, - const cls::rbd::GroupSnapshot& group_snap, - const std::string& group_id, - const std::string& group_header_oid) { - - CephContext *cct = (CephContext *)group_ioctx.cct(); - std::vector ioctxs; - std::vector ictxs; - std::vector on_finishes; - int r, ret_code; - - cls::rbd::GroupImageSnapshotNamespace ne{group_ioctx.get_id(), group_id, - group_snap.id}; - - ldout(cct, 20) << "Removing snapshots" << dendl; - int snap_count = group_snap.snaps.size(); - - for (int i = 0; i < snap_count; ++i) { - librados::IoCtx image_io_ctx; - r = util::create_ioctx(group_ioctx, "image", group_snap.snaps[i].pool, {}, - &image_io_ctx); - if (r < 0) { - return r; - } - ioctxs.push_back(std::move(image_io_ctx)); - } - - for (int i = 0; i < snap_count; ++i) { - librbd::ImageCtx* image_ctx = new ImageCtx("", group_snap.snaps[i].image_id, - nullptr, ioctxs[i], false); - - C_SaferCond* on_finish = new C_SaferCond; - - image_ctx->state->open(0, on_finish); - - ictxs.push_back(image_ctx); - on_finishes.push_back(on_finish); - } - - ret_code = 0; - for (int i = 0; i < snap_count; ++i) { - r = on_finishes[i]->wait(); - delete on_finishes[i]; - if (r < 0) { - ictxs[i] = nullptr; - ret_code = r; - } - } - if (ret_code != 0) { - goto finish; - } - - ldout(cct, 20) << "Opened participating images. " << - "Deleting snapshots themselves." << dendl; - for (int i = 0; i < snap_count; ++i) { - ImageCtx *ictx = ictxs[i]; - on_finishes[i] = new C_SaferCond; - - std::string snap_name; - ictx->image_lock.lock_shared(); - snap_t snap_id = get_group_snap_id(ictx, ne); - r = ictx->get_snap_name(snap_id, &snap_name); - ictx->image_lock.unlock_shared(); - - if (r >= 0) { - ldout(cct, 20) << "removing individual snapshot from image " << ictx->name - << dendl; - ictx->operations->snap_remove(ne, snap_name, on_finishes[i]); - } else { - // We are ok to ignore missing image snapshots. The snapshot could have - // been inconsistent in the first place. - on_finishes[i]->complete(0); - } - } - - for (int i = 0; i < snap_count; ++i) { - r = on_finishes[i]->wait(); - delete on_finishes[i]; - if (r < 0 && r != -ENOENT) { - // if previous attempts to remove this snapshot failed then the image's - // snapshot may not exist - lderr(cct) << "Failed deleting image snapshot. Ret code: " << r << dendl; - ret_code = r; - } - } - - if (ret_code != 0) { - goto finish; - } - - ldout(cct, 20) << "Removed images snapshots removing snapshot record." - << dendl; - - r = cls_client::group_snap_remove(&group_ioctx, group_header_oid, - group_snap.id); - if (r < 0) { - ret_code = r; - goto finish; - } - -finish: - for (int i = 0; i < snap_count; ++i) { - if (ictxs[i] != nullptr) { - ictxs[i]->state->close(); - } - } - return ret_code; + return Group<>::group_image_list_by_id(group_ioctx, group_id, images); } int group_snap_rollback_by_record(librados::IoCtx& group_ioctx, @@ -302,8 +118,8 @@ int group_snap_rollback_by_record(librados::IoCtx& group_ioctx, for (int i = 0; i < snap_count; ++i) { librados::IoCtx image_io_ctx; - r = util::create_ioctx(group_ioctx, "image", group_snap.snaps[i].pool, {}, - &image_io_ctx); + r = librbd::util::create_ioctx(group_ioctx, "image", + group_snap.snaps[i].pool, {}, &image_io_ctx); if (r < 0) { return r; } @@ -375,7 +191,7 @@ int group_snap_rollback_by_record(librados::IoCtx& group_ioctx, std::shared_lock owner_locker{ictx->owner_lock}; std::string snap_name; ictx->image_lock.lock_shared(); - snap_t snap_id = get_group_snap_id(ictx, ne); + snap_t snap_id = util::get_group_snap_id(ictx, ne); r = ictx->get_snap_name(snap_id, &snap_name); ictx->image_lock.unlock_shared(); @@ -406,57 +222,6 @@ finish: return ret_code; } -template -void notify_unquiesce(std::vector &ictxs, - const std::vector &requests) { - if (requests.empty()) { - return; - } - - ceph_assert(requests.size() == ictxs.size()); - int image_count = ictxs.size(); - std::vector on_finishes(image_count); - - for (int i = 0; i < image_count; ++i) { - ImageCtx *ictx = ictxs[i]; - - ictx->image_watcher->notify_unquiesce(requests[i], &on_finishes[i]); - } - - for (int i = 0; i < image_count; ++i) { - on_finishes[i].wait(); - } -} - -template -int notify_quiesce(std::vector &ictxs, ProgressContext &prog_ctx, - std::vector *requests) { - int image_count = ictxs.size(); - std::vector on_finishes(image_count); - - requests->resize(image_count); - for (int i = 0; i < image_count; ++i) { - auto ictx = ictxs[i]; - - ictx->image_watcher->notify_quiesce(&(*requests)[i], prog_ctx, - &on_finishes[i]); - } - - int ret_code = 0; - for (int i = 0; i < image_count; ++i) { - int r = on_finishes[i].wait(); - if (r < 0) { - ret_code = r; - } - } - - if (ret_code != 0) { - notify_unquiesce(ictxs, *requests); - } - - return ret_code; -} - int GroupSnapshot_to_group_snap_info2( librados::IoCtx& group_ioctx, const std::string& group_id, const cls::rbd::GroupSnapshot& cls_group_snap, @@ -467,8 +232,8 @@ int GroupSnapshot_to_group_snap_info2( for (const auto& snap : cls_group_snap.snaps) { librados::IoCtx image_ioctx; - int r = util::create_ioctx(group_ioctx, "image", snap.pool, {}, - &image_ioctx); + int r = librbd::util::create_ioctx(group_ioctx, "image", snap.pool, {}, + &image_ioctx); if (r < 0) { return r; } @@ -517,24 +282,24 @@ int Group::image_remove_by_id(librados::IoCtx& group_ioctx, { CephContext *cct = (CephContext *)group_ioctx.cct(); ldout(cct, 20) << "io_ctx=" << &group_ioctx - << " group name " << group_name << " image " - << &image_ioctx << " id " << image_id << dendl; + << ", group_name=" << group_name << ", image=" + << &image_ioctx << ", id=" << image_id << dendl; string group_id; int r = cls_client::dir_get_id(&group_ioctx, RBD_GROUP_DIRECTORY, group_name, - &group_id); + &group_id); if (r < 0) { - lderr(cct) << "error reading group id object: " - << cpp_strerror(r) - << dendl; + lderr(cct) << "error getting the group id: " << cpp_strerror(r) << dendl; return r; } ldout(cct, 20) << "removing image from group name " << group_name << " group id " << group_id << dendl; - return group_image_remove(group_ioctx, group_id, image_ioctx, string(image_id)); + r = Group::group_image_remove(group_ioctx, group_id, image_ioctx, image_id); + + return r; } template @@ -544,7 +309,7 @@ int Group::create(librados::IoCtx& io_ctx, const char *group_name) ldout(cct, 2) << "adding group to directory..." << dendl; - std::string group_id = util::generate_image_id(io_ctx); + std::string group_id = librbd::util::generate_image_id(io_ctx); int r = cls_client::group_dir_add(&io_ctx, RBD_GROUP_DIRECTORY, group_name, group_id); if (r < 0) { @@ -554,7 +319,7 @@ int Group::create(librados::IoCtx& io_ctx, const char *group_name) return r; } - std::string group_header_oid = util::group_header_name(group_id); + std::string group_header_oid = librbd::util::group_header_name(group_id); r = io_ctx.create(group_header_oid, true); if (r < 0) { lderr(cct) << "error creating group header: " << cpp_strerror(r) << dendl; @@ -588,7 +353,11 @@ int Group::remove(librados::IoCtx& io_ctx, const char *group_name) lderr(cct) << "error getting id of group" << dendl; return r; } - string group_header_oid = util::group_header_name(group_id); + + r = Mirror::group_disable(io_ctx, group_name, true); + if (r < 0) { + return r; + } std::vector snaps; r = group_snap_list(io_ctx, group_id, false, false, &snaps); @@ -598,14 +367,14 @@ int Group::remove(librados::IoCtx& io_ctx, const char *group_name) } for (auto &snap : snaps) { - r = group_snap_remove_by_record(io_ctx, snap, group_id, group_header_oid); + r = util::group_snap_remove(io_ctx, group_id, snap); if (r < 0) { return r; } } std::vector images; - r = group_image_list(io_ctx, group_name, &images); + r = Group::group_image_list_by_id(io_ctx, group_id, &images); if (r < 0 && r != -ENOENT) { lderr(cct) << "error listing group images" << dendl; return r; @@ -613,20 +382,21 @@ int Group::remove(librados::IoCtx& io_ctx, const char *group_name) for (auto image : images) { IoCtx image_ioctx; - r = util::create_ioctx(io_ctx, "image", image.spec.pool_id, {}, - &image_ioctx); + r = librbd::util::create_ioctx(io_ctx, "image", image.spec.pool_id, {}, + &image_ioctx); if (r < 0) { return r; } - r = group_image_remove(io_ctx, group_id, image_ioctx, image.spec.image_id); + r = Group::group_image_remove(io_ctx, group_id, image_ioctx, + image.spec.image_id); if (r < 0 && r != -ENOENT) { lderr(cct) << "error removing image from a group" << dendl; return r; } } - string header_oid = util::group_header_name(group_id); + string header_oid = librbd::util::group_header_name(group_id); r = io_ctx.remove(header_oid); if (r < 0 && r != -ENOENT) { @@ -645,8 +415,23 @@ int Group::remove(librados::IoCtx& io_ctx, const char *group_name) } template -int Group::list(IoCtx& io_ctx, vector *names) -{ +int Group::list(IoCtx& io_ctx, vector *names) { + std::map name_to_id_map; + + int r = list(io_ctx, &name_to_id_map); + if (r < 0) { + return r; + } + + for (auto &[name, _] : name_to_id_map) { + names->push_back(name); + } + return 0; +} + +template +int Group::list(librados::IoCtx& io_ctx, + std::map *name_to_id_map) { CephContext *cct = (CephContext *)io_ctx.cct(); ldout(cct, 20) << "io_ctx=" << &io_ctx << dendl; @@ -666,10 +451,8 @@ int Group::list(IoCtx& io_ctx, vector *names) } return r; } - for (pair group : groups) { - names->push_back(group.first); - } if (!groups.empty()) { + name_to_id_map->insert(groups.begin(), groups.end()); last_read = groups.rbegin()->first; } r = groups.size(); @@ -702,8 +485,8 @@ int Group::image_add(librados::IoCtx& group_ioctx, const char *group_name, { CephContext *cct = (CephContext *)group_ioctx.cct(); ldout(cct, 20) << "io_ctx=" << &group_ioctx - << " group name " << group_name << " image " - << &image_ioctx << " name " << image_name << dendl; + << ", group_name=" << group_name << ", image=" + << &image_ioctx << ", name=" << image_name << dendl; if (group_ioctx.get_namespace() != image_ioctx.get_namespace()) { lderr(cct) << "group and image cannot be in different namespaces" << dendl; @@ -715,12 +498,10 @@ int Group::image_add(librados::IoCtx& group_ioctx, const char *group_name, int r = cls_client::dir_get_id(&group_ioctx, RBD_GROUP_DIRECTORY, group_name, &group_id); if (r < 0) { - lderr(cct) << "error reading group id object: " - << cpp_strerror(r) - << dendl; + lderr(cct) << "error getting the group id: " << cpp_strerror(r) << dendl; return r; } - string group_header_oid = util::group_header_name(group_id); + string group_header_oid = librbd::util::group_header_name(group_id); ldout(cct, 20) << "adding image to group name " << group_name @@ -736,7 +517,7 @@ int Group::image_add(librados::IoCtx& group_ioctx, const char *group_name, return r; } - string image_header_oid = util::header_name(image_id); + string image_header_oid = librbd::util::header_name(image_id); ldout(cct, 20) << "adding image " << image_name << " image id " << image_header_oid << dendl; @@ -769,10 +550,20 @@ int Group::image_add(librados::IoCtx& group_ioctx, const char *group_name, } ImageWatcher<>::notify_header_update(image_ioctx, image_header_oid); + r = Mirror::group_image_add(group_ioctx, group_id, image_ioctx, image_id); + if (r < 0) { + return r; + } + r = cls_client::group_image_set(&group_ioctx, group_header_oid, attached_st); + if (r < 0) { + lderr(cct) << "error updating image reference to group: " + << cpp_strerror(-r) << dendl; + return r; + } - return r; + return 0; } template @@ -781,8 +572,8 @@ int Group::image_remove(librados::IoCtx& group_ioctx, const char *group_name, { CephContext *cct = (CephContext *)group_ioctx.cct(); ldout(cct, 20) << "io_ctx=" << &group_ioctx - << " group name " << group_name << " image " - << &image_ioctx << " name " << image_name << dendl; + << ", group_name=" << group_name << ", image= " + << &image_ioctx << ", name=" << image_name << dendl; if (group_ioctx.get_namespace() != image_ioctx.get_namespace()) { lderr(cct) << "group and image cannot be in different namespaces" << dendl; @@ -794,9 +585,7 @@ int Group::image_remove(librados::IoCtx& group_ioctx, const char *group_name, int r = cls_client::dir_get_id(&group_ioctx, RBD_GROUP_DIRECTORY, group_name, &group_id); if (r < 0) { - lderr(cct) << "error reading group id object: " - << cpp_strerror(r) - << dendl; + lderr(cct) << "error getting the group id: " << cpp_strerror(r) << dendl; return r; } @@ -812,7 +601,7 @@ int Group::image_remove(librados::IoCtx& group_ioctx, const char *group_name, return r; } - r = group_image_remove(group_ioctx, group_id, image_ioctx, image_id); + r = Group::group_image_remove(group_ioctx, group_id, image_ioctx, image_id); return r; } @@ -820,36 +609,36 @@ int Group::image_remove(librados::IoCtx& group_ioctx, const char *group_name, template int Group::image_list(librados::IoCtx& group_ioctx, const char *group_name, - std::vector* images) + std::vector* images_info) { CephContext *cct = (CephContext *)group_ioctx.cct(); ldout(cct, 20) << "io_ctx=" << &group_ioctx - << " group name " << group_name << dendl; + << ", group_name=" << group_name << dendl; - std::vector image_ids; + std::vector images; - group_image_list(group_ioctx, group_name, &image_ids); + group_image_list(group_ioctx, group_name, &images); - for (auto image_id : image_ids) { + for (auto image : images) { IoCtx ioctx; - int r = util::create_ioctx(group_ioctx, "image", image_id.spec.pool_id, {}, - &ioctx); + int r = librbd::util::create_ioctx(group_ioctx, "image", + image.spec.pool_id, {}, &ioctx); if (r < 0) { return r; } std::string image_name; r = cls_client::dir_get_name(&ioctx, RBD_DIRECTORY, - image_id.spec.image_id, &image_name); + image.spec.image_id, &image_name); if (r < 0) { return r; } - images->push_back( + images_info->push_back( group_image_info_t { image_name, ioctx.get_id(), - static_cast(image_id.state)}); + static_cast(image.state)}); } return 0; @@ -892,8 +681,8 @@ int Group::image_get_group(I *ictx, group_info_t *group_info) if (RBD_GROUP_INVALID_POOL != ictx->group_spec.pool_id) { IoCtx ioctx; - r = util::create_ioctx(ictx->md_ctx, "group", ictx->group_spec.pool_id, {}, - &ioctx); + r = librbd::util::create_ioctx(ictx->md_ctx, "group", + ictx->group_spec.pool_id, {}, &ioctx); if (r < 0) { return r; } @@ -930,7 +719,8 @@ int Group::snap_create(librados::IoCtx& group_ioctx, NoOpProgressContext prog_ctx; uint64_t internal_flags = 0; - int r = util::snap_create_flags_api_to_internal(cct, flags, &internal_flags); + int r = librbd::util::snap_create_flags_api_to_internal(cct, flags, + &internal_flags); if (r < 0) { return r; } @@ -938,21 +728,18 @@ int Group::snap_create(librados::IoCtx& group_ioctx, r = cls_client::dir_get_id(&group_ioctx, RBD_GROUP_DIRECTORY, group_name, &group_id); if (r < 0) { - lderr(cct) << "error reading group id object: " - << cpp_strerror(r) - << dendl; + lderr(cct) << "error getting the group id: " << cpp_strerror(r) << dendl; return r; } std::vector images; - r = group_image_list(group_ioctx, group_name, &images); + r = Group::group_image_list_by_id(group_ioctx, group_id, &images); if (r < 0) { return r; } int image_count = images.size(); ldout(cct, 20) << "Found " << image_count << " images in group" << dendl; - image_snaps = vector(image_count, cls::rbd::ImageSnapshotSpec()); @@ -961,9 +748,9 @@ int Group::snap_create(librados::IoCtx& group_ioctx, image_snaps[i].image_id = images[i].spec.image_id; } - string group_header_oid = util::group_header_name(group_id); + string group_header_oid = librbd::util::group_header_name(group_id); - group_snap.id = util::generate_image_id(group_ioctx); + group_snap.id = librbd::util::generate_image_id(group_ioctx); group_snap.name = string(snap_name); group_snap.state = cls::rbd::GROUP_SNAPSHOT_STATE_INCOMPLETE; group_snap.snaps = image_snaps; @@ -985,8 +772,8 @@ int Group::snap_create(librados::IoCtx& group_ioctx, for (auto image: images) { librados::IoCtx image_io_ctx; - r = util::create_ioctx(group_ioctx, "image", image.spec.pool_id, {}, - &image_io_ctx); + r = librbd::util::create_ioctx(group_ioctx, "image", image.spec.pool_id, {}, + &image_io_ctx); if (r < 0) { ret_code = r; goto finish; @@ -1024,7 +811,7 @@ int Group::snap_create(librados::IoCtx& group_ioctx, if ((internal_flags & SNAP_CREATE_FLAG_SKIP_NOTIFY_QUIESCE) == 0) { ldout(cct, 20) << "Sending quiesce notification" << dendl; - ret_code = notify_quiesce(ictxs, prog_ctx, &quiesce_requests); + ret_code = util::notify_quiesce(ictxs, prog_ctx, &quiesce_requests); if (ret_code != 0 && (internal_flags & SNAP_CREATE_FLAG_IGNORE_NOTIFY_QUIESCE_ERROR) == 0) { goto remove_record; @@ -1062,7 +849,7 @@ int Group::snap_create(librados::IoCtx& group_ioctx, } } if (ret_code != 0) { - notify_unquiesce(ictxs, quiesce_requests); + util::notify_unquiesce(ictxs, quiesce_requests); goto remove_record; } @@ -1091,7 +878,7 @@ int Group::snap_create(librados::IoCtx& group_ioctx, } else { ImageCtx *ictx = ictxs[i]; ictx->image_lock.lock_shared(); - snap_t snap_id = get_group_snap_id(ictx, ne); + snap_t snap_id = util::get_group_snap_id(ictx, ne); ictx->image_lock.unlock_shared(); if (snap_id == CEPH_NOSNAP) { ldout(cct, 20) << "Couldn't find created snapshot with namespace: " @@ -1118,12 +905,12 @@ int Group::snap_create(librados::IoCtx& group_ioctx, } ldout(cct, 20) << "Sending unquiesce notification" << dendl; - notify_unquiesce(ictxs, quiesce_requests); + util::notify_unquiesce(ictxs, quiesce_requests); goto finish; remove_image_snaps: - notify_unquiesce(ictxs, quiesce_requests); + util::notify_unquiesce(ictxs, quiesce_requests); for (int i = 0; i < image_count; ++i) { ImageCtx *ictx = ictxs[i]; @@ -1133,7 +920,7 @@ remove_image_snaps: on_finishes[i] = new C_SaferCond; std::string snap_name; ictx->image_lock.lock_shared(); - snap_t snap_id = get_group_snap_id(ictx, ne); + snap_t snap_id = util::get_group_snap_id(ictx, ne); r = ictx->get_snap_name(snap_id, &snap_name); ictx->image_lock.unlock_shared(); if (r >= 0) { @@ -1181,9 +968,7 @@ int Group::snap_remove(librados::IoCtx& group_ioctx, const char *group_name, int r = cls_client::dir_get_id(&group_ioctx, RBD_GROUP_DIRECTORY, group_name, &group_id); if (r < 0) { - lderr(cct) << "error reading group id object: " - << cpp_strerror(r) - << dendl; + lderr(cct) << "error getting the group id: " << cpp_strerror(r) << dendl; return r; } @@ -1204,9 +989,7 @@ int Group::snap_remove(librados::IoCtx& group_ioctx, const char *group_name, return -ENOENT; } - string group_header_oid = util::group_header_name(group_id); - r = group_snap_remove_by_record(group_ioctx, *group_snap, group_id, - group_header_oid); + r = util::group_snap_remove(group_ioctx, group_id, *group_snap); return r; } @@ -1224,7 +1007,7 @@ int Group::snap_rename(librados::IoCtx& group_ioctx, const char *group_name, if (r == -ENOENT) { return r; } else if (r < 0) { - lderr(cct) << "error reading group id object: " << cpp_strerror(r) << dendl; + lderr(cct) << "error getting the group id: " << cpp_strerror(r) << dendl; return r; } @@ -1246,7 +1029,7 @@ int Group::snap_rename(librados::IoCtx& group_ioctx, const char *group_name, return -ENOENT; } - std::string group_header_oid = util::group_header_name(group_id); + std::string group_header_oid = librbd::util::group_header_name(group_id); group_snap.name = new_snap_name; r = cls_client::group_snap_set(&group_ioctx, group_header_oid, group_snap); if (r < 0) { @@ -1346,8 +1129,7 @@ int Group::snap_rollback(librados::IoCtx& group_ioctx, int r = cls_client::dir_get_id(&group_ioctx, RBD_GROUP_DIRECTORY, group_name, &group_id); if (r < 0) { - lderr(cct) << "error reading group id object: " - << cpp_strerror(r) << dendl; + lderr(cct) << "error getting the group id: " << cpp_strerror(r) << dendl; return r; } @@ -1401,6 +1183,91 @@ int Group::snap_rollback(librados::IoCtx& group_ioctx, return r; } +template +int Group::group_image_list_by_id(librados::IoCtx& group_ioctx, + const std::string &group_id, + std::vector *images) { + CephContext *cct = (CephContext *)group_ioctx.cct(); + + string group_header_oid = librbd::util::group_header_name(group_id); + + ldout(cct, 20) << "listing images in group id " << group_header_oid << dendl; + images->clear(); + + int r = 0; + const int max_read = 1024; + cls::rbd::GroupImageSpec start_last; + do { + std::vector images_page; + + r = cls_client::group_image_list(&group_ioctx, group_header_oid, + start_last, max_read, &images_page); + + if (r < 0) { + lderr(cct) << "error reading image list from group: " + << cpp_strerror(-r) << dendl; + return r; + } + images->insert(images->end(), + images_page.begin(), images_page.end()); + + if (images_page.size() > 0) + start_last = images_page.rbegin()->spec; + + r = images_page.size(); + } while (r == max_read); + + return 0; +} + +template +int Group::group_image_remove(librados::IoCtx& group_ioctx, string group_id, + librados::IoCtx& image_ioctx, string image_id) { + CephContext *cct = (CephContext *)group_ioctx.cct(); + + string group_header_oid = librbd::util::group_header_name(group_id); + + string image_header_oid = librbd::util::header_name(image_id); + + ldout(cct, 20) << "removing image " << image_id + << " image id " << image_header_oid << dendl; + + cls::rbd::GroupSpec group_spec(group_id, group_ioctx.get_id()); + + cls::rbd::GroupImageStatus incomplete_st(image_id, image_ioctx.get_id(), + cls::rbd::GROUP_IMAGE_LINK_STATE_INCOMPLETE); + + cls::rbd::GroupImageSpec spec(image_id, image_ioctx.get_id()); + + int r = cls_client::group_image_set(&group_ioctx, group_header_oid, + incomplete_st); + + if (r < 0) { + lderr(cct) << "couldn't put image into removing state: " + << cpp_strerror(-r) << dendl; + return r; + } + + r = cls_client::image_group_remove(&image_ioctx, image_header_oid, + group_spec); + if ((r < 0) && (r != -ENOENT)) { + lderr(cct) << "couldn't remove group reference from image" + << cpp_strerror(-r) << dendl; + return r; + } else if (r >= 0) { + ImageWatcher::notify_header_update(image_ioctx, image_header_oid); + } + + r = cls_client::group_image_remove(&group_ioctx, group_header_oid, spec); + if (r < 0) { + lderr(cct) << "couldn't remove image from group" + << cpp_strerror(-r) << dendl; + return r; + } + + return 0; +} + } // namespace api } // namespace librbd diff --git a/src/librbd/api/Group.h b/src/librbd/api/Group.h index ffbb9afea1af9..a73d082c8bef8 100644 --- a/src/librbd/api/Group.h +++ b/src/librbd/api/Group.h @@ -4,6 +4,7 @@ #ifndef CEPH_LIBRBD_API_GROUP_H #define CEPH_LIBRBD_API_GROUP_H +#include "cls/rbd/cls_rbd_client.h" #include "include/rbd/librbd.hpp" #include "include/rados/librados_fwd.hpp" #include @@ -23,6 +24,8 @@ struct Group { static int list(librados::IoCtx& io_ctx, std::vector *names); static int get_id(librados::IoCtx& io_ctx, const char *group_name, std::string *group_id); + static int list(librados::IoCtx& io_ctx, + std::map *name_to_id_map); static int rename(librados::IoCtx& io_ctx, const char *src_group_name, const char *dest_group_name); @@ -56,6 +59,12 @@ struct Group { const char *group_name, const char *snap_name, ProgressContext& pctx); + static int group_image_list_by_id(librados::IoCtx& group_ioctx, + const std::string &group_id, + std::vector *images); + static int group_image_remove(librados::IoCtx& group_ioctx, std::string group_id, + librados::IoCtx& image_ioctx, std::string image_id); + }; } // namespace api diff --git a/src/librbd/api/Mirror.cc b/src/librbd/api/Mirror.cc index 99f1506518502..c40a059146707 100644 --- a/src/librbd/api/Mirror.cc +++ b/src/librbd/api/Mirror.cc @@ -12,11 +12,15 @@ #include "librbd/ImageCtx.h" #include "librbd/ImageState.h" #include "librbd/Journal.h" +#include "librbd/ImageWatcher.h" #include "librbd/MirroringWatcher.h" #include "librbd/Operations.h" #include "librbd/Utils.h" +#include "librbd/api/Group.h" #include "librbd/api/Image.h" #include "librbd/api/Namespace.h" +#include "librbd/api/Utils.h" +#include "librbd/group/ListSnapshotsRequest.h" #include "librbd/mirror/DemoteRequest.h" #include "librbd/mirror/DisableRequest.h" #include "librbd/mirror/EnableRequest.h" @@ -300,15 +304,6 @@ int list_mirror_images(librados::IoCtx& io_ctx, return 0; } -template -const char *pool_or_namespace(I *ictx) { - if (!ictx->md_ctx.get_namespace().empty()) { - return "namespace"; - } else { - return "pool"; - } -} - struct C_ImageGetInfo : public Context { mirror_image_info_t *mirror_image_info; Context *on_finish; @@ -410,6 +405,9 @@ template struct C_ImageSnapshotCreate : public Context { I *ictx; uint64_t snap_create_flags; + int64_t group_pool_id; + std::string group_id; + std::string group_snap_id; uint64_t *snap_id; Context *on_finish; @@ -417,9 +415,15 @@ struct C_ImageSnapshotCreate : public Context { mirror::PromotionState promotion_state; std::string primary_mirror_uuid; - C_ImageSnapshotCreate(I *ictx, uint64_t snap_create_flags, uint64_t *snap_id, + C_ImageSnapshotCreate(I *ictx, uint64_t snap_create_flags, + int64_t group_pool_id, + const std::string &group_id, + const std::string &group_snap_id, + uint64_t *snap_id, Context *on_finish) - : ictx(ictx), snap_create_flags(snap_create_flags), snap_id(snap_id), + : ictx(ictx), snap_create_flags(snap_create_flags), + group_pool_id(group_pool_id), group_id(group_id), + group_snap_id(group_snap_id), snap_id(snap_id), on_finish(on_finish) { } @@ -438,30 +442,153 @@ struct C_ImageSnapshotCreate : public Context { auto req = mirror::snapshot::CreatePrimaryRequest::create( ictx, mirror_image.global_image_id, CEPH_NOSNAP, snap_create_flags, 0U, - snap_id, on_finish); + group_pool_id, group_id, group_snap_id, snap_id, on_finish); req->send(); } }; +template +void close_images(std::vector *image_ctxs) { + std::vector on_finishes(image_ctxs->size()); + + for (size_t i = 0; i < image_ctxs->size(); i++) { + (*image_ctxs)[i]->state->close(&on_finishes[i]); + } + + for (auto &on_finish : on_finishes) { + on_finish.wait(); + } + + image_ctxs->clear(); +} + +template +int open_group_images(IoCtx& group_ioctx, const std::string &group_id, + std::vector *image_ctxs) { + std::vector images; + int r = Group::group_image_list_by_id(group_ioctx, group_id, &images); + if (r < 0) { + return r; + } + + std::vector on_finishes(images.size()); + int ret_code = 0; + size_t i = 0; + for ( ; i < images.size(); i++) { + auto &image = images[i]; + librbd::IoCtx image_io_ctx; + r = librbd::util::create_ioctx(group_ioctx, "image", image.spec.pool_id, {}, + &image_io_ctx); + if (r < 0) { + ret_code = r; + break; + } + + librbd::ImageCtx* image_ctx = new ImageCtx("", image.spec.image_id.c_str(), + nullptr, image_io_ctx, false); + + image_ctx->state->open(0, &on_finishes[i]); + + image_ctxs->push_back(image_ctx); + } + + for (size_t j = 0; j < i; j++) { + r = on_finishes[j].wait(); + if (r < 0 && ret_code == 0) { + ret_code = r; + } + } + + if (ret_code != 0) { + close_images(image_ctxs); + return ret_code; + } + + return 0; +} + +std::string prepare_primary_mirror_snap_name(CephContext *cct, + const std::string &global_group_id, + const std::string &snap_id) { + ldout(cct, 10) << "global_group_id: " << global_group_id + << ", snap_id: " << snap_id << dendl; + + std::stringstream ind_snap_name_stream; + ind_snap_name_stream << ".mirror.primary." + << global_group_id << "." << snap_id; + return ind_snap_name_stream.str(); +} + +int get_last_mirror_snapshot_state(librados::IoCtx &group_ioctx, + const std::string &group_id, + cls::rbd::MirrorSnapshotState *state) { + std::vector snaps; + + C_SaferCond cond; + auto req = group::ListSnapshotsRequest<>::create(group_ioctx, group_id, + true, true, + &snaps, &cond); + req->send(); + int r = cond.wait(); + if (r < 0) { + return r; + } + + for (auto it = snaps.rbegin(); it != snaps.rend(); it++) { + auto ns = std::get_if( + &it->snapshot_namespace); + if (ns != nullptr) { + // XXXMG: check primary_mirror_uuid matches? + *state = ns->state; + return 0; + } + } + + return -ENOENT; +} + } // anonymous namespace +template +const char* Mirror::pool_or_namespace(I *ictx) { + if (!ictx->md_ctx.get_namespace().empty()) { + return "namespace"; + } else { + return "pool"; + } +} + template int Mirror::image_enable(I *ictx, mirror_image_mode_t mode, bool relax_same_pool_parent_check) { + return Mirror::image_enable(ictx, {}, mode, + relax_same_pool_parent_check, nullptr); +} + +template +int Mirror::image_enable(I *ictx, + const std::string &group_snap_id, + mirror_image_mode_t mode, + bool relax_same_pool_parent_check, + uint64_t *snap_id) { CephContext *cct = ictx->cct; ldout(cct, 20) << "ictx=" << ictx << " mode=" << mode + << "group_snap=(" << ictx->group_spec.pool_id << " " + << ictx->group_spec.group_id << " " << group_snap_id << ")" << " relax_same_pool_parent_check=" << relax_same_pool_parent_check << dendl; int r = ictx->state->refresh_if_required(); if (r < 0) { + lderr(cct) << "refresh request failed: " << cpp_strerror(r) << dendl; return r; } cls::rbd::MirrorMode mirror_mode; r = cls_client::mirror_mode_get(&ictx->md_ctx, &mirror_mode); if (r < 0) { - lderr(cct) << "cannot enable mirroring: failed to retrieve mirror mode: " + lderr(cct) << "cannot enable mirroring: failed to retrieve mirror mode for: " + << Mirror::pool_or_namespace(ictx) << ", :" << cpp_strerror(r) << dendl; return r; } @@ -469,12 +596,12 @@ int Mirror::image_enable(I *ictx, mirror_image_mode_t mode, if (mirror_mode == cls::rbd::MIRROR_MODE_DISABLED || mirror_mode == cls::rbd::MIRROR_MODE_INIT_ONLY) { lderr(cct) << "cannot enable mirroring: mirroring is not enabled on a " - << pool_or_namespace(ictx) << dendl; + << Mirror::pool_or_namespace(ictx) << dendl; return -EINVAL; } if (mirror_mode != cls::rbd::MIRROR_MODE_IMAGE) { - lderr(cct) << "cannot enable mirroring: " << pool_or_namespace(ictx) + lderr(cct) << "cannot enable mirroring: " << Mirror::pool_or_namespace(ictx) << " is not in image mirror mode" << dendl; return -EINVAL; } @@ -512,7 +639,8 @@ int Mirror::image_enable(I *ictx, mirror_image_mode_t mode, C_SaferCond ctx; auto req = mirror::EnableRequest::create( - ictx, static_cast(mode), "", false, &ctx); + ictx, static_cast(mode), "", false, + ictx->group_spec.pool_id, ictx->group_spec.group_id, group_snap_id, snap_id, &ctx); req->send(); r = ctx.wait(); @@ -531,6 +659,7 @@ int Mirror::image_disable(I *ictx, bool force) { int r = ictx->state->refresh_if_required(); if (r < 0) { + lderr(cct) << "refresh request failed: " << cpp_strerror(r) << dendl; return r; } @@ -557,9 +686,6 @@ int Mirror::image_disable(I *ictx, bool force) { ldout(cct, 20) << "ignoring disable command: mirroring is not enabled for " << "this image" << dendl; return 0; - } else if (r == -EOPNOTSUPP) { - ldout(cct, 5) << "mirroring not supported by OSD" << dendl; - return r; } else if (r < 0) { lderr(cct) << "failed to retrieve mirror image metadata: " << cpp_strerror(r) << dendl; @@ -619,10 +745,10 @@ int Mirror::image_disable(I *ictx, bool force) { if (child_pool_id == -1 || child_pool_id != child_image.pool_id || child_io_ctx.get_namespace() != child_image.pool_namespace) { - r = util::create_ioctx(ictx->md_ctx, "child image", - child_image.pool_id, - child_image.pool_namespace, - &child_io_ctx); + r = librbd::util::create_ioctx(ictx->md_ctx, "child image", + child_image.pool_id, + child_image.pool_namespace, + &child_io_ctx); if (r < 0) { rollback = true; return r; @@ -714,6 +840,14 @@ int Mirror::image_promote(I *ictx, bool force) { template void Mirror::image_promote(I *ictx, bool force, Context *on_finish) { + return Mirror::image_promote(ictx, {}, force, nullptr, on_finish); +} + +template +void Mirror::image_promote(I *ictx, + const std::string &group_snap_id, + bool force, uint64_t *snap_id, + Context *on_finish) { CephContext *cct = ictx->cct; ldout(cct, 20) << "ictx=" << ictx << ", " << "force=" << force << dendl; @@ -732,14 +866,19 @@ void Mirror::image_promote(I *ictx, bool force, Context *on_finish) { on_finish->complete(r); }); - auto on_refresh = new LambdaContext([ictx, force, on_promote](int r) { + auto on_refresh = new LambdaContext( + [ictx, force, group_snap_id, snap_id, on_promote](int r) { if (r < 0) { lderr(ictx->cct) << "refresh failed: " << cpp_strerror(r) << dendl; on_promote->complete(r); return; } - auto req = mirror::PromoteRequest<>::create(*ictx, force, on_promote); + auto req = mirror::PromoteRequest<>::create(*ictx, force, + ictx->group_spec.pool_id, + ictx->group_spec.group_id, + group_snap_id, snap_id, + on_promote); req->send(); }); ictx->state->refresh(on_refresh); @@ -762,6 +901,13 @@ int Mirror::image_demote(I *ictx) { template void Mirror::image_demote(I *ictx, Context *on_finish) { + return Mirror::image_demote(ictx, {}, nullptr, on_finish); +} + +template +void Mirror::image_demote(I *ictx, + const std::string &group_snap_id, + uint64_t *snap_id, Context *on_finish) { CephContext *cct = ictx->cct; ldout(cct, 20) << "ictx=" << ictx << dendl; @@ -774,14 +920,19 @@ void Mirror::image_demote(I *ictx, Context *on_finish) { on_finish->complete(r); }); - auto on_refresh = new LambdaContext([ictx, on_cleanup](int r) { + auto on_refresh = new LambdaContext( + [ictx, group_snap_id, snap_id, on_cleanup](int r) { if (r < 0) { lderr(ictx->cct) << "refresh failed: " << cpp_strerror(r) << dendl; on_cleanup->complete(r); return; } - auto req = mirror::DemoteRequest<>::create(*ictx, on_cleanup); + auto req = mirror::DemoteRequest<>::create(*ictx, + ictx->group_spec.pool_id, + ictx->group_spec.group_id, + group_snap_id, + snap_id, on_cleanup); req->send(); }); @@ -801,6 +952,7 @@ int Mirror::image_resync(I *ictx) { int r = ictx->state->refresh_if_required(); if (r < 0) { + lderr(cct) << "refresh request failed: " << cpp_strerror(r) << dendl; return r; } @@ -819,7 +971,11 @@ int Mirror::image_resync(I *ictx) { lderr(cct) << "failed to retrieve mirroring state, cannot resync: " << cpp_strerror(r) << dendl; return r; - } else if (mirror_image.state != cls::rbd::MIRROR_IMAGE_STATE_ENABLED) { + } + ldout(cct, 20) << "image: " << ictx->name.c_str() + << ", id: " << ictx->id << ", global_image_id: " + << mirror_image.global_image_id << dendl; + if (mirror_image.state != cls::rbd::MIRROR_IMAGE_STATE_ENABLED) { lderr(cct) << "mirroring is not enabled, cannot resync" << dendl; return -EINVAL; } else if (promotion_state == mirror::PROMOTION_STATE_PRIMARY) { @@ -838,6 +994,7 @@ int Mirror::image_resync(I *ictx) { std::string mirror_uuid; r = uuid_get(ictx->md_ctx, &mirror_uuid); if (r < 0) { + lderr(cct) << "get uuid failed" << dendl; return r; } @@ -1193,8 +1350,9 @@ int Mirror::mode_set(librados::IoCtx& io_ctx, for (const auto& img_pair : images) { uint64_t features; uint64_t incompatible_features; - r = cls_client::get_features(&io_ctx, util::header_name(img_pair.second), - true, &features, &incompatible_features); + r = cls_client::get_features( + &io_ctx, librbd::util::header_name(img_pair.second), true, &features, + &incompatible_features); if (r < 0) { lderr(cct) << "error getting features for image " << img_pair.first << ": " << cpp_strerror(r) << dendl; @@ -2135,26 +2293,37 @@ int Mirror::image_snapshot_create(I *ictx, uint32_t flags, template void Mirror::image_snapshot_create(I *ictx, uint32_t flags, uint64_t *snap_id, Context *on_finish) { + return Mirror::image_snapshot_create(ictx, flags, {}, snap_id, + on_finish); +} + +template +void Mirror::image_snapshot_create(I *ictx, uint32_t flags, + const std::string &group_snap_id, + uint64_t *snap_id, Context *on_finish) { CephContext *cct = ictx->cct; ldout(cct, 20) << "ictx=" << ictx << dendl; uint64_t snap_create_flags = 0; - int r = util::snap_create_flags_api_to_internal(cct, flags, - &snap_create_flags); + int r = librbd::util::snap_create_flags_api_to_internal(cct, flags, + &snap_create_flags); if (r < 0) { on_finish->complete(r); return; } auto on_refresh = new LambdaContext( - [ictx, snap_create_flags, snap_id, on_finish](int r) { + [ictx, snap_create_flags, group_snap_id, snap_id, on_finish](int r) { if (r < 0) { lderr(ictx->cct) << "refresh failed: " << cpp_strerror(r) << dendl; on_finish->complete(r); return; } - auto ctx = new C_ImageSnapshotCreate(ictx, snap_create_flags, snap_id, + auto ctx = new C_ImageSnapshotCreate(ictx, snap_create_flags, + ictx->group_spec.pool_id, + ictx->group_spec.group_id, + group_snap_id, snap_id, on_finish); auto req = mirror::GetInfoRequest::create(*ictx, &ctx->mirror_image, &ctx->promotion_state, @@ -2170,6 +2339,1550 @@ void Mirror::image_snapshot_create(I *ictx, uint32_t flags, } } +template +int Mirror::group_list(IoCtx& io_ctx, std::vector *names) { + CephContext *cct = reinterpret_cast(io_ctx.cct()); + + std::set group_ids; + std::string last_read = ""; + int max_read = 1024; + int r; + do { + std::map mirror_groups; + r = cls_client::mirror_group_list(&io_ctx, last_read, max_read, + &mirror_groups); + if (r < 0 && r != -ENOENT) { + lderr(cct) << "error listing mirrored image directory: " + << cpp_strerror(r) << dendl; + return r; + } + for (auto &[group_id, mirror_group] : mirror_groups) { + group_ids.insert(group_id); + } + if (!mirror_groups.empty()) { + last_read = mirror_groups.rbegin()->first; + } + r = mirror_groups.size(); + } while (r == max_read); + + if (group_ids.empty()) { + return 0; + } + + std::map name_to_id_map; + r = Group::list(io_ctx, &name_to_id_map); + if (r < 0) { + return r; + } + + for (auto &[name, group_id] : name_to_id_map) { + if (group_ids.count(group_id) > 0) { + names->push_back(name); + } + } + return 0; +} + +template +int Mirror::group_enable(IoCtx& group_ioctx, const char *group_name, + mirror_image_mode_t mirror_image_mode) { + CephContext *cct = (CephContext *)group_ioctx.cct(); + ldout(cct, 20) << "io_ctx=" << &group_ioctx + << ", group_name=" << group_name + << ", namespace=" << group_ioctx.get_namespace() + << ", mirror_image_mode=" << mirror_image_mode << dendl; + + if (mirror_image_mode != RBD_MIRROR_IMAGE_MODE_SNAPSHOT) { + return -EOPNOTSUPP; + } + + std::string group_id; + int r = cls_client::dir_get_id(&group_ioctx, RBD_GROUP_DIRECTORY, + group_name, &group_id); + if (r < 0) { + lderr(cct) << "error getting the group id: " << cpp_strerror(r) << dendl; + return r; + } + + cls::rbd::MirrorMode mirror_mode; + r = cls_client::mirror_mode_get(&group_ioctx, &mirror_mode); + if (r < 0) { + lderr(cct) << "cannot enable mirroring: failed to retrieve mirror mode: " + << cpp_strerror(r) << dendl; + return r; + } + + if (mirror_mode != cls::rbd::MIRROR_MODE_IMAGE) { + lderr(cct) << "cannot enable mirroring, as it is not in the image mirror mode" << dendl; + return -EINVAL; + } + + cls::rbd::MirrorGroup mirror_group; + r = cls_client::mirror_group_get(&group_ioctx, group_id, &mirror_group); + if (r == -EOPNOTSUPP) { + ldout(cct, 5) << "group mirroring not supported by OSD" << dendl; + return r; + } else if (r < 0 && r != -ENOENT) { + lderr(cct) << "failed to retrieve mirror group metadata: " + << cpp_strerror(r) << dendl; + return r; + } + + auto mode = static_cast( + mirror_group.mirror_image_mode); + if (mode != mirror_image_mode) { + lderr(cct) << "mirroring for group " << group_name + << " already enabled with different mirror image mode " << mode + << dendl; + return -EINVAL; + } + + if (mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED) { + ldout(cct, 10) << "mirroring for group " << group_name + << " already enabled" << dendl; + return 0; + } + + if (mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLING) { + lderr(cct) << "enabling mirroring for group " << group_name + << " either in progress or was interrupted" << dendl; + return -EINVAL; + } + + // XXXMG: remove code duplication + auto ns = group_ioctx.get_namespace(); + group_ioctx.set_namespace(""); + std::vector peers; + r = cls_client::mirror_peer_list(&group_ioctx, &peers); + if (r < 0) { + lderr(cct) << "error reading mirror peers: " << cpp_strerror(r) << dendl; + return r; + } + group_ioctx.set_namespace(ns); + + std::set mirror_peer_uuids; + for (auto &peer : peers) { + ldout(cct, 0) << "peer: " << peer << dendl; + if (peer.mirror_peer_direction == cls::rbd::MIRROR_PEER_DIRECTION_RX) { + continue; + } + mirror_peer_uuids.insert(peer.uuid); + } + + if (mirror_peer_uuids.empty()) { + lderr(cct) << "no mirror tx peers configured for the pool" << dendl; + return -EINVAL; + } + + std::vector image_ctxs; + r = open_group_images(group_ioctx, group_id, &image_ctxs); + if (r < 0) { + return r; + } + + std::vector mirror_images(image_ctxs.size()); + std::vector on_finishes(image_ctxs.size()); + + for (size_t i = 0; i < on_finishes.size(); i++) { + image_get_info(image_ctxs[i], &mirror_images[i], &on_finishes[i]); + } + + int ret_code = 0; + for (size_t i = 0; i < on_finishes.size(); i++) { + r = on_finishes[i].wait(); + if (r < 0) { + if (ret_code != 0) { + ret_code = r; + } + } else if (mirror_images[i].state == RBD_MIRROR_IMAGE_ENABLED) { + lderr(cct) << "image " << image_ctxs[i]->name << " has mirroring enabled" + << dendl; + ret_code = -EINVAL; + } + } + + if (ret_code != 0) { + close_images(&image_ctxs); + return ret_code; + } + + uuid_d uuid_gen; + uuid_gen.generate_random(); + mirror_group = {uuid_gen.to_string(), + static_cast(mirror_image_mode), + cls::rbd::MIRROR_GROUP_STATE_ENABLING}; + r = cls_client::mirror_group_set(&group_ioctx, group_id, mirror_group); + if (r < 0) { + lderr(cct) << "failed to set mirroring group metadata: " + << cpp_strerror(r) << dendl; + close_images(&image_ctxs); + return r; + } + + std::string group_snap_id = librbd::util::generate_image_id(group_ioctx); + cls::rbd::GroupSnapshot group_snap{ + group_snap_id, + cls::rbd::MirrorGroupSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, + mirror_peer_uuids, {}, {}}, + prepare_primary_mirror_snap_name(cct, uuid_gen.to_string(), + group_snap_id), + cls::rbd::GROUP_SNAPSHOT_STATE_INCOMPLETE}; + + for (auto image_ctx: image_ctxs) { + group_snap.snaps.emplace_back(image_ctx->md_ctx.get_id(), image_ctx->id, + CEPH_NOSNAP); + } + + std::string group_header_oid = librbd::util::group_header_name(group_id); + r = cls_client::group_snap_set(&group_ioctx, group_header_oid, group_snap); + if (r < 0) { + lderr(cct) << "failed to set group snapshot metadata: " << cpp_strerror(r) + << dendl; + close_images(&image_ctxs); + return r; + } + + std::vector snap_ids(image_ctxs.size()); + + for (size_t i = 0; i < image_ctxs.size(); i++) { + r = image_enable(image_ctxs[i], group_ioctx.get_id(), group_id, + group_snap_id, mirror_image_mode, false, &snap_ids[i]); + if (r < 0) { + break; + } + } + + auto image_count = image_ctxs.size(); + + close_images(&image_ctxs); + + if (r < 0) { + int rr = cls_client::group_snap_remove(&group_ioctx, group_header_oid, + group_snap.id); + if (rr < 0) { + lderr(cct) << "failed to remove group snapshot metadata: " + << cpp_strerror(rr) << dendl; + } + + // TODO: try to remove the created image snapshots + return r; + } + + group_snap.state = cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE; + r = cls_client::group_snap_set(&group_ioctx, group_header_oid, group_snap); + if (r < 0) { + lderr(cct) << "failed to update group snapshot metadata: " + << cpp_strerror(r) << dendl; + } + + mirror_group.state = cls::rbd::MIRROR_GROUP_STATE_ENABLED; + r = cls_client::mirror_group_set(&group_ioctx, group_id, mirror_group); + if (r < 0) { + lderr(cct) << "failed to update mirroring group metadata: " + << cpp_strerror(r) << dendl; + return r; + } + + r = MirroringWatcher::notify_group_updated( + group_ioctx, cls::rbd::MIRROR_GROUP_STATE_ENABLED, group_id, + mirror_group.global_group_id, image_count); + if (r < 0) { + lderr(cct) << "failed to notify mirroring group=" << group_name + << " updated: " << cpp_strerror(r) << dendl; + // not fatal + } + + return 0; +} + +template +int Mirror::group_disable(IoCtx& group_ioctx, const char *group_name, + bool force) { + CephContext *cct = (CephContext *)group_ioctx.cct(); + ldout(cct, 20) << "io_ctx=" << &group_ioctx + << ", group_name=" << group_name + << ", force=" << force << dendl; + + std::string group_id; + int r = cls_client::dir_get_id(&group_ioctx, RBD_GROUP_DIRECTORY, + group_name, &group_id); + if (r < 0) { + lderr(cct) << "error getting the group id: " << cpp_strerror(r) << dendl; + return r; + } + + cls::rbd::MirrorGroup mirror_group; + r = cls_client::mirror_group_get(&group_ioctx, group_id, &mirror_group); + if (r == -EOPNOTSUPP) { + ldout(cct, 5) << "group mirroring not supported by OSD" << dendl; + return r; + } else if (r == -ENOENT) { + ldout(cct, 10) << "mirroring for group " << group_name + << " already disabled" << dendl; + return 0; + } else if (r < 0) { + lderr(cct) << "failed to retrieve mirror group metadata: " + << cpp_strerror(r) << dendl; + return r; + } else if (mirror_group.mirror_image_mode != + cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT) { + auto mode = static_cast( + mirror_group.mirror_image_mode); + lderr(cct) << "cannot disable, mirror mode is set to: " << mode << dendl; + return -EOPNOTSUPP; + } + + cls::rbd::MirrorSnapshotState state; + r = get_last_mirror_snapshot_state(group_ioctx, group_id, &state); + if (r == -ENOENT) { + ldout(cct, 10) << "mirroring for group " << group_name + << " already disabled" << dendl; + return 0; + } else if (r < 0) { + lderr(cct) << "failed to get last mirror snapshot state: " + << cpp_strerror(r) << dendl; + return r; + } + + if (state == cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY && !force) { + lderr(cct) << "group " << group_name + << " is non-primary, ideally disable it from primary cluster " + << " or if you know what you are doing, add a force flag" + << dendl; + return -EINVAL; + } + + std::vector image_ctxs; + r = open_group_images(group_ioctx, group_id, &image_ctxs); + if (r < 0) { + return r; + } + + mirror_group.state = cls::rbd::MIRROR_GROUP_STATE_DISABLING; + r = cls_client::mirror_group_set(&group_ioctx, group_id, mirror_group); + if (r < 0) { + lderr(cct) << "failed to update mirroring group metadata: " + << cpp_strerror(r) << dendl; + close_images(&image_ctxs); + return r; + } + + r = MirroringWatcher::notify_group_updated( + group_ioctx, cls::rbd::MIRROR_GROUP_STATE_DISABLED, group_id, + mirror_group.global_group_id, image_ctxs.size()); + if (r < 0) { + lderr(cct) << "failed to notify mirroring group=" << group_name + << " updated: " << cpp_strerror(r) << dendl; + // not fatal + } + + for (auto image_ctx : image_ctxs) { + ldout(cct, 10) << "attempting to disable image with id " << image_ctx->id + << ": " << cpp_strerror(r) << dendl; + r = image_disable(image_ctx, force); + if (r < 0) { + break; + } + } + + auto image_count = image_ctxs.size(); + + close_images(&image_ctxs); + + if (r < 0) { + lderr(cct) << "failed to disable one or more images: " + << cpp_strerror(r) << dendl; + return r; + } + + std::vector snaps; + C_SaferCond cond; + auto req = group::ListSnapshotsRequest<>::create(group_ioctx, group_id, + true, true, + &snaps, &cond); + req->send(); + r = cond.wait(); + if (r < 0) { + lderr(cct) << "failed to list group snapshots: " + << cpp_strerror(r) << dendl; + // ignore + } + + for (auto &snap : snaps) { + auto ns = std::get_if( + &snap.snapshot_namespace); + if (ns == nullptr) { + continue; + } + r = util::group_snap_remove(group_ioctx, group_id, snap); + if (r < 0) { + lderr(cct) << "failed to remove group snapshot metadata: " + << cpp_strerror(r) << dendl; + // ignore + } + } + + r = cls_client::mirror_group_remove(&group_ioctx, group_id); + if (r < 0 && r != -ENOENT) { + lderr(cct) << "failed to remove mirroring group metadata: " + << cpp_strerror(r) << dendl; + return r; + } + + r = MirroringWatcher::notify_group_updated( + group_ioctx, cls::rbd::MIRROR_GROUP_STATE_DISABLED, group_id, + mirror_group.global_group_id, image_count); + if (r < 0) { + lderr(cct) << "failed to notify mirroring group=" << group_name + << " updated: " << cpp_strerror(r) << dendl; + + // not fatal + } + + return 0; +} + +template +int Mirror::group_promote(IoCtx& group_ioctx, const char *group_name, + bool force) { + CephContext *cct = (CephContext *)group_ioctx.cct(); + ldout(cct, 20) << "io_ctx=" << &group_ioctx + << ", group_name=" << group_name + << ", force=" << force << dendl; + + std::string group_id; + int r = cls_client::dir_get_id(&group_ioctx, RBD_GROUP_DIRECTORY, + group_name, &group_id); + if (r < 0) { + lderr(cct) << "error getting the group id: " << cpp_strerror(r) << dendl; + return r; + } + + cls::rbd::MirrorGroup mirror_group; + r = cls_client::mirror_group_get(&group_ioctx, group_id, &mirror_group); + if (r == -ENOENT) { + ldout(cct, 10) << "mirroring for group " << group_name + << " disabled" << dendl; + return -EINVAL; + } else if (r < 0) { + lderr(cct) << "failed to retrieve mirror group metadata: " + << cpp_strerror(r) << dendl; + return r; + } else if (mirror_group.mirror_image_mode != + cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT) { + auto mode = static_cast( + mirror_group.mirror_image_mode); + lderr(cct) << "cannot promote, mirror mode is set to: " << mode << dendl; + return -EOPNOTSUPP; + } + + cls::rbd::MirrorSnapshotState state; + r = get_last_mirror_snapshot_state(group_ioctx, group_id, &state); + if (r == -ENOENT) { + state = cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED; // XXXMG? + r = 0; + } + if (r < 0) { + return r; + } + + if (state == cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY) { + lderr(cct) << "group " << group_name << " is already primary" << dendl; + return -EINVAL; + } else if (state == cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY && !force) { + lderr(cct) << "group " << group_name + << " is still primary within a remote cluster" << dendl; + return -EBUSY; + } + + std::vector peers; + r = cls_client::mirror_peer_list(&group_ioctx, &peers); + if (r < 0) { + lderr(cct) << "error reading mirror peers: " << cpp_strerror(r) << dendl; + return r; + } + + std::set mirror_peer_uuids; + for (auto &peer : peers) { + if (peer.mirror_peer_direction == cls::rbd::MIRROR_PEER_DIRECTION_RX) { + continue; + } + mirror_peer_uuids.insert(peer.uuid); + } + + if (mirror_peer_uuids.empty()) { + lderr(cct) << "no mirror tx peers configured for the pool" << dendl; + return -EINVAL; + } + + std::vector image_ctxs; + r = open_group_images(group_ioctx, group_id, &image_ctxs); + if (r < 0) { + return r; + } + + std::string group_snap_id = librbd::util::generate_image_id(group_ioctx); + + cls::rbd::GroupSnapshot group_snap{ + group_snap_id, + cls::rbd::MirrorGroupSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, + mirror_peer_uuids, {}, {}}, + prepare_primary_mirror_snap_name(cct, mirror_group.global_group_id, + group_snap_id), + cls::rbd::GROUP_SNAPSHOT_STATE_INCOMPLETE}; + + for (auto image_ctx: image_ctxs) { + group_snap.snaps.emplace_back(image_ctx->md_ctx.get_id(), image_ctx->id, + CEPH_NOSNAP); + } + + std::string group_header_oid = librbd::util::group_header_name(group_id); + r = cls_client::group_snap_set(&group_ioctx, group_header_oid, group_snap); + if (r < 0) { + lderr(cct) << "failed to set group snapshot metadata: " << cpp_strerror(r) + << dendl; + close_images(&image_ctxs); + return r; + } + + // XXXMG: Requesting exclusive locks for images? + // XXXMG: notify quiesce? + + std::vector snap_ids(image_ctxs.size()); + std::vector on_finishes(image_ctxs.size()); + + for (size_t i = 0; i < on_finishes.size(); i++) { + image_promote(image_ctxs[i], group_ioctx.get_id(), group_id, group_snap_id, + force, &snap_ids[i], &on_finishes[i]); + } + + int ret_code = 0; + for (size_t i = 0; i < on_finishes.size(); i++) { + r = on_finishes[i].wait(); + if (r < 0) { + if (ret_code == 0) { + ret_code = r; + } + } else { + group_snap.snaps[i].snap_id = snap_ids[i]; + } + } + + if (ret_code < 0) { + r = cls_client::group_snap_remove(&group_ioctx, group_header_oid, + group_snap.id); + if (r < 0) { + lderr(cct) << "failed to remove group snapshot metadata: " + << cpp_strerror(r) << dendl; + } + } else { + group_snap.state = cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE; + r = cls_client::group_snap_set(&group_ioctx, group_header_oid, group_snap); + if (r < 0) { + lderr(cct) << "failed to update group snapshot metadata: " + << cpp_strerror(r) << dendl; + ret_code = r; + } + } + + close_images(&image_ctxs); + + return ret_code; +} + +template +int Mirror::group_demote(IoCtx& group_ioctx, const char *group_name) { + CephContext *cct = (CephContext *)group_ioctx.cct(); + ldout(cct, 20) << "io_ctx=" << &group_ioctx + << ", group_name=" << group_name << dendl; + + std::string group_id; + int r = cls_client::dir_get_id(&group_ioctx, RBD_GROUP_DIRECTORY, + group_name, &group_id); + if (r < 0) { + lderr(cct) << "error getting the group id: " << cpp_strerror(r) << dendl; + return r; + } + + cls::rbd::MirrorGroup mirror_group; + r = cls_client::mirror_group_get(&group_ioctx, group_id, &mirror_group); + if (r == -ENOENT) { + ldout(cct, 10) << "mirroring for group " << group_name + << " disabled" << dendl; + return -EINVAL; + } else if (r < 0) { + lderr(cct) << "failed to retrieve mirror group metadata: " + << cpp_strerror(r) << dendl; + return r; + } else if (mirror_group.mirror_image_mode != + cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT) { + auto mode = static_cast( + mirror_group.mirror_image_mode); + lderr(cct) << "cannot demote, mirror mode is set to: " << mode << dendl; + return -EOPNOTSUPP; + } + + cls::rbd::MirrorSnapshotState state; + r = get_last_mirror_snapshot_state(group_ioctx, group_id, &state); + if (r == -ENOENT) { + state = cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY; + r = 0; + } + if (r < 0) { + return r; + } + + if (state != cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY) { + lderr(cct) << "group " << group_name << " is not primary" << dendl; + return -EINVAL; + } + + std::vector peers; + r = cls_client::mirror_peer_list(&group_ioctx, &peers); + if (r < 0) { + lderr(cct) << "error reading mirror peers: " << cpp_strerror(r) << dendl; + return r; + } + + std::set mirror_peer_uuids; + for (auto &peer : peers) { + if (peer.mirror_peer_direction == cls::rbd::MIRROR_PEER_DIRECTION_RX) { + continue; + } + mirror_peer_uuids.insert(peer.uuid); + } + + if (mirror_peer_uuids.empty()) { + lderr(cct) << "no mirror tx peers configured for the pool" << dendl; + return -EINVAL; + } + + std::vector image_ctxs; + r = open_group_images(group_ioctx, group_id, &image_ctxs); + if (r < 0) { + return r; + } + + std::string group_snap_id = librbd::util::generate_image_id(group_ioctx); + + cls::rbd::GroupSnapshot group_snap{ + group_snap_id, + cls::rbd::MirrorGroupSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED, + mirror_peer_uuids, {}, {}}, + prepare_primary_mirror_snap_name(cct, mirror_group.global_group_id, + group_snap_id), + cls::rbd::GROUP_SNAPSHOT_STATE_INCOMPLETE}; + + for (auto image_ctx: image_ctxs) { + group_snap.snaps.emplace_back(image_ctx->md_ctx.get_id(), image_ctx->id, + CEPH_NOSNAP); + } + + std::string group_header_oid = librbd::util::group_header_name(group_id); + r = cls_client::group_snap_set(&group_ioctx, group_header_oid, group_snap); + if (r < 0) { + lderr(cct) << "failed to set group snapshot metadata: " << cpp_strerror(r) + << dendl; + close_images(&image_ctxs); + return r; + } + + // XXXMG: Requesting exclusive locks for images? + // XXXMG: notify quiesce? + + std::vector snap_ids(image_ctxs.size()); + std::vector on_finishes(image_ctxs.size()); + + for (size_t i = 0; i < on_finishes.size(); i++) { + image_demote(image_ctxs[i], group_ioctx.get_id(), group_id, group_snap_id, + &snap_ids[i], &on_finishes[i]); + } + + int ret_code = 0; + for (size_t i = 0; i < on_finishes.size(); i++) { + r = on_finishes[i].wait(); + if (r < 0) { + if (ret_code == 0) { + ret_code = r; + } + } else { + group_snap.snaps[i].snap_id = snap_ids[i]; + } + } + + if (ret_code < 0) { + r = cls_client::group_snap_remove(&group_ioctx, group_header_oid, + group_snap.id); + if (r < 0) { + lderr(cct) << "failed to remove group snapshot metadata: " + << cpp_strerror(r) << dendl; + } + } else { + group_snap.state = cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE; + r = cls_client::group_snap_set(&group_ioctx, group_header_oid, group_snap); + if (r < 0) { + lderr(cct) << "failed to update group snapshot metadata: " + << cpp_strerror(r) << dendl; + ret_code = r; + } + } + + close_images(&image_ctxs); + + return ret_code; +} + +template +int Mirror::group_resync(IoCtx& group_ioctx, const char *group_name) { + CephContext *cct = (CephContext *)group_ioctx.cct(); + ldout(cct, 20) << "io_ctx=" << &group_ioctx + << ", group_name=" << group_name << dendl; + + std::string group_id; + int r = cls_client::dir_get_id(&group_ioctx, RBD_GROUP_DIRECTORY, + group_name, &group_id); + if (r < 0) { + lderr(cct) << "error getting the group id: " << cpp_strerror(r) << dendl; + return r; + } + + cls::rbd::MirrorGroup mirror_group; + r = cls_client::mirror_group_get(&group_ioctx, group_id, &mirror_group); + if (r == -ENOENT) { + ldout(cct, 10) << "mirroring for group " << group_name + << " disabled" << dendl; + return -EINVAL; + } else if (r < 0) { + lderr(cct) << "failed to retrieve mirror group metadata: " + << cpp_strerror(r) << dendl; + return r; + } else if (mirror_group.mirror_image_mode != + cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT) { + auto mode = static_cast( + mirror_group.mirror_image_mode); + lderr(cct) << "cannot resync, mirror mode is set to: " << mode << dendl; + return -EOPNOTSUPP; + } + + cls::rbd::MirrorSnapshotState state; + r = get_last_mirror_snapshot_state(group_ioctx, group_id, &state); + if (r == -ENOENT) { + state = cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY; + r = 0; + } + if (r < 0) { + return r; + } + + if (state == cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY) { + lderr(cct) << "group " << group_name + << " is primary, cannot resync to itself" << dendl; + return -EINVAL; + } + + // TODO: implement the group resync functionality + + return 0; +} + +template +int Mirror::group_snapshot_create(IoCtx& group_ioctx, const char *group_name, + uint32_t flags, std::string *snap_id) { + CephContext *cct = (CephContext *)group_ioctx.cct(); + ldout(cct, 20) << "io_ctx=" << &group_ioctx + << ", group_name=" << group_name + << ", flags=" << flags << dendl; + + uint64_t internal_flags = 0; + int r = librbd::util::snap_create_flags_api_to_internal(cct, flags, + &internal_flags); + if (r < 0) { + return r; + } + + std::string group_id; + r = cls_client::dir_get_id(&group_ioctx, RBD_GROUP_DIRECTORY, group_name, + &group_id); + if (r < 0) { + lderr(cct) << "error getting the group id: " << cpp_strerror(r) << dendl; + return r; + } + + auto ns = group_ioctx.get_namespace(); + group_ioctx.set_namespace(""); + std::vector peers; + r = cls_client::mirror_peer_list(&group_ioctx, &peers); + if (r < 0) { + lderr(cct) << "error reading mirror peers: " << cpp_strerror(r) << dendl; + return r; + } + group_ioctx.set_namespace(ns); + + std::set mirror_peer_uuids; + for (auto &peer : peers) { + if (peer.mirror_peer_direction == cls::rbd::MIRROR_PEER_DIRECTION_RX) { + continue; + } + mirror_peer_uuids.insert(peer.uuid); + } + + if (mirror_peer_uuids.empty()) { + lderr(cct) << "no mirror tx peers configured for the pool" << dendl; + return -EINVAL; + } + + cls::rbd::MirrorGroup mirror_group; + r = cls_client::mirror_group_get(&group_ioctx, group_id, &mirror_group); + if (r == -ENOENT) { + ldout(cct, 10) << "mirroring for group " << group_name + << " disabled" << dendl; + return -EINVAL; + } else if (r < 0) { + lderr(cct) << "failed to retrieve mirror group metadata: " + << cpp_strerror(r) << dendl; + return r; + } else if (mirror_group.mirror_image_mode != + cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT) { + auto mode = static_cast( + mirror_group.mirror_image_mode); + lderr(cct) << "cannot create snapshot, mirror mode is set to: " + << mode << dendl; + return -EOPNOTSUPP; + } + + cls::rbd::MirrorSnapshotState state; + r = get_last_mirror_snapshot_state(group_ioctx, group_id, &state); + if (r == -ENOENT) { + state = cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY; + r = 0; + } + if (r < 0) { + return r; + } + + if (state != cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY) { + lderr(cct) << "group " << group_name << " is not primary" << dendl; + return -EINVAL; + } + + std::vector image_ctxs; + r = open_group_images(group_ioctx, group_id, &image_ctxs); + if (r < 0) { + return r; + } + + std::string group_snap_id = librbd::util::generate_image_id(group_ioctx); + + cls::rbd::GroupSnapshot group_snap{ + group_snap_id, + cls::rbd::MirrorGroupSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, + mirror_peer_uuids, {}, {}}, + prepare_primary_mirror_snap_name(cct, mirror_group.global_group_id, + group_snap_id), + cls::rbd::GROUP_SNAPSHOT_STATE_INCOMPLETE}; + + for (auto image_ctx: image_ctxs) { + group_snap.snaps.emplace_back(image_ctx->md_ctx.get_id(), image_ctx->id, + CEPH_NOSNAP); + } + + std::string group_header_oid = librbd::util::group_header_name(group_id); + r = cls_client::group_snap_set(&group_ioctx, group_header_oid, group_snap); + if (r < 0) { + lderr(cct) << "failed to set group snapshot metadata: " << cpp_strerror(r) + << dendl; + close_images(&image_ctxs); + return r; + } + + // XXXMG: Requesting exclusive locks for images? + + std::vector quiesce_requests; + if ((internal_flags & SNAP_CREATE_FLAG_SKIP_NOTIFY_QUIESCE) == 0) { + NoOpProgressContext prog_ctx; + r = util::notify_quiesce(image_ctxs, prog_ctx, &quiesce_requests); + if (r < 0 && + (internal_flags & SNAP_CREATE_FLAG_IGNORE_NOTIFY_QUIESCE_ERROR) == 0) { + close_images(&image_ctxs); + return r; + } + } + + std::vector snap_ids(image_ctxs.size()); + std::vector on_finishes(image_ctxs.size()); + + for (size_t i = 0; i < on_finishes.size(); i++) { + image_snapshot_create(image_ctxs[i], RBD_SNAP_CREATE_SKIP_QUIESCE, + group_ioctx.get_id(), group_id, group_snap_id, + &snap_ids[i], &on_finishes[i]); + } + + int ret_code = 0; + for (size_t i = 0; i < on_finishes.size(); i++) { + r = on_finishes[i].wait(); + if (r < 0) { + if (ret_code == 0) { + ret_code = r; + } + } else { + group_snap.snaps[i].snap_id = snap_ids[i]; + } + } + + if (ret_code < 0) { + r = cls_client::group_snap_remove(&group_ioctx, group_header_oid, + group_snap.id); + if (r < 0) { + lderr(cct) << "failed to remove group snapshot metadata: " + << cpp_strerror(r) << dendl; + } + } else { + group_snap.state = cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE; + r = cls_client::group_snap_set(&group_ioctx, group_header_oid, group_snap); + if (r < 0) { + lderr(cct) << "failed to update group snapshot metadata: " + << cpp_strerror(r) << dendl; + ret_code = r; + } + } + + if (!quiesce_requests.empty()) { + util::notify_unquiesce(image_ctxs, quiesce_requests); + } + + // TODO: try to remove created snapshots in a failure case + + close_images(&image_ctxs); + + if (ret_code < 0) { + return ret_code; + } + + *snap_id = group_snap.id; + + return 0; +} + +template +int Mirror::group_image_add(IoCtx &group_ioctx, + const std::string &group_id, + IoCtx &image_ioctx, + const std::string &image_id) { + CephContext *cct = (CephContext *)group_ioctx.cct(); + ldout(cct, 20) << "group io_ctx=" << &group_ioctx << ", group_id=" << group_id + << ", image io_ctx=" << &image_ioctx << ", image_id=" + << image_id << dendl; + + cls::rbd::MirrorGroup mirror_info; + if (cls_client::mirror_group_get(&group_ioctx, group_id, &mirror_info) < 0 || + mirror_info.state != cls::rbd::MIRROR_GROUP_STATE_ENABLED) { + return 0; + } + + // XXXMG: remove code duplication + + auto ns = group_ioctx.get_namespace(); + group_ioctx.set_namespace(""); + std::vector peers; + r = cls_client::mirror_peer_list(&group_ioctx, &peers); + if (r < 0) { + lderr(cct) << "error reading mirror peers: " << cpp_strerror(r) << dendl; + return r; + } + group_ioctx.set_namespace(ns); + + std::set mirror_peer_uuids; + for (auto &peer : peers) { + if (peer.mirror_peer_direction == cls::rbd::MIRROR_PEER_DIRECTION_RX) { + continue; + } + mirror_peer_uuids.insert(peer.uuid); + } + + if (mirror_peer_uuids.empty()) { + lderr(cct) << "no mirror tx peers configured for the pool" << dendl; + return -EINVAL; + } + + std::vector image_ctxs; + r = open_group_images(group_ioctx, group_id, &image_ctxs); + if (r < 0) { + return r; + } + + std::string group_snap_id = librbd::util::generate_image_id(group_ioctx); + + cls::rbd::GroupSnapshot group_snap{ + group_snap_id, + cls::rbd::MirrorGroupSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, + mirror_peer_uuids, {}, {}}, + prepare_primary_mirror_snap_name(cct, mirror_info.global_group_id, + group_snap_id), + cls::rbd::GROUP_SNAPSHOT_STATE_INCOMPLETE}; + + for (auto image_ctx: image_ctxs) { + group_snap.snaps.emplace_back(image_ctx->md_ctx.get_id(), image_ctx->id, + CEPH_NOSNAP); + } + + std::string group_header_oid = librbd::util::group_header_name(group_id); + r = cls_client::group_snap_set(&group_ioctx, group_header_oid, group_snap); + if (r < 0) { + lderr(cct) << "failed to set group snapshot metadata: " << cpp_strerror(r) + << dendl; + close_images(&image_ctxs); + return r; + } + + // XXXMG: notify quiesce? + + std::vector snap_ids(image_ctxs.size()); + std::vector on_finishes(image_ctxs.size()); + + auto mode = static_cast( + mirror_info.mirror_image_mode); + for (size_t i = 0; i < image_ctxs.size(); i++) { + if (image_ctxs[i]->md_ctx.get_id() == image_ioctx.get_id() && + image_ctxs[i]->id == image_id) { + r = image_enable(image_ctxs[i], group_ioctx.get_id(), group_id, + group_snap_id, mode, false, &snap_ids[i]); + on_finishes[i].complete(r); + } else { + image_snapshot_create(image_ctxs[i], RBD_SNAP_CREATE_SKIP_QUIESCE, + group_ioctx.get_id(), group_id, group_snap_id, + &snap_ids[i], &on_finishes[i]); + } + } + + int ret_code = 0; + for (size_t i = 0; i < on_finishes.size(); i++) { + r = on_finishes[i].wait(); + if (r < 0) { + if (ret_code == 0) { + ret_code = r; + } + } else { + group_snap.snaps[i].snap_id = snap_ids[i]; + } + } + + auto image_count = image_ctxs.size(); + + close_images(&image_ctxs); + + if (ret_code < 0) { + r = cls_client::group_snap_remove(&group_ioctx, group_header_oid, + group_snap.id); + if (r < 0) { + lderr(cct) << "failed to remove group snapshot metadata: " + << cpp_strerror(r) << dendl; + } + + // TODO: try to remove the created image snapshots + return ret_code; + } + + group_snap.state = cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE; + r = cls_client::group_snap_set(&group_ioctx, group_header_oid, group_snap); + if (r < 0) { + lderr(cct) << "failed to update group snapshot metadata: " + << cpp_strerror(r) << dendl; + return r; + } + + r = MirroringWatcher::notify_group_updated( + group_ioctx, cls::rbd::MIRROR_GROUP_STATE_ENABLED, group_id, + mirror_info.global_group_id, image_count); + if (r < 0) { + lderr(cct) << "failed to notify mirroring group_id=" << group_id + << " updated: " << cpp_strerror(r) << dendl; + // not fatal + } + + return 0; +} + +template +int Mirror::group_image_remove(IoCtx &group_ioctx, + const std::string &group_id, + IoCtx &image_ioctx, + const std::string &image_id) { + CephContext *cct = (CephContext *)group_ioctx.cct(); + ldout(cct, 20) << "group io_ctx=" << &group_ioctx << ", group_id=" << group_id + << ", image io_ctx=" << &image_ioctx << ", image_id=" + << image_id << dendl; + + cls::rbd::MirrorGroup mirror_info; + if (cls_client::mirror_group_get(&group_ioctx, group_id, &mirror_info) < 0 || + mirror_info.state != cls::rbd::MIRROR_GROUP_STATE_ENABLED) { + return 0; + } + + // XXXMG: remove code duplication + + auto ns = group_ioctx.get_namespace(); + group_ioctx.set_namespace(""); + std::vector peers; + r = cls_client::mirror_peer_list(&group_ioctx, &peers); + if (r < 0) { + lderr(cct) << "error reading mirror peers: " << cpp_strerror(r) << dendl; + return r; + } + group_ioctx.set_namespace(ns); + + std::set mirror_peer_uuids; + for (auto &peer : peers) { + if (peer.mirror_peer_direction == cls::rbd::MIRROR_PEER_DIRECTION_RX) { + continue; + } + mirror_peer_uuids.insert(peer.uuid); + } + + if (mirror_peer_uuids.empty()) { + lderr(cct) << "no mirror tx peers configured for the pool" << dendl; + return -EINVAL; + } + + std::vector image_ctxs; + r = open_group_images(group_ioctx, group_id, &image_ctxs); + if (r < 0) { + return r; + } + + std::string group_snap_id = librbd::util::generate_image_id(group_ioctx); + + cls::rbd::GroupSnapshot group_snap{ + group_snap_id, + cls::rbd::MirrorGroupSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, + mirror_peer_uuids, {}, {}}, + prepare_primary_mirror_snap_name(cct, mirror_info.global_group_id, + group_snap_id), + cls::rbd::GROUP_SNAPSHOT_STATE_INCOMPLETE}; + + for (auto image_ctx: image_ctxs) { + if (image_ctx->md_ctx.get_id() == image_ioctx.get_id() && + image_ctx->id == image_id) { + continue; + } + group_snap.snaps.emplace_back(image_ctx->md_ctx.get_id(), image_ctx->id, + CEPH_NOSNAP); + } + + std::string group_header_oid = librbd::util::group_header_name(group_id); + r = cls_client::group_snap_set(&group_ioctx, group_header_oid, group_snap); + if (r < 0) { + lderr(cct) << "failed to set group snapshot metadata: " << cpp_strerror(r) + << dendl; + close_images(&image_ctxs); + return r; + } + + // XXXMG: notify quiesce? + + std::vector snap_ids(image_ctxs.size()); + std::vector on_finishes(image_ctxs.size()); + + for (size_t i = 0; i < image_ctxs.size(); i++) { + if (image_ctxs[i]->md_ctx.get_id() == image_ioctx.get_id() && + image_ctxs[i]->id == image_id) { + r = image_disable(image_ctxs[i], true); + snap_ids[i] = CEPH_NOSNAP; + on_finishes[i].complete(r); + } else { + image_snapshot_create(image_ctxs[i], RBD_SNAP_CREATE_SKIP_QUIESCE, + group_ioctx.get_id(), group_id, group_snap_id, + &snap_ids[i], &on_finishes[i]); + } + } + + int ret_code = 0; + group_snap.snaps.clear(); + for (size_t i = 0; i < on_finishes.size(); i++) { + r = on_finishes[i].wait(); + if (r < 0) { + if (ret_code == 0) { + ret_code = r; + } + } + if (image_ctxs[i]->md_ctx.get_id() == image_ioctx.get_id() && + image_ctxs[i]->id == image_id) { + continue; + } + group_snap.snaps.emplace_back(image_ctxs[i]->md_ctx.get_id(), + image_ctxs[i]->id, snap_ids[i]); + } + + auto image_count = image_ctxs.size(); + + close_images(&image_ctxs); + + if (ret_code < 0) { + r = cls_client::group_snap_remove(&group_ioctx, group_header_oid, + group_snap.id); + if (r < 0) { + lderr(cct) << "failed to remove group snapshot metadata: " + << cpp_strerror(r) << dendl; + } + + // TODO: try to remove the created image snapshots + return ret_code; + } + + group_snap.state = cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE; + r = cls_client::group_snap_set(&group_ioctx, group_header_oid, group_snap); + if (r < 0) { + lderr(cct) << "failed to update group snapshot metadata: " + << cpp_strerror(r) << dendl; + return r; + } + + r = MirroringWatcher::notify_group_updated( + group_ioctx, cls::rbd::MIRROR_GROUP_STATE_ENABLED, group_id, + mirror_info.global_group_id, image_count); + if (r < 0) { + lderr(cct) << "failed to notify mirroring group_id=" << group_id + << " updated: " << cpp_strerror(r) << dendl; + // not fatal + } + + return 0; +} + +template +int Mirror::group_status_list(librados::IoCtx& io_ctx, + const std::string &start_id, size_t max, + IdToMirrorGroupStatus *groups) { + CephContext *cct = reinterpret_cast(io_ctx.cct()); + ldout(cct, 20) << dendl; + + std::map id_to_name; + int max_read = 1024; + std::string last_read = ""; + int r; + do { + std::map groups; + r = cls_client::group_dir_list(&io_ctx, RBD_GROUP_DIRECTORY, last_read, + max_read, &groups); + if (r < 0) { + if (r != -ENOENT) { + lderr(cct) << "error listing groups in directory: " + << cpp_strerror(r) << dendl; + } else { + r = 0; + } + return r; + } + for (auto &[name, group_id] : groups) { + id_to_name[group_id] = name; + } + r = groups.size(); + } while (r == max_read); + + std::map groups_internal; + std::map statuses_internal; + + r = librbd::cls_client::mirror_group_status_list(&io_ctx, start_id, max, + &groups_internal, + &statuses_internal); + if (r < 0 && r != -ENOENT) { + lderr(cct) << "failed to list mirror group statuses: " + << cpp_strerror(r) << dendl; + return r; + } + + const std::string STATUS_NOT_FOUND("status not found"); + for (auto &[group_id, info] : groups_internal) { + auto &group_name = id_to_name[group_id]; + if (group_name.empty()) { + lderr(cct) << "failed to resolve name for group " << group_id << ", " + << "using group id as name" << dendl; + group_name = group_id; + } + + mirror_group_global_status_t &status = (*groups)[group_id]; + status.name = group_name; + status.info.global_id = info.global_group_id; + status.info.mirror_image_mode = + static_cast(info.mirror_image_mode); + status.info.state = static_cast(info.state); + + bool found_local_site_status = false; + auto s_it = statuses_internal.find(group_id); + if (s_it != statuses_internal.end()) { + auto &status_internal = s_it->second; + + status.site_statuses.resize( + status_internal.mirror_group_site_statuses.size()); + size_t idx = 0; + for (auto &s : status_internal.mirror_group_site_statuses) { + mirror_group_site_status_t &site_status = status.site_statuses[idx++]; + site_status.mirror_uuid = s.mirror_uuid; + site_status.state = static_cast(s.state); + site_status.description = s.description; + for (auto &[spec, si] : s.mirror_images) { + auto &mirror_image = + site_status.mirror_images[{spec.pool_id, spec.global_image_id}]; + mirror_image.mirror_uuid = si.mirror_uuid; + mirror_image.state = static_cast(si.state); + mirror_image.description = si.description; + mirror_image.last_update = si.last_update.sec(); + mirror_image.up = si.up; + } + site_status.last_update = s.last_update.sec(); + site_status.up = s.up; + if (s.mirror_uuid == cls::rbd::MirrorGroupSiteStatus::LOCAL_MIRROR_UUID) { + found_local_site_status = true; + } + } + } + + if (!found_local_site_status) { + status.site_statuses.push_back(mirror_group_site_status_t{ + cls::rbd::MirrorGroupSiteStatus::LOCAL_MIRROR_UUID, + MIRROR_GROUP_STATUS_STATE_UNKNOWN, + STATUS_NOT_FOUND, {}, {}, false}); + } + } + + return 0; +} + +template +int Mirror::group_status_summary(librados::IoCtx& io_ctx, + MirrorGroupStatusStates *states) { + CephContext *cct = reinterpret_cast(io_ctx.cct()); + ldout(cct, 20) << dendl; + + std::vector mirror_peers; + int r = cls_client::mirror_peer_list(&io_ctx, &mirror_peers); + if (r < 0 && r != -ENOENT) { + lderr(cct) << "failed to list mirror peers: " << cpp_strerror(r) << dendl; + return r; + } + + std::map states_internal; + r = cls_client::mirror_group_status_get_summary(&io_ctx, mirror_peers, + &states_internal); + if (r < 0 && r != -ENOENT) { + lderr(cct) << "failed to get mirror status summary: " + << cpp_strerror(r) << dendl; + return r; + } + for (auto &s : states_internal) { + (*states)[static_cast(s.first)] = s.second; + } + return 0; +} + +template +int Mirror::group_instance_id_list(librados::IoCtx& io_ctx, + const std::string &start_group_id, + size_t max, + std::map *ids) { + CephContext *cct = reinterpret_cast(io_ctx.cct()); + ldout(cct, 20) << dendl; + + std::map instances; + int r = librbd::cls_client::mirror_group_instance_list( + &io_ctx, start_group_id, max, &instances); + if (r < 0 && r != -ENOENT) { + lderr(cct) << "failed to list mirror group instances: " << cpp_strerror(r) + << dendl; + return r; + } + + for (auto it : instances) { + (*ids)[it.first] = stringify(it.second.name.num()); + } + + return 0; +} + +template +int Mirror::group_info_list(librados::IoCtx& io_ctx, + mirror_image_mode_t *mode_filter, + const std::string &start_id, + size_t max, + std::map *entries) { + CephContext *cct = reinterpret_cast(io_ctx.cct()); + ldout(cct, 20) << "pool=" << io_ctx.get_pool_name() << ", mode_filter=" + << (mode_filter ? stringify(*mode_filter) : "null") + << ", start_id=" << start_id << ", max=" << max << dendl; + + std::string last_read = start_id; + entries->clear(); + + while (entries->size() < max) { + std::map groups; + std::map statuses; + + int r = librbd::cls_client::mirror_group_status_list(&io_ctx, last_read, + max, &groups, + &statuses); + if (r < 0 && r != -ENOENT) { + lderr(cct) << "failed to list mirror image statuses: " + << cpp_strerror(r) << dendl; + return r; + } + + for (auto &[group_id, group] : groups) { + auto mode = static_cast(group.mirror_image_mode); + + if ((mode_filter && mode != *mode_filter) || + group.state != cls::rbd::MIRROR_GROUP_STATE_ENABLED) { + continue; + } + + auto &info = (*entries)[group_id]; + info.global_id = group.global_group_id; + info.mirror_image_mode = mode; + info.state = static_cast(group.state); + + if (entries->size() == max) { + break; + } + } + + if (groups.size() != max) { + break; + } + + last_read = groups.rbegin()->first; + } + + return 0; +} + +template +int Mirror::group_get_info(librados::IoCtx& io_ctx, + const std::string &group_name, + mirror_group_info_t *mirror_group_info) { + CephContext *cct((CephContext *)io_ctx.cct()); + ldout(cct, 20) << "group_name=" << group_name << dendl; + + std::string group_id; + int r = cls_client::dir_get_id(&io_ctx, RBD_GROUP_DIRECTORY, group_name, + &group_id); + if (r < 0) { + lderr(cct) << "error getting id of group " << group_name << ": " + << cpp_strerror(r) << dendl; + return r; + } + + cls::rbd::MirrorGroup mirror_group; + r = cls_client::mirror_group_get(&io_ctx, group_id, &mirror_group); + if (r < 0) { + lderr(cct) << "failed to get mirror group info: " << cpp_strerror(r) + << dendl; + return r; + } + + mirror_group_info->global_id = mirror_group.global_group_id; + mirror_group_info->mirror_image_mode = + static_cast(mirror_group.mirror_image_mode); + mirror_group_info->state = + static_cast(mirror_group.state); + + return 0; +} + +template +int Mirror::group_get_status(librados::IoCtx& io_ctx, + const std::string &group_name, + mirror_group_global_status_t *status) { + CephContext *cct((CephContext *)io_ctx.cct()); + ldout(cct, 20) << "group_name=" << group_name << dendl; + + status->name = group_name; + int r = group_get_info(io_ctx, group_name, &status->info); + if (r < 0) { + lderr(cct) << "error getting info of group " << group_name << ": " + << cpp_strerror(r) << dendl; + return r; + } + + cls::rbd::MirrorGroupStatus status_internal; + r = cls_client::mirror_group_status_get(&io_ctx, status->info.global_id, + &status_internal); + if (r < 0 && r != -ENOENT) { + lderr(cct) << "failed to get mirror group status: " << cpp_strerror(r) + << dendl; + return r; + } + + status->site_statuses.resize( + status_internal.mirror_group_site_statuses.size()); + size_t idx = 0; + for (auto &s : status_internal.mirror_group_site_statuses) { + mirror_group_site_status_t &site_status = status->site_statuses[idx++]; + site_status.mirror_uuid = s.mirror_uuid; + site_status.state = static_cast(s.state); + site_status.description = s.description; + for (auto &[spec, si] : s.mirror_images) { + auto &mirror_image = site_status.mirror_images[{spec.pool_id, + spec.global_image_id}]; + mirror_image.mirror_uuid = si.mirror_uuid; + mirror_image.state = static_cast(si.state); + mirror_image.description = si.description; + mirror_image.last_update = si.last_update.sec(); + mirror_image.up = si.up; + } + site_status.last_update = s.last_update.sec(); + site_status.up = s.up; + } + return 0; +} + +template +int Mirror::group_get_instance_id(librados::IoCtx& io_ctx, + const std::string &group_name, + std::string *instance_id) { + CephContext *cct((CephContext *)io_ctx.cct()); + ldout(cct, 20) << "group_name=" << group_name << dendl; + + std::string group_id; + int r = cls_client::dir_get_id(&io_ctx, RBD_GROUP_DIRECTORY, group_name, + &group_id); + if (r < 0) { + lderr(cct) << "error getting id of group " << group_name << ": " + << cpp_strerror(r) << dendl; + return r; + } + + cls::rbd::MirrorGroup mirror_group; + r = cls_client::mirror_group_get(&io_ctx, group_id, &mirror_group); + if (r < 0 && r != -ENOENT) { + lderr(cct) << "failed to retrieve mirroring state: " << cpp_strerror(r) + << dendl; + return r; + } else if (mirror_group.state != cls::rbd::MIRROR_GROUP_STATE_ENABLED) { + lderr(cct) << "mirroring is not currently enabled" << dendl; + return -EINVAL; + } + + entity_inst_t instance; + r = cls_client::mirror_group_instance_get(&io_ctx, + mirror_group.global_group_id, + &instance); + if (r < 0) { + if (r != -ENOENT && r != -ESTALE) { + lderr(cct) << "failed to get mirror group instance: " << cpp_strerror(r) + << dendl; + } + return r; + } + + *instance_id = stringify(instance.name.num()); + return 0; +} + } // namespace api } // namespace librbd diff --git a/src/librbd/api/Mirror.h b/src/librbd/api/Mirror.h index 6e84247b67846..66fcb2b010395 100644 --- a/src/librbd/api/Mirror.h +++ b/src/librbd/api/Mirror.h @@ -23,7 +23,9 @@ struct Mirror { typedef std::map Attributes; typedef std::map IdToMirrorImageGlobalStatus; + typedef std::map IdToMirrorGroupStatus; typedef std::map MirrorImageStatusStates; + typedef std::map MirrorGroupStatusStates; static int site_name_get(librados::Rados& rados, std::string* name); static int site_name_set(librados::Rados& rados, const std::string& name); @@ -67,6 +69,7 @@ struct Mirror { static int peer_site_set_attributes(librados::IoCtx& io_ctx, const std::string &uuid, const Attributes& attributes); + static const char *pool_or_namespace(ImageCtxT *ictx); static int image_global_status_list(librados::IoCtx& io_ctx, const std::string &start_id, size_t max, @@ -87,11 +90,22 @@ struct Mirror { static int image_enable(ImageCtxT *ictx, mirror_image_mode_t mode, bool relax_same_pool_parent_check); + static int image_enable(ImageCtxT *ictx, + const std::string &group_snap_id, + mirror_image_mode_t mode, + bool relax_same_pool_parent_check, + uint64_t *snap_id); static int image_disable(ImageCtxT *ictx, bool force); static int image_promote(ImageCtxT *ictx, bool force); static void image_promote(ImageCtxT *ictx, bool force, Context *on_finish); + static void image_promote(ImageCtxT *ictx, + const std::string &group_snap_id, bool force, + uint64_t *snap_id, Context *on_finish); static int image_demote(ImageCtxT *ictx); static void image_demote(ImageCtxT *ictx, Context *on_finish); + static void image_demote(ImageCtxT *ictx, + const std::string &group_snap_id, uint64_t *snap_id, + Context *on_finish); static int image_resync(ImageCtxT *ictx); static int image_get_info(ImageCtxT *ictx, mirror_image_info_t *mirror_image_info); @@ -121,6 +135,51 @@ struct Mirror { uint64_t *snap_id); static void image_snapshot_create(ImageCtxT *ictx, uint32_t flags, uint64_t *snap_id, Context *on_finish); + static void image_snapshot_create(ImageCtxT *ictx, uint32_t flags, + const std::string &group_snap_id, + uint64_t *snap_id, Context *on_finish); + + static int group_list(IoCtx &io_ctx, std::vector *names); + static int group_enable(IoCtx &group_ioctx, const char *group_name, + mirror_image_mode_t group_image_mode); + static int group_disable(IoCtx &group_ioctx, const char *group_name, + bool force); + static int group_promote(IoCtx &group_ioctx, const char *group_name, + bool force); + static int group_demote(IoCtx &group_ioctx, const char *group_name); + static int group_resync(IoCtx &group_ioctx, const char *group_name); + static int group_snapshot_create(IoCtx& group_ioctx, const char *group_name, + uint32_t flags, std::string *snap_id); + + static int group_image_add(IoCtx &group_ioctx, const std::string &group_id, + IoCtx &image_ioctx, const std::string &image_id); + static int group_image_remove(IoCtx &group_ioctx, const std::string &group_id, + IoCtx &image_ioctx, const std::string &image_id); + + static int group_status_list(librados::IoCtx& io_ctx, + const std::string &start_id, size_t max, + IdToMirrorGroupStatus *groups); + static int group_status_summary(librados::IoCtx& io_ctx, + MirrorGroupStatusStates *states); + static int group_instance_id_list(librados::IoCtx& io_ctx, + const std::string &start_group_id, + size_t max, + std::map *ids); + static int group_info_list(librados::IoCtx& io_ctx, + mirror_image_mode_t *mode_filter, + const std::string &start_id, + size_t max, + std::map *entries); + static int group_get_info(librados::IoCtx& io_ctx, + const std::string &group_name, + mirror_group_info_t *mirror_group_info); + static int group_get_status(librados::IoCtx& io_ctx, + const std::string &group_name, + mirror_group_global_status_t *status); + static int group_get_instance_id(librados::IoCtx& io_ctx, + const std::string &group_name, + std::string *instance_id); }; } // namespace api diff --git a/src/librbd/api/Utils.cc b/src/librbd/api/Utils.cc index 056b6b435f628..f5a7c57edf989 100644 --- a/src/librbd/api/Utils.cc +++ b/src/librbd/api/Utils.cc @@ -2,8 +2,13 @@ // vim: ts=8 sw=2 smarttab #include "librbd/api/Utils.h" +#include "cls/rbd/cls_rbd_client.h" +#include "common/Cond.h" #include "common/dout.h" +#include "librbd/ImageState.h" +#include "librbd/ImageWatcher.h" +#include "librbd/Utils.h" #if defined(HAVE_LIBCRYPTSETUP) #include "librbd/crypto/luks/LUKSEncryptionFormat.h" #endif @@ -92,6 +97,192 @@ int create_encryption_format( return 0; } +template +int notify_quiesce(std::vector &ictxs, ProgressContext &prog_ctx, + std::vector *requests) { + int image_count = ictxs.size(); + if (image_count == 0) { + return 0; + } + + std::vector on_finishes(image_count); + + requests->resize(image_count); + for (int i = 0; i < image_count; ++i) { + auto ictx = ictxs[i]; + + ictx->image_watcher->notify_quiesce(&(*requests)[i], prog_ctx, + &on_finishes[i]); + } + + int ret_code = 0; + for (int i = 0; i < image_count; ++i) { + int r = on_finishes[i].wait(); + if (r < 0) { + ret_code = r; + } + } + + if (ret_code != 0) { + notify_unquiesce(ictxs, *requests); + } + + return ret_code; +} + +template +void notify_unquiesce(std::vector &ictxs, + const std::vector &requests) { + if (requests.empty()) { + return; + } + + ceph_assert(requests.size() == ictxs.size()); + int image_count = ictxs.size(); + std::vector on_finishes(image_count); + + for (int i = 0; i < image_count; ++i) { + ImageCtx *ictx = ictxs[i]; + + ictx->image_watcher->notify_unquiesce(requests[i], &on_finishes[i]); + } + + for (int i = 0; i < image_count; ++i) { + on_finishes[i].wait(); + } +} + +template +librados::snap_t get_group_snap_id( + I *ictx, const cls::rbd::SnapshotNamespace& in_snap_namespace) { + ceph_assert(ceph_mutex_is_locked(ictx->image_lock)); + auto it = ictx->snap_ids.lower_bound({cls::rbd::GroupImageSnapshotNamespace{}, + ""}); + for (; it != ictx->snap_ids.end(); ++it) { + if (it->first.first == in_snap_namespace) { + return it->second; + } else if (std::get_if( + &it->first.first) == nullptr) { + break; + } + } + return CEPH_NOSNAP; +} + +int group_snap_remove(librados::IoCtx& group_ioctx, const std::string& group_id, + const cls::rbd::GroupSnapshot& group_snap) { + CephContext *cct = (CephContext *)group_ioctx.cct(); + std::string group_header_oid = librbd::util::group_header_name(group_id); + std::vector on_finishes; + int r, ret_code; + + std::vector ictxs; + + cls::rbd::GroupImageSnapshotNamespace snap_namespace{group_ioctx.get_id(), + group_id, group_snap.id}; + + ldout(cct, 20) << "Removing snapshot with group snap_id: " << group_snap.id + << ", and group_id: " << group_id << dendl; + int snap_count = group_snap.snaps.size(); + + for (int i = 0; i < snap_count; ++i) { + librbd::IoCtx image_io_ctx; + r = librbd::util::create_ioctx(group_ioctx, "image", + group_snap.snaps[i].pool, {}, &image_io_ctx); + if (r < 0) { + ret_code = r; + goto finish; + } + + librbd::ImageCtx* image_ctx = new ImageCtx("", group_snap.snaps[i].image_id, + nullptr, image_io_ctx, false); + + C_SaferCond* on_finish = new C_SaferCond; + + image_ctx->state->open(0, on_finish); + + ictxs.push_back(image_ctx); + on_finishes.push_back(on_finish); + } + + ret_code = 0; + for (int i = 0; i < snap_count; ++i) { + r = on_finishes[i]->wait(); + delete on_finishes[i]; + if (r < 0) { + ictxs[i] = nullptr; + if (r != -ENOENT) { + ret_code = r; + } + } + } + if (ret_code != 0) { + goto finish; + } + + ldout(cct, 20) << "Opened participating images. " << + "Deleting snapshots themselves." << dendl; + + for (int i = 0; i < snap_count; ++i) { + ImageCtx *ictx = ictxs[i]; + on_finishes[i] = new C_SaferCond; + + if (ictx == nullptr) { + on_finishes[i]->complete(0); + continue; + } + + std::string snap_name; + ictx->image_lock.lock_shared(); + auto snap_id = get_group_snap_id(ictx, snap_namespace); + r = ictx->get_snap_name(snap_id, &snap_name); + ictx->image_lock.unlock_shared(); + + if (r >= 0) { + ldout(cct, 20) << "removing individual snapshot: " << snap_name + << ", on image: " << ictx->name << dendl; + ictx->operations->snap_remove(snap_namespace, snap_name, on_finishes[i]); + } else { + // We are ok to ignore missing image snapshots. The snapshot could have + // been inconsistent in the first place. + on_finishes[i]->complete(0); + } + } + + for (int i = 0; i < snap_count; ++i) { + r = on_finishes[i]->wait(); + delete on_finishes[i]; + if (r < 0 && r != -ENOENT) { + // if previous attempts to remove this snapshot failed then the image's + // snapshot may not exist + lderr(cct) << "Failed deleting image snapshot. Ret code: " << r << dendl; + ret_code = r; + } + } + + if (ret_code != 0) { + goto finish; + } + + ldout(cct, 20) << "Removed images snapshots removing snapshot record." + << dendl; + + r = cls_client::group_snap_remove(&group_ioctx, group_header_oid, + group_snap.id); + if (r < 0) { + ret_code = r; + goto finish; + } + +finish: + for (int i = 0; i < snap_count; ++i) { + if (ictxs[i] != nullptr) { + ictxs[i]->state->close(); + } + } + return ret_code; +} + } // namespace util } // namespace api } // namespace librbd @@ -100,3 +291,14 @@ template int librbd::api::util::create_encryption_format( CephContext* cct, encryption_format_t format, encryption_options_t opts, size_t opts_size, bool c_api, crypto::EncryptionFormat** result_format); + +template int librbd::api::util::notify_quiesce( + std::vector &ictxs, librbd::ProgressContext &prog_ctx, + std::vector *requests); +template void librbd::api::util::notify_unquiesce( + std::vector &ictxs, + const std::vector &requests); + +template librados::snap_t librbd::api::util::get_group_snap_id( + librbd::ImageCtx *ictx, + const cls::rbd::SnapshotNamespace &in_snap_namespace); diff --git a/src/librbd/api/Utils.h b/src/librbd/api/Utils.h index 8f8c22290a992..cb8ef6d714fc1 100644 --- a/src/librbd/api/Utils.h +++ b/src/librbd/api/Utils.h @@ -21,6 +21,19 @@ int create_encryption_format( encryption_options_t opts, size_t opts_size, bool c_api, crypto::EncryptionFormat** result_format); +template +int notify_quiesce(std::vector &ictxs, ProgressContext &prog_ctx, + std::vector *requests); +template +void notify_unquiesce(std::vector &ictxs, + const std::vector &requests); + +template +librados::snap_t get_group_snap_id( + ImageCtxT *ictx, const cls::rbd::SnapshotNamespace& in_snap_namespace); +int group_snap_remove(librados::IoCtx& group_ioctx, const std::string& group_id, + const cls::rbd::GroupSnapshot& group_snap); + } // namespace util } // namespace api } // namespace librbd diff --git a/src/librbd/librbd.cc b/src/librbd/librbd.cc index 01ea33a5bd0bc..57b23410df56c 100644 --- a/src/librbd/librbd.cc +++ b/src/librbd/librbd.cc @@ -338,6 +338,21 @@ int mirror_image_global_status_cpp_to_c( #pragma GCC diagnostic pop +void mirror_image_site_status_cpp_to_c( + const librbd::mirror_image_site_status_t &cpp_status, + rbd_mirror_image_site_status_t *c_status) { + c_status->mirror_uuid = strdup(cpp_status.mirror_uuid.c_str()); + c_status->state = cpp_status.state; + c_status->description = strdup(cpp_status.description.c_str()); + c_status->last_update = cpp_status.last_update; + c_status->up = cpp_status.up; +} + +void mirror_image_site_status_cleanup(rbd_mirror_image_site_status_t *status) { + free(status->mirror_uuid); + free(status->description); +} + void mirror_image_global_status_cpp_to_c( const librbd::mirror_image_global_status_t &cpp_status, rbd_mirror_image_global_status_t *c_status) { @@ -349,14 +364,75 @@ void mirror_image_global_status_cpp_to_c( cpp_status.site_statuses.size(), sizeof(rbd_mirror_image_site_status_t)); auto idx = 0U; - for (auto it = cpp_status.site_statuses.begin(); - it != cpp_status.site_statuses.end(); ++it) { - auto& s_status = c_status->site_statuses[idx++]; - s_status.mirror_uuid = strdup(it->mirror_uuid.c_str()); - s_status.state = it->state; - s_status.description = strdup(it->description.c_str()); - s_status.last_update = it->last_update; - s_status.up = it->up; + for (auto &site_status : cpp_status.site_statuses) { + mirror_image_site_status_cpp_to_c(site_status, + &c_status->site_statuses[idx++]); + } +} + +void mirror_group_info_cpp_to_c(const librbd::mirror_group_info_t &cpp_info, + rbd_mirror_group_info_t *c_info) { + c_info->global_id = strdup(cpp_info.global_id.c_str()); + c_info->mirror_image_mode = cpp_info.mirror_image_mode; + c_info->state = cpp_info.state; + c_info->primary = cpp_info.primary; +} + +void mirror_group_site_status_cpp_to_c( + const librbd::mirror_group_site_status_t &cpp_status, + rbd_mirror_group_site_status_t *c_status) { + + c_status->mirror_uuid = strdup(cpp_status.mirror_uuid.c_str()); + c_status->state = cpp_status.state; + c_status->description = strdup(cpp_status.description.c_str()); + + c_status->mirror_image_count = cpp_status.mirror_images.size(); + c_status->mirror_image_pool_ids = (int64_t *)calloc( + c_status->mirror_image_count, sizeof(int64_t)); + c_status->mirror_image_global_ids = (char **)calloc( + c_status->mirror_image_count, sizeof(char *)); + c_status->mirror_images = (rbd_mirror_image_site_status_t*)calloc( + c_status->mirror_image_count, sizeof(rbd_mirror_image_site_status_t)); + + auto idx = 0U; + for (auto &[p, image] : cpp_status.mirror_images) { + c_status->mirror_image_pool_ids[idx] = p.first; + c_status->mirror_image_global_ids[idx] = strdup(p.second.c_str()); + mirror_image_site_status_cpp_to_c(image, &c_status->mirror_images[idx++]); + } + + c_status->last_update = cpp_status.last_update; + c_status->up = cpp_status.up; +} + +void mirror_group_site_status_cleanup(rbd_mirror_group_site_status_t *status) { + free(status->mirror_uuid); + free(status->description); + + for (auto idx = 0U; idx < status->mirror_image_count; ++idx) { + free(status->mirror_image_global_ids[idx]); + mirror_image_site_status_cleanup(&status->mirror_images[idx]); + } + + free(status->mirror_image_pool_ids); + free(status->mirror_image_global_ids); + free(status->mirror_images); +} + +void mirror_group_status_cpp_to_c( + const librbd::mirror_group_global_status_t &cpp_status, + rbd_mirror_group_global_status_t *c_status) { + c_status->name = strdup(cpp_status.name.c_str()); + mirror_group_info_cpp_to_c(cpp_status.info, &c_status->info); + + c_status->site_statuses_count = cpp_status.site_statuses.size(); + c_status->site_statuses = (rbd_mirror_group_site_status_t*)calloc( + cpp_status.site_statuses.size(), sizeof(rbd_mirror_group_site_status_t)); + + auto idx = 0U; + for (auto &cpp_site_status : cpp_status.site_statuses) { + mirror_group_site_status_cpp_to_c(cpp_site_status, + &c_status->site_statuses[idx++]); } } @@ -1303,6 +1379,33 @@ namespace librbd { max, entries); } + int RBD::mirror_group_info_list( + IoCtx& io_ctx, mirror_image_mode_t *mode_filter, + const std::string &start_id, size_t max, + std::map *entries) { + return librbd::api::Mirror<>::group_info_list(io_ctx, mode_filter, start_id, + max, entries); + } + + int RBD::mirror_group_global_status_list( + IoCtx& io_ctx, const std::string &start_id, size_t max, + std::map *groups) { + return librbd::api::Mirror<>::group_status_list(io_ctx, start_id, max, + groups); + } + + int RBD::mirror_group_status_summary( + IoCtx& io_ctx, std::map *states) { + return librbd::api::Mirror<>::group_status_summary(io_ctx, states); + } + + int RBD::mirror_group_instance_id_list( + IoCtx& io_ctx, const std::string &start_id, size_t max, + std::map *instance_ids) { + return librbd::api::Mirror<>::group_instance_id_list(io_ctx, start_id, max, + instance_ids); + } + int RBD::group_create(IoCtx& io_ctx, const char *group_name) { TracepointProvider::initialize(get_cct(io_ctx)); @@ -1541,6 +1644,70 @@ namespace librbd { return r; } + int RBD::mirror_group_list(IoCtx& io_ctx, std::vector *names) { + return librbd::api::Mirror<>::group_list(io_ctx, names); + } + + int RBD::mirror_group_enable(IoCtx& group_ioctx, const char *group_name, + mirror_image_mode_t mirror_image_mode) { + return librbd::api::Mirror<>::group_enable(group_ioctx, group_name, + mirror_image_mode); + } + + int RBD::mirror_group_disable(IoCtx& group_ioctx, const char *group_name, + bool force) { + return librbd::api::Mirror<>::group_disable(group_ioctx, group_name, force); + } + + int RBD::mirror_group_promote(IoCtx& group_ioctx, const char *group_name, + bool force) { + return librbd::api::Mirror<>::group_promote(group_ioctx, group_name, force); + } + + int RBD::mirror_group_demote(IoCtx& group_ioctx, const char *group_name) { + return librbd::api::Mirror<>::group_demote(group_ioctx, group_name); + } + + int RBD::mirror_group_resync(IoCtx& group_ioctx, const char *group_name) { + return librbd::api::Mirror<>::group_resync(group_ioctx, group_name); + } + + int RBD::mirror_group_create_snapshot(IoCtx& group_ioctx, + const char *group_name, + uint32_t flags, std::string *snap_id) { + return librbd::api::Mirror<>::group_snapshot_create(group_ioctx, group_name, + flags, snap_id); + } + + int RBD::mirror_group_get_info(IoCtx& group_ioctx, const char *group_name, + mirror_group_info_t *mirror_group_info, + size_t info_size) { + if (sizeof(mirror_group_info_t) != info_size) { + return -ERANGE; + } + + return librbd::api::Mirror<>::group_get_info(group_ioctx, group_name, + mirror_group_info); + } + + int RBD::mirror_group_get_status(IoCtx& group_ioctx, const char *group_name, + mirror_group_global_status_t *status, + size_t status_size) { + if (sizeof(mirror_group_global_status_t) != status_size) { + return -ERANGE; + } + + return librbd::api::Mirror<>::group_get_status(group_ioctx, group_name, + status); + } + + int RBD::mirror_group_get_instance_id(IoCtx& group_ioctx, + const char *group_name, + std::string *instance_id) { + return librbd::api::Mirror<>::group_get_instance_id(group_ioctx, group_name, + instance_id); + } + int RBD::pool_metadata_get(IoCtx& ioctx, const std::string &key, std::string *value) { @@ -3703,8 +3870,7 @@ extern "C" void rbd_mirror_image_global_status_cleanup( free(global_status->name); rbd_mirror_image_get_info_cleanup(&global_status->info); for (auto idx = 0U; idx < global_status->site_statuses_count; ++idx) { - free(global_status->site_statuses[idx].mirror_uuid); - free(global_status->site_statuses[idx].description); + mirror_image_site_status_cleanup(&global_status->site_statuses[idx]); } free(global_status->site_statuses); } @@ -3834,19 +4000,21 @@ extern "C" int rbd_mirror_image_status_summary(rados_ioctx_t p, librados::IoCtx io_ctx; librados::IoCtx::from_rados_ioctx_t(p, io_ctx); - std::map states_; - int r = librbd::api::Mirror<>::image_status_summary(io_ctx, &states_); + std::map cpp_states; + int r = librbd::api::Mirror<>::image_status_summary(io_ctx, &cpp_states); if (r < 0) { return r; } + if (cpp_states.size() > *maxlen) { + *maxlen = cpp_states.size(); + return -ERANGE; + } + size_t i = 0; - for (auto &it : states_) { - if (i == *maxlen) { - return -ERANGE; - } - states[i] = it.first; - counts[i] = it.second; + for (auto &[state, count] : cpp_states) { + states[i] = state; + counts[i] = count; i++; } *maxlen = i; @@ -3922,6 +4090,145 @@ extern "C" void rbd_mirror_image_info_list_cleanup( } } +extern "C" int rbd_mirror_group_global_status_list(rados_ioctx_t p, + const char *start_id, size_t max, char **group_ids, + rbd_mirror_group_global_status_t *groups, size_t *len) { + librados::IoCtx io_ctx; + librados::IoCtx::from_rados_ioctx_t(p, io_ctx); + std::map cpp_groups; + + int r = librbd::api::Mirror<>::group_status_list(io_ctx, start_id, max, + &cpp_groups); + if (r < 0) { + return r; + } + + size_t i = 0; + for (auto &[group_id, cpp_group] : cpp_groups) { + ceph_assert(i < max); + group_ids[i] = strdup(group_id.c_str()); + mirror_group_status_cpp_to_c(cpp_group, &groups[i]); + i++; + } + *len = i; + return 0; +} + +extern "C" void rbd_mirror_group_status_cleanup( + rbd_mirror_group_global_status_t *status) { + free(status->name); + rbd_mirror_group_get_info_cleanup(&status->info); + for (auto idx = 0U; idx < status->site_statuses_count; ++idx) { + mirror_group_site_status_cleanup(&status->site_statuses[idx]); + } + free(status->site_statuses); +} + +extern "C" void rbd_mirror_group_global_status_list_cleanup( + char **group_ids, rbd_mirror_group_global_status_t *groups, size_t len) { + for (size_t i = 0; i < len; i++) { + free(group_ids[i]); + rbd_mirror_group_status_cleanup(&groups[i]); + } +} + +extern "C" int rbd_mirror_group_status_summary( + rados_ioctx_t p, rbd_mirror_group_status_state_t *states, int *counts, + size_t *maxlen) { + librados::IoCtx io_ctx; + librados::IoCtx::from_rados_ioctx_t(p, io_ctx); + + std::map cpp_states; + int r = librbd::api::Mirror<>::group_status_summary(io_ctx, &cpp_states); + if (r < 0) { + return r; + } + + if (cpp_states.size() > *maxlen) { + *maxlen = cpp_states.size(); + return -ERANGE; + } + + size_t i = 0; + for (auto &[state, count] : cpp_states) { + states[i] = state; + counts[i] = count; + i++; + } + *maxlen = i; + return 0; +} + +extern "C" int rbd_mirror_group_instance_id_list(rados_ioctx_t p, + const char *start_id, + size_t max, char **group_ids, + char **instance_ids, + size_t *len) { + librados::IoCtx io_ctx; + librados::IoCtx::from_rados_ioctx_t(p, io_ctx); + std::map cpp_instance_ids; + + int r = librbd::api::Mirror<>::group_instance_id_list(io_ctx, start_id, max, + &cpp_instance_ids); + if (r < 0) { + return r; + } + + ceph_assert(cpp_instance_ids.size() <= max); + + size_t i = 0; + for (auto &[group_id, instance_id] : cpp_instance_ids) { + group_ids[i] = strdup(group_id.c_str()); + instance_ids[i] = strdup(instance_id.c_str()); + i++; + } + *len = i; + return 0; +} + +extern "C" void rbd_mirror_group_instance_id_list_cleanup(char **group_ids, + char **instance_ids, + size_t len) { + for (size_t i = 0; i < len; i++) { + free(group_ids[i]); + free(instance_ids[i]); + } +} + +extern "C" int rbd_mirror_group_info_list( + rados_ioctx_t p, rbd_mirror_image_mode_t *mode_filter, + const char *start_id, size_t max, char **group_ids, + rbd_mirror_group_info_t *info_entries, size_t *num_entries) { + librados::IoCtx io_ctx; + librados::IoCtx::from_rados_ioctx_t(p, io_ctx); + std::map cpp_entries; + + int r = librbd::api::Mirror<>::group_info_list(io_ctx, mode_filter, start_id, + max, &cpp_entries); + if (r < 0) { + return r; + } + + ceph_assert(cpp_entries.size() <= max); + + for (auto &[group_id, info] : cpp_entries) { + *(group_ids++) = strdup(group_id.c_str()); + mirror_group_info_cpp_to_c(info, info_entries++); + } + *num_entries = cpp_entries.size(); + + return 0; +} + +CEPH_RBD_API void rbd_mirror_group_info_list_cleanup( + char **group_ids, rbd_mirror_group_info_t *info_entries, + size_t num_entries) { + for (size_t i = 0; i < num_entries; i++) { + free(group_ids[i]); + rbd_mirror_group_get_info_cleanup(&info_entries[i]); + } +} + /* helpers */ extern "C" void rbd_image_spec_cleanup(rbd_image_spec_t *image) @@ -7543,6 +7850,180 @@ extern "C" int rbd_group_snap_rollback_with_progress(rados_ioctx_t group_p, return r; } +extern "C" int rbd_mirror_group_list(rados_ioctx_t p, char *names, + size_t *size) +{ + librados::IoCtx io_ctx; + librados::IoCtx::from_rados_ioctx_t(p, io_ctx); + + std::vector cpp_names; + int r = librbd::api::Mirror<>::group_list(io_ctx, &cpp_names); + + if (r < 0) { + return r; + } + + size_t expected_size = 0; + + for (size_t i = 0; i < cpp_names.size(); i++) { + expected_size += cpp_names[i].size() + 1; + } + if (*size < expected_size) { + *size = expected_size; + return -ERANGE; + } + + if (names == NULL) { + return -EINVAL; + } + + for (int i = 0; i < (int)cpp_names.size(); i++) { + const char* name = cpp_names[i].c_str(); + strcpy(names, name); + names += strlen(names) + 1; + } + return expected_size; +} + +extern "C" int rbd_mirror_group_enable(rados_ioctx_t group_p, + const char *group_name, + rbd_mirror_image_mode_t mirror_image_mode) +{ + librados::IoCtx group_ioctx; + librados::IoCtx::from_rados_ioctx_t(group_p, group_ioctx); + + return librbd::api::Mirror<>::group_enable(group_ioctx, group_name, + mirror_image_mode); +} + +extern "C" int rbd_mirror_group_disable(rados_ioctx_t group_p, + const char *group_name, bool force) +{ + librados::IoCtx group_ioctx; + librados::IoCtx::from_rados_ioctx_t(group_p, group_ioctx); + + return librbd::api::Mirror<>::group_disable(group_ioctx, group_name, force); +} + +extern "C" int rbd_mirror_group_promote(rados_ioctx_t group_p, + const char *group_name, bool force) +{ + librados::IoCtx group_ioctx; + librados::IoCtx::from_rados_ioctx_t(group_p, group_ioctx); + + return librbd::api::Mirror<>::group_promote(group_ioctx, group_name, force); +} + +extern "C" int rbd_mirror_group_demote(rados_ioctx_t group_p, + const char *group_name) +{ + librados::IoCtx group_ioctx; + librados::IoCtx::from_rados_ioctx_t(group_p, group_ioctx); + + return librbd::api::Mirror<>::group_demote(group_ioctx, group_name); +} + +extern "C" int rbd_mirror_group_resync(rados_ioctx_t group_p, + const char *group_name) +{ + librados::IoCtx group_ioctx; + librados::IoCtx::from_rados_ioctx_t(group_p, group_ioctx); + + return librbd::api::Mirror<>::group_resync(group_ioctx, group_name); +} + +extern "C" int rbd_mirror_group_create_snapshot(rados_ioctx_t group_p, + const char *group_name, + uint32_t flags, + char **snap_id) +{ + librados::IoCtx group_ioctx; + librados::IoCtx::from_rados_ioctx_t(group_p, group_ioctx); + + std::string cpp_snap_id; + int r = librbd::api::Mirror<>::group_snapshot_create(group_ioctx, group_name, + flags, &cpp_snap_id); + if (r < 0) { + return r; + } + + if (snap_id != NULL) { + *snap_id = strdup(cpp_snap_id.c_str()); + } + + return 0; +} + +extern "C" int rbd_mirror_group_get_info( + rados_ioctx_t group_p, const char *group_name, + rbd_mirror_group_info_t *mirror_group_info, size_t info_size) { + librados::IoCtx group_ioctx; + librados::IoCtx::from_rados_ioctx_t(group_p, group_ioctx); + + if (sizeof(rbd_mirror_group_info_t) != info_size) { + return -ERANGE; + } + + librbd::mirror_group_info_t cpp_mirror_group; + int r = librbd::api::Mirror<>::group_get_info(group_ioctx, group_name, + &cpp_mirror_group); + if (r < 0) { + return r; + } + + mirror_group_info_cpp_to_c(cpp_mirror_group, mirror_group_info); + return 0; +} + +extern "C" void rbd_mirror_group_get_info_cleanup( + rbd_mirror_group_info_t *mirror_group_info) { + free(mirror_group_info->global_id); +} + +extern "C" int rbd_mirror_group_get_global_status( + rados_ioctx_t group_p, const char *group_name, + rbd_mirror_group_global_status_t *status, size_t status_size) { + librados::IoCtx group_ioctx; + librados::IoCtx::from_rados_ioctx_t(group_p, group_ioctx); + + if (sizeof(rbd_mirror_group_global_status_t) != status_size) { + return -ERANGE; + } + + librbd::mirror_group_global_status_t cpp_status; + int r = librbd::api::Mirror<>::group_get_status(group_ioctx, group_name, + &cpp_status); + if (r < 0) { + return r; + } + + mirror_group_status_cpp_to_c(cpp_status, status); + return 0; +} + +extern "C" int rbd_mirror_group_get_instance_id( + rados_ioctx_t group_p, const char *group_name, char *instance_id, + size_t *instance_id_max_length) { + librados::IoCtx group_ioctx; + librados::IoCtx::from_rados_ioctx_t(group_p, group_ioctx); + + std::string cpp_instance_id; + int r = librbd::api::Mirror<>::group_get_instance_id(group_ioctx, group_name, + &cpp_instance_id); + if (r < 0) { + return r; + } + + if (cpp_instance_id.size() >= *instance_id_max_length) { + *instance_id_max_length = cpp_instance_id.size() + 1; + return -ERANGE; + } + + strcpy(instance_id, cpp_instance_id.c_str()); + *instance_id_max_length = cpp_instance_id.size() + 1; + return 0; +} + extern "C" int rbd_snap_get_namespace_type(rbd_image_t image, uint64_t snap_id, rbd_snap_namespace_type_t *namespace_type) { diff --git a/src/librbd/mirror/DemoteRequest.cc b/src/librbd/mirror/DemoteRequest.cc index fd24bb6989601..408d35d02c0b0 100644 --- a/src/librbd/mirror/DemoteRequest.cc +++ b/src/librbd/mirror/DemoteRequest.cc @@ -138,7 +138,8 @@ void DemoteRequest::demote() { Journal::demote(&m_image_ctx, ctx); } else if (m_mirror_image.mode == cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT) { auto req = mirror::snapshot::DemoteRequest::create( - &m_image_ctx, m_mirror_image.global_image_id, ctx); + &m_image_ctx, m_mirror_image.global_image_id, m_group_pool_id, m_group_id, + m_group_snap_id, m_snap_id, ctx); req->send(); } else { lderr(cct) << "unknown image mirror mode: " << m_mirror_image.mode << dendl; diff --git a/src/librbd/mirror/DemoteRequest.h b/src/librbd/mirror/DemoteRequest.h index ab92390684524..6c357983292dc 100644 --- a/src/librbd/mirror/DemoteRequest.h +++ b/src/librbd/mirror/DemoteRequest.h @@ -18,12 +18,23 @@ namespace mirror { template class DemoteRequest { public: + static DemoteRequest *create(ImageCtxT &image_ctx, int64_t group_pool_id, + const std::string &group_id, + const std::string &group_snap_id, + uint64_t *snap_id, Context *on_finish) { + return new DemoteRequest(image_ctx, group_pool_id, group_id, + group_snap_id, snap_id, on_finish); + } static DemoteRequest *create(ImageCtxT &image_ctx, Context *on_finish) { - return new DemoteRequest(image_ctx, on_finish); + return new DemoteRequest(image_ctx, -1, {}, {}, nullptr, on_finish); } - DemoteRequest(ImageCtxT &image_ctx, Context *on_finish) - : m_image_ctx(image_ctx), m_on_finish(on_finish) { + DemoteRequest(ImageCtxT &image_ctx, int64_t group_pool_id, + const std::string &group_id, const std::string &group_snap_id, + uint64_t *snap_id, Context *on_finish) + : m_image_ctx(image_ctx), m_group_pool_id(group_pool_id), + m_group_id(group_id), m_group_snap_id(group_snap_id), m_snap_id(snap_id), + m_on_finish(on_finish) { } void send(); @@ -53,6 +64,10 @@ private: */ ImageCtxT &m_image_ctx; + const int64_t m_group_pool_id; + const std::string m_group_id; + const std::string m_group_snap_id; + uint64_t *m_snap_id; Context *m_on_finish; int m_ret_val = 0; diff --git a/src/librbd/mirror/EnableRequest.cc b/src/librbd/mirror/EnableRequest.cc index 5fc1dffb149aa..306bee54fd141 100644 --- a/src/librbd/mirror/EnableRequest.cc +++ b/src/librbd/mirror/EnableRequest.cc @@ -30,11 +30,16 @@ EnableRequest::EnableRequest(librados::IoCtx &io_ctx, const std::string &non_primary_global_image_id, bool image_clean, asio::ContextWQ *op_work_queue, + int64_t group_pool_id, + const std::string &group_id, + const std::string &group_snap_id, + uint64_t *snap_id, Context *on_finish) : m_io_ctx(io_ctx), m_image_id(image_id), m_image_ctx(image_ctx), m_mode(mode), m_non_primary_global_image_id(non_primary_global_image_id), m_image_clean(image_clean), m_op_work_queue(op_work_queue), - m_on_finish(on_finish), + m_group_pool_id(group_pool_id), m_group_id(group_id), + m_group_snap_id(group_snap_id), m_snap_id(snap_id), m_on_finish(on_finish), m_cct(reinterpret_cast(io_ctx.cct())) { } @@ -96,6 +101,7 @@ void EnableRequest::handle_get_mirror_image(int r) { return; } } else if (r == -ENOENT) { + m_mirror_image.group_spec = {m_group_id, m_group_pool_id}; m_mirror_image.mode = m_mode; if (m_non_primary_global_image_id.empty()) { uuid_d uuid_gen; @@ -205,7 +211,8 @@ void EnableRequest::create_primary_snapshot() { auto req = snapshot::CreatePrimaryRequest::create( m_image_ctx, m_mirror_image.global_image_id, (m_image_clean ? 0 : CEPH_NOSNAP), snap_create_flags, - snapshot::CREATE_PRIMARY_FLAG_IGNORE_EMPTY_PEERS, &m_snap_id, ctx); + snapshot::CREATE_PRIMARY_FLAG_IGNORE_EMPTY_PEERS, m_group_pool_id, + m_group_id, m_group_snap_id, m_snap_id, ctx); req->send(); } diff --git a/src/librbd/mirror/EnableRequest.h b/src/librbd/mirror/EnableRequest.h index 391028e6e5db3..e058d5023cfc9 100644 --- a/src/librbd/mirror/EnableRequest.h +++ b/src/librbd/mirror/EnableRequest.h @@ -24,13 +24,26 @@ namespace mirror { template class EnableRequest { public: + static EnableRequest *create(ImageCtxT *image_ctx, + cls::rbd::MirrorImageMode mode, + const std::string &non_primary_global_image_id, + bool image_clean, int64_t group_pool_id, + const std::string &group_id, + const std::string &group_snap_id, + uint64_t *snap_id, Context *on_finish) { + return new EnableRequest(image_ctx->md_ctx, image_ctx->id, image_ctx, mode, + non_primary_global_image_id, image_clean, + image_ctx->op_work_queue, group_pool_id, group_id, + group_snap_id, snap_id, on_finish); + } static EnableRequest *create(ImageCtxT *image_ctx, cls::rbd::MirrorImageMode mode, const std::string &non_primary_global_image_id, bool image_clean, Context *on_finish) { return new EnableRequest(image_ctx->md_ctx, image_ctx->id, image_ctx, mode, non_primary_global_image_id, image_clean, - image_ctx->op_work_queue, on_finish); + image_ctx->op_work_queue, -1, {}, {}, nullptr, + on_finish); } static EnableRequest *create(librados::IoCtx &io_ctx, const std::string &image_id, @@ -40,7 +53,7 @@ public: Context *on_finish) { return new EnableRequest(io_ctx, image_id, nullptr, mode, non_primary_global_image_id, image_clean, - op_work_queue, on_finish); + op_work_queue, -1, {}, {}, nullptr, on_finish); } void send(); @@ -82,15 +95,21 @@ private: ImageCtxT* image_ctx, cls::rbd::MirrorImageMode mode, const std::string &non_primary_global_image_id, bool image_clean, asio::ContextWQ *op_work_queue, + int64_t group_pool_id, const std::string &group_id, + const std::string &group_snap_id, uint64_t *snap_id, Context *on_finish); librados::IoCtx &m_io_ctx; - std::string m_image_id; + const std::string m_image_id; ImageCtxT* m_image_ctx; - cls::rbd::MirrorImageMode m_mode; - std::string m_non_primary_global_image_id; - bool m_image_clean; + const cls::rbd::MirrorImageMode m_mode; + const std::string m_non_primary_global_image_id; + const bool m_image_clean; asio::ContextWQ *m_op_work_queue; + const int64_t m_group_pool_id; + const std::string m_group_id; + const std::string m_group_snap_id; + uint64_t *m_snap_id; Context *m_on_finish; CephContext *m_cct = nullptr; @@ -101,7 +120,6 @@ private: bool m_close_image = false; bool m_is_primary = false; - uint64_t m_snap_id = CEPH_NOSNAP; void get_mirror_image(); void handle_get_mirror_image(int r); diff --git a/src/librbd/mirror/ImageStateUpdateRequest.cc b/src/librbd/mirror/ImageStateUpdateRequest.cc index 34a1903e2601c..31d3b9f87fa24 100644 --- a/src/librbd/mirror/ImageStateUpdateRequest.cc +++ b/src/librbd/mirror/ImageStateUpdateRequest.cc @@ -44,7 +44,7 @@ void ImageStateUpdateRequest::get_mirror_image() { return; } - get_group(); + set_mirror_image(); return; } @@ -85,95 +85,6 @@ void ImageStateUpdateRequest::handle_get_mirror_image(int r) { return; } - get_group(); -} - -template -void ImageStateUpdateRequest::get_group() { - ldout(m_cct, 10) << dendl; - librados::ObjectReadOperation op; - cls_client::image_group_get_start(&op); - - auto comp = create_rados_callback< - ImageStateUpdateRequest, - &ImageStateUpdateRequest::handle_get_group>(this); - m_out_bl.clear(); - int r = m_io_ctx.aio_operate(util::header_name(m_image_id), comp, &op, - &m_out_bl); - ceph_assert(r == 0); - comp->release(); -} - -template -void ImageStateUpdateRequest::handle_get_group(int r) { - ldout(m_cct, 10) << "r=" << r << dendl; - - if (r == 0) { - auto iter = m_out_bl.cbegin(); - r = cls_client::image_group_get_finish(&iter, &m_group_spec); - } - - if (r < 0) { - lderr(m_cct) << "failed to retrieve image group: " << cpp_strerror(r) - << dendl; - finish(r); - return; - } - - get_mirror_group(); -} - -template -void ImageStateUpdateRequest::get_mirror_group() { - if (!m_group_spec.is_valid()) { - m_mirror_group.state = cls::rbd::MIRROR_GROUP_STATE_DISABLED; - set_mirror_image(); - return; - } else if (m_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED || - m_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLING) { - ceph_assert(m_group_spec.is_valid()); - m_mirror_image.group_spec = m_group_spec; - } - - ldout(m_cct, 10) << dendl; - - int r = util::create_ioctx(m_io_ctx, "group", m_group_spec.pool_id, {}, - &m_group_io_ctx); - if (r < 0) { - finish(r); - return; - } - - librados::ObjectReadOperation op; - cls_client::mirror_group_get_start(&op, m_group_spec.group_id); - - auto comp = create_rados_callback< - ImageStateUpdateRequest, - &ImageStateUpdateRequest::handle_get_mirror_group>(this); - m_out_bl.clear(); - r = m_group_io_ctx.aio_operate(RBD_MIRRORING, comp, &op, &m_out_bl); - ceph_assert(r == 0); - comp->release(); -} - -template -void ImageStateUpdateRequest::handle_get_mirror_group(int r) { - ldout(m_cct, 10) << "r=" << r << dendl; - - if (r == 0) { - auto iter = m_out_bl.cbegin(); - r = cls_client::mirror_group_get_finish(&iter, &m_mirror_group); - } - - if (r == -ENOENT) { - m_mirror_group.state = cls::rbd::MIRROR_GROUP_STATE_DISABLED; - } else if (r < 0) { - lderr(m_cct) << "failed to retrieve group mirroring state: " - << cpp_strerror(r) << dendl; - finish(r); - return; - } - set_mirror_image(); } @@ -210,7 +121,7 @@ void ImageStateUpdateRequest::handle_set_mirror_image(int r) { template void ImageStateUpdateRequest::notify_mirroring_watcher() { // skip image notification if mirroring for the image group is disabled - if (m_mirror_group.state != cls::rbd::MIRROR_GROUP_STATE_DISABLED) { + if (m_mirror_image.group_spec.is_valid()) { finish(0); return; } diff --git a/src/librbd/mirror/ImageStateUpdateRequest.h b/src/librbd/mirror/ImageStateUpdateRequest.h index 1db953d2da57e..3de98e1a3b1a9 100644 --- a/src/librbd/mirror/ImageStateUpdateRequest.h +++ b/src/librbd/mirror/ImageStateUpdateRequest.h @@ -51,12 +51,6 @@ private: * GET_MIRROR_IMAGE * | * v - * GET_GROUP - * | - * v - * GET_MIRROR_GROUP (skip if no group) - * | - * v * SET_MIRROR_IMAGE * | * v @@ -76,19 +70,10 @@ private: CephContext* m_cct; bufferlist m_out_bl; - librados::IoCtx m_group_io_ctx; - cls::rbd::GroupSpec m_group_spec; - cls::rbd::MirrorGroup m_mirror_group; void get_mirror_image(); void handle_get_mirror_image(int r); - void get_group(); - void handle_get_group(int r); - - void get_mirror_group(); - void handle_get_mirror_group(int r); - void set_mirror_image(); void handle_set_mirror_image(int r); diff --git a/src/librbd/mirror/PromoteRequest.cc b/src/librbd/mirror/PromoteRequest.cc index 62b528390a3ae..e25da85dceb4b 100644 --- a/src/librbd/mirror/PromoteRequest.cc +++ b/src/librbd/mirror/PromoteRequest.cc @@ -79,7 +79,8 @@ void PromoteRequest::promote() { Journal::promote(&m_image_ctx, ctx); } else if (m_mirror_image.mode == cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT) { auto req = mirror::snapshot::PromoteRequest::create( - &m_image_ctx, m_mirror_image.global_image_id, ctx); + &m_image_ctx, m_mirror_image.global_image_id, m_group_pool_id, m_group_id, + m_group_snap_id, m_snap_id, ctx); req->send(); } else { lderr(cct) << "unknown image mirror mode: " << m_mirror_image.mode << dendl; diff --git a/src/librbd/mirror/PromoteRequest.h b/src/librbd/mirror/PromoteRequest.h index c54f3bb76aca1..f5e1e015cbc25 100644 --- a/src/librbd/mirror/PromoteRequest.h +++ b/src/librbd/mirror/PromoteRequest.h @@ -18,13 +18,25 @@ namespace mirror { template class PromoteRequest { public: + static PromoteRequest *create(ImageCtxT &image_ctx, bool force, + int64_t group_pool_id, + const std::string &group_id, + const std::string &group_snap_id, + uint64_t *snap_id, Context *on_finish) { + return new PromoteRequest(image_ctx, force, group_pool_id, group_id, + group_snap_id, snap_id, on_finish); + } static PromoteRequest *create(ImageCtxT &image_ctx, bool force, Context *on_finish) { - return new PromoteRequest(image_ctx, force, on_finish); + return new PromoteRequest(image_ctx, force, -1, {}, {}, nullptr, on_finish); } - PromoteRequest(ImageCtxT &image_ctx, bool force, Context *on_finish) - : m_image_ctx(image_ctx), m_force(force), m_on_finish(on_finish) { + PromoteRequest(ImageCtxT &image_ctx, bool force, int64_t group_pool_id, + const std::string &group_id, const std::string &group_snap_id, + uint64_t *snap_id, Context *on_finish) + : m_image_ctx(image_ctx), m_force(force), m_group_pool_id(group_pool_id), + m_group_id(group_id), m_group_snap_id(group_snap_id), m_snap_id(snap_id), + m_on_finish(on_finish) { } void send(); @@ -51,7 +63,11 @@ private: */ ImageCtxT &m_image_ctx; - bool m_force; + const bool m_force; + const int64_t m_group_pool_id; + const std::string m_group_id; + const std::string m_group_snap_id; + uint64_t *m_snap_id; Context *m_on_finish; cls::rbd::MirrorImage m_mirror_image; diff --git a/src/librbd/mirror/snapshot/CreatePrimaryRequest.cc b/src/librbd/mirror/snapshot/CreatePrimaryRequest.cc index 1e4391d2a259f..9db6143462367 100644 --- a/src/librbd/mirror/snapshot/CreatePrimaryRequest.cc +++ b/src/librbd/mirror/snapshot/CreatePrimaryRequest.cc @@ -31,11 +31,13 @@ template CreatePrimaryRequest::CreatePrimaryRequest( I *image_ctx, const std::string& global_image_id, uint64_t clean_since_snap_id, uint64_t snap_create_flags, uint32_t flags, - uint64_t *snap_id, Context *on_finish) + int64_t group_pool_id, const std::string &group_id, + const std::string &group_snap_id, uint64_t *snap_id, Context *on_finish) : m_image_ctx(image_ctx), m_global_image_id(global_image_id), m_clean_since_snap_id(clean_since_snap_id), - m_snap_create_flags(snap_create_flags), m_flags(flags), m_snap_id(snap_id), - m_on_finish(on_finish) { + m_snap_create_flags(snap_create_flags), m_flags(flags), + m_group_pool_id(group_pool_id), m_group_id(group_id), + m_group_snap_id(group_snap_id), m_snap_id(snap_id), m_on_finish(on_finish) { m_default_ns_ctx.dup(m_image_ctx->md_ctx); m_default_ns_ctx.set_namespace(""); } @@ -117,6 +119,8 @@ void CreatePrimaryRequest::create_snapshot() { cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED : cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY), m_mirror_peer_uuids, "", m_clean_since_snap_id}; + ns.group_spec = {m_group_id, m_group_pool_id}; + ns.group_snap_id = m_group_snap_id; CephContext *cct = m_image_ctx->cct; ldout(cct, 15) << "name=" << m_snap_name << ", " diff --git a/src/librbd/mirror/snapshot/CreatePrimaryRequest.h b/src/librbd/mirror/snapshot/CreatePrimaryRequest.h index b8e84cf2b74b2..2b5b8b25ce1c3 100644 --- a/src/librbd/mirror/snapshot/CreatePrimaryRequest.h +++ b/src/librbd/mirror/snapshot/CreatePrimaryRequest.h @@ -25,6 +25,21 @@ namespace snapshot { template class CreatePrimaryRequest { public: + static CreatePrimaryRequest *create(ImageCtxT *image_ctx, + const std::string& global_image_id, + uint64_t clean_since_snap_id, + uint64_t snap_create_flags, + uint32_t flags, int64_t group_pool_id, + const std::string &group_id, + const std::string &group_snap_id, + uint64_t *snap_id, + Context *on_finish) { + return new CreatePrimaryRequest(image_ctx, global_image_id, + clean_since_snap_id, snap_create_flags, + flags, group_pool_id, group_id, + group_snap_id, snap_id, on_finish); + } + static CreatePrimaryRequest *create(ImageCtxT *image_ctx, const std::string& global_image_id, uint64_t clean_since_snap_id, @@ -32,14 +47,17 @@ public: uint32_t flags, uint64_t *snap_id, Context *on_finish) { return new CreatePrimaryRequest(image_ctx, global_image_id, - clean_since_snap_id, snap_create_flags, flags, - snap_id, on_finish); + clean_since_snap_id, snap_create_flags, + flags, -1, {}, {}, snap_id, on_finish); } CreatePrimaryRequest(ImageCtxT *image_ctx, const std::string& global_image_id, uint64_t clean_since_snap_id, uint64_t snap_create_flags, - uint32_t flags, uint64_t *snap_id, Context *on_finish); + uint32_t flags, int64_t group_pool_id, + const std::string &group_id, + const std::string &group_snap_id, uint64_t *snap_id, + Context *on_finish); void send(); @@ -68,10 +86,13 @@ private: */ ImageCtxT *m_image_ctx; - std::string m_global_image_id; - uint64_t m_clean_since_snap_id; + const std::string m_global_image_id; + const uint64_t m_clean_since_snap_id; const uint64_t m_snap_create_flags; const uint32_t m_flags; + const int64_t m_group_pool_id; + const std::string m_group_id; + const std::string m_group_snap_id; uint64_t *m_snap_id; Context *m_on_finish; diff --git a/src/librbd/mirror/snapshot/DemoteRequest.cc b/src/librbd/mirror/snapshot/DemoteRequest.cc index ccaa33c83492b..466fea5a7376c 100644 --- a/src/librbd/mirror/snapshot/DemoteRequest.cc +++ b/src/librbd/mirror/snapshot/DemoteRequest.cc @@ -75,7 +75,8 @@ void DemoteRequest::create_snapshot() { m_image_ctx, m_global_image_id, CEPH_NOSNAP, SNAP_CREATE_FLAG_SKIP_NOTIFY_QUIESCE, (snapshot::CREATE_PRIMARY_FLAG_IGNORE_EMPTY_PEERS | - snapshot::CREATE_PRIMARY_FLAG_DEMOTED), nullptr, ctx); + snapshot::CREATE_PRIMARY_FLAG_DEMOTED), m_group_pool_id, + m_group_id, m_group_snap_id, m_snap_id, ctx); req->send(); } diff --git a/src/librbd/mirror/snapshot/DemoteRequest.h b/src/librbd/mirror/snapshot/DemoteRequest.h index 63c9356458dc2..28dc346ea477d 100644 --- a/src/librbd/mirror/snapshot/DemoteRequest.h +++ b/src/librbd/mirror/snapshot/DemoteRequest.h @@ -21,15 +21,29 @@ namespace snapshot { template class DemoteRequest { public: + static DemoteRequest *create(ImageCtxT *image_ctx, + const std::string& global_image_id, + int64_t group_pool_id, + const std::string &group_id, + const std::string &group_snap_id, + uint64_t *snap_id, Context *on_finish) { + return new DemoteRequest(image_ctx, global_image_id, group_pool_id, + group_id, group_snap_id, snap_id, on_finish); + } static DemoteRequest *create(ImageCtxT *image_ctx, const std::string& global_image_id, Context *on_finish) { - return new DemoteRequest(image_ctx, global_image_id, on_finish); + return new DemoteRequest(image_ctx, global_image_id, -1, {}, {}, nullptr, + on_finish); } DemoteRequest(ImageCtxT *image_ctx, const std::string& global_image_id, + int64_t group_pool_id, const std::string &group_id, + const std::string &group_snap_id, uint64_t *snap_id, Context *on_finish) : m_image_ctx(image_ctx), m_global_image_id(global_image_id), + m_group_pool_id(group_pool_id), m_group_id(group_id), + m_group_snap_id(group_snap_id), m_snap_id(snap_id), m_on_finish(on_finish) { } @@ -54,7 +68,11 @@ private: */ ImageCtxT *m_image_ctx; - std::string m_global_image_id; + const std::string m_global_image_id; + const int64_t m_group_pool_id; + const std::string m_group_id; + const std::string m_group_snap_id; + uint64_t *m_snap_id; Context *m_on_finish; void enable_non_primary_feature(); diff --git a/src/librbd/mirror/snapshot/PromoteRequest.cc b/src/librbd/mirror/snapshot/PromoteRequest.cc index 1a6b1c81bc43d..068c96a307dfb 100644 --- a/src/librbd/mirror/snapshot/PromoteRequest.cc +++ b/src/librbd/mirror/snapshot/PromoteRequest.cc @@ -301,7 +301,8 @@ void PromoteRequest::create_promote_snapshot() { m_image_ctx, m_global_image_id, CEPH_NOSNAP, SNAP_CREATE_FLAG_SKIP_NOTIFY_QUIESCE, (snapshot::CREATE_PRIMARY_FLAG_IGNORE_EMPTY_PEERS | - snapshot::CREATE_PRIMARY_FLAG_FORCE), nullptr, ctx); + snapshot::CREATE_PRIMARY_FLAG_FORCE), m_group_pool_id, + m_group_id, m_group_snap_id, m_snap_id, ctx); req->send(); } diff --git a/src/librbd/mirror/snapshot/PromoteRequest.h b/src/librbd/mirror/snapshot/PromoteRequest.h index 1d9a862a0afb3..7ecb18673b425 100644 --- a/src/librbd/mirror/snapshot/PromoteRequest.h +++ b/src/librbd/mirror/snapshot/PromoteRequest.h @@ -25,15 +25,29 @@ namespace snapshot { template class PromoteRequest { public: + static PromoteRequest *create(ImageCtxT *image_ctx, + const std::string& global_image_id, + int64_t group_pool_id, + const std::string &group_id, + const std::string &group_snap_id, + uint64_t *snap_id, Context *on_finish) { + return new PromoteRequest(image_ctx, global_image_id, group_pool_id, + group_id, group_snap_id, snap_id, on_finish); + } static PromoteRequest *create(ImageCtxT *image_ctx, const std::string& global_image_id, Context *on_finish) { - return new PromoteRequest(image_ctx, global_image_id, on_finish); + return new PromoteRequest(image_ctx, global_image_id, -1, {}, {}, nullptr, + on_finish); } PromoteRequest(ImageCtxT *image_ctx, const std::string& global_image_id, + int64_t group_pool_id, const std::string &group_id, + const std::string &group_snap_id, uint64_t *snap_id, Context *on_finish) : m_image_ctx(image_ctx), m_global_image_id(global_image_id), + m_group_pool_id(group_pool_id), m_group_id(group_id), + m_group_snap_id(group_snap_id), m_snap_id(snap_id), m_on_finish(on_finish) { } @@ -79,7 +93,11 @@ private: */ ImageCtxT *m_image_ctx; - std::string m_global_image_id; + const std::string m_global_image_id; + const int64_t m_group_pool_id; + const std::string m_group_id; + const std::string m_group_snap_id; + uint64_t *m_snap_id; Context *m_on_finish; uint64_t m_rollback_snap_id = CEPH_NOSNAP; diff --git a/src/librbd/mirroring_watcher/Types.cc b/src/librbd/mirroring_watcher/Types.cc index c92c6fda8c504..759644abaff7e 100644 --- a/src/librbd/mirroring_watcher/Types.cc +++ b/src/librbd/mirroring_watcher/Types.cc @@ -73,6 +73,7 @@ void GroupUpdatedPayload::encode(bufferlist &bl) const { encode(static_cast(mirror_group_state), bl); encode(group_id, bl); encode(global_group_id, bl); + encode(image_count, bl); } void GroupUpdatedPayload::decode(__u8 version, bufferlist::const_iterator &iter) { @@ -83,12 +84,14 @@ void GroupUpdatedPayload::decode(__u8 version, bufferlist::const_iterator &iter) mirror_group_state_decode); decode(group_id, iter); decode(global_group_id, iter); + decode(image_count, iter); } void GroupUpdatedPayload::dump(Formatter *f) const { f->dump_stream("mirror_group_state") << mirror_group_state; f->dump_string("group_id", group_id); f->dump_string("global_group_id", global_group_id); + f->dump_unsigned("image_count", image_count); } void UnknownPayload::encode(bufferlist &bl) const { @@ -142,7 +145,7 @@ void NotifyMessage::generate_test_instances(std::list &o) { o.push_back(new NotifyMessage(ImageUpdatedPayload(cls::rbd::MIRROR_IMAGE_STATE_DISABLING, "image id", "global image id"))); o.push_back(new NotifyMessage(GroupUpdatedPayload(cls::rbd::MIRROR_GROUP_STATE_DISABLING, - "group id", "global group id"))); + "group id", "global group id", 2))); } std::ostream &operator<<(std::ostream &out, const NotifyOp &op) { diff --git a/src/librbd/mirroring_watcher/Types.h b/src/librbd/mirroring_watcher/Types.h index 5b5efb14a9c98..7f266f00bc48a 100644 --- a/src/librbd/mirroring_watcher/Types.h +++ b/src/librbd/mirroring_watcher/Types.h @@ -69,14 +69,15 @@ struct GroupUpdatedPayload { cls::rbd::MIRROR_GROUP_STATE_ENABLED; std::string group_id; std::string global_group_id; + size_t image_count = 0; GroupUpdatedPayload() { } GroupUpdatedPayload(cls::rbd::MirrorGroupState mirror_group_state, const std::string &group_id, - const std::string &global_group_id) + const std::string &global_group_id, size_t image_count) : mirror_group_state(mirror_group_state), group_id(group_id), - global_group_id(global_group_id) { + global_group_id(global_group_id), image_count(image_count) { } void encode(bufferlist &bl) const; diff --git a/src/test/librbd/mirror/snapshot/test_mock_CreatePrimaryRequest.cc b/src/test/librbd/mirror/snapshot/test_mock_CreatePrimaryRequest.cc index 8bfdcdeb1d8ab..3690fb5d4a1ca 100644 --- a/src/test/librbd/mirror/snapshot/test_mock_CreatePrimaryRequest.cc +++ b/src/test/librbd/mirror/snapshot/test_mock_CreatePrimaryRequest.cc @@ -228,7 +228,7 @@ TEST_F(TestMockMirrorSnapshotCreatePrimaryRequest, Success) { C_SaferCond ctx; auto req = new MockCreatePrimaryRequest(&mock_image_ctx, "gid", CEPH_NOSNAP, - 0U, 0U, nullptr, &ctx); + 0U, 0U, -1, {}, {}, nullptr, &ctx); req->send(); ASSERT_EQ(0, ctx.wait()); } @@ -258,7 +258,7 @@ TEST_F(TestMockMirrorSnapshotCreatePrimaryRequest, SuccessPrimary) { C_SaferCond ctx; auto req = new MockCreatePrimaryRequest(&mock_image_ctx, "gid", CEPH_NOSNAP, - 0U, 0U, nullptr, &ctx); + 0U, 0U, -1, {}, {}, nullptr, &ctx); req->send(); ASSERT_EQ(0, ctx.wait()); } @@ -288,7 +288,7 @@ TEST_F(TestMockMirrorSnapshotCreatePrimaryRequest, SuccessPrimaryDemoted) { C_SaferCond ctx; auto req = new MockCreatePrimaryRequest(&mock_image_ctx, "gid", CEPH_NOSNAP, - 0U, 0U, nullptr, &ctx); + 0U, 0U, -1, {}, {}, nullptr, &ctx); req->send(); ASSERT_EQ(0, ctx.wait()); } @@ -319,7 +319,7 @@ TEST_F(TestMockMirrorSnapshotCreatePrimaryRequest, SuccessNonPrimaryDemoted) { C_SaferCond ctx; auto req = new MockCreatePrimaryRequest(&mock_image_ctx, "gid", CEPH_NOSNAP, - 0U, 0U, nullptr, &ctx); + 0U, 0U, -1, {}, {}, nullptr, &ctx); req->send(); ASSERT_EQ(0, ctx.wait()); } @@ -352,7 +352,7 @@ TEST_F(TestMockMirrorSnapshotCreatePrimaryRequest, SuccessPrimaryBelowMaxSnapsho C_SaferCond ctx; auto req = new MockCreatePrimaryRequest(&mock_image_ctx, "gid", CEPH_NOSNAP, - 0U, 0U, nullptr, &ctx); + 0U, 0U, -1, {}, {}, nullptr, &ctx); req->send(); ASSERT_EQ(0, ctx.wait()); } @@ -386,8 +386,9 @@ TEST_F(TestMockMirrorSnapshotCreatePrimaryRequest, SuccessPrimaryBelowMaxSnapsho expect_refresh_image(mock_image_ctx, 0); C_SaferCond ctx; - auto req = new MockCreatePrimaryRequest(&mock_image_ctx, "gid", CEPH_NOSNAP, - 0U, 0U, nullptr, &ctx); + auto req = MockCreatePrimaryRequest::create(&mock_image_ctx, "gid", + CEPH_NOSNAP, 0U, 0U, -1, {}, {}, + nullptr, &ctx); req->send(); ASSERT_EQ(0, ctx.wait()); } @@ -407,8 +408,9 @@ TEST_F(TestMockMirrorSnapshotCreatePrimaryRequest, CanNotError) { expect_can_create_primary_snapshot(mock_utils, false, false, false); C_SaferCond ctx; - auto req = new MockCreatePrimaryRequest(&mock_image_ctx, "gid", CEPH_NOSNAP, - 0U, 0U, nullptr, &ctx); + auto req = MockCreatePrimaryRequest::create(&mock_image_ctx, "gid", + CEPH_NOSNAP, 0U, 0U, -1, {}, {}, + nullptr, &ctx); req->send(); ASSERT_EQ(-EINVAL, ctx.wait()); } @@ -431,8 +433,9 @@ TEST_F(TestMockMirrorSnapshotCreatePrimaryRequest, GetMirrorPeersError) { "mirror", "mirror uuid"}}, -EINVAL); C_SaferCond ctx; - auto req = new MockCreatePrimaryRequest(&mock_image_ctx, "gid", CEPH_NOSNAP, - 0U, 0U, nullptr, &ctx); + auto req = MockCreatePrimaryRequest::create(&mock_image_ctx, "gid", + CEPH_NOSNAP, 0U, 0U, -1, {}, {}, + nullptr, &ctx); req->send(); ASSERT_EQ(-EINVAL, ctx.wait()); } @@ -456,8 +459,9 @@ TEST_F(TestMockMirrorSnapshotCreatePrimaryRequest, CreateSnapshotError) { expect_create_snapshot(mock_image_ctx, -EINVAL); C_SaferCond ctx; - auto req = new MockCreatePrimaryRequest(&mock_image_ctx, "gid", CEPH_NOSNAP, - 0U, 0U, nullptr, &ctx); + auto req = MockCreatePrimaryRequest::create(&mock_image_ctx, "gid", + CEPH_NOSNAP, 0U, 0U, -1, {}, {}, + nullptr, &ctx); req->send(); ASSERT_EQ(-EINVAL, ctx.wait()); } @@ -492,7 +496,7 @@ TEST_F(TestMockMirrorSnapshotCreatePrimaryRequest, SuccessUnlinkPrimaryNoPeer) { C_SaferCond ctx; auto req = new MockCreatePrimaryRequest(&mock_image_ctx, "gid", CEPH_NOSNAP, - 0U, 0U, nullptr, &ctx); + 0U, 0U, -1, {}, {}, nullptr, &ctx); req->send(); ASSERT_EQ(0, ctx.wait()); } @@ -527,7 +531,7 @@ TEST_F(TestMockMirrorSnapshotCreatePrimaryRequest, SuccessUnlinkPrimaryDemotedNo C_SaferCond ctx; auto req = new MockCreatePrimaryRequest(&mock_image_ctx, "gid", CEPH_NOSNAP, - 0U, 0U, nullptr, &ctx); + 0U, 0U, -1, {}, {}, nullptr, &ctx); req->send(); ASSERT_EQ(0, ctx.wait()); } @@ -563,7 +567,7 @@ TEST_F(TestMockMirrorSnapshotCreatePrimaryRequest, SuccessUnlinkNonPrimaryDemote C_SaferCond ctx; auto req = new MockCreatePrimaryRequest(&mock_image_ctx, "gid", CEPH_NOSNAP, - 0U, 0U, nullptr, &ctx); + 0U, 0U, -1, {}, {}, nullptr, &ctx); req->send(); ASSERT_EQ(0, ctx.wait()); } @@ -598,7 +602,7 @@ TEST_F(TestMockMirrorSnapshotCreatePrimaryRequest, SuccessUnlinkOrphanNoPeer) { C_SaferCond ctx; auto req = new MockCreatePrimaryRequest(&mock_image_ctx, "gid", CEPH_NOSNAP, - 0U, 0U, nullptr, &ctx); + 0U, 0U, -1, {}, {}, nullptr, &ctx); req->send(); ASSERT_EQ(0, ctx.wait()); } @@ -633,7 +637,7 @@ TEST_F(TestMockMirrorSnapshotCreatePrimaryRequest, SuccessUnlinkPrimaryIncomplet C_SaferCond ctx; auto req = new MockCreatePrimaryRequest(&mock_image_ctx, "gid", CEPH_NOSNAP, - 0U, 0U, nullptr, &ctx); + 0U, 0U, -1, {}, {}, nullptr, &ctx); req->send(); ASSERT_EQ(0, ctx.wait()); } @@ -670,8 +674,9 @@ TEST_F(TestMockMirrorSnapshotCreatePrimaryRequest, SuccessUnlinkPrimaryMaxSnapsh true, true, true, 0); C_SaferCond ctx; - auto req = new MockCreatePrimaryRequest(&mock_image_ctx, "gid", CEPH_NOSNAP, - 0U, 0U, nullptr, &ctx); + auto req = MockCreatePrimaryRequest::create(&mock_image_ctx, "gid", + CEPH_NOSNAP, 0U, 0U, -1, {}, {}, + nullptr, &ctx); req->send(); ASSERT_EQ(0, ctx.wait()); } @@ -710,8 +715,9 @@ TEST_F(TestMockMirrorSnapshotCreatePrimaryRequest, SuccessUnlinkPrimaryMaxSnapsh true, true, true, 0); C_SaferCond ctx; - auto req = new MockCreatePrimaryRequest(&mock_image_ctx, "gid", CEPH_NOSNAP, - 0U, 0U, nullptr, &ctx); + auto req = MockCreatePrimaryRequest::create(&mock_image_ctx, "gid", + CEPH_NOSNAP, 0U, 0U, -1, {}, {}, + nullptr, &ctx); req->send(); ASSERT_EQ(0, ctx.wait()); } @@ -754,7 +760,7 @@ TEST_F(TestMockMirrorSnapshotCreatePrimaryRequest, SuccessUnlinkMultiplePeers) { C_SaferCond ctx; auto req = new MockCreatePrimaryRequest(&mock_image_ctx, "gid", CEPH_NOSNAP, - 0U, 0U, nullptr, &ctx); + 0U, 0U, -1, {}, {}, nullptr, &ctx); req->send(); ASSERT_EQ(0, ctx.wait()); } @@ -795,8 +801,9 @@ TEST_F(TestMockMirrorSnapshotCreatePrimaryRequest, SuccessUnlinkMultipleSnapshot false, true, true, 0); C_SaferCond ctx; - auto req = new MockCreatePrimaryRequest(&mock_image_ctx, "gid", CEPH_NOSNAP, - 0U, 0U, nullptr, &ctx); + auto req = MockCreatePrimaryRequest::create(&mock_image_ctx, "gid", + CEPH_NOSNAP, 0U, 0U, -1, {}, {}, + nullptr, &ctx); req->send(); ASSERT_EQ(0, ctx.wait()); } diff --git a/src/test/librbd/mirror/snapshot/test_mock_PromoteRequest.cc b/src/test/librbd/mirror/snapshot/test_mock_PromoteRequest.cc index af9c349339b46..6dcde3d25f516 100644 --- a/src/test/librbd/mirror/snapshot/test_mock_PromoteRequest.cc +++ b/src/test/librbd/mirror/snapshot/test_mock_PromoteRequest.cc @@ -124,7 +124,11 @@ struct CreatePrimaryRequest { const std::string& global_image_id, uint64_t clean_since_snap_id, uint64_t snap_create_flags, - uint32_t flags, uint64_t *snap_id, + uint32_t flags, + int64_t group_pool_id, + const std::string &group_id, + const std::string &group_snap_id, + uint64_t *snap_id, Context *on_finish) { ceph_assert(s_instance != nullptr); s_instance->demoted = ((flags & CREATE_PRIMARY_FLAG_DEMOTED) != 0); @@ -280,7 +284,7 @@ TEST_F(TestMockMirrorSnapshotPromoteRequest, Success) { expect_create_promote_snapshot(mock_image_ctx, mock_create_primary_request, 0); C_SaferCond ctx; - auto req = new MockPromoteRequest(&mock_image_ctx, "gid", &ctx); + auto req = MockPromoteRequest::create(&mock_image_ctx, "gid", &ctx); req->send(); ASSERT_EQ(0, ctx.wait()); } @@ -318,7 +322,7 @@ TEST_F(TestMockMirrorSnapshotPromoteRequest, SuccessForce) { expect_release_lock(mock_image_ctx, 0); C_SaferCond ctx; - auto req = new MockPromoteRequest(&mock_image_ctx, "gid", &ctx); + auto req = MockPromoteRequest::create(&mock_image_ctx, "gid", &ctx); req->send(); ASSERT_EQ(0, ctx.wait()); } @@ -357,7 +361,7 @@ TEST_F(TestMockMirrorSnapshotPromoteRequest, SuccessRollback) { expect_release_lock(mock_image_ctx, 0); C_SaferCond ctx; - auto req = new MockPromoteRequest(&mock_image_ctx, "gid", &ctx); + auto req = MockPromoteRequest::create(&mock_image_ctx, "gid", &ctx); req->send(); ASSERT_EQ(0, ctx.wait()); } @@ -378,7 +382,7 @@ TEST_F(TestMockMirrorSnapshotPromoteRequest, ErrorCannotRollback) { false); C_SaferCond ctx; - auto req = new MockPromoteRequest(&mock_image_ctx, "gid", &ctx); + auto req = MockPromoteRequest::create(&mock_image_ctx, "gid", &ctx); req->send(); ASSERT_EQ(-EINVAL, ctx.wait()); } diff --git a/src/test/librbd/test_Groups.cc b/src/test/librbd/test_Groups.cc index a95785ad7a39e..91fa09783312b 100644 --- a/src/test/librbd/test_Groups.cc +++ b/src/test/librbd/test_Groups.cc @@ -880,3 +880,77 @@ TEST_F(TestGroup, snap_list_internal) ASSERT_EQ(0, rbd.group_remove(ioctx, group_name)); } + +TEST_F(TestGroup, mirrorPP) +{ + REQUIRE_FORMAT_V2(); + + std::string peer_uuid; + ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_IMAGE)); + ASSERT_EQ(0, m_rbd.mirror_peer_site_add(m_ioctx, &peer_uuid, + RBD_MIRROR_PEER_DIRECTION_RX_TX, + "cluster", "client")); + const char *group_name = "snap_group"; + ASSERT_EQ(0, m_rbd.group_create(m_ioctx, group_name)); + ASSERT_EQ(0, m_rbd.group_image_add(m_ioctx, group_name, m_ioctx, + m_image_name.c_str())); + + std::vector group_snaps; + ASSERT_EQ(0, m_rbd.group_snap_list(m_ioctx, group_name, &group_snaps, + sizeof(librbd::group_snap_info_t))); + ASSERT_EQ(0U, group_snaps.size()); + + std::string snap_id; + + ASSERT_EQ(-EINVAL, m_rbd.mirror_group_create_snapshot(m_ioctx, group_name, 0, + &snap_id)); + ASSERT_EQ(0, m_rbd.group_snap_list(m_ioctx, group_name, &group_snaps, + sizeof(librbd::group_snap_info_t))); + ASSERT_EQ(0U, group_snaps.size()); + + ASSERT_EQ(0, m_rbd.mirror_group_enable(m_ioctx, group_name, + RBD_MIRROR_IMAGE_MODE_SNAPSHOT)); + + librbd::Image image; + ASSERT_EQ(0, m_rbd.open(m_ioctx, image, m_image_name.c_str())); + + librbd::mirror_image_mode_t mode; + ASSERT_EQ(0, image.mirror_image_get_mode(&mode)); + ASSERT_EQ(RBD_MIRROR_IMAGE_MODE_SNAPSHOT, mode); + + std::vector snaps; + ASSERT_EQ(0, image.snap_list(snaps)); + ASSERT_EQ(1U, snaps.size()); + + ASSERT_EQ(0, m_rbd.mirror_group_create_snapshot(m_ioctx, group_name, 0, + &snap_id)); + snaps.clear(); + ASSERT_EQ(0, image.snap_list(snaps)); + ASSERT_EQ(2U, snaps.size()); + librbd::snap_namespace_type_t snap_ns_type; + ASSERT_EQ(0, image.snap_get_namespace_type(snaps[1].id, &snap_ns_type)); + ASSERT_EQ(RBD_SNAP_NAMESPACE_TYPE_MIRROR, snap_ns_type); + librbd::snap_mirror_namespace_t mirror_snap; + ASSERT_EQ(0, image.snap_get_mirror_namespace(snaps[1].id, &mirror_snap, + sizeof(mirror_snap))); + + ASSERT_EQ(0, m_rbd.group_snap_list(m_ioctx, group_name, &group_snaps, + sizeof(librbd::group_snap_info_t))); + ASSERT_EQ(2U, group_snaps.size()); + + ASSERT_EQ(0, m_rbd.mirror_group_demote(m_ioctx, group_name)); + + librbd::mirror_image_info_t mirror_info; + ASSERT_EQ(0, image.mirror_image_get_info(&mirror_info, sizeof(mirror_info))); + ASSERT_FALSE(mirror_info.primary); + + ASSERT_EQ(0, m_rbd.mirror_group_resync(m_ioctx, group_name)); + + ASSERT_EQ(0, m_rbd.mirror_group_promote(m_ioctx, group_name, false)); + ASSERT_EQ(0, image.mirror_image_get_info(&mirror_info, sizeof(mirror_info))); + ASSERT_TRUE(mirror_info.primary); + + ASSERT_EQ(0, m_rbd.mirror_group_disable(m_ioctx, group_name, false)); + + ASSERT_EQ(0, m_rbd.group_remove(m_ioctx, group_name)); +} diff --git a/src/test/librbd/test_MirroringWatcher.cc b/src/test/librbd/test_MirroringWatcher.cc index 4fe4fca2ce988..490517ff8d822 100644 --- a/src/test/librbd/test_MirroringWatcher.cc +++ b/src/test/librbd/test_MirroringWatcher.cc @@ -28,9 +28,10 @@ struct MockMirroringWatcher : public MirroringWatcher<> { MOCK_METHOD3(handle_image_updated, void(cls::rbd::MirrorImageState, const std::string &, const std::string &)); - MOCK_METHOD3(handle_group_updated, void(cls::rbd::MirrorGroupState, + MOCK_METHOD4(handle_group_updated, void(cls::rbd::MirrorGroupState, const std::string &, - const std::string &)); + const std::string &, + size_t)); }; } // anonymous namespace diff --git a/src/test/rbd_mirror/test_mock_PoolWatcher.cc b/src/test/rbd_mirror/test_mock_PoolWatcher.cc index b2f4f91f47f44..8653a412f5db0 100644 --- a/src/test/rbd_mirror/test_mock_PoolWatcher.cc +++ b/src/test/rbd_mirror/test_mock_PoolWatcher.cc @@ -67,7 +67,8 @@ struct MirroringWatcher { const std::string &global_image_id) = 0; virtual void handle_group_updated(cls::rbd::MirrorGroupState state, const std::string &remote_group_id, - const std::string &global_group_id) = 0; + const std::string &global_group_id, + size_t image_count) = 0; bool is_unregistered() const { return MockMirroringWatcher::get_instance().is_unregistered(); diff --git a/src/tools/rbd_mirror/PoolWatcher.cc b/src/tools/rbd_mirror/PoolWatcher.cc index 94343e95faf11..deab4f20ad60d 100644 --- a/src/tools/rbd_mirror/PoolWatcher.cc +++ b/src/tools/rbd_mirror/PoolWatcher.cc @@ -65,10 +65,11 @@ public: void handle_group_updated(cls::rbd::MirrorGroupState state, const std::string &group_id, - const std::string &global_group_id) override { + const std::string &global_group_id, + size_t image_count) override { bool enabled = (state == cls::rbd::MIRROR_GROUP_STATE_ENABLED); - m_pool_watcher->handle_group_updated(group_id, global_group_id, - enabled); + m_pool_watcher->handle_group_updated(group_id, global_group_id, enabled, + image_count); } private: @@ -364,9 +365,10 @@ void PoolWatcher::handle_image_updated(const std::string &id, template void PoolWatcher::handle_group_updated(const std::string &id, const std::string &global_group_id, - bool enabled) { + bool enabled, size_t image_count) { dout(10) << "group_id=" << id << ", " << "global_group_id=" << global_group_id << ", " + << "image_count=" << image_count << ", " << "enabled=" << enabled << dendl; // TODO diff --git a/src/tools/rbd_mirror/PoolWatcher.h b/src/tools/rbd_mirror/PoolWatcher.h index af1825b5799ec..b8d96eefbdc58 100644 --- a/src/tools/rbd_mirror/PoolWatcher.h +++ b/src/tools/rbd_mirror/PoolWatcher.h @@ -149,7 +149,7 @@ private: bool enabled); void handle_group_updated(const std::string &group_id, const std::string &global_group_id, - bool enabled); + bool enabled, size_t image_count); void schedule_listener(); void notify_listener(); -- 2.39.5