From 7252da9ce479b46a0dc6a352adfa4b027b9eba11 Mon Sep 17 00:00:00 2001 From: VinayBhaskar-V Date: Fri, 28 Feb 2025 03:38:55 +0530 Subject: [PATCH] librbd: get_group_snap_get_mirror_namespace() API + groups in "rbd mirror pool status" Co-authored-by: Ilya Dryomov Signed-off-by: VinayBhaskar-V Signed-off-by: Ilya Dryomov --- qa/workunits/rbd/rbd_mirror_bootstrap.sh | 16 +-- src/include/rbd/librbd.h | 13 ++ src/include/rbd/librbd.hpp | 10 ++ src/librbd/api/Group.cc | 62 ++++++++ src/librbd/api/Group.h | 3 + src/librbd/librbd.cc | 50 +++++++ src/pybind/rbd/c_rbd.pxd | 14 ++ src/pybind/rbd/mock_rbd.pxi | 7 + src/pybind/rbd/rbd.pyx | 43 ++++++ src/tools/rbd/action/Group.cc | 70 ++++++++- src/tools/rbd/action/MirrorGroup.cc | 2 +- src/tools/rbd/action/MirrorPool.cc | 172 ++++++++++++++++++++++- 12 files changed, 445 insertions(+), 17 deletions(-) diff --git a/qa/workunits/rbd/rbd_mirror_bootstrap.sh b/qa/workunits/rbd/rbd_mirror_bootstrap.sh index 8eaedf03653..d76b571fa87 100755 --- a/qa/workunits/rbd/rbd_mirror_bootstrap.sh +++ b/qa/workunits/rbd/rbd_mirror_bootstrap.sh @@ -43,9 +43,9 @@ wait_for_replay_complete ${CLUSTER2} ${CLUSTER1} ${POOL} ${POOL} image1 wait_for_replaying_status_in_pool_dir ${CLUSTER2} ${POOL} image1 POOL_STATUS=$(get_pool_status_json ${CLUSTER1} ${POOL}) -jq -e '.summary.states == {"replaying": 1}' <<< ${POOL_STATUS} +jq -e '.summary.image_states == {"replaying": 1}' <<< ${POOL_STATUS} POOL_STATUS=$(get_pool_status_json ${CLUSTER2} ${POOL}) -jq -e '.summary.states == {"replaying": 1}' <<< ${POOL_STATUS} +jq -e '.summary.image_states == {"replaying": 1}' <<< ${POOL_STATUS} wait_for_image_replay_started ${CLUSTER2} ${POOL}/${NS1} image1 write_image ${CLUSTER1} ${POOL}/${NS1} image1 100 @@ -53,9 +53,9 @@ wait_for_replay_complete ${CLUSTER2} ${CLUSTER1} ${POOL}/${NS1} ${POOL}/${NS1} i wait_for_replaying_status_in_pool_dir ${CLUSTER2} ${POOL}/${NS1} image1 POOL_STATUS=$(get_pool_status_json ${CLUSTER1} ${POOL}/${NS1}) -jq -e '.summary.states == {"replaying": 1}' <<< ${POOL_STATUS} +jq -e '.summary.image_states == {"replaying": 1}' <<< ${POOL_STATUS} POOL_STATUS=$(get_pool_status_json ${CLUSTER2} ${POOL}/${NS1}) -jq -e '.summary.states == {"replaying": 1}' <<< ${POOL_STATUS} +jq -e '.summary.image_states == {"replaying": 1}' <<< ${POOL_STATUS} testlog "TEST: verify rx-tx direction" # both rx-tx peers are added immediately by "rbd mirror pool peer bootstrap import" @@ -78,9 +78,9 @@ wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${PARENT_POOL} ${PARENT_POOL} i wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${PARENT_POOL} image2 POOL_STATUS=$(get_pool_status_json ${CLUSTER1} ${PARENT_POOL}) -jq -e '.summary.states == {"replaying": 2}' <<< ${POOL_STATUS} +jq -e '.summary.image_states == {"replaying": 2}' <<< ${POOL_STATUS} POOL_STATUS=$(get_pool_status_json ${CLUSTER2} ${PARENT_POOL}) -jq -e '.summary.states == {"replaying": 2}' <<< ${POOL_STATUS} +jq -e '.summary.image_states == {"replaying": 2}' <<< ${POOL_STATUS} wait_for_image_replay_started ${CLUSTER2} ${PARENT_POOL}/${NS1} image1 write_image ${CLUSTER1} ${PARENT_POOL}/${NS1} image1 100 @@ -93,9 +93,9 @@ wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${PARENT_POOL}/${NS1} ${PARENT_ wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${PARENT_POOL}/${NS1} image2 POOL_STATUS=$(get_pool_status_json ${CLUSTER1} ${PARENT_POOL}/${NS1}) -jq -e '.summary.states == {"replaying": 2}' <<< ${POOL_STATUS} +jq -e '.summary.image_states == {"replaying": 2}' <<< ${POOL_STATUS} POOL_STATUS=$(get_pool_status_json ${CLUSTER2} ${PARENT_POOL}/${NS1}) -jq -e '.summary.states == {"replaying": 2}' <<< ${POOL_STATUS} +jq -e '.summary.image_states == {"replaying": 2}' <<< ${POOL_STATUS} testlog "TEST: pool replayer and callout cleanup when peer is updated" test_health_state ${CLUSTER1} ${PARENT_POOL} 'OK' diff --git a/src/include/rbd/librbd.h b/src/include/rbd/librbd.h index 5d6414a3574..b2562e6e7ec 100644 --- a/src/include/rbd/librbd.h +++ b/src/include/rbd/librbd.h @@ -356,6 +356,14 @@ typedef struct { uint64_t last_copied_object_number; } rbd_snap_mirror_namespace_t; +typedef struct { + rbd_snap_mirror_state_t state; + size_t mirror_peer_uuids_count; + char* mirror_peer_uuids; + char* primary_mirror_uuid; + char* primary_snap_id; +} rbd_group_snap_mirror_namespace_t; + typedef enum { RBD_LOCK_MODE_EXCLUSIVE = 0, RBD_LOCK_MODE_SHARED = 1, @@ -1647,6 +1655,11 @@ CEPH_RBD_API int rbd_mirror_group_list(rados_ioctx_t p, char *names, size_t *size); CEPH_RBD_API int rbd_mirror_group_enable(rados_ioctx_t p, const char *name, rbd_mirror_image_mode_t mirror_image_mode); +CEPH_RBD_API int rbd_group_snap_get_mirror_namespace( + rados_ioctx_t p, const char *group_name, const char *snap_id, + rbd_group_snap_mirror_namespace_t *mirror_namespace); +CEPH_RBD_API void rbd_group_snap_mirror_namespace_cleanup( + rbd_group_snap_mirror_namespace_t *mirror_namespace); CEPH_RBD_API int rbd_mirror_group_disable(rados_ioctx_t p, const char *name, bool force); CEPH_RBD_API int rbd_mirror_group_promote(rados_ioctx_t p, diff --git a/src/include/rbd/librbd.hpp b/src/include/rbd/librbd.hpp index d6327996305..bab57b55ede 100644 --- a/src/include/rbd/librbd.hpp +++ b/src/include/rbd/librbd.hpp @@ -211,6 +211,13 @@ namespace librbd { typedef rbd_image_info_t image_info_t; + typedef struct { + snap_mirror_state_t state; + std::set mirror_peer_uuids; + std::string primary_mirror_uuid; + std::string primary_snap_id; + } group_snap_mirror_namespace_t; + class CEPH_RBD_API ProgressContext { public: @@ -524,6 +531,9 @@ public: int mirror_group_list(IoCtx& io_ctx, std::vector *names); int mirror_group_enable(IoCtx& io_ctx, const char *group_name, mirror_image_mode_t mirror_image_mode); + int group_snap_get_mirror_namespace( + IoCtx& group_ioctx, const char *group_name, const char *snap_id, + group_snap_mirror_namespace_t* mirror_namespace); int mirror_group_disable(IoCtx& io_ctx, const char *group_name, bool force); int mirror_group_promote(IoCtx& io_ctx, const char *group_name, bool force); int mirror_group_demote(IoCtx& io_ctx, const char *group_name); diff --git a/src/librbd/api/Group.cc b/src/librbd/api/Group.cc index 8d772a7d7f9..5b7052a353f 100644 --- a/src/librbd/api/Group.cc +++ b/src/librbd/api/Group.cc @@ -83,6 +83,28 @@ std::string calc_ind_image_snap_name(uint64_t pool_id, return ind_snap_name_stream.str(); } +class GetGroupMirrorVisitor { +public: + group_snap_mirror_namespace_t *mirror_snap; + + explicit GetGroupMirrorVisitor(group_snap_mirror_namespace_t *mirror_snap) + : mirror_snap(mirror_snap) {} + + template + inline int operator()(const T&) const { + return -EINVAL; + } + + inline int operator()( + const cls::rbd::GroupSnapshotNamespaceMirror& snap_namespace) { + mirror_snap->state = static_cast(snap_namespace.state); + mirror_snap->mirror_peer_uuids = snap_namespace.mirror_peer_uuids; + mirror_snap->primary_mirror_uuid = snap_namespace.primary_mirror_uuid; + mirror_snap->primary_snap_id = snap_namespace.primary_snap_id; + return 0; + } +}; + int group_image_list(librados::IoCtx& group_ioctx, const std::string &group_name, std::vector *images) { @@ -1271,6 +1293,46 @@ int Group::group_image_list_by_id(librados::IoCtx& group_ioctx, return 0; } +template +int Group::snap_get_mirror_namespace( + librados::IoCtx& group_ioctx, const char *group_name, const char *snap_id, + group_snap_mirror_namespace_t* mirror_namespace) { + CephContext *cct = (CephContext *)group_ioctx.cct(); + + std::string group_id; + int r = cls_client::dir_get_id(&group_ioctx, RBD_GROUP_DIRECTORY, + group_name, &group_id); + if (r < 0) { + lderr(cct) << "error getting the group id: " << cpp_strerror(r) << dendl; + return r; + } + + std::vector cls_group_snaps; + r = group_snap_list(group_ioctx, group_id, false, false, &cls_group_snaps); + if (r < 0) { + return r; + } + + const cls::rbd::GroupSnapshot *cls_group_snap_ptr = nullptr; + for (const auto& cls_group_snap : cls_group_snaps) { + if (cls_group_snap.id == snap_id) { + cls_group_snap_ptr = &cls_group_snap; + break; + } + } + if (cls_group_snap_ptr == nullptr) { + return -ENOENT; + } + + GetGroupMirrorVisitor visitor(mirror_namespace); + r = cls_group_snap_ptr->snapshot_namespace.visit(visitor); + if (r < 0) { + return r; + } + + return 0; +} + template int Group::group_image_remove(librados::IoCtx& group_ioctx, string group_id, librados::IoCtx& image_ioctx, diff --git a/src/librbd/api/Group.h b/src/librbd/api/Group.h index f09b55dc3a0..73646989112 100644 --- a/src/librbd/api/Group.h +++ b/src/librbd/api/Group.h @@ -69,6 +69,9 @@ struct Group { librados::IoCtx& image_ioctx, std::string image_id); + static int snap_get_mirror_namespace(librados::IoCtx& group_ioctx, + const char *group_name, const char *snap_id, + group_snap_mirror_namespace_t* mirror_namespace); }; } // namespace api diff --git a/src/librbd/librbd.cc b/src/librbd/librbd.cc index df1572524ac..51f87401846 100644 --- a/src/librbd/librbd.cc +++ b/src/librbd/librbd.cc @@ -1653,6 +1653,14 @@ namespace librbd { false, snaps); } + int RBD::group_snap_get_mirror_namespace( + IoCtx& group_ioctx, const char* group_name, const char* snap_id, + group_snap_mirror_namespace_t* mirror_namespace) { + return librbd::api::Group<>::snap_get_mirror_namespace(group_ioctx, + group_name, snap_id, + mirror_namespace); + } + int RBD::group_snap_get_info(IoCtx& group_ioctx, const char *group_name, const char *snap_name, group_snap_info2_t *group_snap) { @@ -7983,6 +7991,48 @@ extern "C" int rbd_group_snap_rollback(rados_ioctx_t group_p, return r; } +extern "C" int rbd_group_snap_get_mirror_namespace( + rados_ioctx_t group_p, const char *group_name, const char *snap_id, + rbd_group_snap_mirror_namespace_t* mirror_namespace) +{ + librados::IoCtx group_ioctx; + librados::IoCtx::from_rados_ioctx_t(group_p, group_ioctx); + + librbd::group_snap_mirror_namespace_t mirror_namespace_cpp; + int r = librbd::api::Group<>::snap_get_mirror_namespace( + group_ioctx, group_name, snap_id, &mirror_namespace_cpp); + if (r < 0) { + return r; + } + + mirror_namespace->state = mirror_namespace_cpp.state; + mirror_namespace->primary_mirror_uuid = + strdup(mirror_namespace_cpp.primary_mirror_uuid.c_str()); + mirror_namespace->primary_snap_id = + strdup(mirror_namespace_cpp.primary_snap_id.c_str()); + mirror_namespace->mirror_peer_uuids_count = + mirror_namespace_cpp.mirror_peer_uuids.size(); + size_t len = 0; + for (auto &peer : mirror_namespace_cpp.mirror_peer_uuids) { + len += peer.size() + 1; + } + mirror_namespace->mirror_peer_uuids = (char *)malloc(len); + char *p = mirror_namespace->mirror_peer_uuids; + for (auto &peer : mirror_namespace_cpp.mirror_peer_uuids) { + strncpy(p, peer.c_str(), peer.size() + 1); + p += peer.size() + 1; + } + + return 0; +} + +extern "C" void rbd_group_snap_mirror_namespace_cleanup( + rbd_group_snap_mirror_namespace_t *mirror_namespace) { + free(mirror_namespace->primary_mirror_uuid); + free(mirror_namespace->primary_snap_id); + free(mirror_namespace->mirror_peer_uuids); +} + extern "C" int rbd_group_snap_rollback_with_progress(rados_ioctx_t group_p, const char *group_name, const char *snap_name, diff --git a/src/pybind/rbd/c_rbd.pxd b/src/pybind/rbd/c_rbd.pxd index 0cca9c154f9..0dbc7199618 100644 --- a/src/pybind/rbd/c_rbd.pxd +++ b/src/pybind/rbd/c_rbd.pxd @@ -247,6 +247,7 @@ cdef extern from "rbd/librbd.h" nogil: ctypedef enum rbd_group_snap_namespace_type_t: _RBD_GROUP_SNAP_NAMESPACE_TYPE_USER "RBD_GROUP_SNAP_NAMESPACE_TYPE_USER" + _RBD_GROUP_SNAP_NAMESPACE_TYPE_MIRROR "RBD_GROUP_SNAP_NAMESPACE_TYPE_MIRROR" ctypedef struct rbd_group_image_snap_info_t: char *image_name @@ -262,6 +263,13 @@ cdef extern from "rbd/librbd.h" nogil: size_t image_snaps_count rbd_group_image_snap_info_t *image_snaps + ctypedef struct rbd_group_snap_mirror_namespace_t: + rbd_snap_mirror_state_t state; + size_t mirror_peer_uuids_count; + char* mirror_peer_uuids; + char* primary_mirror_uuid; + char* primary_snap_id; + ctypedef enum rbd_image_migration_state_t: _RBD_IMAGE_MIGRATION_STATE_UNKNOWN "RBD_IMAGE_MIGRATION_STATE_UNKNOWN" _RBD_IMAGE_MIGRATION_STATE_ERROR "RBD_IMAGE_MIGRATION_STATE_ERROR" @@ -794,6 +802,12 @@ cdef extern from "rbd/librbd.h" nogil: int rbd_group_snap_rollback(rados_ioctx_t group_p, const char *group_name, const char *snap_name) + int rbd_group_snap_get_mirror_namespace( + rados_ioctx_t group_p, const char *group_name, const char *snap_id, + rbd_group_snap_mirror_namespace_t* mirror_namespace) + void rbd_group_snap_mirror_namespace_cleanup( + rbd_group_snap_mirror_namespace_t *mirror_namespace) + int rbd_watchers_list(rbd_image_t image, rbd_image_watcher_t *watchers, size_t *max_watchers) void rbd_watchers_list_cleanup(rbd_image_watcher_t *watchers, diff --git a/src/pybind/rbd/mock_rbd.pxi b/src/pybind/rbd/mock_rbd.pxi index 1f77ee9d7eb..e0c3f7b9da2 100644 --- a/src/pybind/rbd/mock_rbd.pxi +++ b/src/pybind/rbd/mock_rbd.pxi @@ -935,6 +935,13 @@ cdef nogil: int rbd_group_snap_rollback(rados_ioctx_t group_p, const char *group_name, const char *snap_name): pass + int rbd_group_snap_get_mirror_namespace( + rados_ioctx_t group_p, const char *group_name, const char *snap_id, + rbd_group_snap_mirror_namespace_t* mirror_namespace): + pass + void rbd_group_snap_mirror_namespace_cleanup( + rbd_group_snap_mirror_namespace_t *mirror_namespace): + pass int rbd_watchers_list(rbd_image_t image, rbd_image_watcher_t *watchers, size_t *max_watchers): pass diff --git a/src/pybind/rbd/rbd.pyx b/src/pybind/rbd/rbd.pyx index 5e272e52289..2776a15090c 100644 --- a/src/pybind/rbd/rbd.pyx +++ b/src/pybind/rbd/rbd.pyx @@ -3035,6 +3035,49 @@ cdef class Group(object): if ret != 0: raise make_ex(ret, 'error rolling back group to snapshot', group_errno_to_exception) + def group_snap_get_mirror_namespace(self, snap_id): + """ + get the group snap mirror namespace details. + :param snap_id: the snapshot id of the mirror group snapshot + :type snap_id: str + + :returns: dict - contains the following keys: + + * ``state`` (int) - the group snapshot state + + * ``mirror_peer_uuids`` (list) - mirror peer uuids + + * ``primary_mirror_uuid`` (str) - primary mirror uuid + + * ``primary_snap_id`` (str) - primary snapshot id + + """ + snap_id = cstr(snap_id, 'snap_id') + cdef: + rbd_group_snap_mirror_namespace_t sn + char *_snap_id = snap_id + with nogil: + ret = rbd_group_snap_get_mirror_namespace( + self._ioctx, self._name, _snap_id, &sn) + if ret != 0: + raise make_ex(ret, 'error getting snapshot mirror ' + 'namespace for group: %s, snap_id: %s' % + (self.name, snap_id)) + uuids = [] + cdef char *p = sn.mirror_peer_uuids + for i in range(sn.mirror_peer_uuids_count): + uuid = decode_cstr(p) + uuids.append(uuid) + p += len(uuid) + 1 + info = { + 'state' : sn.state, + 'mirror_peer_uuids' : uuids, + 'primary_mirror_uuid' : decode_cstr(sn.primary_mirror_uuid), + 'primary_snap_id' : decode_cstr(sn.primary_snap_id), + } + rbd_group_snap_mirror_namespace_cleanup(&sn) + return info + def mirror_group_create_snapshot(self, flags=0): """ Create mirror group snapshot. diff --git a/src/tools/rbd/action/Group.cc b/src/tools/rbd/action/Group.cc index 79df51d1400..02d0a66d305 100644 --- a/src/tools/rbd/action/Group.cc +++ b/src/tools/rbd/action/Group.cc @@ -82,6 +82,19 @@ std::string get_group_snap_state_name(rbd_group_snap_state_t state) } } +std::string get_group_snap_namespace_name( + librbd::group_snap_namespace_type_t type) +{ + switch (type) { + case RBD_GROUP_SNAP_NAMESPACE_TYPE_USER: + return "user"; + case RBD_GROUP_SNAP_NAMESPACE_TYPE_MIRROR: + return "mirror"; + default: + return "unknown (" + stringify(type) + ")"; + } +} + int execute_create(const po::variables_map &vm, const std::vector &ceph_global_init_args) { size_t arg_index = 0; @@ -760,18 +773,73 @@ int execute_group_snap_list(const po::variables_map &vm, t.define_column("ID", TextTable::LEFT, TextTable::LEFT); t.define_column("NAME", TextTable::LEFT, TextTable::LEFT); t.define_column("STATE", TextTable::LEFT, TextTable::RIGHT); + t.define_column("NAMESPACE", TextTable::LEFT, TextTable::LEFT); } for (const auto& snap : snaps) { auto state_string = get_group_snap_state_name(snap.state); + auto type_string = get_group_snap_namespace_name(snap.namespace_type); + librbd::group_snap_mirror_namespace_t mirror_snap; + std::string mirror_snap_state = "unknown"; + if (snap.namespace_type == RBD_GROUP_SNAP_NAMESPACE_TYPE_MIRROR) { + r = rbd.group_snap_get_mirror_namespace(io_ctx, group_name.c_str(), + snap.id.c_str(), &mirror_snap); + if (r < 0) { + return r; + } + switch (mirror_snap.state) { + case RBD_SNAP_MIRROR_STATE_PRIMARY: + mirror_snap_state = "primary"; + break; + case RBD_SNAP_MIRROR_STATE_NON_PRIMARY: + mirror_snap_state = "non-primary"; + break; + case RBD_SNAP_MIRROR_STATE_PRIMARY_DEMOTED: + case RBD_SNAP_MIRROR_STATE_NON_PRIMARY_DEMOTED: + mirror_snap_state = "demoted"; + break; + } + } + if (f) { f->open_object_section("group_snap"); f->dump_string("id", snap.id); f->dump_string("snapshot", snap.name); f->dump_string("state", state_string); + f->open_object_section("namespace"); + f->dump_string("type", type_string); + if (snap.namespace_type == RBD_GROUP_SNAP_NAMESPACE_TYPE_MIRROR) { + f->dump_string("state", mirror_snap_state); + f->open_array_section("mirror_peer_uuids"); + for (auto &uuid : mirror_snap.mirror_peer_uuids) { + f->dump_string("peer_uuid", uuid); + } + f->close_section(); + if (mirror_snap.state == RBD_SNAP_MIRROR_STATE_NON_PRIMARY || + mirror_snap.state == RBD_SNAP_MIRROR_STATE_NON_PRIMARY_DEMOTED) { + f->dump_string("primary_mirror_uuid", + mirror_snap.primary_mirror_uuid); + f->dump_string("primary_snap_id", mirror_snap.primary_snap_id); + } + } + f->close_section(); // namespace f->close_section(); } else { - t << snap.id << snap.name << state_string << TextTable::endrow; + t << snap.id << snap.name << state_string; + std::ostringstream oss; + oss << type_string; + if (snap.namespace_type == RBD_GROUP_SNAP_NAMESPACE_TYPE_MIRROR) { + oss << " (" << mirror_snap_state << " " + << "peer_uuids:[" << mirror_snap.mirror_peer_uuids << "]"; + if (mirror_snap.state == RBD_SNAP_MIRROR_STATE_NON_PRIMARY || + mirror_snap.state == RBD_SNAP_MIRROR_STATE_NON_PRIMARY_DEMOTED) { + oss << " " << mirror_snap.primary_mirror_uuid << ":" + << mirror_snap.primary_snap_id; + } + oss << ")"; + } + t << oss.str(); + t << TextTable::endrow; } } diff --git a/src/tools/rbd/action/MirrorGroup.cc b/src/tools/rbd/action/MirrorGroup.cc index 1b18d366bc9..4aaeefdb087 100644 --- a/src/tools/rbd/action/MirrorGroup.cc +++ b/src/tools/rbd/action/MirrorGroup.cc @@ -442,7 +442,7 @@ int execute_status(const po::variables_map &vm, auto name_it = peer_mirror_uuids_to_name.find(status.mirror_uuid); formatter->dump_string("site_name", (name_it != peer_mirror_uuids_to_name.end() ? name_it->second : "")); - formatter->dump_string("mirror_uuids", status.mirror_uuid); + formatter->dump_string("mirror_uuid", status.mirror_uuid); formatter->dump_string("state", utils::mirror_group_site_status_state( status)); diff --git a/src/tools/rbd/action/MirrorPool.cc b/src/tools/rbd/action/MirrorPool.cc index ad6ce780677..86e83680a12 100644 --- a/src/tools/rbd/action/MirrorPool.cc +++ b/src/tools/rbd/action/MirrorPool.cc @@ -829,6 +829,36 @@ int get_mirror_image_status( return 0; } +int get_mirror_group_status( + librados::IoCtx& io_ctx, uint32_t* total_groups, + std::map* mirror_group_states, + MirrorHealth* mirror_group_health) { + librbd::RBD rbd; + int r = rbd.mirror_group_status_summary(io_ctx, mirror_group_states); + if (r < 0) { + std::cerr << "rbd: failed to get status summary for mirrored groups: " + << cpp_strerror(r) << std::endl; + return r; + } + + *mirror_group_health = MIRROR_HEALTH_OK; + for (auto &it : *mirror_group_states) { + auto &state = it.first; + if (*mirror_group_health < MIRROR_HEALTH_WARNING && + (state != MIRROR_GROUP_STATUS_STATE_REPLAYING && + state != MIRROR_GROUP_STATUS_STATE_STOPPED)) { + *mirror_group_health = MIRROR_HEALTH_WARNING; + } + if (*mirror_group_health < MIRROR_HEALTH_ERROR && + state == MIRROR_GROUP_STATUS_STATE_ERROR) { + *mirror_group_health = MIRROR_HEALTH_ERROR; + } + *total_groups += it.second; + } + + return 0; +} + } // anonymous namespace void get_peer_bootstrap_create_arguments(po::options_description *positional, @@ -1576,13 +1606,21 @@ int execute_status(const po::variables_map &vm, librbd::RBD rbd; uint32_t total_images = 0; + uint32_t total_groups = 0; std::map mirror_image_states; + std::map mirror_group_states; MirrorHealth mirror_image_health = MIRROR_HEALTH_UNKNOWN; + MirrorHealth mirror_group_health = MIRROR_HEALTH_UNKNOWN; r = get_mirror_image_status(io_ctx, &total_images, &mirror_image_states, &mirror_image_health); if (r < 0) { return r; } + r = get_mirror_group_status(io_ctx, &total_groups, &mirror_group_states, + &mirror_group_health); + if (r < 0) { + return r; + } MirrorDaemonServiceInfo daemon_service_info(io_ctx); daemon_service_info.init(); @@ -1590,7 +1628,9 @@ int execute_status(const po::variables_map &vm, MirrorHealth mirror_daemon_health = daemon_service_info.get_daemon_health(); auto mirror_services = daemon_service_info.get_mirror_services(); - auto mirror_health = std::max(mirror_image_health, mirror_daemon_health); + auto mirror_health = std::max(std::max(mirror_image_health, + mirror_group_health), + mirror_daemon_health); if (formatter != nullptr) { formatter->open_object_section("status"); @@ -1598,26 +1638,37 @@ int execute_status(const po::variables_map &vm, formatter->dump_stream("health") << mirror_health; formatter->dump_stream("daemon_health") << mirror_daemon_health; formatter->dump_stream("image_health") << mirror_image_health; - formatter->open_object_section("states"); + formatter->dump_stream("group_health") << mirror_group_health; + formatter->open_object_section("image_states"); for (auto &it : mirror_image_states) { std::string state_name = utils::mirror_image_status_state(it.first); formatter->dump_int(state_name.c_str(), it.second); } - formatter->close_section(); // states + formatter->close_section(); // image_states + formatter->open_object_section("group_states"); + for (auto &it : mirror_group_states) { + std::string state_name = utils::mirror_group_status_state(it.first); + formatter->dump_int(state_name.c_str(), it.second); + } + formatter->close_section(); // group_states formatter->close_section(); // summary } else { std::cout << "health: " << mirror_health << std::endl; std::cout << "daemon health: " << mirror_daemon_health << std::endl; std::cout << "image health: " << mirror_image_health << std::endl; + std::cout << "group health: " << mirror_group_health << std::endl; std::cout << "images: " << total_images << " total" << std::endl; for (auto &it : mirror_image_states) { std::cout << " " << it.second << " " << utils::mirror_image_status_state(it.first) << std::endl; } + std::cout << "groups: " << total_groups << " total" << std::endl; + for (auto &it : mirror_group_states) { + std::cout << " " << it.second << " " + << utils::mirror_group_status_state(it.first) << std::endl; + } } - int ret = 0; - if (verbose) { // dump per-daemon status if (formatter != nullptr) { @@ -1706,14 +1757,121 @@ int execute_status(const po::variables_map &vm, ImageRequestGenerator generator( io_ctx, instance_ids, mirror_peers, peer_mirror_uuids_to_name, daemon_service_info, formatter, &saw_image); - ret = generator.execute(); + r = generator.execute(); + if (r < 0) { + return r; + } if (formatter != nullptr) { formatter->close_section(); // images + formatter->open_array_section("groups"); } else { if (saw_image == false) { std::cout << std::endl << " none" << std::endl; } + std::cout << std::endl << "GROUPS"; + } + + std::map mirror_groups; + r = rbd.mirror_group_global_status_list(io_ctx, "", 1024, &mirror_groups); + if (r < 0) { + std::cerr << "rbd: failed to get group status list: " + << cpp_strerror(r) << std::endl; + return r; + } + + for (auto& [group_id, group_global_status] : mirror_groups) { + utils::populate_unknown_mirror_group_site_statuses(mirror_peers, + &group_global_status); + + librbd::mirror_group_site_status_t group_local_status; + int local_site_r = utils::get_local_mirror_group_status( + group_global_status, &group_local_status); + group_global_status.site_statuses.erase( + std::remove_if(group_global_status.site_statuses.begin(), + group_global_status.site_statuses.end(), + [](auto& group_status) { + return (group_status.mirror_uuid == + RBD_MIRROR_GROUP_STATUS_LOCAL_MIRROR_UUID); + }), + group_global_status.site_statuses.end()); + + if (formatter != nullptr) { + formatter->open_object_section("group"); + formatter->dump_string("name", group_global_status.name); + formatter->dump_string("global_id", group_global_status.info.global_id); + if (local_site_r >= 0) { + formatter->dump_string("state", utils::mirror_group_site_status_state( + group_local_status)); + formatter->dump_string("description", group_local_status.description); + formatter->dump_string("last_update", utils::timestr( + group_local_status.last_update)); + } + if (!group_global_status.site_statuses.empty()) { + formatter->open_array_section("peer_sites"); + for (auto& status : group_global_status.site_statuses) { + formatter->open_object_section("peer_site"); + + auto name_it = peer_mirror_uuids_to_name.find(status.mirror_uuid); + formatter->dump_string("site_name", + (name_it != peer_mirror_uuids_to_name.end() ? + name_it->second : "")); + formatter->dump_string("mirror_uuid", status.mirror_uuid); + + formatter->dump_string( + "state", utils::mirror_group_site_status_state(status)); + formatter->dump_string("description", status.description); + formatter->dump_string("last_update", utils::timestr( + status.last_update)); + formatter->close_section(); // peer_site + } + formatter->close_section(); // peer_sites + } + formatter->close_section(); // group + } else { + std::cout << std::endl + << group_global_status.name << ":" << std::endl + << " global_id: " << group_global_status.info.global_id + << std::endl; + if (local_site_r >= 0) { + std::cout << " state: " + << utils::mirror_group_site_status_state(group_local_status) + << std::endl + << " description: " << group_local_status.description + << std::endl + << " last_update: " << utils::timestr( + group_local_status.last_update) << std::endl; + } + if (!group_global_status.site_statuses.empty()) { + std::cout << " peer_sites:" << std::endl; + bool first_site = true; + for (auto& site : group_global_status.site_statuses) { + if (!first_site) { + std::cout << std::endl; + } + first_site = false; + + auto name_it = peer_mirror_uuids_to_name.find(site.mirror_uuid); + std::cout << " name: " + << (name_it != peer_mirror_uuids_to_name.end() ? + name_it->second : site.mirror_uuid) + << std::endl + << " state: " << utils::mirror_group_site_status_state( + site) << std::endl + << " description: " << site.description << std::endl + << " last_update: " << utils::timestr( + site.last_update) << std::endl; + } + } + } + } + + if (formatter != nullptr) { + formatter->close_section(); // groups + } else { + if (mirror_groups.empty()) { + std::cout << std::endl << " none" << std::endl; + } } } @@ -1722,7 +1880,7 @@ int execute_status(const po::variables_map &vm, formatter->flush(std::cout); } - return ret; + return 0; } void get_promote_arguments(po::options_description *positional, -- 2.39.5