From: Prasanna Kumar Kalever Date: Fri, 7 Mar 2025 09:51:59 +0000 (+0530) Subject: rbd-mirror: group remove status as part of Group Replayer X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=cfee4191f9bb4b607b66d1ea8be677e2d92f31db;p=ceph.git rbd-mirror: group remove status as part of Group Replayer Signed-off-by: Prasanna Kumar Kalever --- diff --git a/src/cls/rbd/cls_rbd_client.cc b/src/cls/rbd/cls_rbd_client.cc index f23cf974e2a19..c8d6d52b640f5 100644 --- a/src/cls/rbd/cls_rbd_client.cc +++ b/src/cls/rbd/cls_rbd_client.cc @@ -2892,6 +2892,20 @@ int mirror_group_status_get_summary_finish( return 0; } +int mirror_group_status_remove(librados::IoCtx *ioctx, + const std::string &global_group_id) { + librados::ObjectWriteOperation op; + mirror_group_status_remove(&op, global_group_id); + return ioctx->operate(RBD_MIRRORING, &op); +} + +void mirror_group_status_remove(librados::ObjectWriteOperation *op, + const std::string &global_group_id) { + bufferlist bl; + encode(global_group_id, bl); + op->exec("rbd", "mirror_group_status_remove", bl); +} + int mirror_group_status_remove_down(librados::IoCtx *ioctx) { librados::ObjectWriteOperation op; mirror_group_status_remove_down(&op); diff --git a/src/cls/rbd/cls_rbd_client.h b/src/cls/rbd/cls_rbd_client.h index d491acd452434..c15095bf68fd9 100644 --- a/src/cls/rbd/cls_rbd_client.h +++ b/src/cls/rbd/cls_rbd_client.h @@ -618,6 +618,10 @@ void mirror_group_status_get_summary_start( int mirror_group_status_get_summary_finish( ceph::buffer::list::const_iterator *iter, std::map *states); +int mirror_group_status_remove(librados::IoCtx *ioctx, + const std::string &global_group_id); +void mirror_group_status_remove(librados::ObjectWriteOperation *op, + const std::string &global_group_id); int mirror_group_status_remove_down(librados::IoCtx *ioctx); void mirror_group_status_remove_down(librados::ObjectWriteOperation *op); diff --git a/src/tools/rbd_mirror/GroupReplayer.cc b/src/tools/rbd_mirror/GroupReplayer.cc index 6dfda75c19d18..dc50e93f2a38b 100644 --- a/src/tools/rbd_mirror/GroupReplayer.cc +++ b/src/tools/rbd_mirror/GroupReplayer.cc @@ -327,6 +327,8 @@ void GroupReplayer::start(Context *on_finish, bool manual, bool restart) { m_image_replayer_index.clear(); m_manual_stop = false; m_finished = false; + m_delete_requested = false; + m_status_removed = false; ceph_assert(m_on_start_finish == nullptr); std::swap(m_on_start_finish, on_finish); } @@ -715,6 +717,7 @@ void GroupReplayer::finish_start_fail(int r, const std::string &desc) { if (r == -ECANCELED) { dout(10) << "start canceled" << dendl; } else if (r == -ENOENT) { + m_delete_requested = true; dout(10) << "mirroring group removed" << dendl; } else if (r == -EREMOTEIO) { dout(10) << "mirroring group demoted" << dendl; @@ -788,6 +791,23 @@ void GroupReplayer::shut_down(int r) { template void GroupReplayer::handle_shut_down(int r) { + dout(10) << "r=" << r << dendl; + + if (r == -ENOENT) { // group removed + if (!m_resync_requested) { + set_finished(true); + } + unregister_admin_socket_hook(); + } + + if (!m_status_removed) { + auto ctx = new LambdaContext([this, r](int) { + m_status_removed = true; + handle_shut_down(r); + }); + remove_group_status(m_delete_requested, ctx); + return; + } dout(10) << "stop complete" << dendl; Context *on_start = nullptr; @@ -801,13 +821,6 @@ void GroupReplayer::handle_shut_down(int r) { m_state = STATE_STOPPED; } - if (r == -ENOENT) { // group removed - if (!m_resync_requested) { - set_finished(true); - } - unregister_admin_socket_hook(); - } - if (on_start != nullptr) { dout(10) << "on start finish complete, r=" << r << dendl; on_start->complete(r); @@ -819,7 +832,6 @@ void GroupReplayer::handle_shut_down(int r) { } } - template void GroupReplayer::register_admin_socket_hook() { GroupReplayerAdminSocketHook *asok_hook; @@ -878,6 +890,50 @@ void GroupReplayer::reregister_admin_socket_hook() { register_admin_socket_hook(); } +template +void GroupReplayer::remove_group_status(bool force, Context *on_finish) +{ + auto ctx = new LambdaContext([this, force, on_finish](int) { + remove_group_status_remote(force, on_finish); + }); + + if (m_local_status_updater->mirror_group_exists(m_global_group_id)) { + dout(15) << "removing local mirror group status: " + << m_global_group_id << dendl; + if (force) { + m_local_status_updater->remove_mirror_group_status( + m_global_group_id, true, ctx); + } else { + m_local_status_updater->remove_refresh_mirror_group_status( + m_global_group_id, ctx); + } + return; + } + + ctx->complete(0); +} + +template +void GroupReplayer::remove_group_status_remote(bool force, Context *on_finish) +{ + if (m_remote_group_peer.mirror_status_updater != nullptr && + m_remote_group_peer.mirror_status_updater->mirror_group_exists(m_global_group_id)) { + dout(15) << "removing remote mirror group status: " + << m_global_group_id << dendl; + if (force) { + m_remote_group_peer.mirror_status_updater->remove_mirror_group_status( + m_global_group_id, true, on_finish); + } else { + m_remote_group_peer.mirror_status_updater->remove_refresh_mirror_group_status( + m_global_group_id, on_finish); + } + return; + } + if (on_finish) { + on_finish->complete(0); + } +} + template void GroupReplayer::set_mirror_group_status_update( cls::rbd::MirrorGroupStatusState state, const std::string &desc) { diff --git a/src/tools/rbd_mirror/GroupReplayer.h b/src/tools/rbd_mirror/GroupReplayer.h index ee36e7192b51c..21cff6c0090d9 100644 --- a/src/tools/rbd_mirror/GroupReplayer.h +++ b/src/tools/rbd_mirror/GroupReplayer.h @@ -217,6 +217,8 @@ private: bool m_manual_stop = false; bool m_finished = false; + bool m_delete_requested = false; + bool m_status_removed = false; AdminSocketHook *m_asok_hook = nullptr; @@ -273,6 +275,9 @@ private: void unregister_admin_socket_hook(); void reregister_admin_socket_hook(); + void remove_group_status(bool force, Context *on_finish); + void remove_group_status_remote(bool force, Context *on_finish); + void set_mirror_group_status_update(cls::rbd::MirrorGroupStatusState state, const std::string &desc); void wait_for_ops(); diff --git a/src/tools/rbd_mirror/MirrorStatusUpdater.cc b/src/tools/rbd_mirror/MirrorStatusUpdater.cc index 08db0dd5fa854..8f453d1e5b24c 100644 --- a/src/tools/rbd_mirror/MirrorStatusUpdater.cc +++ b/src/tools/rbd_mirror/MirrorStatusUpdater.cc @@ -277,18 +277,36 @@ void MirrorStatusUpdater::set_mirror_group_status( } } +template +void MirrorStatusUpdater::remove_refresh_mirror_group_status( + const std::string& global_group_id, + Context* on_finish) { + dout(15) << "global_group_id=" << global_group_id << dendl; + + if (try_remove_mirror_group_status(global_group_id, false, false, + on_finish)) { + m_threads->work_queue->queue(on_finish, 0); + } +} + template void MirrorStatusUpdater::remove_mirror_group_status( - const std::string& global_group_id, Context* on_finish) { - if (try_remove_mirror_group_status(global_group_id, on_finish)) { + const std::string& global_group_id, bool immediate_update, + Context* on_finish) { + dout(15) << "global_group_id=" << global_group_id << dendl; + if (try_remove_mirror_group_status(global_group_id, true, immediate_update, + on_finish)) { m_threads->work_queue->queue(on_finish, 0); } } template bool MirrorStatusUpdater::try_remove_mirror_group_status( - const std::string& global_group_id, Context* on_finish) { - dout(15) << "global_group_id=" << global_group_id << dendl; + const std::string& global_group_id, bool queue_update, + bool immediate_update, Context* on_finish) { + dout(15) << "global_group_id=" << global_group_id << ", " + << "queue_update=" << queue_update << ", " + << "immediate_update=" << immediate_update << dendl; std::unique_lock locker(m_lock); if ((m_update_in_flight && @@ -297,8 +315,10 @@ bool MirrorStatusUpdater::try_remove_mirror_group_status( m_update_global_group_ids.count(global_group_id) > 0)) { // if update is scheduled/in-progress, wait for it to complete on_finish = new LambdaContext( - [this, global_group_id, on_finish](int r) { - if (try_remove_mirror_group_status(global_group_id, on_finish)) { + [this, global_group_id, queue_update, immediate_update, + on_finish](int r) { + if (try_remove_mirror_group_status(global_group_id, queue_update, + immediate_update, on_finish)) { on_finish->complete(0); } }); @@ -307,6 +327,12 @@ bool MirrorStatusUpdater::try_remove_mirror_group_status( } m_global_group_status.erase(global_group_id); + if (queue_update) { + m_update_global_group_ids.insert(global_group_id); + if (immediate_update) { + queue_update_task(std::move(locker)); + } + } return true; } @@ -498,6 +524,8 @@ void MirrorStatusUpdater::update_task(int r) { auto status_it = global_group_status.find(global_group_id); if (status_it == global_group_status.end()) { + librbd::cls_client::mirror_group_status_remove(&op, global_group_id); + ++op_count; continue; } diff --git a/src/tools/rbd_mirror/MirrorStatusUpdater.h b/src/tools/rbd_mirror/MirrorStatusUpdater.h index ce1f9a1dd17a3..998dcb1628c23 100644 --- a/src/tools/rbd_mirror/MirrorStatusUpdater.h +++ b/src/tools/rbd_mirror/MirrorStatusUpdater.h @@ -54,7 +54,9 @@ public: const cls::rbd::MirrorGroupSiteStatus& mirror_group_site_status, bool immediate_update); void remove_mirror_group_status(const std::string& global_group_id, - Context* on_finish); + bool immediate_update, Context* on_finish); + void remove_refresh_mirror_group_status(const std::string& global_group_id, + Context* on_finish); bool mirror_group_image_exists(const std::string& global_group_id, int64_t image_pool_id, @@ -123,6 +125,7 @@ private: Context* on_finish); bool try_remove_mirror_group_status(const std::string& global_image_id, + bool queue_update, bool immediate_update, Context* on_finish); void init_mirror_status_watcher(Context* on_finish);