From 5c0c2883a73e89daac70046f8e290250928ddc0c Mon Sep 17 00:00:00 2001
From: Prasanna Kumar Kalever <prasanna.kalever@redhat.com>
Date: Fri, 14 Feb 2025 16:21:10 +0530
Subject: [PATCH] rbd_mirror: cleanup group status keys in the rbd_mirroring
 object

Keys & Values for "gremote_status_global_*" and "gstatus_global_*" are
getting readded in the rbd_mirroring object after they were removed at
group disable time as part of group_status_set(), as it doesn't defend
for disabled groups today.

Also librbd::cls_client::mirror_group_status_remove_down() was added in
the code but not levearaged, hence added code for unhappy path cleanup
as part of MirrorStatusWatcher::init() like it is calls
librbd::cls_client::mirror_image_status_remove_down() today.

fixes: issue#16

Signed-off-by: Prasanna Kumar Kalever <prasanna.kalever@redhat.com>
---
 src/cls/rbd/cls_rbd.cc                      | 18 +++++++-
 src/tools/rbd_mirror/MirrorStatusWatcher.cc | 48 +++++++++++++++++++--
 src/tools/rbd_mirror/MirrorStatusWatcher.h  |  4 ++
 3 files changed, 66 insertions(+), 4 deletions(-)

diff --git a/src/cls/rbd/cls_rbd.cc b/src/cls/rbd/cls_rbd.cc
index e578710e8c089..618f66f38b932 100644
--- a/src/cls/rbd/cls_rbd.cc
+++ b/src/cls/rbd/cls_rbd.cc
@@ -5985,7 +5985,23 @@ int group_status_set(cls_method_context_t hctx, const string &global_group_id,
   ondisk_status.up = false;
   ondisk_status.last_update = ceph_clock_now();
 
-  int r = cls_get_request_origin(hctx, &ondisk_status.origin);
+  std::string group_id;
+  int r = group_get_group_id(hctx, global_group_id, &group_id);
+  if (r < 0) {
+    return 0;
+  }
+
+  cls::rbd::MirrorGroup group;
+  r = group_get(hctx, group_id, &group);
+  if (r < 0) {
+    return 0;
+  }
+
+  if (group.state != cls::rbd::MIRROR_GROUP_STATE_ENABLED) {
+    return 0;
+  }
+
+  r = cls_get_request_origin(hctx, &ondisk_status.origin);
   ceph_assert(r == 0);
 
   bufferlist bl;
diff --git a/src/tools/rbd_mirror/MirrorStatusWatcher.cc b/src/tools/rbd_mirror/MirrorStatusWatcher.cc
index 3b78e14495c65..225ed1bf3bf3a 100644
--- a/src/tools/rbd_mirror/MirrorStatusWatcher.cc
+++ b/src/tools/rbd_mirror/MirrorStatusWatcher.cc
@@ -42,13 +42,55 @@ void MirrorStatusWatcher<I>::init(Context *on_finish) {
       register_watch(on_finish);
     });
 
+  remove_down_image_status(on_finish);
+}
+
+template <typename I>
+void MirrorStatusWatcher<I>::remove_down_image_status(Context *on_finish) {
+  dout(20) << dendl;
+
   librados::ObjectWriteOperation op;
   librbd::cls_client::mirror_image_status_remove_down(&op);
-  librados::AioCompletion *aio_comp = create_rados_callback(on_finish);
+  auto comp = create_rados_callback(
+    new LambdaContext([this, on_finish](int r) {
+      handle_remove_down_image_status(r, on_finish);
+    }));
 
-  int r = m_ioctx.aio_operate(RBD_MIRRORING, aio_comp, &op);
+  int r = m_ioctx.aio_operate(RBD_MIRRORING, comp, &op);
   ceph_assert(r == 0);
-  aio_comp->release();
+  comp->release();
+}
+
+template <typename I>
+void MirrorStatusWatcher<I>::handle_remove_down_image_status(int r,
+                                                             Context *on_finish) {
+  dout(20) << "r=" << r << dendl;
+
+  remove_down_group_status(on_finish);
+}
+
+template <typename I>
+void MirrorStatusWatcher<I>::remove_down_group_status(Context *on_finish) {
+  dout(20) << dendl;
+
+  librados::ObjectWriteOperation op;
+  librbd::cls_client::mirror_group_status_remove_down(&op);
+  auto comp = create_rados_callback(
+    new LambdaContext([this, on_finish](int r) {
+      handle_remove_down_group_status(r, on_finish);
+    }));
+
+  int r = m_ioctx.aio_operate(RBD_MIRRORING, comp, &op);
+  ceph_assert(r == 0);
+  comp->release();
+}
+
+template <typename I>
+void MirrorStatusWatcher<I>::handle_remove_down_group_status(int r,
+                                                             Context *on_finish) {
+  dout(20) << "r=" << r << dendl;
+
+  on_finish->complete(r);
 }
 
 template <typename I>
diff --git a/src/tools/rbd_mirror/MirrorStatusWatcher.h b/src/tools/rbd_mirror/MirrorStatusWatcher.h
index 3335e9e63ad6a..609e661e2bdbf 100644
--- a/src/tools/rbd_mirror/MirrorStatusWatcher.h
+++ b/src/tools/rbd_mirror/MirrorStatusWatcher.h
@@ -30,6 +30,10 @@ public:
   ~MirrorStatusWatcher() override;
 
   void init(Context *on_finish);
+  void remove_down_image_status(Context *on_finish);
+  void handle_remove_down_image_status(int r, Context *on_finish);
+  void remove_down_group_status(Context *on_finish);
+  void handle_remove_down_group_status(int r, Context *on_finish);
   void shut_down(Context *on_finish);
 
 protected:
-- 
2.39.5