rbd-mirror: group remove status as part of Group Replayer

Signed-off-by: Prasanna Kumar Kalever <prasanna.kalever@redhat.com>

diff --git a/src/cls/rbd/cls_rbd_client.cc b/src/cls/rbd/cls_rbd_client.cc
index f23cf974e2a194475d851c252d28edafce8eefde..c8d6d52b640f55b530586e3b2d996fd01c69bb32 100644 (file)
--- a/src/cls/rbd/cls_rbd_client.cc
+++ b/src/cls/rbd/cls_rbd_client.cc
@@ -2892,6 +2892,20 @@ int mirror_group_status_get_summary_finish(
   return 0;
 }
 
+int mirror_group_status_remove(librados::IoCtx *ioctx,
+                               const std::string &global_group_id) {
+  librados::ObjectWriteOperation op;
+  mirror_group_status_remove(&op, global_group_id);
+  return ioctx->operate(RBD_MIRRORING, &op);
+}
+
+void mirror_group_status_remove(librados::ObjectWriteOperation *op,
+                                const std::string &global_group_id) {
+  bufferlist bl;
+  encode(global_group_id, bl);
+  op->exec("rbd", "mirror_group_status_remove", bl);
+}
+
 int mirror_group_status_remove_down(librados::IoCtx *ioctx) {
   librados::ObjectWriteOperation op;
   mirror_group_status_remove_down(&op);

diff --git a/src/cls/rbd/cls_rbd_client.h b/src/cls/rbd/cls_rbd_client.h
index d491acd452434ab0c53cbbc58cd7c6fba1db435d..c15095bf68fd9ebfd9dddea363989b2ef6554304 100644 (file)
--- a/src/cls/rbd/cls_rbd_client.h
+++ b/src/cls/rbd/cls_rbd_client.h
@@ -618,6 +618,10 @@ void mirror_group_status_get_summary_start(
 int mirror_group_status_get_summary_finish(
     ceph::buffer::list::const_iterator *iter,
     std::map<cls::rbd::MirrorGroupStatusState, int32_t> *states);
+int mirror_group_status_remove(librados::IoCtx *ioctx,
+                               const std::string &global_group_id);
+void mirror_group_status_remove(librados::ObjectWriteOperation *op,
+                                const std::string &global_group_id);
 int mirror_group_status_remove_down(librados::IoCtx *ioctx);
 void mirror_group_status_remove_down(librados::ObjectWriteOperation *op);
 

diff --git a/src/tools/rbd_mirror/GroupReplayer.cc b/src/tools/rbd_mirror/GroupReplayer.cc
index 6dfda75c19d18558d285dbc193b77773b85d154d..dc50e93f2a38b3d2e72bcc817ea546b897c9255a 100644 (file)
--- a/src/tools/rbd_mirror/GroupReplayer.cc
+++ b/src/tools/rbd_mirror/GroupReplayer.cc
@@ -327,6 +327,8 @@ void GroupReplayer<I>::start(Context *on_finish, bool manual, bool restart) {
       m_image_replayer_index.clear();
       m_manual_stop = false;
       m_finished = false;
+      m_delete_requested = false;
+      m_status_removed = false;
       ceph_assert(m_on_start_finish == nullptr);
       std::swap(m_on_start_finish, on_finish);
     }
@@ -715,6 +717,7 @@ void GroupReplayer<I>::finish_start_fail(int r, const std::string &desc) {
        if (r == -ECANCELED) {
          dout(10) << "start canceled" << dendl;
        } else if (r == -ENOENT) {
+          m_delete_requested = true;
          dout(10) << "mirroring group removed" << dendl;
        } else if (r == -EREMOTEIO) {
          dout(10) << "mirroring group demoted" << dendl;
@@ -788,6 +791,23 @@ void GroupReplayer<I>::shut_down(int r) {
 
 template <typename I>
 void GroupReplayer<I>::handle_shut_down(int r) {
+  dout(10) << "r=" << r << dendl;
+
+  if (r == -ENOENT) { // group removed
+    if (!m_resync_requested) {
+      set_finished(true);
+    }
+    unregister_admin_socket_hook();
+  }
+
+  if (!m_status_removed) {
+    auto ctx = new LambdaContext([this, r](int) {
+      m_status_removed = true;
+      handle_shut_down(r);
+    });
+    remove_group_status(m_delete_requested, ctx);
+    return;
+  }
 
   dout(10) << "stop complete" << dendl;
   Context *on_start = nullptr;
@@ -801,13 +821,6 @@ void GroupReplayer<I>::handle_shut_down(int r) {
     m_state = STATE_STOPPED;
   }
 
-  if (r == -ENOENT) { // group removed
-    if (!m_resync_requested) {
-      set_finished(true);
-    }
-    unregister_admin_socket_hook();
-  }
-
   if (on_start != nullptr) {
     dout(10) << "on start finish complete, r=" << r << dendl;
     on_start->complete(r);
@@ -819,7 +832,6 @@ void GroupReplayer<I>::handle_shut_down(int r) {
   }
 }
 
-
 template <typename I>
 void GroupReplayer<I>::register_admin_socket_hook() {
   GroupReplayerAdminSocketHook<I> *asok_hook;
@@ -878,6 +890,50 @@ void GroupReplayer<I>::reregister_admin_socket_hook() {
   register_admin_socket_hook();
 }
 
+template <typename I>
+void GroupReplayer<I>::remove_group_status(bool force, Context *on_finish)
+{
+  auto ctx = new LambdaContext([this, force, on_finish](int) {
+    remove_group_status_remote(force, on_finish);
+  });
+
+  if (m_local_status_updater->mirror_group_exists(m_global_group_id)) {
+    dout(15) << "removing local mirror group status: "
+             << m_global_group_id << dendl;
+    if (force) {
+      m_local_status_updater->remove_mirror_group_status(
+          m_global_group_id, true, ctx);
+    } else {
+      m_local_status_updater->remove_refresh_mirror_group_status(
+          m_global_group_id, ctx);
+    }
+    return;
+  }
+
+  ctx->complete(0);
+}
+
+template <typename I>
+void GroupReplayer<I>::remove_group_status_remote(bool force, Context *on_finish)
+{
+  if (m_remote_group_peer.mirror_status_updater != nullptr &&
+      m_remote_group_peer.mirror_status_updater->mirror_group_exists(m_global_group_id)) {
+    dout(15) << "removing remote mirror group status: "
+             << m_global_group_id << dendl;
+    if (force) {
+      m_remote_group_peer.mirror_status_updater->remove_mirror_group_status(
+          m_global_group_id, true, on_finish);
+    } else {
+      m_remote_group_peer.mirror_status_updater->remove_refresh_mirror_group_status(
+          m_global_group_id, on_finish);
+    }
+    return;
+  }
+  if (on_finish) {
+    on_finish->complete(0);
+  }
+}
+
 template <typename I>
 void GroupReplayer<I>::set_mirror_group_status_update(
     cls::rbd::MirrorGroupStatusState state, const std::string &desc) {

diff --git a/src/tools/rbd_mirror/GroupReplayer.h b/src/tools/rbd_mirror/GroupReplayer.h
index ee36e7192b51c30af3b83178e053010f09b2ee2c..21cff6c0090d979d556acfef7cdddeae3dd45258 100644 (file)
--- a/src/tools/rbd_mirror/GroupReplayer.h
+++ b/src/tools/rbd_mirror/GroupReplayer.h
@@ -217,6 +217,8 @@ private:
   bool m_manual_stop = false;
   bool m_finished = false;
 
+  bool m_delete_requested = false;
+  bool m_status_removed = false;
 
   AdminSocketHook *m_asok_hook = nullptr;
 
@@ -273,6 +275,9 @@ private:
   void unregister_admin_socket_hook();
   void reregister_admin_socket_hook();
 
+  void remove_group_status(bool force, Context *on_finish);
+  void remove_group_status_remote(bool force, Context *on_finish);
+
   void set_mirror_group_status_update(cls::rbd::MirrorGroupStatusState state,
                                       const std::string &desc);
   void wait_for_ops();

diff --git a/src/tools/rbd_mirror/MirrorStatusUpdater.cc b/src/tools/rbd_mirror/MirrorStatusUpdater.cc
index 08db0dd5fa85405142cbc476169598d77a15ff8a..8f453d1e5b24cda1ceed7afd6c27cbf9730486e4 100644 (file)
--- a/src/tools/rbd_mirror/MirrorStatusUpdater.cc
+++ b/src/tools/rbd_mirror/MirrorStatusUpdater.cc
@@ -277,18 +277,36 @@ void MirrorStatusUpdater<I>::set_mirror_group_status(
   }
 }
 
+template <typename I>
+void MirrorStatusUpdater<I>::remove_refresh_mirror_group_status(
+    const std::string& global_group_id,
+    Context* on_finish) {
+  dout(15) << "global_group_id=" << global_group_id << dendl;
+
+  if (try_remove_mirror_group_status(global_group_id, false, false,
+                                     on_finish)) {
+    m_threads->work_queue->queue(on_finish, 0);
+  }
+}
+
 template <typename I>
 void MirrorStatusUpdater<I>::remove_mirror_group_status(
-    const std::string& global_group_id, Context* on_finish) {
-  if (try_remove_mirror_group_status(global_group_id, on_finish)) {
+    const std::string& global_group_id, bool immediate_update,
+    Context* on_finish) {
+  dout(15) << "global_group_id=" << global_group_id << dendl;
+  if (try_remove_mirror_group_status(global_group_id, true, immediate_update,
+                                     on_finish)) {
     m_threads->work_queue->queue(on_finish, 0);
   }
 }
 
 template <typename I>
 bool MirrorStatusUpdater<I>::try_remove_mirror_group_status(
-    const std::string& global_group_id, Context* on_finish) {
-  dout(15) << "global_group_id=" << global_group_id << dendl;
+    const std::string& global_group_id, bool queue_update,
+    bool immediate_update, Context* on_finish) {
+  dout(15) << "global_group_id=" << global_group_id << ", "
+           << "queue_update=" << queue_update << ", "
+           << "immediate_update=" << immediate_update << dendl;
 
   std::unique_lock locker(m_lock);
   if ((m_update_in_flight &&
@@ -297,8 +315,10 @@ bool MirrorStatusUpdater<I>::try_remove_mirror_group_status(
        m_update_global_group_ids.count(global_group_id) > 0)) {
     // if update is scheduled/in-progress, wait for it to complete
     on_finish = new LambdaContext(
-      [this, global_group_id, on_finish](int r) {
-        if (try_remove_mirror_group_status(global_group_id, on_finish)) {
+      [this, global_group_id, queue_update, immediate_update,
+             on_finish](int r) {
+        if (try_remove_mirror_group_status(global_group_id, queue_update,
+                                           immediate_update, on_finish)) {
           on_finish->complete(0);
         }
       });
@@ -307,6 +327,12 @@ bool MirrorStatusUpdater<I>::try_remove_mirror_group_status(
   }
 
   m_global_group_status.erase(global_group_id);
+  if (queue_update) {
+    m_update_global_group_ids.insert(global_group_id);
+    if (immediate_update) {
+      queue_update_task(std::move(locker));
+    }
+  }
   return true;
 }
 
@@ -498,6 +524,8 @@ void MirrorStatusUpdater<I>::update_task(int r) {
 
       auto status_it = global_group_status.find(global_group_id);
       if (status_it == global_group_status.end()) {
+        librbd::cls_client::mirror_group_status_remove(&op, global_group_id);
+        ++op_count;
         continue;
       }
 

diff --git a/src/tools/rbd_mirror/MirrorStatusUpdater.h b/src/tools/rbd_mirror/MirrorStatusUpdater.h
index ce1f9a1dd17a34338a34be827b05db60e23e6aea..998dcb1628c23189eb5e063a8cc33e87403f921f 100644 (file)
--- a/src/tools/rbd_mirror/MirrorStatusUpdater.h
+++ b/src/tools/rbd_mirror/MirrorStatusUpdater.h
@@ -54,7 +54,9 @@ public:
       const cls::rbd::MirrorGroupSiteStatus& mirror_group_site_status,
       bool immediate_update);
   void remove_mirror_group_status(const std::string& global_group_id,
-                                  Context* on_finish);
+                                  bool immediate_update, Context* on_finish);
+  void remove_refresh_mirror_group_status(const std::string& global_group_id,
+                                          Context* on_finish);
 
   bool mirror_group_image_exists(const std::string& global_group_id,
                                  int64_t image_pool_id,
@@ -123,6 +125,7 @@ private:
                                       Context* on_finish);
 
   bool try_remove_mirror_group_status(const std::string& global_image_id,
+                                      bool queue_update, bool immediate_update,
                                       Context* on_finish);
 
   void init_mirror_status_watcher(Context* on_finish);