From: Prasanna Kumar Kalever <prasanna.kalever@redhat.com>
Date: Fri, 3 May 2024 08:16:53 +0000 (+0530)
Subject: rbd-mirror: implement group resync functionality
X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=77a0e1110826f85c125da1ac5faf6c93277d3255;p=ceph.git

rbd-mirror: implement group resync functionality

Signed-off-by: Prasanna Kumar Kalever <prasanna.kalever@redhat.com>
---

diff --git a/src/cls/rbd/cls_rbd.cc b/src/cls/rbd/cls_rbd.cc
index bb11907bf8e32..fd89befb3c7a4 100644
--- a/src/cls/rbd/cls_rbd.cc
+++ b/src/cls/rbd/cls_rbd.cc
@@ -4674,6 +4674,11 @@ std::string group_global_key(const string &global_id) {
   return GROUP_GLOBAL_KEY_PREFIX + global_id;
 }
 
+std::string group_resync_key(const std::string& global_group_id,
+                             const std::string& group_name) {
+  return group_name + "_" + global_group_id;
+}
+
 std::string group_remote_status_global_key(const std::string& global_id,
                                            const std::string& mirror_uuid) {
   return GROUP_REMOTE_STATUS_GLOBAL_KEY_PREFIX + global_id + "_" + mirror_uuid;
@@ -7656,6 +7661,127 @@ int mirror_group_list(cls_method_context_t hctx, bufferlist *in,
   return 0;
 }
 
+/**
+ * Input:
+ * @param global_group_id (std::string)
+ * @param global_name (std::string)
+ *
+ * Output:
+ * @param group_id (std::string)
+ * @returns 0 on success, negative error code on failure
+ */
+int mirror_group_resync_get(cls_method_context_t hctx, bufferlist *in,
+                            bufferlist *out) {
+  std::string global_group_id;
+  std::string group_name;
+  try {
+    auto it = in->cbegin();
+    decode(global_group_id, it);
+    decode(group_name, it);
+  } catch (const ceph::buffer::error &err) {
+    return -EINVAL;
+  }
+
+  std::string search = mirror::group_resync_key(global_group_id, group_name);
+  std::string last_read = group_name;
+  int max_read = 5;
+  bool more = true;
+  do {
+    std::set<std::string> keys;
+    int r = cls_cxx_map_get_keys(hctx, last_read, max_read, &keys, &more);
+    if (r < 0) {
+      CLS_ERR("error reading group resync keys, global_id '%s': '%s'",
+          global_group_id.c_str(), cpp_strerror(r).c_str());
+      return r;
+    }
+
+    for (auto& key : keys) {
+      if (key == search) {
+        r = cls_cxx_map_get_val(hctx, key, out);
+        if (r < 0) {
+          CLS_ERR("error reading group_id for group resync, global_id '%s': '%s'",
+              global_group_id.c_str(), cpp_strerror(r).c_str());
+          return r;
+        }
+        return 0;
+      }
+    }
+
+    if (!keys.empty()) {
+      last_read = *keys.rbegin();
+    }
+  } while (more);
+
+  // shouldn't reach here
+  return -EINVAL;
+}
+
+/**
+ * Input:
+ * @param global_group_id (std::string)
+ * @param global_name (std::string)
+ * @param group_id (std::string)
+ *
+ * Output:
+ * @returns 0 on success, negative error code on failure
+ */
+int mirror_group_resync_set(cls_method_context_t hctx, bufferlist *in,
+		            bufferlist *out) {
+  std::string global_group_id;
+  std::string group_name;
+  std::string group_id;
+  try {
+    auto it = in->cbegin();
+    decode(global_group_id, it);
+    decode(group_name, it);
+    decode(group_id, it);
+  } catch (const ceph::buffer::error &err) {
+    return -EINVAL;
+  }
+  std::string key = mirror::group_resync_key(global_group_id, group_name);
+  bufferlist val_bl;
+  encode(group_id, val_bl);
+  int r = cls_cxx_map_set_val(hctx, key, &val_bl);
+  if (r < 0) {
+    CLS_ERR("error setting key %s on mirror group resync object: %s",
+            key.c_str(), cpp_strerror(r).c_str());
+    return r;
+  }
+
+  return 0;
+}
+
+/**
+ * Input:
+ * @param global_group_id (std::string)
+ * @param global_name (std::string)
+ *
+ * Output:
+ * @returns 0 on success, negative error code on failure
+ */
+int mirror_group_resync_remove(cls_method_context_t hctx, bufferlist *in,
+			       bufferlist *out) {
+  std::string global_group_id;
+  std::string group_name;
+  try {
+    auto it = in->cbegin();
+    decode(global_group_id, it);
+    decode(group_name, it);
+  } catch (const ceph::buffer::error &err) {
+    return -EINVAL;
+  }
+
+  std::string key = mirror::group_resync_key(global_group_id, group_name);
+  int r = cls_cxx_map_remove_key(hctx, key);
+  if (r < 0) {
+    CLS_ERR("error removing key %s from mirror group resync object: %s",
+            key.c_str(), cpp_strerror(r).c_str());
+    return r;
+  }
+
+  return 0;
+}
+
 /**
  * Input:
  * @param global_id (std::string)
@@ -9573,6 +9699,9 @@ CLS_INIT(rbd)
   cls_method_handle_t h_mirror_image_snapshot_unlink_peer;
   cls_method_handle_t h_mirror_image_snapshot_set_copy_progress;
   cls_method_handle_t h_mirror_group_list;
+  cls_method_handle_t h_mirror_group_resync_get;
+  cls_method_handle_t h_mirror_group_resync_set;
+  cls_method_handle_t h_mirror_group_resync_remove;
   cls_method_handle_t h_mirror_group_get_group_id;
   cls_method_handle_t h_mirror_group_get;
   cls_method_handle_t h_mirror_group_set;
@@ -9949,6 +10078,16 @@ CLS_INIT(rbd)
                           &h_mirror_image_snapshot_set_copy_progress);
   cls_register_cxx_method(h_class, "mirror_group_list", CLS_METHOD_RD,
                           mirror_group_list, &h_mirror_group_list);
+  cls_register_cxx_method(h_class, "mirror_group_resync_get", CLS_METHOD_RD,
+                          mirror_group_resync_get,
+                          &h_mirror_group_resync_get);
+  cls_register_cxx_method(h_class, "mirror_group_resync_set",
+                          CLS_METHOD_RD | CLS_METHOD_WR,
+                          mirror_group_resync_set, &h_mirror_group_resync_set);
+  cls_register_cxx_method(h_class, "mirror_group_resync_remove",
+                          CLS_METHOD_RD | CLS_METHOD_WR,
+                          mirror_group_resync_remove,
+                          &h_mirror_group_resync_remove);
   cls_register_cxx_method(h_class, "mirror_group_get_group_id", CLS_METHOD_RD,
                           mirror_group_get_group_id,
                           &h_mirror_group_get_group_id);
diff --git a/src/cls/rbd/cls_rbd_client.cc b/src/cls/rbd/cls_rbd_client.cc
index 57b2528447295..d618289905a71 100644
--- a/src/cls/rbd/cls_rbd_client.cc
+++ b/src/cls/rbd/cls_rbd_client.cc
@@ -2643,6 +2643,93 @@ int mirror_group_list(librados::IoCtx *ioctx,
   return mirror_group_list_finish(&bl_it, groups);
 }
 
+void mirror_group_resync_get_start(librados::ObjectReadOperation *op,
+                                   const std::string &global_group_id,
+                                   const std::string &group_name)
+{
+  bufferlist in_bl;
+  encode(global_group_id, in_bl);
+  encode(group_name, in_bl);
+  op->exec("rbd", "mirror_group_resync_get", in_bl);
+}
+
+int mirror_group_resync_get_finish(bufferlist::const_iterator *it,
+                                   std::string *group_id) {
+  try {
+    decode(*group_id, *it);
+  } catch (const ceph::buffer::error &err) {
+    return -EBADMSG;
+  }
+  return 0;
+}
+
+int mirror_group_resync_get(librados::IoCtx *ioctx,
+                            const std::string &global_group_id,
+                            const std::string &group_name,
+                            std::string *group_id)
+{
+  librados::ObjectReadOperation op;
+  mirror_group_resync_get_start(&op, global_group_id, group_name);
+
+  bufferlist out_bl;
+  int r = ioctx->operate(RBD_GROUP_RESYNC, &op, &out_bl);
+  if (r < 0) {
+    return r;
+  }
+
+  auto bl_it = out_bl.cbegin();
+  return mirror_group_resync_get_finish(&bl_it, group_id);
+}
+
+void mirror_group_resync_set(librados::ObjectWriteOperation *op,
+                             const std::string &global_group_id,
+                             const std::string &group_name,
+                             const std::string &group_id) {
+  bufferlist bl;
+  encode(global_group_id, bl);
+  encode(group_name, bl);
+  encode(group_id, bl);
+
+  op->exec("rbd", "mirror_group_resync_set", bl);
+}
+
+int mirror_group_resync_set(librados::IoCtx *ioctx,
+                             const std::string &global_group_id,
+                             const std::string &group_name,
+                             const std::string &group_id) {
+  librados::ObjectWriteOperation op;
+  mirror_group_resync_set(&op, global_group_id, group_name, group_id);
+
+  int r = ioctx->operate(RBD_GROUP_RESYNC, &op);
+  if (r < 0) {
+    return r;
+  }
+  return 0;
+}
+
+void mirror_group_resync_remove(librados::ObjectWriteOperation *op,
+                                const std::string &global_group_id,
+                                const std::string &group_name) {
+  bufferlist bl;
+  encode(global_group_id, bl);
+  encode(group_name, bl);
+
+  op->exec("rbd", "mirror_group_resync_remove", bl);
+}
+
+int mirror_group_resync_remove(librados::IoCtx *ioctx,
+                               const std::string &global_group_id,
+                               const std::string &group_name) {
+  librados::ObjectWriteOperation op;
+  mirror_group_resync_remove(&op, global_group_id, group_name);
+
+  int r = ioctx->operate(RBD_GROUP_RESYNC, &op);
+  if (r < 0) {
+    return r;
+  }
+  return 0;
+}
+
 void mirror_group_get_group_id_start(librados::ObjectReadOperation *op,
                                      const std::string &global_group_id) {
   bufferlist in_bl;
diff --git a/src/cls/rbd/cls_rbd_client.h b/src/cls/rbd/cls_rbd_client.h
index ebe59cd364c7f..2744e80811594 100644
--- a/src/cls/rbd/cls_rbd_client.h
+++ b/src/cls/rbd/cls_rbd_client.h
@@ -562,6 +562,29 @@ int mirror_group_list_finish(ceph::buffer::list::const_iterator *it,
 int mirror_group_list(librados::IoCtx *ioctx,
                       const std::string &start, uint64_t max_return,
                       std::map<std::string, cls::rbd::MirrorGroup> *groups);
+void mirror_group_resync_get_start(librados::ObjectReadOperation *op,
+                                   const std::string &global_group_id,
+                                   const std::string &group_name);
+int mirror_group_resync_get_finish(bufferlist::const_iterator *it,
+                                   std::string *group_id);
+int mirror_group_resync_get(librados::IoCtx *ioctx,
+                            const std::string &global_group_id,
+                            const std::string &group_name,
+                            std::string *group_id);
+void mirror_group_resync_set(librados::ObjectWriteOperation *op,
+                             const std::string &global_group_id,
+                             const std::string &group_name,
+                             const std::string &group_id);
+int mirror_group_resync_set(librados::IoCtx *ioctx,
+                             const std::string &global_group_id,
+                             const std::string &group_name,
+                             const std::string &group_id);
+void mirror_group_resync_remove(librados::ObjectWriteOperation *op,
+                                const std::string &global_group_id,
+                                const std::string &group_name);
+int mirror_group_resync_remove(librados::IoCtx *ioctx,
+                               const std::string &global_group_id,
+                               const std::string &group_name);
 void mirror_group_get_group_id_start(librados::ObjectReadOperation *op,
                                      const std::string &global_group_id);
 int mirror_group_get_group_id_finish(ceph::buffer::list::const_iterator *it,
diff --git a/src/include/rbd_types.h b/src/include/rbd_types.h
index 35a1a8bc3c3bd..7abf2600bf0ce 100644
--- a/src/include/rbd_types.h
+++ b/src/include/rbd_types.h
@@ -110,6 +110,8 @@
 
 #define RBD_GROUP_DIRECTORY "rbd_group_directory"
 
+#define RBD_GROUP_RESYNC "rbd_group_resync"
+
 #define RBD_TRASH "rbd_trash"
 
 /**
diff --git a/src/librbd/api/Mirror.cc b/src/librbd/api/Mirror.cc
index 03dc0ff3025e9..41932de61590c 100644
--- a/src/librbd/api/Mirror.cc
+++ b/src/librbd/api/Mirror.cc
@@ -3201,7 +3201,6 @@ int Mirror<I>::group_resync(IoCtx& group_ioctx, const char *group_name) {
   CephContext *cct = (CephContext *)group_ioctx.cct();
   ldout(cct, 20) << "io_ctx=" << &group_ioctx
 		 << ", group_name=" << group_name << dendl;
-
   std::string group_id;
   int r = cls_client::dir_get_id(&group_ioctx, RBD_GROUP_DIRECTORY,
 				 group_name, &group_id);
@@ -3239,12 +3238,37 @@ int Mirror<I>::group_resync(IoCtx& group_ioctx, const char *group_name) {
   }
 
   if (state == cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY) {
-    lderr(cct) << "group " << group_name
+    lderr(cct) << "group=" << group_name
                << " is primary, cannot resync to itself" << dendl;
     return -EINVAL;
   }
 
-  // TODO: implement the group resync functionality
+  r = cls_client::mirror_group_resync_set(&group_ioctx,
+                                          mirror_group.global_group_id,
+                                          group_name, group_id);
+  if (r < 0) {
+    lderr(cct) << "setting group resync with global_group_id="
+               << mirror_group.global_group_id << " failed: "
+               << cpp_strerror(r) << dendl;
+    return r;
+  }
+
+  std::vector<cls::rbd::GroupImageStatus> images;
+  r = Group<I>::group_image_list_by_id(group_ioctx, group_id, &images);
+  if (r < 0) {
+    lderr(cct) << "listing images in group=" << group_name
+               << " failed: " << cpp_strerror(r) << dendl;
+    return r;
+  }
+
+  r = MirroringWatcher<I>::notify_group_updated(
+        group_ioctx, cls::rbd::MIRROR_GROUP_STATE_DISABLED, group_id,
+        mirror_group.global_group_id, images.size());
+  if (r < 0) {
+    lderr(cct) << "failed to notify mirroring group=" << group_name
+               << " updated: " << cpp_strerror(r) << dendl;
+    // not fatal
+  }
 
   return 0;
 }
diff --git a/src/test/rbd_mirror/test_mock_InstanceReplayer.cc b/src/test/rbd_mirror/test_mock_InstanceReplayer.cc
index 9c671dc050aee..d5fae11501b39 100644
--- a/src/test/rbd_mirror/test_mock_InstanceReplayer.cc
+++ b/src/test/rbd_mirror/test_mock_InstanceReplayer.cc
@@ -89,13 +89,16 @@ struct GroupReplayer<librbd::MockTestImageCtx> {
   }
 
   MOCK_METHOD0(destroy, void());
-  MOCK_METHOD2(start, void(Context *, bool));
+  MOCK_METHOD4(start, void(Context *, bool, bool, bool));
   MOCK_METHOD2(stop, void(Context *, bool));
-  MOCK_METHOD1(restart, void(Context*));
+  MOCK_METHOD2(restart, void(Context*, bool));
   MOCK_METHOD0(flush, void());
+  MOCK_METHOD0(sync_group_names, void());
   MOCK_METHOD1(print_status, void(Formatter *));
   MOCK_METHOD1(add_peer, void(const Peer<librbd::MockTestImageCtx>& peer));
+  MOCK_METHOD0(get_name, const std::string &());
   MOCK_METHOD0(get_global_group_id, const std::string &());
+  MOCK_METHOD0(get_local_group_id, const std::string &());
   MOCK_METHOD0(is_running, bool());
   MOCK_METHOD0(is_stopped, bool());
   MOCK_METHOD0(is_blocklisted, bool());
diff --git a/src/tools/rbd_mirror/GroupReplayer.cc b/src/tools/rbd_mirror/GroupReplayer.cc
index 061be0bc1d87c..2e78515f8fed5 100644
--- a/src/tools/rbd_mirror/GroupReplayer.cc
+++ b/src/tools/rbd_mirror/GroupReplayer.cc
@@ -312,9 +312,10 @@ void GroupReplayer<I>::set_state_description(int r, const std::string &desc) {
 }
 
 template <typename I>
-void GroupReplayer<I>::start(Context *on_finish, bool manual, bool restart) {
+void GroupReplayer<I>::start(Context *on_finish, bool manual,
+                             bool restart, bool resync) {
   dout(10) << "on_finish=" << on_finish << ", manual=" << manual
-           << ", restart=" << restart << dendl;
+           << ", restart=" << restart << ", resync=" << resync << dendl;
 
   int r = 0;
   {
@@ -338,6 +339,10 @@ void GroupReplayer<I>::start(Context *on_finish, bool manual, bool restart) {
       m_image_replayer_index.clear();
       m_get_remote_group_snap_ret_vals.clear();
       m_manual_stop = false;
+      m_finished = false;
+      if (resync) {
+        m_resync_requested = true;
+      }
       ceph_assert(m_on_start_finish == nullptr);
       std::swap(m_on_start_finish, on_finish);
     }
@@ -453,20 +458,28 @@ void GroupReplayer<I>::stop(Context *on_finish, bool manual, bool restart) {
 }
 
 template <typename I>
-void GroupReplayer<I>::restart(Context *on_finish) {
-  dout(10) << dendl;
+void GroupReplayer<I>::restart(Context *on_finish, bool resync) {
+  dout(10) << "resync=" << resync << dendl;
   {
     std::lock_guard locker{m_lock};
+    if (m_resync_requested) {
+      dout(10) << "resync is already in progress, cancelling restart" << dendl;
+      on_finish->complete(-ECANCELED);
+      return;
+    }
     m_restart_requested = true;
     m_on_start_finish = nullptr;
+    if (resync) {
+      m_resync_requested = true;
+    }
   }
 
   auto ctx = new LambdaContext(
-    [this, on_finish](int r) {
+    [this, on_finish, resync](int r) {
       if (r < 0) {
 	// Try start anyway.
       }
-      start(on_finish, true, true);
+      start(on_finish, true, true, resync);
     });
   stop(ctx, false, true);
 }
@@ -536,9 +549,9 @@ void GroupReplayer<I>::bootstrap_group() {
     m_threads, m_local_io_ctx, m_remote_group_peer.io_ctx, m_global_group_id,
     m_local_mirror_uuid, m_instance_watcher, m_local_status_updater,
     m_remote_group_peer.mirror_status_updater, m_cache_manager_handler,
-    m_pool_meta_cache, &m_local_group_id, &m_remote_group_id,
-    &m_local_group_snaps, &m_local_group_ctx, &m_image_replayers,
-    &m_image_replayer_index, ctx);
+    m_pool_meta_cache, m_resync_requested, &m_local_group_id,
+    &m_remote_group_id, &m_local_group_snaps, &m_local_group_ctx,
+    &m_image_replayers, &m_image_replayer_index, ctx);
 
   request->get();
   m_bootstrap_request = request;
@@ -554,6 +567,7 @@ void GroupReplayer<I>::handle_bootstrap_group(int r) {
   dout(10) << "r=" << r << dendl;
   {
     std::lock_guard locker{m_lock};
+    m_resync_requested = false;
     if (m_state == STATE_STOPPING || m_state == STATE_STOPPED) {
       dout(10) << "stop prevailed" <<dendl;
       return;
@@ -1094,8 +1108,6 @@ void GroupReplayer<I>::create_mirror_snapshot_start(
   auto requests_it = m_create_snap_requests.find(remote_group_snap_id);
 
   if (requests_it == m_create_snap_requests.end()) {
-    ceph_assert(m_local_group_snaps.count(remote_group_snap_id) == 0);
-
     requests_it = m_create_snap_requests.insert(
         {remote_group_snap_id, {}}).first;
 
@@ -1250,7 +1262,6 @@ void GroupReplayer<I>::handle_get_remote_group_snapshot(
   }
 
   m_get_remote_group_snap_ret_vals[remote_group_snap_id] = r;
-
   maybe_create_mirror_snapshot(locker, remote_group_snap_id);
 }
 
diff --git a/src/tools/rbd_mirror/GroupReplayer.h b/src/tools/rbd_mirror/GroupReplayer.h
index 92c76459bf4f1..c498b239e6ddc 100644
--- a/src/tools/rbd_mirror/GroupReplayer.h
+++ b/src/tools/rbd_mirror/GroupReplayer.h
@@ -79,7 +79,7 @@ public:
 
   std::string get_name() const {
     std::lock_guard l{m_lock};
-    return m_group_spec;
+    return m_local_group_name;
   }
   void set_state_description(int r, const std::string &desc);
 
@@ -114,12 +114,15 @@ public:
   inline const std::string& get_global_group_id() const {
     return m_global_group_id;
   }
+  inline const std::string& get_local_group_id() const {
+    return m_local_group_id;
+  }
 
   void start(Context *on_finish = nullptr, bool manual = false,
-             bool restart = false);
+             bool restart = false, bool resync = false);
   void stop(Context *on_finish = nullptr, bool manual = false,
             bool restart = false);
-  void restart(Context *on_finish = nullptr);
+  void restart(Context *on_finish = nullptr, bool resync = false);
   void flush();
 
   void print_status(Formatter *f);
@@ -233,6 +236,7 @@ private:
   Context *m_on_start_finish = nullptr;
   Context *m_on_stop_finish = nullptr;
   bool m_stop_requested = false;
+  bool m_resync_requested = false;
   bool m_restart_requested = false;
   bool m_manual_stop = false;
   bool m_finished = false;
diff --git a/src/tools/rbd_mirror/InstanceReplayer.cc b/src/tools/rbd_mirror/InstanceReplayer.cc
index e85521f8d0549..e0821c8d02ae3 100644
--- a/src/tools/rbd_mirror/InstanceReplayer.cc
+++ b/src/tools/rbd_mirror/InstanceReplayer.cc
@@ -2,6 +2,7 @@
 // vim: ts=8 sw=2 smarttab
 
 #include "include/stringify.h"
+#include "cls/rbd/cls_rbd_client.h"
 #include "common/Cond.h"
 #include "common/Timer.h"
 #include "common/debug.h"
@@ -654,10 +655,25 @@ void InstanceReplayer<I>::start_group_replayer(
   ceph_assert(ceph_mutex_is_locked(m_lock));
 
   std::string global_group_id = group_replayer->get_global_group_id();
+  std::string group_name = group_replayer->get_name();
+  std::string group_id;
+  bool resync_requested = false;
+  int r = librbd::cls_client::mirror_group_resync_get(&m_local_io_ctx,
+                                                      global_group_id,
+                                                      group_name,
+                                                      &group_id);
+  if (r < 0) {
+    derr << "getting mirror group resync for global_group_id="
+         << global_group_id << " failed: " << cpp_strerror(r) << dendl;
+  } else if (r == 0) {
+    if (group_id == group_replayer->get_local_group_id()) {
+      resync_requested = true;
+    }
+  }
   if (!group_replayer->is_stopped()) {
-    if (group_replayer->needs_restart()) {
-      stop_group_replayer(group_replayer, new C_TrackedOp(m_async_op_tracker,
-                                                          nullptr));
+    if (group_replayer->needs_restart() || resync_requested) {
+      group_replayer->restart(new C_TrackedOp(m_async_op_tracker, nullptr),
+                              resync_requested);
     } else {
       group_replayer->sync_group_names();
     }
@@ -669,17 +685,28 @@ void InstanceReplayer<I>::start_group_replayer(
     return;
   } else if (group_replayer->is_finished()) {
     // TODO temporary until policy integrated
-    dout(5) << "removing group replayer for global_group_id="
-            << global_group_id << dendl;
-    m_group_replayers.erase(group_replayer->get_global_group_id());
-    group_replayer->destroy();
-    return;
+    if (resync_requested) {
+      resync_requested = false;
+      r = librbd::cls_client::mirror_group_resync_remove(&m_local_io_ctx,
+                                                         global_group_id,
+                                                         group_name);
+      if (r < 0) {
+        derr << "removing mirror group resync for global_group_id="
+             << global_group_id << " failed: " << cpp_strerror(r) << dendl;
+      }
+    } else {
+      dout(5) << "removing group replayer for global_group_id="
+              << global_group_id << dendl;
+      m_group_replayers.erase(group_replayer->get_global_group_id());
+      group_replayer->destroy();
+      return;
+    }
   } else if (m_manual_stop) {
     return;
   }
-
   dout(10) << "global_group_id=" << global_group_id << dendl;
-  group_replayer->start(new C_TrackedOp(m_async_op_tracker, nullptr), false);
+  group_replayer->start(new C_TrackedOp(m_async_op_tracker, nullptr),
+                        false, false, resync_requested);
 }
 
 template <typename I>
diff --git a/src/tools/rbd_mirror/group_replayer/BootstrapRequest.cc b/src/tools/rbd_mirror/group_replayer/BootstrapRequest.cc
index aae48f8b7e927..dabe6c04b1227 100644
--- a/src/tools/rbd_mirror/group_replayer/BootstrapRequest.cc
+++ b/src/tools/rbd_mirror/group_replayer/BootstrapRequest.cc
@@ -78,6 +78,7 @@ BootstrapRequest<I>::BootstrapRequest(
     MirrorStatusUpdater<I> *remote_status_updater,
     journal::CacheManagerHandler *cache_manager_handler,
     PoolMetaCache *pool_meta_cache,
+    bool resync_requested,
     std::string *local_group_id,
     std::string *remote_group_id,
     std::map<std::string, cls::rbd::GroupSnapshot> *local_group_snaps,
@@ -98,6 +99,7 @@ BootstrapRequest<I>::BootstrapRequest(
     m_remote_status_updater(remote_status_updater),
     m_cache_manager_handler(cache_manager_handler),
     m_pool_meta_cache(pool_meta_cache),
+    m_resync_requested(resync_requested),
     m_local_group_id(local_group_id),
     m_remote_group_id(remote_group_id),
     m_local_group_snaps(local_group_snaps),
@@ -110,7 +112,11 @@ BootstrapRequest<I>::BootstrapRequest(
 
 template <typename I>
 void BootstrapRequest<I>::send() {
-  get_remote_group_id();
+  if (m_resync_requested) {
+    get_local_group_id();
+  } else {
+    get_remote_group_id();
+  }
 }
 
 template <typename I>
@@ -720,10 +726,11 @@ void BootstrapRequest<I>::handle_list_local_group_snapshots(int r) {
     if (r == -ENOENT) {
       derr << "failed to find local mirror group snapshot" << dendl;
     } else {
-      if (state == cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED) {
+      if (m_remote_mirror_group_primary &&
+          state == cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED) {
         // if local snapshot is primary demoted, check if there is demote snapshot
         // in remote, if not then split brain
-        if (!is_demoted_snap_exists(remote_group_snaps)) {
+        if (!is_demoted_snap_exists(remote_group_snaps) && !m_resync_requested) {
           finish(-EEXIST);
           return;
         }
@@ -954,7 +961,8 @@ void BootstrapRequest<I>::move_local_image_to_trash() {
       &BootstrapRequest<I>::handle_move_local_image_to_trash>(this);
 
   auto req = image_deleter::TrashMoveRequest<I>::create(
-      m_image_io_ctx, global_image_id, false, m_threads->work_queue, ctx);
+      m_image_io_ctx, global_image_id, m_resync_requested,
+      m_threads->work_queue, ctx);
   req->send();
 }
 
diff --git a/src/tools/rbd_mirror/group_replayer/BootstrapRequest.h b/src/tools/rbd_mirror/group_replayer/BootstrapRequest.h
index 171ad1dcef080..a3c834c926b5e 100644
--- a/src/tools/rbd_mirror/group_replayer/BootstrapRequest.h
+++ b/src/tools/rbd_mirror/group_replayer/BootstrapRequest.h
@@ -45,6 +45,7 @@ public:
       MirrorStatusUpdater<ImageCtxT> *remote_status_updater,
       journal::CacheManagerHandler *cache_manager_handler,
       PoolMetaCache *pool_meta_cache,
+      bool resync_requested,
       std::string *local_group_id,
       std::string *remote_group_id,
       std::map<std::string, cls::rbd::GroupSnapshot> *local_group_snaps,
@@ -55,9 +56,9 @@ public:
     return new BootstrapRequest(
       threads, local_io_ctx, remote_io_ctx, global_group_id, local_mirror_uuid,
       instance_watcher, local_status_updater, remote_status_updater,
-      cache_manager_handler, pool_meta_cache, local_group_id, remote_group_id,
-      local_group_snaps, local_group_ctx, image_replayers, image_replayer_index,
-      on_finish);
+      cache_manager_handler, pool_meta_cache, resync_requested, local_group_id,
+      remote_group_id, local_group_snaps, local_group_ctx, image_replayers,
+      image_replayer_index, on_finish);
   }
 
   BootstrapRequest(
@@ -71,6 +72,7 @@ public:
       MirrorStatusUpdater<ImageCtxT> *remote_status_updater,
       journal::CacheManagerHandler *cache_manager_handler,
       PoolMetaCache *pool_meta_cache,
+      bool resync_requested,
       std::string *local_group_id,
       std::string *remote_group_id,
       std::map<std::string, cls::rbd::GroupSnapshot> *local_group_snaps,
@@ -167,6 +169,7 @@ private:
   MirrorStatusUpdater<ImageCtxT> *m_remote_status_updater;
   journal::CacheManagerHandler *m_cache_manager_handler;
   PoolMetaCache *m_pool_meta_cache;
+  bool m_resync_requested = false;
   std::string *m_local_group_id;
   std::string *m_remote_group_id;
   std::map<std::string, cls::rbd::GroupSnapshot> *m_local_group_snaps;
diff --git a/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.cc b/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.cc
index a0a9babd63f73..4695ad0f6c63b 100644
--- a/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.cc
+++ b/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.cc
@@ -538,7 +538,8 @@ void Replayer<I>::scan_local_mirror_snapshots(
 
     m_prune_snap_id = *prune_snap_ids.begin();
     dout(5) << "pruning unused non-primary snapshot " << m_prune_snap_id << dendl;
-    unlink_group_snapshot();
+    prune_non_primary_snapshot();
+    //unlink_group_snapshot();  //PK: FIXME
     return;
   }