rbd-mirror: fix issues around resync

Following issues are fixed:
* Allow to flag resync, but if remote is not primary do not resync or
  even delete the local group.
   - Wait for remote to turn primary, if it turns primary, then continue
     to resync.
   - Just in case if the same site is made primary right after issuing
     resync, then clear that flag immediatly.
* Revert some old code in PoolWatcher, unintentional edits/changes.
* Do not send MIRROR_GROUP_STATE_DISABLED notification from group_resync
  API, this will lead to release_group(). Credits to Nithya Balachandran
  for pointing about this notification deatils.

Signed-off-by: Prasanna Kumar Kalever <prasanna.kalever@redhat.com>

diff --git a/src/librbd/api/Mirror.cc b/src/librbd/api/Mirror.cc
index 382e500f22aac1d241b4a41b84bc33734480ee2b..1f57947ecd212dc05c638a69a5114bf357bcf409 100644 (file)
--- a/src/librbd/api/Mirror.cc
+++ b/src/librbd/api/Mirror.cc
@@ -3121,7 +3121,27 @@ int Mirror<I>::group_promote(IoCtx& group_ioctx, const char *group_name,
     lderr(cct) << "group " << group_name
                << " is still primary within a remote cluster" << dendl;
     return -EBUSY;
-  } else if (force) {
+  }
+
+  std::string group_header_oid = librbd::util::group_header_name(group_id);
+  std::string value;
+  r = librbd::cls_client::metadata_get(&group_ioctx, group_header_oid,
+                                       RBD_GROUP_RESYNC, &value);
+  if (r < 0 && r != -ENOENT) {
+    lderr(cct) << "failed reading metadata: " << RBD_GROUP_RESYNC << " : "
+               << cpp_strerror(r) << dendl;
+  } else if (r == 0) {
+    ldout(cct, 5) << "found previous resync group request, clearing it"
+                  << dendl;
+    r = cls_client::metadata_remove(&group_ioctx, group_header_oid,
+                                    RBD_GROUP_RESYNC);
+    if (r < 0) {
+      lderr(cct) << "failed removing metadata: " << RBD_GROUP_RESYNC << " : "
+                 << cpp_strerror(r) << dendl;
+    }
+  }
+
+  if (force) {
     std::vector<cls::rbd::GroupImageStatus> images;
     r = Group<I>::group_image_list_by_id(group_ioctx, group_id, &images);
     if (r < 0) {
@@ -3282,7 +3302,6 @@ int Mirror<I>::group_promote(IoCtx& group_ioctx, const char *group_name,
     }
   }
 
-  std::string group_header_oid = librbd::util::group_header_name(group_id);
   if (ret_code < 0) {
     // undo
     ldout(cct, 20) << "undoing group promote: " << ret_code << dendl;
@@ -3570,23 +3589,6 @@ int Mirror<I>::group_resync(IoCtx& group_ioctx, const char *group_name) {
     return r;
   }
 
-  std::vector<cls::rbd::GroupImageStatus> images;
-  r = Group<I>::group_image_list_by_id(group_ioctx, group_id, &images);
-  if (r < 0) {
-    lderr(cct) << "listing images in group=" << group_name
-               << " failed: " << cpp_strerror(r) << dendl;
-    return r;
-  }
-
-  r = MirroringWatcher<I>::notify_group_updated(
-        group_ioctx, cls::rbd::MIRROR_GROUP_STATE_DISABLED, group_id,
-        mirror_group.global_group_id, images.size());
-  if (r < 0) {
-    lderr(cct) << "failed to notify mirroring group=" << group_name
-               << " updated: " << cpp_strerror(r) << dendl;
-    // not fatal
-  }
-
   return 0;
 }
 

diff --git a/src/tools/rbd_mirror/PoolWatcher.cc b/src/tools/rbd_mirror/PoolWatcher.cc
index 0c8bab188fe814eca1adfeddcf06811695e3ae9e..c6ebd5e7a9c0608b199e04060a42dae6d32c34d9 100644 (file)
--- a/src/tools/rbd_mirror/PoolWatcher.cc
+++ b/src/tools/rbd_mirror/PoolWatcher.cc
@@ -482,7 +482,7 @@ void PoolWatcher<I>::notify_listener() {
       // If previous entity is not there in current set of entities or if
       // their id's don't match then consider its removed
       if (it == m_pending_entities.end() || it->second != id ||
-          it->first.count < entity.count) {
+          it->first.count != entity.count) {
         removed_entities.insert(entity);
       }
     }
@@ -491,7 +491,7 @@ void PoolWatcher<I>::notify_listener() {
       // If current entity is not there in previous set of entities or if
       // their id's don't match then consider its added
       if (it == m_entities.end() || it->second != id ||
-          it->first.count < entity.count) {
+          it->first.count != entity.count) {
         added_entities.insert(entity);
       }
     }

diff --git a/src/tools/rbd_mirror/group_replayer/BootstrapRequest.cc b/src/tools/rbd_mirror/group_replayer/BootstrapRequest.cc
index 4aae3eeb555f0683f0c2094bd3cf3248dc6ec9c9..28b6756bab3c016a1ed650b524f510e88fb02629 100644 (file)
--- a/src/tools/rbd_mirror/group_replayer/BootstrapRequest.cc
+++ b/src/tools/rbd_mirror/group_replayer/BootstrapRequest.cc
@@ -112,27 +112,7 @@ BootstrapRequest<I>::BootstrapRequest(
 
 template <typename I>
 void BootstrapRequest<I>::send() {
-  *m_resync_requested = false;
-
-  if (m_local_group_id && !m_local_group_id->empty()) {
-    std::string group_header_oid = librbd::util::group_header_name(
-        *m_local_group_id);
-    std::string value;
-    int r = librbd::cls_client::metadata_get(&m_local_io_ctx, group_header_oid,
-                                             RBD_GROUP_RESYNC, &value);
-    if (r < 0 && r != -ENOENT) {
-      derr << "failed reading metadata: " << cpp_strerror(r) << dendl;
-    } else if (r == 0) {
-      dout(10) << "local group resync requested" << dendl;
-      *m_resync_requested = true;
-    }
-  }
-
-  if (*m_resync_requested) {
-    get_local_group_id();
-  } else {
-    get_remote_group_id();
-  }
+  get_remote_group_id();
 }
 
 template <typename I>
@@ -357,6 +337,29 @@ void BootstrapRequest<I>::handle_list_remote_group_snapshots(int r) {
     m_remote_mirror_group_primary = (state == cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY);
   }
 
+  *m_resync_requested = false;
+  if (m_local_group_id && !m_local_group_id->empty()) {
+    std::string group_header_oid = librbd::util::group_header_name(
+        *m_local_group_id);
+    std::string value;
+    int r = librbd::cls_client::metadata_get(&m_local_io_ctx, group_header_oid,
+                                             RBD_GROUP_RESYNC, &value);
+    if (r < 0 && r != -ENOENT) {
+      derr << "failed reading metadata: " << cpp_strerror(r) << dendl;
+    } else if (r == 0) {
+      dout(10) << "local group resync requested : " << m_local_group_id
+               << dendl;
+      if (m_remote_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED &&
+          m_remote_mirror_group_primary) {
+        *m_resync_requested = true;
+        list_remote_group();
+        return;
+      }
+      dout(10) << "turns out remote is not primary, we cannot resync, will retry later"
+               << dendl;
+    }
+  }
+
   if (m_remote_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED &&
       m_remote_mirror_group_primary) {
     list_remote_group();
@@ -910,7 +913,8 @@ void BootstrapRequest<I>::handle_get_local_mirror_image(int r) {
                m_remote_mirror_group_primary &&
                m_local_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED &&
                m_local_mirror_group.global_group_id == m_global_group_id &&
-               has_remote_image(spec.pool_id, mirror_image.global_image_id)) {
+               has_remote_image(spec.pool_id, mirror_image.global_image_id)
+               && *m_resync_requested == false) {
       dout(10) << "add secondary to replayer queue: " << spec.pool_id << " "
                << spec.image_id << " " << mirror_image.global_image_id
                << dendl;
@@ -922,7 +926,6 @@ void BootstrapRequest<I>::handle_get_local_mirror_image(int r) {
         finish(-ERESTART);
         return;
       }
-
       dout(10) << "add to trash queue: " << spec.pool_id << " "
                << spec.image_id << " " << mirror_image.global_image_id
                << dendl;
@@ -1024,7 +1027,8 @@ void BootstrapRequest<I>::handle_move_local_image_to_trash(int r) {
 template <typename I>
 void BootstrapRequest<I>::disable_local_mirror_group() {
   if (m_local_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED &&
-      m_local_mirror_group.global_group_id == m_global_group_id) {
+      m_local_mirror_group.global_group_id == m_global_group_id &&
+      *m_resync_requested == false) {
     finish(0);
     return;
   }
@@ -1093,7 +1097,7 @@ void BootstrapRequest<I>::handle_remove_local_mirror_group(int r) {
   m_local_mirror_group.state = cls::rbd::MIRROR_GROUP_STATE_DISABLED;
   if (r != -ENOENT && (m_local_mirror_group.global_group_id == m_global_group_id) &&
       (m_remote_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED &&
-       m_remote_mirror_group_primary)) {
+       m_remote_mirror_group_primary) && *m_resync_requested == false) {
     create_local_mirror_group();
   } else {
     remove_local_group();