]> git.apps.os.sepia.ceph.com Git - ceph.git/commitdiff
rbd-mirror: remove group snaps on primary at snapshot creation
authorPrasanna Kumar Kalever <prasanna.kalever@redhat.com>
Sun, 21 Jul 2024 15:52:12 +0000 (21:22 +0530)
committerPrasanna Kumar Kalever <prasanna.kalever@redhat.com>
Thu, 24 Apr 2025 15:56:24 +0000 (21:26 +0530)
Signed-off-by: Prasanna Kumar Kalever <prasanna.kalever@redhat.com>
src/librbd/CMakeLists.txt
src/librbd/api/Mirror.cc
src/librbd/group/UnlinkPeerGroupRequest.cc [new file with mode: 0644]
src/librbd/group/UnlinkPeerGroupRequest.h [new file with mode: 0644]
src/librbd/mirror/snapshot/UnlinkPeerRequest.cc
src/librbd/mirror/snapshot/UnlinkPeerRequest.h
src/tools/rbd_mirror/group_replayer/Replayer.cc

index b787b72bd42633503bb8b68126198841865965e4..62c234b733fdaf7da8fbb873683cb6805681b1c6 100644 (file)
@@ -80,6 +80,7 @@ set(librbd_internal_srcs
   group/ListSnapshotsRequest.cc
   group/AddImageRequest.cc
   group/RemoveImageRequest.cc
+  group/UnlinkPeerGroupRequest.cc
   image/AttachChildRequest.cc
   image/AttachParentRequest.cc
   image/CloneRequest.cc
index 22f09a691a90c870be56acbd633aa251db7df31a..61b68582766e4d9fa3fd69222bf3265bae61b4ca 100644 (file)
@@ -22,6 +22,7 @@
 #include "librbd/api/Namespace.h"
 #include "librbd/api/Utils.h"
 #include "librbd/group/ListSnapshotsRequest.h"
+#include "librbd/group/UnlinkPeerGroupRequest.h"
 #include "librbd/mirror/DemoteRequest.h"
 #include "librbd/mirror/DisableRequest.h"
 #include "librbd/mirror/EnableRequest.h"
@@ -3035,6 +3036,13 @@ int Mirror<I>::group_promote(IoCtx& group_ioctx, const char *group_name,
     util::notify_unquiesce(image_ctxs, quiesce_requests);
   }
 
+  if (!ret_code) {
+    C_SaferCond cond;
+    auto req = group::UnlinkPeerGroupRequest<I>::create(
+        group_ioctx, group_id, &image_ctxs, &cond);
+    req->send();
+    cond.wait();
+  }
   close_images(&image_ctxs);
 
   return ret_code;
@@ -3191,6 +3199,13 @@ int Mirror<I>::group_demote(IoCtx& group_ioctx,
     util::notify_unquiesce(image_ctxs, quiesce_requests);
   }
 
+  if (!ret_code) {
+    C_SaferCond cond;
+    auto req = group::UnlinkPeerGroupRequest<I>::create(
+        group_ioctx, group_id, &image_ctxs, &cond);
+    req->send();
+    cond.wait();
+  }
   close_images(&image_ctxs);
 
   return ret_code;
@@ -3387,6 +3402,13 @@ int Mirror<I>::group_snapshot_create(IoCtx& group_ioctx, const char *group_name,
     util::notify_unquiesce(image_ctxs, quiesce_requests);
   }
 
+  if (!ret_code) {
+    C_SaferCond cond;
+    auto req = group::UnlinkPeerGroupRequest<I>::create(
+        group_ioctx, group_id, &image_ctxs, &cond);
+    req->send();
+    cond.wait();
+  }
   close_images(&image_ctxs);
 
   return ret_code;
@@ -3495,6 +3517,13 @@ int Mirror<I>::group_image_add(IoCtx &group_ioctx,
     util::notify_unquiesce(image_ctxs, quiesce_requests);
   }
 
+  if (!ret_code) {
+    C_SaferCond cond;
+    auto req = group::UnlinkPeerGroupRequest<I>::create(
+        group_ioctx, group_id, &image_ctxs, &cond);
+    req->send();
+    cond.wait();
+  }
   close_images(&image_ctxs);
 
   if (!ret_code) {
@@ -3627,6 +3656,13 @@ int Mirror<I>::group_image_remove(IoCtx &group_ioctx,
     util::notify_unquiesce(image_ctxs, quiesce_requests);
   }
 
+  if (!ret_code) {
+    C_SaferCond cond;
+    auto req = group::UnlinkPeerGroupRequest<I>::create(
+        group_ioctx, group_id, &image_ctxs, &cond);
+    req->send();
+    cond.wait();
+  }
   close_images(&image_ctxs);
 
   if (!ret_code) {
diff --git a/src/librbd/group/UnlinkPeerGroupRequest.cc b/src/librbd/group/UnlinkPeerGroupRequest.cc
new file mode 100644 (file)
index 0000000..c04a319
--- /dev/null
@@ -0,0 +1,171 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "include/ceph_assert.h"
+#include "common/Cond.h"
+#include "common/dout.h"
+#include "common/errno.h"
+#include "common/ceph_context.h"
+#include "cls/rbd/cls_rbd_client.h"
+#include "librbd/Operations.h"
+#include "librbd/Utils.h"
+#include "librbd/api/Utils.h"
+#include "librbd/api/Group.h"
+#include "librbd/group/ListSnapshotsRequest.h"
+#include "librbd/group/UnlinkPeerGroupRequest.h"
+#include "librbd/mirror/snapshot/UnlinkPeerRequest.h"
+
+#define dout_subsys ceph_subsys_rbd
+#undef dout_prefix
+#define dout_prefix *_dout << "librbd::group::UnlinkPeerGroupRequest: " << this \
+                           << " " << __func__ << ": "
+
+namespace librbd {
+namespace group {
+
+using util::create_rados_callback;
+using util::create_context_callback;
+
+template <typename I>
+void UnlinkPeerGroupRequest<I>::send() {
+  CephContext *cct = (CephContext *)m_group_io_ctx.cct();
+  ldout(cct, 10) << dendl;
+  unlink_peer();
+}
+
+template <typename I>
+void UnlinkPeerGroupRequest<I>::unlink_peer() {
+  CephContext *cct = (CephContext *)m_group_io_ctx.cct();
+  ldout(cct, 10) << dendl;
+
+  std::vector<cls::rbd::GroupSnapshot> snaps;
+  C_SaferCond cond;
+  auto req = group::ListSnapshotsRequest<>::create(m_group_io_ctx, m_group_id,
+                                                   true, true, &snaps, &cond);
+  req->send();
+  cond.wait();
+  uint64_t count = 0;
+  auto unlink_snap = snaps.end();
+  auto unlink_unsynced_snap = snaps.end();
+  bool unlink_unsynced = false;
+  for (auto it = snaps.begin(); it != snaps.end(); it++) {
+    auto ns = std::get_if<cls::rbd::MirrorGroupSnapshotNamespace>(
+        &it->snapshot_namespace);
+    if (ns != nullptr) {
+      if (ns->state != cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY) {
+        continue;
+      }
+      count++;
+      if (count == 3) {
+        unlink_unsynced_snap = it;
+      }
+      ceph_assert(count <= 5);
+
+      if (ns->mirror_peer_uuids.empty()) {
+        auto next_snap = std::next(it);
+        if (next_snap != snaps.end()) {
+          unlink_snap = it;
+          break;
+        }
+      }
+    }
+    // TODO: fix the hardcoded max_snaps value
+    if (count == 5) {
+      unlink_unsynced = true;
+    }
+  }
+
+  if (unlink_snap != snaps.end()) {
+    remove_group_snapshot(*unlink_snap);
+  } else if (unlink_unsynced && unlink_unsynced_snap != snaps.end()) {
+    remove_group_snapshot(*unlink_unsynced_snap);
+  } else {
+    finish(0);
+  }
+}
+
+template <typename I>
+void UnlinkPeerGroupRequest<I>::remove_group_snapshot(
+                              cls::rbd::GroupSnapshot group_snap) {
+  CephContext *cct = (CephContext *)m_group_io_ctx.cct();
+  ldout(cct, 10) << dendl;
+
+  for (auto &snap : group_snap.snaps) {
+    if (snap.snap_id == CEPH_NOSNAP) {
+      continue;
+    }
+    ImageCtx *ictx = nullptr;
+    for (size_t i = 0; i < m_image_ctxs->size(); ++i) {
+      ictx = (*m_image_ctxs)[i];
+      if (ictx->id  != snap.image_id) {
+        ictx = nullptr;
+      } else {
+        break;
+      }
+    }
+    if (!ictx) {
+      continue;
+    }
+    ldout(cct, 10) << "removing individual snapshot: "
+                   << snap.snap_id << ", from image id:" << snap.image_id << dendl;
+    remove_image_snapshot(ictx, snap.snap_id);
+  }
+
+  int r = cls_client::group_snap_remove(&m_group_io_ctx,
+      librbd::util::group_header_name(m_group_id), group_snap.id);
+  if (r < 0) {
+    lderr(cct) << "failed to remove group snapshot metadata: "
+               << cpp_strerror(r) << dendl;
+  }
+
+  unlink_peer();
+}
+
+template <typename I>
+void UnlinkPeerGroupRequest<I>::remove_image_snapshot(
+    ImageCtx *image_ctx, uint64_t snap_id) {
+  CephContext *cct = (CephContext *)m_group_io_ctx.cct();
+  ldout(cct, 10) << snap_id << dendl;
+
+  image_ctx->image_lock.lock_shared();
+  int r = -ENOENT;
+  cls::rbd::SnapshotNamespace snap_namespace;
+  std::string snap_name;
+  for (auto snap_it = image_ctx->snap_info.find(snap_id);
+       snap_it != image_ctx->snap_info.end(); ++snap_it) {
+    if (snap_it->first == snap_id) {
+      r = 0;
+      snap_namespace = snap_it->second.snap_namespace;
+      snap_name = snap_it->second.name;
+    }
+  }
+
+  if (r == -ENOENT) {
+    ldout(cct, 10) << "missing snapshot: snap_id=" << snap_id << dendl;
+    image_ctx->image_lock.unlock_shared();
+    return;
+  }
+
+  auto mirror_ns = std::get_if<cls::rbd::MirrorSnapshotNamespace>(
+    &snap_namespace);
+  if (mirror_ns == nullptr) {
+    lderr(cct) << "not mirror snapshot (snap_id=" << snap_id << ")" << dendl;
+    image_ctx->image_lock.unlock_shared();
+    return;
+  }
+  image_ctx->image_lock.unlock_shared();
+  image_ctx->operations->snap_remove(snap_namespace, snap_name.c_str());
+}
+
+template <typename I>
+void UnlinkPeerGroupRequest<I>::finish(int r) {
+  CephContext *cct = (CephContext *)m_group_io_ctx.cct();
+  ldout(cct, 10) << "r=" << r << dendl;
+
+  m_on_finish->complete(r);
+}
+
+} // namespace group
+} // namespace librbd
+
+template class librbd::group::UnlinkPeerGroupRequest<librbd::ImageCtx>;
diff --git a/src/librbd/group/UnlinkPeerGroupRequest.h b/src/librbd/group/UnlinkPeerGroupRequest.h
new file mode 100644 (file)
index 0000000..b211a9e
--- /dev/null
@@ -0,0 +1,61 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CEPH_LIBRBD_UNLINK_PEER_GROUP_REQUST_H
+#define CEPH_LIBRBD_UNLINK_PEER_GROUP_REQUST_H
+
+#include "include/int_types.h"
+#include "include/types.h"
+#include "include/rados/librados.hpp"
+#include "cls/rbd/cls_rbd_types.h"
+
+#include <string>
+#include <vector>
+
+class Context;
+
+namespace librbd {
+
+struct ImageCtx;
+
+namespace group {
+
+template <typename ImageCtxT = librbd::ImageCtx>
+class UnlinkPeerGroupRequest {
+public:
+  static UnlinkPeerGroupRequest *create(
+      librados::IoCtx &group_io_ctx, const std::string &group_id,
+      std::vector<ImageCtx *> *image_ctxs,
+      Context *on_finish) {
+    return new UnlinkPeerGroupRequest(group_io_ctx, group_id,
+                                      image_ctxs, on_finish);
+  }
+
+  UnlinkPeerGroupRequest(librados::IoCtx &group_io_ctx,
+                       const std::string &group_id,
+                       std::vector<ImageCtx *> *image_ctxs,
+                       Context *on_finish)
+    : m_group_io_ctx(group_io_ctx), m_group_id(group_id),
+      m_image_ctxs(image_ctxs), m_on_finish(on_finish) {
+  }
+
+  void send();
+
+private:
+  librados::IoCtx &m_group_io_ctx;
+  const std::string m_group_id;
+  std::vector<ImageCtx *> *m_image_ctxs;
+  Context *m_on_finish;
+
+  void unlink_peer();
+  void remove_group_snapshot(cls::rbd::GroupSnapshot group_snap);
+  void remove_image_snapshot(ImageCtx *image_ctx, uint64_t snap_id);
+  void finish(int r);
+};
+
+} // namespace group
+} // namespace librbd
+
+extern template class librbd::group::UnlinkPeerGroupRequest<librbd::ImageCtx>;
+
+#endif // CEPH_LIBRBD_UNLINK_PEER_GROUP_REQUST_H
index adb431f0f13205de93a4ed111323a6f1f834fdda..35313f6277981bb6876724dcc3b316043aea2d91 100644 (file)
@@ -103,7 +103,7 @@ void UnlinkPeerRequest<I>::unlink_peer() {
       have_newer_mirror_snapshot) {
     if (m_allow_remove) {
       m_image_ctx->image_lock.unlock_shared();
-      unlink_group_snapshot(snap_namespace, snap_name);
+      remove_snapshot(snap_namespace, snap_name);
       return;
     } else {
       ldout(cct, 15) << "skipping removal of snapshot: snap_id=" << m_snap_id
@@ -187,57 +187,6 @@ void UnlinkPeerRequest<I>::handle_notify_update(int r) {
   refresh_image();
 }
 
-template <typename I>
-void UnlinkPeerRequest<I>::unlink_group_snapshot(
-    const cls::rbd::SnapshotNamespace& snap_namespace,
-    const std::string& snap_name) {
-  CephContext *cct = m_image_ctx->cct;
-  ldout(cct, 15) << "snap_name: " << snap_name << dendl;
-
-  auto info = std::get_if<cls::rbd::MirrorSnapshotNamespace>(&snap_namespace);
-  if (!info->group_spec.is_valid()) {
-    remove_snapshot(snap_namespace, snap_name);
-    return;
-  }
-
-  int r = util::create_ioctx(m_image_ctx->md_ctx, "group", info->group_spec.pool_id,
-      {}, &m_group_io_ctx);
-  if (r < 0) {
-    remove_snapshot(snap_namespace, snap_name);
-    return;
-  }
-
-  librados::ObjectWriteOperation op;
-  cls::rbd::ImageSnapshotSpec image_snap = {m_image_ctx->md_ctx.get_id(),
-                                            m_image_ctx->id, m_snap_id};
-  librbd::cls_client::group_snap_unlink(&op, info->group_snap_id, image_snap);
-  auto ctx = new LambdaContext([this, snap_namespace, snap_name](int r) {
-      handle_unlink_group_snapshot(snap_namespace, snap_name, r);
-    });
-  auto aio_comp = create_rados_callback(ctx);
-  r = m_group_io_ctx.aio_operate(
-      util::group_header_name(info->group_spec.group_id), aio_comp, &op);
-  ceph_assert(r == 0);
-  aio_comp->release();
-}
-
-template <typename I>
-void UnlinkPeerRequest<I>::handle_unlink_group_snapshot(
-    const cls::rbd::SnapshotNamespace& snap_namespace,
-    const std::string& snap_name, int r) {
-  CephContext *cct = m_image_ctx->cct;
-  ldout(cct, 15) << "r=" << r << dendl;
-
-  if (r < 0 && r != -ENOENT) {
-    lderr(cct) << "failed to unlink group snapshot: " << cpp_strerror(r)
-               << dendl;
-    finish(r);
-    return;
-  }
-
-  remove_snapshot(snap_namespace, snap_name);
-}
-
 template <typename I>
 void UnlinkPeerRequest<I>::remove_snapshot(
     const cls::rbd::SnapshotNamespace& snap_namespace,
index 8690efadc30200702defa9747d3f0aa11a120908..53178dabe7dc4c5a2bf98ae9ec4511e09ecb649a 100644 (file)
@@ -84,12 +84,6 @@ private:
   void notify_update();
   void handle_notify_update(int r);
 
-  void unlink_group_snapshot(const cls::rbd::SnapshotNamespace& snap_namespace,
-                             const std::string& snap_name);
-  void handle_unlink_group_snapshot(
-      const cls::rbd::SnapshotNamespace& snap_namespace,
-      const std::string& snap_name, int r);
-
   void remove_snapshot(const cls::rbd::SnapshotNamespace& snap_namespace,
                        const std::string& snap_name);
   void handle_remove_snapshot(int r);
index 44b4002008cbf69fde65d26b869c4d8e8273bf37..d903fdff834afdf909af9e9f90dd4b8813d42a89 100644 (file)
@@ -596,7 +596,32 @@ void Replayer<I>::handle_mirror_snapshot_complete(
     on_finish->complete(0);
     return;
   }
-  unlink_group_snapshots(remote_group_snap_id);
+
+  // remove mirror_peer_uuids from remote snap
+  auto itr = std::find_if(
+      m_remote_group_snaps.begin(), m_remote_group_snaps.end(),
+      [remote_group_snap_id](const cls::rbd::GroupSnapshot &s) {
+      return s.id == remote_group_snap_id;
+      });
+
+  ceph_assert(itr != m_remote_group_snaps.end());
+  auto rns = std::get_if<cls::rbd::MirrorGroupSnapshotNamespace>(
+      &itr->snapshot_namespace);
+  if (rns != nullptr) {
+    rns->mirror_peer_uuids.clear();
+    auto comp = create_rados_callback(
+        new LambdaContext([this, remote_group_snap_id](int r) {
+          unlink_group_snapshots(remote_group_snap_id);
+        }));
+
+    librados::ObjectWriteOperation op;
+    librbd::cls_client::group_snap_set(&op, *itr);
+    int r = m_remote_io_ctx.aio_operate(
+        librbd::util::group_header_name(m_remote_group_id), comp, &op);
+    ceph_assert(r == 0);
+    comp->release();
+  }
+
   on_finish->complete(0);
 }