]> git.apps.os.sepia.ceph.com Git - ceph-ci.git/commitdiff
librbd/mirror/snapshot: add async create mirror group snapshot
authorRamana Raja <rraja@redhat.com>
Sun, 12 Jan 2025 21:01:23 +0000 (16:01 -0500)
committerIlya Dryomov <idryomov@gmail.com>
Sun, 28 Sep 2025 18:25:00 +0000 (20:25 +0200)
Signed-off-by: Ramana Raja <rraja@redhat.com>
src/librbd/CMakeLists.txt
src/librbd/api/Mirror.cc
src/librbd/api/Mirror.h
src/librbd/mirror/snapshot/GroupCreatePrimaryRequest.cc [new file with mode: 0644]
src/librbd/mirror/snapshot/GroupCreatePrimaryRequest.h [new file with mode: 0644]

index 50c0e6c74051aef8761bc3acd8ae6cf0d54567af..2200cf5e115450381da028149481980ae6f39de9 100644 (file)
@@ -161,6 +161,7 @@ set(librbd_internal_srcs
   mirror/snapshot/CreatePrimaryRequest.cc
   mirror/snapshot/DemoteRequest.cc
   mirror/snapshot/GetImageStateRequest.cc
+  mirror/snapshot/GroupCreatePrimaryRequest.cc
   mirror/snapshot/GroupGetInfoRequest.cc
   mirror/snapshot/GroupPrepareImagesRequest.cc
   mirror/snapshot/ImageMeta.cc
index eb456f98e09cc58eaaa45f8ce754959b2e902105..675d332ae4d460fa62fd7325a8cd6fa488f1f2bd 100644 (file)
@@ -33,6 +33,7 @@
 #include "librbd/mirror/Types.h"
 #include "librbd/MirroringWatcher.h"
 #include "librbd/mirror/snapshot/CreatePrimaryRequest.h"
+#include "librbd/mirror/snapshot/GroupCreatePrimaryRequest.h"
 #include "librbd/mirror/snapshot/GroupGetInfoRequest.h"
 #include "librbd/mirror/snapshot/ImageMeta.h"
 #include "librbd/mirror/snapshot/UnlinkPeerRequest.h"
@@ -3428,129 +3429,36 @@ int Mirror<I>::group_resync(IoCtx& group_ioctx, const char *group_name) {
 }
 
 template <typename I>
-int Mirror<I>::group_snapshot_create(IoCtx& group_ioctx, const char *group_name,
-                                     uint32_t flags, std::string *snap_id) {
+void Mirror<I>::group_snapshot_create(IoCtx& group_ioctx,
+                                      const std::string& group_name,
+                                      uint32_t flags, std::string *snap_id,
+                                      Context *on_finish) {
   CephContext *cct = (CephContext *)group_ioctx.cct();
-  ldout(cct, 20) << "io_ctx=" << &group_ioctx
+  ldout(cct, 20) << "group io_ctx=" << &group_ioctx
                 << ", group_name=" << group_name
                 << ", flags=" << flags << dendl;
 
-  std::string group_id;
-  int r = cls_client::dir_get_id(&group_ioctx, RBD_GROUP_DIRECTORY,
-                                 group_name, &group_id);
-  if (r < 0) {
-    lderr(cct) << "error getting the group id: " << cpp_strerror(r) << dendl;
-    return r;
-  }
-
-  cls::rbd::MirrorGroup mirror_group;
-  r = cls_client::mirror_group_get(&group_ioctx, group_id, &mirror_group);
-  if (r == -ENOENT) {
-    ldout(cct, 10) << "mirroring for group " << group_name
-                   << " disabled" << dendl;
-    return -EINVAL;
-  } else if (r < 0) {
-    lderr(cct) << "failed to retrieve mirror group metadata: "
-               << cpp_strerror(r) << dendl;
-    return r;
-  } else if (mirror_group.mirror_image_mode !=
-             cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT) {
-    auto mode = static_cast<rbd_mirror_image_mode_t>(
-        mirror_group.mirror_image_mode);
-    lderr(cct) << "cannot create snapshot, mirror mode is set to: "
-               << mode << dendl;
-    return -EOPNOTSUPP;
-  }
-
-  cls::rbd::MirrorSnapshotState state;
-  r = get_last_mirror_snapshot_state(group_ioctx, group_id, &state);
-  if (r == -ENOENT) {
-    state = cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY;
-    r = 0;
-  }
-  if (r < 0) {
-    return r;
-  }
-
-  if (state != cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY) {
-    lderr(cct) << "group " << group_name << " is not primary" << dendl;
-    return -EINVAL;
-  }
-
-  std::string group_snap_id = librbd::util::generate_image_id(group_ioctx);
-  cls::rbd::GroupSnapshot group_snap{
-      group_snap_id,
-      cls::rbd::GroupSnapshotNamespaceMirror{},
-      prepare_primary_mirror_snap_name(cct, mirror_group.global_group_id,
-                                       group_snap_id),
-      cls::rbd::GROUP_SNAPSHOT_STATE_INCOMPLETE};
-
-  std::vector<uint64_t> quiesce_requests;
-  std::vector<I *> image_ctxs;
-  r = prepare_group_images(group_ioctx, group_id, &image_ctxs,
-                           &group_snap, quiesce_requests,
-                           cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY,
-                           flags);
-  if (r != 0) {
-    return r;
-  }
-
-  int ret_code = 0;
-  std::vector<uint64_t> snap_ids(image_ctxs.size(), CEPH_NOSNAP);
-  std::vector<C_SaferCond*> on_finishes(image_ctxs.size(), nullptr);
-
-  for (size_t i = 0; i < image_ctxs.size(); i++) {
-    C_SaferCond* on_finish = new C_SaferCond;
-    image_snapshot_create(image_ctxs[i], RBD_SNAP_CREATE_SKIP_QUIESCE,
-                          group_snap_id, &snap_ids[i], on_finish);
-    on_finishes[i] = on_finish;
-  }
-
-  for (size_t i = 0; i < image_ctxs.size(); i++) {
-    r = 0;
-    if (on_finishes[i]) {
-      r = on_finishes[i]->wait();
-      delete on_finishes[i];
-    }
-    if (r < 0) {
-      if (ret_code == 0) {
-        ret_code = r;
-      }
-    } else {
-      group_snap.snaps[i].snap_id = snap_ids[i];
-    }
-  }
-
-  std::string group_header_oid = librbd::util::group_header_name(group_id);
-  if (ret_code != 0) {
-    // undo
-    ldout(cct, 20) << "undoing group create snapshot: " << ret_code << dendl;
-    remove_interim_snapshots(group_ioctx, group_header_oid, &image_ctxs, &group_snap);
-  } else{
-    group_snap.state = cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE;
-    r = cls_client::group_snap_set(&group_ioctx, group_header_oid, group_snap);
-    if (r < 0) {
-      lderr(cct) << "failed to update group snapshot metadata: "
-                 << cpp_strerror(r) << dendl;
-    }
+  auto req = mirror::snapshot::GroupCreatePrimaryRequest<I>::create(
+    group_ioctx, group_name, flags, snap_id, on_finish);
+  req->send();
+}
 
-    *snap_id = group_snap.id;
-  }
+template <typename I>
+int Mirror<I>::group_snapshot_create(IoCtx& group_ioctx,
+                                     const std::string& group_name,
+                                     uint32_t flags, std::string *snap_id) {
+  CephContext *cct = (CephContext *)group_ioctx.cct();
 
-  if (!quiesce_requests.empty()) {
-    util::notify_unquiesce(image_ctxs, quiesce_requests);
-  }
+  C_SaferCond ctx;
+  group_snapshot_create(group_ioctx, group_name, flags, snap_id, &ctx);
 
-  if (!ret_code) {
-    C_SaferCond cond;
-    auto req = group::UnlinkPeerGroupRequest<I>::create(
-        group_ioctx, group_id, &image_ctxs, &cond);
-    req->send();
-    cond.wait();
+  int r = ctx.wait();
+  if (r < 0) {
+    lderr(cct) << "failed to create mirror snapshot for group '" << group_name
+               << "': " << cpp_strerror(r) << dendl;
   }
-  close_images(&image_ctxs);
 
-  return ret_code;
+  return r;
 }
 
 template <typename I>
index 7a006905f25f0cf1b2c05f679e978f440b744bfb..6881f77bb7fdae1681188f534fb6cc911c97023d 100644 (file)
@@ -150,7 +150,12 @@ struct Mirror {
   static int group_demote(IoCtx &group_ioctx, const char *group_name,
                           uint32_t flags);
   static int group_resync(IoCtx &group_ioctx, const char *group_name);
-  static int group_snapshot_create(IoCtx& group_ioctx, const char *group_name,
+  static void group_snapshot_create(IoCtx& group_ioctx,
+                                    const std::string& group_name,
+                                    uint32_t flags, std::string *snap_id,
+                                    Context *on_finish);
+  static int group_snapshot_create(IoCtx& group_ioctx,
+                                   const std::string& group_name,
                                    uint32_t flags, std::string *snap_id);
 
   static int group_image_add(IoCtx &group_ioctx, const std::string &group_id,
diff --git a/src/librbd/mirror/snapshot/GroupCreatePrimaryRequest.cc b/src/librbd/mirror/snapshot/GroupCreatePrimaryRequest.cc
new file mode 100644 (file)
index 0000000..b778ed1
--- /dev/null
@@ -0,0 +1,576 @@
+// -*- mode:c++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "librbd/mirror/snapshot/GroupCreatePrimaryRequest.h"
+#include "include/ceph_assert.h"
+#include "common/dout.h"
+#include "include/Context.h"
+#include "common/Cond.h"
+#include "common/errno.h"
+#include "librbd/api/Group.h"
+#include "librbd/internal.h"
+#include "common/ceph_context.h"
+#include "cls/rbd/cls_rbd_client.h"
+#include "librbd/ImageCtx.h"
+#include "librbd/Operations.h"
+#include "librbd/ImageState.h"
+#include "librbd/Utils.h"
+#include "librbd/group/UnlinkPeerGroupRequest.h"
+#include "librbd/group/ListSnapshotsRequest.h"
+#include "librbd/mirror/GetInfoRequest.h"
+#include "librbd/mirror/snapshot/CreatePrimaryRequest.h"
+#include "librbd/mirror/snapshot/GroupPrepareImagesRequest.h"
+#include "librbd/mirror/Types.h"
+
+#define dout_subsys ceph_subsys_rbd
+#undef dout_prefix
+#define dout_prefix *_dout << "librbd::mirror::snapshot::GroupCreatePrimaryRequest: " \
+                           << " " << __func__ << ": "
+
+namespace librbd {
+namespace mirror {
+namespace snapshot {
+
+using librbd::util::create_rados_callback;
+
+
+std::string prepare_primary_mirror_snap_name(CephContext *cct,
+                                             const std::string &global_group_id,
+                                             const std::string &snap_id) {
+  ldout(cct, 10) << "global_group_id: " << global_group_id
+                 << ", snap_id: " << snap_id << dendl;
+
+  std::stringstream ind_snap_name_stream;
+  ind_snap_name_stream << ".mirror.primary."
+                       << global_group_id << "." << snap_id;
+  return ind_snap_name_stream.str();
+}
+
+
+template <typename I>
+struct C_ImageSnapshotCreate2 : public Context {
+  I *ictx;
+  uint64_t snap_create_flags;
+  int64_t group_pool_id;
+  std::string group_id;
+  std::string group_snap_id;
+  uint64_t *snap_id;
+  Context *on_finish;
+
+  cls::rbd::MirrorImage mirror_image;
+  mirror::PromotionState promotion_state;
+  std::string primary_mirror_uuid;
+
+  C_ImageSnapshotCreate2(I *ictx, uint64_t snap_create_flags,
+                         int64_t group_pool_id,
+                         const std::string &group_id,
+                         const std::string &group_snap_id,
+                         uint64_t *snap_id,
+                         Context *on_finish)
+    : ictx(ictx), snap_create_flags(snap_create_flags),
+      group_pool_id(group_pool_id), group_id(group_id),
+      group_snap_id(group_snap_id), snap_id(snap_id),
+      on_finish(on_finish) {
+  }
+
+  void finish(int r) override {
+    if (r < 0 && r != -ENOENT) {
+      on_finish->complete(r);
+      return;
+    }
+
+    if (mirror_image.mode != cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT ||
+        mirror_image.state != cls::rbd::MIRROR_IMAGE_STATE_ENABLED) {
+      lderr(ictx->cct) << "snapshot based mirroring is not enabled" << dendl;
+      on_finish->complete(-EINVAL);
+      return;
+    }
+
+    auto req = mirror::snapshot::CreatePrimaryRequest<I>::create(
+      ictx, mirror_image.global_image_id, CEPH_NOSNAP, snap_create_flags, 0U,
+      group_pool_id, group_id, group_snap_id, snap_id, on_finish);
+    req->send();
+  }
+};
+
+
+template <typename I>
+void image_snapshot_create2(I *ictx, uint32_t flags,
+                            const std::string &group_snap_id,
+                            uint64_t *snap_id, Context *on_finish) {
+  CephContext *cct = ictx->cct;
+  ldout(cct, 10) << "ictx=" << ictx << dendl;
+
+  uint64_t snap_create_flags = 0;
+  int r = librbd::util::snap_create_flags_api_to_internal(cct, flags,
+                                                          &snap_create_flags);
+  if (r < 0) {
+    on_finish->complete(r);
+    return;
+  }
+
+  auto on_refresh = new LambdaContext(
+    [ictx, snap_create_flags, group_snap_id, snap_id, on_finish](int r) {
+      if (r < 0) {
+        lderr(ictx->cct) << "refresh failed: " << cpp_strerror(r) << dendl;
+        on_finish->complete(r);
+        return;
+      }
+
+      auto ctx = new C_ImageSnapshotCreate2<I>(ictx, snap_create_flags,
+                                               ictx->group_spec.pool_id,
+                                               ictx->group_spec.group_id,
+                                               group_snap_id, snap_id,
+                                               on_finish);
+      auto req = mirror::GetInfoRequest<I>::create(*ictx, &ctx->mirror_image,
+                                                   &ctx->promotion_state,
+                                                   &ctx->primary_mirror_uuid,
+                                                   ctx);
+      req->send();
+    });
+
+  if (ictx->state->is_refresh_required()) {
+    ictx->state->refresh(on_refresh);
+  } else {
+    on_refresh->complete(0);
+  }
+}
+
+
+template <typename I>
+void GroupCreatePrimaryRequest<I>::send() {
+ get_group_id();
+}
+
+template <typename I>
+void GroupCreatePrimaryRequest<I>::get_group_id() {
+  ldout(m_cct, 10) << dendl;
+
+  librados::ObjectReadOperation op;
+  cls_client::dir_get_id_start(&op, m_group_name);
+
+  auto comp = create_rados_callback<
+      GroupCreatePrimaryRequest<I>,
+      &GroupCreatePrimaryRequest<I>::handle_get_group_id>(this);
+
+  m_outbl.clear();
+  int r = m_group_ioctx.aio_operate(RBD_GROUP_DIRECTORY, comp, &op, &m_outbl);
+  ceph_assert(r == 0);
+  comp->release();
+}
+
+template <typename I>
+void GroupCreatePrimaryRequest<I>::handle_get_group_id(int r) {
+  ldout(m_cct, 10) << "r=" << r << dendl;
+
+  if (r < 0) {
+    lderr(m_cct) << "failed to get ID of group '" << m_group_name
+                 << "': " << cpp_strerror(r) << dendl;
+    finish(r);
+    return;
+  }
+
+  auto it = m_outbl.cbegin();
+  r = cls_client::dir_get_id_finish(&it, &m_group_id);
+  if (r < 0) {
+    lderr(m_cct) << "failed to get ID of group '" << m_group_name
+                 << "': " << cpp_strerror(r) << dendl;
+    finish(r);
+    return;
+  }
+
+  get_mirror_group();
+}
+
+template <typename I>
+void GroupCreatePrimaryRequest<I>::get_mirror_group() {
+  ldout(m_cct, 10) << dendl;
+
+  librados::ObjectReadOperation op;
+  cls_client::mirror_group_get_start(&op, m_group_id);
+
+  auto comp = create_rados_callback<
+      GroupCreatePrimaryRequest<I>,
+      &GroupCreatePrimaryRequest<I>::handle_get_mirror_group>(this);
+
+  m_outbl.clear();
+  int r = m_group_ioctx.aio_operate(RBD_MIRRORING, comp, &op, &m_outbl);
+  ceph_assert(r == 0);
+  comp->release();
+}
+
+template <typename I>
+void GroupCreatePrimaryRequest<I>::handle_get_mirror_group(int r) {
+  ldout(m_cct, 10) << "r=" << r << dendl;
+
+  if (r == -ENOENT) {
+    ldout(m_cct, 10) << "mirroring for group '" << m_group_name
+                     << "' disabled" << dendl;
+    finish(-EINVAL);
+    return;
+  } else if (r < 0) {
+    lderr(m_cct) << "failed to retrieve mirror group metadata for group '"
+                 << m_group_name << "': " << cpp_strerror(r) << dendl;
+    finish(r);
+    return;
+  }
+
+  auto it = m_outbl.cbegin();
+  r = cls_client::mirror_group_get_finish(&it, &m_mirror_group);
+  if (r < 0) {
+    lderr(m_cct) << "failed to retrieve mirror group metadata for group '"
+                 << m_group_name << "': " << cpp_strerror(r) << dendl;
+    finish(r);
+    return;
+  }
+
+  if (m_mirror_group.mirror_image_mode !=
+      cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT) {
+    auto mode = static_cast<rbd_mirror_image_mode_t>(
+        m_mirror_group.mirror_image_mode);
+    lderr(m_cct) << "cannot create snapshot, mirror mode is set to: "
+                 << mode << dendl;
+    finish(-EOPNOTSUPP);
+    return;
+  }
+
+  get_last_mirror_snapshot_state();
+}
+
+template <typename I>
+void GroupCreatePrimaryRequest<I>::get_last_mirror_snapshot_state() {
+  ldout(m_cct, 10) << dendl;
+
+  auto ctx = util::create_context_callback<
+    GroupCreatePrimaryRequest<I>,
+    &GroupCreatePrimaryRequest<I>::handle_get_last_mirror_snapshot_state>(
+      this);
+
+  auto req = group::ListSnapshotsRequest<I>::create(
+    m_group_ioctx, m_group_id, true, true, &m_existing_group_snaps, ctx);
+
+  req->send();
+}
+
+template <typename I>
+void GroupCreatePrimaryRequest<I>::handle_get_last_mirror_snapshot_state(
+    int r) {
+  ldout(m_cct, 10) << dendl;
+
+  if (r < 0) {
+    lderr(m_cct) << "failed to list group snapshots of group '" << m_group_name
+               << "': " << cpp_strerror(r) << dendl;
+    finish(r);
+    return;
+  }
+
+  cls::rbd::MirrorSnapshotState state =
+    cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY;
+  for (auto it = m_existing_group_snaps.rbegin();
+       it != m_existing_group_snaps.rend(); it++) {
+    auto ns = std::get_if<cls::rbd::GroupSnapshotNamespaceMirror>(
+      &it->snapshot_namespace);
+    if (ns != nullptr) {
+      // XXXMG: check primary_mirror_uuid matches?
+      ldout(m_cct, 10) <<  "the state of existing group snap is: " << ns->state
+                     << dendl;
+      state = ns->state;
+      break;
+    }
+  }
+
+  if (state != cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY) {
+    lderr(m_cct) << "group " << m_group_name << " is not primary" << dendl;
+    finish(-EINVAL);
+    return;
+  }
+
+  generate_group_snap();
+}
+
+template <typename I>
+void GroupCreatePrimaryRequest<I>::generate_group_snap() {
+  ldout(m_cct, 10) << dendl;
+
+  m_group_snap.id = librbd::util::generate_image_id(m_group_ioctx);
+  m_group_snap.snapshot_namespace = cls::rbd::GroupSnapshotNamespaceMirror{};
+  m_group_snap.name = prepare_primary_mirror_snap_name(
+    m_cct, m_mirror_group.global_group_id, m_group_snap.id);
+
+  prepare_group_images();
+}
+
+template <typename I>
+void GroupCreatePrimaryRequest<I>::prepare_group_images() {
+  ldout(m_cct, 10) << dendl;
+
+  auto ctx = util::create_context_callback<
+    GroupCreatePrimaryRequest<I>,
+    &GroupCreatePrimaryRequest<I>::handle_prepare_group_images>(this);
+
+  auto req = mirror::snapshot::GroupPrepareImagesRequest<I>::create(
+    m_group_ioctx, m_group_id, cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY,
+    m_flags, &m_group_snap, &m_image_ctxs, &m_quiesce_requests, ctx);
+
+  req->send();
+}
+
+template <typename I>
+void GroupCreatePrimaryRequest<I>::handle_prepare_group_images(int r) {
+  ldout(m_cct, 10) << "r=" << r << dendl;
+
+  if (r < 0) {
+    lderr(m_cct) << "failed to prepare group images '" << m_group_name
+                 << "': " << cpp_strerror(r) << dendl;
+    finish(r);
+    return;
+  }
+
+  create_image_snaps();
+}
+
+template <typename I>
+void GroupCreatePrimaryRequest<I>::create_image_snaps() {
+  ldout(m_cct, 10) << "group name '" << m_group_name << "' group ID '"
+                   << m_group_id
+                   << "' group snap ID '" << m_group_snap.id << dendl;
+
+  auto ctx = librbd::util::create_context_callback<
+    GroupCreatePrimaryRequest<I>,
+    &GroupCreatePrimaryRequest<I>::handle_create_image_snaps>(this);
+  auto gather_ctx = new C_Gather(m_cct, ctx);
+
+  m_image_snap_ids.resize(m_image_ctxs.size(), CEPH_NOSNAP);
+
+  for (size_t i = 0; i < m_image_ctxs.size(); i++) {
+    image_snapshot_create2(m_image_ctxs[i], RBD_SNAP_CREATE_SKIP_QUIESCE,
+                           m_group_snap.id, &m_image_snap_ids[i],
+                           gather_ctx->new_sub());
+  }
+
+  gather_ctx->activate();
+}
+
+template <typename I>
+void GroupCreatePrimaryRequest<I>::handle_create_image_snaps(int r) {
+  ldout(m_cct, 10) << "r=" << r << dendl;
+
+  std::string group_header_oid = librbd::util::group_header_name(m_group_id);
+
+  if (r < 0) {
+    lderr(m_cct) << "failed to create image snaps: "
+                 << cpp_strerror(r) << dendl;
+
+    if (m_ret_code == 0) {
+      m_ret_code = r;
+    }
+
+    ldout(m_cct, 10) << "undoing group create snapshot: " << r << dendl;
+    remove_incomplete_group_snap();
+    return;
+  } else {
+    for (size_t i = 0; i < m_image_ctxs.size(); i++) {
+      m_group_snap.snaps[i].snap_id = m_image_snap_ids[i];
+    }
+
+    m_group_snap.state = cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE;
+    r = cls_client::group_snap_set(&m_group_ioctx, group_header_oid,
+                                   m_group_snap);
+    if (r < 0) {
+      lderr(m_cct) << "failed to update group snapshot metadata: "
+                 << cpp_strerror(r) << dendl;
+      if (m_ret_code == 0) {
+        m_ret_code = r;
+      }
+
+      ldout(m_cct, 10) << "undoing group create snapshot: " << r << dendl;
+      remove_incomplete_group_snap();
+      return;
+    }
+
+    *m_snap_id = m_group_snap.id;
+  }
+
+  if (!m_quiesce_requests.empty()) {
+    notify_unquiesce();
+    return;
+  }
+
+  if (m_ret_code == 0) {
+    unlink_peer_group();
+    return;
+  }
+
+  close_images();
+}
+
+template <typename I>
+void GroupCreatePrimaryRequest<I>::remove_incomplete_group_snap() {
+  ldout(m_cct, 10) << dendl;
+
+  auto ctx = librbd::util::create_context_callback<
+    GroupCreatePrimaryRequest<I>,
+    &GroupCreatePrimaryRequest<I>::handle_remove_incomplete_group_snap>(this);
+  auto gather_ctx = new C_Gather(m_cct, ctx);
+
+
+  for (size_t i = 0; i < m_image_ctxs.size(); ++i) {
+    if (m_group_snap.snaps[i].snap_id == CEPH_NOSNAP) {
+      continue;
+    }
+
+    librbd::ImageCtx *ictx = m_image_ctxs[i];
+
+    std::shared_lock image_locker{ictx->image_lock};
+    auto info = ictx->get_snap_info(
+      m_group_snap.snaps[i].snap_id);
+    ceph_assert(info != nullptr);
+    image_locker.unlock();
+
+    ldout(m_cct, 10) << "removing individual snapshot: "
+                     << info->name << dendl;
+
+    ictx->operations->snap_remove(info->snap_namespace,
+                                  info->name,
+                                  gather_ctx->new_sub());
+  }
+
+  gather_ctx->activate();
+}
+
+template <typename I>
+void GroupCreatePrimaryRequest<I>::handle_remove_incomplete_group_snap(int r) {
+  ldout(m_cct, 10) << "r=" << r << dendl;
+
+  // if previous attempts to remove this snapshot failed then the
+  // image's snapshot may not exist
+  if (r < 0 && r != -ENOENT) {
+    lderr(m_cct) << "failed cleaning up group member image snapshots: "
+                 << cpp_strerror(r) << dendl;
+  }
+
+  if (r == 0) {
+    r = cls_client::group_snap_remove(
+      &m_group_ioctx,
+      librbd::util::group_header_name(m_group_id),
+      m_group_snap.id);
+
+    if (r < 0) {
+      lderr(m_cct) << "failed to remove group snapshot metadata: "
+                   << cpp_strerror(r) << dendl;
+    }
+  }
+
+  if (!m_quiesce_requests.empty()) {
+    notify_unquiesce();
+    return;
+  }
+
+  close_images();
+}
+
+template <typename I>
+void GroupCreatePrimaryRequest<I>::notify_unquiesce() {
+  ldout(m_cct, 10) << dendl;
+
+  ceph_assert(m_quiesce_requests.size() == m_image_ctxs.size());
+
+  auto ctx = librbd::util::create_context_callback<
+    GroupCreatePrimaryRequest<I>,
+    &GroupCreatePrimaryRequest<I>::handle_notify_unquiesce>(this);
+  auto gather_ctx = new C_Gather(m_cct, ctx);
+
+  int image_count = m_image_ctxs.size();
+  for (int i = 0; i < image_count; ++i) {
+    auto ictx = m_image_ctxs[i];
+    ictx->image_watcher->notify_unquiesce(m_quiesce_requests[i],
+                                          gather_ctx->new_sub());
+  }
+
+  gather_ctx->activate();
+}
+
+template <typename I>
+void GroupCreatePrimaryRequest<I>::handle_notify_unquiesce(int r) {
+  ldout(m_cct, 10) << "r=" << r << dendl;
+
+  if (r < 0) {
+    lderr(m_cct) << "failed to notify the unquiesce requests: "
+                 << cpp_strerror(r) << dendl;
+  }
+
+  if (m_ret_code == 0) {
+    unlink_peer_group();
+    return;
+  }
+
+  close_images();
+}
+
+template <typename I>
+void GroupCreatePrimaryRequest<I>::unlink_peer_group() {
+  ldout(m_cct, 10) << dendl;
+
+  auto ctx = librbd::util::create_context_callback<
+    GroupCreatePrimaryRequest<I>,
+    &GroupCreatePrimaryRequest<I>::handle_unlink_peer_group>(this);
+
+  auto req = group::UnlinkPeerGroupRequest<I>::create(
+    m_group_ioctx, m_group_id, &m_image_ctxs, ctx);
+
+  req->send();
+}
+
+template <typename I>
+void GroupCreatePrimaryRequest<I>::handle_unlink_peer_group(int r) {
+  ldout(m_cct, 10) << "r=" << r << dendl;
+
+  if (r < 0) {
+    lderr(m_cct) << "failed to unlink peer group: " << cpp_strerror(r)
+                 << dendl;
+  }
+
+  close_images();
+}
+
+template <typename I>
+void GroupCreatePrimaryRequest<I>::close_images() {
+  ldout(m_cct, 10) << dendl;
+
+  auto ctx = librbd::util::create_context_callback<
+    GroupCreatePrimaryRequest<I>,
+    &GroupCreatePrimaryRequest<I>::handle_close_images>(this);
+  auto gather_ctx = new C_Gather(m_cct, ctx);
+
+  for (auto ictx: m_image_ctxs) {
+    ictx->state->close(gather_ctx->new_sub());
+  }
+
+  gather_ctx->activate();
+}
+
+template <typename I>
+void GroupCreatePrimaryRequest<I>::handle_close_images(int r) {
+  ldout(m_cct, 10) << "r=" << r << dendl;
+
+  if (r < 0) {
+    lderr(m_cct) << "failed to close images: " << cpp_strerror(r) << dendl;
+  }
+
+  m_image_ctxs.clear();
+  finish(m_ret_code);
+}
+
+template <typename I>
+void GroupCreatePrimaryRequest<I>::finish(int r) {
+  ldout(m_cct, 10) << "r=" << r << dendl;
+
+  m_on_finish->complete(r);
+  delete this;
+}
+
+} // namespace snapshot
+} // namespace mirror
+} // namespace librbd
+
+template class librbd::mirror::snapshot::GroupCreatePrimaryRequest<librbd::ImageCtx>;
diff --git a/src/librbd/mirror/snapshot/GroupCreatePrimaryRequest.h b/src/librbd/mirror/snapshot/GroupCreatePrimaryRequest.h
new file mode 100644 (file)
index 0000000..d8567a0
--- /dev/null
@@ -0,0 +1,139 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CEPH_LIBRBD_MIRROR_SNAPSHOT_GROUP_CREATE_PRIMARY_REQUEST_H
+#define CEPH_LIBRBD_MIRROR_SNAPSHOT_GROUP_CREATE_PRIMARY_REQUEST_H
+
+#include "include/buffer.h"
+#include "include/rados/librados.hpp"
+#include "librbd/ImageWatcher.h"
+#include "cls/rbd/cls_rbd_types.h"
+#include "librbd/internal.h"
+#include "librbd/mirror/snapshot/Types.h"
+
+#include <string>
+#include <set>
+
+struct Context;
+
+namespace librbd {
+
+struct ImageCtx;
+
+namespace mirror {
+namespace snapshot {
+
+template <typename ImageCtxT = librbd::ImageCtx>
+class GroupCreatePrimaryRequest {
+public:
+  static GroupCreatePrimaryRequest *create(librados::IoCtx& group_ioctx,
+                                    const std::string& group_name,
+                                    uint32_t flags, std::string *snap_id,
+                                    Context *on_finish) {
+    return new GroupCreatePrimaryRequest(group_ioctx, group_name, flags,
+                                         snap_id, on_finish);
+  }
+
+  GroupCreatePrimaryRequest(librados::IoCtx& group_ioctx,
+                     const std::string& group_name,
+                     uint32_t flags, std::string *snap_id,
+                     Context *on_finish)
+    : m_group_ioctx(group_ioctx), m_group_name(group_name), m_flags(flags),
+      m_snap_id(snap_id), m_on_finish(on_finish) {
+    m_cct = (CephContext *)group_ioctx.cct();
+  }
+
+  void send();
+
+private:
+  // TODO: Complete the diagram
+  /**
+   * @verbatim
+   *
+   * <start>
+   *    |
+   *    v
+   * GET GROUP ID
+   *    |
+   *    v
+   * GET LAST MIRROR SNAPSHOT STATE
+   *    |
+   *    v
+   * PREPARE GROUP IMAGES
+   *    |
+   *    |                 (on error)
+   * CREATE IMAGE SNAPS . . . . . . . REMOVE INCOMPLETE GROUP SNAP
+   *    |                                  .
+   *    v                                  .
+   * NOTIFY UNQUIESCE < . . . . . . . . .  .
+   *    |
+   *    v
+   * UNLINK PEER GROUP
+   *    |
+   *    v
+   * CLOSE IMAGES
+   *    |
+   *    v
+   * <finish>
+   *
+   * @endverbatim
+   */
+
+  librados::IoCtx m_group_ioctx;
+  const std::string m_group_name;
+  const uint32_t m_flags;
+  std::string *m_snap_id;
+  Context *m_on_finish;
+
+  CephContext *m_cct;
+
+  bufferlist m_outbl;
+  std::string m_group_id;
+  cls::rbd::MirrorGroup m_mirror_group;
+  std::vector<cls::rbd::GroupSnapshot> m_existing_group_snaps;
+  cls::rbd::GroupSnapshot m_group_snap;
+  std::vector<ImageCtx *> m_image_ctxs;
+  std::vector<uint64_t> m_quiesce_requests;
+  std::vector<uint64_t> m_image_snap_ids;
+  int m_ret_code=0;
+
+  void get_group_id();
+  void handle_get_group_id(int r);
+
+  void get_mirror_group();
+  void handle_get_mirror_group(int r);
+
+  void get_last_mirror_snapshot_state();
+  void handle_get_last_mirror_snapshot_state(int r);
+
+  void generate_group_snap();
+
+  void prepare_group_images();
+  void handle_prepare_group_images(int r);
+
+  void create_image_snaps();
+  void handle_create_image_snaps(int r);
+
+  void notify_unquiesce();
+  void handle_notify_unquiesce(int r);
+
+  void unlink_peer_group();
+  void handle_unlink_peer_group(int r);
+
+  void close_images();
+  void handle_close_images(int r);
+
+  void finish(int r);
+
+  // cleanup
+  void remove_incomplete_group_snap();
+  void handle_remove_incomplete_group_snap(int r);
+};
+
+} // namespace snapshot
+} // namespace mirror
+} // namespace librbd
+
+extern template class librbd::mirror::snapshot::GroupCreatePrimaryRequest<librbd::ImageCtx>;
+
+#endif // CEPH_LIBRBD_MIRROR_SNAPSHOT_GROUP_CREATE_PRIMARY_REQUEST_H