return r;
}
+void group_snap_remove(librados::ObjectWriteOperation *op,
+ const std::string &snap_id)
+{
+ using ceph::encode;
+ bufferlist inbl, outbl;
+ encode(snap_id, inbl);
+ op->exec("rbd", "group_snap_remove", inbl);
+}
+
int group_snap_remove(librados::IoCtx *ioctx, const std::string &oid,
const std::string &snap_id)
{
const cls::rbd::GroupSnapshot &snapshot);
int group_snap_set(librados::IoCtx *ioctx, const std::string &oid,
const cls::rbd::GroupSnapshot &snapshot);
+void group_snap_remove(librados::ObjectWriteOperation *op,
+ const std::string &snap_id);
int group_snap_remove(librados::IoCtx *ioctx, const std::string &oid,
const std::string &snap_id);
void group_snap_unlink(librados::ObjectWriteOperation *op,
group/ListSnapshotsRequest.cc
group/AddImageRequest.cc
group/RemoveImageRequest.cc
- group/UnlinkPeerGroupRequest.cc
image/AttachChildRequest.cc
image/AttachParentRequest.cc
image/CloneRequest.cc
mirror/snapshot/DemoteRequest.cc
mirror/snapshot/GetImageStateRequest.cc
mirror/snapshot/GroupCreatePrimaryRequest.cc
- mirror/snapshot/GroupPrepareImagesRequest.cc
+ mirror/snapshot/GroupUnlinkPeerRequest.cc
mirror/snapshot/ImageMeta.cc
mirror/snapshot/PromoteRequest.cc
mirror/snapshot/RemoveImageStateRequest.cc
#include "librbd/api/Namespace.h"
#include "librbd/api/Utils.h"
#include "librbd/group/ListSnapshotsRequest.h"
-#include "librbd/group/UnlinkPeerGroupRequest.h"
#include "librbd/mirror/DemoteRequest.h"
#include "librbd/mirror/DisableRequest.h"
#include "librbd/mirror/EnableRequest.h"
#include "librbd/mirror/snapshot/GroupCreatePrimaryRequest.h"
#include "librbd/mirror/snapshot/ImageMeta.h"
#include "librbd/mirror/snapshot/UnlinkPeerRequest.h"
+#include "librbd/mirror/snapshot/GroupUnlinkPeerRequest.h"
#include "librbd/mirror/snapshot/Utils.h"
#include <boost/algorithm/string/trim.hpp>
#include <boost/algorithm/string/replace.hpp>
}
};
+template <typename I>
+struct C_GroupSnapshotCreate : public Context {
+ IoCtx group_ioctx;
+ std::string group_name;
+ uint64_t flags;
+ std::string *group_snap_id;
+ Context *on_finish;
+
+ cls::rbd::MirrorGroup mirror_group;
+ mirror::PromotionState promotion_state;
+
+ C_GroupSnapshotCreate(IoCtx& group_ioctx, const std::string group_name,
+ uint64_t snap_create_flags,
+ std::string *group_snap_id,
+ Context *on_finish)
+ : group_ioctx(group_ioctx), group_name(group_name),
+ flags(snap_create_flags), group_snap_id(group_snap_id),
+ on_finish(on_finish) {
+ }
+
+ void finish(int r) override {
+ if (r < 0 && r != -ENOENT) {
+ on_finish->complete(r);
+ return;
+ }
+
+ if(mirror_group.state != cls::rbd::MIRROR_GROUP_STATE_ENABLED ||
+ mirror_group.mirror_image_mode != cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT) {
+ CephContext *cct = (CephContext *)group_ioctx.cct();
+ lderr(cct) << "snapshot based mirroring is not enabled for "
+ << group_name << dendl;
+ on_finish->complete(-EINVAL);
+ return;
+ }
+
+ auto req = mirror::snapshot::GroupCreatePrimaryRequest<I>::create(
+ group_ioctx, group_name, flags, group_snap_id, on_finish);
+ req->send();
+ }
+};
+
} // anonymous namespace
template <typename I>
CephContext *cct = reinterpret_cast<CephContext *>(io_ctx.cct());
ldout(cct, 20) << dendl;
+ std::string local_namespace = io_ctx.get_namespace();
+
+ if (local_namespace.empty() && !remote_namespace.empty()) {
+ lderr(cct) << "cannot mirror the default namespace to a "
+ << "non-default namespace." << dendl;
+ return -EINVAL;
+ }
+
+ if (!local_namespace.empty() && remote_namespace.empty()) {
+ lderr(cct) << "cannot mirror a non-default namespace to the default "
+ << "namespace." << dendl;
+ return -EINVAL;
+ }
+
int r = cls_client::mirror_remote_namespace_set(&io_ctx, remote_namespace);
if (r < 0) {
lderr(cct) << "failed to set remote mirror namespace: "
<< cpp_strerror(r) << dendl;
return r;
- }
+ }
return 0;
}
cls::rbd::GroupSnapshot *group_snap,
std::vector<uint64_t> &quiesce_requests,
cls::rbd::MirrorSnapshotState state,
+ std::set<std::string> *mirror_peer_uuids,
uint32_t flags) {
CephContext *cct = (CephContext *)group_ioctx.cct();
ldout(cct, 20) << dendl;
}
group_ioctx.set_namespace(ns);
- std::set<std::string> mirror_peer_uuids;
for (auto &peer : peers) {
if (peer.mirror_peer_direction == cls::rbd::MIRROR_PEER_DIRECTION_RX) {
continue;
}
- mirror_peer_uuids.insert(peer.uuid);
+ mirror_peer_uuids->insert(peer.uuid);
}
- if (mirror_peer_uuids.empty()) {
+ if (mirror_peer_uuids->empty()) {
lderr(cct) << "no mirror tx peers configured for the pool" << dendl;
return -EINVAL;
}
}
group_snap->snapshot_namespace = cls::rbd::GroupSnapshotNamespaceMirror{
- state, mirror_peer_uuids, {}, {}};
+ state, *mirror_peer_uuids, {}, {}};
for (auto image_ctx: *image_ctxs) {
group_snap->snaps.emplace_back(image_ctx->md_ctx.get_id(), image_ctx->id,
std::vector<uint64_t> quiesce_requests;
std::vector<I *> image_ctxs;
+ std::set<std::string> mirror_peer_uuids;
r = prepare_group_images(group_ioctx, group_id, &image_ctxs,
&group_snap, quiesce_requests,
cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY,
+ &mirror_peer_uuids,
flags);
if (r != 0) {
return r;
std::vector<uint64_t> quiesce_requests;
std::vector<I *> image_ctxs;
+ std::set<std::string> mirror_peer_uuids;
r = prepare_group_images(group_ioctx, group_id, &image_ctxs,
&group_snap, quiesce_requests,
cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY,
- flags);
+ &mirror_peer_uuids, flags);
if (r != 0) {
return r;
}
r = prepare_group_images(group_ioctx, group_id, &image_ctxs,
&group_snap, quiesce_requests,
cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED,
- flags);
+ &mirror_peer_uuids, flags);
if (r != 0) {
return r;
}
if (!ret_code) {
C_SaferCond cond;
- auto req = group::UnlinkPeerGroupRequest<I>::create(
- group_ioctx, group_id, &image_ctxs, &cond);
+ auto req = mirror::snapshot::GroupUnlinkPeerRequest<I>::create(
+ group_ioctx, group_id, &mirror_peer_uuids, &image_ctxs, &cond);
req->send();
cond.wait();
}
std::vector<uint64_t> quiesce_requests;
std::vector<I *> image_ctxs;
+ std::set<std::string> mirror_peer_uuids;
r = prepare_group_images(group_ioctx, group_id, &image_ctxs,
&group_snap, quiesce_requests,
cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED,
- flags);
+ &mirror_peer_uuids, flags);
if (r != 0) {
return r;
}
r = prepare_group_images(group_ioctx, group_id, &image_ctxs,
&group_snap, quiesce_requests,
cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED,
- flags);
+ &mirror_peer_uuids, flags);
if (r != 0) {
return r;
}
if (!ret_code) {
C_SaferCond cond;
- auto req = group::UnlinkPeerGroupRequest<I>::create(
- group_ioctx, group_id, &image_ctxs, &cond);
+ auto req = mirror::snapshot::GroupUnlinkPeerRequest<I>::create(
+ group_ioctx, group_id, &mirror_peer_uuids, &image_ctxs, &cond);
req->send();
cond.wait();
}
<< ", group_name=" << group_name
<< ", flags=" << flags << dendl;
- auto req = mirror::snapshot::GroupCreatePrimaryRequest<I>::create(
- group_ioctx, group_name, flags, snap_id, on_finish);
+ uint64_t snap_create_flags = 0;
+ int r = librbd::util::snap_create_flags_api_to_internal(cct, flags,
+ &snap_create_flags);
+ if (r < 0) {
+ on_finish->complete(r);
+ return;
+ }
+ auto ctx = new C_GroupSnapshotCreate<I>(group_ioctx, group_name,
+ snap_create_flags,
+ snap_id,
+ on_finish);
+
+ auto req = mirror::GroupGetInfoRequest<I>::create(
+ group_ioctx, group_name, &ctx->mirror_group, &ctx->promotion_state, ctx);
req->send();
}
std::vector<uint64_t> quiesce_requests;
std::vector<I *> image_ctxs;
+ std::set<std::string> mirror_peer_uuids;
r = prepare_group_images(group_ioctx, group_id, &image_ctxs,
&group_snap, quiesce_requests,
cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY,
- flags);
+ &mirror_peer_uuids, flags);
if (r != 0) {
return r;
}
if (!ret_code) {
C_SaferCond cond;
- auto req = group::UnlinkPeerGroupRequest<I>::create(
- group_ioctx, group_id, &image_ctxs, &cond);
+ auto req = mirror::snapshot::GroupUnlinkPeerRequest<I>::create(
+ group_ioctx, group_id, &mirror_peer_uuids, &image_ctxs, &cond);
req->send();
cond.wait();
}
std::vector<uint64_t> quiesce_requests;
std::vector<I *> image_ctxs;
+ std::set<std::string> mirror_peer_uuids;
int r = prepare_group_images(group_ioctx, group_id, &image_ctxs,
&group_snap, quiesce_requests,
cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY,
- flags);
+ &mirror_peer_uuids, flags);
if (r != 0) {
return r;
}
if (!ret_code) {
C_SaferCond cond;
- auto req = group::UnlinkPeerGroupRequest<I>::create(
- group_ioctx, group_id, &image_ctxs, &cond);
+ auto req = mirror::snapshot::GroupUnlinkPeerRequest<I>::create(
+ group_ioctx, group_id, &mirror_peer_uuids, &image_ctxs, &cond);
req->send();
cond.wait();
}
+++ /dev/null
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-
-#include "include/ceph_assert.h"
-#include "include/Context.h"
-#include "common/Cond.h"
-#include "common/dout.h"
-#include "common/errno.h"
-#include "common/ceph_context.h"
-#include "cls/rbd/cls_rbd_client.h"
-#include "librbd/Operations.h"
-#include "librbd/Utils.h"
-#include "librbd/api/Utils.h"
-#include "librbd/api/Group.h"
-#include "librbd/group/ListSnapshotsRequest.h"
-#include "librbd/group/UnlinkPeerGroupRequest.h"
-#include "librbd/mirror/snapshot/UnlinkPeerRequest.h"
-
-#define dout_subsys ceph_subsys_rbd
-#undef dout_prefix
-#define dout_prefix *_dout << "librbd::group::UnlinkPeerGroupRequest: " << this \
- << " " << __func__ << ": "
-
-namespace librbd {
-namespace group {
-
-using util::create_rados_callback;
-using util::create_context_callback;
-
-template <typename I>
-void UnlinkPeerGroupRequest<I>::send() {
- ldout(m_cct, 10) << dendl;
- list_group_snaps();
-}
-
-template <typename I>
-void UnlinkPeerGroupRequest<I>::list_group_snaps() {
- ldout(m_cct, 10) << dendl;
-
- auto ctx = util::create_context_callback<
- UnlinkPeerGroupRequest<I>,
- &UnlinkPeerGroupRequest<I>::handle_list_group_snaps>(
- this);
-
- m_group_snaps.clear();
- auto req = group::ListSnapshotsRequest<I>::create(
- m_group_io_ctx, m_group_id, true, true, &m_group_snaps, ctx);
-
- req->send();
-}
-
-template <typename I>
-void UnlinkPeerGroupRequest<I>::handle_list_group_snaps(int r) {
- ldout(m_cct, 10) << "r=" << r << dendl;
-
- if (r < 0) {
- lderr(m_cct) << "failed to list group snapshots of group ID '"
- << m_group_id << "': " << cpp_strerror(r) << dendl;
- finish(r);
- return;
- }
-
- unlink_peer();
-}
-
-template <typename I>
-void UnlinkPeerGroupRequest<I>::unlink_peer() {
- ldout(m_cct, 10) << dendl;
-
- uint64_t count = 0;
- auto unlink_snap = m_group_snaps.end();
- auto unlink_unsynced_snap = m_group_snaps.end();
- bool unlink_unsynced = false;
- for (auto it = m_group_snaps.begin(); it != m_group_snaps.end(); it++) {
- auto ns = std::get_if<cls::rbd::GroupSnapshotNamespaceMirror>(
- &it->snapshot_namespace);
- if (ns != nullptr) {
- // FIXME: after relocate, on new primary the previous primary demoted
- // snap is not getting deleted, until the next demotion.
- if (ns->state != cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY &&
- ns->state != cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY) {
- continue;
- }
- count++;
- if (count == 3) {
- unlink_unsynced_snap = it;
- }
- ceph_assert(count <= 5);
-
- // FIXME: This logic of unlinking group snaps will be moved to Group Replayer
- // and will be done by secondary, just added a half backed fix (do not
- // want to spend time on this as this code will be removed) to avoid
- // deleting the previous group snap. This logic will change to makesure
- // we always have a previous completly synced group snap on primary.
- if (ns->mirror_peer_uuids.empty()) {
- auto next_snap = std::next(it);
- if (next_snap != m_group_snaps.end() &&
- next_snap->state != cls::rbd::GROUP_SNAPSHOT_STATE_INCOMPLETE) {
- next_snap = std::next(next_snap);
- if (next_snap != m_group_snaps.end() &&
- next_snap->state != cls::rbd::GROUP_SNAPSHOT_STATE_INCOMPLETE) {
- unlink_snap = it;
- break;
- }
- }
- }
- }
- // TODO: fix the hardcoded max_snaps value
- if (count == 5) {
- unlink_unsynced = true;
- }
- }
-
- if (unlink_snap != m_group_snaps.end()) {
- remove_group_snapshot(*unlink_snap);
- } else if (unlink_unsynced && unlink_unsynced_snap != m_group_snaps.end()) {
- remove_group_snapshot(*unlink_unsynced_snap);
- } else {
- finish(0);
- }
-}
-
-template <typename I>
-void UnlinkPeerGroupRequest<I>::remove_group_snapshot(
- cls::rbd::GroupSnapshot group_snap) {
- ldout(m_cct, 10) << "group snap id: " << group_snap.id << dendl;
-
- m_remove_gp_snap_id = group_snap.id;
-
- auto ctx = librbd::util::create_context_callback<
- UnlinkPeerGroupRequest<I>,
- &UnlinkPeerGroupRequest<I>::handle_remove_group_snapshot>(this);
- auto gather_ctx = new C_Gather(m_cct, ctx);
-
- for (auto &snap : group_snap.snaps) {
- if (snap.snap_id == CEPH_NOSNAP) {
- continue;
- }
- ImageCtx *ictx = nullptr;
- for (size_t i = 0; i < m_image_ctxs->size(); ++i) {
- ictx = (*m_image_ctxs)[i];
- if (ictx->id != snap.image_id) {
- ictx = nullptr;
- } else {
- break;
- }
- }
- if (!ictx) {
- continue;
- }
- ldout(m_cct, 10) << "removing individual snapshot: "
- << snap.snap_id << ", from image id:" << snap.image_id
- << dendl;
- remove_image_snapshot(ictx, snap.snap_id, gather_ctx);
- }
-
- gather_ctx->activate();
-}
-
-template <typename I>
-void UnlinkPeerGroupRequest<I>::handle_remove_group_snapshot(int r) {
- ldout(m_cct, 10) << "r=" << r << dendl;
-
- if (r < 0) {
- lderr(m_cct) << "failed to remove group snapshot metadata: "
- << cpp_strerror(r) << dendl;
- }
-
- r = cls_client::group_snap_remove(&m_group_io_ctx,
- librbd::util::group_header_name(m_group_id), m_remove_gp_snap_id);
- if (r < 0) {
- lderr(m_cct) << "failed to remove group snapshot metadata: "
- << cpp_strerror(r) << dendl;
- }
-
- list_group_snaps();
-}
-
-template <typename I>
-void UnlinkPeerGroupRequest<I>::remove_image_snapshot(
- ImageCtx *image_ctx, uint64_t snap_id, C_Gather *gather_ctx) {
- ldout(m_cct, 10) << snap_id << dendl;
-
- image_ctx->image_lock.lock_shared();
- int r = -ENOENT;
- cls::rbd::SnapshotNamespace snap_namespace;
- std::string snap_name;
- for (auto snap_it = image_ctx->snap_info.find(snap_id);
- snap_it != image_ctx->snap_info.end(); ++snap_it) {
- if (snap_it->first == snap_id) {
- r = 0;
- snap_namespace = snap_it->second.snap_namespace;
- snap_name = snap_it->second.name;
- }
- }
-
- if (r == -ENOENT) {
- ldout(m_cct, 10) << "missing snapshot: snap_id=" << snap_id << dendl;
- image_ctx->image_lock.unlock_shared();
- return;
- }
-
- auto mirror_ns = std::get_if<cls::rbd::MirrorSnapshotNamespace>(
- &snap_namespace);
- if (mirror_ns == nullptr) {
- lderr(m_cct) << "not mirror snapshot (snap_id=" << snap_id << ")" << dendl;
- image_ctx->image_lock.unlock_shared();
- return;
- }
- image_ctx->image_lock.unlock_shared();
- image_ctx->operations->snap_remove(snap_namespace, snap_name.c_str(),
- gather_ctx->new_sub());
-}
-
-template <typename I>
-void UnlinkPeerGroupRequest<I>::finish(int r) {
- ldout(m_cct, 10) << "r=" << r << dendl;
-
- m_on_finish->complete(r);
-}
-
-} // namespace group
-} // namespace librbd
-
-template class librbd::group::UnlinkPeerGroupRequest<librbd::ImageCtx>;
+++ /dev/null
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-
-#ifndef CEPH_LIBRBD_UNLINK_PEER_GROUP_REQUST_H
-#define CEPH_LIBRBD_UNLINK_PEER_GROUP_REQUST_H
-
-#include "include/int_types.h"
-#include "include/types.h"
-#include "include/rados/librados.hpp"
-#include "cls/rbd/cls_rbd_types.h"
-
-#include <string>
-#include <vector>
-
-class Context;
-
-namespace librbd {
-
-struct ImageCtx;
-
-namespace group {
-
-template <typename ImageCtxT = librbd::ImageCtx>
-class UnlinkPeerGroupRequest {
-public:
- static UnlinkPeerGroupRequest *create(
- librados::IoCtx &group_io_ctx, const std::string &group_id,
- std::vector<ImageCtx *> *image_ctxs,
- Context *on_finish) {
- return new UnlinkPeerGroupRequest(group_io_ctx, group_id,
- image_ctxs, on_finish);
- }
-
- UnlinkPeerGroupRequest(librados::IoCtx &group_io_ctx,
- const std::string &group_id,
- std::vector<ImageCtx *> *image_ctxs,
- Context *on_finish)
- : m_group_io_ctx(group_io_ctx), m_group_id(group_id),
- m_image_ctxs(image_ctxs), m_on_finish(on_finish) {
- m_cct = (CephContext *)group_io_ctx.cct();
- }
-
- void send();
-
-private:
- librados::IoCtx &m_group_io_ctx;
- const std::string m_group_id;
- std::vector<ImageCtx *> *m_image_ctxs;
- Context *m_on_finish;
-
- CephContext *m_cct;
-
- std::vector<cls::rbd::GroupSnapshot> m_group_snaps;
- std::string m_remove_gp_snap_id;
-
- void list_group_snaps();
- void handle_list_group_snaps(int r);
-
- void unlink_peer();
-
- void remove_group_snapshot(cls::rbd::GroupSnapshot group_snap);
- void remove_image_snapshot(ImageCtx *image_ctx, uint64_t snap_id,
- C_Gather *gather_ctx);
- void handle_remove_group_snapshot(int r);
-
- void finish(int r);
-};
-
-} // namespace group
-} // namespace librbd
-
-extern template class librbd::group::UnlinkPeerGroupRequest<librbd::ImageCtx>;
-
-#endif // CEPH_LIBRBD_UNLINK_PEER_GROUP_REQUST_H
#include "librbd/internal.h"
#include "common/ceph_context.h"
#include "cls/rbd/cls_rbd_client.h"
+#include "librbd/ExclusiveLock.h"
#include "librbd/ImageCtx.h"
#include "librbd/Operations.h"
#include "librbd/ImageState.h"
#include "librbd/Utils.h"
-#include "librbd/group/UnlinkPeerGroupRequest.h"
#include "librbd/group/ListSnapshotsRequest.h"
#include "librbd/mirror/GetInfoRequest.h"
#include "librbd/mirror/snapshot/CreatePrimaryRequest.h"
-#include "librbd/mirror/snapshot/GroupPrepareImagesRequest.h"
+#include "librbd/mirror/snapshot/GroupUnlinkPeerRequest.h"
#include "librbd/mirror/Types.h"
#define dout_subsys ceph_subsys_rbd
namespace mirror {
namespace snapshot {
+namespace {
+
+const uint32_t MAX_RETURN = 1024;
+
+} // anonymous namespace
+
using librbd::util::create_rados_callback;
template <typename I>
-struct C_ImageSnapshotCreate2 : public Context {
+struct C_ImageSnapshotCreate : public Context {
I *ictx;
uint64_t snap_create_flags;
int64_t group_pool_id;
mirror::PromotionState promotion_state;
std::string primary_mirror_uuid;
- C_ImageSnapshotCreate2(I *ictx, uint64_t snap_create_flags,
+ C_ImageSnapshotCreate(I *ictx, uint64_t snap_create_flags,
int64_t group_pool_id,
const std::string &group_id,
const std::string &group_snap_id,
if (mirror_image.mode != cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT ||
mirror_image.state != cls::rbd::MIRROR_IMAGE_STATE_ENABLED) {
- lderr(ictx->cct) << "snapshot based mirroring is not enabled" << dendl;
+ lderr(ictx->cct) << "snapshot based mirroring is not enabled for "
+ << ictx->id << dendl;
on_finish->complete(-EINVAL);
return;
}
template <typename I>
-void image_snapshot_create2(I *ictx, uint32_t flags,
+void image_snapshot_create(I *ictx, uint64_t snap_create_flags,
const std::string &group_snap_id,
uint64_t *snap_id, Context *on_finish) {
CephContext *cct = ictx->cct;
ldout(cct, 10) << "ictx=" << ictx << dendl;
- uint64_t snap_create_flags = 0;
- int r = librbd::util::snap_create_flags_api_to_internal(cct, flags,
- &snap_create_flags);
- if (r < 0) {
- on_finish->complete(r);
- return;
- }
-
auto on_refresh = new LambdaContext(
[ictx, snap_create_flags, group_snap_id, snap_id, on_finish](int r) {
if (r < 0) {
on_finish->complete(r);
return;
}
-
- auto ctx = new C_ImageSnapshotCreate2<I>(ictx, snap_create_flags,
+//TODO: validate the images earlier.
+ auto ctx = new C_ImageSnapshotCreate<I>(ictx, snap_create_flags,
ictx->group_spec.pool_id,
ictx->group_spec.group_id,
group_snap_id, snap_id,
template <typename I>
void GroupCreatePrimaryRequest<I>::send() {
- get_group_id();
+ ldout(m_cct, 10) << dendl;
+
+ get_group_id();
}
template <typename I>
break;
}
}
-
+// TODO: Do away with this check as we will eventually use this
+// class when promoting groups as well
if (state != cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY) {
lderr(m_cct) << "group " << m_group_name << " is not primary" << dendl;
finish(-EINVAL);
return;
}
+ get_mirror_peer_list();
+}
+
+template <typename I>
+void GroupCreatePrimaryRequest<I>::get_mirror_peer_list() {
+ ldout(m_cct, 10) << dendl;
+
+ m_default_ns_ioctx.dup(m_group_ioctx);
+ m_default_ns_ioctx.set_namespace("");
+
+ librados::ObjectReadOperation op;
+ cls_client::mirror_peer_list_start(&op);
+
+ auto comp = create_rados_callback<
+ GroupCreatePrimaryRequest<I>,
+ &GroupCreatePrimaryRequest<I>::handle_get_mirror_peer_list>(this);
+
+ m_outbl.clear();
+ int r = m_default_ns_ioctx.aio_operate(RBD_MIRRORING, comp, &op, &m_outbl);
+ ceph_assert(r == 0);
+ comp->release();
+}
+
+template <typename I>
+void GroupCreatePrimaryRequest<I>::handle_get_mirror_peer_list(int r) {
+ ldout(m_cct, 10) << "r=" << r << dendl;
+
+ std::vector<cls::rbd::MirrorPeer> peers;
+ if (r == 0) {
+ auto it = m_outbl.cbegin();
+ r = cls_client::mirror_peer_list_finish(&it, &peers);
+ }
+
+ if (r < 0) {
+ lderr(m_cct) << "error listing mirror peers" << cpp_strerror(r) << dendl;
+ finish(r);
+ return;
+ }
+
+ for (auto &peer : peers) {
+ if (peer.mirror_peer_direction == cls::rbd::MIRROR_PEER_DIRECTION_RX) {
+ continue;
+ }
+ m_mirror_peer_uuids.insert(peer.uuid);
+ }
+
+ if (m_mirror_peer_uuids.empty()) {
+ lderr(m_cct) << "no mirror tx peers configured for the pool" << dendl;
+ finish(-EINVAL);
+ return;
+ }
+
+ list_group_images();
+}
+
+
+template <typename I>
+void GroupCreatePrimaryRequest<I>::list_group_images() {
+ ldout(m_cct, 10) << dendl;
+
+ librados::ObjectReadOperation op;
+ cls_client::group_image_list_start(&op, m_start_after, MAX_RETURN);
+
+ auto comp = create_rados_callback<
+ GroupCreatePrimaryRequest<I>,
+ &GroupCreatePrimaryRequest<I>::handle_list_group_images>(this);
+
+ m_outbl.clear();
+ int r = m_group_ioctx.aio_operate(
+ librbd::util::group_header_name(m_group_id), comp, &op, &m_outbl);
+ ceph_assert(r == 0);
+ comp->release();
+}
+
+template <typename I>
+void GroupCreatePrimaryRequest<I>::handle_list_group_images(int r) {
+ ldout(m_cct, 10) << "r=" << r << dendl;
+
+ std::vector<cls::rbd::GroupImageStatus> images;
+ if (r == 0) {
+ auto iter = m_outbl.cbegin();
+ r = cls_client::group_image_list_finish(&iter, &images);
+ }
+
+ if (r < 0) {
+ lderr(m_cct) << "error listing images in group: " << cpp_strerror(r)
+ << dendl;
+ finish(r);
+ return;
+ }
+
+ auto image_count = images.size();
+ m_images.insert(m_images.end(), images.begin(), images.end());
+ if (image_count == MAX_RETURN) {
+ m_start_after = images.rbegin()->spec;
+ list_group_images();
+ return;
+ }
+
+ open_group_images();
+}
+
+template <typename I>
+void GroupCreatePrimaryRequest<I>::open_group_images() {
+ ldout(m_cct, 10) << dendl;
+
+ if(m_images.empty()) {
+ generate_group_snap();
+ return;
+ }
+
+ auto ctx = librbd::util::create_context_callback<
+ GroupCreatePrimaryRequest<I>,
+ &GroupCreatePrimaryRequest<I>::handle_open_group_images>(this);
+ auto gather_ctx = new C_Gather(m_cct, ctx);
+
+ int r = 0;
+ for (size_t i = 0; i < m_images.size(); i++) {
+ auto &image = m_images[i];
+ librbd::IoCtx image_io_ctx;
+ r = librbd::util::create_ioctx(m_group_ioctx, "image",
+ image.spec.pool_id, {},
+ &image_io_ctx);
+ if (r < 0) {
+ m_ret_code = r;
+ break;
+ }
+
+ librbd::ImageCtx* image_ctx = new ImageCtx("", image.spec.image_id.c_str(),
+ nullptr, image_io_ctx, false);
+
+ m_image_ctxs.push_back(image_ctx);
+ image_ctx->state->open(0, gather_ctx->new_sub());
+ }
+
+ gather_ctx->activate();
+}
+
+template <typename I>
+void GroupCreatePrimaryRequest<I>::handle_open_group_images(int r) {
+ ldout(m_cct, 10) << "r=" << r << dendl;
+
+ if (r < 0 && m_ret_code == 0) {
+ m_ret_code = r;
+ }
+
+ if (m_ret_code < 0) {
+ lderr(m_cct) << "failed to open group images: " << cpp_strerror(m_ret_code)
+ << dendl;
+ close_images();
+ return;
+ }
+
generate_group_snap();
}
m_group_snap.name = prepare_primary_mirror_snap_name(
m_cct, m_mirror_group.global_group_id, m_group_snap.id);
- prepare_group_images();
+ // TODO: Fix this to handle primary demoted snaps
+ cls::rbd::MirrorSnapshotState state = cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY;
+
+ // Create incomplete group snap
+ m_group_snap.snapshot_namespace = cls::rbd::GroupSnapshotNamespaceMirror{
+ state, m_mirror_peer_uuids, {}, {}};
+
+ for (auto image_ctx: m_image_ctxs) {
+ m_group_snap.snaps.emplace_back(image_ctx->md_ctx.get_id(), image_ctx->id,
+ CEPH_NOSNAP);
+ }
+
+ set_snap_metadata();
}
template <typename I>
-void GroupCreatePrimaryRequest<I>::prepare_group_images() {
+void GroupCreatePrimaryRequest<I>::set_snap_metadata() {
ldout(m_cct, 10) << dendl;
- auto ctx = util::create_context_callback<
+ librados::ObjectWriteOperation op;
+ cls_client::group_snap_set(&op, m_group_snap);
+
+ auto aio_comp = create_rados_callback<
GroupCreatePrimaryRequest<I>,
- &GroupCreatePrimaryRequest<I>::handle_prepare_group_images>(this);
+ &GroupCreatePrimaryRequest<I>::handle_set_snap_metadata>(this);
+ int r = m_group_ioctx.aio_operate(librbd::util::group_header_name(m_group_id),
+ aio_comp, &op);
+ ceph_assert(r == 0);
+ aio_comp->release();
+}
- auto req = mirror::snapshot::GroupPrepareImagesRequest<I>::create(
- m_group_ioctx, m_group_id, cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY,
- m_flags, &m_group_snap, &m_image_ctxs, &m_quiesce_requests, ctx);
+template <typename I>
+void GroupCreatePrimaryRequest<I>::handle_set_snap_metadata(int r) {
+ ldout(m_cct, 10) << "r=" << r << dendl;
+ if (r < 0) {
+ lderr(m_cct) << "failed to set group snapshot metadata: " << cpp_strerror(r)
+ << dendl;
+ m_ret_code = r;
+ if (m_group_snap.state == cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE) {
+ remove_incomplete_group_snap();
+ } else {
+ close_images();
+ }
+ return;
+ }
- req->send();
+ if (m_group_snap.state == cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE) {
+ release_image_exclusive_locks();
+ } else {
+ notify_quiesce();
+ }
+}
+
+template <typename I>
+void GroupCreatePrimaryRequest<I>::notify_quiesce() {
+ ldout(m_cct, 10) << dendl;
+
+ if ((m_internal_flags & SNAP_CREATE_FLAG_SKIP_NOTIFY_QUIESCE) != 0) {
+ acquire_image_exclusive_locks();
+ return;
+ }
+ auto ctx = librbd::util::create_context_callback<
+ GroupCreatePrimaryRequest<I>,
+ &GroupCreatePrimaryRequest<I>::handle_notify_quiesce>(this);
+ auto gather_ctx = new C_Gather(m_cct, ctx);
+
+ int image_count = m_image_ctxs.size();
+ m_quiesce_requests.resize(image_count);
+
+ for (int i = 0; i < image_count; ++i) {
+ auto ictx = (m_image_ctxs)[i];
+ ictx->image_watcher->notify_quiesce(&(m_quiesce_requests)[i], m_prog_ctx,
+ gather_ctx->new_sub());
+ }
+
+ gather_ctx->activate();
}
template <typename I>
-void GroupCreatePrimaryRequest<I>::handle_prepare_group_images(int r) {
+void GroupCreatePrimaryRequest<I>::handle_notify_quiesce(int r) {
ldout(m_cct, 10) << "r=" << r << dendl;
- if (r < 0) {
- lderr(m_cct) << "failed to prepare group images '" << m_group_name
- << "': " << cpp_strerror(r) << dendl;
- finish(r);
+ if (r < 0 &&
+ (m_internal_flags & SNAP_CREATE_FLAG_IGNORE_NOTIFY_QUIESCE_ERROR) == 0) {
+ m_ret_code = r;
+ notify_unquiesce();
return;
}
+ acquire_image_exclusive_locks();
+}
+
+template <typename I>
+void GroupCreatePrimaryRequest<I>::acquire_image_exclusive_locks() {
+ ldout(m_cct, 10) << dendl;
+
+ m_release_locks = true;
+
+ auto ctx = librbd::util::create_context_callback<
+ GroupCreatePrimaryRequest<I>,
+ &GroupCreatePrimaryRequest<I>::handle_acquire_image_exclusive_locks>(this);
+ auto gather_ctx = new C_Gather(m_cct, ctx);
+
+ for (auto ictx: m_image_ctxs) {
+ std::shared_lock owner_lock{ictx->owner_lock};
+ if (ictx->exclusive_lock != nullptr) {
+ ictx->exclusive_lock->block_requests(-EBUSY);
+ ictx->exclusive_lock->acquire_lock(gather_ctx->new_sub());
+ }
+ }
+
+ gather_ctx->activate();
+}
+
+template <typename I>
+void GroupCreatePrimaryRequest<I>::handle_acquire_image_exclusive_locks(int r) {
+ ldout(m_cct, 10) << "r=" << r << dendl;
+
+ if (r < 0) {
+ lderr(m_cct) << "failed to acquire image exclusive locks: "
+ << cpp_strerror(r) << dendl;
+ m_ret_code = r;
+ remove_snap_metadata();
+ return;
+ }
create_image_snaps();
}
+template <typename I>
+void GroupCreatePrimaryRequest<I>::remove_snap_metadata() {
+ ldout(m_cct, 10) << dendl;
+ librados::ObjectWriteOperation op;
+ cls_client::group_snap_remove(&op, m_group_snap.id);
+
+ auto aio_comp = create_rados_callback<
+ GroupCreatePrimaryRequest<I>,
+ &GroupCreatePrimaryRequest<I>::handle_remove_snap_metadata>(this);
+ int r = m_group_ioctx.aio_operate(librbd::util::group_header_name(m_group_id),
+ aio_comp, &op);
+ ceph_assert(r == 0);
+ aio_comp->release();
+}
+
+template <typename I>
+void GroupCreatePrimaryRequest<I>::handle_remove_snap_metadata(int r) {
+ ldout(m_cct, 10) << " r=" << r << dendl;
+ if (r < 0) {
+ // ignore error
+ lderr(m_cct) << "failed to remove group snapshot metadata: "
+ << cpp_strerror(r) << dendl;
+ }
+ release_image_exclusive_locks();
+}
+
template <typename I>
void GroupCreatePrimaryRequest<I>::create_image_snaps() {
- ldout(m_cct, 10) << "group name '" << m_group_name << "' group ID '"
- << m_group_id
- << "' group snap ID '" << m_group_snap.id << dendl;
+ ldout(m_cct, 10) << "group name: " << m_group_name
+ << ", group ID: " << m_group_id
+ << ", group snap ID: " << m_group_snap.id << dendl;
auto ctx = librbd::util::create_context_callback<
GroupCreatePrimaryRequest<I>,
m_image_snap_ids.resize(m_image_ctxs.size(), CEPH_NOSNAP);
for (size_t i = 0; i < m_image_ctxs.size(); i++) {
- image_snapshot_create2(m_image_ctxs[i], RBD_SNAP_CREATE_SKIP_QUIESCE,
- m_group_snap.id, &m_image_snap_ids[i],
- gather_ctx->new_sub());
+ image_snapshot_create(m_image_ctxs[i], SNAP_CREATE_FLAG_SKIP_NOTIFY_QUIESCE,
+ m_group_snap.id, &m_image_snap_ids[i],
+ gather_ctx->new_sub());
}
gather_ctx->activate();
std::string group_header_oid = librbd::util::group_header_name(m_group_id);
+ for (size_t i = 0; i < m_image_ctxs.size(); i++) {
+ m_group_snap.snaps[i].snap_id = m_image_snap_ids[i];
+ }
+
if (r < 0) {
lderr(m_cct) << "failed to create image snaps: "
<< cpp_strerror(r) << dendl;
remove_incomplete_group_snap();
return;
} else {
- for (size_t i = 0; i < m_image_ctxs.size(); i++) {
- m_group_snap.snaps[i].snap_id = m_image_snap_ids[i];
- }
-
m_group_snap.state = cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE;
- r = cls_client::group_snap_set(&m_group_ioctx, group_header_oid,
- m_group_snap);
- if (r < 0) {
- lderr(m_cct) << "failed to update group snapshot metadata: "
- << cpp_strerror(r) << dendl;
- if (m_ret_code == 0) {
- m_ret_code = r;
- }
-
- ldout(m_cct, 10) << "undoing group create snapshot: " << r << dendl;
- remove_incomplete_group_snap();
- return;
- }
-
*m_snap_id = m_group_snap.id;
- }
-
- if (!m_quiesce_requests.empty()) {
- notify_unquiesce();
- return;
- }
- if (m_ret_code == 0) {
- unlink_peer_group();
- return;
+ set_snap_metadata();
}
-
- close_images();
}
template <typename I>
void GroupCreatePrimaryRequest<I>::remove_incomplete_group_snap() {
ldout(m_cct, 10) << dendl;
+
auto ctx = librbd::util::create_context_callback<
GroupCreatePrimaryRequest<I>,
&GroupCreatePrimaryRequest<I>::handle_remove_incomplete_group_snap>(this);
for (size_t i = 0; i < m_image_ctxs.size(); ++i) {
- if (m_group_snap.snaps[i].snap_id == CEPH_NOSNAP) {
+ auto snap_id = m_group_snap.snaps[i].snap_id;
+ if (snap_id == CEPH_NOSNAP) {
continue;
}
librbd::ImageCtx *ictx = m_image_ctxs[i];
std::shared_lock image_locker{ictx->image_lock};
- auto info = ictx->get_snap_info(
- m_group_snap.snaps[i].snap_id);
+ auto info = ictx->get_snap_info(snap_id);
ceph_assert(info != nullptr);
image_locker.unlock();
ldout(m_cct, 10) << "removing individual snapshot: "
- << info->name << dendl;
+ << snap_id << dendl;
ictx->operations->snap_remove(info->snap_namespace,
info->name,
// if previous attempts to remove this snapshot failed then the
// image's snapshot may not exist
if (r < 0 && r != -ENOENT) {
- lderr(m_cct) << "failed cleaning up group member image snapshots: "
+ lderr(m_cct) << "failed to remove group member image snapshots: "
<< cpp_strerror(r) << dendl;
}
if (r == 0) {
- r = cls_client::group_snap_remove(
- &m_group_ioctx,
- librbd::util::group_header_name(m_group_id),
- m_group_snap.id);
-
- if (r < 0) {
- lderr(m_cct) << "failed to remove group snapshot metadata: "
- << cpp_strerror(r) << dendl;
- }
- }
-
- if (!m_quiesce_requests.empty()) {
- notify_unquiesce();
+ remove_snap_metadata();
return;
}
- close_images();
+ release_image_exclusive_locks();
}
template <typename I>
void GroupCreatePrimaryRequest<I>::notify_unquiesce() {
ldout(m_cct, 10) << dendl;
+ if (m_quiesce_requests.empty()) {
+ unlink_peer_group();
+ return;
+ }
+
ceph_assert(m_quiesce_requests.size() == m_image_ctxs.size());
auto ctx = librbd::util::create_context_callback<
void GroupCreatePrimaryRequest<I>::unlink_peer_group() {
ldout(m_cct, 10) << dendl;
+ // Unlink snaps only if the current snap was created successfully
+ if (m_ret_code != 0) {
+ close_images();
+ return;
+ }
+
auto ctx = librbd::util::create_context_callback<
GroupCreatePrimaryRequest<I>,
&GroupCreatePrimaryRequest<I>::handle_unlink_peer_group>(this);
- auto req = group::UnlinkPeerGroupRequest<I>::create(
- m_group_ioctx, m_group_id, &m_image_ctxs, ctx);
+ auto req = GroupUnlinkPeerRequest<I>::create(
+ m_group_ioctx, m_group_id, &m_mirror_peer_uuids, &m_image_ctxs, ctx);
req->send();
}
ldout(m_cct, 10) << "r=" << r << dendl;
if (r < 0) {
- lderr(m_cct) << "failed to unlink peer group: " << cpp_strerror(r)
+ lderr(m_cct) << "failed to unlink group peers: " << cpp_strerror(r)
<< dendl;
}
close_images();
}
+template <typename I>
+void GroupCreatePrimaryRequest<I>::release_image_exclusive_locks() {
+ ldout(m_cct, 10) << dendl;
+
+ if(!m_release_locks){
+ notify_unquiesce();
+ return;
+ }
+ auto ctx = librbd::util::create_context_callback<
+ GroupCreatePrimaryRequest<I>,
+ &GroupCreatePrimaryRequest<I>::handle_release_image_exclusive_locks>(this);
+ auto gather_ctx = new C_Gather(m_cct, ctx);
+
+ for (auto ictx: m_image_ctxs) {
+ std::shared_lock owner_lock{ictx->owner_lock};
+ if (ictx->exclusive_lock != nullptr) {
+ ictx->exclusive_lock->release_lock(gather_ctx->new_sub());
+ }
+ }
+
+ gather_ctx->activate();
+}
+
+template <typename I>
+void GroupCreatePrimaryRequest<I>::handle_release_image_exclusive_locks(int r) {
+ ldout(m_cct, 10) << "r=" << r << dendl;
+
+ if (r < 0) {
+ lderr(m_cct) << "failed to release exclusive locks for images: "
+ << cpp_strerror(r) << dendl;
+ }
+ notify_unquiesce();
+}
+
template <typename I>
void GroupCreatePrimaryRequest<I>::close_images() {
ldout(m_cct, 10) << dendl;
+ if(m_image_ctxs.empty()) {
+ finish(m_ret_code);
+ return;
+ }
+
auto ctx = librbd::util::create_context_callback<
GroupCreatePrimaryRequest<I>,
&GroupCreatePrimaryRequest<I>::handle_close_images>(this);
class GroupCreatePrimaryRequest {
public:
static GroupCreatePrimaryRequest *create(librados::IoCtx& group_ioctx,
- const std::string& group_name,
- uint32_t flags, std::string *snap_id,
- Context *on_finish) {
+ const std::string& group_name,
+ uint64_t flags, std::string *snap_id,
+ Context *on_finish) {
return new GroupCreatePrimaryRequest(group_ioctx, group_name, flags,
snap_id, on_finish);
}
-
+// TODO: Allow demoted flag?
GroupCreatePrimaryRequest(librados::IoCtx& group_ioctx,
const std::string& group_name,
uint32_t flags, std::string *snap_id,
* GET LAST MIRROR SNAPSHOT STATE
* |
* v
- * PREPARE GROUP IMAGES
+ * LIST_GROUP_IMAGES
+ * |
+ * v
+ * OPEN_IMAGES
+ * |
+ * v
+ * SET_GROUP_INCOMPLETE_SNAP
+ * |
+ * v
+ * NOTIFY_QUIESCE (optional)
* |
- * | (on error)
- * CREATE IMAGE SNAPS . . . . . . . REMOVE INCOMPLETE GROUP SNAP
- * | .
- * v .
- * NOTIFY UNQUIESCE < . . . . . . . . . .
+ * v
+ * ACQUIRE_IMAGE_EXCLUSIVE_LOCKS
+ * |
+ * v
+ * CREATE IMAGE SNAPS
+ * |
+ * v
+ * SET_GROUP_COMPLETE_SNAP
+ * |
+ * v
+ * RELEASE_IMAGE_EXCLUSIVE_LOCKS
+ * |
+ * v
+ * NOTIFY UNQUIESCE
* |
* v
* UNLINK PEER GROUP
librados::IoCtx m_group_ioctx;
const std::string m_group_name;
- const uint32_t m_flags;
+ const uint64_t m_flags;
std::string *m_snap_id;
Context *m_on_finish;
std::vector<ImageCtx *> m_image_ctxs;
std::vector<uint64_t> m_quiesce_requests;
std::vector<uint64_t> m_image_snap_ids;
+ std::set<std::string> m_mirror_peer_uuids;
+ librados::IoCtx m_default_ns_ioctx;
+ uint64_t m_internal_flags;
int m_ret_code=0;
+ cls::rbd::GroupImageSpec m_start_after;
+ std::vector<cls::rbd::GroupImageStatus> m_images;
+ NoOpProgressContext m_prog_ctx;
+ bool m_release_locks = false;
void get_group_id();
void handle_get_group_id(int r);
void generate_group_snap();
- void prepare_group_images();
- void handle_prepare_group_images(int r);
-
void create_image_snaps();
void handle_create_image_snaps(int r);
void close_images();
void handle_close_images(int r);
- void finish(int r);
+ void get_mirror_peer_list();
+ void handle_get_mirror_peer_list(int r);
+
+ void list_group_images();
+ void handle_list_group_images(int r);
+
+ void open_group_images();
+ void handle_open_group_images(int r);
+
+ void set_snap_metadata();
+ void handle_set_snap_metadata(int r);
+
+ void notify_quiesce();
+ void handle_notify_quiesce(int r);
+
+ void acquire_image_exclusive_locks();
+ void handle_acquire_image_exclusive_locks(int r);
+
+ void release_image_exclusive_locks();
+ void handle_release_image_exclusive_locks(int r);
+
// cleanup
void remove_incomplete_group_snap();
void handle_remove_incomplete_group_snap(int r);
+
+ void remove_snap_metadata();
+ void handle_remove_snap_metadata(int r);
+
+ void finish(int r);
+
};
} // namespace snapshot
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "include/ceph_assert.h"
+#include "common/Cond.h"
+#include "common/dout.h"
+#include "common/errno.h"
+#include "common/ceph_context.h"
+#include "cls/rbd/cls_rbd_client.h"
+#include "librbd/Operations.h"
+#include "librbd/Utils.h"
+#include "librbd/api/Utils.h"
+#include "librbd/api/Group.h"
+#include "librbd/group/ListSnapshotsRequest.h"
+//#include "librbd/mirror/snapshot/UnlinkPeerRequest.h"
+#include "librbd/mirror/snapshot/GroupUnlinkPeerRequest.h"
+
+#define dout_subsys ceph_subsys_rbd
+#undef dout_prefix
+#define dout_prefix *_dout << "librbd::mirror::snapshot::GroupUnlinkPeerRequest: " << this \
+ << " " << __func__ << ": "
+
+namespace librbd {
+namespace mirror {
+namespace snapshot {
+
+using util::create_rados_callback;
+using util::create_context_callback;
+
+template <typename I>
+void GroupUnlinkPeerRequest<I>::send() {
+ ldout(m_cct, 10) << dendl;
+ m_max_snaps =
+ m_cct->_conf.get_val<uint64_t>("rbd_mirroring_max_mirroring_snapshots");
+
+ list_group_snaps();
+}
+
+
+template <typename I>
+void GroupUnlinkPeerRequest<I>::unlink_peer() {
+ ldout(m_cct, 10) << "rbd_mirroring_max_mirroring_snapshots = " << m_max_snaps << dendl;
+
+ uint64_t count = 0;
+ auto unlink_snap = m_group_snaps.end();
+ // First pass : cleanup snaps that have no peer_uuids or are incomplete
+ for (auto peer: *m_mirror_peer_uuids){
+ for (auto it = m_group_snaps.begin(); it != m_group_snaps.end(); it++) {
+ auto ns = std::get_if<cls::rbd::GroupSnapshotNamespaceMirror>(
+ &it->snapshot_namespace);
+ if (ns == nullptr) {
+ continue;
+ }
+
+ if (ns->mirror_peer_uuids.empty() ||
+ (ns->mirror_peer_uuids.count(peer) != 0 &&
+ ns->is_primary() &&
+ it->state == cls::rbd::GROUP_SNAPSHOT_STATE_INCOMPLETE)){
+ unlink_snap = it;
+ process_snapshot(*unlink_snap, peer);
+ return;
+ }
+ }
+ }
+
+ for (auto peer: *m_mirror_peer_uuids){
+ for (auto it = m_group_snaps.begin(); it != m_group_snaps.end(); it++) {
+ auto ns = std::get_if<cls::rbd::GroupSnapshotNamespaceMirror>(
+ &it->snapshot_namespace);
+ if (ns == nullptr) {
+ continue;
+ }
+ // FIXME: after relocate, on new primary the previous primary demoted
+ // snap is not getting deleted, until the next demotion.
+ if (ns->state != cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY) {
+ // Reset the count if the group was demoted.
+ count = 0;
+ continue;
+ }
+
+ if (ns->mirror_peer_uuids.count(peer) == 0) {
+ continue;
+ }
+
+ count++;
+ if (count == m_max_snaps) {
+ unlink_snap = it;
+ }
+ if (count > m_max_snaps) {
+ process_snapshot(*unlink_snap, peer);
+ return;
+ }
+ }
+ }
+
+ ldout(m_cct, 10)<< "no more snaps" << dendl;
+ finish(0);
+}
+
+
+template <typename I>
+void GroupUnlinkPeerRequest<I>::list_group_snaps() {
+ ldout(m_cct, 10) << dendl;
+
+ auto ctx = util::create_context_callback<
+ GroupUnlinkPeerRequest<I>,
+ &GroupUnlinkPeerRequest<I>::handle_list_group_snaps>(
+ this);
+
+ m_group_snaps.clear();
+ auto req = group::ListSnapshotsRequest<I>::create(
+ m_group_io_ctx, m_group_id, true, true, &m_group_snaps, ctx);
+
+ req->send();
+}
+
+
+
+template <typename I>
+void GroupUnlinkPeerRequest<I>::handle_list_group_snaps(int r) {
+ ldout(m_cct, 10) << "r=" << r << dendl;
+
+ if (r < 0) {
+ lderr(m_cct) << "failed to list group snapshots of group ID '"
+ << m_group_id << "': " << cpp_strerror(r) << dendl;
+ finish(r);
+ return;
+ }
+
+ unlink_peer();
+}
+
+template <typename I>
+void GroupUnlinkPeerRequest<I>::process_snapshot(cls::rbd::GroupSnapshot group_snap,
+ std::string mirror_peer_uuid) {
+ ldout(m_cct, 10) << "snap id: " << group_snap.id << dendl;
+ bool found = false;
+ bool has_newer_mirror_snap = false;
+
+ for (auto it = m_group_snaps.begin(); it != m_group_snaps.end(); it++) {
+ if (it->id == group_snap.id) {
+ found = true;
+ } else if (found) {
+ auto ns = std::get_if<cls::rbd::GroupSnapshotNamespaceMirror>(
+ &it->snapshot_namespace);
+ if (ns != nullptr) {
+ has_newer_mirror_snap = true;
+ break;
+ }
+ }
+ }
+
+ if (!found) {
+ ldout(m_cct, 15) << "missing snapshot: snap_id=" << group_snap.id << dendl;
+ finish(-ENOENT);
+ return;
+ }
+
+ if (has_newer_mirror_snap) {
+ remove_group_snapshot(group_snap);
+ } else {
+ remove_peer_uuid(group_snap, mirror_peer_uuid);
+ }
+}
+
+
+template <typename I>
+void GroupUnlinkPeerRequest<I>::remove_peer_uuid(
+ cls::rbd::GroupSnapshot group_snap,
+ std::string mirror_peer_uuid) {
+ ldout(m_cct, 10) << dendl;
+
+ auto aio_comp = create_rados_callback<
+ GroupUnlinkPeerRequest<I>,
+ &GroupUnlinkPeerRequest<I>::handle_remove_peer_uuid>(this);
+
+ auto ns = std::get_if<cls::rbd::GroupSnapshotNamespaceMirror>(
+ &group_snap.snapshot_namespace);
+ ns->mirror_peer_uuids.erase(mirror_peer_uuid);
+
+ librados::ObjectWriteOperation op;
+ librbd::cls_client::group_snap_set(&op, group_snap);
+ int r = m_group_io_ctx.aio_operate(
+ librbd::util::group_header_name(m_group_id), aio_comp, &op);
+ ceph_assert(r == 0);
+ aio_comp->release();
+}
+
+template <typename I>
+void GroupUnlinkPeerRequest<I>::handle_remove_peer_uuid(int r) {
+ ldout(m_cct, 10) << "r=" << r << dendl;
+
+ if (r < 0) {
+ lderr(m_cct) << "failed to remove group snapshot mirror peer: "
+ << cpp_strerror(r) << dendl;
+ finish(r);
+ return;
+ }
+ list_group_snaps();
+}
+
+
+template <typename I>
+void GroupUnlinkPeerRequest<I>::remove_group_snapshot(
+ cls::rbd::GroupSnapshot group_snap) {
+ ldout(m_cct, 10) << "group snap id: " << group_snap.id << dendl;
+
+ //TODO: Handle dynamic group membership
+ if (!m_image_ctxs->empty() && m_image_ctx_map.empty()) {
+ for (size_t i = 0; i < m_image_ctxs->size(); ++i) {
+ ImageCtx *ictx = (*m_image_ctxs)[i];
+ m_image_ctx_map[ictx->id] = ictx;
+ }
+ }
+
+ auto ctx = create_context_callback<
+ GroupUnlinkPeerRequest,
+ &GroupUnlinkPeerRequest<I>::handle_remove_group_snapshot>(this);
+
+ m_group_snap_id = group_snap.id;
+
+ C_Gather *gather_ctx = new C_Gather(g_ceph_context, ctx);
+ for (auto &snap : group_snap.snaps) {
+ if (snap.snap_id == CEPH_NOSNAP) {
+ continue;
+ }
+ ImageCtx *ictx = nullptr;
+ if (m_image_ctx_map.find(snap.image_id) != m_image_ctx_map.end()) {
+ ictx = m_image_ctx_map[snap.image_id];
+ }
+
+ if (!ictx) {
+ ldout(m_cct, 10) << "failed to remove individual snapshot: " << dendl;
+ continue;
+ }
+
+ ldout(m_cct, 10) << "removing individual snapshot: "
+ << snap.snap_id << ", from image id:"
+ << snap.image_id << dendl;
+ remove_image_snapshot(ictx, snap.snap_id, gather_ctx);
+ }
+
+ gather_ctx->activate();
+
+}
+
+template <typename I>
+void GroupUnlinkPeerRequest<I>::handle_remove_group_snapshot(int r) {
+ ldout(m_cct, 10) << "r=" << r << dendl;
+
+ if (r < 0) {
+ lderr(m_cct) << "failed to remove image snapshot metadata: "
+ << cpp_strerror(r) << dendl;
+ finish(r);
+ return;
+ }
+
+ remove_snap_metadata();
+}
+
+template <typename I>
+void GroupUnlinkPeerRequest<I>::remove_snap_metadata() {
+ ldout(m_cct, 10) << dendl;
+
+ librados::ObjectWriteOperation op;
+ cls_client::group_snap_remove(&op,m_group_snap_id);
+
+ auto aio_comp = create_rados_callback<
+ GroupUnlinkPeerRequest<I>,
+ &GroupUnlinkPeerRequest<I>::handle_remove_snap_metadata>(this);
+ int r = m_group_io_ctx.aio_operate(librbd::util::group_header_name(m_group_id),
+ aio_comp, &op);
+ ceph_assert(r == 0);
+ aio_comp->release();
+}
+
+template <typename I>
+void GroupUnlinkPeerRequest<I>::handle_remove_snap_metadata(int r) {
+ ldout(m_cct, 10) << "r=" << r << dendl;
+
+ if (r < 0) {
+ lderr(m_cct) << "failed to remove group snapshot metadata: "
+ << cpp_strerror(r) << dendl;
+ finish(r);
+ return;
+ }
+ list_group_snaps();
+}
+
+template <typename I>
+void GroupUnlinkPeerRequest<I>::remove_image_snapshot(ImageCtx *image_ctx,
+ uint64_t snap_id,
+ C_Gather *gather_ctx) {
+ ldout(m_cct, 10) << snap_id << dendl;
+
+ int r = -ENOENT;
+ cls::rbd::SnapshotNamespace snap_namespace;
+ std::string snap_name;
+
+ image_ctx->image_lock.lock_shared();
+ auto snap_it = image_ctx->snap_info.find(snap_id);
+ if (snap_it != image_ctx->snap_info.end()) {
+ r = 0;
+ snap_namespace = snap_it->second.snap_namespace;
+ snap_name = snap_it->second.name;
+ }
+
+ image_ctx->image_lock.unlock_shared();
+ if (r == -ENOENT) {
+ ldout(m_cct, 10) << "missing snapshot: snap_id=" << snap_id << dendl;
+ return;
+ }
+ auto mirror_ns = std::get_if<cls::rbd::MirrorSnapshotNamespace>(
+ &snap_namespace);
+ if (mirror_ns == nullptr) {
+ lderr(m_cct) << "not mirror snapshot (snap_id=" << snap_id << ")" << dendl;
+ return;
+ }
+ image_ctx->operations->snap_remove(snap_namespace, snap_name.c_str(),
+ gather_ctx->new_sub());
+}
+
+template <typename I>
+void GroupUnlinkPeerRequest<I>::finish(int r) {
+ ldout(m_cct, 10) << "r=" << r << dendl;
+
+ m_on_finish->complete(r);
+}
+
+} // namespace snapshot
+} // namespace mirror
+} // namespace librbd
+
+template class librbd::mirror::snapshot::GroupUnlinkPeerRequest<librbd::ImageCtx>;
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CEPH_LIBRBD_MIRROR_SNAPSHOT_GROUP_UNLINK_PEER_REQUEST_H
+#define CEPH_LIBRBD_MIRROR_SNAPSHOT_GROUP_UNLINK_PEER_REQUEST_H
+
+#include "include/int_types.h"
+#include "include/types.h"
+#include "include/rados/librados.hpp"
+#include "cls/rbd/cls_rbd_types.h"
+
+#include <string>
+#include <vector>
+
+class Context;
+
+namespace librbd {
+
+struct ImageCtx;
+
+namespace mirror {
+namespace snapshot {
+
+template <typename ImageCtxT = librbd::ImageCtx>
+class GroupUnlinkPeerRequest {
+public:
+ static GroupUnlinkPeerRequest *create(
+ librados::IoCtx &group_io_ctx, const std::string &group_id,
+ std::set<std::string> *mirror_peer_uuids,
+ std::vector<ImageCtx *> *image_ctxs,
+ Context *on_finish) {
+ return new GroupUnlinkPeerRequest(group_io_ctx, group_id,
+ mirror_peer_uuids, image_ctxs,
+ on_finish);
+ }
+
+ GroupUnlinkPeerRequest(librados::IoCtx &group_io_ctx,
+ const std::string &group_id,
+ std::set<std::string> *mirror_peer_uuids,
+ std::vector<ImageCtx *> *image_ctxs,
+ Context *on_finish)
+ : m_group_io_ctx(group_io_ctx), m_group_id(group_id),
+ m_mirror_peer_uuids(mirror_peer_uuids), m_image_ctxs(image_ctxs),
+ m_on_finish(on_finish) {
+ m_cct = (CephContext *)group_io_ctx.cct();
+ }
+
+ void send();
+
+private:
+ librados::IoCtx &m_group_io_ctx;
+ const std::string m_group_id;
+ std::set<std::string> *m_mirror_peer_uuids;
+ std::vector<ImageCtx *> *m_image_ctxs;
+ Context *m_on_finish;
+
+ uint64_t m_max_snaps;
+ CephContext *m_cct;
+
+ std::vector<cls::rbd::GroupSnapshot> m_group_snaps;
+ std::map<std::string, ImageCtx *> m_image_ctx_map;
+ std::string m_group_snap_id;
+
+ void unlink_peer();
+
+ void list_group_snaps();
+ void handle_list_group_snaps(int r);
+
+ void process_snapshot(cls::rbd::GroupSnapshot group_snap,
+ std::string mirror_peer_uuid);
+
+ void remove_peer_uuid(cls::rbd::GroupSnapshot group_snap,
+ std::string mirror_peer_uuid);
+ void handle_remove_peer_uuid(int r);
+
+ void remove_group_snapshot(cls::rbd::GroupSnapshot group_snap);
+ void handle_remove_group_snapshot(int r);
+
+ void remove_snap_metadata();
+ void handle_remove_snap_metadata(int r);
+
+ void remove_image_snapshot(ImageCtx *image_ctx, uint64_t snap_id,
+ C_Gather *ctx);
+ void finish(int r);
+};
+
+} // namespace snapshot
+} // namespace mirror
+} // namespace librbd
+
+extern template class librbd::mirror::snapshot::GroupUnlinkPeerRequest<librbd::ImageCtx>;
+
+#endif // CEPH_LIBRBD_MIRROR_SNAPSHOT_GROUP_UNLINK_PEER_REQUEST_H
projects / ceph.git / commitdiff