};
r = image::snapshot::iterate(hctx, pre_check_lambda);
- if (r < 0) {
+ if (r < 0 && r != -EEXIST) {
return r;
}
image_ctxs[i]->id, snap_ids[i]);
}
+ auto it = std::find_if(
+ group_snap.snaps.begin(), group_snap.snaps.end(),
+ [image_id](const cls::rbd::ImageSnapshotSpec &s) {
+ return image_id == s.image_id;
+ });
+ if (it != group_snap.snaps.end()) {
+ group_snap.snaps.erase(it);
+ }
+
std::string group_header_oid = librbd::util::group_header_name(group_id);
if (ret_code < 0) {
// undo
CreateNonPrimaryRequest<I>::CreateNonPrimaryRequest(
I* image_ctx, bool demoted, const std::string &primary_mirror_uuid,
uint64_t primary_snap_id, const SnapSeqs& snap_seqs,
- int64_t group_pool_id, const std::string &group_id,
- const std::string &group_snap_id, const ImageState &image_state,
- uint64_t *snap_id, Context *on_finish)
+ const ImageState &image_state, uint64_t *snap_id, Context *on_finish)
: m_image_ctx(image_ctx), m_demoted(demoted),
m_primary_mirror_uuid(primary_mirror_uuid),
m_primary_snap_id(primary_snap_id), m_snap_seqs(snap_seqs),
- m_group_pool_id(group_pool_id), m_group_id(group_id),
- m_group_snap_id(group_snap_id), m_image_state(image_state),
- m_snap_id(snap_id), m_on_finish(on_finish) {
+ m_image_state(image_state), m_snap_id(snap_id), m_on_finish(on_finish) {
m_default_ns_ctx.dup(m_image_ctx->md_ctx);
m_default_ns_ctx.set_namespace("");
}
ns.mirror_peer_uuids = m_mirror_peer_uuids;
}
ns.snap_seqs = m_snap_seqs;
- ns.group_spec = {m_group_id, m_group_pool_id};
- ns.group_snap_id = m_group_snap_id;
ns.complete = is_orphan();
ldout(cct, 15) << "ns=" << ns << dendl;
const std::string &primary_mirror_uuid,
uint64_t primary_snap_id,
const SnapSeqs& snap_seqs,
- int64_t group_pool_id,
- const std::string &group_id,
- const std::string &group_snap_id,
const ImageState &image_state,
uint64_t *snap_id,
Context *on_finish) {
return new CreateNonPrimaryRequest(image_ctx, demoted, primary_mirror_uuid,
primary_snap_id, snap_seqs,
- group_pool_id, group_id, group_snap_id,
image_state, snap_id, on_finish);
}
const std::string &primary_mirror_uuid,
uint64_t primary_snap_id,
const SnapSeqs& snap_seqs,
- int64_t group_pool_id,
- const std::string &group_id,
- const std::string &group_snap_id,
const ImageState &image_state, uint64_t *snap_id,
Context *on_finish);
const std::string m_primary_mirror_uuid;
const uint64_t m_primary_snap_id;
const SnapSeqs m_snap_seqs;
- const int64_t m_group_pool_id;
- const std::string m_group_id;
- const std::string m_group_snap_id;
const ImageState m_image_state;
uint64_t *m_snap_id;
Context *m_on_finish;
std::string peer_uuid;
uint64_t snap_id = CEPH_NOSNAP;
- CephContext *cct = m_image_ctx->cct;
{
std::shared_lock image_locker{m_image_ctx->image_lock};
if (info->mirror_peer_uuids.empty() ||
(info->mirror_peer_uuids.count(peer) != 0 &&
info->is_primary() && !info->complete)) {
+ if (info->group_spec.is_valid() || !info->group_snap_id.empty()) {
+ // snap is part of a group snap
+ continue;
+ }
peer_uuid = peer;
snap_id = snap_info_pair.first;
goto do_unlink;
for (const auto& peer : m_mirror_peer_uuids) {
size_t count = 0;
uint64_t unlink_snap_id = 0;
- uint64_t prev_snap_id = 0;
- std::string prev_group_snap_id;
for (const auto& snap_info_pair : m_image_ctx->snap_info) {
auto info = std::get_if<cls::rbd::MirrorSnapshotNamespace>(
&snap_info_pair.second.snap_namespace);
// promotion
count = 0;
unlink_snap_id = 0;
- prev_snap_id = 0;
continue;
}
if (info->mirror_peer_uuids.count(peer) == 0) {
// snapshot is not linked with this peer
continue;
}
- if (prev_snap_id) {
- librados::IoCtx m_group_io_ctx;
- int r = librbd::util::create_ioctx(m_image_ctx->md_ctx,
- "group", info->group_spec.pool_id,
- {}, &m_group_io_ctx);
- if (r < 0) {
- return;
- }
- cls::rbd::GroupSnapshot prev_group_snap;
- std::string group_header_oid = librbd::util::group_header_name(
- info->group_spec.group_id);
- r = cls_client::group_snap_get_by_id(&m_group_io_ctx,
- group_header_oid,
- prev_group_snap_id,
- &prev_group_snap);
- if (r < 0) {
- lderr(cct) << "failed to retrieve group snapshot: "
- << cpp_strerror(r) << dendl;
- prev_snap_id = snap_info_pair.first;
- prev_group_snap_id = info->group_snap_id;
- continue;
- }
- if (prev_group_snap.state != cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE) {
- peer_uuid = peer;
- snap_id = prev_snap_id;
- r = cls_client::group_snap_remove(&m_group_io_ctx,
- group_header_oid,
- prev_group_snap.id);
- if (r < 0) {
- lderr(cct) << "failed to remove group snapshot metadata: "
- << cpp_strerror(r) << dendl;
- }
- goto do_unlink;
- }
+ if (info->group_spec.is_valid() || !info->group_snap_id.empty()) {
+ // snap is part of a group snap
+ continue;
}
count++;
if (count == max_snapshots) {
snap_id = unlink_snap_id;
goto do_unlink;
}
- prev_snap_id = 0;
- if (info->group_spec.is_valid() && !info->group_snap_id.empty()) {
- prev_snap_id = snap_info_pair.first;
- prev_group_snap_id = info->group_snap_id;
- }
}
}
}
return;
do_unlink:
+ CephContext *cct = m_image_ctx->cct;
ldout(cct, 15) << "peer=" << peer_uuid << ", snap_id=" << snap_id << dendl;
auto ctx = create_context_callback<
&PromoteRequest<I>::handle_create_orphan_snapshot>(this);
auto req = CreateNonPrimaryRequest<I>::create(
- m_image_ctx, false, "", CEPH_NOSNAP, {}, -1, {}, {}, {}, nullptr, ctx);
+ m_image_ctx, false, "", CEPH_NOSNAP, {}, {}, nullptr, ctx);
req->send();
}
C_SaferCond ctx;
auto req = new MockCreateNonPrimaryRequest(&mock_image_ctx, false,
- "mirror_uuid", 123, {{1, 2}}, -1, {},
- {}, {}, nullptr, &ctx);
+ "mirror_uuid", 123, {{1, 2}}, {},
+ nullptr, &ctx);
req->send();
ASSERT_EQ(0, ctx.wait());
}
C_SaferCond ctx;
auto req = new MockCreateNonPrimaryRequest(&mock_image_ctx, true,
- "mirror_uuid", 123, {{1, 2}}, -1,
- {}, {}, {}, nullptr, &ctx);
+ "mirror_uuid", 123, {{1, 2}}, {},
+ nullptr, &ctx);
req->send();
ASSERT_EQ(0, ctx.wait());
}
C_SaferCond ctx;
auto req = new MockCreateNonPrimaryRequest(&mock_image_ctx, false,
- "mirror_uuid", 123, {{1, 2}}, -1,
- {}, {}, {}, nullptr, &ctx);
+ "mirror_uuid", 123, {{1, 2}}, {},
+ nullptr, &ctx);
req->send();
ASSERT_EQ(-EINVAL, ctx.wait());
}
C_SaferCond ctx;
auto req = new MockCreateNonPrimaryRequest(&mock_image_ctx, false,
- "mirror_uuid", 123, {{1, 2}}, -1,
- {}, {}, {}, nullptr, &ctx);
+ "mirror_uuid", 123, {{1, 2}}, {},
+ nullptr, &ctx);
req->send();
ASSERT_EQ(-EINVAL, ctx.wait());
}
C_SaferCond ctx;
auto req = new MockCreateNonPrimaryRequest(&mock_image_ctx, false,
- "mirror_uuid", 123, {{1, 2}}, -1,
- {}, {}, {}, nullptr, &ctx);
+ "mirror_uuid", 123, {{1, 2}}, {},
+ nullptr, &ctx);
req->send();
ASSERT_EQ(-EINVAL, ctx.wait());
}
C_SaferCond ctx;
auto req = new MockCreateNonPrimaryRequest(&mock_image_ctx, true,
- "mirror_uuid", 123, {{1, 2}}, -1,
- {}, {}, {}, nullptr, &ctx);
+ "mirror_uuid", 123, {{1, 2}}, {},
+ nullptr, &ctx);
req->send();
ASSERT_EQ(-EPERM, ctx.wait());
}
C_SaferCond ctx;
auto req = new MockCreateNonPrimaryRequest(&mock_image_ctx, false,
- "mirror_uuid", 123, {{1, 2}}, -1,
- {}, {}, {}, nullptr, &ctx);
+ "mirror_uuid", 123, {{1, 2}}, {},
+ nullptr, &ctx);
req->send();
ASSERT_EQ(-EINVAL, ctx.wait());
}
C_SaferCond ctx;
auto req = new MockCreateNonPrimaryRequest(&mock_image_ctx, false,
- "mirror_uuid", 123, {{1, 2}}, -1,
- {}, {}, {}, nullptr, &ctx);
+ "mirror_uuid", 123, {{1, 2}}, {},
+ nullptr, &ctx);
req->send();
ASSERT_EQ(-EINVAL, ctx.wait());
}
const std::string &primary_mirror_uuid,
uint64_t primary_snap_id,
SnapSeqs snap_seqs,
- int64_t group_pool_id,
- const std::string &group_id,
- const std::string &group_snap_id,
const ImageState &image_state,
uint64_t *snap_id,
Context *on_finish) {
struct MockReplayerListener : public image_replayer::ReplayerListener {
MOCK_METHOD0(handle_notification, void());
- MOCK_METHOD5(create_mirror_snapshot_start,
- void(const cls::rbd::MirrorSnapshotNamespace &,
- int64_t *, std::string *, std::string *, Context *));
- MOCK_METHOD3(create_mirror_snapshot_finish, void(const std::string &,
- uint64_t, Context *));
};
} // anonymous namespace
struct MockReplayerListener : public image_replayer::ReplayerListener {
MOCK_METHOD0(handle_notification, void());
- MOCK_METHOD5(create_mirror_snapshot_start,
- void(const cls::rbd::MirrorSnapshotNamespace &,
- int64_t *, std::string *, std::string *, Context *));
- MOCK_METHOD3(create_mirror_snapshot_finish, void(const std::string &,
- uint64_t, Context *));
};
} // anonymous namespace
}));
}
- void expect_unlink_group_snapshot(librbd::MockTestImageCtx& mock_image_ctx,
- uint64_t snap_id) {
- EXPECT_CALL(mock_image_ctx, get_snap_info(snap_id))
- .WillOnce(Invoke([&mock_image_ctx](uint64_t snap_id) -> librbd::SnapInfo* {
- auto it = mock_image_ctx.snap_info.find(snap_id);
- if (it == mock_image_ctx.snap_info.end()) {
- return nullptr;
- }
- return &it->second;
- }));
- }
-
void expect_prune_non_primary_snapshot(librbd::MockTestImageCtx& mock_image_ctx,
uint64_t snap_id, int r) {
EXPECT_CALL(mock_image_ctx, get_snap_info(snap_id))
"", CEPH_NOSNAP, true, 0, {}},
0, {}, 0, 0, {}}}
}, 0);
- expect_unlink_group_snapshot(mock_local_image_ctx, 11);
expect_prune_non_primary_snapshot(mock_local_image_ctx, 11, 0);
// idle
Throttler.cc
Types.cc
group_replayer/BootstrapRequest.cc
+ group_replayer/Replayer.cc
image_deleter/SnapshotPurgeRequest.cc
image_deleter/TrashMoveRequest.cc
image_deleter/TrashRemoveRequest.cc
#include "tools/rbd_mirror/MirrorStatusUpdater.h"
#include "tools/rbd_mirror/Threads.h"
#include "tools/rbd_mirror/group_replayer/BootstrapRequest.h"
+#include "tools/rbd_mirror/group_replayer/Replayer.h"
#include "tools/rbd_mirror/image_replayer/Utils.h"
#include "GroupReplayer.h"
return;
}
+ C_SaferCond ctx;
+ create_group_replayer(&ctx);
+ ctx.wait();
+}
+
+template <typename I>
+void GroupReplayer<I>::create_group_replayer(Context *on_finish) {
+ dout(10) << dendl;
+
+ auto ctx = new LambdaContext(
+ [this, on_finish](int r) {
+ handle_create_group_replayer(r, on_finish);
+ });
+
+ m_replayer = group_replayer::Replayer<I>::create(
+ m_threads, m_local_io_ctx, m_remote_group_peer.io_ctx, m_global_group_id,
+ m_local_mirror_uuid, m_remote_group_peer.uuid, m_pool_meta_cache,
+ m_local_group_id, m_remote_group_id, &m_image_replayers);
+
+ m_replayer->init(ctx);
+}
+
+template <typename I>
+void GroupReplayer<I>::handle_create_group_replayer(int r, Context *on_finish) {
+ dout(10) << "r=" << r << dendl;
+
+ if (m_state == STATE_STOPPING || m_state == STATE_STOPPED) {
+ dout(10) << "stop prevailed" <<dendl;
+ on_finish->complete(r);
+ return;
+ }
+ on_finish->complete(0);
start_image_replayers();
}
cls::rbd::MIRROR_GROUP_STATUS_STATE_STARTING_REPLAY, "starting replay");
auto ctx = create_context_callback<
- GroupReplayer, &GroupReplayer<I>::handle_start_image_replayers>(this);
+ GroupReplayer, &GroupReplayer<I>::handle_start_image_replayers>(this);
C_Gather *gather_ctx = new C_Gather(g_ceph_context, ctx);
{
std::lock_guard locker{m_lock};
finish_start(0, "");
}
+template <typename I>
+void GroupReplayer<I>::stop_group_replayer(Context *on_finish) {
+ dout(10) << dendl;
+
+ Context *ctx = new LambdaContext(
+ [this, on_finish](int r) {
+ handle_stop_group_replayer(r, on_finish);
+ });
+
+ if (m_replayer != nullptr) {
+ m_replayer->shut_down(ctx);
+ return;
+ }
+ on_finish->complete(0);
+}
+
+template <typename I>
+void GroupReplayer<I>::handle_stop_group_replayer(int r, Context *on_finish) {
+ dout(10) << "r=" << r << dendl;
+
+ if (on_finish != nullptr) {
+ on_finish->complete(0);
+ }
+}
+
template <typename I>
void GroupReplayer<I>::stop_image_replayers() {
dout(10) << m_image_replayers.size() << dendl;
cls::rbd::MIRROR_GROUP_STATUS_STATE_STOPPING_REPLAY, "stopping");
auto ctx = create_context_callback<
- GroupReplayer, &GroupReplayer<I>::handle_stop_image_replayers>(this);
+ GroupReplayer, &GroupReplayer<I>::handle_stop_image_replayers>(this);
+
C_Gather *gather_ctx = new C_Gather(g_ceph_context, ctx);
{
std::lock_guard locker{m_lock};
set_mirror_group_status_update(
cls::rbd::MIRROR_GROUP_STATUS_STATE_STOPPED, "stopped");
- if (on_finish != nullptr) {
- on_finish->complete(0);
- }
+ stop_group_replayer(on_finish);
}
template <typename I>
}
}
-template <typename I>
-void GroupReplayer<I>::create_regular_group_snapshot(
- const std::string &remote_group_snap_name,
- const std::string &remote_group_snap_id,
- std::vector<cls::rbd::GroupImageStatus> *local_images,
- Context *on_finish) {
- // each image will have one snapshot specific to group snap, and so for each
- // image get a ImageSnapshotSpec and prepare a vector
- // for image :: <images in that group> {
- // * get snap whos name has group snap_id for that we can list snaps and
- // filter with remote_group_snap_id
- // * get its { pool_id, snap_id, image_id }
- // }
- // finally write to the object
- dout(10) << dendl;
- librados::ObjectWriteOperation op;
- cls::rbd::GroupSnapshot group_snap{
- remote_group_snap_id, // keeping it same as remote group snap id
- cls::rbd::UserGroupSnapshotNamespace{},
- remote_group_snap_name,
- cls::rbd::GROUP_SNAPSHOT_STATE_INCOMPLETE};
- librbd::cls_client::group_snap_set(&op, group_snap);
- if (m_local_group_snaps.find(group_snap.id) == m_local_group_snaps.end()) {
- m_local_group_snaps.insert(make_pair(group_snap.id, group_snap));
- }
-
- std::vector<cls::rbd::ImageSnapshotSpec> local_image_snap_specs;
- local_image_snap_specs = std::vector<cls::rbd::ImageSnapshotSpec>(
- local_images->size(), cls::rbd::ImageSnapshotSpec());
- for (auto& image : *local_images) {
- std::string image_header_oid = librbd::util::header_name(
- image.spec.image_id);
- ::SnapContext snapc;
- int r = librbd::cls_client::get_snapcontext(&m_local_io_ctx,
- image_header_oid, &snapc);
- if (r < 0) {
- derr << "get snap context failed: " << cpp_strerror(r) << dendl;
- on_finish->complete(r);
- return;
- }
-
- auto image_snap_name = ".group." + std::to_string(image.spec.pool_id) +
- "_" + m_remote_group_id + "_" + remote_group_snap_id;
- // stored in reverse order
- for (auto snap_id : snapc.snaps) {
- cls::rbd::SnapshotInfo snap_info;
- r = librbd::cls_client::snapshot_get(&m_local_io_ctx, image_header_oid,
- snap_id, &snap_info);
- if (r < 0) {
- derr << "failed getting snap info for snap id: " << snap_id
- << ", : " << cpp_strerror(r) << dendl;
- on_finish->complete(r);
- return;
- }
-
- // extract { pool_id, snap_id, image_id }
- if (snap_info.name == image_snap_name) {
- cls::rbd::ImageSnapshotSpec snap_spec;
- snap_spec.pool = image.spec.pool_id;
- snap_spec.image_id = image.spec.image_id;
- snap_spec.snap_id = snap_info.id;
-
- local_image_snap_specs.push_back(snap_spec);
- }
- }
- }
-
- group_snap.snaps = local_image_snap_specs;
- group_snap.state = cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE;
- librbd::cls_client::group_snap_set(&op, group_snap);
- m_local_group_snaps[group_snap.id] = group_snap;
-
- auto comp = create_rados_callback(
- new LambdaContext([this, on_finish](int r) {
- handle_create_regular_group_snapshot(r, on_finish);
- }));
- int r = m_local_io_ctx.aio_operate(
- librbd::util::group_header_name(m_local_group_ctx.group_id), comp, &op);
- ceph_assert(r == 0);
- comp->release();
-}
-
-template <typename I>
-void GroupReplayer<I>::handle_create_regular_group_snapshot(
- int r, Context *on_finish) {
- dout(10) << "r=" << r << dendl;
-
- if (r < 0) {
- derr << "error creating local non-primary group snapshot: "
- << cpp_strerror(r) << dendl;
- }
-
- on_finish->complete(0);
-}
-
-template <typename I>
-void GroupReplayer<I>::list_remote_group_snapshots(Context *on_finish) {
- dout(10) << dendl;
-
- remote_group_snaps.clear();
- auto ctx = new LambdaContext(
- [this, on_finish] (int r) {
- handle_list_remote_group_snapshots(r, on_finish);
- });
-
- auto req = librbd::group::ListSnapshotsRequest<I>::create(
- m_remote_group_peer.io_ctx, m_remote_group_id, &remote_group_snaps, ctx);
- req->send();
-}
-
-template <typename I>
-void GroupReplayer<I>::handle_list_remote_group_snapshots(int r,
- Context *on_finish) {
- dout(10) << "r=" << r << dendl;
- std::unique_lock locker{m_lock};
-
- if (r < 0) {
- derr << "error listing remote mirror group snapshots: " << cpp_strerror(r)
- << dendl;
- on_finish->complete(r);
- return;
- }
-
- m_remote_group_snaps.clear();
- for (auto it : remote_group_snaps) {
- dout(10) << "found remote group snap id: " << it.id << dendl;
- m_remote_group_snaps.insert(make_pair(it.id, it));
- }
-
- std::vector<cls::rbd::GroupImageStatus> local_images;
- std::vector<C_SaferCond*> on_finishes;
- for (auto it = m_remote_group_snaps.begin(); it != m_remote_group_snaps.end(); ++it) {
- auto snap_type = cls::rbd::get_group_snap_namespace_type(
- it->second.snapshot_namespace);
- if (snap_type == cls::rbd::GROUP_SNAPSHOT_NAMESPACE_TYPE_USER) {
- dout(10) << "found user snap, snap name: " << it->second.name
- << ", remote group snap id: " << it->second.id << dendl;
- if (local_images.empty()) {
- r = local_group_image_list_by_id(&local_images);
- if (r < 0) {
- locker.unlock();
- on_finish->complete(r);
- return;
- }
- }
- if (m_local_group_snaps.find(it->second.id) == m_local_group_snaps.end()) {
- C_SaferCond* ctx = new C_SaferCond;
- create_regular_group_snapshot(it->second.name,
- it->second.id, &local_images, ctx);
- on_finishes.push_back(ctx);
- }
- }
- }
-
- for (auto &finish : on_finishes) {
- finish->wait();
- }
-
- locker.unlock();
- on_finish->complete(0);
-}
-
-template <typename I>
-int GroupReplayer<I>::local_group_image_list_by_id(
- std::vector<cls::rbd::GroupImageStatus> *image_ids) {
- std::string group_header_oid = librbd::util::group_header_name(
- m_local_group_ctx.group_id);
-
- dout(10) << "listing images in local group id " << group_header_oid << dendl;
- image_ids->clear();
-
- int r = 0;
- const int max_read = 1024;
- cls::rbd::GroupImageSpec start_last;
- do {
- std::vector<cls::rbd::GroupImageStatus> image_ids_page;
-
- r = librbd::cls_client::group_image_list(&m_local_io_ctx, group_header_oid,
- start_last, max_read,
- &image_ids_page);
-
- if (r < 0) {
- derr << "error reading image list from local group: "
- << cpp_strerror(-r) << dendl;
- return r;
- }
- image_ids->insert(image_ids->end(), image_ids_page.begin(),
- image_ids_page.end());
-
- if (image_ids_page.size() > 0)
- start_last = image_ids_page.rbegin()->spec;
-
- r = image_ids_page.size();
- } while (r == max_read);
-
- return 0;
-}
-
-template <typename I>
-void GroupReplayer<I>::create_mirror_snapshot_start(
- const cls::rbd::MirrorSnapshotNamespace &remote_group_snap_ns,
- ImageReplayer<I> *image_replayer, int64_t *local_group_pool_id,
- std::string *local_group_id, std::string *local_group_snap_id,
- Context *on_finish) {
- dout(20) << remote_group_snap_ns << " " << image_replayer << dendl;
-
- ceph_assert(remote_group_snap_ns.is_primary());
-
- int r = 0;
- std::unique_lock locker{m_lock};
- auto &remote_group_snap_id = remote_group_snap_ns.group_snap_id;
- if (m_local_group_snaps.find(remote_group_snap_id) != m_local_group_snaps.end() &&
- m_local_group_snaps[remote_group_snap_id].state == cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE) {
- dout(20) << "group snapshot: " << remote_group_snap_id << " already exists"
- << dendl;
- r = -EEXIST;
- } else if (m_state == STATE_STARTING) {
- derr << "group replayer is not ready yet, m_state: " << m_state << dendl;
- r = -EAGAIN;
- } else if (m_state != STATE_REPLAYING) {
- derr << "group replayer is not in replaying state, m_state: "
- << m_state << dendl;
- r = -ESTALE;
- }
-
- if (r != 0) {
- locker.unlock();
- on_finish->complete(r);
- return;
- }
-
- auto requests_it = m_create_snap_requests.find(remote_group_snap_id);
-
- if (requests_it == m_create_snap_requests.end()) {
- requests_it = m_create_snap_requests.insert(
- {remote_group_snap_id, {}}).first;
-
- auto snap_state =
- remote_group_snap_ns.state == cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY ?
- cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY :
- cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY_DEMOTED;
-
- m_local_group_snaps[remote_group_snap_id] =
- {remote_group_snap_id,
- cls::rbd::MirrorGroupSnapshotNamespace{
- snap_state, {}, m_remote_group_peer.uuid, remote_group_snap_id},
- {}, cls::rbd::GROUP_SNAPSHOT_STATE_INCOMPLETE};
-
- librados::ObjectReadOperation op;
- librbd::cls_client::group_snap_get_by_id_start(&op, remote_group_snap_id);
- auto ctx = new C_GetRemoteGroupSnap(this, remote_group_snap_id);
- auto comp = create_rados_callback(ctx);
-
- r = m_remote_group_peer.io_ctx.aio_operate(
- librbd::util::group_header_name(m_remote_group_id), comp, &op,
- &ctx->bl);
- ceph_assert(r == 0);
- comp->release();
- }
-
- ceph_assert(requests_it->second.count(image_replayer) == 0);
- requests_it->second[image_replayer] = on_finish;
-
- ceph_assert(m_local_group_snaps.count(remote_group_snap_id) > 0);
- auto &local_group_snap = m_local_group_snaps[remote_group_snap_id];
-
- local_group_snap.snaps.emplace_back(image_replayer->get_local_pool_id(),
- image_replayer->get_local_image_id(),
- CEPH_NOSNAP);
- ceph_assert(!local_group_snap.snaps.back().image_id.empty());
-
- *local_group_pool_id = m_local_io_ctx.get_id();
- *local_group_id = m_local_group_ctx.group_id;
- *local_group_snap_id = m_local_group_snaps[remote_group_snap_id].id;
-
- if (m_get_remote_group_snap_ret_vals.count(remote_group_snap_id) == 0) {
- dout(20) << "getting remote group snap is still in-progress" << dendl;
- locker.unlock();
- return;
- }
-
- maybe_create_mirror_snapshot(locker, remote_group_snap_id);
-}
-
-template <typename I>
-void GroupReplayer<I>::maybe_create_mirror_snapshot(
- std::unique_lock<ceph::mutex>& locker,
- const std::string &remote_group_snap_id) {
- ceph_assert(ceph_mutex_is_locked_by_me(m_lock));
-
- dout(20) << remote_group_snap_id << dendl;
-
- auto &remote_group_snap = m_remote_group_snaps[remote_group_snap_id];
- ceph_assert(!remote_group_snap.id.empty());
-
- ceph_assert(m_create_snap_requests.count(remote_group_snap_id) > 0);
- auto &create_snap_requests = m_create_snap_requests[remote_group_snap_id];
-
- for (auto &s : remote_group_snap.snaps) {
- auto it = m_image_replayer_index.find({s.pool, s.image_id});
- if (it == m_image_replayer_index.end()) {
- continue;
- }
-
- if (create_snap_requests.count(it->second) == 0) {
- dout(20) << "waiting for create_mirror_snapshot_start "
- << remote_group_snap_id << " from " << it->second << dendl;
- locker.unlock();
- return;
- }
- }
-
- ceph_assert(m_local_group_snaps.count(remote_group_snap_id) > 0);
- auto &local_group_snap = m_local_group_snaps[remote_group_snap_id];
-
- dout(20) << local_group_snap.id << " " << local_group_snap.name << dendl;
-
- bool inserted = m_pending_snap_create.insert(remote_group_snap_id).second;
- ceph_assert(inserted);
-
- ceph_assert(m_get_remote_group_snap_ret_vals.count(remote_group_snap_id) > 0);
- int r = m_get_remote_group_snap_ret_vals[remote_group_snap_id];
- m_get_remote_group_snap_ret_vals.erase(remote_group_snap_id);
- if (r < 0) {
- locker.unlock();
- handle_create_mirror_snapshot_start(remote_group_snap_id, r);
- return;
- }
-
- librados::ObjectWriteOperation op;
- librbd::cls_client::group_snap_set(&op, local_group_snap);
- auto comp = create_rados_callback(
- new LambdaContext([this, remote_group_snap_id](int r) {
- handle_create_mirror_snapshot_start(remote_group_snap_id, r);
- }));
-
- r = m_local_io_ctx.aio_operate(
- librbd::util::group_header_name(m_local_group_ctx.group_id), comp, &op);
- locker.unlock();
- ceph_assert(r == 0);
- comp->release();
-}
-
-template <typename I>
-void GroupReplayer<I>::handle_get_remote_group_snapshot(
- const std::string &remote_group_snap_id, bufferlist &out_bl, int r) {
- dout(20) << remote_group_snap_id << " r=" << r << dendl;
-
- auto &remote_group_snap = m_remote_group_snaps[remote_group_snap_id];
-
- std::unique_lock locker{m_lock};
- if (r == 0) {
- auto iter = out_bl.cbegin();
- r = librbd::cls_client::group_snap_get_by_id_finish(
- &iter, &remote_group_snap);
- }
-
- bool complete = (remote_group_snap.state == cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE);
- if (r < 0) {
- derr << "failed to get remote group snapshot: " << cpp_strerror(r) << dendl;
- } else if (!complete) {
- derr << "incomplete remote group snapshot: " << remote_group_snap_id
- << dendl;
- r = -EAGAIN;
- } else {
- m_local_group_snaps[remote_group_snap_id].name = m_bootstrap_request->prepare_non_primary_mirror_snap_name(
- m_global_group_id,
- m_local_group_snaps[remote_group_snap_id].id);
- }
-
- if (m_state == STATE_STOPPING) {
- dout(20) << "interrupted" << dendl;
- m_local_group_snaps.erase(remote_group_snap_id);
- auto create_snap_requests = m_create_snap_requests[remote_group_snap_id];
- m_create_snap_requests.erase(remote_group_snap_id);
- bool shut_down_replay = m_pending_snap_create.empty() &&
- m_create_snap_requests.empty();
- locker.unlock();
- for (auto &[_, on_finish] : create_snap_requests) {
- on_finish->complete(r);
- }
- if (shut_down_replay) {
- stop_image_replayers();
- }
- return;
- }
-
- m_get_remote_group_snap_ret_vals[remote_group_snap_id] = r;
- maybe_create_mirror_snapshot(locker, remote_group_snap_id);
-}
-
-template <typename I>
-void GroupReplayer<I>::handle_create_mirror_snapshot_start(
- const std::string &remote_group_snap_id, int r) {
- dout(20) << remote_group_snap_id << " r=" << r << dendl;
-
- std::unique_lock locker{m_lock};
- ceph_assert(m_pending_snap_create.count(remote_group_snap_id) > 0);
-
- ceph_assert(m_create_snap_requests.count(remote_group_snap_id) > 0);
- auto create_snap_requests = m_create_snap_requests[remote_group_snap_id];
- m_create_snap_requests.erase(remote_group_snap_id);
-
- bool shut_down_replay = false;
- if (r == -EEXIST) {
- dout(20) << "group snapshot: " << remote_group_snap_id << " already exists"
- << dendl;
- r = 0;
- } else if (r < 0) {
- m_pending_snap_create.erase(remote_group_snap_id);
- m_local_group_snaps.erase(remote_group_snap_id);
- shut_down_replay = m_state == STATE_STOPPING && !m_restart_requested &&
- m_pending_snap_create.empty() &&
- m_create_snap_requests.empty();
- }
- locker.unlock();
-
- for (auto &[_, on_finish] : create_snap_requests) {
- on_finish->complete(r);
- }
- if (shut_down_replay) {
- stop_image_replayers();
- }
-}
-
-template <typename I>
-void GroupReplayer<I>::create_mirror_snapshot_finish(
- const std::string &remote_group_snap_id, ImageReplayer<I> *image_replayer,
- uint64_t snap_id, Context *on_finish) {
- dout(20) << remote_group_snap_id << " " << image_replayer << " snap_id="
- << snap_id << dendl;
-
- std::lock_guard locker{m_lock};
- if (m_local_group_snaps.find(remote_group_snap_id) != m_local_group_snaps.end() &&
- m_local_group_snaps[remote_group_snap_id].state == cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE) {
- on_finish->complete(-EEXIST);
- return;
- }
-
- ceph_assert(m_pending_snap_create.count(remote_group_snap_id) > 0);
-
- auto &create_snap_requests = m_create_snap_requests[remote_group_snap_id];
-
- ceph_assert(create_snap_requests.count(image_replayer) == 0);
- create_snap_requests[image_replayer] = on_finish;
-
- auto pool = image_replayer->get_local_pool_id();
- auto image_id = image_replayer->get_local_image_id();
- auto &local_group_snap = m_local_group_snaps[remote_group_snap_id];
- auto it = std::find_if(
- local_group_snap.snaps.begin(), local_group_snap.snaps.end(),
- [&pool, &image_id](const cls::rbd::ImageSnapshotSpec &s) {
- return pool == s.pool && image_id == s.image_id;
- });
- ceph_assert(it != local_group_snap.snaps.end());
- it->snap_id = snap_id;
-
- if (create_snap_requests.size() < local_group_snap.snaps.size()) {
- return;
- }
-
- local_group_snap.state = cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE;
-
- dout(20) << local_group_snap.id << " " << local_group_snap.name << dendl;
-
- librados::ObjectWriteOperation op;
- librbd::cls_client::group_snap_set(&op, local_group_snap);
- auto comp = create_rados_callback(
- new LambdaContext([this, remote_group_snap_id](int r) {
- handle_create_mirror_snapshot_finish(remote_group_snap_id, r);
- }));
-
- int r = m_local_io_ctx.aio_operate(
- librbd::util::group_header_name(m_local_group_ctx.group_id), comp, &op);
- ceph_assert(r == 0);
- comp->release();
-}
-
-template <typename I>
-void GroupReplayer<I>::handle_create_mirror_snapshot_finish(
- const std::string &remote_group_snap_id, int r) {
- dout(20) << remote_group_snap_id << " r=" << r << dendl;
-
- std::unique_lock locker{m_lock};
- auto count = m_pending_snap_create.erase(remote_group_snap_id);
- ceph_assert(count > 0);
-
- auto create_snap_requests = m_create_snap_requests[remote_group_snap_id];
- m_create_snap_requests.erase(remote_group_snap_id);
- bool shut_down_replay = m_state == STATE_STOPPING && !m_restart_requested &&
- m_pending_snap_create.empty() &&
- m_create_snap_requests.empty();
- locker.unlock();
-
- for (auto &[_, on_finish] : create_snap_requests) {
- on_finish->complete(r);
- }
-
- if (shut_down_replay) {
- stop_image_replayers();
- }
-}
-
} // namespace mirror
} // namespace rbd
#include "common/ceph_mutex.h"
#include "include/rados/librados.hpp"
#include "tools/rbd_mirror/Types.h"
+#include "tools/rbd_mirror/group_replayer/Replayer.h"
#include "tools/rbd_mirror/image_replayer/Types.h"
#include <boost/optional.hpp>
#include <string>
Listener(GroupReplayer *group_replayer) : group_replayer(group_replayer) {
}
- void list_remote_group_snapshots(Context *on_finish) override {
- group_replayer->list_remote_group_snapshots(on_finish);
- }
-
- void create_mirror_snapshot_start(
- const cls::rbd::MirrorSnapshotNamespace &remote_group_snap_ns,
- void *arg, int64_t *local_group_pool_id, std::string *local_group_id,
- std::string *local_group_snap_id, Context *on_finish) override {
- group_replayer->create_mirror_snapshot_start(
- remote_group_snap_ns, static_cast<ImageReplayer<ImageCtxT> *>(arg),
- local_group_pool_id, local_group_id, local_group_snap_id, on_finish);
- }
-
- void create_mirror_snapshot_finish(const std::string &remote_group_snap_id,
- void *arg, uint64_t snap_id,
- Context *on_finish) override {
- group_replayer->create_mirror_snapshot_finish(
- remote_group_snap_id, static_cast<ImageReplayer<ImageCtxT> *>(arg),
- snap_id, on_finish);
- }
- };
-
- struct C_GetRemoteGroupSnap : public Context {
- GroupReplayer *group_replayer;
- std::string group_snap_id;
- bufferlist bl;
-
- C_GetRemoteGroupSnap(GroupReplayer *group_replayer,
- const std::string &group_snap_id)
- : group_replayer(group_replayer), group_snap_id(group_snap_id) {
- }
-
- void finish(int r) override {
- group_replayer->handle_get_remote_group_snapshot(group_snap_id, bl, r);
+ void notify_group_snap_image_complete(
+ int64_t local_pool_id,
+ const std::string &local_image_id,
+ const std::string &remote_group_snap_id,
+ uint64_t local_snap_id) override {
+ group_replayer->m_replayer->notify_group_snap_image_complete(
+ local_pool_id, local_image_id, remote_group_snap_id, local_snap_id);
}
};
AdminSocketHook *m_asok_hook = nullptr;
group_replayer::BootstrapRequest<ImageCtxT> *m_bootstrap_request = nullptr;
+ group_replayer::Replayer<ImageCtxT> *m_replayer = nullptr;
std::list<std::pair<librados::IoCtx, ImageReplayer<ImageCtxT> *>> m_image_replayers;
Listener m_listener = {this};
void bootstrap_group();
void handle_bootstrap_group(int r);
+ void create_group_replayer(Context *on_finish);
+ void handle_create_group_replayer(int r, Context *on_finish);
+
void start_image_replayers();
void handle_start_image_replayers(int r);
bool finish_start_if_interrupted(ceph::mutex &lock);
void finish_start(int r, const std::string &desc);
+ void stop_group_replayer(Context *on_finish);
+ void handle_stop_group_replayer(int r, Context *on_finish);
+
void stop_image_replayers();
void handle_stop_image_replayers(int r);
void set_mirror_group_status_update(cls::rbd::MirrorGroupStatusState state,
const std::string &desc);
-
- void create_regular_group_snapshot(const std::string &remote_snap_name,
- const std::string &remote_snap_id,
- std::vector<cls::rbd::GroupImageStatus> *local_images,
- Context *on_finish);
- void handle_create_regular_group_snapshot(int r, Context *on_finish);
- void list_remote_group_snapshots(Context *on_finish);
- void handle_list_remote_group_snapshots(int r, Context *on_finish);
- int local_group_image_list_by_id(std::vector<cls::rbd::GroupImageStatus> *image_ids);
- void create_mirror_snapshot_start(
- const cls::rbd::MirrorSnapshotNamespace &remote_group_snap_ns,
- ImageReplayer<ImageCtxT> *image_replayer, int64_t *local_group_pool_id,
- std::string *local_group_id, std::string *local_group_snap_id,
- Context *on_finish);
- void handle_create_mirror_snapshot_start(
- const std::string &remote_group_snap_id, int r);
- void handle_get_remote_group_snapshot(
- const std::string &remote_group_snap_id, bufferlist &out_bl, int r);
- void maybe_create_mirror_snapshot(
- std::unique_lock<ceph::mutex>& locker,
- const std::string &remote_group_snap_id);
-
- void create_mirror_snapshot_finish(
- const std::string &remote_group_snap_id,
- ImageReplayer<ImageCtxT> *image_replayer,
- uint64_t snap_id, Context *on_finish);
- void handle_create_mirror_snapshot_finish(
- const std::string &remote_group_snap_id, int r);
};
} // namespace mirror
image_replayer->handle_replayer_notification();
}
- void list_remote_group_snapshots(Context *on_finish) override {
- if (local_group_ctx == nullptr) {
- on_finish->complete(0);
- return;
- }
-
- local_group_ctx->listener->list_remote_group_snapshots(on_finish);
- }
-
- void create_mirror_snapshot_start(
- const cls::rbd::MirrorSnapshotNamespace &remote_group_snap_ns,
- int64_t *local_group_pool_id, std::string *local_group_id,
- std::string *local_group_snap_id, Context *on_finish) override {
- if (local_group_ctx == nullptr) {
- on_finish->complete(0);
- return;
- }
-
- local_group_ctx->listener->create_mirror_snapshot_start(
- remote_group_snap_ns, image_replayer, local_group_pool_id,
- local_group_id, local_group_snap_id, on_finish);
- }
-
- void create_mirror_snapshot_finish(const std::string &remote_group_snap_id,
- uint64_t snap_id,
- Context *on_finish) override {
- if (local_group_ctx == nullptr) {
- on_finish->complete(0);
- return;
- }
-
- local_group_ctx->listener->create_mirror_snapshot_finish(
- remote_group_snap_id, image_replayer, snap_id, on_finish);
+ void notify_group_snap_image_complete(
+ int64_t local_pool_id,
+ const std::string &local_image_id,
+ const std::string &remote_group_snap_id,
+ uint64_t local_snap_id) override {
+ local_group_ctx->listener->notify_group_snap_image_complete(local_pool_id,
+ local_image_id, remote_group_snap_id, local_snap_id);
}
};
ImageReplayer<I>::~ImageReplayer()
{
unregister_admin_socket_hook();
- ceph_assert(m_state_builder == nullptr);
+ //ceph_assert(m_state_builder == nullptr);
ceph_assert(m_on_start_finish == nullptr);
ceph_assert(m_on_stop_contexts.empty());
ceph_assert(m_bootstrap_request == nullptr);
}
}
+template <typename I>
+void ImageReplayer<I>::prune_snapshot(uint64_t snap_id) {
+ std::unique_lock locker(m_lock);
+ if (m_replayer != nullptr) {
+ m_replayer->prune_snapshot(snap_id);
+ }
+}
+
template <typename I>
void ImageReplayer<I>::set_state_description(int r, const std::string &desc) {
dout(10) << "r=" << r << ", desc=" << desc << dendl;
void ImageReplayer<I>::stop(Context *on_finish, bool manual, bool restart)
{
dout(10) << "on_finish=" << on_finish << ", manual=" << manual
- << ", restart=" << restart << dendl;
+ << ", restart=" << restart << ", state=" << m_state << dendl;
image_replayer::BootstrapRequest<I> *bootstrap_request = nullptr;
bool shut_down_replay = false;
cancel_update_mirror_image_replay_status();
set_state_description(r, desc);
- update_mirror_image_status(true, boost::none);
+ update_mirror_image_status(false, boost::none);
shut_down(0);
}
template <typename I>
void ImageReplayer<I>::shut_down(int r) {
- dout(10) << "r=" << r << dendl;
+ dout(10) << "r=" << r << ", state=" << m_state << dendl;
{
std::lock_guard locker{m_lock};
bool resync_requested = false;
bool delete_requested = false;
bool unregister_asok_hook = false;
+ dout(10) << "r=" << r << dendl;
{
std::lock_guard locker{m_lock};
inline const std::string& get_global_image_id() const {
return m_global_image_id;
}
- inline const std::string& get_local_image_id() const {
- return m_state_builder->local_image_id;
+ inline const std::string get_local_image_id() const {
+ std::string image_id;
+ if (m_state_builder) {
+ image_id = m_state_builder->local_image_id;
+ }
+ return image_id;
}
void start(Context *on_finish, bool manual = false, bool restart = false);
void print_status(Formatter *f);
+ void prune_snapshot(uint64_t snap_id);
+
protected:
/**
* @verbatim
virtual ~Listener() {
}
- virtual void list_remote_group_snapshots(Context *on_finish) = 0;
-
- virtual void create_mirror_snapshot_start(
- const cls::rbd::MirrorSnapshotNamespace &remote_group_snap_ns,
- void *arg, int64_t *local_group_pool_id, std::string *local_group_id,
- std::string *local_group_snap_id, Context *on_finish) = 0;
- virtual void create_mirror_snapshot_finish(
- const std::string &remote_group_snap_id, void *arg, uint64_t snap_id,
- Context *on_finish) = 0;
+ virtual void notify_group_snap_image_complete(
+ int64_t local_pool_id,
+ const std::string &local_image_id,
+ const std::string &remote_group_snap_id,
+ uint64_t local_snap_id) = 0;
};
std::string name;
void BootstrapRequest<I>::handle_move_local_image_to_trash(int r) {
dout(10) << "r=" << r << dendl;
- if (m_canceled) {
- finish(-ECANCELED);
- return;
- }
-
if (r < 0 && r != -ENOENT) {
derr << "error moving mirror image to trash: " << cpp_strerror(r) << dendl;
finish(r);
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "Replayer.h"
+#include "common/Cond.h"
+#include "common/debug.h"
+#include "common/errno.h"
+#include "common/perf_counters.h"
+#include "common/perf_counters_key.h"
+#include "librbd/asio/ContextWQ.h"
+#include "librbd/group/ListSnapshotsRequest.h"
+#include "include/stringify.h"
+#include "common/Timer.h"
+#include "cls/rbd/cls_rbd_client.h"
+#include "json_spirit/json_spirit.h"
+#include "librbd/Utils.h"
+#include "tools/rbd_mirror/ImageReplayer.h"
+#include "tools/rbd_mirror/PoolMetaCache.h"
+#include "tools/rbd_mirror/Threads.h"
+
+
+#define dout_context g_ceph_context
+#define dout_subsys ceph_subsys_rbd_mirror
+#undef dout_prefix
+#define dout_prefix *_dout << "rbd::mirror::group_replayer::Replayer: " \
+ << this << " " << __func__ << ": "
+
+namespace rbd {
+namespace mirror {
+namespace group_replayer {
+
+using librbd::util::create_async_context_callback;
+using librbd::util::create_context_callback;
+using librbd::util::create_rados_callback;
+
+template <typename I>
+Replayer<I>::Replayer(
+ Threads<I>* threads,
+ librados::IoCtx &local_io_ctx,
+ librados::IoCtx &remote_io_ctx,
+ const std::string &global_group_id,
+ const std::string& local_mirror_uuid,
+ const std::string& remote_mirror_uuid,
+ PoolMetaCache* pool_meta_cache,
+ std::string local_group_id,
+ std::string remote_group_id,
+ std::list<std::pair<librados::IoCtx, ImageReplayer<I> *>> *image_replayers)
+ : m_threads(threads),
+ m_local_io_ctx(local_io_ctx),
+ m_remote_io_ctx(remote_io_ctx),
+ m_global_group_id(global_group_id),
+ m_local_mirror_uuid(local_mirror_uuid),
+ m_remote_mirror_uuid(remote_mirror_uuid),
+ m_pool_meta_cache(pool_meta_cache),
+ m_local_group_id(local_group_id),
+ m_remote_group_id(remote_group_id),
+ m_image_replayers(image_replayers),
+ m_lock(ceph::make_mutex(librbd::util::unique_lock_name(
+ "rbd::mirror::group_replayer::Replayer", this))) {
+ dout(10) << m_global_group_id << dendl;
+}
+
+template <typename I>
+Replayer<I>::~Replayer() {
+ dout(10) << m_global_group_id << dendl;
+
+ ceph_assert(m_state == STATE_COMPLETE);
+}
+
+template <typename I>
+void Replayer<I>::schedule_load_group_snapshots() {
+ dout(10) << dendl;
+
+ auto ctx = new LambdaContext(
+ [this](int r) {
+ load_local_group_snapshots();
+ });
+ std::lock_guard timer_locker{m_threads->timer_lock};
+ m_threads->timer->add_event_after(1, ctx);
+}
+
+template <typename I>
+void Replayer<I>::notify_group_snap_image_complete(
+ int64_t local_pool_id,
+ const std::string &local_image_id,
+ const std::string &remote_group_snap_id,
+ uint64_t local_snap_id) {
+
+ dout(10) << "local_pool_id=" << local_pool_id
+ << ", local_image_id=" << local_image_id
+ << ", remote_group_snap_id=" << remote_group_snap_id
+ << ", local_image_snap_id=" << local_snap_id << dendl;
+
+ std::unique_lock locker{m_lock};
+ if (m_state != STATE_IDLE && m_state != STATE_REPLAYING) {
+ locker.unlock();
+ derr << "replayer is not running, missed the notification" << dendl;
+ return;
+ }
+
+ cls::rbd::ImageSnapshotSpec spec;
+ spec.pool = local_pool_id;
+ spec.image_id = local_image_id;
+ spec.snap_id = local_snap_id;
+ m_pending_group_snaps[remote_group_snap_id].push_back({spec, false});
+ if (m_state == STATE_IDLE) {
+ locker.unlock();
+ load_local_group_snapshots();
+ return;
+ }
+ locker.unlock();
+}
+
+template <typename I>
+int Replayer<I>::local_group_image_list_by_id(
+ std::vector<cls::rbd::GroupImageStatus> *image_ids) {
+ std::string group_header_oid = librbd::util::group_header_name(
+ m_local_group_id);
+
+ dout(10) << "listing images in local group id " << group_header_oid << dendl;
+ image_ids->clear();
+
+ int r = 0;
+ const int max_read = 1024;
+ cls::rbd::GroupImageSpec start_last;
+ do {
+ std::vector<cls::rbd::GroupImageStatus> image_ids_page;
+
+ r = librbd::cls_client::group_image_list(&m_local_io_ctx, group_header_oid,
+ start_last, max_read,
+ &image_ids_page);
+
+ if (r < 0) {
+ derr << "error reading image list from local group: "
+ << cpp_strerror(-r) << dendl;
+ return r;
+ }
+ image_ids->insert(image_ids->end(), image_ids_page.begin(),
+ image_ids_page.end());
+
+ if (image_ids_page.size() > 0)
+ start_last = image_ids_page.rbegin()->spec;
+
+ r = image_ids_page.size();
+ } while (r == max_read);
+
+ return 0;
+}
+
+template <typename I>
+void Replayer<I>::init(Context* on_finish) {
+ dout(10) << m_global_group_id << dendl;
+
+ ceph_assert(m_state == STATE_INIT);
+
+ RemotePoolMeta remote_pool_meta;
+ int r = m_pool_meta_cache->get_remote_pool_meta(
+ m_remote_io_ctx.get_id(), &remote_pool_meta);
+ if (r < 0 || remote_pool_meta.mirror_peer_uuid.empty()) {
+ derr << "failed to retrieve mirror peer uuid from remote pool" << dendl;
+ m_state = STATE_COMPLETE;
+ m_threads->work_queue->queue(on_finish, r);
+ return;
+ }
+
+ m_remote_mirror_peer_uuid = remote_pool_meta.mirror_peer_uuid;
+ dout(10) << "remote_mirror_peer_uuid=" << m_remote_mirror_peer_uuid << dendl;
+
+ on_finish->complete(0);
+ load_local_group_snapshots();
+}
+
+template <typename I>
+void Replayer<I>::load_local_group_snapshots() {
+ dout(10) << "m_local_group_id=" << m_local_group_id << dendl;
+
+ if (m_state != STATE_COMPLETE) {
+ m_state = STATE_REPLAYING;
+ }
+
+ m_local_group_snaps.clear();
+ auto ctx = create_context_callback<
+ Replayer<I>,
+ &Replayer<I>::handle_load_local_group_snapshots>(this);
+
+ auto req = librbd::group::ListSnapshotsRequest<I>::create(m_local_io_ctx,
+ m_local_group_id, true, true, &m_local_group_snaps, ctx);
+ req->send();
+}
+
+template <typename I>
+void Replayer<I>::handle_load_local_group_snapshots(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ if (r < 0) {
+ derr << "error listing local mirror group snapshots: " << cpp_strerror(r)
+ << dendl;
+ schedule_load_group_snapshots();
+ return;
+ }
+
+ std::unique_lock locker{m_lock};
+ for (auto it = m_local_group_snaps.rbegin();
+ it != m_local_group_snaps.rend(); it++) {
+ auto ns = std::get_if<cls::rbd::MirrorGroupSnapshotNamespace>(
+ &it->snapshot_namespace);
+ if (ns == nullptr) {
+ continue;
+ }
+ if (ns->state != cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY) {
+ break; // Not a primary, continue the group replayer
+ }
+ ceph_assert(m_state == STATE_REPLAYING);
+ m_state = STATE_IDLE;
+ locker.unlock();
+ return;
+ }
+
+ load_remote_group_snapshots();
+}
+
+template <typename I>
+void Replayer<I>::load_remote_group_snapshots() {
+ dout(10) << "m_remote_group_id=" << m_remote_group_id << dendl;
+
+ m_remote_group_snaps.clear();
+ auto ctx = new LambdaContext(
+ [this] (int r) {
+ handle_load_remote_group_snapshots(r);
+ });
+
+ auto req = librbd::group::ListSnapshotsRequest<I>::create(m_remote_io_ctx,
+ m_remote_group_id, true, true, &m_remote_group_snaps, ctx);
+ req->send();
+}
+
+template <typename I>
+void Replayer<I>::handle_load_remote_group_snapshots(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ if (r < 0) {
+ derr << "error listing remote mirror group snapshots: " << cpp_strerror(r)
+ << dendl;
+ load_remote_group_snapshots();
+ return;
+ }
+
+ std::unique_lock locker{m_lock};
+ scan_for_unsynced_group_snapshots(locker);
+}
+
+template <typename I>
+void Replayer<I>::scan_for_unsynced_group_snapshots(
+ std::unique_lock<ceph::mutex> &locker) {
+ dout(10) << dendl;
+ ceph_assert(ceph_mutex_is_locked_by_me(m_lock));
+
+ if (!m_pending_group_snaps.empty()) {
+ bool complete_req = false;
+ for (auto &snap : m_pending_group_snaps) {
+ // skip if the snap is not discovered by Replayer yet
+ auto id = snap.first;
+ auto itl = std::find_if(
+ m_local_group_snaps.begin(), m_local_group_snaps.end(),
+ [id](const cls::rbd::GroupSnapshot &s) {
+ return s.id == id;
+ });
+ if (itl == m_local_group_snaps.end()) {
+ continue;
+ }
+ if (snap.second.empty()) {
+ C_SaferCond *ctx = new C_SaferCond;
+ mirror_snapshot_complete(snap.first, nullptr, ctx);
+ ctx->wait();
+ continue;
+ }
+ for (auto &i : snap.second) {
+ cls::rbd::ImageSnapshotSpec spec = i.first;
+ int64_t pool = spec.pool;
+ std::string image_id = spec.image_id;
+ auto it = std::find_if(
+ itl->snaps.begin(), itl->snaps.end(),
+ [&pool, &image_id](const cls::rbd::ImageSnapshotSpec &s) {
+ return pool == s.pool && image_id == s.image_id;
+ });
+ if (it == itl->snaps.end()) {
+ complete_req = true;
+ C_SaferCond *ctx = new C_SaferCond;
+ mirror_snapshot_complete(snap.first, &spec, ctx);
+ ctx->wait();
+ i.second = true; // ack
+ }
+ }
+ }
+ if (complete_req) {
+ locker.unlock();
+ schedule_load_group_snapshots();
+ return;
+ }
+ }
+
+ bool found = false;
+ bool syncs_upto_date = false;
+ if (m_remote_group_snaps.empty()) {
+ goto out;
+ }
+
+ // check if we have a matching snap on remote to start with.
+ for (auto local_snap = m_local_group_snaps.rbegin();
+ local_snap != m_local_group_snaps.rend(); ++local_snap) {
+ auto snap_type = cls::rbd::get_group_snap_namespace_type(
+ local_snap->snapshot_namespace);
+ auto local_snap_ns = std::get_if<cls::rbd::MirrorGroupSnapshotNamespace>(
+ &local_snap->snapshot_namespace);
+
+ auto next_remote_snap = m_remote_group_snaps.end();
+ if (snap_type == cls::rbd::GROUP_SNAPSHOT_NAMESPACE_TYPE_USER ||
+ local_snap_ns->is_non_primary() ||
+ local_snap_ns->state == cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED) {
+ for (auto remote_snap = m_remote_group_snaps.begin();
+ remote_snap != m_remote_group_snaps.end(); ++remote_snap) {
+ if (local_snap->id == remote_snap->id) {
+ next_remote_snap = std::next(remote_snap);
+ found = true;
+ break;
+ }
+ }
+ }
+ if (found && next_remote_snap == m_remote_group_snaps.end()) {
+ syncs_upto_date = true;
+ break;
+ }
+ if (next_remote_snap != m_remote_group_snaps.end()) {
+ auto id = next_remote_snap->id;
+ auto itl = std::find_if(
+ m_local_group_snaps.begin(), m_local_group_snaps.end(),
+ [id](const cls::rbd::GroupSnapshot &s) {
+ return s.id == id;
+ });
+ if (found && itl == m_local_group_snaps.end()) {
+ try_create_group_snapshot(*next_remote_snap);
+ locker.unlock();
+ return;
+ }
+ }
+ found = false;
+ }
+ if (!syncs_upto_date) {
+ dout(10) << "non of the local snaps match remote" << dendl;
+ auto remote_snap = m_remote_group_snaps.rbegin();
+ for(; remote_snap != m_remote_group_snaps.rend(); ++remote_snap) {
+ auto prev_remote_snap = std::next(remote_snap);
+ if (prev_remote_snap == m_remote_group_snaps.rend()) {
+ break;
+ }
+ auto snap_type = cls::rbd::get_group_snap_namespace_type(
+ prev_remote_snap->snapshot_namespace);
+ if (snap_type != cls::rbd::GROUP_SNAPSHOT_NAMESPACE_TYPE_MIRROR) {
+ continue;
+ }
+ auto prev_remote_snap_ns = std::get_if<cls::rbd::MirrorGroupSnapshotNamespace>(
+ &prev_remote_snap->snapshot_namespace);
+ if (prev_remote_snap_ns->is_demoted()) {
+ break;
+ }
+ }
+ auto id = remote_snap->id;
+ auto itl = std::find_if(
+ m_local_group_snaps.begin(), m_local_group_snaps.end(),
+ [id](const cls::rbd::GroupSnapshot &s) {
+ return s.id == id;
+ });
+ if (remote_snap != m_remote_group_snaps.rend() &&
+ itl == m_local_group_snaps.end()) {
+ try_create_group_snapshot(*remote_snap);
+ locker.unlock();
+ return;
+ }
+ }
+
+ for (auto &snap : m_pending_group_snaps) {
+ dout(10) << "snap: " << snap.first << dendl;
+ auto snap_spec = snap.second;
+ for (auto &it : snap_spec) {
+ if (it.second) { // already ack by Replayer
+ continue;
+ }
+ dout(10) << "replayer is working on pending snaps" << dendl;
+ locker.unlock();
+ schedule_load_group_snapshots();
+ return;
+ }
+ }
+
+out:
+ if (m_on_shutdown) {
+ locker.unlock();
+ m_on_shutdown->complete(0);
+ return;
+ }
+
+ dout(10) << "all remote snapshots synced, idling waiting for new snapshot"
+ << dendl;
+
+ ceph_assert(m_state == STATE_REPLAYING);
+ m_state = STATE_IDLE;
+ locker.unlock();
+}
+
+template <typename I>
+std::string Replayer<I>::prepare_non_primary_mirror_snap_name(
+ const std::string &global_group_id,
+ const std::string &snap_id) {
+ dout(5) << "global_group_id: " << global_group_id
+ << ", snap_id: " << snap_id << dendl;
+ std::stringstream ind_snap_name_stream;
+ ind_snap_name_stream << ".mirror.non-primary."
+ << global_group_id << "." << snap_id;
+ return ind_snap_name_stream.str();
+}
+
+template <typename I>
+void Replayer<I>::try_create_group_snapshot(cls::rbd::GroupSnapshot snap) {
+ dout(10) << snap.id << dendl;
+
+ auto snap_type = cls::rbd::get_group_snap_namespace_type(
+ snap.snapshot_namespace);
+ if (snap_type == cls::rbd::GROUP_SNAPSHOT_NAMESPACE_TYPE_MIRROR) {
+ auto snap_ns = std::get_if<cls::rbd::MirrorGroupSnapshotNamespace>(
+ &snap.snapshot_namespace);
+ if (snap_ns->is_non_primary()) {
+ dout(10) << "remote group snapshot: " << snap.id << "is non primary"
+ << dendl;
+ return;
+ }
+ auto snap_state =
+ snap_ns->state == cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY ?
+ cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY :
+ cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY_DEMOTED;
+ create_mirror_snapshot(snap.id, snap_state);
+ } else if (snap_type == cls::rbd::GROUP_SNAPSHOT_NAMESPACE_TYPE_USER) {
+ dout(10) << "found user snap, snap name: " << snap.name
+ << ", remote group snap id: " << snap.id << dendl;
+ std::vector<cls::rbd::GroupImageStatus> local_images;
+ int r = local_group_image_list_by_id(&local_images);
+ if (r < 0) {
+ derr << "failed group image list: " << cpp_strerror(r) << dendl;
+ return;
+ }
+ C_SaferCond *ctx = new C_SaferCond;
+ mirror_regular_snapshot(snap.name, snap.id, &local_images, ctx);
+ ctx->wait();
+ }
+}
+
+template <typename I>
+void Replayer<I>::create_mirror_snapshot(
+ const std::string &remote_group_snap_id,
+ const cls::rbd::MirrorSnapshotState &snap_state) {
+ dout(10) << remote_group_snap_id << dendl;
+
+ auto itl = std::find_if(
+ m_local_group_snaps.begin(), m_local_group_snaps.end(),
+ [remote_group_snap_id](const cls::rbd::GroupSnapshot &s) {
+ return s.id == remote_group_snap_id;
+ });
+
+ if (itl != m_local_group_snaps.end() &&
+ itl->state == cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE) {
+ dout(20) << "group snapshot: " << remote_group_snap_id << " already exists"
+ << dendl;
+ schedule_load_group_snapshots();
+ return;
+ }
+
+ auto requests_it = m_create_snap_requests.find(remote_group_snap_id);
+ if (requests_it == m_create_snap_requests.end()) {
+ requests_it = m_create_snap_requests.insert(
+ {remote_group_snap_id, {}}).first;
+ cls::rbd::GroupSnapshot local_snap =
+ {remote_group_snap_id,
+ cls::rbd::MirrorGroupSnapshotNamespace{
+ snap_state, {}, m_remote_mirror_uuid, remote_group_snap_id},
+ {}, cls::rbd::GROUP_SNAPSHOT_STATE_INCOMPLETE};
+ local_snap.name = prepare_non_primary_mirror_snap_name(m_global_group_id,
+ remote_group_snap_id);
+ m_local_group_snaps.push_back(local_snap);
+
+ auto comp = create_rados_callback(
+ new LambdaContext([this, remote_group_snap_id](int r) {
+ handle_create_mirror_snapshot(remote_group_snap_id, r);
+ }));
+
+ librados::ObjectWriteOperation op;
+ librbd::cls_client::group_snap_set(&op, local_snap);
+ int r = m_local_io_ctx.aio_operate(
+ librbd::util::group_header_name(m_local_group_id), comp, &op);
+ ceph_assert(r == 0);
+ comp->release();
+ } else {
+ schedule_load_group_snapshots();
+ }
+}
+
+template <typename I>
+void Replayer<I>::handle_create_mirror_snapshot(
+ const std::string &remote_group_snap_id, int r) {
+ dout(10) << remote_group_snap_id << ", r=" << r << dendl;
+
+ std::unique_lock locker{m_lock};
+ auto snap_id = remote_group_snap_id;
+ auto itr = std::find_if(
+ m_remote_group_snaps.begin(), m_remote_group_snaps.end(),
+ [snap_id](const cls::rbd::GroupSnapshot &s) {
+ return s.id == snap_id;
+ });
+
+ if(itr != m_remote_group_snaps.end()) {
+ if (itr->snaps.size() == 0) {
+ dout(10) << "remote snap with no image snaps: " << snap_id << dendl;
+ m_pending_group_snaps[remote_group_snap_id].push_back({});
+ }
+ }
+ locker.unlock();
+ schedule_load_group_snapshots();
+}
+
+template <typename I>
+void Replayer<I>::mirror_snapshot_complete(
+ const std::string &remote_group_snap_id,
+ cls::rbd::ImageSnapshotSpec *spec,
+ Context *on_finish) {
+
+ ceph_assert(ceph_mutex_is_locked_by_me(m_lock));
+
+ auto itr = std::find_if(
+ m_remote_group_snaps.begin(), m_remote_group_snaps.end(),
+ [remote_group_snap_id](const cls::rbd::GroupSnapshot &s) {
+ return s.id == remote_group_snap_id;
+ });
+
+ ceph_assert(itr != m_remote_group_snaps.end());
+ auto itl = std::find_if(
+ m_local_group_snaps.begin(), m_local_group_snaps.end(),
+ [remote_group_snap_id](const cls::rbd::GroupSnapshot &s) {
+ return s.id == remote_group_snap_id;
+ });
+ if (itr->snaps.size() != 0) {
+ // update image snap
+ C_SaferCond *ctx = new C_SaferCond;
+ update_image_snapshot(remote_group_snap_id, *spec, ctx);
+ ctx->wait();
+
+ // update the group snap with snap spec
+ itl->snaps.push_back(*spec);
+ }
+
+ if (itr->snaps.size() == itl->snaps.size()) {
+ m_create_snap_requests.erase(remote_group_snap_id);
+ m_pending_group_snaps.erase(remote_group_snap_id);
+ itl->state = cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE;
+ }
+
+ dout(10) << "local group snap info: "
+ << "id: " << itl->id
+ << ", name: " << itl->name
+ << ", state: " << itl->state
+ << ", snaps.size: " << itl->snaps.size()
+ << dendl;
+ auto comp = create_rados_callback(
+ new LambdaContext([this, remote_group_snap_id, on_finish](int r) {
+ handle_mirror_snapshot_complete(r, remote_group_snap_id, on_finish);
+ }));
+
+ librados::ObjectWriteOperation op;
+ librbd::cls_client::group_snap_set(&op, *itl);
+ int r = m_local_io_ctx.aio_operate(
+ librbd::util::group_header_name(m_local_group_id), comp, &op);
+ ceph_assert(r == 0);
+ comp->release();
+}
+
+template <typename I>
+void Replayer<I>::handle_mirror_snapshot_complete(
+ int r, const std::string &remote_group_snap_id, Context *on_finish) {
+ dout(10) << remote_group_snap_id << ", r=" << r << dendl;
+
+ auto itl = std::find_if(
+ m_local_group_snaps.begin(), m_local_group_snaps.end(),
+ [remote_group_snap_id](const cls::rbd::GroupSnapshot &s) {
+ return s.id == remote_group_snap_id;
+ });
+
+ if (itl->state !=
+ cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE) {
+ on_finish->complete(0);
+ return;
+ }
+ unlink_group_snapshots(remote_group_snap_id);
+ on_finish->complete(0);
+}
+
+template <typename I>
+void Replayer<I>::unlink_group_snapshots(
+ const std::string &remote_group_snap_id) {
+ if (m_image_replayers->empty()) {
+ return;
+ }
+ dout(10) << dendl;
+ int r;
+ bool unlink_snap;
+ for (auto &snap : m_local_group_snaps) {
+ if (snap.id == remote_group_snap_id) {
+ break;
+ }
+ auto snap_type = cls::rbd::get_group_snap_namespace_type(
+ snap.snapshot_namespace);
+ if (snap_type == cls::rbd::GROUP_SNAPSHOT_NAMESPACE_TYPE_USER) {
+ bool unlink_user_snap = true;
+ for (auto &remote_snap : m_remote_group_snaps) {
+ if (remote_snap.name == snap.name) {
+ unlink_user_snap = false;
+ break;
+ }
+ }
+ if (!unlink_user_snap) {
+ continue;
+ }
+ dout(10) << "unlinking regular group snap in-progress: "
+ << snap.name << ", with id: " << snap.id << dendl;
+ }
+ dout(10) << "attempting to unlink image snaps from group snap: "
+ << snap.id << dendl;
+ unlink_snap = true;
+ for (auto &spec : snap.snaps) {
+ std::string image_header_oid = librbd::util::header_name(spec.image_id);
+ cls::rbd::SnapshotInfo snap_info;
+ r = librbd::cls_client::snapshot_get(&m_local_io_ctx, image_header_oid,
+ spec.snap_id, &snap_info);
+ if (r == -ENOENT) {
+ continue;
+ } else if (r < 0) {
+ derr << "failed getting snap info for snap id: " << spec.snap_id
+ << ", : " << cpp_strerror(r) << dendl;
+ }
+ unlink_snap = false;
+ for (auto it = m_image_replayers->begin();
+ it != m_image_replayers->end(); ++it) {
+ auto image_replayer = it->second;
+ if (!image_replayer) {
+ continue;
+ }
+ auto local_image_id = image_replayer->get_local_image_id();
+ if (local_image_id.empty() || local_image_id != spec.image_id) {
+ continue;
+ }
+ dout(10) << "pruning: " << spec.snap_id << dendl;
+ image_replayer->prune_snapshot(spec.snap_id);
+ break;
+ }
+ }
+ if (!unlink_snap) {
+ continue;
+ }
+ dout(10) << "all image snaps are pruned, finally unlinking group snap: "
+ << snap.id << dendl;
+ r = librbd::cls_client::group_snap_remove(&m_local_io_ctx,
+ librbd::util::group_header_name(m_local_group_id), snap.id);
+ if (r < 0) {
+ derr << "failed to remove group snapshot : "
+ << snap.id << " : " << cpp_strerror(r) << dendl;
+ }
+ }
+}
+
+template <typename I>
+void Replayer<I>::update_image_snapshot(
+ const std::string &remote_group_snap_id,
+ cls::rbd::ImageSnapshotSpec spec,
+ Context *on_finish) {
+ dout(10) << "local group snap info: "
+ << "image snap id: " << spec.snap_id
+ << ", image id: " << spec.image_id
+ << ", group snap id: " << remote_group_snap_id
+ << dendl;
+ std::string image_header_oid = librbd::util::header_name(spec.image_id);
+ cls::rbd::SnapshotInfo snap_info;
+ int r = librbd::cls_client::snapshot_get(&m_local_io_ctx, image_header_oid,
+ spec.snap_id, &snap_info);
+ if (r < 0) {
+ derr << "failed getting snap info for snap id: " << spec.snap_id
+ << ", : " << cpp_strerror(r) << dendl;
+ }
+ auto mirror_ns = std::get_if<cls::rbd::MirrorSnapshotNamespace>(
+ &snap_info.snapshot_namespace);
+ ceph_assert(mirror_ns != nullptr);
+ mirror_ns->group_spec = {m_local_group_id, spec.pool};
+ mirror_ns->group_snap_id = remote_group_snap_id;
+
+ // write to disk
+ librados::ObjectWriteOperation op;
+ librbd::cls_client::snapshot_add(&op, snap_info.id, snap_info.name,
+ *mirror_ns);
+ auto comp = create_rados_callback(
+ new LambdaContext([this, snap_info, on_finish](int r) {
+ handle_update_image_snapshot(r, snap_info.id, on_finish);
+ }));
+ r = m_local_io_ctx.aio_operate(image_header_oid, comp, &op);
+ ceph_assert(r == 0);
+ comp->release();
+}
+
+template <typename I>
+void Replayer<I>::handle_update_image_snapshot(
+ int r, uint64_t local_snap_id, Context *on_finish) {
+ dout(10) << "snap id: " << local_snap_id << ", r=" << r << dendl;
+ on_finish->complete(r);
+}
+
+template <typename I>
+void Replayer<I>::mirror_regular_snapshot(
+ const std::string &remote_group_snap_name,
+ const std::string &remote_group_snap_id,
+ std::vector<cls::rbd::GroupImageStatus> *local_images,
+ Context *on_finish) {
+ // each image will have one snapshot specific to group snap, and so for each
+ // image get a ImageSnapshotSpec and prepare a vector
+ // for image :: <images in that group> {
+ // * get snap whos name has group snap_id for that we can list snaps and
+ // filter with remote_group_snap_id
+ // * get its { pool_id, snap_id, image_id }
+ // }
+ // finally write to the object
+ dout(10) << remote_group_snap_id << dendl;
+ librados::ObjectWriteOperation op;
+ cls::rbd::GroupSnapshot group_snap{
+ remote_group_snap_id, // keeping it same as remote group snap id
+ cls::rbd::UserGroupSnapshotNamespace{},
+ remote_group_snap_name,
+ cls::rbd::GROUP_SNAPSHOT_STATE_INCOMPLETE};
+
+ std::vector<cls::rbd::ImageSnapshotSpec> local_image_snap_specs;
+ local_image_snap_specs = std::vector<cls::rbd::ImageSnapshotSpec>(
+ local_images->size(), cls::rbd::ImageSnapshotSpec());
+ for (auto& image : *local_images) {
+ std::string image_header_oid = librbd::util::header_name(
+ image.spec.image_id);
+ ::SnapContext snapc;
+ int r = librbd::cls_client::get_snapcontext(&m_local_io_ctx,
+ image_header_oid, &snapc);
+ if (r < 0) {
+ derr << "get snap context failed: " << cpp_strerror(r) << dendl;
+ on_finish->complete(r);
+ return;
+ }
+
+ auto image_snap_name = ".group." + std::to_string(image.spec.pool_id) +
+ "_" + m_remote_group_id + "_" + remote_group_snap_id;
+ // stored in reverse order
+ for (auto snap_id : snapc.snaps) {
+ cls::rbd::SnapshotInfo snap_info;
+ r = librbd::cls_client::snapshot_get(&m_local_io_ctx, image_header_oid,
+ snap_id, &snap_info);
+ if (r < 0) {
+ derr << "failed getting snap info for snap id: " << snap_id
+ << ", : " << cpp_strerror(r) << dendl;
+ on_finish->complete(r);
+ return;
+ }
+
+ // extract { pool_id, snap_id, image_id }
+ if (snap_info.name == image_snap_name) {
+ cls::rbd::ImageSnapshotSpec snap_spec;
+ snap_spec.pool = image.spec.pool_id;
+ snap_spec.image_id = image.spec.image_id;
+ snap_spec.snap_id = snap_info.id;
+
+ local_image_snap_specs.push_back(snap_spec);
+ }
+ }
+ }
+
+ group_snap.snaps = local_image_snap_specs;
+ group_snap.state = cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE;
+ librbd::cls_client::group_snap_set(&op, group_snap);
+ m_local_group_snaps.push_back(group_snap);
+
+ auto comp = create_rados_callback(
+ new LambdaContext([this, on_finish](int r) {
+ handle_mirror_regular_snapshot(r, on_finish);
+ }));
+ int r = m_local_io_ctx.aio_operate(
+ librbd::util::group_header_name(m_local_group_id), comp, &op);
+ ceph_assert(r == 0);
+ comp->release();
+}
+
+template <typename I>
+void Replayer<I>::handle_mirror_regular_snapshot(
+ int r, Context *on_finish) {
+ dout(10) << "r=" << r << dendl;
+
+ if (r < 0) {
+ derr << "error creating local non-primary group snapshot: "
+ << cpp_strerror(r) << dendl;
+ }
+ on_finish->complete(0);
+
+ schedule_load_group_snapshots();
+}
+
+template <typename I>
+void Replayer<I>::shut_down(Context* on_finish) {
+ dout(10) << dendl;
+
+ std::unique_lock locker{m_lock};
+ m_on_shutdown = on_finish;
+ auto state = STATE_COMPLETE;
+ std::swap(m_state, state);
+
+ if (state == STATE_REPLAYING) {
+ // if there are any pending snaps
+ dout(10) << "shut down pending" << dendl;
+ return;
+ }
+ locker.unlock();
+ m_on_shutdown->complete(0);
+ return;
+}
+
+
+} // namespace group_replayer
+} // namespace mirror
+} // namespace rbd
+
+template class rbd::mirror::group_replayer::Replayer<librbd::ImageCtx>;
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef RBD_MIRROR_GROUP_REPLAYER_REPLAYER_H
+#define RBD_MIRROR_GROUP_REPLAYER_REPLAYER_H
+
+#include "tools/rbd_mirror/image_replayer/Replayer.h"
+#include "common/ceph_mutex.h"
+#include "cls/rbd/cls_rbd_types.h"
+#include "include/rados/librados.hpp"
+#include "librbd/mirror/snapshot/Types.h"
+#include "tools/rbd_mirror/image_replayer/Types.h"
+#include <string>
+
+class Context;
+namespace librbd { class ImageCtx; }
+
+namespace rbd {
+namespace mirror {
+
+template <typename> class ImageReplayer;
+class PoolMetaCache;
+template <typename> struct Threads;
+
+namespace group_replayer {
+
+template <typename ImageCtxT = librbd::ImageCtx>
+class Replayer {
+public:
+ static Replayer* create(
+ Threads<ImageCtxT>* threads,
+ librados::IoCtx &local_io_ctx,
+ librados::IoCtx &remote_io_ctx,
+ const std::string &global_group_id,
+ const std::string& local_mirror_uuid,
+ const std::string& remote_mirror_uuid,
+ PoolMetaCache* pool_meta_cache,
+ std::string local_group_id,
+ std::string remote_group_id,
+ std::list<std::pair<librados::IoCtx, ImageReplayer<ImageCtxT> *>> *image_replayers) {
+ return new Replayer(threads, local_io_ctx, remote_io_ctx, global_group_id,
+ local_mirror_uuid, remote_mirror_uuid, pool_meta_cache, local_group_id,
+ remote_group_id, image_replayers);
+ }
+
+ Replayer(
+ Threads<ImageCtxT>* threads,
+ librados::IoCtx &local_io_ctx,
+ librados::IoCtx &remote_io_ctx,
+ const std::string &global_group_id,
+ const std::string& local_mirror_uuid,
+ const std::string& remote_mirror_uuid,
+ PoolMetaCache* pool_meta_cache,
+ std::string local_group_id,
+ std::string remote_group_id,
+ std::list<std::pair<librados::IoCtx, ImageReplayer<ImageCtxT> *>> *image_replayers);
+ ~Replayer();
+
+ void destroy() {
+ delete this;
+ }
+ void init(Context* on_finish);
+ void shut_down(Context* on_finish);
+
+ bool is_replaying() const {
+ std::unique_lock locker{m_lock};
+ return (m_state == STATE_REPLAYING || m_state == STATE_IDLE);
+ }
+
+ void notify_group_snap_image_complete(
+ int64_t local_pool_id,
+ const std::string &local_image_id,
+ const std::string &remote_group_snap_id,
+ uint64_t local_snap_id);
+
+private:
+ enum State {
+ STATE_INIT,
+ STATE_REPLAYING,
+ STATE_IDLE,
+ STATE_COMPLETE
+ };
+
+ Threads<ImageCtxT> *m_threads;
+ librados::IoCtx &m_local_io_ctx;
+ librados::IoCtx &m_remote_io_ctx;
+ std::string m_global_group_id;
+ std::string m_local_mirror_uuid;
+ std::string m_remote_mirror_uuid;
+ PoolMetaCache* m_pool_meta_cache;
+ std::string m_local_group_id;
+ std::string m_remote_group_id;
+ std::list<std::pair<librados::IoCtx, ImageReplayer<ImageCtxT> *>> *m_image_replayers;
+
+ mutable ceph::mutex m_lock;
+
+ State m_state = STATE_INIT;
+ Context* m_on_shutdown = nullptr;
+ std::string m_remote_mirror_peer_uuid;
+
+ std::vector<cls::rbd::GroupSnapshot> m_local_group_snaps;
+ std::vector<cls::rbd::GroupSnapshot> m_remote_group_snaps;
+
+ // map of <group_snap_id, pair<GroupSnapshot, on_finish>>
+ std::map<std::string, std::pair<cls::rbd::GroupSnapshot, Context *>> m_create_snap_requests;
+
+ // map of <group_snap_id, vec<pair<cls::rbd::ImageSnapshotSpec, bool>>>
+ std::map<std::string, std::vector<std::pair<cls::rbd::ImageSnapshotSpec, bool>>> m_pending_group_snaps;
+
+ int local_group_image_list_by_id(
+ std::vector<cls::rbd::GroupImageStatus> *image_ids);
+
+ void schedule_load_group_snapshots();
+
+ void load_local_group_snapshots();
+ void handle_load_local_group_snapshots(int r);
+
+ void load_remote_group_snapshots();
+ void handle_load_remote_group_snapshots(int r);
+
+ void scan_for_unsynced_group_snapshots(std::unique_lock<ceph::mutex>& locker);
+
+ void try_create_group_snapshot(cls::rbd::GroupSnapshot snap);
+
+ void create_mirror_snapshot(
+ const std::string &remote_group_snap_id,
+ const cls::rbd::MirrorSnapshotState &snap_state);
+ void handle_create_mirror_snapshot(
+ const std::string &remote_group_snap_id, int r);
+
+ std::string prepare_non_primary_mirror_snap_name(
+ const std::string &global_group_id, const std::string &snap_id);
+
+ void mirror_snapshot_complete(
+ const std::string &remote_group_snap_id,
+ cls::rbd::ImageSnapshotSpec *spec,
+ Context *on_finish);
+ void handle_mirror_snapshot_complete(
+ int r, const std::string &remote_group_snap_id, Context *on_finish);
+
+ void unlink_group_snapshots(const std::string &remote_group_snap_id);
+
+ void update_image_snapshot(
+ const std::string &remote_group_snap_id,
+ cls::rbd::ImageSnapshotSpec spec,
+ Context *on_finish);
+ void handle_update_image_snapshot(
+ int r, uint64_t local_snap_id, Context *on_finish);
+
+ void mirror_regular_snapshot(
+ const std::string &remote_group_snap_name,
+ const std::string &remote_group_snap_id,
+ std::vector<cls::rbd::GroupImageStatus> *local_images,
+ Context *on_finish);
+ void handle_mirror_regular_snapshot(int r, Context *on_finish);
+};
+
+} // namespace group_replayer
+} // namespace mirror
+} // namespace rbd
+
+extern template class rbd::mirror::group_replayer::Replayer<librbd::ImageCtx>;
+
+#endif // RBD_MIRROR_GROUP_REPLAYER_REPLAYER_H
#ifndef RBD_MIRROR_IMAGE_REPLAYER_REPLAYER_H
#define RBD_MIRROR_IMAGE_REPLAYER_REPLAYER_H
+#include <cstdint>
#include <string>
struct Context;
virtual int get_error_code() const = 0;
virtual std::string get_error_description() const = 0;
+
+ virtual void prune_snapshot(uint64_t) = 0;
};
} // namespace image_replayer
virtual ~ReplayerListener() {}
virtual void handle_notification() = 0;
- virtual void list_remote_group_snapshots(Context *on_finish) = 0;
-
- virtual void create_mirror_snapshot_start(
- const cls::rbd::MirrorSnapshotNamespace &remote_group_snap_ns,
- int64_t *local_group_pool_id, std::string *local_group_id,
- std::string *local_group_snap_id, Context *on_finish) = 0;
- virtual void create_mirror_snapshot_finish(
- const std::string &remote_group_snap_id, uint64_t snap_id,
- Context *on_finish) = 0;
+ virtual void notify_group_snap_image_complete(
+ int64_t local_pool_id,
+ const std::string &local_image_id,
+ const std::string &remote_group_snap_id,
+ uint64_t local_snap_id) = 0;
};
} // namespace image_replayer
return m_image_spec;
}
+ void prune_snapshot(uint64_t snap_id) {
+ }
+
private:
/**
* @verbatim
m_remote_snap_id_end = CEPH_NOSNAP;
m_remote_mirror_snap_ns = {};
- std::set<uint64_t> prune_snap_ids;
-
auto local_image_ctx = m_state_builder->local_image_ctx;
std::shared_lock image_locker{local_image_ctx->image_lock};
for (auto snap_info_it = local_image_ctx->snap_info.begin();
m_local_snap_id_start = local_snap_id;
ceph_assert(m_local_snap_id_end == CEPH_NOSNAP);
- if (mirror_ns->mirror_peer_uuids.empty()) {
+ if (mirror_ns->mirror_peer_uuids.empty() &&
+ (!mirror_ns->group_spec.is_valid() &&
+ mirror_ns->group_snap_id.empty())) {
// no other peer will attempt to sync to this snapshot so store as
// a candidate for removal
- prune_snap_ids.insert(local_snap_id);
+ m_prune_snap_ids.insert(local_snap_id);
}
} else if (mirror_ns->last_copied_object_number == 0 &&
m_local_snap_id_start > 0) {
// the first non-primary snapshot since we know its snapshot is
// well-formed because otherwise the mirror-image-state would have
// forced an image deletion.
- prune_snap_ids.clear();
- prune_snap_ids.insert(local_snap_id);
+ if(!mirror_ns->group_spec.is_valid() &&
+ mirror_ns->group_snap_id.empty()) {
+ m_prune_snap_ids.clear();
+ m_prune_snap_ids.insert(local_snap_id);
+ }
break;
} else {
// start snap will be last complete mirror snapshot or initial
if (m_local_snap_id_start > 0) {
// remove candidate that is required for delta snapshot sync
- prune_snap_ids.erase(m_local_snap_id_start);
+ m_prune_snap_ids.erase(m_local_snap_id_start);
}
- if (!prune_snap_ids.empty()) {
+ if (!m_prune_snap_ids.empty()) {
locker->unlock();
- m_prune_snap_id = *prune_snap_ids.begin();
- dout(5) << "pruning unused non-primary snapshot " << m_prune_snap_id << dendl;
- prune_non_primary_snapshot();
- //unlink_group_snapshot(); //PK: FIXME
+ auto prune_snap_id = *m_prune_snap_ids.begin();
+ dout(5) << "pruning unused non-primary snapshot " << prune_snap_id << dendl;
+ prune_non_primary_snapshot(prune_snap_id);
return;
}
}
template <typename I>
-void Replayer<I>::unlink_group_snapshot() {
- auto local_image_ctx = m_state_builder->local_image_ctx;
- cls::rbd::SnapshotNamespace snap_namespace;
- std::string snap_name;
- int r = 0;
- {
- std::shared_lock image_locker{local_image_ctx->image_lock};
-
- auto snap_info = local_image_ctx->get_snap_info(m_prune_snap_id);
- if (!snap_info) {
- r = -ENOENT;
- } else {
- snap_namespace = snap_info->snap_namespace;
- snap_name = snap_info->name;
- }
- }
-
- if (r == -ENOENT) {
- dout(15) << "failed to locate snapshot " << m_prune_snap_id << dendl;
- prune_non_primary_snapshot();
- return;
- }
-
- auto info = std::get_if<cls::rbd::MirrorSnapshotNamespace>(&snap_namespace);
- if (!info->group_spec.is_valid()) {
- prune_non_primary_snapshot();
- return;
- }
-
- dout(15) << "image_snap_id=" << m_prune_snap_id << dendl;
-
- r = librbd::util::create_ioctx(local_image_ctx->md_ctx, "group",
- info->group_spec.pool_id, {}, &m_group_io_ctx);
- if (r < 0) {
- prune_non_primary_snapshot();
- return;
- }
-
- librados::ObjectWriteOperation op;
- cls::rbd::ImageSnapshotSpec image_snap = {local_image_ctx->md_ctx.get_id(),
- local_image_ctx->id,
- m_prune_snap_id};
- librbd::cls_client::group_snap_unlink(&op, info->group_snap_id, image_snap);
- auto aio_comp = create_rados_callback<
- Replayer<I>,
- &Replayer<I>::handle_unlink_group_snapshot>(this);
- r = m_group_io_ctx.aio_operate(
- librbd::util::group_header_name(info->group_spec.group_id), aio_comp, &op);
- ceph_assert(r == 0);
- aio_comp->release();
-}
-
-template <typename I>
-void Replayer<I>::handle_unlink_group_snapshot(int r) {
- dout(15) << "r=" << r << dendl;
-
- if (r < 0 && r != -ENOENT) {
- derr << "failed to unlink group snapshot: " << cpp_strerror(r)
- << dendl;
- handle_replay_complete(r, "failed to unlink group snapshot");
- return;
- }
-
- prune_non_primary_snapshot();
-}
-
-template <typename I>
-void Replayer<I>::prune_non_primary_snapshot() {
- dout(10) << "snap_id=" << m_prune_snap_id << dendl;
+void Replayer<I>::prune_non_primary_snapshot(uint64_t snap_id) {
+ dout(10) << "snap_id=" << snap_id << dendl;
auto local_image_ctx = m_state_builder->local_image_ctx;
bool snap_valid = false;
{
std::shared_lock image_locker{local_image_ctx->image_lock};
- auto snap_info = local_image_ctx->get_snap_info(m_prune_snap_id);
+ auto snap_info = local_image_ctx->get_snap_info(snap_id);
if (snap_info != nullptr) {
snap_valid = true;
snap_namespace = snap_info->snap_namespace;
snap_name = snap_info->name;
-
- ceph_assert(std::holds_alternative<cls::rbd::MirrorSnapshotNamespace>(
- snap_namespace));
}
}
load_local_image_meta();
return;
}
+ m_prune_snap_ids.erase(snap_id);
auto ctx = create_context_callback<
Replayer<I>, &Replayer<I>::handle_prune_non_primary_snapshot>(this);
return;
}
- refresh_remote_group_snapshot_list();
+ create_non_primary_snapshot();
}
template <typename I>
request_sync();
}
-template <typename I>
-void Replayer<I>::refresh_remote_group_snapshot_list() {
- if (!m_remote_mirror_snap_ns.group_spec.is_valid() ||
- m_remote_mirror_snap_ns.group_snap_id.empty()) {
- create_non_primary_snapshot();
- return;
- }
-
- dout(10) << dendl;
-
- auto ctx = new LambdaContext(
- [this](int r) {
- handle_refresh_remote_group_snapshot_list(r);
- });
-
- m_replayer_listener->list_remote_group_snapshots(ctx);
-}
-
-template <typename I>
-void Replayer<I>::handle_refresh_remote_group_snapshot_list(int r) {
- dout(10) << "r=" << r << dendl;
-
- if (r < 0) {
- dout(15) << "restarting replayer" << dendl;
- load_local_image_meta();
- return;
- }
-
- create_group_snap_start();
-}
-
-template <typename I>
-void Replayer<I>::create_group_snap_start() {
- dout(10) << dendl;
-
- auto ctx = create_context_callback<
- Replayer<I>, &Replayer<I>::handle_create_group_snap_start>(this);
-
- m_replayer_listener->create_mirror_snapshot_start(
- m_remote_mirror_snap_ns, &m_local_group_pool_id,
- &m_local_group_id, &m_local_group_snap_id, ctx);
-}
-
-template <typename I>
-void Replayer<I>::handle_create_group_snap_start(int r) {
- dout(10) << "r=" << r << dendl;
-
- if (r < 0 && r != -EEXIST) {
- if (r == -EAGAIN) {
- auto ctx = new LambdaContext(
- [this](int r) {
- // retry after 1 sec
- refresh_remote_group_snapshot_list();
- });
- std::lock_guard timer_locker{m_threads->timer_lock};
- m_threads->timer->add_event_after(1, ctx);
- } else if (r == -ESTALE) {
- dout(15) << "waiting for shut down" << dendl;
- handle_replay_complete(r, "waiting for shut down");
- } else {
- derr << "failed to create group snapshot: " << cpp_strerror(r) << dendl;
- handle_replay_complete(r, "failed to create group snapshot");
- }
- return;
- }
-
- create_non_primary_snapshot();
-}
template <typename I>
void Replayer<I>::create_non_primary_snapshot() {
auto req = librbd::mirror::snapshot::CreateNonPrimaryRequest<I>::create(
local_image_ctx, m_remote_mirror_snap_ns.is_demoted(),
m_state_builder->remote_mirror_uuid, m_remote_snap_id_end,
- m_local_mirror_snap_ns.snap_seqs, m_local_group_pool_id, m_local_group_id,
- m_local_group_snap_id, m_image_state, &m_local_snap_id_end, ctx);
+ m_local_mirror_snap_ns.snap_seqs, m_image_state, &m_local_snap_id_end, ctx);
req->send();
}
dout(15) << "local_snap_id_end=" << m_local_snap_id_end << dendl;
- create_group_snap_finish();
-}
-
-template <typename I>
-void Replayer<I>::create_group_snap_finish() {
- if (!m_remote_mirror_snap_ns.group_spec.is_valid() ||
- m_remote_mirror_snap_ns.group_snap_id.empty()) {
- update_mirror_image_state();
- return;
- }
-
- dout(10) << dendl;
-
- auto ctx = create_context_callback<
- Replayer<I>, &Replayer<I>::handle_create_group_snap_finish>(this);
-
- m_replayer_listener->create_mirror_snapshot_finish(
- m_remote_mirror_snap_ns.group_snap_id, m_local_snap_id_end, ctx);
-}
-
-template <typename I>
-void Replayer<I>::handle_create_group_snap_finish(int r) {
- dout(10) << "r=" << r << dendl;
-
- if (r < 0 && r != -EEXIST) {
- derr << "failed to create group snapshot: " << cpp_strerror(r) << dendl;
- handle_replay_complete(r, "failed to create group snapshot");
- return;
- }
-
update_mirror_image_state();
}
return;
}
+ notify_group_snap_image_complete();
+}
+
+template <typename I>
+void Replayer<I>::notify_group_snap_image_complete() {
+ if (!m_remote_mirror_snap_ns.group_spec.is_valid() ||
+ m_remote_mirror_snap_ns.group_snap_id.empty()) {
+ std::unique_lock locker{m_lock};
+ update_non_primary_snapshot(true);
+ return;
+ }
+
+ dout(10) << "image_id=" << m_state_builder->local_image_ctx->id
+ << ", remote_group_snap_id=" << m_remote_mirror_snap_ns.group_snap_id
+ << ", local_image_snap_id=" << m_local_snap_id_end << dendl;
+
+ m_replayer_listener->notify_group_snap_image_complete(
+ m_state_builder->local_image_ctx->md_ctx.get_id(),
+ m_state_builder->local_image_ctx->id,
+ m_remote_mirror_snap_ns.group_snap_id,
+ m_local_snap_id_end);
+
std::unique_lock locker{m_lock};
update_non_primary_snapshot(true);
}
#include "common/ceph_mutex.h"
#include "common/AsyncOpTracker.h"
#include "cls/rbd/cls_rbd_types.h"
-#include "include/rados/librados.hpp"
#include "librbd/mirror/snapshot/Types.h"
#include "tools/rbd_mirror/image_replayer/TimeRollingMean.h"
#include <boost/accumulators/accumulators.hpp>
return m_image_spec;
}
+ void prune_snapshot(uint64_t snap_id) {
+ std::unique_lock locker(m_lock);
+ m_prune_snap_ids.insert(snap_id);
+ }
+
private:
/**
* @verbatim
* REFRESH_REMOTE_IMAGE |
* | |
* | (unused non-primary snapshot) |
- * |\--------------> UNLINK_GROUP_SNAPSHOT |
- * | | (skip if no group) |
- * | v |
- * | PRUNE_NON_PRIMARY_SNAPSHOT---/|
+ * |\--------------> PRUNE_NON_PRIMARY_SNAPSHOT---/|
* | |
* | (interrupted sync) |
* |\--------------> GET_LOCAL_IMAGE_STATE ------\ |
utime_t m_snapshot_replay_start;
uint32_t m_pending_snapshots = 0;
+ std::set<uint64_t> m_prune_snap_ids;
bool m_remote_image_updated = false;
bool m_updating_sync_point = false;
PerfCounters *m_perf_counters = nullptr;
- uint64_t m_prune_snap_id = CEPH_NOSNAP;
- librados::IoCtx m_group_io_ctx;
-
void load_local_image_meta();
void handle_load_local_image_meta(int r);
void scan_local_mirror_snapshots(std::unique_lock<ceph::mutex>* locker);
void scan_remote_mirror_snapshots(std::unique_lock<ceph::mutex>* locker);
- void unlink_group_snapshot();
- void handle_unlink_group_snapshot(int r);
-
- void prune_non_primary_snapshot();
+ void prune_non_primary_snapshot(uint64_t snap_id);
void handle_prune_non_primary_snapshot(int r);
void copy_snapshots();
void get_local_image_state();
void handle_get_local_image_state(int r);
- void refresh_remote_group_snapshot_list();
- void handle_refresh_remote_group_snapshot_list(int r);
-
- void create_group_snap_start();
- void handle_create_group_snap_start(int r);
-
void create_non_primary_snapshot();
void handle_create_non_primary_snapshot(int r);
- void create_group_snap_finish();
- void handle_create_group_snap_finish(int r);
-
void update_mirror_image_state();
void handle_update_mirror_image_state(int r);
void update_non_primary_snapshot(bool complete);
void handle_update_non_primary_snapshot(bool complete, int r);
+ void notify_group_snap_image_complete();
+
void notify_image_update();
void handle_notify_image_update(int r);
projects / ceph.git / commitdiff