static CreateImageRequest* create(Threads<librbd::MockTestImageCtx>* threads,
librados::IoCtx &local_io_ctx,
+ GroupCtx *local_group_ctx,
const std::string &global_image_id,
const std::string &remote_mirror_uuid,
const std::string &local_image_name,
const std::string& global_image_id,
Context* on_finish) {
return new MockCreateLocalImageRequest(
- &mock_threads, m_local_io_ctx, m_mock_remote_image_ctx,
+ &mock_threads, m_local_io_ctx, nullptr, m_mock_remote_image_ctx,
global_image_id, nullptr, nullptr, &mock_state_builder,
on_finish);
}
static CreateImageRequest* create(Threads<librbd::MockTestImageCtx>* threads,
librados::IoCtx &local_io_ctx,
+ GroupCtx *local_group_ctx,
const std::string &global_image_id,
const std::string &remote_mirror_uuid,
const std::string &local_image_name,
const std::string& global_image_id,
Context* on_finish) {
return new MockCreateLocalImageRequest(
- &mock_threads, m_local_io_ctx, m_mock_remote_image_ctx,
+ &mock_threads, m_local_io_ctx, nullptr, m_mock_remote_image_ctx,
global_image_id, &m_pool_meta_cache, nullptr, &mock_state_builder,
on_finish);
}
static ImageSync* create(
Threads<librbd::MockTestImageCtx>* threads,
+ GroupCtx *local_group_ctx,
librbd::MockTestImageCtx *local_image_ctx,
librbd::MockTestImageCtx *remote_image_ctx,
const std::string &local_mirror_uuid,
MOCK_CONST_METHOD0(replay_requires_remote_image, bool());
MOCK_METHOD1(close_remote_image, void(Context*));
- MOCK_METHOD6(create_local_image_request,
+ MOCK_METHOD7(create_local_image_request,
BaseRequest*(Threads<librbd::MockTestImageCtx>*,
librados::IoCtx&,
+ GroupCtx*,
const std::string&,
PoolMetaCache*,
ProgressContext*,
void expect_create_local_image(MockStateBuilder& mock_state_builder,
const std::string& local_image_id, int r) {
EXPECT_CALL(mock_state_builder,
- create_local_image_request(_, _, _, _, _, _))
- .WillOnce(WithArg<5>(
+ create_local_image_request(_, _, _, _, _, _, _))
+ .WillOnce(WithArg<6>(
Invoke([&mock_state_builder, local_image_id, r](Context* ctx) {
if (r >= 0) {
mock_state_builder.local_image_id = local_image_id;
return new MockBootstrapRequest(mock_threads,
m_local_io_ctx,
m_remote_io_ctx,
+ nullptr,
mock_instance_watcher,
global_image_id,
local_mirror_uuid,
const std::string &local_image_id,
librbd::MockTestImageCtx &mock_remote_image_ctx,
Context *on_finish) {
- return new MockCreateImageRequest(mock_threads, m_local_io_ctx,
+ return new MockCreateImageRequest(mock_threads, m_local_io_ctx, nullptr,
global_image_id, remote_mirror_uuid,
local_image_name, local_image_id,
&mock_remote_image_ctx,
}
void create_replayer() {
- m_replayer = new ImageReplayer<>(m_local_ioctx, m_local_mirror_uuid,
- m_global_image_id, m_threads.get(),
- m_instance_watcher, m_local_status_updater,
- nullptr, &m_pool_meta_cache);
+ m_replayer = new ImageReplayer<>(m_local_ioctx, nullptr,
+ m_local_mirror_uuid, m_global_image_id,
+ m_threads.get(), m_instance_watcher,
+ m_local_status_updater, nullptr,
+ &m_pool_meta_cache);
m_replayer->add_peer({"peer uuid", m_remote_ioctx,
{m_remote_mirror_uuid, "remote mirror peer uuid"},
nullptr});
}
ImageSync<> *create_request(Context *ctx) {
- return new ImageSync<>(m_threads, m_local_image_ctx, m_remote_image_ctx,
- "mirror-uuid", m_sync_point_handler,
- m_instance_watcher, nullptr, ctx);
+ return new ImageSync<>(m_threads, nullptr, m_local_image_ctx,
+ m_remote_image_ctx, "mirror-uuid",
+ m_sync_point_handler, m_instance_watcher, nullptr,
+ ctx);
}
librbd::ImageCtx *m_remote_image_ctx;
template <>
struct MirrorStatusUpdater<librbd::MockTestImageCtx> {
- MOCK_METHOD1(exists, bool(const std::string&));
+ MOCK_METHOD1(mirror_image_exists, bool(const std::string&));
MOCK_METHOD3(set_mirror_image_status,
void(const std::string&, const cls::rbd::MirrorImageSiteStatus&,
bool));
Context*));
MOCK_METHOD3(remove_mirror_image_status, void(const std::string&, bool,
Context*));
+ MOCK_METHOD1(mirror_group_exists, bool(const std::string&));
+ MOCK_METHOD3(set_mirror_group_status,
+ void(const std::string&, const cls::rbd::MirrorGroupSiteStatus&,
+ bool));
+ MOCK_METHOD2(remove_mirror_group_status, void(const std::string&, Context*));
+ MOCK_METHOD3(mirror_group_image_exists,
+ bool(const std::string&, int64_t, const std::string&));
+ MOCK_METHOD5(set_mirror_group_image_status,
+ void(const std::string&, int64_t, const std::string&,
+ const cls::rbd::MirrorImageSiteStatus&, bool));
+ MOCK_METHOD4(remove_mirror_group_image_status,
+ void(const std::string&, int64_t, const std::string&, Context*));
};
template <>
Threads<librbd::MockTestImageCtx>* threads,
librados::IoCtx &local_io_ctx,
librados::IoCtx& remote_io_ctx,
+ rbd::mirror::GroupCtx *local_group_ctx,
rbd::mirror::InstanceWatcher<librbd::MockTestImageCtx> *instance_watcher,
const std::string &global_image_id,
const std::string &local_mirror_uuid,
}
void expect_mirror_image_status_exists(bool exists) {
- EXPECT_CALL(m_local_status_updater, exists(_))
+ EXPECT_CALL(m_local_status_updater, mirror_image_exists(_))
.WillOnce(Return(exists));
- EXPECT_CALL(m_remote_status_updater, exists(_))
+ EXPECT_CALL(m_remote_status_updater, mirror_image_exists(_))
.WillOnce(Return(exists));
}
void create_image_replayer(MockThreads &mock_threads) {
m_image_replayer = new MockImageReplayer(
- m_local_io_ctx, "local_mirror_uuid", "global image id",
+ m_local_io_ctx, nullptr, "local_mirror_uuid", "global image id",
&mock_threads, &m_instance_watcher, &m_local_status_updater, nullptr,
nullptr);
m_image_replayer->add_peer({"peer_uuid", m_remote_io_ctx,
MockSyncPointHandler& mock_sync_point_handler,
MockInstanceWatcher &mock_instance_watcher,
Context *ctx) {
- return new MockImageSync(&mock_threads, &mock_local_image_ctx,
+ return new MockImageSync(&mock_threads, nullptr, &mock_local_image_ctx,
&mock_remote_image_ctx,
"mirror-uuid", &mock_sync_point_handler,
&mock_instance_watcher, nullptr, ctx);
MOCK_METHOD0(is_running, bool());
MOCK_METHOD0(is_stopped, bool());
MOCK_METHOD0(is_blocklisted, bool());
+ MOCK_METHOD0(needs_restart, bool());
MOCK_CONST_METHOD0(is_finished, bool());
MOCK_METHOD1(set_finished, void(bool));
std::string global_image_id;
static ImageReplayer *create(
- librados::IoCtx &local_io_ctx, const std::string &local_mirror_uuid,
+ librados::IoCtx &local_io_ctx,
+ GroupCtx *local_group_ctx,
+ const std::string &local_mirror_uuid,
const std::string &global_image_id,
Threads<librbd::MockTestImageCtx> *threads,
InstanceWatcher<librbd::MockTestImageCtx> *instance_watcher,
Threads.cc
Throttler.cc
Types.cc
+ group_replayer/BootstrapRequest.cc
image_deleter/SnapshotPurgeRequest.cc
image_deleter/TrashMoveRequest.cc
image_deleter/TrashRemoveRequest.cc
#include "common/debug.h"
#include "common/errno.h"
#include "librbd/ImageCtx.h"
+#include "librbd/Utils.h"
+#include "librbd/asio/ContextWQ.h"
+#include "tools/rbd_mirror/ImageReplayer.h"
+#include "tools/rbd_mirror/MirrorStatusUpdater.h"
+#include "tools/rbd_mirror/Threads.h"
+#include "tools/rbd_mirror/group_replayer/BootstrapRequest.h"
#include "tools/rbd_mirror/image_replayer/Utils.h"
#include "GroupReplayer.h"
namespace rbd {
namespace mirror {
+using librbd::util::create_context_callback;
+using librbd::util::unique_lock_name;
+
namespace {
+std::string calc_ind_mirror_snap_name(uint64_t pool_id,
+ const std::string &group_id,
+ const std::string &snap_id)
+{
+ std::stringstream ind_snap_name_stream;
+ ind_snap_name_stream << ".mirror." << std::hex << pool_id << "_"
+ << group_id << "_" << snap_id;
+ return ind_snap_name_stream.str();
+}
+
template <typename I>
class GroupReplayerAdminSocketCommand {
public:
} // anonymous namespace
-template <typename I>
-std::ostream &operator<<(std::ostream &os,
- const typename GroupReplayer<I>::State &state) {
- switch (state) {
- case GroupReplayer<I>::STATE_STARTING:
- os << "Starting";
- break;
- case GroupReplayer<I>::STATE_REPLAYING:
- os << "Replaying";
- break;
- case GroupReplayer<I>::STATE_STOPPING:
- os << "Stopping";
- break;
- case GroupReplayer<I>::STATE_STOPPED:
- os << "Stopped";
- break;
- default:
- os << "Unknown (" << static_cast<uint32_t>(state) << ")";
- break;
- }
- return os;
-}
-
template <typename I>
std::ostream &operator<<(std::ostream &os, const GroupReplayer<I> &replayer) {
std::string nspace = replayer.get_namespace();
m_cache_manager_handler(cache_manager_handler),
m_pool_meta_cache(pool_meta_cache),
m_local_group_name(global_group_id),
- m_lock(ceph::make_mutex("rbd::mirror::GroupReplayer " +
- stringify(local_io_ctx.get_id()) + " " + global_group_id)) {
+ m_lock(ceph::make_mutex(unique_lock_name("GroupReplayer::m_lock", this))) {
// Register asok commands using a temporary "remote_pool_name/global_group_id"
// name. When the group name becomes known on start the asok commands will be
// re-registered using "remote_pool_name/remote_group_name" name.
unregister_admin_socket_hook();
ceph_assert(m_on_start_finish == nullptr);
ceph_assert(m_on_stop_finish == nullptr);
+ ceph_assert(m_bootstrap_request == nullptr);
}
+template <typename I>
+bool GroupReplayer<I>::needs_restart() const {
+ std::lock_guard locker{m_lock};
+
+ if (m_state != STATE_REPLAYING) {
+ return false;
+ }
+
+ for (auto &[_, image_replayer] : m_image_replayers) {
+ if (image_replayer->is_stopped()) {
+ return true;
+ }
+ }
+
+ return false;
+}
template <typename I>
image_replayer::HealthState GroupReplayer<I>::get_health_state() const {
m_last_r = 0;
m_state_desc.clear();
m_manual_stop = false;
+ ceph_assert(m_on_start_finish == nullptr);
+ std::swap(m_on_start_finish, on_finish);
}
}
return;
}
- // TODO
- on_finish->complete(0);
+ bootstrap_group();
}
template <typename I>
dout(10) << "on_finish=" << on_finish << ", manual=" << manual
<< ", restart=" << restart << dendl;
+ group_replayer::BootstrapRequest<I> *bootstrap_request = nullptr;
+ bool shut_down_replay = false;
bool running = true;
{
std::lock_guard locker{m_lock};
if (!is_stopped_()) {
if (m_state == STATE_STARTING) {
dout(10) << "canceling start" << dendl;
+ if (m_bootstrap_request != nullptr) {
+ bootstrap_request = m_bootstrap_request;
+ bootstrap_request->get();
+ }
} else {
dout(10) << "interrupting replay" << dendl;
+ shut_down_replay = true;
+ m_state = STATE_STOPPING;
}
+
+ ceph_assert(m_on_stop_finish == nullptr);
+ std::swap(m_on_stop_finish, on_finish);
+ m_stop_requested = true;
+ m_manual_stop = manual;
}
}
}
+ if (bootstrap_request != nullptr) {
+ dout(10) << "canceling bootstrap" << dendl;
+ bootstrap_request->cancel();
+ bootstrap_request->put();
+ }
+
if (!running) {
dout(20) << "not running" << dendl;
if (on_finish) {
return;
}
- // TODO
- on_finish->complete(0);
+ if (shut_down_replay) {
+ stop_image_replayers();
+ } else if (on_finish != nullptr) {
+ // XXXMG: clean up
+ {
+ std::lock_guard locker{m_lock};
+ m_stop_requested = false;
+ }
+ on_finish->complete(0);
+ }
}
template <typename I>
}
dout(10) << dendl;
- // TODO
+
+ for (auto &[_, image_replayer] : m_image_replayers) {
+ image_replayer->flush();
+ }
}
template <typename I>
f->open_object_section("group_replayer");
f->dump_string("name", m_group_spec);
- f->dump_stream("state") << m_state;
- f->close_section();
+ auto state = m_state;
+ if (m_local_group_ctx.primary && state == STATE_REPLAYING) { // XXXMG
+ state = STATE_STOPPED;
+ }
+ f->dump_string("state", state_to_string(state));
+ f->open_array_section("image_replayers");
+ for (auto &[_, image_replayer] : m_image_replayers) {
+ image_replayer->print_status(f);
+ }
+ f->close_section(); // image_replayers
+ f->close_section(); // group_replayer
+}
+
+template <typename I>
+void GroupReplayer<I>::bootstrap_group() {
+ dout(10) << dendl;
+
+ std::unique_lock locker{m_lock};
+ if (m_peers.empty()) {
+ locker.unlock();
+
+ dout(5) << "no peer clusters" << dendl;
+ finish_start(-ENOENT, "no peer clusters");
+ return;
+ }
+
+ // TODO need to support multiple remote groups
+ ceph_assert(!m_peers.empty());
+ m_remote_group_peer = *m_peers.begin();
+
+ if (finish_start_if_interrupted(m_lock)) {
+ return;
+ }
+
+ ceph_assert(m_image_replayers.empty());
+
+ auto ctx = create_context_callback<
+ GroupReplayer,
+ &GroupReplayer<I>::handle_bootstrap_group>(this);
+ auto request = group_replayer::BootstrapRequest<I>::create(
+ m_threads, m_local_io_ctx, m_remote_group_peer.io_ctx, m_global_group_id,
+ m_local_mirror_uuid, m_instance_watcher, m_local_status_updater,
+ m_remote_group_peer.mirror_status_updater, m_cache_manager_handler,
+ m_pool_meta_cache, &m_local_group_ctx, &m_image_replayers, ctx);
+
+ request->get();
+ m_bootstrap_request = request;
+ locker.unlock();
+
+ set_mirror_group_status_update(
+ cls::rbd::MIRROR_GROUP_STATUS_STATE_STARTING_REPLAY, "bootstrapping");
+ request->send();
+}
+
+template <typename I>
+void GroupReplayer<I>::handle_bootstrap_group(int r) {
+ dout(10) << "r=" << r << dendl;
+ {
+ std::lock_guard locker{m_lock};
+ m_bootstrap_request->put();
+ m_bootstrap_request = nullptr;
+ }
+
+ if (finish_start_if_interrupted()) {
+ return;
+ } else if (r == -ENOENT) {
+ finish_start(r, "group removed");
+ return;
+ } else if (r == -EREMOTEIO) {
+ finish_start(r, "remote group is non-primary");
+ return;
+ } else if (r == -EEXIST) {
+ finish_start(r, "split-brain detected");
+ return;
+ } else if (r < 0) {
+ finish_start(r, "bootstrap failed");
+ return;
+ }
+
+ {
+ std::lock_guard locker{m_lock};
+ m_local_group_name = m_local_group_ctx.name;
+ }
+ reregister_admin_socket_hook();
+
+ start_image_replayers();
+}
+
+template <typename I>
+void GroupReplayer<I>::start_image_replayers() {
+ dout(10) << m_image_replayers.size() << dendl;
+
+ set_mirror_group_status_update(
+ cls::rbd::MIRROR_GROUP_STATUS_STATE_STARTING_REPLAY, "starting replay");
+
+ auto ctx = create_context_callback<
+ GroupReplayer, &GroupReplayer<I>::handle_start_image_replayers>(this);
+ C_Gather *gather_ctx = new C_Gather(g_ceph_context, ctx);
+ {
+ std::lock_guard locker{m_lock};
+ for (auto &[_, image_replayer] : m_image_replayers) {
+ image_replayer->start(gather_ctx->new_sub(), false);
+ }
+ }
+ gather_ctx->activate();
+}
+
+template <typename I>
+void GroupReplayer<I>::handle_start_image_replayers(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ if (finish_start_if_interrupted()) {
+ return;
+ } else if (r < 0) {
+ finish_start(r, "");
+ return;
+ }
+
+ finish_start(0, "");
+}
+
+template <typename I>
+void GroupReplayer<I>::stop_image_replayers() {
+ dout(10) << m_image_replayers.size() << dendl;
+
+ set_mirror_group_status_update(
+ cls::rbd::MIRROR_GROUP_STATUS_STATE_STOPPING_REPLAY, "stopping");
+
+ auto ctx = create_context_callback<
+ GroupReplayer, &GroupReplayer<I>::handle_stop_image_replayers>(this);
+ C_Gather *gather_ctx = new C_Gather(g_ceph_context, ctx);
+ {
+ std::lock_guard locker{m_lock};
+ for (auto &[_, image_replayer] : m_image_replayers) {
+ image_replayer->stop(gather_ctx->new_sub(), false);
+ }
+ }
+ gather_ctx->activate();
+}
+
+template <typename I>
+void GroupReplayer<I>::handle_stop_image_replayers(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ Context *on_finish = nullptr;
+ {
+ std::lock_guard locker{m_lock};
+ ceph_assert(m_state == STATE_STOPPING);
+ m_stop_requested = false;
+ m_state = STATE_STOPPED;
+ std::swap(on_finish, m_on_stop_finish);
+
+ for (auto &[_, image_replayer] : m_image_replayers) {
+ delete image_replayer;
+ }
+ m_image_replayers.clear();
+ }
+
+ set_mirror_group_status_update(
+ cls::rbd::MIRROR_GROUP_STATUS_STATE_STOPPED, "stopped");
+
+ if (on_finish != nullptr) {
+ on_finish->complete(0);
+ }
+}
+
+template <typename I>
+bool GroupReplayer<I>::finish_start_if_interrupted() {
+ std::lock_guard locker{m_lock};
+
+ return finish_start_if_interrupted(m_lock);
+}
+
+template <typename I>
+bool GroupReplayer<I>::finish_start_if_interrupted(ceph::mutex &lock) {
+ ceph_assert(ceph_mutex_is_locked_by_me(m_lock));
+ ceph_assert(m_state == STATE_STARTING);
+ if (!m_stop_requested) {
+ return false;
+ }
+
+ finish_start(-ECANCELED, "");
+ return true;
+}
+
+template <typename I>
+void GroupReplayer<I>::finish_start(int r, const std::string &desc) {
+ dout(10) << "r=" << r << ", desc=" << desc << dendl;
+ Context *ctx = new LambdaContext(
+ [this, r, desc](int _r) {
+ Context *on_finish = nullptr;
+ {
+ std::lock_guard locker{m_lock};
+ ceph_assert(m_state == STATE_STARTING);
+ m_state = STATE_REPLAYING;
+ std::swap(m_on_start_finish, on_finish);
+ m_state_desc = desc;
+ if (r < 0) {
+ auto state = cls::rbd::MIRROR_GROUP_STATUS_STATE_STOPPED;
+ if (r == -ECANCELED) {
+ dout(10) << "start canceled" << dendl;
+ } else if (r == -ENOENT) {
+ dout(10) << "mirroring group removed" << dendl;
+ } else if (r == -EREMOTEIO) {
+ dout(10) << "mirroring group demoted" << dendl;
+ } else {
+ derr << "start failed: " << cpp_strerror(r) << dendl;
+ state = cls::rbd::MIRROR_GROUP_STATUS_STATE_ERROR;
+ }
+ on_finish = new LambdaContext(
+ [this, r, state, desc, on_finish](int) {
+ set_mirror_group_status_update(state, desc);
+
+ if (r == -ENOENT) {
+ set_finished(true);
+ }
+ if (on_finish != nullptr) {
+ on_finish->complete(r);
+ }
+ });
+ }
+ }
+
+ if (r < 0) {
+ stop(on_finish, false, false);
+ return;
+ }
+
+ if (m_local_group_ctx.primary) { // XXXMG
+ set_mirror_group_status_update(
+ cls::rbd::MIRROR_GROUP_STATUS_STATE_STOPPED,
+ "local group is primary");
+ } else {
+ set_mirror_group_status_update(
+ cls::rbd::MIRROR_GROUP_STATUS_STATE_REPLAYING, "replaying");
+ }
+
+ if (on_finish != nullptr) {
+ on_finish->complete(0);
+ }
+ });
+ m_threads->work_queue->queue(ctx, 0);
}
+
template <typename I>
void GroupReplayer<I>::register_admin_socket_hook() {
GroupReplayerAdminSocketHook<I> *asok_hook;
register_admin_socket_hook();
}
+template <typename I>
+void GroupReplayer<I>::set_mirror_group_status_update(
+ cls::rbd::MirrorGroupStatusState state, const std::string &desc) {
+ dout(20) << "state=" << state << ", description=" << desc << dendl;
+
+ cls::rbd::MirrorGroupSiteStatus local_status;
+ local_status.state = state;
+ local_status.description = desc;
+ local_status.up = true;
+
+ auto remote_status = local_status;
+
+ {
+ std::unique_lock locker{m_lock};
+ for (auto &[_, ir] : m_image_replayers) {
+ cls::rbd::MirrorImageSiteStatus mirror_image;
+ if (ir->is_running()) {
+ if (ir->is_replaying()) {
+ mirror_image.state = cls::rbd::MIRROR_IMAGE_STATUS_STATE_REPLAYING;
+ } else {
+ mirror_image.state = cls::rbd::MIRROR_IMAGE_STATUS_STATE_STARTING_REPLAY;
+ }
+ } else if (ir->is_stopped()) {
+ mirror_image.state = cls::rbd::MIRROR_IMAGE_STATUS_STATE_STOPPED;
+ } else {
+ mirror_image.state = cls::rbd::MIRROR_IMAGE_STATUS_STATE_STOPPING_REPLAY;
+ }
+ mirror_image.description = ir->get_state_description();
+
+ local_status.mirror_images[{ir->get_local_pool_id(),
+ ir->get_global_image_id()}] = mirror_image;
+ auto remote_pool_id = ir->get_remote_pool_id();
+ if (remote_pool_id >= 0) {
+ remote_status.mirror_images[{remote_pool_id,
+ ir->get_global_image_id()}] = mirror_image;
+ }
+ }
+ }
+
+ m_local_status_updater->set_mirror_group_status(m_global_group_id,
+ local_status, true);
+ if (m_remote_group_peer.mirror_status_updater != nullptr) {
+ m_remote_group_peer.mirror_status_updater->set_mirror_group_status(
+ m_global_group_id, remote_status, true);
+ }
+}
+
+template <typename I>
+void GroupReplayer<I>::create_mirror_snapshot_start(
+ const std::string &remote_group_snap_id, ImageReplayer<I> *image_replayer,
+ int64_t *local_group_pool_id, std::string *local_group_id,
+ std::string *local_group_snap_id, Context *on_finish) {
+ dout(20) << remote_group_snap_id << " " << image_replayer << dendl;
+
+ std::unique_lock locker{m_lock};
+ ceph_assert(m_pending_snap_create == false);
+
+ if (m_state != STATE_REPLAYING) {
+ derr << "not in replaying state" << dendl;
+ locker.unlock();
+ on_finish->complete(-ESTALE);
+ return;
+ }
+
+ if (m_remote_group_snap_id.empty()) {
+ ceph_assert(m_create_snap_requests.empty());
+ m_remote_group_snap_id = remote_group_snap_id;
+
+ // XXXMG: make the same name as the primary snapshot has
+ m_group_snap = {librbd::util::generate_uuid(m_local_io_ctx),
+ cls::rbd::MirrorGroupSnapshotNamespace{
+ cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY,
+ {}, m_remote_group_peer.uuid, m_remote_group_snap_id},
+ calc_ind_mirror_snap_name(
+ m_remote_group_peer.io_ctx.get_id(), m_remote_group_id,
+ m_remote_group_snap_id),
+ cls::rbd::GROUP_SNAPSHOT_STATE_INCOMPLETE};
+ }
+
+ ceph_assert(m_create_snap_requests.count(image_replayer) == 0);
+ m_create_snap_requests[image_replayer] = on_finish;
+
+ *local_group_pool_id = m_local_io_ctx.get_id();
+ *local_group_id = m_local_group_ctx.group_id;
+ *local_group_snap_id = m_group_snap.id;
+
+ if (m_remote_group_snap_id != remote_group_snap_id) {
+ dout(15) << "request snap_id " << remote_group_snap_id
+ << " does not match us " << m_remote_group_snap_id
+ << " -- restarting" << dendl;
+ auto create_snap_requests = m_create_snap_requests;
+ m_create_snap_requests.clear();
+ locker.unlock();
+ for (auto &[_, on_finish] : create_snap_requests) {
+ on_finish->complete(-EAGAIN);
+ }
+ return;
+ }
+
+ // XXXMG: m_image_replayers.size() will not always work
+ if (m_create_snap_requests.size() < m_image_replayers.size()) {
+ return;
+ }
+
+ m_pending_snap_create = true;
+
+ for (auto &[io_ctx, image_replayer] : m_image_replayers) {
+ m_group_snap.snaps.emplace_back(io_ctx.get_id(),
+ image_replayer->get_local_image_id(),
+ CEPH_NOSNAP);
+ ceph_assert(!m_group_snap.snaps.back().image_id.empty());
+ }
+
+ dout(20) << m_group_snap.id << " " << m_group_snap.name << dendl;
+
+ librados::ObjectWriteOperation op;
+ librbd::cls_client::group_snap_set(&op, m_group_snap);
+ auto comp = create_rados_callback<
+ GroupReplayer<I>,
+ &GroupReplayer<I>::handle_create_mirror_snapshot_start>(this);
+
+ int r = m_local_io_ctx.aio_operate(
+ librbd::util::group_header_name(m_local_group_ctx.group_id), comp, &op);
+ ceph_assert(r == 0);
+ comp->release();
+}
+
+template <typename I>
+void GroupReplayer<I>::handle_create_mirror_snapshot_start(int r) {
+ dout(20) << m_remote_group_snap_id << " r=" << r << dendl;
+
+ std::unique_lock locker{m_lock};
+ ceph_assert(m_pending_snap_create == true);
+
+ auto create_snap_requests = m_create_snap_requests;
+ m_create_snap_requests.clear();
+ locker.unlock();
+
+ for (auto &[_, on_finish] : create_snap_requests) {
+ on_finish->complete(r);
+ }
+}
+
+template <typename I>
+void GroupReplayer<I>::create_mirror_snapshot_finish(
+ const std::string &remote_group_snap_id, ImageReplayer<I> *image_replayer,
+ uint64_t snap_id, Context *on_finish) {
+ dout(20) << remote_group_snap_id << " " << image_replayer << " snap_id="
+ << snap_id << dendl;
+
+ std::lock_guard locker{m_lock};
+ ceph_assert(m_pending_snap_create == true);
+ ceph_assert(m_state == STATE_REPLAYING || m_state == STATE_STOPPING);
+ ceph_assert(m_remote_group_snap_id == remote_group_snap_id);
+ ceph_assert(m_create_snap_requests.count(image_replayer) == 0);
+
+ m_create_snap_requests[image_replayer] = on_finish;
+
+ auto pool = image_replayer->get_local_pool_id();
+ auto image_id = image_replayer->get_local_image_id();
+ auto it = std::find_if(
+ m_group_snap.snaps.begin(), m_group_snap.snaps.end(),
+ [&pool, &image_id](const cls::rbd::ImageSnapshotSpec &s) {
+ return pool == s.pool && image_id == s.image_id;
+ });
+ ceph_assert(it != m_group_snap.snaps.end());
+ it->snap_id = snap_id;
+
+ if (m_create_snap_requests.size() < m_image_replayers.size()) {
+ return;
+ }
+
+ m_group_snap.state = cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE;
+
+ dout(20) << m_group_snap.id << " " << m_group_snap.name << dendl;
+
+ librados::ObjectWriteOperation op;
+ librbd::cls_client::group_snap_set(&op, m_group_snap);
+ auto comp = create_rados_callback<
+ GroupReplayer<I>,
+ &GroupReplayer<I>::handle_create_mirror_snapshot_finish>(this);
+
+ int r = m_local_io_ctx.aio_operate(
+ librbd::util::group_header_name(m_local_group_ctx.group_id), comp, &op);
+ ceph_assert(r == 0);
+ comp->release();
+}
+
+template <typename I>
+void GroupReplayer<I>::handle_create_mirror_snapshot_finish(int r) {
+ dout(20) << m_remote_group_snap_id << " r=" << r << dendl;
+
+ std::unique_lock locker{m_lock};
+ ceph_assert(m_pending_snap_create == true);
+ m_pending_snap_create = false;
+ m_remote_group_snap_id.clear();
+ bool stopping = (m_state == STATE_STOPPING);
+ auto create_snap_requests = m_create_snap_requests;
+ m_create_snap_requests.clear();
+ locker.unlock();
+
+ for (auto &[_, on_finish] : create_snap_requests) {
+ on_finish->complete(r);
+ }
+
+ if (stopping) {
+ stop_image_replayers();
+ }
+}
+
} // namespace mirror
} // namespace rbd
#ifndef CEPH_RBD_MIRROR_GROUP_REPLAYER_H
#define CEPH_RBD_MIRROR_GROUP_REPLAYER_H
+#include "cls/rbd/cls_rbd_types.h"
#include "common/ceph_mutex.h"
#include "include/rados/librados.hpp"
#include "tools/rbd_mirror/Types.h"
#include "tools/rbd_mirror/image_replayer/Types.h"
#include <boost/optional.hpp>
#include <string>
+#include <list>
class AdminSocketHook;
namespace rbd {
namespace mirror {
+template <typename> class ImageReplayer;
template <typename> struct InstanceWatcher;
template <typename> struct MirrorStatusUpdater;
struct PoolMetaCache;
template <typename> struct Threads;
+namespace group_replayer {
+ template <typename> class BootstrapRequest;
+}
+
/**
* Replays changes from a remote cluster for a single group.
*/
GroupReplayer(const GroupReplayer&) = delete;
GroupReplayer& operator=(const GroupReplayer&) = delete;
- bool is_stopped() { std::lock_guard l{m_lock}; return is_stopped_(); }
- bool is_running() { std::lock_guard l{m_lock}; return is_running_(); }
- bool is_replaying() { std::lock_guard l{m_lock}; return is_replaying_(); }
+ bool is_stopped() const {
+ std::lock_guard l{m_lock};
+ return is_stopped_();
+ }
+ bool is_running() const {
+ std::lock_guard l{m_lock};
+ return is_running_();
+ }
+ bool is_replaying() const {
+ std::lock_guard l{m_lock};
+ return is_replaying_();
+ }
- std::string get_name() { std::lock_guard l{m_lock}; return m_group_spec; };
+ std::string get_name() const {
+ std::lock_guard l{m_lock};
+ return m_group_spec;
+ }
void set_state_description(int r, const std::string &desc);
// TODO temporary until policy handles release of group replayers
return (m_last_r == -EBLOCKLISTED);
}
+ bool needs_restart() const;
+
image_replayer::HealthState get_health_state() const;
void add_peer(const Peer<ImageCtxT>& peer);
* <starting> *
* | *
* v (error) *
+ * BOOTSTRAP_GROUP * * * * * * * * * * * * * * * * * * * *
+ * | *
+ * v (error) *
+ * START_IMAGE_REPLAYERS * * * * * * * * * * * * * * * * *
+ * |
+ * v
+ * REPLAYING
+ * |
+ * v
+ * STOP_IMAGE_REPLAYERS
+ * |
+ * v
* <stopped>
*
* @endverbatim
std::string m_global_group_id;
Threads<ImageCtxT> *m_threads;
InstanceWatcher<ImageCtxT> *m_instance_watcher;
- MirrorStatusUpdater<ImageCtxT>* m_local_status_updater;
+ MirrorStatusUpdater<ImageCtxT> *m_local_status_updater;
journal::CacheManagerHandler *m_cache_manager_handler;
PoolMetaCache* m_pool_meta_cache;
std::string m_local_group_name;
std::string m_group_spec;
+ GroupCtx m_local_group_ctx;
Peers m_peers;
+ Peer<ImageCtxT> m_remote_group_peer;
mutable ceph::mutex m_lock;
State m_state = STATE_STOPPED;
AdminSocketHook *m_asok_hook = nullptr;
+ group_replayer::BootstrapRequest<ImageCtxT> *m_bootstrap_request = nullptr;
+ std::list<std::pair<librados::IoCtx, ImageReplayer<ImageCtxT> *>> m_image_replayers;
+
+ static std::string state_to_string(const State &state) {
+ switch (state) {
+ case STATE_STARTING:
+ return "Starting";
+ case STATE_REPLAYING:
+ return "Replaying";
+ case STATE_STOPPING:
+ return "Stopping";
+ case STATE_STOPPED:
+ return "Stopped";
+ default:
+ return "Unknown (" + stringify(static_cast<uint32_t>(state)) + ")";
+ }
+ }
+
bool is_stopped_() const {
return m_state == STATE_STOPPED;
}
return (m_state == STATE_REPLAYING);
}
+ void bootstrap_group();
+ void handle_bootstrap_group(int r);
+
+ void start_image_replayers();
+ void handle_start_image_replayers(int r);
+
+ bool finish_start_if_interrupted();
+ bool finish_start_if_interrupted(ceph::mutex &lock);
+ void finish_start(int r, const std::string &desc);
+
+ void stop_image_replayers();
+ void handle_stop_image_replayers(int r);
+
void register_admin_socket_hook();
void unregister_admin_socket_hook();
void reregister_admin_socket_hook();
+
+ void set_mirror_group_status_update(cls::rbd::MirrorGroupStatusState state,
+ const std::string &desc);
};
} // namespace mirror
m_threads->work_queue,
new C_NotifyInstance(this, global_id, true)));
} else if (update.entity.type == MIRROR_ENTITY_TYPE_GROUP) {
- // TODO
+ m_listener.acquire_group(
+ update.entity.global_id, update.instance_id,
+ create_async_context_callback(
+ m_threads->work_queue,
+ new C_NotifyInstance(this, global_id, true)));
} else {
ceph_abort_msgf("invalid mirror entity type: %d",
(int)update.entity.type);
m_threads->work_queue,
new C_NotifyInstance(this, global_id, true)));
} else if (update.entity.type == MIRROR_ENTITY_TYPE_GROUP) {
- // TODO
+ m_listener.release_group(
+ update.entity.global_id, update.instance_id,
+ create_async_context_callback(
+ m_threads->work_queue,
+ new C_NotifyInstance(this, global_id, true)));
} else {
ceph_abort_msgf("invalid mirror entity type: %d",
(int)update.entity.type);
m_threads->work_queue,
new C_NotifyInstance(this, global_id, false)));
} else if (update.entity.type == MIRROR_ENTITY_TYPE_GROUP) {
- // TODO
+ m_listener.remove_group(
+ mirror_uuid, update.entity.global_id, update.instance_id,
+ create_async_context_callback(
+ m_threads->work_queue,
+ new C_NotifyInstance(this, global_id, false)));
} else {
ceph_abort_msgf("invalid mirror entity type: %d",
(int)update.entity.type);
}
if (entity_mapped && peer_removed && !mirror_uuid.empty()) {
- // peer image has been deleted
+ // peer entity has been deleted or local non-image entity needs restart
to_remove.emplace_back(entity, info.instance_id);
}
template <typename I>
ImageReplayer<I>::ImageReplayer(
- librados::IoCtx &local_io_ctx, const std::string &local_mirror_uuid,
+ librados::IoCtx &local_io_ctx, GroupCtx *local_group_ctx,
+ const std::string &local_mirror_uuid,
const std::string &global_image_id, Threads<I> *threads,
InstanceWatcher<I> *instance_watcher,
MirrorStatusUpdater<I>* local_status_updater,
journal::CacheManagerHandler *cache_manager_handler,
PoolMetaCache* pool_meta_cache) :
- m_local_io_ctx(local_io_ctx), m_local_mirror_uuid(local_mirror_uuid),
- m_global_image_id(global_image_id), m_threads(threads),
- m_instance_watcher(instance_watcher),
+ m_local_io_ctx(local_io_ctx), m_local_group_ctx(local_group_ctx),
+ m_local_mirror_uuid(local_mirror_uuid), m_global_image_id(global_image_id),
+ m_threads(threads), m_instance_watcher(instance_watcher),
m_local_status_updater(local_status_updater),
m_cache_manager_handler(cache_manager_handler),
m_pool_meta_cache(pool_meta_cache),
auto ctx = create_context_callback<
ImageReplayer, &ImageReplayer<I>::handle_bootstrap>(this);
auto request = image_replayer::BootstrapRequest<I>::create(
- m_threads, m_local_io_ctx, m_remote_image_peer.io_ctx, m_instance_watcher,
- m_global_image_id, m_local_mirror_uuid,
+ m_threads, m_local_io_ctx, m_remote_image_peer.io_ctx, m_local_group_ctx,
+ m_instance_watcher, m_global_image_id, m_local_mirror_uuid,
m_remote_image_peer.remote_pool_meta, m_cache_manager_handler,
m_pool_meta_cache, &m_progress_cxt, &m_state_builder, &m_resync_requested,
ctx);
}
dout(15) << "status=" << status << dendl;
- m_local_status_updater->set_mirror_image_status(m_global_image_id, status,
- force);
- if (m_remote_image_peer.mirror_status_updater != nullptr) {
- m_remote_image_peer.mirror_status_updater->set_mirror_image_status(
- m_global_image_id, status, force);
+ if (m_local_group_ctx != nullptr) {
+ m_local_status_updater->set_mirror_group_image_status(
+ m_local_group_ctx->global_group_id, m_local_io_ctx.get_id(),
+ m_global_image_id, status, force);
+ if (m_remote_image_peer.mirror_status_updater != nullptr) {
+ m_remote_image_peer.mirror_status_updater->set_mirror_group_image_status(
+ m_local_group_ctx->global_group_id, m_remote_image_peer.io_ctx.get_id(),
+ m_global_image_id, status, force);
+ }
+ } else {
+ m_local_status_updater->set_mirror_image_status(m_global_image_id, status,
+ force);
+ if (m_remote_image_peer.mirror_status_updater != nullptr) {
+ m_remote_image_peer.mirror_status_updater->set_mirror_image_status(
+ m_global_image_id, status, force);
+ }
}
m_in_flight_op_tracker.finish_op();
return;
}
+ if (m_local_group_ctx != nullptr) {
+ if (m_local_status_updater->mirror_group_image_exists(
+ m_local_group_ctx->global_group_id, m_local_io_ctx.get_id(),
+ m_global_image_id)) {
+ dout(15) << "removing local mirror group image status" << dendl;
+ auto ctx = new LambdaContext([this, r](int) {
+ handle_shut_down(r);
+ });
+ m_local_status_updater->remove_mirror_group_image_status(
+ m_local_group_ctx->global_group_id, m_local_io_ctx.get_id(),
+ m_global_image_id, ctx);
+ return;
+ }
+
+ if (m_remote_image_peer.mirror_status_updater != nullptr &&
+ m_remote_image_peer.mirror_status_updater->mirror_group_image_exists(
+ m_local_group_ctx->global_group_id,
+ m_remote_image_peer.io_ctx.get_id(), m_global_image_id)) {
+ dout(15) << "removing remote mirror group image status" << dendl;
+ auto ctx = new LambdaContext([this, r](int) {
+ handle_shut_down(r);
+ });
+ m_remote_image_peer.mirror_status_updater->remove_mirror_group_image_status(
+ m_local_group_ctx->global_group_id,
+ m_remote_image_peer.io_ctx.get_id(), m_global_image_id, ctx);
+ return;
+ }
+ } else {
+ if (m_local_status_updater->mirror_image_exists(m_global_image_id)) {
+ dout(15) << "removing local mirror image status" << dendl;
+ auto ctx = new LambdaContext([this, r](int) {
+ handle_shut_down(r);
+ });
+ m_local_status_updater->remove_mirror_image_status(m_global_image_id,
+ true, ctx);
+ return;
+ }
+
+ if (m_remote_image_peer.mirror_status_updater != nullptr &&
+ m_remote_image_peer.mirror_status_updater->mirror_image_exists(
+ m_global_image_id)) {
+ dout(15) << "removing remote mirror image status" << dendl;
+ auto ctx = new LambdaContext([this, r](int) {
+ handle_shut_down(r);
+ });
+ m_remote_image_peer.mirror_status_updater->remove_mirror_image_status(
+ m_global_image_id, true, ctx);
+ return;
+ }
+ }
+
if (m_state_builder != nullptr) {
m_state_builder->destroy();
m_state_builder = nullptr;
remove_image_status_remote(force, on_finish);
});
- if (m_local_status_updater->exists(m_global_image_id)) {
+ if (m_local_status_updater->mirror_image_exists(m_global_image_id)) {
dout(15) << "removing local mirror image status" << dendl;
if (force) {
m_local_status_updater->remove_mirror_image_status(
void ImageReplayer<I>::remove_image_status_remote(bool force, Context *on_finish)
{
if (m_remote_image_peer.mirror_status_updater != nullptr &&
- m_remote_image_peer.mirror_status_updater->exists(m_global_image_id)) {
+ m_remote_image_peer.mirror_status_updater->mirror_image_exists(m_global_image_id)) {
dout(15) << "removing remote mirror image status" << dendl;
if (force) {
m_remote_image_peer.mirror_status_updater->remove_mirror_image_status(
class ImageReplayer {
public:
static ImageReplayer *create(
- librados::IoCtx &local_io_ctx, const std::string &local_mirror_uuid,
- const std::string &global_image_id, Threads<ImageCtxT> *threads,
- InstanceWatcher<ImageCtxT> *instance_watcher,
+ librados::IoCtx &local_io_ctx, GroupCtx *local_group_ctx,
+ const std::string &local_mirror_uuid, const std::string &global_image_id,
+ Threads<ImageCtxT> *threads, InstanceWatcher<ImageCtxT> *instance_watcher,
MirrorStatusUpdater<ImageCtxT>* local_status_updater,
journal::CacheManagerHandler *cache_manager_handler,
PoolMetaCache* pool_meta_cache) {
- return new ImageReplayer(local_io_ctx, local_mirror_uuid, global_image_id,
- threads, instance_watcher, local_status_updater,
- cache_manager_handler, pool_meta_cache);
+ return new ImageReplayer(local_io_ctx, local_group_ctx, local_mirror_uuid,
+ global_image_id, threads, instance_watcher,
+ local_status_updater, cache_manager_handler,
+ pool_meta_cache);
}
void destroy() {
delete this;
}
ImageReplayer(librados::IoCtx &local_io_ctx,
+ GroupCtx *local_group_ctx,
const std::string &local_mirror_uuid,
const std::string &global_image_id,
Threads<ImageCtxT> *threads,
ImageReplayer(const ImageReplayer&) = delete;
ImageReplayer& operator=(const ImageReplayer&) = delete;
- bool is_stopped() { std::lock_guard l{m_lock}; return is_stopped_(); }
- bool is_running() { std::lock_guard l{m_lock}; return is_running_(); }
- bool is_replaying() { std::lock_guard l{m_lock}; return is_replaying_(); }
+ inline bool is_stopped() const {
+ std::lock_guard l{m_lock};
+ return is_stopped_();
+ }
+ inline bool is_running() const {
+ std::lock_guard l{m_lock};
+ return is_running_();
+ }
+ inline bool is_replaying() const {
+ std::lock_guard l{m_lock};
+ return is_replaying_();
+ }
- std::string get_name() { std::lock_guard l{m_lock}; return m_image_spec; };
+ inline std::string get_name() const {
+ std::lock_guard l{m_lock};
+ return m_image_spec;
+ };
+ inline std::string get_state_description() const {
+ std::lock_guard l{m_lock};
+ return m_state_desc;
+ }
void set_state_description(int r, const std::string &desc);
// TODO temporary until policy handles release of image replayers
inline int64_t get_local_pool_id() const {
return m_local_io_ctx.get_id();
}
+ inline int64_t get_remote_pool_id() const {
+ std::unique_lock locker{m_lock};
+ if (m_remote_image_peer.io_ctx.is_valid()) {
+ return m_remote_image_peer.io_ctx.get_id();
+ }
+ if (!m_peers.empty()) {
+ return m_peers.begin()->io_ctx.get_id();
+ }
+ return -1;
+ }
inline const std::string& get_global_image_id() const {
return m_global_image_id;
}
};
librados::IoCtx &m_local_io_ctx;
+ GroupCtx *m_local_group_ctx;
std::string m_local_mirror_uuid;
std::string m_global_image_id;
Threads<ImageCtxT> *m_threads;
#include "librbd/asio/ContextWQ.h"
#include "librbd/deep_copy/Handler.h"
#include "tools/rbd_mirror/Threads.h"
+#include "tools/rbd_mirror/Types.h"
#include "tools/rbd_mirror/image_sync/SyncPointCreateRequest.h"
#include "tools/rbd_mirror/image_sync/SyncPointPruneRequest.h"
#include "tools/rbd_mirror/image_sync/Types.h"
template <typename I>
ImageSync<I>::ImageSync(
Threads<I>* threads,
+ GroupCtx *local_group_ctx,
I *local_image_ctx,
I *remote_image_ctx,
const std::string &local_mirror_uuid,
m_update_sync_point_interval(
m_local_image_ctx->cct->_conf.template get_val<double>(
"rbd_mirror_sync_point_update_age")) {
+ if (local_group_ctx == nullptr) {
+ m_sync_id = m_local_image_ctx->id;
+ } else {
+ m_sync_id = stringify(m_local_image_ctx->md_ctx.get_id()) + ":" +
+ m_local_image_ctx->id;
+ }
}
template <typename I>
m_canceled = true;
- if (m_instance_watcher->cancel_sync_request(m_local_image_ctx->id)) {
+ if (m_instance_watcher->cancel_sync_request(m_sync_id)) {
return;
}
Context *ctx = create_async_context_callback(
m_threads->work_queue, create_context_callback<
ImageSync<I>, &ImageSync<I>::handle_notify_sync_request>(this));
- m_instance_watcher->notify_sync_request(m_local_image_ctx->id, ctx);
+ m_instance_watcher->notify_sync_request(m_sync_id, ctx);
m_lock.unlock();
}
void ImageSync<I>::finish(int r) {
dout(20) << ": r=" << r << dendl;
- m_instance_watcher->notify_sync_complete(m_local_image_ctx->id);
+ m_instance_watcher->notify_sync_complete(m_sync_id);
CancelableRequest::finish(r);
}
class ProgressContext;
template <typename> class InstanceWatcher;
template <typename> class Threads;
+struct GroupCtx;
namespace image_sync { struct SyncPointHandler; }
public:
static ImageSync* create(
Threads<ImageCtxT>* threads,
+ GroupCtx *local_group_ctx,
ImageCtxT *local_image_ctx,
ImageCtxT *remote_image_ctx,
const std::string &local_mirror_uuid,
InstanceWatcher<ImageCtxT> *instance_watcher,
ProgressContext *progress_ctx,
Context *on_finish) {
- return new ImageSync(threads, local_image_ctx, remote_image_ctx,
- local_mirror_uuid, sync_point_handler,
- instance_watcher, progress_ctx, on_finish);
+ return new ImageSync(threads, local_group_ctx, local_image_ctx,
+ remote_image_ctx, local_mirror_uuid,
+ sync_point_handler, instance_watcher, progress_ctx,
+ on_finish);
}
ImageSync(
Threads<ImageCtxT>* threads,
+ GroupCtx *local_group_ctx,
ImageCtxT *local_image_ctx,
ImageCtxT *remote_image_ctx,
const std::string &local_mirror_uuid,
ceph::mutex m_lock;
bool m_canceled = false;
+ std::string m_sync_id;
librbd::DeepCopyRequest<ImageCtxT> *m_image_copy_request = nullptr;
ImageCopyProgressHandler *m_image_copy_prog_handler = nullptr;
Context *ctx = new LambdaContext(
[this] (int r) {
+ cancel_group_state_check_task();
cancel_image_state_check_task();
wait_for_ops();
});
std::lock_guard locker{m_lock};
C_Gather *gather_ctx = new C_Gather(g_ceph_context, on_finish);
+ dout(10) << "group_replayers: { " << m_group_replayers << " }" << dendl;
for (auto it = m_group_replayers.begin(); it != m_group_replayers.end();
it = m_group_replayers.erase(it)) {
auto group_replayer = it->second;
auto it = m_image_replayers.find(global_image_id);
if (it == m_image_replayers.end()) {
auto image_replayer = ImageReplayer<I>::create(
- m_local_io_ctx, m_local_mirror_uuid, global_image_id,
+ m_local_io_ctx, nullptr, m_local_mirror_uuid, global_image_id,
m_threads, instance_watcher, m_local_status_updater,
m_cache_manager_handler, m_pool_meta_cache);
std::lock_guard locker{m_lock};
ceph_assert(m_on_shut_down == nullptr);
+ dout(10) << "group_replayers: { " << m_group_replayers << " }" << dendl;
auto it = m_group_replayers.find(global_group_id);
if (it == m_group_replayers.end()) {
dout(5) << global_group_id << ": not found" << dendl;
}
m_image_replayers.clear();
- ceph_assert(m_on_shut_down != nullptr);
std::swap(on_finish, m_on_shut_down);
}
- on_finish->complete(r);
+ if (on_finish) {
+ on_finish->complete(r);
+ }
}
template <typename I>
template <typename I>
void InstanceReplayer<I>::start_group_replayer(
GroupReplayer<I> *group_replayer) {
+ dout(10) << dendl;
ceph_assert(ceph_mutex_is_locked(m_lock));
std::string global_group_id = group_replayer->get_global_group_id();
if (!group_replayer->is_stopped()) {
+ if (group_replayer->needs_restart()) {
+ stop_group_replayer(group_replayer, new C_TrackedOp(m_async_op_tracker,
+ nullptr));
+ }
return;
} else if (group_replayer->is_blocklisted()) {
derr << "global_group_id=" << global_group_id << ": blocklisted detected "
std::lock_guard locker{m_lock};
if (m_on_shut_down != nullptr) {
+ m_async_op_tracker.finish_op();
return;
}
}
m_group_replayers.clear();
- ceph_assert(m_on_shut_down != nullptr);
std::swap(on_finish, m_on_shut_down);
}
- on_finish->complete(r);
+ if (on_finish) {
+ on_finish->complete(r);
+ }
}
template <typename I>
ceph_assert(r == 0);
std::lock_guard locker{m_lock};
+ stop_group_replayers();
stop_image_replayers();
}
}
template <typename I>
-bool MirrorStatusUpdater<I>::exists(const std::string& global_image_id) {
- dout(15) << "global_image_id=" << global_image_id << dendl;
-
+bool MirrorStatusUpdater<I>::mirror_image_exists(
+ const std::string& global_image_id) const {
std::unique_lock locker(m_lock);
- return (m_global_image_status.count(global_image_id) > 0);
+ bool exists = (m_global_image_status.count(global_image_id) > 0);
+
+ dout(15) << "global_image_id=" << global_image_id << ": " << exists << dendl;
+
+ return exists;
}
template <typename I>
return true;
}
+template <typename I>
+bool MirrorStatusUpdater<I>::mirror_group_exists(
+ const std::string& global_group_id) const {
+ std::unique_lock locker(m_lock);
+ bool exists = (m_global_group_status.count(global_group_id) > 0);
+
+ dout(15) << "global_group_id=" << global_group_id << ": " << exists << dendl;
+
+ return exists;
+}
+
+template <typename I>
+void MirrorStatusUpdater<I>::set_mirror_group_status(
+ const std::string& global_group_id,
+ const cls::rbd::MirrorGroupSiteStatus& mirror_group_site_status,
+ bool immediate_update) {
+ dout(15) << "global_group_id=" << global_group_id << ", "
+ << "mirror_group_site_status=" << mirror_group_site_status << dendl;
+
+ std::unique_lock locker(m_lock);
+
+ m_global_group_status[global_group_id] = mirror_group_site_status;
+ if (immediate_update) {
+ m_update_global_group_ids.insert(global_group_id);
+ queue_update_task(std::move(locker));
+ }
+}
+
+template <typename I>
+void MirrorStatusUpdater<I>::remove_mirror_group_status(
+ const std::string& global_group_id, Context* on_finish) {
+ if (try_remove_mirror_group_status(global_group_id, on_finish)) {
+ m_threads->work_queue->queue(on_finish, 0);
+ }
+}
+
+template <typename I>
+bool MirrorStatusUpdater<I>::try_remove_mirror_group_status(
+ const std::string& global_group_id, Context* on_finish) {
+ dout(15) << "global_group_id=" << global_group_id << dendl;
+
+ std::unique_lock locker(m_lock);
+ if ((m_update_in_flight &&
+ m_updating_global_group_ids.count(global_group_id) > 0) ||
+ ((m_update_in_progress || m_update_requested) &&
+ m_update_global_group_ids.count(global_group_id) > 0)) {
+ // if update is scheduled/in-progress, wait for it to complete
+ on_finish = new LambdaContext(
+ [this, global_group_id, on_finish](int r) {
+ if (try_remove_mirror_group_status(global_group_id, on_finish)) {
+ on_finish->complete(0);
+ }
+ });
+ m_update_on_finish_ctxs.push_back(on_finish);
+ return false;
+ }
+
+ m_global_group_status.erase(global_group_id);
+ return true;
+}
+
+template <typename I>
+bool MirrorStatusUpdater<I>::mirror_group_image_exists(
+ const std::string& global_group_id, int64_t image_pool_id,
+ const std::string& global_image_id) const {
+ std::unique_lock locker(m_lock);
+ auto it = m_global_group_status.find(global_group_id);
+ bool exists = (it != m_global_group_status.end() &&
+ it->second.mirror_images.count({image_pool_id,
+ global_image_id}) > 0);
+
+ dout(15) << "global_group_id=" << global_group_id
+ << ", image_pool_id=" << image_pool_id
+ << ", global_image_id=" << global_image_id
+ << ": " << exists << dendl;
+ return exists;
+}
+
+template <typename I>
+void MirrorStatusUpdater<I>::set_mirror_group_image_status(
+ const std::string& global_group_id,
+ int64_t image_pool_id, const std::string& global_image_id,
+ const cls::rbd::MirrorImageSiteStatus& mirror_image_site_status,
+ bool immediate_update) {
+ dout(15) << "global_group_id=" << global_group_id << ", "
+ << "image_pool_id=" << image_pool_id << ", "
+ << "global_image_id=" << global_image_id << ", "
+ << "mirror_image_site_status=" << mirror_image_site_status << dendl;
+
+ std::unique_lock locker(m_lock);
+
+ auto it = m_global_group_status.find(global_group_id);
+ ceph_assert(it != m_global_group_status.end());
+
+ it->second.mirror_images[{image_pool_id, global_image_id}] =
+ mirror_image_site_status;
+ if (immediate_update) {
+ m_update_global_group_ids.insert(global_group_id);
+ queue_update_task(std::move(locker));
+ }
+}
+
+template <typename I>
+void MirrorStatusUpdater<I>::remove_mirror_group_image_status(
+ const std::string& global_group_id, int64_t image_pool_id,
+ const std::string& global_image_id, Context* on_finish) {
+ dout(15) << "global_group_id=" << global_group_id << ", "
+ << "image_pool_id=" << image_pool_id << ", "
+ << "global_image_id=" << global_image_id << dendl;
+
+ std::unique_lock locker(m_lock);
+
+ auto it = m_global_group_status.find(global_group_id);
+ ceph_assert(it != m_global_group_status.end());
+
+ it->second.mirror_images.erase({image_pool_id, global_image_id});
+ m_update_global_group_ids.insert(global_group_id);
+ m_update_on_finish_ctxs.push_back(on_finish);
+ queue_update_task(std::move(locker));
+}
+
template <typename I>
void MirrorStatusUpdater<I>::schedule_timer_task() {
dout(10) << dendl;
schedule_timer_task();
std::unique_lock locker(m_lock);
- for (auto& pair : m_global_image_status) {
- m_update_global_image_ids.insert(pair.first);
+ for (auto &[global_image_id, _] : m_global_image_status) {
+ m_update_global_image_ids.insert(global_image_id);
+ }
+ for (auto&[global_group_id, _] : m_global_group_status) {
+ m_update_global_group_ids.insert(global_group_id);
}
queue_update_task(std::move(locker));
std::swap(m_updating_global_image_ids, m_update_global_image_ids);
auto updating_global_image_ids = m_updating_global_image_ids;
auto global_image_status = m_global_image_status;
+
+ std::swap(m_updating_global_group_ids, m_update_global_group_ids);
+ auto updating_global_group_ids = m_updating_global_group_ids;
+ auto global_group_status = m_global_group_status;
locker.unlock();
Context* ctx = create_context_callback<
MirrorStatusUpdater<I>,
&MirrorStatusUpdater<I>::handle_update_task>(this);
- if (updating_global_image_ids.empty()) {
+ if (updating_global_image_ids.empty() && updating_global_group_ids.empty()) {
ctx->complete(0);
return;
}
aio_comp->release();
}
+ auto group_it = updating_global_group_ids.begin();
+ while (group_it != updating_global_group_ids.end()) {
+ librados::ObjectWriteOperation op;
+ uint32_t op_count = 0;
+
+ while (group_it != updating_global_group_ids.end() &&
+ op_count < MAX_UPDATES_PER_OP) {
+ auto &global_group_id = *group_it;
+ ++group_it;
+
+ auto status_it = global_group_status.find(global_group_id);
+ if (status_it == global_group_status.end()) {
+ continue;
+ }
+
+ status_it->second.mirror_uuid = m_local_mirror_uuid;
+ librbd::cls_client::mirror_group_status_set(&op, global_group_id,
+ status_it->second);
+ ++op_count;
+ }
+
+ auto aio_comp = create_rados_callback(gather->new_sub());
+ int r = m_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op);
+ ceph_assert(r == 0);
+ aio_comp->release();
+ }
+
gather->activate();
}
m_update_in_flight = false;
m_updating_global_image_ids.clear();
+ m_updating_global_group_ids.clear();
if (m_update_requested) {
m_update_requested = false;
void init(Context* on_finish);
void shut_down(Context* on_finish);
- bool exists(const std::string& global_image_id);
+ bool mirror_image_exists(const std::string& global_image_id) const;
void set_mirror_image_status(
const std::string& global_image_id,
const cls::rbd::MirrorImageSiteStatus& mirror_image_site_status,
void remove_refresh_mirror_image_status(const std::string& global_image_id,
Context* on_finish);
+ bool mirror_group_exists(const std::string& global_group_id) const;
+ void set_mirror_group_status(
+ const std::string& global_group_id,
+ const cls::rbd::MirrorGroupSiteStatus& mirror_group_site_status,
+ bool immediate_update);
+ void remove_mirror_group_status(const std::string& global_group_id,
+ Context* on_finish);
+
+ bool mirror_group_image_exists(const std::string& global_group_id,
+ int64_t image_pool_id,
+ const std::string& global_image_id) const;
+ void set_mirror_group_image_status(
+ const std::string& global_group_id,
+ int64_t image_pool_id, const std::string& global_image_id,
+ const cls::rbd::MirrorImageSiteStatus& mirror_image_site_status,
+ bool immediate_update);
+ void remove_mirror_group_image_status(
+ const std::string& global_group_id, int64_t image_pool_id,
+ const std::string& global_image_id, Context* on_finish);
+
private:
/**
* @verbatim
typedef std::set<std::string> GlobalImageIds;
typedef std::map<std::string, cls::rbd::MirrorImageSiteStatus>
GlobalImageStatus;
+ typedef std::set<std::string> GlobalGroupIds;
+ typedef std::map<std::string, cls::rbd::MirrorGroupSiteStatus>
+ GlobalGroupStatus;
librados::IoCtx m_io_ctx;
Threads<ImageCtxT>* m_threads;
Context* m_timer_task = nullptr;
- ceph::mutex m_lock;
+ mutable ceph::mutex m_lock;
bool m_initialized = false;
GlobalImageIds m_update_global_image_ids;
GlobalImageStatus m_global_image_status;
+ GlobalGroupIds m_update_global_group_ids;
+ GlobalGroupStatus m_global_group_status;
+
bool m_update_in_progress = false;
bool m_update_in_flight = false;
bool m_update_requested = false;
Contexts m_update_on_finish_ctxs;
GlobalImageIds m_updating_global_image_ids;
+ GlobalImageIds m_updating_global_group_ids;
bool try_remove_mirror_image_status(const std::string& global_image_id,
bool queue_update, bool immediate_update,
Context* on_finish);
+ bool try_remove_mirror_group_status(const std::string& global_image_id,
+ Context* on_finish);
+
void init_mirror_status_watcher(Context* on_finish);
void handle_init_mirror_status_watcher(int r, Context* on_finish);
const std::string &global_group_id,
const std::string &instance_id,
Context* on_finish) {
- ceph_assert(!mirror_uuid.empty());
dout(5) << "mirror_uuid=" << mirror_uuid << ", "
<< "global_group_id=" << global_group_id << ", "
<< "instance_id=" << instance_id << dendl;
namespace rbd {
namespace mirror {
+bool PoolMetaCache::local_pool_meta_exists(int64_t pool_id) const {
+ dout(15) << "pool_id=" << pool_id << dendl;
+
+ std::unique_lock locker(m_lock);
+ return m_local_pool_metas.count(pool_id) > 0;
+}
+
+bool PoolMetaCache::remote_pool_meta_exists(int64_t pool_id) const {
+ dout(15) << "pool_id=" << pool_id << dendl;
+
+ std::unique_lock locker(m_lock);
+ return m_remote_pool_metas.count(pool_id) > 0;
+}
+
int PoolMetaCache::get_local_pool_meta(
int64_t pool_id,
LocalPoolMeta* local_pool_meta) const {
PoolMetaCache(const PoolMetaCache&) = delete;
PoolMetaCache& operator=(const PoolMetaCache&) = delete;
+ bool local_pool_meta_exists(int64_t pool_id) const;
+ bool remote_pool_meta_exists(int64_t pool_id) const;
int get_local_pool_meta(int64_t pool_id,
LocalPoolMeta* local_pool_meta) const;
void set_local_pool_meta(int64_t pool_id,
std::ostream& operator<<(std::ostream& os,
const RemotePoolMeta& remote_pool_meta) {
- return os << "mirror_uuid=" << remote_pool_meta.mirror_uuid << ", "
- "mirror_peer_uuid=" << remote_pool_meta.mirror_peer_uuid;
+ return os << "mirror_uuid=" << remote_pool_meta.mirror_uuid
+ << ", mirror_peer_uuid=" << remote_pool_meta.mirror_peer_uuid;
}
std::ostream& operator<<(std::ostream& os, const PeerSpec &peer) {
return os << "uuid: " << peer.uuid
- << " cluster: " << peer.cluster_name
- << " client: " << peer.client_name;
+ << " cluster: " << peer.cluster_name
+ << " client: " << peer.client_name;
+}
+
+std::ostream& operator<<(std::ostream& os, const GroupCtx &group_ctx) {
+ return os << "name: " << group_ctx.name
+ << ", group_id: " << group_ctx.group_id
+ << ", global_group_id: " << group_ctx.global_group_id;
}
} // namespace mirror
#include <iostream>
#include <memory>
+#include <optional>
#include <set>
#include <string>
#include <vector>
std::ostream& operator<<(std::ostream& os, const PeerSpec &peer);
+struct GroupCtx {
+ std::string name;
+ std::string group_id;
+ std::string global_group_id;
+ bool primary = false;
+ mutable librados::IoCtx io_ctx;
+
+ GroupCtx() {
+ }
+
+ GroupCtx(const std::string &name, const std::string &group_id,
+ const std::string &global_group_id, bool primary,
+ librados::IoCtx &io_ctx_)
+ : name(name), group_id(group_id), global_group_id(global_group_id),
+ primary(primary) {
+ io_ctx.dup(io_ctx_);
+ }
+};
+
+std::ostream& operator<<(std::ostream& lhs, const GroupCtx &group_ctx);
+
} // namespace mirror
} // namespace rbd
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "include/compat.h"
+#include "BootstrapRequest.h"
+#include "common/debug.h"
+#include "common/dout.h"
+#include "common/errno.h"
+#include "cls/rbd/cls_rbd_client.h"
+#include "librbd/internal.h"
+#include "librbd/group/ListSnapshotsRequest.h"
+#include "librbd/group/RemoveImageRequest.h"
+#include "librbd/Utils.h"
+#include "tools/rbd_mirror/ImageReplayer.h"
+#include "tools/rbd_mirror/PoolMetaCache.h"
+#include "tools/rbd_mirror/Threads.h"
+#include "tools/rbd_mirror/image_deleter/TrashMoveRequest.h"
+
+#define dout_context g_ceph_context
+#define dout_subsys ceph_subsys_rbd_mirror
+#undef dout_prefix
+#define dout_prefix *_dout << "rbd::mirror::group_replayer::" \
+ << "BootstrapRequest: " << this << " " \
+ << __func__ << ": "
+
+namespace rbd {
+namespace mirror {
+namespace group_replayer {
+
+using librbd::util::create_context_callback;
+using librbd::util::create_rados_callback;
+
+namespace {
+
+static const uint32_t MAX_RETURN = 1024;
+
+int get_last_mirror_snapshot_state(
+ const std::vector<cls::rbd::GroupSnapshot> &snaps,
+ cls::rbd::MirrorSnapshotState *state) {
+ for (auto it = snaps.rbegin(); it != snaps.rend(); it++) {
+ auto ns = std::get_if<cls::rbd::MirrorGroupSnapshotNamespace>(
+ &it->snapshot_namespace);
+ if (ns != nullptr) {
+ // XXXMG: check primary_mirror_uuid matches?
+ *state = ns->state;
+ return 0;
+ }
+ }
+
+ return -ENOENT;
+}
+
+} // anonymous namespace
+
+template <typename I>
+BootstrapRequest<I>::BootstrapRequest(
+ Threads<I> *threads,
+ librados::IoCtx &local_io_ctx,
+ librados::IoCtx &remote_io_ctx,
+ const std::string &global_group_id,
+ const std::string &local_mirror_uuid,
+ InstanceWatcher<I> *instance_watcher,
+ MirrorStatusUpdater<I> *local_status_updater,
+ MirrorStatusUpdater<I> *remote_status_updater,
+ journal::CacheManagerHandler *cache_manager_handler,
+ PoolMetaCache *pool_meta_cache,
+ GroupCtx *local_group_ctx,
+ std::list<std::pair<librados::IoCtx, ImageReplayer<I> *>> *image_replayers,
+ Context* on_finish)
+ : CancelableRequest("rbd::mirror::group_replayer::BootstrapRequest",
+ reinterpret_cast<CephContext*>(local_io_ctx.cct()),
+ on_finish),
+ m_threads(threads),
+ m_local_io_ctx(local_io_ctx),
+ m_remote_io_ctx(remote_io_ctx),
+ m_global_group_id(global_group_id),
+ m_local_mirror_uuid(local_mirror_uuid),
+ m_instance_watcher(instance_watcher),
+ m_local_status_updater(local_status_updater),
+ m_remote_status_updater(remote_status_updater),
+ m_cache_manager_handler(cache_manager_handler),
+ m_pool_meta_cache(pool_meta_cache),
+ m_local_group_ctx(local_group_ctx),
+ m_image_replayers(image_replayers),
+ m_on_finish(on_finish) {
+ dout(10) << "global_group_id=" << m_global_group_id << dendl;
+}
+
+template <typename I>
+void BootstrapRequest<I>::send() {
+ get_remote_group_id();
+}
+
+template <typename I>
+void BootstrapRequest<I>::cancel() {
+ dout(10) << dendl;
+
+ m_canceled = true;
+}
+
+template <typename I>
+std::string BootstrapRequest<I>::prepare_non_primary_mirror_snap_name(
+ const std::string &global_group_id,
+ const std::string &snap_id) {
+ dout(5) << "global_group_id: " << global_group_id
+ << ", snap_id: " << snap_id << dendl;
+ std::stringstream ind_snap_name_stream;
+ ind_snap_name_stream << ".mirror.non-primary."
+ << global_group_id << "." << snap_id;
+ return ind_snap_name_stream.str();
+}
+
+template <typename I>
+bool BootstrapRequest<I>::has_remote_image(
+ int64_t local_pool_id, const std::string &global_image_id) const {
+
+ std::string pool_name;
+ int r = librados::Rados(m_local_io_ctx).pool_reverse_lookup(local_pool_id,
+ &pool_name);
+ if (r < 0) {
+ return false;
+ }
+ int64_t remote_pool_id =
+ librados::Rados(m_remote_io_ctx).pool_lookup(pool_name.c_str());
+ if (remote_pool_id < 0) {
+ return false;
+ }
+
+ return m_remote_images.count({remote_pool_id, global_image_id}) > 0;
+}
+
+template <typename I>
+void BootstrapRequest<I>::get_remote_group_id() {
+ dout(10) << dendl;
+
+ librados::ObjectReadOperation op;
+ librbd::cls_client::mirror_group_get_group_id_start(&op, m_global_group_id);
+ m_out_bl.clear();
+ auto comp = create_rados_callback<
+ BootstrapRequest<I>,
+ &BootstrapRequest<I>::handle_get_remote_group_id>(this);
+
+ int r = m_remote_io_ctx.aio_operate(RBD_MIRRORING, comp, &op, &m_out_bl);
+ ceph_assert(r == 0);
+ comp->release();
+}
+
+template <typename I>
+void BootstrapRequest<I>::handle_get_remote_group_id(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ if (m_canceled) {
+ finish(-ECANCELED);
+ return;
+ }
+
+ if (r == -ENOENT) {
+ get_local_group_id();
+ return;
+ }
+
+ if (r == 0) {
+ auto iter = m_out_bl.cbegin();
+ r = librbd::cls_client::mirror_group_get_group_id_finish(
+ &iter, &m_remote_group_id);
+ }
+
+ if (r < 0) {
+ derr << "error getting remote group id: " << cpp_strerror(r) << dendl;
+ finish(r);
+ return;
+ }
+
+ get_remote_group_name();
+}
+
+template <typename I>
+void BootstrapRequest<I>::get_remote_group_name() {
+ dout(10) << dendl;
+
+ librados::ObjectReadOperation op;
+ librbd::cls_client::dir_get_name_start(&op, m_remote_group_id);
+ m_out_bl.clear();
+ auto comp = create_rados_callback<
+ BootstrapRequest<I>,
+ &BootstrapRequest<I>::handle_get_remote_group_name>(this);
+
+ int r = m_remote_io_ctx.aio_operate(RBD_GROUP_DIRECTORY, comp, &op,
+ &m_out_bl);
+ ceph_assert(r == 0);
+ comp->release();
+}
+
+template <typename I>
+void BootstrapRequest<I>::handle_get_remote_group_name(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ if (m_canceled) {
+ finish(-ECANCELED);
+ return;
+ }
+
+ if (r == -ENOENT) {
+ get_local_group_id();
+ return;
+ }
+
+ if (r == 0) {
+ auto iter = m_out_bl.cbegin();
+ r = librbd::cls_client::dir_get_name_finish(&iter, &m_group_name);
+ }
+
+ if (r < 0) {
+ derr << "error getting remote group name: " << cpp_strerror(r) << dendl;
+ finish(r);
+ return;
+ }
+
+ get_remote_mirror_group();
+}
+
+template <typename I>
+void BootstrapRequest<I>::get_remote_mirror_group() {
+ dout(10) << dendl;
+
+ librados::ObjectReadOperation op;
+ librbd::cls_client::mirror_group_get_start(&op, m_remote_group_id);
+ m_out_bl.clear();
+ auto comp = create_rados_callback<
+ BootstrapRequest<I>,
+ &BootstrapRequest<I>::handle_get_remote_mirror_group>(this);
+
+ int r = m_remote_io_ctx.aio_operate(RBD_MIRRORING, comp, &op, &m_out_bl);
+ ceph_assert(r == 0);
+ comp->release();
+}
+
+template <typename I>
+void BootstrapRequest<I>::handle_get_remote_mirror_group(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ if (m_canceled) {
+ finish(-ECANCELED);
+ return;
+ }
+
+ if (r == -ENOENT) {
+ m_remote_mirror_group.state = cls::rbd::MIRROR_GROUP_STATE_DISABLED;
+ get_local_group_id();
+ return;
+ }
+
+ if (r == 0) {
+ auto iter = m_out_bl.cbegin();
+ r = librbd::cls_client::mirror_group_get_finish(&iter,
+ &m_remote_mirror_group);
+ }
+
+ if (r < 0 && r != -ENOENT) {
+ derr << "error getting remote mirror group: " << cpp_strerror(r) << dendl;
+ finish(r);
+ return;
+ }
+
+ if (m_remote_mirror_group.global_group_id != m_global_group_id) {
+ derr << "invalid global group id: "
+ << m_remote_mirror_group.global_group_id << dendl;
+ finish(-EINVAL);
+ return;
+ }
+
+ list_remote_group_snapshots();
+}
+
+template <typename I>
+void BootstrapRequest<I>::list_remote_group_snapshots() {
+ dout(10) << dendl;
+
+ auto ctx = create_context_callback<
+ BootstrapRequest<I>,
+ &BootstrapRequest<I>::handle_list_remote_group_snapshots>(this);
+
+ auto req = librbd::group::ListSnapshotsRequest<I>::create(m_remote_io_ctx,
+ *m_remote_group_id, true, true, &m_remote_group_snaps, ctx);
+ req->send();
+}
+
+template <typename I>
+void BootstrapRequest<I>::handle_list_remote_group_snapshots(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ if (m_canceled) {
+ finish(-ECANCELED);
+ return;
+ }
+
+ if (r < 0) {
+ derr << "error listing remote mirror group snapshots: " << cpp_strerror(r)
+ << dendl;
+ finish(r);
+ return;
+ }
+
+ if (m_remote_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED) {
+ cls::rbd::MirrorSnapshotState state;
+ r = get_last_mirror_snapshot_state(m_remote_group_snaps, &state);
+ if (r == -ENOENT) {
+ derr << "failed to find remote mirror group snapshot" << dendl;
+ finish(-EINVAL);
+ return;
+ }
+ ceph_assert(r == 0);
+ m_remote_mirror_group_primary = (state == cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY);
+ }
+
+ if (m_remote_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED &&
+ m_remote_mirror_group_primary) {
+ list_remote_group();
+ } else {
+ get_local_group_id();
+ }
+}
+
+template <typename I>
+void BootstrapRequest<I>::list_remote_group() {
+ dout(10) << dendl;
+
+ librados::ObjectReadOperation op;
+ cls::rbd::GroupImageSpec start_after;
+ if (!m_images.empty()) {
+ start_after = m_images.rbegin()->spec;
+ }
+ librbd::cls_client::group_image_list_start(&op, start_after, MAX_RETURN);
+
+ auto comp = create_rados_callback<
+ BootstrapRequest<I>,
+ &BootstrapRequest<I>::handle_list_remote_group>(this);
+ m_out_bl.clear();
+ int r = m_remote_io_ctx.aio_operate(
+ librbd::util::group_header_name(m_remote_group_id), comp, &op, &m_out_bl);
+ ceph_assert(r == 0);
+ comp->release();
+}
+
+template <typename I>
+void BootstrapRequest<I>::handle_list_remote_group(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ if (m_canceled) {
+ finish(-ECANCELED);
+ return;
+ }
+
+ std::vector<cls::rbd::GroupImageStatus> images;
+ if (r == 0) {
+ auto iter = m_out_bl.cbegin();
+ r = librbd::cls_client::group_image_list_finish(&iter, &images);
+ }
+
+ if (r < 0 && r != -ENOENT) {
+ derr << "error listing remote group: " << cpp_strerror(r) << dendl;
+ finish(r);
+ return;
+ }
+
+ m_images.insert(m_images.end(), images.begin(), images.end());
+
+ if (images.size() == MAX_RETURN) {
+ list_remote_group();
+ return;
+ }
+
+ get_remote_mirror_image();
+}
+
+template <typename I>
+void BootstrapRequest<I>::get_remote_mirror_image() {
+ while (!m_images.empty() &&
+ m_images.front().state != cls::rbd::GROUP_IMAGE_LINK_STATE_ATTACHED) {
+ dout(20) << "skip " << m_images.front().spec.pool_id << " "
+ << m_images.front().spec.image_id << dendl;
+ m_images.pop_front();
+ }
+
+ if (m_images.empty()) {
+ get_local_group_id();
+ return;
+ }
+
+ auto &spec = m_images.front().spec;
+
+ dout(10) << spec.pool_id << " " << spec.image_id << dendl;
+
+ if (!m_pool_meta_cache->remote_pool_meta_exists(spec.pool_id)) {
+ derr << "failed to find remote image pool in meta cache" << dendl;
+ finish(-ENOENT);
+ return;
+ }
+
+ int r = librbd::util::create_ioctx(m_remote_io_ctx, "remote image pool",
+ spec.pool_id, {}, &m_image_io_ctx);
+ if (r < 0) {
+ derr << "failed to open remote image pool " << spec.pool_id << ": "
+ << cpp_strerror(r) << dendl;
+ finish(r);
+ return;
+ }
+
+ librados::ObjectReadOperation op;
+ librbd::cls_client::mirror_image_get_start(&op, spec.image_id);
+ m_out_bl.clear();
+ auto comp = create_rados_callback<
+ BootstrapRequest<I>,
+ &BootstrapRequest<I>::handle_get_remote_mirror_image>(this);
+
+ r = m_image_io_ctx.aio_operate(RBD_MIRRORING, comp, &op, &m_out_bl);
+ ceph_assert(r == 0);
+ comp->release();
+}
+
+template <typename I>
+void BootstrapRequest<I>::handle_get_remote_mirror_image(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ if (m_canceled) {
+ finish(-ECANCELED);
+ return;
+ }
+
+ auto &spec = m_images.front().spec;
+ cls::rbd::MirrorImage mirror_image;
+
+ if (r == 0) {
+ auto iter = m_out_bl.cbegin();
+ r = librbd::cls_client::mirror_image_get_finish(&iter, &mirror_image);
+ }
+
+ if (r < 0 && r != -ENOENT) {
+ derr << "error getting remote mirror image: " << cpp_strerror(r) << dendl;
+ finish(r);
+ return;
+ }
+
+ m_remote_images.insert({spec.pool_id, mirror_image.global_image_id});
+
+ m_images.pop_front();
+
+ get_remote_mirror_image();
+}
+
+template <typename I>
+void BootstrapRequest<I>::get_local_group_id() {
+ dout(10) << dendl;
+
+ librados::ObjectReadOperation op;
+ librbd::cls_client::mirror_group_get_group_id_start(&op, m_global_group_id);
+ m_out_bl.clear();
+ auto comp = create_rados_callback<
+ BootstrapRequest<I>,
+ &BootstrapRequest<I>::handle_get_local_group_id>(this);
+
+ int r = m_local_io_ctx.aio_operate(RBD_MIRRORING, comp, &op, &m_out_bl);
+ ceph_assert(r == 0);
+ comp->release();
+}
+
+template <typename I>
+void BootstrapRequest<I>::handle_get_local_group_id(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ if (m_canceled) {
+ finish(-ECANCELED);
+ return;
+ }
+
+ if (r == -ENOENT &&
+ m_remote_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED &&
+ m_remote_mirror_group_primary) {
+ ceph_assert(!m_group_name.empty());
+ get_local_group_id_by_name();
+ return;
+ }
+
+ if (r == 0) {
+ auto iter = m_out_bl.cbegin();
+ r = librbd::cls_client::mirror_group_get_group_id_finish(
+ &iter, &m_local_group_id);
+ }
+
+ if (r < 0) {
+ if (r != -ENOENT) {
+ derr << "error getting local group id: " << cpp_strerror(r) << dendl;
+ } else {
+ m_local_mirror_group.state = cls::rbd::MIRROR_GROUP_STATE_DISABLED;
+ r = 0;
+ }
+ finish(r);
+ return;
+ }
+
+ get_local_group_name();
+}
+
+template <typename I>
+void BootstrapRequest<I>::get_local_group_name() {
+ dout(10) << dendl;
+
+ librados::ObjectReadOperation op;
+ librbd::cls_client::dir_get_name_start(&op, m_local_group_id);
+ m_out_bl.clear();
+ auto comp = create_rados_callback<
+ BootstrapRequest<I>,
+ &BootstrapRequest<I>::handle_get_local_group_name>(this);
+
+ int r = m_local_io_ctx.aio_operate(RBD_GROUP_DIRECTORY, comp, &op,
+ &m_out_bl);
+ ceph_assert(r == 0);
+ comp->release();
+}
+
+template <typename I>
+void BootstrapRequest<I>::handle_get_local_group_name(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ if (m_canceled) {
+ finish(-ECANCELED);
+ return;
+ }
+
+ std::string local_group_name;
+
+ if (r == 0) {
+ auto iter = m_out_bl.cbegin();
+ r = librbd::cls_client::dir_get_name_finish(&iter, &local_group_name);
+ }
+
+ if (r < 0) {
+ derr << "error getting local group name: " << cpp_strerror(r) << dendl;
+ if (r == -ENOENT) {
+ r = -EEXIST; // split-brain
+ }
+ finish(r);
+ return;
+ }
+
+ if (m_group_name.empty()) {
+ m_group_name = local_group_name;
+ } else if (m_group_name != local_group_name) {
+ // should never happen
+ derr << "local group name '" << local_group_name << "' does not match "
+ << "remote group name '" << m_group_name << "'" << dendl;
+ finish(-EEXIST); // split-brain
+ return;
+ }
+
+ get_local_mirror_group();
+}
+
+template <typename I>
+void BootstrapRequest<I>::get_local_group_id_by_name() {
+ dout(10) << dendl;
+
+ librados::ObjectReadOperation op;
+ librbd::cls_client::dir_get_id_start(&op, m_group_name);
+ m_out_bl.clear();
+ auto comp = create_rados_callback<
+ BootstrapRequest<I>,
+ &BootstrapRequest<I>::handle_get_local_group_id_by_name>(this);
+
+ int r = m_local_io_ctx.aio_operate(RBD_GROUP_DIRECTORY, comp, &op, &m_out_bl);
+ ceph_assert(r == 0);
+ comp->release();
+}
+
+template <typename I>
+void BootstrapRequest<I>::handle_get_local_group_id_by_name(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ if (m_canceled) {
+ finish(-ECANCELED);
+ return;
+ }
+
+ if (r == -ENOENT) {
+ create_local_group_id();
+ return;
+ }
+
+ if (r == 0) {
+ auto iter = m_out_bl.cbegin();
+ r = librbd::cls_client::dir_get_id_finish(&iter, &m_local_group_id);
+ }
+
+ if (r < 0) {
+ derr << "error getting local group id: " << cpp_strerror(r) << dendl;
+ finish(r);
+ return;
+ }
+
+ m_local_group_id_by_name = true;
+ get_local_mirror_group();
+}
+
+template <typename I>
+void BootstrapRequest<I>::get_local_mirror_group() {
+ dout(10) << dendl;
+
+ librados::ObjectReadOperation op;
+ librbd::cls_client::mirror_group_get_start(&op, m_local_group_id);
+ m_out_bl.clear();
+ auto comp = create_rados_callback<
+ BootstrapRequest<I>,
+ &BootstrapRequest<I>::handle_get_local_mirror_group>(this);
+
+ int r = m_local_io_ctx.aio_operate(RBD_MIRRORING, comp, &op, &m_out_bl);
+ ceph_assert(r == 0);
+ comp->release();
+}
+
+template <typename I>
+void BootstrapRequest<I>::handle_get_local_mirror_group(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ if (m_canceled) {
+ finish(-ECANCELED);
+ return;
+ }
+
+ if (r == 0) {
+ auto iter = m_out_bl.cbegin();
+ r = librbd::cls_client::mirror_group_get_finish(&iter,
+ &m_local_mirror_group);
+ }
+
+ if (r == -ENOENT) {
+ if (m_local_group_id_by_name) {
+ derr << "local group is not mirrored" << dendl;
+ finish(-EINVAL);
+ return;
+ }
+ if (m_remote_mirror_group.state != cls::rbd::MIRROR_GROUP_STATE_ENABLED ||
+ !m_remote_mirror_group_primary) {
+ derr << "can't find primary for group: " << m_group_name << dendl;
+ finish(-EEXIST); // split-brain
+ return;
+ }
+ r = 0;
+ }
+
+ if (r < 0) {
+ derr << "error getting local mirror group: " << cpp_strerror(r) << dendl;
+ finish(r);
+ return;
+ }
+
+ dout(20) << m_local_mirror_group << dendl;
+
+ list_local_group_snapshots();
+}
+
+template <typename I>
+void BootstrapRequest<I>::list_local_group_snapshots() {
+ dout(10) << dendl;
+
+ auto ctx = create_context_callback<
+ BootstrapRequest<I>,
+ &BootstrapRequest<I>::handle_list_local_group_snapshots>(this);
+
+ auto req = librbd::group::ListSnapshotsRequest<I>::create(m_local_io_ctx,
+ m_local_group_id, true, true, &m_local_group_snaps, ctx);
+ req->send();
+}
+
+template <typename I>
+void BootstrapRequest<I>::handle_list_local_group_snapshots(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ if (m_canceled) {
+ finish(-ECANCELED);
+ return;
+ }
+
+ if (r < 0) {
+ derr << "error listing local mirror group snapshots: " << cpp_strerror(r)
+ << dendl;
+ finish(r);
+ return;
+ }
+
+ if (m_local_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED) {
+ cls::rbd::MirrorSnapshotState state;
+ r = get_last_mirror_snapshot_state(m_local_group_snaps, &state);
+ if (r == -ENOENT) {
+ derr << "failed to find local mirror group snapshot" << dendl;
+ finish(-EINVAL);
+ return;
+ }
+ ceph_assert(r == 0);
+ m_local_mirror_group_primary = (state == cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY);
+ }
+
+ if (m_remote_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED) {
+ if (m_remote_mirror_group_primary) {
+ if (m_local_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED &&
+ m_local_mirror_group_primary) {
+ derr << "both remote and local groups are primary" << dendl;
+ finish(-EEXIST); // split-brain
+ return;
+ }
+ } else if (m_local_mirror_group.state != cls::rbd::MIRROR_GROUP_STATE_ENABLED ||
+ !m_local_mirror_group_primary) {
+ derr << "both remote and local groups are not primary" << dendl;
+ finish(-EREMOTEIO);
+ return;
+ }
+ } else if (m_local_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED &&
+ !m_local_mirror_group_primary) {
+ // trigger group removal
+ m_local_mirror_group.state = cls::rbd::MIRROR_GROUP_STATE_DISABLED;
+ }
+
+ list_local_group();
+}
+
+template <typename I>
+void BootstrapRequest<I>::list_local_group() {
+ dout(10) << dendl;
+
+ librados::ObjectReadOperation op;
+ cls::rbd::GroupImageSpec start_after;
+ if (!m_images.empty()) {
+ start_after = m_images.rbegin()->spec;
+ }
+ librbd::cls_client::group_image_list_start(&op, start_after, MAX_RETURN);
+ auto comp = create_rados_callback<
+ BootstrapRequest<I>,
+ &BootstrapRequest<I>::handle_list_local_group>(this);
+ m_out_bl.clear();
+ int r = m_local_io_ctx.aio_operate(
+ librbd::util::group_header_name(m_local_group_id), comp, &op, &m_out_bl);
+ ceph_assert(r == 0);
+ comp->release();
+}
+
+template <typename I>
+void BootstrapRequest<I>::handle_list_local_group(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ if (m_canceled) {
+ finish(-ECANCELED);
+ return;
+ }
+
+ std::vector<cls::rbd::GroupImageStatus> images;
+ if (r == 0) {
+ auto iter = m_out_bl.cbegin();
+ r = librbd::cls_client::group_image_list_finish(&iter, &images);
+ }
+
+ if (r < 0 && r != -ENOENT) {
+ derr << "error listing local group: " << cpp_strerror(r) << dendl;
+ finish(r);
+ return;
+ }
+
+ m_images.insert(m_images.end(), images.begin(), images.end());
+
+ if (images.size() == MAX_RETURN) {
+ list_local_group();
+ return;
+ }
+
+ get_local_mirror_image();
+}
+
+template <typename I>
+void BootstrapRequest<I>::get_local_mirror_image() {
+ if (m_images.empty()) {
+ remove_local_image_from_group();
+ return;
+ }
+
+ auto &spec = m_images.front().spec;
+
+ dout(10) << spec.pool_id << " " << spec.image_id << dendl;
+
+ int r = librbd::util::create_ioctx(m_local_io_ctx, "local image pool",
+ spec.pool_id, {}, &m_image_io_ctx);
+ if (r < 0) {
+ derr << "failed to open local image pool " << spec.pool_id << ": "
+ << cpp_strerror(r) << dendl;
+ finish(r);
+ return;
+ }
+
+ librados::ObjectReadOperation op;
+ librbd::cls_client::mirror_image_get_start(&op, spec.image_id);
+ m_out_bl.clear();
+ auto comp = create_rados_callback<
+ BootstrapRequest<I>,
+ &BootstrapRequest<I>::handle_get_local_mirror_image>(this);
+
+ r = m_image_io_ctx.aio_operate(RBD_MIRRORING, comp, &op, &m_out_bl);
+ ceph_assert(r == 0);
+ comp->release();
+}
+
+template <typename I>
+void BootstrapRequest<I>::handle_get_local_mirror_image(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ if (m_canceled) {
+ finish(-ECANCELED);
+ return;
+ }
+
+ auto &spec = m_images.front().spec;
+ cls::rbd::MirrorImage mirror_image;
+
+ if (r == 0) {
+ auto iter = m_out_bl.cbegin();
+ r = librbd::cls_client::mirror_image_get_finish(&iter, &mirror_image);
+ }
+
+ if (r < 0) {
+ if (r != -ENOENT) {
+ derr << "error getting local mirror image: " << cpp_strerror(r) << dendl;
+ finish(r);
+ return;
+ }
+ } else {
+ if (m_local_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED &&
+ m_local_mirror_group_primary) {
+ dout(10) << "add primary to replayer queue: " << spec.pool_id << " "
+ << spec.image_id << " " << mirror_image.global_image_id
+ << dendl;
+ m_local_images.insert({spec.pool_id, mirror_image.global_image_id});
+ } else if (m_remote_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED &&
+ m_remote_mirror_group_primary &&
+ m_local_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED &&
+ m_local_mirror_group.global_group_id == m_global_group_id &&
+ has_remote_image(spec.pool_id, mirror_image.global_image_id)) {
+ dout(10) << "add secondary to replayer queue: " << spec.pool_id << " "
+ << spec.image_id << " " << mirror_image.global_image_id
+ << dendl;
+ m_local_images.insert({spec.pool_id, mirror_image.global_image_id});
+ } else {
+ dout(10) << "add to trash queue: " << spec.pool_id << " "
+ << spec.image_id << " " << mirror_image.global_image_id
+ << dendl;
+ m_local_trash_images[{spec.pool_id, mirror_image.global_image_id}] =
+ spec.image_id;
+ }
+ }
+
+ m_images.pop_front();
+
+ get_local_mirror_image();
+}
+
+template <typename I>
+void BootstrapRequest<I>::remove_local_image_from_group() {
+ if (m_local_trash_images.empty()) {
+ disable_local_mirror_group();
+ return;
+ }
+
+ auto &[pool_id, global_image_id] = m_local_trash_images.begin()->first;
+ auto &image_id = m_local_trash_images.begin()->second;
+
+ dout(10) << "pool_id=" << pool_id << ", image_id=" << image_id << dendl;
+
+ int r = librbd::util::create_ioctx(m_local_io_ctx, "local image pool",
+ pool_id, {}, &m_image_io_ctx);
+ if (r < 0) {
+ derr << "failed to open local image pool " << pool_id << ": "
+ << cpp_strerror(r) << dendl;
+ handle_remove_local_image_from_group(-ENOENT);
+ return;
+ }
+
+ auto ctx = create_context_callback<
+ BootstrapRequest<I>,
+ &BootstrapRequest<I>::handle_remove_local_image_from_group>(this);
+
+ auto req = librbd::group::RemoveImageRequest<I>::create(
+ m_local_io_ctx, m_local_group_id, m_image_io_ctx, image_id, ctx);
+ req->send();
+}
+
+template <typename I>
+void BootstrapRequest<I>::handle_remove_local_image_from_group(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ if (r < 0 && r != -ENOENT) {
+ derr << "error removing mirror image from group: " << cpp_strerror(r) << dendl;
+ finish(r);
+ return;
+ }
+
+ move_local_image_to_trash();
+}
+
+template <typename I>
+void BootstrapRequest<I>::move_local_image_to_trash() {
+ ceph_assert(!m_local_trash_images.empty());
+ auto &[pool_id, global_image_id] = m_local_trash_images.begin()->first;
+
+ dout(10) << "pool_id=" << pool_id << ", global_image_id=" << global_image_id
+ << dendl;
+
+ int r = librbd::util::create_ioctx(m_local_io_ctx, "local image pool",
+ pool_id, {}, &m_image_io_ctx);
+ if (r < 0) {
+ derr << "failed to open local image pool " << pool_id << ": "
+ << cpp_strerror(r) << dendl;
+ handle_move_local_image_to_trash(-ENOENT);
+ return;
+ }
+
+ auto ctx = create_context_callback<
+ BootstrapRequest<I>,
+ &BootstrapRequest<I>::handle_move_local_image_to_trash>(this);
+
+ auto req = image_deleter::TrashMoveRequest<I>::create(
+ m_image_io_ctx, global_image_id, false, m_threads->work_queue, ctx);
+ req->send();
+}
+
+template <typename I>
+void BootstrapRequest<I>::handle_move_local_image_to_trash(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ if (m_canceled) {
+ finish(-ECANCELED);
+ return;
+ }
+
+ if (r < 0 && r != -ENOENT) {
+ derr << "error moving mirror image to trash: " << cpp_strerror(r) << dendl;
+ finish(r);
+ return;
+ }
+
+ m_local_trash_images.erase(m_local_trash_images.begin());
+
+ remove_local_image_from_group();
+}
+
+template <typename I>
+void BootstrapRequest<I>::disable_local_mirror_group() {
+ if (m_local_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED &&
+ m_local_mirror_group.global_group_id == m_global_group_id) {
+ finish(0);
+ return;
+ }
+
+ dout(10) << dendl;
+
+ librados::ObjectWriteOperation op;
+ m_local_mirror_group.state = cls::rbd::MIRROR_GROUP_STATE_DISABLING;
+ librbd::cls_client::mirror_group_set(&op, m_local_group_id,
+ m_local_mirror_group);
+
+ auto comp = create_rados_callback<
+ BootstrapRequest<I>,
+ &BootstrapRequest<I>::handle_disable_local_mirror_group>(this);
+
+ int r = m_local_io_ctx.aio_operate(RBD_MIRRORING, comp, &op);
+ ceph_assert(r == 0);
+ comp->release();
+}
+
+template <typename I>
+void BootstrapRequest<I>::handle_disable_local_mirror_group(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ if (r < 0 && r != -ENOENT) {
+ derr << "error disabling local mirror group: " << cpp_strerror(r) << dendl;
+ finish(r);
+ return;
+ }
+
+ remove_local_mirror_group();
+}
+
+template <typename I>
+void BootstrapRequest<I>::remove_local_mirror_group() {
+ if (m_local_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED &&
+ m_local_mirror_group.global_group_id == m_global_group_id) {
+ finish(0);
+ return;
+ }
+
+ dout(10) << dendl;
+
+ librados::ObjectWriteOperation op;
+ librbd::cls_client::mirror_group_remove(&op, m_local_group_id);
+
+ auto comp = create_rados_callback<
+ BootstrapRequest<I>,
+ &BootstrapRequest<I>::handle_remove_local_mirror_group>(this);
+
+ int r = m_local_io_ctx.aio_operate(RBD_MIRRORING, comp, &op);
+ ceph_assert(r == 0);
+ comp->release();
+}
+
+template <typename I>
+void BootstrapRequest<I>::handle_remove_local_mirror_group(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ if (r < 0 && r != -ENOENT) {
+ derr << "error removing local mirror group: " << cpp_strerror(r) << dendl;
+ finish(r);
+ return;
+ }
+
+ m_local_mirror_group.state = cls::rbd::MIRROR_GROUP_STATE_DISABLED;
+
+ if (m_remote_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED &&
+ m_remote_mirror_group_primary) {
+ create_local_mirror_group();
+ } else {
+ remove_local_group();
+ }
+}
+
+template <typename I>
+void BootstrapRequest<I>::remove_local_group() {
+ dout(10) << m_group_name << " " << m_local_group_id << dendl;
+
+ ceph_assert(!m_local_group_id.empty());
+ ceph_assert(!m_group_name.empty());
+
+ librados::ObjectWriteOperation op;
+ op.remove();
+ auto comp = create_rados_callback<
+ BootstrapRequest<I>,
+ &BootstrapRequest<I>::handle_remove_local_group>(this);
+
+ int r = m_local_io_ctx.aio_operate(
+ librbd::util::group_header_name(m_local_group_id), comp, &op);
+ ceph_assert(r == 0);
+ comp->release();
+}
+
+template <typename I>
+void BootstrapRequest<I>::handle_remove_local_group(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ if (r < 0 && r != -ENOENT) {
+ derr << "error removing local group: " << cpp_strerror(r) << dendl;
+ finish(r);
+ return;
+ }
+
+ remove_local_group_id();
+}
+
+template <typename I>
+void BootstrapRequest<I>::remove_local_group_id() {
+ dout(10) << dendl;
+
+ librados::ObjectWriteOperation op;
+ librbd::cls_client::group_dir_remove(&op, m_group_name, m_local_group_id);
+
+ auto comp = create_rados_callback<
+ BootstrapRequest<I>,
+ &BootstrapRequest<I>::handle_remove_local_group_id>(this);
+
+ int r = m_local_io_ctx.aio_operate(RBD_GROUP_DIRECTORY, comp, &op);
+ ceph_assert(r == 0);
+ comp->release();
+}
+
+template <typename I>
+void BootstrapRequest<I>::handle_remove_local_group_id(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ if (r < 0 && r != -ENOENT) {
+ derr << "error removing local group id: " << cpp_strerror(r) << dendl;
+ finish(r);
+ return;
+ }
+
+ finish(0);
+}
+
+
+template <typename I>
+void BootstrapRequest<I>::create_local_group_id() {
+ dout(10) << dendl;
+
+ m_local_group_id = librbd::util::generate_uuid(m_local_io_ctx);
+
+ librados::ObjectWriteOperation op;
+ librbd::cls_client::group_dir_add(&op, m_group_name, m_local_group_id);
+
+ auto comp = create_rados_callback<
+ BootstrapRequest<I>,
+ &BootstrapRequest<I>::handle_create_local_group_id>(this);
+
+ int r = m_local_io_ctx.aio_operate(RBD_GROUP_DIRECTORY, comp, &op);
+ ceph_assert(r == 0);
+ comp->release();
+}
+
+template <typename I>
+void BootstrapRequest<I>::handle_create_local_group_id(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ if (r < 0) {
+ derr << "error creating local group id: " << cpp_strerror(r) << dendl;
+ finish(r);
+ return;
+ }
+
+ create_local_group();
+}
+
+template <typename I>
+void BootstrapRequest<I>::create_local_group() {
+ dout(10) << dendl;
+
+ librados::ObjectWriteOperation op;
+ op.create(true);
+ auto comp = create_rados_callback<
+ BootstrapRequest<I>,
+ &BootstrapRequest<I>::handle_create_local_group>(this);
+
+ int r = m_local_io_ctx.aio_operate(
+ librbd::util::group_header_name(m_local_group_id), comp, &op);
+ ceph_assert(r == 0);
+ comp->release();
+}
+
+template <typename I>
+void BootstrapRequest<I>::handle_create_local_group(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ if (r < 0) {
+ derr << "error creating local group: " << cpp_strerror(r) << dendl;
+ finish(r);
+ return;
+ }
+
+ create_local_mirror_group();
+}
+
+template <typename I>
+void BootstrapRequest<I>::create_local_mirror_group() {
+ dout(10) << dendl;
+
+ ceph_assert(
+ m_remote_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED &&
+ m_remote_mirror_group_primary);
+
+ librados::ObjectWriteOperation op;
+ m_local_mirror_group = {m_global_group_id,
+ m_remote_mirror_group.mirror_image_mode,
+ cls::rbd::MIRROR_GROUP_STATE_ENABLED};
+ librbd::cls_client::mirror_group_set(&op, m_local_group_id,
+ m_local_mirror_group);
+ auto comp = create_rados_callback<
+ BootstrapRequest<I>,
+ &BootstrapRequest<I>::handle_create_local_mirror_group>(this);
+
+ int r = m_local_io_ctx.aio_operate(RBD_MIRRORING, comp, &op);
+ ceph_assert(r == 0);
+ comp->release();
+}
+
+template <typename I>
+void BootstrapRequest<I>::handle_create_local_mirror_group(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ if (r < 0) {
+ derr << "error creating local mirror group: " << cpp_strerror(r) << dendl;
+ finish(r);
+ return;
+ }
+
+ create_local_non_primary_group_snapshot();
+}
+
+template <typename I>
+void BootstrapRequest<I>::create_local_non_primary_group_snapshot() {
+ dout(10) << dendl;
+
+ RemotePoolMeta remote_pool_meta;
+ int r = m_pool_meta_cache->get_remote_pool_meta(m_remote_io_ctx.get_id(),
+ &remote_pool_meta);
+ if (r < 0 || remote_pool_meta.mirror_peer_uuid.empty()) {
+ derr << "failed to retrieve mirror peer uuid from remote image pool"
+ << dendl;
+ finish(r < 0 ? r : -EINVAL);
+ return;
+ }
+
+ librados::ObjectWriteOperation op;
+ std::string group_snap_id = librbd::util::generate_image_id(m_local_io_ctx);
+ cls::rbd::GroupSnapshot group_snap{
+ group_snap_id,
+ cls::rbd::MirrorGroupSnapshotNamespace{
+ cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY,
+ {}, remote_pool_meta.mirror_peer_uuid, {}},
+ prepare_non_primary_mirror_snap_name(m_global_group_id, group_snap_id),
+ cls::rbd::GROUP_SNAPSHOT_STATE_INCOMPLETE};
+ librbd::cls_client::group_snap_set(&op, group_snap);
+ group_snap.state = cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE;
+ librbd::cls_client::group_snap_set(&op, group_snap);
+
+ auto comp = create_rados_callback<
+ BootstrapRequest<I>,
+ &BootstrapRequest<I>::handle_create_local_non_primary_group_snapshot>(this);
+
+ r = m_local_io_ctx.aio_operate(
+ librbd::util::group_header_name(*m_local_group_id), comp, &op);
+ ceph_assert(r == 0);
+ comp->release();
+}
+
+template <typename I>
+void BootstrapRequest<I>::handle_create_local_non_primary_group_snapshot(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ if (r < 0) {
+ derr << "error creating local non-primary group snapshot: "
+ << cpp_strerror(r) << dendl;
+ finish(r);
+ return;
+ }
+
+ finish(0);
+}
+
+template <typename I>
+void BootstrapRequest<I>::finish(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ ceph_assert(r != -ENOENT);
+ if (r == 0) {
+ if (m_local_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_DISABLED) {
+ r = -ENOENT;
+ } else {
+ *m_local_group_ctx = {m_group_name, m_local_group_id, m_global_group_id,
+ m_local_mirror_group_primary, m_local_io_ctx};
+ r = create_replayers();
+ }
+ }
+
+ m_on_finish->complete(r);
+}
+
+template <typename I>
+int BootstrapRequest<I>::create_replayers() {
+ dout(10) << dendl;
+
+ int r = 0;
+ if (m_remote_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED &&
+ m_remote_mirror_group_primary) {
+ for (auto &[remote_pool_id, global_image_id] : m_remote_images) {
+ m_image_replayers->emplace_back(librados::IoCtx(), nullptr);
+ auto &local_io_ctx = m_image_replayers->back().first;
+ auto &image_replayer = m_image_replayers->back().second;
+
+ RemotePoolMeta remote_pool_meta;
+ r = m_pool_meta_cache->get_remote_pool_meta(remote_pool_id,
+ &remote_pool_meta);
+ if (r < 0 || remote_pool_meta.mirror_peer_uuid.empty()) {
+ derr << "failed to retrieve mirror peer uuid from remote image pool"
+ << dendl;
+ r = -ENOENT;
+ break;
+ }
+
+ librados::IoCtx remote_io_ctx;
+ r = librbd::util::create_ioctx(m_remote_io_ctx, "remote image pool",
+ remote_pool_id, {}, &remote_io_ctx);
+ if (r < 0) {
+ derr << "failed to open remote image pool " << remote_pool_id << ": "
+ << cpp_strerror(r) << dendl;
+ if (r == -ENOENT) {
+ r = -EINVAL;
+ }
+ break;
+ }
+
+ int64_t local_pool_id = librados::Rados(m_local_io_ctx).pool_lookup(
+ remote_io_ctx.get_pool_name().c_str());
+
+ LocalPoolMeta local_pool_meta;
+ r = m_pool_meta_cache->get_local_pool_meta(local_pool_id,
+ &local_pool_meta);
+ if (r < 0 || local_pool_meta.mirror_uuid.empty()) {
+ if (r == 0 || r == -ENOENT) {
+ r = -EINVAL;
+ }
+ derr << "failed to retrieve mirror uuid from local image pool" << dendl;
+ break;
+ }
+
+ r = librbd::util::create_ioctx(m_local_io_ctx, "local image pool",
+ local_pool_id, {}, &local_io_ctx);
+ if (r < 0) {
+ derr << "failed to open local image pool " << local_pool_id << ": "
+ << cpp_strerror(r) << dendl;
+ if (r == -ENOENT) {
+ r = -EINVAL;
+ }
+ break;
+ }
+
+ image_replayer = ImageReplayer<I>::create(
+ local_io_ctx, m_local_group_ctx, local_pool_meta.mirror_uuid,
+ global_image_id, m_threads, m_instance_watcher, m_local_status_updater,
+ m_cache_manager_handler, m_pool_meta_cache);
+
+ // TODO only a single peer is currently supported
+ image_replayer->add_peer({local_pool_meta.mirror_uuid, remote_io_ctx,
+ remote_pool_meta, m_remote_status_updater});
+ }
+ } else if (m_local_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED &&
+ m_local_mirror_group_primary) {
+ for (auto &[local_pool_id, global_image_id] : m_local_images) {
+ m_image_replayers->emplace_back(librados::IoCtx(), nullptr);
+ auto &local_io_ctx = m_image_replayers->back().first;
+ auto &image_replayer = m_image_replayers->back().second;
+
+ LocalPoolMeta local_pool_meta;
+ r = m_pool_meta_cache->get_local_pool_meta(local_pool_id,
+ &local_pool_meta);
+ if (r < 0 || local_pool_meta.mirror_uuid.empty()) {
+ if (r == 0 || r == -ENOENT) {
+ r = -EINVAL;
+ }
+ derr << "failed to retrieve mirror uuid from local image pool" << dendl;
+ break;
+ }
+
+ r = librbd::util::create_ioctx(m_local_io_ctx, "local image pool",
+ local_pool_id, {}, &local_io_ctx);
+ if (r < 0) {
+ derr << "failed to open local image pool " << local_pool_id << ": "
+ << cpp_strerror(r) << dendl;
+ if (r == -ENOENT) {
+ r = -EINVAL;
+ }
+ break;
+ }
+
+ int64_t remote_pool_id = librados::Rados(m_remote_io_ctx).pool_lookup(
+ local_io_ctx.get_pool_name().c_str());
+
+ RemotePoolMeta remote_pool_meta;
+ r = m_pool_meta_cache->get_remote_pool_meta(remote_pool_id,
+ &remote_pool_meta);
+ if (r < 0 || remote_pool_meta.mirror_peer_uuid.empty()) {
+ derr << "failed to retrieve mirror peer uuid from remote image pool"
+ << dendl;
+ r = -ENOENT;
+ break;
+ }
+
+ librados::IoCtx remote_io_ctx;
+ r = librbd::util::create_ioctx(m_remote_io_ctx, "remote image pool",
+ remote_pool_id, {}, &remote_io_ctx);
+ if (r < 0) {
+ derr << "failed to open remote image pool " << remote_pool_id << ": "
+ << cpp_strerror(r) << dendl;
+ if (r == -ENOENT) {
+ r = -EINVAL;
+ }
+ break;
+ }
+
+ image_replayer = ImageReplayer<I>::create(
+ local_io_ctx, m_local_group_ctx, local_pool_meta.mirror_uuid,
+ global_image_id, m_threads, m_instance_watcher, m_local_status_updater,
+ m_cache_manager_handler, m_pool_meta_cache);
+
+ // TODO only a single peer is currently supported
+ image_replayer->add_peer({local_pool_meta.mirror_uuid, remote_io_ctx,
+ remote_pool_meta, m_remote_status_updater});
+ }
+ } else {
+ ceph_abort();
+ }
+
+ if (r < 0) {
+ for (auto &[_, image_replayer] : *m_image_replayers) {
+ delete image_replayer;
+ }
+ m_image_replayers->clear();
+ return r;
+ }
+
+ return 0;
+}
+
+} // namespace group_replayer
+} // namespace mirror
+} // namespace rbd
+
+template class rbd::mirror::group_replayer::BootstrapRequest<librbd::ImageCtx>;
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef RBD_MIRROR_GROUP_REPLAYER_BOOTSTRAP_REQUEST_H
+#define RBD_MIRROR_GROUP_REPLAYER_BOOTSTRAP_REQUEST_H
+
+#include "include/rados/librados.hpp"
+#include "cls/rbd/cls_rbd_types.h"
+#include "tools/rbd_mirror/CancelableRequest.h"
+
+#include <atomic>
+#include <list>
+#include <set>
+#include <string>
+
+
+class Context;
+
+namespace journal { struct CacheManagerHandler; }
+namespace librbd { class ImageCtx; }
+
+namespace rbd {
+namespace mirror {
+
+struct GroupCtx;
+template <typename> struct ImageReplayer;
+template <typename> struct InstanceWatcher;
+template <typename> struct MirrorStatusUpdater;
+struct PoolMetaCache;
+template <typename> struct Threads;
+
+namespace group_replayer {
+
+template <typename ImageCtxT = librbd::ImageCtx>
+class BootstrapRequest : public CancelableRequest {
+public:
+ static BootstrapRequest *create(
+ Threads<ImageCtxT> *threads,
+ librados::IoCtx &local_io_ctx,
+ librados::IoCtx &remote_io_ctx,
+ const std::string &global_group_id,
+ const std::string &local_mirror_uuid,
+ InstanceWatcher<ImageCtxT> *instance_watcher,
+ MirrorStatusUpdater<ImageCtxT> *local_status_updater,
+ MirrorStatusUpdater<ImageCtxT> *remote_status_updater,
+ journal::CacheManagerHandler *cache_manager_handler,
+ PoolMetaCache *pool_meta_cache,
+ GroupCtx *local_group_ctx,
+ std::list<std::pair<librados::IoCtx, ImageReplayer<ImageCtxT> *>> *image_replayers,
+ Context *on_finish) {
+ return new BootstrapRequest(
+ threads, local_io_ctx, remote_io_ctx, global_group_id, local_mirror_uuid,
+ instance_watcher, local_status_updater, remote_status_updater,
+ cache_manager_handler, pool_meta_cache, local_group_ctx, image_replayers,
+ on_finish);
+ }
+
+ BootstrapRequest(
+ Threads<ImageCtxT> *threads,
+ librados::IoCtx &local_io_ctx,
+ librados::IoCtx &remote_io_ctx,
+ const std::string &global_group_id,
+ const std::string &local_mirror_uuid,
+ InstanceWatcher<ImageCtxT> *instance_watcher,
+ MirrorStatusUpdater<ImageCtxT> *local_status_updater,
+ MirrorStatusUpdater<ImageCtxT> *remote_status_updater,
+ journal::CacheManagerHandler *cache_manager_handler,
+ PoolMetaCache *pool_meta_cache,
+ GroupCtx *local_group_ctx,
+ std::list<std::pair<librados::IoCtx, ImageReplayer<ImageCtxT> *>> *image_replayers,
+ Context* on_finish);
+
+ void send() override;
+ void cancel() override;
+ std::string prepare_non_primary_mirror_snap_name(
+ const std::string &global_group_id, const std::string &snap_id);
+
+private:
+ /**
+ * @verbatim
+ *
+ * <start>
+ * |
+ * v
+ * GET_REMOTE_GROUP_ID * * * * * * * * * * *
+ * | (noent) *
+ * v v
+ * GET_REMOTE_GROUP_NAME * * * * * * * * * *
+ * | (noent) *
+ * v v
+ * GET_REMOTE_MIRROR_GROUP * * * * * * * * *
+ * | (noent or not primary) *
+ * v v
+ * LIST_REMOTE_GROUP_SNAPSHOTS * * * * * * *
+ * | (noent) *
+ * v v
+ * LIST_REMOTE_GROUP * * * * * * * * * * * *
+ * | (repeat if neeeded) (noent) *
+ * v v
+ * GET_REMOTE_MIRROR_IMAGE * * * * * * * * *
+ * | (repeat for every image) (noent) *
+ * | v
+ * |/< * * * * * * * * * * * * * * * * * *
+ * v
+ * GET_LOCAL_GROUP_ID * * * * * * * * * * * *
+ * | (noent) *
+ * v *
+ * GET_LOCAL_GROUP_NAME *
+ * | *
+ * v v
+ * GET_LOCAL_MIRROR_GROUP <------------- GET_LOCAL_GROUP_ID_BY_NAME
+ * | * (noent)
+ * v (noent) *
+ * LIST_LOCAL_GROUP_SNAPSHOTS * * * *
+ * | * *
+ * v (noent) * *
+ * LIST_LOCAL_GROUP * * * * * * * * *
+ * | (repeat if neeeded) * *
+ * v * *
+ * GET_LOCAL_MIRROR_IMAGE * *
+ * | (repeat for every image) * *
+ * v * *
+ * REMOVE_LOCAL_IMAGE_FROM_GROUP * *
+ * | ^ * *
+ * v | * v
+ * MOVE_LOCAL_IMAGE_TO_TRASH -/ * CREATE_LOCAL_GROUP_ID
+ * | (repeat for every * |
+ * | stale image) * v
+ * |\----\ * * > CREATE_LOCAL_GROUP
+ * | | (if stale |
+ * | v or removing) |
+ * | DISABLE_LOCAL_MIRROR_GROUP |
+ * | | |
+ * | v v
+ * | REMOVE_LOCAL_MIRROR_GROUP ----> CREATE_LOCAL_MIRROR_GROUP
+ * | | (if stale) |
+ * | v v
+ * | REMOVE_LOCAL_GROUP CREATE_LOCAL_NON_PRIMARY_GROUP_SNAPSHOT
+ * | | (if removing) |
+ * | v |
+ * | REMOVE_LOCAL_GROUP_ID |
+ * | | (if removing) |
+ * v v |
+ * <finish> <-------------------------------/
+ *
+ * @endverbatim
+ */
+
+ typedef std::pair<int64_t /*pool_id*/, std::string /*global_image_id*/> GlobalImageId;
+
+ Threads<ImageCtxT>* m_threads;
+ librados::IoCtx &m_local_io_ctx;
+ librados::IoCtx &m_remote_io_ctx;
+ std::string m_global_group_id;
+ std::string m_local_mirror_uuid;
+ InstanceWatcher<ImageCtxT> *m_instance_watcher;
+ MirrorStatusUpdater<ImageCtxT> *m_local_status_updater;
+ MirrorStatusUpdater<ImageCtxT> *m_remote_status_updater;
+ journal::CacheManagerHandler *m_cache_manager_handler;
+ PoolMetaCache *m_pool_meta_cache;
+ GroupCtx *m_local_group_ctx;
+ std::list<std::pair<librados::IoCtx, ImageReplayer<ImageCtxT> *>> *m_image_replayers;
+ Context *m_on_finish;
+
+ std::atomic<bool> m_canceled = false;
+
+ std::string m_group_name;
+ std::string m_remote_group_id;
+ std::string m_local_group_id;
+ bool m_local_group_id_by_name = false;
+ cls::rbd::MirrorGroup m_remote_mirror_group;
+ cls::rbd::MirrorGroup m_local_mirror_group;
+ std::vector<cls::rbd::GroupSnapshot> m_remote_group_snaps;
+ std::vector<cls::rbd::GroupSnapshot> m_local_group_snaps;
+ bool m_remote_mirror_group_primary = false;
+ bool m_local_mirror_group_primary = false;
+ std::list<cls::rbd::GroupImageStatus> m_images;
+ librados::IoCtx m_image_io_ctx;
+
+ std::set<GlobalImageId> m_remote_images;
+ std::set<GlobalImageId> m_local_images;
+ std::map<GlobalImageId, std::string> m_local_trash_images;
+
+ bufferlist m_out_bl;
+
+ bool has_remote_image(int64_t local_pool_id,
+ const std::string &global_image_id) const;
+
+ void get_remote_group_id();
+ void handle_get_remote_group_id(int r);
+
+ void get_remote_group_name();
+ void handle_get_remote_group_name(int r);
+
+ void get_remote_mirror_group();
+ void handle_get_remote_mirror_group(int r);
+
+ void get_remote_mirror_image();
+ void handle_get_remote_mirror_image(int r);
+
+ void list_remote_group_snapshots();
+ void handle_list_remote_group_snapshots(int r);
+
+ void list_remote_group();
+ void handle_list_remote_group(int r);
+
+ void get_local_group_id();
+ void handle_get_local_group_id(int r);
+
+ void get_local_group_id_by_name();
+ void handle_get_local_group_id_by_name(int r);
+
+ void get_local_group_name();
+ void handle_get_local_group_name(int r);
+
+ void get_local_mirror_group();
+ void handle_get_local_mirror_group(int r);
+
+ void list_local_group_snapshots();
+ void handle_list_local_group_snapshots(int r);
+
+ void list_local_group();
+ void handle_list_local_group(int r);
+
+ void get_local_mirror_image();
+ void handle_get_local_mirror_image(int r);
+
+ void remove_local_image_from_group();
+ void handle_remove_local_image_from_group(int r);
+
+ void move_local_image_to_trash();
+ void handle_move_local_image_to_trash(int r);
+
+ void remove_local_mirror_image();
+ void handle_remove_local_mirror_image(int r);
+
+ void disable_local_mirror_group();
+ void handle_disable_local_mirror_group(int r);
+
+ void remove_local_mirror_group();
+ void handle_remove_local_mirror_group(int r);
+
+ void remove_local_group();
+ void handle_remove_local_group(int r);
+
+ void remove_local_group_id();
+ void handle_remove_local_group_id(int r);
+
+ void create_local_group_id();
+ void handle_create_local_group_id(int r);
+
+ void create_local_group();
+ void handle_create_local_group(int r);
+
+ void create_local_mirror_group();
+ void handle_create_local_mirror_group(int r);
+
+ void create_local_non_primary_group_snapshot();
+ void handle_create_local_non_primary_group_snapshot(int r);
+
+ void finish(int r);
+
+ int create_replayers();
+};
+
+} // namespace group_replayer
+} // namespace mirror
+} // namespace rbd
+
+extern template class rbd::mirror::group_replayer::BootstrapRequest<librbd::ImageCtx>;
+
+#endif // RBD_MIRROR_GROUP_REPLAYER_BOOTSTRAP_REQUEST_H
Threads<I>* threads,
librados::IoCtx& local_io_ctx,
librados::IoCtx& remote_io_ctx,
+ GroupCtx *local_group_ctx,
InstanceWatcher<I>* instance_watcher,
const std::string& global_image_id,
const std::string& local_mirror_uuid,
m_threads(threads),
m_local_io_ctx(local_io_ctx),
m_remote_io_ctx(remote_io_ctx),
+ m_local_group_ctx(local_group_ctx),
m_instance_watcher(instance_watcher),
m_global_image_id(global_image_id),
m_local_mirror_uuid(local_mirror_uuid),
BootstrapRequest<I>,
&BootstrapRequest<I>::handle_create_local_image>(this);
auto request = (*m_state_builder)->create_local_image_request(
- m_threads, m_local_io_ctx, m_global_image_id, m_pool_meta_cache,
- m_progress_ctx, ctx);
+ m_threads, m_local_io_ctx, m_local_group_ctx, m_global_image_id,
+ m_pool_meta_cache, m_progress_ctx, ctx);
request->send();
}
Context *ctx = create_context_callback<
BootstrapRequest<I>, &BootstrapRequest<I>::handle_image_sync>(this);
m_image_sync = ImageSync<I>::create(
- m_threads, state_builder->local_image_ctx, state_builder->remote_image_ctx,
- m_local_mirror_uuid, sync_point_handler, m_instance_watcher,
- m_progress_ctx, ctx);
+ m_threads, m_local_group_ctx, state_builder->local_image_ctx,
+ state_builder->remote_image_ctx, m_local_mirror_uuid, sync_point_handler,
+ m_instance_watcher, m_progress_ctx, ctx);
m_image_sync->get();
locker.unlock();
Threads<ImageCtxT>* threads,
librados::IoCtx& local_io_ctx,
librados::IoCtx& remote_io_ctx,
+ GroupCtx *local_group_ctx,
InstanceWatcher<ImageCtxT>* instance_watcher,
const std::string& global_image_id,
const std::string& local_mirror_uuid,
bool* do_resync,
Context* on_finish) {
return new BootstrapRequest(
- threads, local_io_ctx, remote_io_ctx, instance_watcher, global_image_id,
- local_mirror_uuid, remote_pool_meta, cache_manager_handler,
- pool_meta_cache, progress_ctx, state_builder, do_resync, on_finish);
+ threads, local_io_ctx, remote_io_ctx, local_group_ctx, instance_watcher,
+ global_image_id, local_mirror_uuid, remote_pool_meta,
+ cache_manager_handler, pool_meta_cache, progress_ctx, state_builder,
+ do_resync, on_finish);
}
BootstrapRequest(
Threads<ImageCtxT>* threads,
librados::IoCtx& local_io_ctx,
librados::IoCtx& remote_io_ctx,
+ GroupCtx *local_group_ctx,
InstanceWatcher<ImageCtxT>* instance_watcher,
const std::string& global_image_id,
const std::string& local_mirror_uuid,
Threads<ImageCtxT>* m_threads;
librados::IoCtx &m_local_io_ctx;
librados::IoCtx &m_remote_io_ctx;
+ GroupCtx *m_local_group_ctx;
InstanceWatcher<ImageCtxT> *m_instance_watcher;
std::string m_global_image_id;
std::string m_local_mirror_uuid;
CreateImageRequest<I>::CreateImageRequest(
Threads<I>* threads,
librados::IoCtx &local_io_ctx,
+ GroupCtx *local_group_ctx,
const std::string &global_image_id,
const std::string &remote_mirror_uuid,
const std::string &local_image_name,
cls::rbd::MirrorImageMode mirror_image_mode,
Context *on_finish)
: m_threads(threads), m_local_io_ctx(local_io_ctx),
- m_global_image_id(global_image_id),
+ m_local_group_ctx(local_group_ctx), m_global_image_id(global_image_id),
m_remote_mirror_uuid(remote_mirror_uuid),
m_local_image_name(local_image_name), m_local_image_id(local_image_id),
m_remote_image_ctx(remote_image_ctx),
}
image_options->set(RBD_IMAGE_OPTION_CLONE_FORMAT, clone_format);
}
+
+ if (m_local_group_ctx != nullptr) {
+ image_options->set(RBD_IMAGE_OPTION_GROUP_NAME, m_local_group_ctx->name);
+ image_options->set(RBD_IMAGE_OPTION_GROUP_POOL,
+ m_local_group_ctx->io_ctx.get_pool_name());
+ }
}
} // namespace image_replayer
namespace mirror {
class PoolMetaCache;
+struct GroupCtx;
template <typename> struct Threads;
namespace image_replayer {
static CreateImageRequest *create(
Threads<ImageCtxT> *threads,
librados::IoCtx &local_io_ctx,
+ GroupCtx *local_group_ctx,
const std::string &global_image_id,
const std::string &remote_mirror_uuid,
const std::string &local_image_name,
PoolMetaCache* pool_meta_cache,
cls::rbd::MirrorImageMode mirror_image_mode,
Context *on_finish) {
- return new CreateImageRequest(threads, local_io_ctx, global_image_id,
- remote_mirror_uuid, local_image_name,
- local_image_id, remote_image_ctx,
- pool_meta_cache, mirror_image_mode,
- on_finish);
+ return new CreateImageRequest(threads, local_io_ctx, local_group_ctx,
+ global_image_id, remote_mirror_uuid,
+ local_image_name, local_image_id,
+ remote_image_ctx, pool_meta_cache,
+ mirror_image_mode, on_finish);
}
CreateImageRequest(
Threads<ImageCtxT> *threads, librados::IoCtx &local_io_ctx,
+ GroupCtx *local_group_ctx,
const std::string &global_image_id,
const std::string &remote_mirror_uuid,
const std::string &local_image_name,
Threads<ImageCtxT> *m_threads;
librados::IoCtx &m_local_io_ctx;
+ GroupCtx *m_local_group_ctx;
std::string m_global_image_id;
std::string m_remote_mirror_uuid;
std::string m_local_image_name;
namespace mirror {
struct BaseRequest;
+struct GroupCtx;
template <typename> class InstanceWatcher;
struct PoolMetaCache;
struct ProgressContext;
virtual BaseRequest* create_local_image_request(
Threads<ImageCtxT>* threads,
librados::IoCtx& local_io_ctx,
+ GroupCtx *local_group_ctx,
const std::string& global_image_id,
PoolMetaCache* pool_meta_cache,
ProgressContext* progress_ctx,
CreateLocalImageRequest<I>,
&CreateLocalImageRequest<I>::handle_create_local_image>(this);
auto request = CreateImageRequest<I>::create(
- m_threads, m_local_io_ctx, m_global_image_id,
+ m_threads, m_local_io_ctx, m_local_group_ctx, m_global_image_id,
m_state_builder->remote_mirror_uuid, image_name,
m_state_builder->local_image_id, m_remote_image_ctx,
m_pool_meta_cache, cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, ctx);
class PoolMetaCache;
class ProgressContext;
+struct GroupCtx;
template <typename> struct Threads;
namespace image_replayer {
static CreateLocalImageRequest* create(
Threads<ImageCtxT>* threads,
librados::IoCtx& local_io_ctx,
+ GroupCtx *local_group_ctx,
ImageCtxT* remote_image_ctx,
const std::string& global_image_id,
PoolMetaCache* pool_meta_cache,
ProgressContext* progress_ctx,
StateBuilder<ImageCtxT>* state_builder,
Context* on_finish) {
- return new CreateLocalImageRequest(threads, local_io_ctx, remote_image_ctx,
- global_image_id, pool_meta_cache,
- progress_ctx, state_builder, on_finish);
+ return new CreateLocalImageRequest(threads, local_io_ctx, local_group_ctx,
+ remote_image_ctx, global_image_id,
+ pool_meta_cache, progress_ctx,
+ state_builder, on_finish);
}
CreateLocalImageRequest(
Threads<ImageCtxT>* threads,
librados::IoCtx& local_io_ctx,
+ GroupCtx *local_group_ctx,
ImageCtxT* remote_image_ctx,
const std::string& global_image_id,
PoolMetaCache* pool_meta_cache,
: BaseRequest(on_finish),
m_threads(threads),
m_local_io_ctx(local_io_ctx),
+ m_local_group_ctx(local_group_ctx),
m_remote_image_ctx(remote_image_ctx),
m_global_image_id(global_image_id),
m_pool_meta_cache(pool_meta_cache),
Threads<ImageCtxT>* m_threads;
librados::IoCtx& m_local_io_ctx;
+ GroupCtx *m_local_group_ctx;
ImageCtxT* m_remote_image_ctx;
std::string m_global_image_id;
PoolMetaCache* m_pool_meta_cache;
BaseRequest* StateBuilder<I>::create_local_image_request(
Threads<I>* threads,
librados::IoCtx& local_io_ctx,
+ GroupCtx *local_group_ctx,
const std::string& global_image_id,
PoolMetaCache* pool_meta_cache,
ProgressContext* progress_ctx,
Context* on_finish) {
return CreateLocalImageRequest<I>::create(
- threads, local_io_ctx, this->remote_image_ctx, this->global_image_id,
- pool_meta_cache, progress_ctx, this, on_finish);
+ threads, local_io_ctx, local_group_ctx, this->remote_image_ctx,
+ this->global_image_id, pool_meta_cache, progress_ctx, this, on_finish);
}
template <typename I>
BaseRequest* create_local_image_request(
Threads<ImageCtxT>* threads,
librados::IoCtx& local_io_ctx,
+ GroupCtx *local_group_ctx,
const std::string& global_image_id,
PoolMetaCache* pool_meta_cache,
ProgressContext* progress_ctx,
#include "librbd/ImageCtx.h"
#include "librbd/Utils.h"
#include "tools/rbd_mirror/ProgressContext.h"
+#include "tools/rbd_mirror/Types.h"
#include "tools/rbd_mirror/image_replayer/CreateImageRequest.h"
#include "tools/rbd_mirror/image_replayer/snapshot/StateBuilder.h"
// need to send 'disabling' since the cls methods will fail if we aren't
// in that state
+ cls::rbd::GroupSpec group_spec;
+ if (m_local_group_ctx != nullptr) {
+ group_spec = {m_local_group_ctx->group_id,
+ m_local_group_ctx->io_ctx.get_id()};
+ }
cls::rbd::MirrorImage mirror_image{
- cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT, m_global_image_id,
+ cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT, m_global_image_id, group_spec,
cls::rbd::MIRROR_IMAGE_STATE_DISABLING};
librados::ObjectWriteOperation op;
librbd::cls_client::mirror_image_set(&op, m_state_builder->local_image_id,
dout(10) << "local_image_id=" << m_state_builder->local_image_id << dendl;
update_progress("ADD_MIRROR_IMAGE");
+ cls::rbd::GroupSpec group_spec;
+ if (m_local_group_ctx != nullptr) {
+ group_spec = {m_local_group_ctx->group_id,
+ m_local_group_ctx->io_ctx.get_id()};
+ }
// use 'creating' to track a partially constructed image. it will
// be switched to 'enabled' once the image is fully created
cls::rbd::MirrorImage mirror_image{
- cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT, m_global_image_id,
+ cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT, m_global_image_id, group_spec,
cls::rbd::MIRROR_IMAGE_STATE_CREATING};
librados::ObjectWriteOperation op;
librbd::cls_client::mirror_image_set(&op, m_state_builder->local_image_id,
CreateLocalImageRequest<I>,
&CreateLocalImageRequest<I>::handle_create_local_image>(this);
auto request = CreateImageRequest<I>::create(
- m_threads, m_local_io_ctx, m_global_image_id,
+ m_threads, m_local_io_ctx, m_local_group_ctx, m_global_image_id,
m_state_builder->remote_mirror_uuid, image_name,
m_state_builder->local_image_id, m_remote_image_ctx,
m_pool_meta_cache, cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT, ctx);
class PoolMetaCache;
class ProgressContext;
+struct GroupCtx;
template <typename> struct Threads;
namespace image_replayer {
static CreateLocalImageRequest* create(
Threads<ImageCtxT>* threads,
librados::IoCtx& local_io_ctx,
+ GroupCtx *local_group_ctx,
ImageCtxT* remote_image_ctx,
const std::string& global_image_id,
PoolMetaCache* pool_meta_cache,
ProgressContext* progress_ctx,
StateBuilder<ImageCtxT>* state_builder,
Context* on_finish) {
- return new CreateLocalImageRequest(threads, local_io_ctx, remote_image_ctx,
- global_image_id, pool_meta_cache,
- progress_ctx, state_builder, on_finish);
+ return new CreateLocalImageRequest(threads, local_io_ctx, local_group_ctx,
+ remote_image_ctx, global_image_id,
+ pool_meta_cache, progress_ctx,
+ state_builder, on_finish);
}
CreateLocalImageRequest(
Threads<ImageCtxT>* threads,
librados::IoCtx& local_io_ctx,
+ GroupCtx *local_group_ctx,
ImageCtxT* remote_image_ctx,
const std::string& global_image_id,
PoolMetaCache* pool_meta_cache,
: BaseRequest(on_finish),
m_threads(threads),
m_local_io_ctx(local_io_ctx),
+ m_local_group_ctx(local_group_ctx),
m_remote_image_ctx(remote_image_ctx),
m_global_image_id(global_image_id),
m_pool_meta_cache(pool_meta_cache),
Threads<ImageCtxT>* m_threads;
librados::IoCtx& m_local_io_ctx;
+ GroupCtx *m_local_group_ctx;
ImageCtxT* m_remote_image_ctx;
std::string m_global_image_id;
PoolMetaCache* m_pool_meta_cache;
BaseRequest* StateBuilder<I>::create_local_image_request(
Threads<I>* threads,
librados::IoCtx& local_io_ctx,
+ GroupCtx *local_group_ctx,
const std::string& global_image_id,
PoolMetaCache* pool_meta_cache,
ProgressContext* progress_ctx,
Context* on_finish) {
return CreateLocalImageRequest<I>::create(
- threads, local_io_ctx, this->remote_image_ctx, global_image_id,
- pool_meta_cache, progress_ctx, this, on_finish);
+ threads, local_io_ctx, local_group_ctx, this->remote_image_ctx,
+ global_image_id, pool_meta_cache, progress_ctx, this, on_finish);
}
template <typename I>
BaseRequest* create_local_image_request(
Threads<ImageCtxT>* threads,
librados::IoCtx& local_io_ctx,
+ GroupCtx *local_group_ctx,
const std::string& global_image_id,
PoolMetaCache* pool_meta_cache,
ProgressContext* progress_ctx,
projects / ceph.git / commitdiff