mirror_group_demote ${CLUSTER1} ${POOL}/${group}
test_fields_in_group_info ${CLUSTER1} ${POOL}/${group} 'snapshot' 'enabled' 'false'
test_fields_in_group_info ${CLUSTER2} ${POOL}/${group} 'snapshot' 'enabled' 'true'
-wait_for_group_status_in_pool_dir ${CLUSTER1} ${POOL}/${group} 'up+error' 0 'split-brain detected'
+wait_for_group_status_in_pool_dir ${CLUSTER1} ${POOL}/${group} 'up+error' 1 'split-brain'
get_id_from_group_info ${CLUSTER1} ${POOL}/${group} group_id_before
mirror_group_resync ${CLUSTER1} ${POOL}/${group}
MIRROR_GROUP_STATE_ENABLING = 1,
MIRROR_GROUP_STATE_ENABLED = 2,
MIRROR_GROUP_STATE_DISABLED = 3,
+ MIRROR_GROUP_STATE_CREATING = 4,
};
struct MirrorGroup {
RBD_MIRROR_GROUP_DISABLING = 0,
RBD_MIRROR_GROUP_ENABLING = 1,
RBD_MIRROR_GROUP_ENABLED = 2,
- RBD_MIRROR_GROUP_DISABLED = 3
+ RBD_MIRROR_GROUP_DISABLED = 3,
+ RBD_MIRROR_GROUP_CREATING = 4
} rbd_mirror_group_state_t;
typedef struct {
on_finish);
auto req = mirror::GroupGetInfoRequest<I>::create(
- group_ioctx, group_name, &ctx->mirror_group, &ctx->promotion_state, ctx);
+ group_ioctx, group_name, "", &ctx->mirror_group, &ctx->promotion_state, ctx);
req->send();
}
auto ctx = new C_GroupGetInfo(mirror_group_info, on_finish);
auto req = mirror::GroupGetInfoRequest<I>::create(
- io_ctx, group_name, &ctx->mirror_group, &ctx->promotion_state, ctx);
+ io_ctx, group_name, "", &ctx->mirror_group, &ctx->promotion_state, ctx);
req->send();
}
template <typename I>
void GroupGetInfoRequest<I>::send() {
- get_id();
+ auto cct = reinterpret_cast<CephContext *>(m_group_ioctx.cct());
+ ldout(cct, 10) << dendl;
+
+ if (m_group_name.empty() && m_group_id.empty()) {
+ lderr(cct) << "both group name and group id cannot be empty" << dendl;
+ finish(-EINVAL);
+ }
+
+ if (m_group_id.empty()) {
+ get_id();
+ } else {
+ get_info();
+ }
}
template <typename I>
ldout(cct, 10) << "r=" << r << dendl;
m_mirror_group->state = cls::rbd::MIRROR_GROUP_STATE_DISABLED;
- *m_promotion_state = PROMOTION_STATE_NON_PRIMARY;
+ *m_promotion_state = PROMOTION_STATE_UNKNOWN;
if (r == 0) {
auto it = m_outbl.cbegin();
return;
}
if (r < 0) {
- lderr(cct) << "failed to get mirror info of group '" << m_group_name
+ lderr(cct) << "failed to get mirror info of group '" << m_group_id
<< "': " << cpp_strerror(r) << dendl;
finish(r);
return;
}
+ if (m_mirror_group->state == cls::rbd::MIRROR_GROUP_STATE_CREATING) {
+ // No snapshots will have been created and it is likely that the
+ // group has not been created either.
+ finish(0);
+ return;
+ }
+
get_last_mirror_snapshot_state();
}
auto cct = reinterpret_cast<CephContext *>(m_group_ioctx.cct());
ldout(cct, 10) << dendl;
- if (r < 0) {
- lderr(cct) << "failed to list group snapshots of group '" << m_group_name
+ // This could return -ENOENT if the group creation was interrupted
+ if (r < 0 && r != -ENOENT) {
+ lderr(cct) << "failed to list group snapshots of group '" << m_group_id
<< "': " << cpp_strerror(r) << dendl;
finish(r);
return;
public:
static GroupGetInfoRequest *create(librados::IoCtx& group_ioctx,
const std::string& group_name,
+ const std::string& group_id,
cls::rbd::MirrorGroup *mirror_group,
PromotionState *promotion_state,
Context *on_finish) {
- return new GroupGetInfoRequest(group_ioctx, group_name, mirror_group,
- promotion_state, on_finish);
+ return new GroupGetInfoRequest(group_ioctx, group_name, group_id,
+ mirror_group, promotion_state, on_finish);
}
GroupGetInfoRequest(librados::IoCtx& group_ioctx,
const std::string& group_name,
+ const std::string& group_id,
cls::rbd::MirrorGroup *mirror_group,
PromotionState *promotion_state,
Context *on_finish)
: m_group_ioctx(group_ioctx), m_group_name(group_name),
- m_mirror_group(mirror_group), m_promotion_state(promotion_state),
- m_on_finish(on_finish) {
+ m_group_id(group_id), m_mirror_group(mirror_group),
+ m_promotion_state(promotion_state), m_on_finish(on_finish) {
}
void send();
&group_snap.snapshot_namespace);
ns->mirror_peer_uuids.erase(mirror_peer_uuid);
+ m_group_snap_id = group_snap.id;
librados::ObjectWriteOperation op;
librbd::cls_client::group_snap_set(&op, group_snap);
int r = m_group_io_ctx.aio_operate(
return "enabled";
case RBD_MIRROR_GROUP_DISABLED:
return "disabled";
+ case RBD_MIRROR_GROUP_CREATING:
+ return "creating";
default:
return "unknown";
}
Throttler.cc
Types.cc
group_replayer/BootstrapRequest.cc
+ group_replayer/CreateLocalGroupRequest.cc
+ group_replayer/GroupMirrorStateUpdateRequest.cc
+ group_replayer/GroupStateBuilder.cc
+ group_replayer/PrepareLocalGroupRequest.cc
+ group_replayer/PrepareRemoteGroupRequest.cc
+ group_replayer/RemoveLocalGroupRequest.cc
group_replayer/Replayer.cc
image_deleter/SnapshotPurgeRequest.cc
image_deleter/TrashMoveRequest.cc
#include "tools/rbd_mirror/MirrorStatusUpdater.h"
#include "tools/rbd_mirror/Threads.h"
#include "tools/rbd_mirror/group_replayer/BootstrapRequest.h"
+#include "tools/rbd_mirror/group_replayer/GroupStateBuilder.h"
#include "tools/rbd_mirror/group_replayer/Replayer.h"
#include "tools/rbd_mirror/image_replayer/Utils.h"
#include "GroupReplayer.h"
std::string remote_group_name;
int r = librbd::cls_client::dir_get_name(&m_local_io_ctx,
RBD_GROUP_DIRECTORY,
- m_local_group_id,
+ m_state_builder->local_group_id,
&local_group_name);
if (r < 0) {
derr << "failed to retrieve local group name: "
r = librbd::cls_client::dir_get_name(&m_remote_group_peer.io_ctx,
RBD_GROUP_DIRECTORY,
- m_remote_group_id,
+ m_state_builder->remote_group_id,
&remote_group_name);
if (r < 0) {
derr << "failed to retrieve remote group name: "
RBD_GROUP_DIRECTORY,
local_group_name,
remote_group_name,
- m_local_group_id);
+ m_state_builder->local_group_id);
if (r < 0) {
derr << "error renaming group from directory"
<< cpp_strerror(r) << dendl;
m_state = STATE_STARTING;
m_last_r = 0;
m_state_desc.clear();
- m_local_group_snaps.clear();
ceph_assert(m_replayer_check_task == nullptr);
+ ceph_assert(m_state_builder == nullptr);
ceph_assert(m_replayer == nullptr);
if (m_destroy_replayers) {
ceph_assert(m_image_replayers.empty());
}
m_destroy_replayers = false;
- // FIXME: replayer index is not used
- m_image_replayer_index.clear();
m_manual_stop = false;
m_finished = false;
m_delete_requested = false;
state = STATE_STOPPED;
}
f->dump_string("state", state_to_string(state));
-/*
- f->open_array_section("image_replayers");
- for (auto &[_, image_replayer] : m_image_replayers) {
- image_replayer->print_status(f);
- }
- f->close_section(); // image_replayers
-*/
f->close_section(); // group_replayer
}
}
ceph_assert(m_replayer == nullptr);
+ ceph_assert(m_state_builder == nullptr);
auto ctx = create_context_callback<
GroupReplayer,
m_threads, m_local_io_ctx, m_remote_group_peer.io_ctx, m_global_group_id,
m_local_mirror_uuid, m_instance_watcher, m_local_status_updater,
m_remote_group_peer.mirror_status_updater, m_cache_manager_handler,
- m_pool_meta_cache, &m_resync_requested, &m_local_group_id,
- &m_remote_group_id, &m_local_group_snaps, &m_local_group_ctx,
- &m_image_replayers, &m_image_replayer_index, ctx);
+ m_pool_meta_cache, &m_resync_requested, &m_local_group_ctx,
+ &m_image_replayers, &m_state_builder, ctx);
request->get();
m_bootstrap_request = request;
locker.unlock();
set_state_description(0, "bootstrapping");
- update_mirror_group_status(false, boost::none);
+ update_mirror_group_status(true, boost::none);
request->send();
}
} else if (r == -ENOENT) {
finish_start_fail(r, "group removed");
return;
+ } else if (r == -ENOLINK) {
+ finish_start_fail(r, "remote group does not exist");
+ return;
} else if (r == -EREMOTEIO) {
m_destroy_replayers = true;
- finish_start_fail(r, "remote group is non-primary");
+ finish_start_fail(r, "remote group is not primary");
return;
} else if (r == -EEXIST) {
finish_start_fail(r, "split-brain detected");
m_replayer = group_replayer::Replayer<I>::create(
m_threads, m_local_io_ctx, m_remote_group_peer.io_ctx, m_global_group_id,
- m_local_mirror_uuid, m_pool_meta_cache, m_local_group_id, m_remote_group_id,
- &m_local_group_ctx, &m_image_replayers);
+ m_local_mirror_uuid, m_pool_meta_cache, m_state_builder->local_group_id,
+ m_state_builder->remote_group_id, &m_local_group_ctx, &m_image_replayers);
m_replayer->init(ctx);
}
}
}
+template <typename I>
+void GroupReplayer<I>::handle_replayer_notification() {
+ dout(10) << dendl;
+
+ std::unique_lock locker{m_lock};
+ if (m_state != STATE_REPLAYING) {
+ // might be attempting to shut down
+ return;
+ }
+ // replayer cannot be shut down while notification is in-flight
+ ceph_assert(m_replayer != nullptr);
+ locker.unlock();
+
+ if (!m_replayer->is_replaying()) {
+ auto error_code = m_replayer->get_error_code();
+ auto error_description = m_replayer->get_error_description();
+ dout(10) << "replay interrupted: "
+ << "r=" << error_code << ", "
+ << "error=" << error_description << dendl;
+ //FIXME : destroying the IRs if split brain
+ /*
+ if (error_code == -EEXIST) {
+ m_destroy_replayers = true;
+ } */
+ on_stop_replay(error_code, error_description);
+ return;
+ }
+}
+
template <typename I>
bool GroupReplayer<I>::finish_start_if_interrupted() {
std::lock_guard locker{m_lock};
return;
}
+ if (m_state_builder != nullptr) {
+ m_state_builder->destroy();
+ m_state_builder = nullptr;
+ }
+
dout(10) << "stop complete" << dendl;
Context *on_start = nullptr;
std::list<Context *> on_stop_contexts;
default:
ceph_assert(!"invalid state");
}
+
auto remote_status = local_status;
{
std::lock_guard locker{m_lock};
(!m_local_group_ctx.primary && local_status.state == cls::rbd::MIRROR_GROUP_STATUS_STATE_REPLAYING))) {
for (auto &image_site_status : images_status) {
if (image_site_status.second.state == cls::rbd::MIRROR_IMAGE_STATUS_STATE_ERROR) {
- dout(10) << "ImageReplayer with global image id: " << image_site_status.first.global_image_id
- << " in error state, with description: " << image_site_status.second.description
- << " marking group replayer to error state with global group id: " << m_global_group_id
- << dendl;
+ dout(10) << "ImageReplayer in error state. global image id: "
+ << image_site_status.first.global_image_id
+ << ", description: " << image_site_status.second.description
+ << dendl;
+ dout(10) << "setting group replayer state to error : global group id: "
+ << m_global_group_id << dendl;
+
local_status.state = cls::rbd::MIRROR_GROUP_STATUS_STATE_ERROR;
local_status.description = "image in error state";
mirror_group_status_state = local_status.state;
}
}
}
+
{
std::lock_guard locker{m_lock};
m_mirror_group_status_state = mirror_group_status_state;
namespace group_replayer {
template <typename> class BootstrapRequest;
+ template <typename> class GroupStateBuilder;
}
/**
return m_global_group_id;
}
inline const std::string& get_local_group_id() const {
- return m_local_group_id;
+ ceph_assert(m_state_builder != nullptr);
+ return m_state_builder->local_group_id;
}
void start(Context *on_finish = nullptr, bool manual = false,
Listener(GroupReplayer *group_replayer) : group_replayer(group_replayer) {
}
- void stop() {
- group_replayer->stop(nullptr, false);
+ void handle_notification() {
+ group_replayer->handle_replayer_notification();
}
};
std::string m_local_group_id;
std::string m_remote_group_id;
+ // FIXME: Find a better way
bool m_destroy_replayers = false;
mutable ceph::mutex m_lock;
Context *m_on_start_finish = nullptr;
std::list<Context *> m_on_stop_contexts;
- Context *m_on_stop_finish = nullptr;
bool m_stop_requested = false;
bool m_resync_requested = false;
bool m_restart_requested = false;
Context* m_replayer_check_task = nullptr;
Context* m_update_status_task = nullptr;
+ group_replayer::GroupStateBuilder<ImageCtxT> *m_state_builder = nullptr;
group_replayer::BootstrapRequest<ImageCtxT> *m_bootstrap_request = nullptr;
group_replayer::Replayer<ImageCtxT> *m_replayer = nullptr;
std::list<std::pair<librados::IoCtx, ImageReplayer<ImageCtxT> *>> m_image_replayers;
Listener m_listener = {this};
- std::map<std::pair<int64_t, std::string>, ImageReplayer<ImageCtxT> *> m_image_replayer_index;
- std::map<std::string, cls::rbd::GroupSnapshot> m_local_group_snaps;
- std::map<std::string, std::map<ImageReplayer<ImageCtxT> *, Context *>> m_create_snap_requests;
- std::set<std::string> m_pending_snap_create;
static std::string state_to_string(const State &state) {
switch (state) {
void set_mirror_group_status_update(bool force,
const OptionalState &opt_state);
- void wait_for_ops();
- void handle_wait_for_ops(int r);
+ void handle_replayer_notification();
void shut_down(int r);
void handle_shut_down(int r);
virtual ~Listener() {
}
- virtual void stop() = 0;
+ virtual void handle_notification() = 0;
};
std::string name;
#include "include/compat.h"
#include "BootstrapRequest.h"
+#include "CreateLocalGroupRequest.h"
+#include "GroupStateBuilder.h"
+#include "PrepareLocalGroupRequest.h"
+#include "PrepareRemoteGroupRequest.h"
+#include "RemoveLocalGroupRequest.h"
#include "common/debug.h"
#include "common/dout.h"
#include "common/errno.h"
using librbd::util::create_context_callback;
using librbd::util::create_rados_callback;
-namespace {
-
-static const uint32_t MAX_RETURN = 1024;
-
-bool is_demoted_snap_exists(
- const std::vector<cls::rbd::GroupSnapshot> &snaps) {
- for (auto it = snaps.rbegin(); it != snaps.rend(); it++) {
- auto ns = std::get_if<cls::rbd::GroupSnapshotNamespaceMirror>(
- &it->snapshot_namespace);
- if (ns != nullptr) {
- if (ns->is_demoted()) {
- return true;
- }
- }
- }
- return false;
-}
-
-int get_last_mirror_snapshot_state(
- const std::vector<cls::rbd::GroupSnapshot> &snaps,
- cls::rbd::MirrorSnapshotState *state) {
- for (auto it = snaps.rbegin(); it != snaps.rend(); it++) {
- auto ns = std::get_if<cls::rbd::GroupSnapshotNamespaceMirror>(
- &it->snapshot_namespace);
- if (ns != nullptr) {
- // XXXMG: check primary_mirror_uuid matches?
- *state = ns->state;
- return 0;
- }
- }
-
- return -ENOENT;
-}
-
-} // anonymous namespace
-
template <typename I>
BootstrapRequest<I>::BootstrapRequest(
Threads<I> *threads,
librados::IoCtx &local_io_ctx,
librados::IoCtx &remote_io_ctx,
const std::string &global_group_id,
- const std::string &local_mirror_uuid,
+ const std::string &local_mirror_uuid, // FIXME: Not used
InstanceWatcher<I> *instance_watcher,
MirrorStatusUpdater<I> *local_status_updater,
MirrorStatusUpdater<I> *remote_status_updater,
journal::CacheManagerHandler *cache_manager_handler,
PoolMetaCache *pool_meta_cache,
bool *resync_requested,
- std::string *local_group_id,
- std::string *remote_group_id,
- std::map<std::string, cls::rbd::GroupSnapshot> *local_group_snaps,
GroupCtx *local_group_ctx,
std::list<std::pair<librados::IoCtx, ImageReplayer<I> *>> *image_replayers,
- std::map<std::pair<int64_t, std::string>, ImageReplayer<I> *> *image_replayer_index,
+ GroupStateBuilder<I> **state_builder,
Context* on_finish)
: CancelableRequest("rbd::mirror::group_replayer::BootstrapRequest",
reinterpret_cast<CephContext*>(local_io_ctx.cct()),
m_local_status_updater(local_status_updater),
m_remote_status_updater(remote_status_updater),
m_cache_manager_handler(cache_manager_handler),
- m_pool_meta_cache(pool_meta_cache),
- m_resync_requested(resync_requested),
- m_local_group_id(local_group_id),
- m_remote_group_id(remote_group_id),
- m_local_group_snaps(local_group_snaps),
- m_local_group_ctx(local_group_ctx),
- m_image_replayers(image_replayers),
- m_image_replayer_index(image_replayer_index),
- m_on_finish(on_finish) {
- dout(10) << "global_group_id=" << m_global_group_id << dendl;
-}
-
-template <typename I>
-void BootstrapRequest<I>::send() {
- get_remote_group_id();
-}
-
-template <typename I>
-void BootstrapRequest<I>::cancel() {
- dout(10) << dendl;
-
- m_canceled = true;
-}
-
-template <typename I>
-bool BootstrapRequest<I>::has_remote_image(
- int64_t local_pool_id, const std::string &global_image_id) const {
-
- std::string pool_name;
- int r = librados::Rados(m_local_io_ctx).pool_reverse_lookup(local_pool_id,
- &pool_name);
- if (r < 0) {
- return false;
- }
- int64_t remote_pool_id =
- librados::Rados(m_remote_io_ctx).pool_lookup(pool_name.c_str());
- if (remote_pool_id < 0) {
- return false;
- }
-
- return m_remote_images.count({remote_pool_id, global_image_id}) > 0;
-}
-
-template <typename I>
-void BootstrapRequest<I>::get_remote_group_id() {
- dout(10) << dendl;
-
- librados::ObjectReadOperation op;
- librbd::cls_client::mirror_group_get_group_id_start(&op, m_global_group_id);
- m_out_bl.clear();
- auto comp = create_rados_callback<
- BootstrapRequest<I>,
- &BootstrapRequest<I>::handle_get_remote_group_id>(this);
-
- int r = m_remote_io_ctx.aio_operate(RBD_MIRRORING, comp, &op, &m_out_bl);
- ceph_assert(r == 0);
- comp->release();
-}
-
-template <typename I>
-void BootstrapRequest<I>::handle_get_remote_group_id(int r) {
- dout(10) << "r=" << r << dendl;
-
- if (m_canceled) {
- finish(-ECANCELED);
- return;
- }
-
- if (r == -ENOENT) {
- get_local_group_id();
- return;
- }
-
- if (r == 0) {
- auto iter = m_out_bl.cbegin();
- r = librbd::cls_client::mirror_group_get_group_id_finish(
- &iter, m_remote_group_id);
- }
-
- if (r < 0) {
- derr << "error getting remote group id: " << cpp_strerror(r) << dendl;
- finish(r);
- return;
- }
-
- get_remote_group_name();
-}
-
-template <typename I>
-void BootstrapRequest<I>::get_remote_group_name() {
- dout(10) << dendl;
-
- librados::ObjectReadOperation op;
- librbd::cls_client::dir_get_name_start(&op, *m_remote_group_id);
- m_out_bl.clear();
- auto comp = create_rados_callback<
- BootstrapRequest<I>,
- &BootstrapRequest<I>::handle_get_remote_group_name>(this);
-
- int r = m_remote_io_ctx.aio_operate(RBD_GROUP_DIRECTORY, comp, &op,
- &m_out_bl);
- ceph_assert(r == 0);
- comp->release();
-}
-
-template <typename I>
-void BootstrapRequest<I>::handle_get_remote_group_name(int r) {
- dout(10) << "r=" << r << dendl;
-
- if (m_canceled) {
- finish(-ECANCELED);
- return;
- }
-
- if (r == -ENOENT) {
- get_local_group_id();
- return;
- }
-
- if (r == 0) {
- auto iter = m_out_bl.cbegin();
- r = librbd::cls_client::dir_get_name_finish(&iter, &m_group_name);
- }
-
- if (r < 0) {
- derr << "error getting remote group name: " << cpp_strerror(r) << dendl;
- finish(r);
- return;
- }
-
- get_remote_mirror_group();
-}
-
-template <typename I>
-void BootstrapRequest<I>::get_remote_mirror_group() {
- dout(10) << dendl;
-
- librados::ObjectReadOperation op;
- librbd::cls_client::mirror_group_get_start(&op, *m_remote_group_id);
- m_out_bl.clear();
- auto comp = create_rados_callback<
- BootstrapRequest<I>,
- &BootstrapRequest<I>::handle_get_remote_mirror_group>(this);
-
- int r = m_remote_io_ctx.aio_operate(RBD_MIRRORING, comp, &op, &m_out_bl);
- ceph_assert(r == 0);
- comp->release();
-}
-
-template <typename I>
-void BootstrapRequest<I>::handle_get_remote_mirror_group(int r) {
- dout(10) << "r=" << r << dendl;
-
- if (m_canceled) {
- finish(-ECANCELED);
- return;
- }
-
- if (r == -ENOENT) {
- m_remote_mirror_group.state = cls::rbd::MIRROR_GROUP_STATE_DISABLED;
- get_local_group_id();
- return;
- }
-
- if (r == 0) {
- auto iter = m_out_bl.cbegin();
- r = librbd::cls_client::mirror_group_get_finish(&iter,
- &m_remote_mirror_group);
- }
-
- if (r < 0 && r != -ENOENT) {
- derr << "error getting remote mirror group: " << cpp_strerror(r) << dendl;
- finish(r);
- return;
- }
-
- if (m_remote_mirror_group.global_group_id != m_global_group_id) {
- derr << "invalid global group id: "
- << m_remote_mirror_group.global_group_id << dendl;
- finish(-EINVAL);
- return;
- }
-
- list_remote_group_snapshots();
-}
-
-template <typename I>
-void BootstrapRequest<I>::list_remote_group_snapshots() {
- dout(10) << dendl;
-
- auto ctx = create_context_callback<
- BootstrapRequest<I>,
- &BootstrapRequest<I>::handle_list_remote_group_snapshots>(this);
-
- auto req = librbd::group::ListSnapshotsRequest<I>::create(m_remote_io_ctx,
- *m_remote_group_id, true, true, &remote_group_snaps, ctx);
- req->send();
-}
-
-template <typename I>
-void BootstrapRequest<I>::handle_list_remote_group_snapshots(int r) {
- dout(10) << "r=" << r << dendl;
-
- if (m_canceled) {
- finish(-ECANCELED);
- return;
- }
-
- if (r < 0) {
- derr << "error listing remote mirror group snapshots: " << cpp_strerror(r)
- << dendl;
- finish(r);
- return;
- }
-
- if (m_remote_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED) {
- cls::rbd::MirrorSnapshotState state;
- r = get_last_mirror_snapshot_state(remote_group_snaps, &state);
- if (r == -ENOENT) {
- derr << "failed to find remote mirror group snapshot" << dendl;
- finish(-EINVAL);
- return;
- }
- ceph_assert(r == 0);
- m_remote_mirror_group_primary = (state == cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY);
- }
-
- *m_resync_requested = false;
- if (m_local_group_id && !m_local_group_id->empty()) {
- std::string group_header_oid = librbd::util::group_header_name(
- *m_local_group_id);
- std::string value;
- int r = librbd::cls_client::metadata_get(&m_local_io_ctx, group_header_oid,
- RBD_GROUP_RESYNC, &value);
- if (r < 0 && r != -ENOENT) {
- derr << "failed reading metadata: " << cpp_strerror(r) << dendl;
- } else if (r == 0) {
- dout(10) << "local group resync requested : " << m_local_group_id
- << dendl;
- if (m_remote_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED &&
- m_remote_mirror_group_primary) {
- *m_resync_requested = true;
- list_remote_group();
- return;
- }
- dout(10) << "turns out remote is not primary, we cannot resync, will retry later"
- << dendl;
- }
- }
-
- if (m_remote_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED &&
- m_remote_mirror_group_primary) {
- list_remote_group();
- } else {
- get_local_group_id();
- }
-}
-
-template <typename I>
-void BootstrapRequest<I>::list_remote_group() {
- dout(10) << dendl;
-
- librados::ObjectReadOperation op;
- cls::rbd::GroupImageSpec start_after;
- if (!m_images.empty()) {
- start_after = m_images.rbegin()->spec;
- }
- librbd::cls_client::group_image_list_start(&op, start_after, MAX_RETURN);
-
- auto comp = create_rados_callback<
- BootstrapRequest<I>,
- &BootstrapRequest<I>::handle_list_remote_group>(this);
- m_out_bl.clear();
- int r = m_remote_io_ctx.aio_operate(
- librbd::util::group_header_name(*m_remote_group_id), comp, &op,
- &m_out_bl);
- ceph_assert(r == 0);
- comp->release();
-}
-
-template <typename I>
-void BootstrapRequest<I>::handle_list_remote_group(int r) {
- dout(10) << "r=" << r << dendl;
-
- if (m_canceled) {
- finish(-ECANCELED);
- return;
- }
-
- std::vector<cls::rbd::GroupImageStatus> images;
- if (r == 0) {
- auto iter = m_out_bl.cbegin();
- r = librbd::cls_client::group_image_list_finish(&iter, &images);
- }
-
- if (r < 0 && r != -ENOENT) {
- derr << "error listing remote group: " << cpp_strerror(r) << dendl;
- finish(r);
- return;
- }
-
- m_images.insert(m_images.end(), images.begin(), images.end());
-
- if (images.size() == MAX_RETURN) {
- list_remote_group();
- return;
- }
-
- get_remote_mirror_image();
-}
-
-template <typename I>
-void BootstrapRequest<I>::get_remote_mirror_image() {
- while (!m_images.empty() &&
- m_images.front().state != cls::rbd::GROUP_IMAGE_LINK_STATE_ATTACHED) {
- dout(20) << "skipping pool_id= " << m_images.front().spec.pool_id
- << ", image_id=" << m_images.front().spec.image_id << dendl;
- m_images.pop_front();
- }
-
- if (m_images.empty()) {
- get_local_group_id();
- return;
- }
-
- auto &spec = m_images.front().spec;
-
- dout(10) << "pool_id=" << spec.pool_id
- << ", image_id=" << spec.image_id << dendl;
-
- if (!m_pool_meta_cache->remote_pool_meta_exists(spec.pool_id)) {
- derr << "failed to find remote image pool in meta cache" << dendl;
- finish(-ENOENT);
- return;
- }
-
- int r = librbd::util::create_ioctx(m_remote_io_ctx, "remote image pool",
- spec.pool_id, {}, &m_image_io_ctx);
- if (r < 0) {
- derr << "failed to open remote image pool " << spec.pool_id << ": "
- << cpp_strerror(r) << dendl;
- finish(r);
- return;
- }
-
- librados::ObjectReadOperation op;
- librbd::cls_client::mirror_image_get_start(&op, spec.image_id);
- m_out_bl.clear();
- auto comp = create_rados_callback<
- BootstrapRequest<I>,
- &BootstrapRequest<I>::handle_get_remote_mirror_image>(this);
-
- r = m_image_io_ctx.aio_operate(RBD_MIRRORING, comp, &op, &m_out_bl);
- ceph_assert(r == 0);
- comp->release();
-}
-
-template <typename I>
-void BootstrapRequest<I>::handle_get_remote_mirror_image(int r) {
- dout(10) << "r=" << r << dendl;
-
- if (m_canceled) {
- finish(-ECANCELED);
- return;
- }
-
- auto &spec = m_images.front().spec;
- cls::rbd::MirrorImage mirror_image;
-
- if (r == 0) {
- auto iter = m_out_bl.cbegin();
- r = librbd::cls_client::mirror_image_get_finish(&iter, &mirror_image);
- }
-
- if (r < 0 && r != -ENOENT) {
- derr << "error getting remote mirror image: " << cpp_strerror(r) << dendl;
- finish(r);
- return;
- }
-
- m_remote_images[{spec.pool_id, mirror_image.global_image_id}] = spec.image_id;
-
- m_images.pop_front();
-
- get_remote_mirror_image();
-}
-
-template <typename I>
-void BootstrapRequest<I>::get_local_group_id() {
- dout(10) << dendl;
-
- librados::ObjectReadOperation op;
- librbd::cls_client::mirror_group_get_group_id_start(&op, m_global_group_id);
- m_out_bl.clear();
- auto comp = create_rados_callback<
- BootstrapRequest<I>,
- &BootstrapRequest<I>::handle_get_local_group_id>(this);
-
- int r = m_local_io_ctx.aio_operate(RBD_MIRRORING, comp, &op, &m_out_bl);
- ceph_assert(r == 0);
- comp->release();
-}
-
-template <typename I>
-void BootstrapRequest<I>::handle_get_local_group_id(int r) {
- dout(10) << "r=" << r << dendl;
-
- if (m_canceled) {
- finish(-ECANCELED);
- return;
- }
-
- if (r == -ENOENT &&
- m_remote_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED &&
- m_remote_mirror_group_primary) {
- ceph_assert(!m_group_name.empty());
- get_local_group_id_by_name();
- return;
- }
-
- if (r == 0) {
- auto iter = m_out_bl.cbegin();
- r = librbd::cls_client::mirror_group_get_group_id_finish(
- &iter, m_local_group_id);
- }
-
- if (r < 0) {
- if (r != -ENOENT) {
- derr << "error getting local group id: " << cpp_strerror(r) << dendl;
- } else {
- m_local_mirror_group.state = cls::rbd::MIRROR_GROUP_STATE_DISABLED;
- r = 0;
- }
- finish(r);
- return;
- }
-
- get_local_group_name();
-}
-
-template <typename I>
-void BootstrapRequest<I>::get_local_group_name() {
- dout(10) << dendl;
-
- librados::ObjectReadOperation op;
- librbd::cls_client::dir_get_name_start(&op, *m_local_group_id);
- m_out_bl.clear();
- auto comp = create_rados_callback<
- BootstrapRequest<I>,
- &BootstrapRequest<I>::handle_get_local_group_name>(this);
-
- int r = m_local_io_ctx.aio_operate(RBD_GROUP_DIRECTORY, comp, &op,
- &m_out_bl);
- ceph_assert(r == 0);
- comp->release();
-}
-
-template <typename I>
-void BootstrapRequest<I>::handle_get_local_group_name(int r) {
- dout(10) << "r=" << r << dendl;
-
- if (m_canceled) {
- finish(-ECANCELED);
- return;
- }
-
- std::string local_group_name;
-
- if (r == 0) {
- auto iter = m_out_bl.cbegin();
- r = librbd::cls_client::dir_get_name_finish(&iter, &local_group_name);
- }
-
- if (r < 0) {
- derr << "error getting local group name: " << cpp_strerror(r) << dendl;
- if (r == -ENOENT) {
- r = -EEXIST; // split-brain
- }
- finish(r);
- return;
- }
-
- if (m_group_name.empty()) {
- m_group_name = local_group_name;
- } else if (m_group_name != local_group_name && m_remote_mirror_group_primary) {
- derr << "local group name '" << local_group_name << "' does not match "
- << "remote group name '" << m_group_name << "'" << dendl;
- finish(-EINVAL);
- return;
- }
-
- get_local_mirror_group();
-}
-
-template <typename I>
-void BootstrapRequest<I>::get_local_group_id_by_name() {
- dout(10) << dendl;
-
- librados::ObjectReadOperation op;
- librbd::cls_client::dir_get_id_start(&op, m_group_name);
- m_out_bl.clear();
- auto comp = create_rados_callback<
- BootstrapRequest<I>,
- &BootstrapRequest<I>::handle_get_local_group_id_by_name>(this);
-
- int r = m_local_io_ctx.aio_operate(RBD_GROUP_DIRECTORY, comp, &op, &m_out_bl);
- ceph_assert(r == 0);
- comp->release();
-}
-
-template <typename I>
-void BootstrapRequest<I>::handle_get_local_group_id_by_name(int r) {
- dout(10) << "r=" << r << dendl;
-
- if (m_canceled) {
- finish(-ECANCELED);
- return;
- }
-
- if (r == -ENOENT) {
- create_local_group_id();
- return;
- }
-
- if (r == 0) {
- auto iter = m_out_bl.cbegin();
- r = librbd::cls_client::dir_get_id_finish(&iter, m_local_group_id);
- }
-
- if (r < 0) {
- derr << "error getting local group id: " << cpp_strerror(r) << dendl;
- finish(r);
- return;
- }
-
- m_local_group_id_by_name = true;
- get_local_mirror_group();
-}
-
-template <typename I>
-void BootstrapRequest<I>::get_local_mirror_group() {
- dout(10) << dendl;
-
- librados::ObjectReadOperation op;
- librbd::cls_client::mirror_group_get_start(&op, *m_local_group_id);
- m_out_bl.clear();
- auto comp = create_rados_callback<
- BootstrapRequest<I>,
- &BootstrapRequest<I>::handle_get_local_mirror_group>(this);
-
- int r = m_local_io_ctx.aio_operate(RBD_MIRRORING, comp, &op, &m_out_bl);
- ceph_assert(r == 0);
- comp->release();
-}
-
-template <typename I>
-void BootstrapRequest<I>::handle_get_local_mirror_group(int r) {
- dout(10) << "r=" << r << dendl;
-
- if (m_canceled) {
- finish(-ECANCELED);
- return;
- }
-
- if (r == 0) {
- auto iter = m_out_bl.cbegin();
- r = librbd::cls_client::mirror_group_get_finish(&iter,
- &m_local_mirror_group);
- }
-
- if (r == -ENOENT) {
- if (m_local_group_id_by_name) {
- derr << "local group is not mirrored" << dendl;
- finish(-EINVAL);
- return;
- }
- if (m_remote_mirror_group.state != cls::rbd::MIRROR_GROUP_STATE_ENABLED ||
- !m_remote_mirror_group_primary) {
- derr << "can't find primary for group: " << m_group_name << dendl;
- finish(-EEXIST); // split-brain
- return;
- }
- r = 0;
- }
-
- if (r < 0) {
- derr << "error getting local mirror group: " << cpp_strerror(r) << dendl;
- finish(r);
- return;
- }
-
- if (m_local_mirror_group.global_group_id != m_global_group_id) {
- finish(-ERESTART);
- return;
- } else if (m_local_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_DISABLING) {
- derr << "group with same name exists: " << m_group_name
- << " and is currently disabling" << dendl;
- finish(-ERESTART); // The other group replayer might be removing the
- // group already, so wait and retry later.
- return;
- }
-
- dout(20) << m_local_mirror_group << dendl;
-
- list_local_group_snapshots();
-}
-
-template <typename I>
-void BootstrapRequest<I>::list_local_group_snapshots() {
- dout(10) << dendl;
-
- auto ctx = create_context_callback<
- BootstrapRequest<I>,
- &BootstrapRequest<I>::handle_list_local_group_snapshots>(this);
-
- auto req = librbd::group::ListSnapshotsRequest<I>::create(m_local_io_ctx,
- *m_local_group_id, true, true, &local_group_snaps, ctx);
- req->send();
-}
-
-template <typename I>
-void BootstrapRequest<I>::handle_list_local_group_snapshots(int r) {
- dout(10) << "r=" << r << dendl;
-
- if (m_canceled) {
- finish(-ECANCELED);
- return;
- }
-
- if (r < 0) {
- derr << "error listing local mirror group snapshots: " << cpp_strerror(r)
- << dendl;
- finish(r);
- return;
- }
-
- for (auto it : local_group_snaps) {
- m_local_group_snaps->insert(make_pair(it.id, it));
- }
-
- if (m_local_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED) {
- cls::rbd::MirrorSnapshotState state;
- r = get_last_mirror_snapshot_state(local_group_snaps, &state);
- if (r == -ENOENT) {
- derr << "failed to find local mirror group snapshot" << dendl;
- } else {
- if (m_remote_mirror_group_primary &&
- state == cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED) {
- // if local snapshot is primary demoted, check if there is demote snapshot
- // in remote, if not then split brain
- if (!is_demoted_snap_exists(remote_group_snaps)
- && *m_resync_requested == false) {
- finish(-EEXIST);
- return;
- }
- }
- }
- m_local_mirror_group_primary = (state == cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY);
- }
-
- if (m_remote_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED) {
- if (m_remote_mirror_group_primary) {
- if (m_local_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED &&
- m_local_mirror_group_primary) {
- derr << "both remote and local groups are primary, global group id: "
- << m_global_group_id << dendl;
- }
- } else if (m_local_mirror_group.state != cls::rbd::MIRROR_GROUP_STATE_ENABLED ||
- !m_local_mirror_group_primary) {
- derr << "both remote and local groups are not primary, global group id: "
- << m_global_group_id << dendl;
- finish(-EREMOTEIO);
- return;
- }
- } else if (m_local_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED &&
- !m_local_mirror_group_primary) {
- // trigger group removal
- derr << "local group is enabled and is not primary, global group id: "
- << m_global_group_id << dendl;
- m_local_mirror_group.state = cls::rbd::MIRROR_GROUP_STATE_DISABLED;
- }
-
- list_local_group();
-}
-
-template <typename I>
-void BootstrapRequest<I>::list_local_group() {
- dout(10) << dendl;
-
- librados::ObjectReadOperation op;
- cls::rbd::GroupImageSpec start_after;
- if (!m_images.empty()) {
- start_after = m_images.rbegin()->spec;
- }
- librbd::cls_client::group_image_list_start(&op, start_after, MAX_RETURN);
- auto comp = create_rados_callback<
- BootstrapRequest<I>,
- &BootstrapRequest<I>::handle_list_local_group>(this);
- m_out_bl.clear();
- int r = m_local_io_ctx.aio_operate(
- librbd::util::group_header_name(*m_local_group_id), comp, &op, &m_out_bl);
- ceph_assert(r == 0);
- comp->release();
-}
-
-template <typename I>
-void BootstrapRequest<I>::handle_list_local_group(int r) {
- dout(10) << "r=" << r << dendl;
-
- if (m_canceled) {
- finish(-ECANCELED);
- return;
- }
-
- std::vector<cls::rbd::GroupImageStatus> images;
- if (r == 0) {
- auto iter = m_out_bl.cbegin();
- r = librbd::cls_client::group_image_list_finish(&iter, &images);
- }
-
- if (r < 0 && r != -ENOENT) {
- derr << "error listing local group: " << cpp_strerror(r) << dendl;
- finish(r);
- return;
- }
-
- m_images.insert(m_images.end(), images.begin(), images.end());
-
- if (images.size() == MAX_RETURN) {
- list_local_group();
- return;
- }
-
- get_local_mirror_image();
-}
-
-template <typename I>
-void BootstrapRequest<I>::get_local_mirror_image() {
- if (m_images.empty()) {
- remove_local_image_from_group();
- return;
- }
-
- auto &spec = m_images.front().spec;
-
- dout(10) << spec.pool_id << " " << spec.image_id << dendl;
-
- int r = librbd::util::create_ioctx(m_local_io_ctx, "local image pool",
- spec.pool_id, {}, &m_image_io_ctx);
- if (r < 0) {
- derr << "failed to open local image pool " << spec.pool_id << ": "
- << cpp_strerror(r) << dendl;
- finish(r);
- return;
- }
-
- librados::ObjectReadOperation op;
- librbd::cls_client::mirror_image_get_start(&op, spec.image_id);
- m_out_bl.clear();
- auto comp = create_rados_callback<
- BootstrapRequest<I>,
- &BootstrapRequest<I>::handle_get_local_mirror_image>(this);
-
- r = m_image_io_ctx.aio_operate(RBD_MIRRORING, comp, &op, &m_out_bl);
- ceph_assert(r == 0);
- comp->release();
-}
-
-template <typename I>
-void BootstrapRequest<I>::handle_get_local_mirror_image(int r) {
- dout(10) << "r=" << r << dendl;
-
- if (m_canceled) {
- finish(-ECANCELED);
- return;
- }
-
- auto &spec = m_images.front().spec;
- cls::rbd::MirrorImage mirror_image;
-
- if (r == 0) {
- auto iter = m_out_bl.cbegin();
- r = librbd::cls_client::mirror_image_get_finish(&iter, &mirror_image);
- }
-
- if (r < 0) {
- if (r != -ENOENT) {
- derr << "error getting local mirror image: " << cpp_strerror(r) << dendl;
- finish(r);
- return;
- }
- } else {
- if (m_local_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED &&
- m_local_mirror_group_primary) {
- dout(10) << "add primary to replayer queue: " << spec.pool_id << " "
- << spec.image_id << " " << mirror_image.global_image_id
- << dendl;
- m_local_images.insert({spec.pool_id, mirror_image.global_image_id});
- } else if (m_remote_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED &&
- m_remote_mirror_group_primary &&
- m_local_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED &&
- m_local_mirror_group.global_group_id == m_global_group_id &&
- has_remote_image(spec.pool_id, mirror_image.global_image_id)
- && *m_resync_requested == false) {
- dout(10) << "add secondary to replayer queue: " << spec.pool_id << " "
- << spec.image_id << " " << mirror_image.global_image_id
- << dendl;
- m_local_images.insert({spec.pool_id, mirror_image.global_image_id});
- } else {
- if (m_local_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_DISABLING) {
- dout(10) << "local group with global_group_id: " << m_local_mirror_group.global_group_id
- << " is in disabling state, will retry later." << dendl;
- finish(-ERESTART);
- return;
- }
- dout(10) << "add to trash queue: " << spec.pool_id << " "
- << spec.image_id << " " << mirror_image.global_image_id
- << dendl;
- m_local_trash_images[{spec.pool_id, mirror_image.global_image_id}] =
- spec.image_id;
- }
- }
-
- m_images.pop_front();
-
- get_local_mirror_image();
-}
-
-template <typename I>
-void BootstrapRequest<I>::remove_local_image_from_group() {
- if (m_local_trash_images.empty()) {
- disable_local_mirror_group();
- return;
- }
-
- auto &[pool_id, global_image_id] = m_local_trash_images.begin()->first;
- auto &image_id = m_local_trash_images.begin()->second;
-
- dout(10) << "pool_id=" << pool_id << ", image_id=" << image_id << dendl;
-
- int r = librbd::util::create_ioctx(m_local_io_ctx, "local image pool",
- pool_id, {}, &m_image_io_ctx);
- if (r < 0) {
- derr << "failed to open local image pool " << pool_id << ": "
- << cpp_strerror(r) << dendl;
- handle_remove_local_image_from_group(-ENOENT);
- return;
- }
-
- auto ctx = create_context_callback<
- BootstrapRequest<I>,
- &BootstrapRequest<I>::handle_remove_local_image_from_group>(this);
-
- auto req = librbd::group::RemoveImageRequest<I>::create(
- m_local_io_ctx, *m_local_group_id, m_image_io_ctx, image_id, ctx);
- req->send();
-}
-
-template <typename I>
-void BootstrapRequest<I>::handle_remove_local_image_from_group(int r) {
- dout(10) << "r=" << r << dendl;
-
- if (r < 0 && r != -ENOENT) {
- derr << "error removing mirror image from group: " << cpp_strerror(r) << dendl;
- finish(r);
- return;
- }
-
- move_local_image_to_trash();
-}
-
-template <typename I>
-void BootstrapRequest<I>::move_local_image_to_trash() {
- ceph_assert(!m_local_trash_images.empty());
- auto &[pool_id, global_image_id] = m_local_trash_images.begin()->first;
-
- dout(10) << "pool_id=" << pool_id << ", global_image_id=" << global_image_id
- << dendl;
-
- int r = librbd::util::create_ioctx(m_local_io_ctx, "local image pool",
- pool_id, {}, &m_image_io_ctx);
- if (r < 0) {
- derr << "failed to open local image pool " << pool_id << ": "
- << cpp_strerror(r) << dendl;
- handle_move_local_image_to_trash(-ENOENT);
- return;
- }
-
- auto ctx = create_context_callback<
- BootstrapRequest<I>,
- &BootstrapRequest<I>::handle_move_local_image_to_trash>(this);
-
- auto req = image_deleter::TrashMoveRequest<I>::create(
- m_image_io_ctx, global_image_id, *m_resync_requested,
- m_threads->work_queue, ctx);
- req->send();
-}
-
-template <typename I>
-void BootstrapRequest<I>::handle_move_local_image_to_trash(int r) {
- dout(10) << "r=" << r << dendl;
-
- if (r < 0 && r != -ENOENT) {
- derr << "error moving mirror image to trash: " << cpp_strerror(r) << dendl;
- finish(r);
- return;
- }
-
- m_local_trash_images.erase(m_local_trash_images.begin());
-
- remove_local_image_from_group();
-}
-
-template <typename I>
-void BootstrapRequest<I>::disable_local_mirror_group() {
- if (m_local_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED &&
- m_local_mirror_group.global_group_id == m_global_group_id &&
- *m_resync_requested == false) {
- finish(0);
- return;
- }
-
- dout(10) << dendl;
-
- librados::ObjectWriteOperation op;
- m_local_mirror_group.state = cls::rbd::MIRROR_GROUP_STATE_DISABLING;
- librbd::cls_client::mirror_group_set(&op, *m_local_group_id,
- m_local_mirror_group);
-
- auto comp = create_rados_callback<
- BootstrapRequest<I>,
- &BootstrapRequest<I>::handle_disable_local_mirror_group>(this);
-
- int r = m_local_io_ctx.aio_operate(RBD_MIRRORING, comp, &op);
- ceph_assert(r == 0);
- comp->release();
+ m_pool_meta_cache(pool_meta_cache),
+ m_resync_requested(resync_requested),
+ m_local_group_ctx(local_group_ctx),
+ m_image_replayers(image_replayers),
+ m_state_builder(state_builder),
+ m_on_finish(on_finish),
+ m_lock(ceph::make_mutex(librbd::util::unique_lock_name(
+ "BootstrapRequest::m_lock", this))){
+ dout(10) << "global_group_id=" << m_global_group_id << dendl;
}
template <typename I>
-void BootstrapRequest<I>::handle_disable_local_mirror_group(int r) {
- dout(10) << "r=" << r << dendl;
-
- if (r < 0 && r != -ENOENT) {
- derr << "error disabling local mirror group: " << cpp_strerror(r) << dendl;
- finish(r);
- return;
- }
+void BootstrapRequest<I>::send() {
+ ceph_assert(*m_state_builder == nullptr);
+// TODO : Create this in PrepareLocalGroupRequest/PrepareRemoteGroupRequest ?
+ *m_state_builder = GroupStateBuilder<I>::create(m_global_group_id);
- remove_local_mirror_group();
+ prepare_local_group();
}
template <typename I>
-void BootstrapRequest<I>::remove_local_mirror_group() {
- if (m_local_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED &&
- m_local_mirror_group.global_group_id == m_global_group_id) {
- finish(0);
- return;
- }
-
+void BootstrapRequest<I>::cancel() {
dout(10) << dendl;
- librados::ObjectWriteOperation op;
- librbd::cls_client::mirror_group_remove(&op, *m_local_group_id);
-
- auto comp = create_rados_callback<
- BootstrapRequest<I>,
- &BootstrapRequest<I>::handle_remove_local_mirror_group>(this);
-
- int r = m_local_io_ctx.aio_operate(RBD_MIRRORING, comp, &op);
- ceph_assert(r == 0);
- comp->release();
-}
-
-template <typename I>
-void BootstrapRequest<I>::handle_remove_local_mirror_group(int r) {
- dout(10) << "r=" << r << dendl;
-
- if (r < 0 && r != -ENOENT) {
- derr << "error removing local mirror group: " << cpp_strerror(r) << dendl;
- finish(r);
- return;
- }
-
- m_local_mirror_group.state = cls::rbd::MIRROR_GROUP_STATE_DISABLED;
- if (r != -ENOENT && (m_local_mirror_group.global_group_id == m_global_group_id) &&
- (m_remote_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED &&
- m_remote_mirror_group_primary) && *m_resync_requested == false) {
- create_local_mirror_group();
- } else {
- remove_local_group();
- }
+ m_canceled = true;
}
template <typename I>
-void BootstrapRequest<I>::remove_local_group() {
- dout(10) << m_group_name << " " << *m_local_group_id << dendl;
-
- ceph_assert(!m_local_group_id->empty());
- ceph_assert(!m_group_name.empty());
-
- librados::ObjectWriteOperation op;
- op.remove();
- auto comp = create_rados_callback<
- BootstrapRequest<I>,
- &BootstrapRequest<I>::handle_remove_local_group>(this);
+void BootstrapRequest<I>::prepare_local_group() {
+ dout(10) << dendl;
- int r = m_local_io_ctx.aio_operate(
- librbd::util::group_header_name(*m_local_group_id), comp, &op);
- ceph_assert(r == 0);
- comp->release();
+ m_local_group_removed = false;
+ auto ctx = create_context_callback<
+ BootstrapRequest, &BootstrapRequest<I>::handle_prepare_local_group>(this);
+ auto req = PrepareLocalGroupRequest<I>::create(
+ m_local_io_ctx, m_global_group_id, &m_prepare_local_group_name,
+ m_state_builder, m_threads->work_queue, ctx);
+ req->send();
}
template <typename I>
-void BootstrapRequest<I>::handle_remove_local_group(int r) {
+void BootstrapRequest<I>::handle_prepare_local_group(int r) {
dout(10) << "r=" << r << dendl;
- if (r < 0 && r != -ENOENT) {
- derr << "error removing local group: " << cpp_strerror(r) << dendl;
+ if (r == -ENOENT) {
+ dout(10) << "local group does not exist" << dendl;
+ } else if (r < 0) {
+ derr << "error preparing local group: " << cpp_strerror(r)
+ << dendl;
finish(r);
return;
}
- remove_local_group_id();
+ if (!m_prepare_local_group_name.empty()) {
+ std::lock_guard locker{m_lock};
+ m_local_group_name = m_prepare_local_group_name;
+ }
+
+ prepare_remote_group();
}
template <typename I>
-void BootstrapRequest<I>::remove_local_group_id() {
+void BootstrapRequest<I>::prepare_remote_group() {
dout(10) << dendl;
- librados::ObjectWriteOperation op;
- librbd::cls_client::group_dir_remove(&op, m_group_name, *m_local_group_id);
-
- auto comp = create_rados_callback<
- BootstrapRequest<I>,
- &BootstrapRequest<I>::handle_remove_local_group_id>(this);
-
- int r = m_local_io_ctx.aio_operate(RBD_GROUP_DIRECTORY, comp, &op);
- ceph_assert(r == 0);
- comp->release();
+ Context *ctx = create_context_callback<
+ BootstrapRequest, &BootstrapRequest<I>::handle_prepare_remote_group>(this);
+ auto req = PrepareRemoteGroupRequest<I>::create(
+ m_remote_io_ctx, m_global_group_id, &m_prepare_remote_group_name,
+ m_state_builder, ctx);
+ req->send();
}
template <typename I>
-void BootstrapRequest<I>::handle_remove_local_group_id(int r) {
+void BootstrapRequest<I>::handle_prepare_remote_group(int r) {
dout(10) << "r=" << r << dendl;
- if (r < 0 && r != -ENOENT) {
- derr << "error removing local group id: " << cpp_strerror(r) << dendl;
+ auto state_builder = *m_state_builder;
+
+ if (state_builder->is_local_primary()) {
+ dout(5) << "local group is primary" << dendl;
+ finish(0);
+ return;
+ } else if (r == -ENOENT) {
+ if (state_builder->remote_group_id.empty()) {
+ if (state_builder->local_group_id.empty()) {
+ // Neither group exists
+ m_local_group_removed = true; //FIXME
+ finish(0);
+ return;
+ } else if (state_builder->local_promotion_state ==
+ librbd::mirror::PROMOTION_STATE_NON_PRIMARY){
+ remove_local_group();
+ return;
+ } else {
+ // Do not remove the group if the promotion state is orphan or unknown
+ finish(-ENOLINK);
+ return;
+ }
+ }
+ } else if (r < 0) {
+ derr << "error preparing remote group: " << cpp_strerror(r)
+ << dendl;
finish(r);
return;
}
- notify_mirroring_watcher();
-}
-
-template <typename I>
-void BootstrapRequest<I>::create_local_group_id() {
- dout(10) << dendl;
-
- *m_local_group_id = librbd::util::generate_image_id(m_local_io_ctx);
-
- librados::ObjectWriteOperation op;
- librbd::cls_client::group_dir_add(&op, m_group_name, *m_local_group_id);
-
- auto comp = create_rados_callback<
- BootstrapRequest<I>,
- &BootstrapRequest<I>::handle_create_local_group_id>(this);
-
- int r = m_local_io_ctx.aio_operate(RBD_GROUP_DIRECTORY, comp, &op);
- ceph_assert(r == 0);
- comp->release();
-}
-
-template <typename I>
-void BootstrapRequest<I>::handle_create_local_group_id(int r) {
- dout(10) << "r=" << r << dendl;
+ if (!state_builder->is_remote_primary()) {
+ if (state_builder->local_group_id.empty()) {
+ // local group does not exist and remote is not primary
+ dout(10) << "local group does not exist and remote group is not primary"
+ << dendl;
+ finish(-EREMOTEIO);
+ return;
+ } else if (!state_builder->is_linked()) {
+ dout(10) << "local group is not non-primary and remote group is not primary"
+ << dendl;
+ finish(-EREMOTEIO);
+ return;
+ }
+ }
- if (r < 0) {
- derr << "error creating local group id: " << cpp_strerror(r) << dendl;
- finish(r);
+ if (state_builder->local_group_id.empty()) {
+ // create the local group
+ create_local_group();
return;
+ } else {
+ // Local group is secondary.
+ if (m_local_group_name != (*m_state_builder)->group_name) {
+ finish(-EINVAL);
+ return;
+ }
+ // See if resync is set.
+ get_local_group_meta();
}
-
- create_local_group();
}
template <typename I>
void BootstrapRequest<I>::create_local_group() {
dout(10) << dendl;
-
- librados::ObjectWriteOperation op;
- op.create(true);
- auto comp = create_rados_callback<
- BootstrapRequest<I>,
- &BootstrapRequest<I>::handle_create_local_group>(this);
-
- int r = m_local_io_ctx.aio_operate(
- librbd::util::group_header_name(*m_local_group_id), comp, &op);
- ceph_assert(r == 0);
- comp->release();
+ auto ctx = create_context_callback<
+ BootstrapRequest, &BootstrapRequest<I>::handle_create_local_group>(this);
+ auto req = CreateLocalGroupRequest<I>::create(
+ m_local_io_ctx, m_global_group_id, *m_state_builder, ctx);
+ req->send();
}
template <typename I>
dout(10) << "r=" << r << dendl;
if (r < 0) {
- derr << "error creating local group: " << cpp_strerror(r) << dendl;
+ derr << "error creating local group: " << cpp_strerror(r)
+ << dendl;
finish(r);
return;
}
-
- create_local_mirror_group();
+ finish(0);
}
template <typename I>
-void BootstrapRequest<I>::create_local_mirror_group() {
+void BootstrapRequest<I>::remove_local_group() {
dout(10) << dendl;
- ceph_assert(
- m_remote_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED &&
- m_remote_mirror_group_primary);
-
- librados::ObjectWriteOperation op;
- m_local_mirror_group = {m_global_group_id,
- m_remote_mirror_group.mirror_image_mode,
- cls::rbd::MIRROR_GROUP_STATE_ENABLED};
- librbd::cls_client::mirror_group_set(&op, *m_local_group_id,
- m_local_mirror_group);
- auto comp = create_rados_callback<
- BootstrapRequest<I>,
- &BootstrapRequest<I>::handle_create_local_mirror_group>(this);
-
- int r = m_local_io_ctx.aio_operate(RBD_MIRRORING, comp, &op);
- ceph_assert(r == 0);
- comp->release();
-}
-
-template <typename I>
-void BootstrapRequest<I>::handle_create_local_mirror_group(int r) {
- dout(10) << "r=" << r << dendl;
-
- if (r < 0) {
- derr << "error creating local mirror group: " << cpp_strerror(r) << dendl;
- finish(r);
- return;
- }
-
- notify_mirroring_watcher();
-}
-
-template <typename I>
-void BootstrapRequest<I>::notify_mirroring_watcher() {
- dout(10) << dendl;
+ ceph_assert(!(*m_state_builder)->local_group_id.empty());
auto ctx = create_context_callback<
- BootstrapRequest<I>,
- &BootstrapRequest<I>::handle_notify_mirroring_watcher>(this);
+ BootstrapRequest,
+ &BootstrapRequest<I>::handle_remove_local_group>(this);
- librbd::MirroringWatcher<I>::notify_group_updated(
- m_local_io_ctx, m_local_mirror_group.state, *m_local_group_id,
- m_global_group_id, m_local_images.size(), ctx);
+ auto req = RemoveLocalGroupRequest<I>::create(
+ m_local_io_ctx, m_global_group_id, *m_resync_requested,
+ m_threads->work_queue, ctx);
+ req->send();
}
template <typename I>
-void BootstrapRequest<I>::handle_notify_mirroring_watcher(int r) {
+void BootstrapRequest<I>::handle_remove_local_group(int r) {
dout(10) << "r=" << r << dendl;
- if (r < 0) {
- derr << "failed to notify mirror group update: " << cpp_strerror(r)
- << dendl;
+ if (r < 0 && r != -ENOENT) {
+ derr << "error removing local group: " << cpp_strerror(r) << dendl;
+ finish(r);
+ return;
}
-
+ m_local_group_removed = true;
finish(0);
}
void BootstrapRequest<I>::finish(int r) {
dout(10) << "r=" << r << dendl;
- ceph_assert(r != -ENOENT);
+ if (m_canceled) {
+ r = -ECANCELED;
+ m_on_finish->complete(r);
+ return;
+ }
+
if (r == 0) {
- if (m_local_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_DISABLED) {
+ if (m_local_group_removed) {
r = -ENOENT;
} else {
- *m_local_group_ctx = {m_group_name, *m_local_group_id, m_global_group_id,
- m_local_mirror_group_primary, m_local_io_ctx};
+ *m_local_group_ctx = {(*m_state_builder)->group_name,
+ (*m_state_builder)->local_group_id,
+ m_global_group_id,
+ (*m_state_builder)->is_local_primary(),
+ m_local_io_ctx};
r = create_replayers();
}
}
- if (r == -ENOENT && m_local_mirror_group.global_group_id != m_global_group_id) {
- r = -ERESTART; // try again
- }
-
m_on_finish->complete(r);
}
return 0;
}
+ auto state_builder = *m_state_builder;
int r = 0;
- if (m_remote_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED &&
- m_remote_mirror_group_primary) {
- for (auto &[p, remote_image_id] : m_remote_images) {
- auto &remote_pool_id = p.first;
- auto &global_image_id = p.second;
+
+ if ((*m_state_builder)->is_local_primary()) {
+ // The ImageReplayers are required to run even when the group is primary in
+ // order to update the image status for the mirror pool status to be healthy.
+ for (auto &[global_image_id, p] : (*m_state_builder)->local_images) {
+ auto &local_pool_id = p.first;
m_image_replayers->emplace_back(librados::IoCtx(), nullptr);
auto &local_io_ctx = m_image_replayers->back().first;
auto &image_replayer = m_image_replayers->back().second;
- RemotePoolMeta remote_pool_meta;
- r = m_pool_meta_cache->get_remote_pool_meta(remote_pool_id,
- &remote_pool_meta);
- if (r < 0 || remote_pool_meta.mirror_peer_uuid.empty()) {
- derr << "failed to retrieve mirror peer uuid from remote image pool"
- << dendl;
- r = -ENOENT;
+ LocalPoolMeta local_pool_meta;
+ r = m_pool_meta_cache->get_local_pool_meta(local_pool_id,
+ &local_pool_meta);
+ if (r < 0 || local_pool_meta.mirror_uuid.empty()) {
+ if (r == 0 || r == -ENOENT) {
+ r = -EINVAL;
+ }
+ derr << "failed to retrieve mirror uuid from local image pool" << dendl;
break;
}
- librados::IoCtx remote_io_ctx;
- r = librbd::util::create_ioctx(m_remote_io_ctx, "remote image pool",
- remote_pool_id, {}, &remote_io_ctx);
+ r = librbd::util::create_ioctx(m_local_io_ctx, "local image pool",
+ local_pool_id, {}, &local_io_ctx);
if (r < 0) {
- derr << "failed to open remote image pool " << remote_pool_id << ": "
+ derr << "failed to open local image pool " << local_pool_id << ": "
<< cpp_strerror(r) << dendl;
if (r == -ENOENT) {
r = -EINVAL;
break;
}
- int64_t local_pool_id = librados::Rados(m_local_io_ctx).pool_lookup(
- remote_io_ctx.get_pool_name().c_str());
+ int64_t remote_pool_id = librados::Rados(m_remote_io_ctx).pool_lookup(
+ local_io_ctx.get_pool_name().c_str());
- LocalPoolMeta local_pool_meta;
- r = m_pool_meta_cache->get_local_pool_meta(local_pool_id,
- &local_pool_meta);
- if (r < 0 || local_pool_meta.mirror_uuid.empty()) {
- if (r == 0 || r == -ENOENT) {
- r = -EINVAL;
- }
- derr << "failed to retrieve mirror uuid from local image pool" << dendl;
+ RemotePoolMeta remote_pool_meta;
+ r = m_pool_meta_cache->get_remote_pool_meta(remote_pool_id,
+ &remote_pool_meta);
+ if (r < 0 || remote_pool_meta.mirror_peer_uuid.empty()) {
+ derr << "failed to retrieve mirror peer uuid from remote image pool"
+ << dendl;
+ r = -ENOENT;
break;
}
- r = librbd::util::create_ioctx(m_local_io_ctx, "local image pool",
- local_pool_id, {}, &local_io_ctx);
+ librados::IoCtx remote_io_ctx;
+ r = librbd::util::create_ioctx(m_remote_io_ctx, "remote image pool",
+ remote_pool_id, {}, &remote_io_ctx);
if (r < 0) {
- derr << "failed to open local image pool " << local_pool_id << ": "
+ derr << "failed to open remote image pool " << remote_pool_id << ": "
<< cpp_strerror(r) << dendl;
if (r == -ENOENT) {
r = -EINVAL;
// TODO only a single peer is currently supported
image_replayer->add_peer({local_pool_meta.mirror_uuid, remote_io_ctx,
remote_pool_meta, m_remote_status_updater});
-
- (*m_image_replayer_index)[{remote_pool_id, remote_image_id}] = image_replayer;
}
- } else if (m_local_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED &&
- m_local_mirror_group_primary) {
- for (auto &[local_pool_id, global_image_id] : m_local_images) {
+ } else if (!state_builder->remote_group_id.empty()) {
+ for (auto &[remote_pool_id, global_image_id] : (*m_state_builder)->remote_images) {
+
m_image_replayers->emplace_back(librados::IoCtx(), nullptr);
auto &local_io_ctx = m_image_replayers->back().first;
auto &image_replayer = m_image_replayers->back().second;
- LocalPoolMeta local_pool_meta;
- r = m_pool_meta_cache->get_local_pool_meta(local_pool_id,
- &local_pool_meta);
- if (r < 0 || local_pool_meta.mirror_uuid.empty()) {
- if (r == 0 || r == -ENOENT) {
- r = -EINVAL;
- }
- derr << "failed to retrieve mirror uuid from local image pool" << dendl;
+ RemotePoolMeta remote_pool_meta;
+ r = m_pool_meta_cache->get_remote_pool_meta(remote_pool_id,
+ &remote_pool_meta);
+ if (r < 0 || remote_pool_meta.mirror_peer_uuid.empty()) {
+ derr << "failed to retrieve mirror peer uuid from remote image pool"
+ << dendl;
+ r = -ENOENT;
break;
}
- r = librbd::util::create_ioctx(m_local_io_ctx, "local image pool",
- local_pool_id, {}, &local_io_ctx);
+ librados::IoCtx remote_io_ctx;
+ r = librbd::util::create_ioctx(m_remote_io_ctx, "remote image pool",
+ remote_pool_id, {}, &remote_io_ctx);
if (r < 0) {
- derr << "failed to open local image pool " << local_pool_id << ": "
+ derr << "failed to open remote image pool " << remote_pool_id << ": "
<< cpp_strerror(r) << dendl;
if (r == -ENOENT) {
r = -EINVAL;
break;
}
- int64_t remote_pool_id = librados::Rados(m_remote_io_ctx).pool_lookup(
- local_io_ctx.get_pool_name().c_str());
+ int64_t local_pool_id = librados::Rados(m_local_io_ctx).pool_lookup(
+ remote_io_ctx.get_pool_name().c_str());
- RemotePoolMeta remote_pool_meta;
- r = m_pool_meta_cache->get_remote_pool_meta(remote_pool_id,
- &remote_pool_meta);
- if (r < 0 || remote_pool_meta.mirror_peer_uuid.empty()) {
- derr << "failed to retrieve mirror peer uuid from remote image pool"
- << dendl;
- r = -ENOENT;
+ LocalPoolMeta local_pool_meta;
+ r = m_pool_meta_cache->get_local_pool_meta(local_pool_id,
+ &local_pool_meta);
+ if (r < 0 || local_pool_meta.mirror_uuid.empty()) {
+ if (r == 0 || r == -ENOENT) {
+ r = -EINVAL;
+ }
+ derr << "failed to retrieve mirror uuid from local image pool" << dendl;
break;
}
- librados::IoCtx remote_io_ctx;
- r = librbd::util::create_ioctx(m_remote_io_ctx, "remote image pool",
- remote_pool_id, {}, &remote_io_ctx);
+ r = librbd::util::create_ioctx(m_local_io_ctx, "local image pool",
+ local_pool_id, {}, &local_io_ctx);
if (r < 0) {
- derr << "failed to open remote image pool " << remote_pool_id << ": "
+ derr << "failed to open local image pool " << local_pool_id << ": "
<< cpp_strerror(r) << dendl;
if (r == -ENOENT) {
r = -EINVAL;
image_replayer->add_peer({local_pool_meta.mirror_uuid, remote_io_ctx,
remote_pool_meta, m_remote_status_updater});
}
- } else {
- ceph_abort();
}
if (r < 0) {
m_image_replayers->clear();
return r;
}
-
return 0;
}
+template <typename I>
+void BootstrapRequest<I>::get_local_group_meta() {
+ dout(10) << dendl;
+
+ *m_resync_requested = false;
+ librados::ObjectReadOperation op;
+ librbd::cls_client::metadata_get_start(&op, RBD_GROUP_RESYNC);
+
+ m_out_bl.clear();
+
+ std::string group_header_oid = librbd::util::group_header_name(
+ (*m_state_builder)->local_group_id);
+ auto aio_comp = create_rados_callback<
+ BootstrapRequest<I>,
+ &BootstrapRequest<I>::handle_get_local_group_meta>(this);
+
+ int r = m_local_io_ctx.aio_operate(group_header_oid, aio_comp,
+ &op, &m_out_bl);
+ ceph_assert(r == 0);
+ aio_comp->release();
+}
+
+template <typename I>
+void BootstrapRequest<I>::handle_get_local_group_meta(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ std::string data;
+ if (r == 0) {
+ auto it = m_out_bl.cbegin();
+ r = librbd::cls_client::metadata_get_finish(&it, &data);
+ if (r == 0) {
+ *m_resync_requested = true;
+ }
+ }
+ if (r != -ENOENT){
+ // ignore this for now ?
+ dout(10) << "failed to get group meta: " << r << dendl;
+ }
+ if (!*m_resync_requested) {
+ finish(0);
+ return;
+ } else {
+ // proceed to remove local group
+ remove_local_group();
+ return;
+ }
+}
+
} // namespace group_replayer
} // namespace mirror
} // namespace rbd
#include "include/rados/librados.hpp"
#include "cls/rbd/cls_rbd_types.h"
#include "tools/rbd_mirror/CancelableRequest.h"
+#include "tools/rbd_mirror/group_replayer/Types.h"
#include <atomic>
#include <list>
namespace group_replayer {
+template <typename> class GroupStateBuilder;
+
template <typename ImageCtxT = librbd::ImageCtx>
class BootstrapRequest : public CancelableRequest {
public:
journal::CacheManagerHandler *cache_manager_handler,
PoolMetaCache *pool_meta_cache,
bool *resync_requested,
- std::string *local_group_id,
- std::string *remote_group_id,
- std::map<std::string, cls::rbd::GroupSnapshot> *local_group_snaps,
GroupCtx *local_group_ctx,
std::list<std::pair<librados::IoCtx, ImageReplayer<ImageCtxT> *>> *image_replayers,
- std::map<std::pair<int64_t, std::string>, ImageReplayer<ImageCtxT> *> *image_replayer_index,
+ GroupStateBuilder<ImageCtxT> **state_builder,
Context *on_finish) {
return new BootstrapRequest(
threads, local_io_ctx, remote_io_ctx, global_group_id, local_mirror_uuid,
instance_watcher, local_status_updater, remote_status_updater,
- cache_manager_handler, pool_meta_cache, resync_requested, local_group_id,
- remote_group_id, local_group_snaps, local_group_ctx, image_replayers,
- image_replayer_index, on_finish);
+ cache_manager_handler, pool_meta_cache, resync_requested,
+ local_group_ctx, image_replayers, state_builder, on_finish);
}
BootstrapRequest(
journal::CacheManagerHandler *cache_manager_handler,
PoolMetaCache *pool_meta_cache,
bool *resync_requested,
- std::string *local_group_id,
- std::string *remote_group_id,
- std::map<std::string, cls::rbd::GroupSnapshot> *local_group_snaps,
GroupCtx *local_group_ctx,
std::list<std::pair<librados::IoCtx, ImageReplayer<ImageCtxT> *>> *image_replayers,
- std::map<std::pair<int64_t, std::string>, ImageReplayer<ImageCtxT> *> *image_replayer_index,
+ GroupStateBuilder<ImageCtxT> **state_builder,
Context* on_finish);
void send() override;
*
* <start>
* |
- * v
- * GET_REMOTE_GROUP_ID * * * * * * * * * * *
- * | (noent) *
- * v v
- * GET_REMOTE_GROUP_NAME * * * * * * * * * *
- * | (noent) *
- * v v
- * GET_REMOTE_MIRROR_GROUP * * * * * * * * *
- * | (noent or not primary) *
- * v v
- * LIST_REMOTE_GROUP_SNAPSHOTS * * * * * * *
- * | (noent) *
- * v v
- * LIST_REMOTE_GROUP * * * * * * * * * * * *
- * | (repeat if neeeded) (noent) *
- * v v
- * GET_REMOTE_MIRROR_IMAGE * * * * * * * * *
- * | (repeat for every image) (noent) *
- * | v
- * |/< * * * * * * * * * * * * * * * * * *
- * v
- * GET_LOCAL_GROUP_ID * * * * * * * * * * * *
- * | (noent) *
- * v *
- * GET_LOCAL_GROUP_NAME *
+ * v (error)
+ * PREPARE_LOCAL_GROUP * * * * * * * * * * *
+ * | *
+ * v (error) *
+ * PREPARE_REMOTE_GROUP_NAME * * * * * * * *
+ * | *
+ * | (remote dne) *
+ * \------------> REMOVE_LOCAL_GROUP * * *
+ * | (if local non-primary) *
+ * | *
+ * | (local dne) *
+ * \------------> CREATE_LOCAL_GROUP * * *
+ * | (if remote primary) *
+ * v | *
+ * CREATE_IMAGE_REPLAYERS <---/ *
* | *
- * v v
- * GET_LOCAL_MIRROR_GROUP <------------- GET_LOCAL_GROUP_ID_BY_NAME
- * | * (noent)
- * v (noent) *
- * LIST_LOCAL_GROUP_SNAPSHOTS * * * *
- * | * *
- * v (noent) * *
- * LIST_LOCAL_GROUP * * * * * * * * *
- * | (repeat if neeeded) * *
- * v * *
- * GET_LOCAL_MIRROR_IMAGE * *
- * | (repeat for every image) * *
- * v * *
- * REMOVE_LOCAL_IMAGE_FROM_GROUP * *
- * | ^ * *
- * v | * v
- * MOVE_LOCAL_IMAGE_TO_TRASH -/ * CREATE_LOCAL_GROUP_ID
- * | (repeat for every * |
- * | stale image) * v
- * |\----\ * * > CREATE_LOCAL_GROUP
- * | | (if stale |
- * | v or removing) |
- * | DISABLE_LOCAL_MIRROR_GROUP |
- * | | |
- * | v v
- * | REMOVE_LOCAL_MIRROR_GROUP ----> CREATE_LOCAL_MIRROR_GROUP
- * | | (if stale) |
- * | v v
- * | REMOVE_LOCAL_GROUP NOFTIFY_MIRRORING_WATCHER
- * | | (if removing) |
- * | v |
- * | REMOVE_LOCAL_GROUP_ID |
- * | | (if removing) |
- * v v |
- * <finish> <-------------------------------/
+ * v *
+ * <finish> < * * * * * * * * * * * * * * * *
*
* @endverbatim
*/
- typedef std::pair<int64_t /*pool_id*/, std::string /*global_image_id*/> GlobalImageId;
-
Threads<ImageCtxT>* m_threads;
librados::IoCtx &m_local_io_ctx;
librados::IoCtx &m_remote_io_ctx;
journal::CacheManagerHandler *m_cache_manager_handler;
PoolMetaCache *m_pool_meta_cache;
bool *m_resync_requested;
- std::string *m_local_group_id;
- std::string *m_remote_group_id;
- std::map<std::string, cls::rbd::GroupSnapshot> *m_local_group_snaps;
GroupCtx *m_local_group_ctx;
std::list<std::pair<librados::IoCtx, ImageReplayer<ImageCtxT> *>> *m_image_replayers;
- std::map<std::pair<int64_t, std::string>, ImageReplayer<ImageCtxT> *> *m_image_replayer_index;
+ GroupStateBuilder<ImageCtxT> **m_state_builder = nullptr;
Context *m_on_finish;
+ mutable ceph::mutex m_lock;
std::atomic<bool> m_canceled = false;
- std::string m_group_name;
- bool m_local_group_id_by_name = false;
- cls::rbd::MirrorGroup m_remote_mirror_group;
- cls::rbd::MirrorGroup m_local_mirror_group;
- std::vector<cls::rbd::GroupSnapshot> remote_group_snaps;
- std::vector<cls::rbd::GroupSnapshot> local_group_snaps;
- bool m_remote_mirror_group_primary = false;
- bool m_local_mirror_group_primary = false;
- std::list<cls::rbd::GroupImageStatus> m_images;
- librados::IoCtx m_image_io_ctx;
-
- std::map<GlobalImageId, std::string> m_remote_images;
- std::set<GlobalImageId> m_local_images;
- std::map<GlobalImageId, std::string> m_local_trash_images;
+ std::string m_local_group_name;
+ std::string m_prepare_local_group_name;
+ std::string m_prepare_remote_group_name;
+ bool m_local_group_removed = false;
bufferlist m_out_bl;
- bool has_remote_image(int64_t local_pool_id,
- const std::string &global_image_id) const;
-
- void get_remote_group_id();
- void handle_get_remote_group_id(int r);
-
- void get_remote_group_name();
- void handle_get_remote_group_name(int r);
-
- void get_remote_mirror_group();
- void handle_get_remote_mirror_group(int r);
-
- void get_remote_mirror_image();
- void handle_get_remote_mirror_image(int r);
-
- void list_remote_group_snapshots();
- void handle_list_remote_group_snapshots(int r);
-
- void list_remote_group();
- void handle_list_remote_group(int r);
-
- void get_local_group_id();
- void handle_get_local_group_id(int r);
-
- void get_local_group_id_by_name();
- void handle_get_local_group_id_by_name(int r);
-
- void get_local_group_name();
- void handle_get_local_group_name(int r);
+ void prepare_local_group();
+ void handle_prepare_local_group(int r);
- void get_local_mirror_group();
- void handle_get_local_mirror_group(int r);
+ void prepare_remote_group();
+ void handle_prepare_remote_group(int r);
- void list_local_group_snapshots();
- void handle_list_local_group_snapshots(int r);
-
- void list_local_group();
- void handle_list_local_group(int r);
-
- void get_local_mirror_image();
- void handle_get_local_mirror_image(int r);
-
- void remove_local_image_from_group();
- void handle_remove_local_image_from_group(int r);
-
- void move_local_image_to_trash();
- void handle_move_local_image_to_trash(int r);
-
- void remove_local_mirror_image();
- void handle_remove_local_mirror_image(int r);
-
- void disable_local_mirror_group();
- void handle_disable_local_mirror_group(int r);
-
- void remove_local_mirror_group();
- void handle_remove_local_mirror_group(int r);
-
- void remove_local_group();
- void handle_remove_local_group(int r);
-
- void remove_local_group_id();
- void handle_remove_local_group_id(int r);
-
- void create_local_group_id();
- void handle_create_local_group_id(int r);
+ void get_local_group_meta();
+ void handle_get_local_group_meta(int r);
void create_local_group();
void handle_create_local_group(int r);
- void create_local_mirror_group();
- void handle_create_local_mirror_group(int r);
+ void remove_local_group();
+ void handle_remove_local_group(int r);
- void notify_mirroring_watcher();
- void handle_notify_mirroring_watcher(int r);
+ int create_replayers();
void finish(int r);
-
- int create_replayers();
};
} // namespace group_replayer
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "CreateLocalGroupRequest.h"
+#include "GroupStateBuilder.h"
+#include "include/rados/librados.hpp"
+#include "common/debug.h"
+#include "common/dout.h"
+#include "common/errno.h"
+#include "cls/rbd/cls_rbd_client.h"
+#include "cls/rbd/cls_rbd_types.h"
+#include "librbd/ImageCtx.h"
+#include "librbd/MirroringWatcher.h"
+#include "librbd/Utils.h"
+
+#define dout_context g_ceph_context
+#define dout_subsys ceph_subsys_rbd_mirror
+#undef dout_prefix
+#define dout_prefix *_dout << "rbd::mirror::group_replayer::" \
+ << "CreateLocalGroupRequest: " << this << " " \
+ << __func__ << ": "
+
+namespace rbd {
+namespace mirror {
+namespace group_replayer {
+
+using librbd::util::create_context_callback;
+using librbd::util::create_rados_callback;
+
+template <typename I>
+void CreateLocalGroupRequest<I>::send() {
+ dout(10) << "m_global_group_id=" << m_global_group_id << dendl;
+ ceph_assert(m_state_builder->local_group_id.empty());
+ add_mirror_group();
+}
+
+template <typename I>
+void CreateLocalGroupRequest<I>::add_mirror_group() {
+ ceph_assert(m_state_builder->local_group_id.empty());
+ m_state_builder->local_group_id =
+ librbd::util::generate_image_id<I>(m_local_io_ctx);
+
+ dout(10) << "local_group_id=" << m_state_builder->local_group_id << dendl;
+
+ // use 'creating' to track a partially constructed group. it will
+ // be switched to 'enabled' once the group is fully created
+ m_mirror_group.global_group_id = m_global_group_id;
+ m_mirror_group.mirror_image_mode = cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT;
+ m_mirror_group.state= cls::rbd::MIRROR_GROUP_STATE_CREATING;
+
+ librados::ObjectWriteOperation op;
+ librbd::cls_client::mirror_group_set(&op, m_state_builder->local_group_id,
+ m_mirror_group);
+
+ auto aio_comp = create_rados_callback<
+ CreateLocalGroupRequest<I>,
+ &CreateLocalGroupRequest<I>::handle_add_mirror_group>(this);
+ int r = m_local_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op);
+ ceph_assert(r == 0);
+ aio_comp->release();
+}
+
+template <typename I>
+void CreateLocalGroupRequest<I>::handle_add_mirror_group(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ if (r < 0) {
+ derr << "failed to register mirror group " << m_global_group_id << ": "
+ << cpp_strerror(r) << dendl;
+ this->finish(r);
+ return;
+ }
+
+ create_local_group_id();
+}
+
+template <typename I>
+void CreateLocalGroupRequest<I>::create_local_group_id() {
+ dout(10) << "group_id=" << m_state_builder->local_group_id
+ << ", group_name=" << m_state_builder->group_name << dendl;
+
+ std::string group_name = m_state_builder->group_name;
+ std::string group_id = m_state_builder->local_group_id;
+
+ librados::ObjectWriteOperation op;
+ librbd::cls_client::group_dir_add(&op, group_name, group_id);
+
+ auto comp = create_rados_callback<
+ CreateLocalGroupRequest<I>,
+ &CreateLocalGroupRequest<I>::handle_create_local_group_id>(this);
+
+ int r = m_local_io_ctx.aio_operate(RBD_GROUP_DIRECTORY, comp, &op);
+ ceph_assert(r == 0);
+ comp->release();
+}
+
+template <typename I>
+void CreateLocalGroupRequest<I>::handle_create_local_group_id(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ if (r < 0) {
+ derr << "error creating local group: " << cpp_strerror(r) << dendl;
+ finish(r);
+ return;
+ }
+
+ create_local_group();
+}
+
+template <typename I>
+void CreateLocalGroupRequest<I>::create_local_group() {
+ dout(10) << dendl;
+
+ librados::ObjectWriteOperation op;
+ op.create(true);
+ auto comp = create_rados_callback<
+ CreateLocalGroupRequest<I>,
+ &CreateLocalGroupRequest<I>::handle_create_local_group>(this);
+
+ int r = m_local_io_ctx.aio_operate(
+ librbd::util::group_header_name(m_state_builder->local_group_id),
+ comp, &op);
+ ceph_assert(r == 0);
+ comp->release();
+}
+
+template <typename I>
+void CreateLocalGroupRequest<I>::handle_create_local_group(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ if (r < 0) {
+ derr << "failed to create local group: " << cpp_strerror(r) << dendl;
+ finish(r);
+ return;
+ }
+
+ // The group mirror state will be set to enabled once the first
+ // non-primary mirror group snap is created.
+ finish(0);
+}
+
+template <typename I>
+void CreateLocalGroupRequest<I>::finish(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ m_on_finish->complete(r);
+ delete this;
+}
+
+} // namespace group_replayer
+} // namespace mirror
+} // namespace rbd
+
+template class rbd::mirror::group_replayer::CreateLocalGroupRequest<librbd::ImageCtx>;
+
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef RBD_MIRROR_GROUP_REPLAYER_CREATE_LOCAL_GROUP_REQUEST_H
+#define RBD_MIRROR_GROUP_REPLAYER_CREATE_LOCAL_GROUP_REQUEST_H
+
+#include "include/buffer.h"
+#include "include/rados/librados_fwd.hpp"
+#include "cls/rbd/cls_rbd_types.h"
+#include <string>
+
+struct Context;
+
+namespace librbd {
+struct ImageCtx;
+namespace asio { struct ContextWQ; }
+} // namespace librbd
+
+namespace rbd {
+namespace mirror {
+namespace group_replayer {
+
+template <typename> class GroupStateBuilder;
+
+template <typename ImageCtxT = librbd::ImageCtx>
+class CreateLocalGroupRequest {
+public:
+ static CreateLocalGroupRequest *create(
+ librados::IoCtx &io_ctx,
+ const std::string &global_group_id,
+ GroupStateBuilder<ImageCtxT>* state_builder,
+ Context *on_finish) {
+ return new CreateLocalGroupRequest(io_ctx, global_group_id,
+ state_builder, on_finish);
+ }
+
+ CreateLocalGroupRequest(
+ librados::IoCtx &io_ctx,
+ const std::string &global_group_id,
+ GroupStateBuilder<ImageCtxT>* state_builder,
+ Context *on_finish)
+ : m_local_io_ctx(io_ctx), m_global_group_id(global_group_id),
+ m_state_builder(state_builder),
+ m_on_finish(on_finish) {
+ }
+
+ void send();
+
+private:
+ /**
+ * @verbatim
+ *
+ * <start>
+ * |
+ * v
+ * ADD_MIRROR_GROUP (state = CREATING)
+ * |
+ * v
+ * CREATE_LOCAL_GROUP_ID
+ * |
+ * v
+ * CREATE_LOCAL_GROUP
+ * |
+ * v
+ * <finish>
+ *
+ * @endverbatim
+ */
+
+ librados::IoCtx &m_local_io_ctx;
+ std::string m_global_group_id;
+ GroupStateBuilder<ImageCtxT>* m_state_builder;
+ Context *m_on_finish;
+
+ cls::rbd::MirrorGroup m_mirror_group;
+
+ void add_mirror_group();
+ void handle_add_mirror_group(int r);
+
+ void create_local_group_id();
+ void handle_create_local_group_id(int r);
+
+ void create_local_group();
+ void handle_create_local_group(int r);
+
+ void finish(int r);
+
+};
+
+} // namespace group_replayer
+} // namespace mirror
+} // namespace rbd
+
+extern template class rbd::mirror::group_replayer::CreateLocalGroupRequest<librbd::ImageCtx>;
+
+#endif // RBD_MIRROR_GROUP_REPLAYER_CREATE_LOCAL_GROUP_REQUEST_H
+
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "GroupMirrorStateUpdateRequest.h"
+#include "GroupStateBuilder.h"
+#include "include/rados/librados.hpp"
+#include "common/debug.h"
+#include "common/dout.h"
+#include "common/errno.h"
+#include "cls/rbd/cls_rbd_client.h"
+#include "cls/rbd/cls_rbd_types.h"
+#include "librbd/ImageCtx.h"
+#include "librbd/MirroringWatcher.h"
+#include "librbd/Utils.h"
+
+#define dout_context g_ceph_context
+#define dout_subsys ceph_subsys_rbd_mirror
+#undef dout_prefix
+#define dout_prefix *_dout << "rbd::mirror::group_replayer::" \
+ << "GroupMirrorStateUpdateRequest: " << this << " " \
+ << __func__ << ": "
+
+namespace rbd {
+namespace mirror {
+namespace group_replayer {
+
+using librbd::util::create_context_callback;
+using librbd::util::create_rados_callback;
+
+template <typename I>
+void GroupMirrorStateUpdateRequest<I>::send() {
+ dout(10) << "m_local_group_id=" << m_local_group_id << dendl;
+ get_mirror_group();
+}
+
+template <typename I>
+void GroupMirrorStateUpdateRequest<I>::get_mirror_group() {
+ dout(10) << dendl;
+
+ librados::ObjectReadOperation op;
+ librbd::cls_client::mirror_group_get_start(&op, m_local_group_id);
+
+ auto aio_comp = create_rados_callback<
+ GroupMirrorStateUpdateRequest<I>,
+ &GroupMirrorStateUpdateRequest<I>::handle_get_mirror_group>(this);
+
+ int r = m_local_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op, &m_out_bl);
+ ceph_assert(r == 0);
+ aio_comp->release();
+}
+
+template <typename I>
+void GroupMirrorStateUpdateRequest<I>::handle_get_mirror_group(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ if (r == 0) {
+ auto it = m_out_bl.cbegin();
+ r = librbd::cls_client::mirror_group_get_finish(&it, &m_mirror_group);
+ }
+
+ if (r == -ENOENT) {
+ dout(20) << "mirroring is disabled" << dendl;
+ finish(r);
+ return;
+ }
+
+ if (r < 0) {
+ derr << "failed to get mirror info of group '" << m_local_group_id << dendl;
+
+ finish(r);
+ return;
+ }
+
+ if (m_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED) {
+ finish(0);
+ return;
+ }
+ enable_mirror_group();
+}
+
+template <typename I>
+void GroupMirrorStateUpdateRequest<I>::enable_mirror_group() {
+ dout(10) << dendl;
+
+ m_mirror_group.state = cls::rbd::MIRROR_GROUP_STATE_ENABLED;
+
+ librados::ObjectWriteOperation op;
+ librbd::cls_client::mirror_group_set(&op, m_local_group_id,
+ m_mirror_group);
+
+ auto aio_comp = create_rados_callback<
+ GroupMirrorStateUpdateRequest<I>,
+ &GroupMirrorStateUpdateRequest<I>::handle_enable_mirror_group>(this);
+ int r = m_local_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op);
+ ceph_assert(r == 0);
+ aio_comp->release();
+}
+
+template <typename I>
+void GroupMirrorStateUpdateRequest<I>::handle_enable_mirror_group(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ if (r < 0) {
+ derr << "failed to mirror enable group " << m_local_group_id << ": "
+ << cpp_strerror(r) << dendl;
+ finish(r);
+ return;
+ }
+
+ notify_mirroring_watcher();
+}
+
+template <typename I>
+void GroupMirrorStateUpdateRequest<I>::notify_mirroring_watcher() {
+ dout(10) << dendl;
+
+ auto ctx = create_context_callback<
+ GroupMirrorStateUpdateRequest<I>,
+ &GroupMirrorStateUpdateRequest<I>::handle_notify_mirroring_watcher>(this);
+
+ librbd::MirroringWatcher<I>::notify_group_updated(
+ m_local_io_ctx, m_mirror_group.state, m_local_group_id,
+ m_mirror_group.global_group_id, m_num_images, ctx);
+}
+
+template <typename I>
+void GroupMirrorStateUpdateRequest<I>::handle_notify_mirroring_watcher(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ if (r < 0) {
+ derr << "failed to notify mirror group update: " << cpp_strerror(r)
+ << dendl;
+ }
+
+ finish(0);
+}
+
+template <typename I>
+void GroupMirrorStateUpdateRequest<I>::finish(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ m_on_finish->complete(r);
+ delete this;
+}
+
+} // namespace group_replayer
+} // namespace mirror
+} // namespace rbd
+
+template class rbd::mirror::group_replayer::GroupMirrorStateUpdateRequest<librbd::ImageCtx>;
+
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef RBD_MIRROR_GROUP_REPLAYER_GROUP_MIRROR_STATE_UPDATE_REQUEST_H
+#define RBD_MIRROR_GROUP_REPLAYER_GROUP_MIRROR_STATE_UPDATE_REQUEST_H
+
+#include "include/buffer.h"
+#include "include/rados/librados_fwd.hpp"
+#include "cls/rbd/cls_rbd_types.h"
+#include <string>
+
+struct Context;
+
+namespace librbd {
+struct ImageCtx;
+namespace asio { struct ContextWQ; }
+} // namespace librbd
+
+namespace rbd {
+namespace mirror {
+namespace group_replayer {
+
+template <typename> class GroupStateBuilder;
+
+template <typename ImageCtxT = librbd::ImageCtx>
+class GroupMirrorStateUpdateRequest {
+public:
+ static GroupMirrorStateUpdateRequest *create(
+ librados::IoCtx &io_ctx,
+ const std::string &local_group_id,
+ uint64_t num_images,
+ Context *on_finish) {
+ return new GroupMirrorStateUpdateRequest(io_ctx, local_group_id,
+ num_images, on_finish);
+ }
+
+ GroupMirrorStateUpdateRequest(
+ librados::IoCtx &io_ctx,
+ const std::string &local_group_id,
+ uint64_t num_images,
+ Context *on_finish)
+ : m_local_io_ctx(io_ctx), m_local_group_id(local_group_id),
+ m_num_images(num_images), m_on_finish(on_finish) {
+ }
+
+ void send();
+
+private:
+ /**
+ * @verbatim
+ *
+ * <start>
+ * |
+ * v
+ * GET_MIRROR_GROUP (state = CREATING)
+ * |
+ * v
+ * ENABLE_MIRROR_GROUP
+ * |
+ * v
+ * NOTIFY_MIRRORING_WATCHER
+ * |
+ * v
+ * <finish>
+ *
+ * @endverbatim
+ */
+
+ librados::IoCtx &m_local_io_ctx;
+ std::string m_local_group_id;
+ uint64_t m_num_images;
+ Context *m_on_finish;
+
+ cls::rbd::MirrorGroup m_mirror_group;
+ bufferlist m_out_bl;
+
+ void get_mirror_group();
+ void handle_get_mirror_group(int r);
+
+ void enable_mirror_group();
+ void handle_enable_mirror_group(int r);
+
+ void notify_mirroring_watcher();
+ void handle_notify_mirroring_watcher(int r);
+
+ void finish(int r);
+
+};
+
+} // namespace group_replayer
+} // namespace mirror
+} // namespace rbd
+
+extern template class rbd::mirror::group_replayer::GroupMirrorStateUpdateRequest<librbd::ImageCtx>;
+
+#endif // RBD_MIRROR_GROUP_REPLAYER_GROUP_MIRROR_STATE_UPDATE_REQUEST_H
+
--- /dev/null
+#include "include/ceph_assert.h"
+#include "include/Context.h"
+#include "common/debug.h"
+#include "common/errno.h"
+#include "librbd/ImageCtx.h"
+#include "GroupStateBuilder.h"
+
+#define dout_context g_ceph_context
+#define dout_subsys ceph_subsys_rbd_mirror
+#undef dout_prefix
+#define dout_prefix *_dout << "rbd::mirror::group_replayer::" \
+ << "GroupStateBuilder: " << this << " " \
+ << __func__ << ": "
+
+namespace rbd {
+namespace mirror {
+namespace group_replayer {
+
+template <typename I>
+GroupStateBuilder<I>::GroupStateBuilder(const std::string& global_group_id)
+ : global_group_id(global_group_id) {
+ dout(10) << "global_group_id=" << global_group_id << dendl;
+}
+
+template <typename I>
+GroupStateBuilder<I>::~GroupStateBuilder() {
+ local_images.clear();
+ remote_images.clear();
+}
+
+template <typename I>
+bool GroupStateBuilder<I>::is_local_primary() const {
+ if (local_promotion_state == librbd::mirror::PROMOTION_STATE_PRIMARY) {
+ ceph_assert(!local_group_id.empty());
+ return true;
+ }
+ return false;
+}
+
+template <typename I>
+bool GroupStateBuilder<I>::is_remote_primary() const {
+ if (remote_promotion_state == librbd::mirror::PROMOTION_STATE_PRIMARY) {
+ ceph_assert(!remote_group_id.empty());
+ return true;
+ }
+ return false;
+}
+
+template <typename I>
+bool GroupStateBuilder<I>::is_linked() const {
+ if (local_promotion_state == librbd::mirror::PROMOTION_STATE_NON_PRIMARY) {
+ ceph_assert(!local_group_id.empty());
+ return true;
+ }
+ return false;
+}
+
+} // namespace group_replayer
+} // namespace mirror
+} // namespace rbd
+
+template class rbd::mirror::group_replayer::GroupStateBuilder<librbd::ImageCtx>;
--- /dev/null
+#ifndef CEPH_RBD_MIRROR_GROUP_REPLAYER_STATE_BUILDER_H
+#define CEPH_RBD_MIRROR_GROUP_REPLAYER_STATE_BUILDER_H
+
+#include "tools/rbd_mirror/group_replayer/Types.h"
+#include "include/rados/librados_fwd.hpp"
+#include "cls/rbd/cls_rbd_types.h"
+#include "librbd/mirror/Types.h"
+
+namespace librbd { struct ImageCtx; }
+
+namespace rbd {
+namespace mirror {
+
+template <typename> class Threads;
+
+namespace group_replayer {
+
+template <typename ImageCtxT = librbd::ImageCtx>
+class GroupStateBuilder {
+
+public:
+ static GroupStateBuilder* create(const std::string& global_image_id) {
+ return new GroupStateBuilder(global_image_id);
+ }
+
+ GroupStateBuilder(const std::string& global_image_id);
+
+ ~GroupStateBuilder();
+
+ void destroy() {
+ delete this;
+ }
+
+ bool is_local_primary() const;
+ bool is_remote_primary() const;
+ bool is_linked() const; // FIXME: Required?
+
+ std::string global_group_id;
+ std::string group_name;
+
+ std::string local_group_id;
+ librbd::mirror::PromotionState local_promotion_state =
+ librbd::mirror::PROMOTION_STATE_UNKNOWN;
+ std::map<std::string /*global-id*/, std::pair<int64_t /*pool_id*/, std::string /*image_id*/>> local_images;
+
+ std::string remote_group_id;
+ librbd::mirror::PromotionState remote_promotion_state =
+ librbd::mirror::PROMOTION_STATE_UNKNOWN;
+ std::string remote_mirror_peer_uuid;
+ std::set<GlobalImageId> remote_images;
+
+};
+
+} // namespace group_replayer
+} // namespace mirror
+} // namespace rbd
+
+extern template class rbd::mirror::group_replayer::GroupStateBuilder<librbd::ImageCtx>;
+
+#endif // CEPH_RBD_MIRROR_GROUP_REPLAYER_STATE_BUILDER_H
+
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "tools/rbd_mirror/group_replayer/PrepareLocalGroupRequest.h"
+#include "tools/rbd_mirror/group_replayer/GroupStateBuilder.h"
+#include "tools/rbd_mirror/group_replayer/RemoveLocalGroupRequest.h"
+#include "include/rados/librados.hpp"
+#include "cls/rbd/cls_rbd_client.h"
+#include "common/debug.h"
+#include "common/errno.h"
+#include "librbd/ImageCtx.h"
+#include "librbd/Utils.h"
+#include "librbd/mirror/GroupGetInfoRequest.h"
+#include "tools/rbd_mirror/Threads.h"
+#include <type_traits>
+
+#define dout_context g_ceph_context
+#define dout_subsys ceph_subsys_rbd_mirror
+#undef dout_prefix
+#define dout_prefix *_dout << "rbd::mirror::group_replayer::" \
+ << "PrepareLocalGroupRequest: " << this << " " \
+ << __func__ << ": "
+namespace {
+ static const uint32_t MAX_RETURN = 1024;
+} // anonymous namespace
+
+namespace rbd {
+namespace mirror {
+namespace group_replayer {
+
+using librbd::util::create_context_callback;
+using librbd::util::create_rados_callback;
+
+template <typename I>
+void PrepareLocalGroupRequest<I>::send() {
+ dout(10) << "global_group_id: " << m_global_group_id << dendl;
+
+ get_local_group_id();
+}
+
+template <typename I>
+void PrepareLocalGroupRequest<I>::get_local_group_id() {
+ dout(10) << dendl;
+
+ librados::ObjectReadOperation op;
+ librbd::cls_client::mirror_group_get_group_id_start(&op, m_global_group_id);
+ m_out_bl.clear();
+ auto comp = create_rados_callback<
+ PrepareLocalGroupRequest<I>,
+ &PrepareLocalGroupRequest<I>::handle_get_local_group_id>(this);
+
+ int r = m_io_ctx.aio_operate(RBD_MIRRORING, comp, &op, &m_out_bl);
+ ceph_assert(r == 0);
+ comp->release();
+}
+
+template <typename I>
+void PrepareLocalGroupRequest<I>::handle_get_local_group_id(int r) {
+ dout(10) << "r=" << r << ", global_group_id: " << m_global_group_id << dendl;
+
+ if (r == 0) {
+ auto iter = m_out_bl.cbegin();
+ r = librbd::cls_client::mirror_group_get_group_id_finish(
+ &iter, &m_local_group_id);
+ }
+
+ if (r < 0) {
+ if (r != -ENOENT) {
+ derr << "error getting local group id: " << cpp_strerror(r) << dendl;
+ }
+ finish(r);
+ return;
+ }
+
+ get_local_group_name();
+}
+
+template <typename I>
+void PrepareLocalGroupRequest<I>::get_local_group_name() {
+ dout(10) << dendl;
+
+ librados::ObjectReadOperation op;
+ librbd::cls_client::dir_get_name_start(&op, m_local_group_id);
+
+ m_out_bl.clear();
+ auto comp = create_rados_callback<
+ PrepareLocalGroupRequest<I>,
+ &PrepareLocalGroupRequest<I>::handle_get_local_group_name>(this);
+
+ int r = m_io_ctx.aio_operate(RBD_GROUP_DIRECTORY, comp, &op,
+ &m_out_bl);
+ ceph_assert(r == 0);
+ comp->release();
+}
+
+template <typename I>
+void PrepareLocalGroupRequest<I>::handle_get_local_group_name(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ *m_local_group_name = "";
+ if (r == 0) {
+ auto iter = m_out_bl.cbegin();
+ r = librbd::cls_client::dir_get_name_finish(&iter, m_local_group_name);
+ }
+
+ if (r == -ENOENT) {
+ // proceed - we should have a mirror group record if we got this far
+ dout(10) << "group does not exist for local group id " << m_local_group_id
+ << dendl;
+ *m_local_group_name = "";
+ } else if (r < 0) {
+ derr << "error getting local group name: " << cpp_strerror(r) << dendl;
+ finish(r);
+ return;
+ }
+
+ get_mirror_info();
+}
+
+template <typename I>
+void PrepareLocalGroupRequest<I>::get_mirror_info() {
+ dout(10) << dendl;
+
+ auto ctx = create_context_callback<
+ PrepareLocalGroupRequest<I>,
+ &PrepareLocalGroupRequest<I>::handle_get_mirror_info>(this);
+
+ auto req = librbd::mirror::GroupGetInfoRequest<I>::create(
+ m_io_ctx, "", m_local_group_id, &m_mirror_group,
+ &m_promotion_state, ctx);
+ req->send();
+}
+
+template <typename I>
+void PrepareLocalGroupRequest<I>::handle_get_mirror_info(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ if (r < 0) {
+ derr << "failed to retrieve local mirror group info: " << cpp_strerror(r)
+ << dendl;
+ finish(r);
+ return;
+ }
+
+ // If the mirror group state is set to CREATING, it means that the group
+ // creation was interrupted.
+ if (m_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_CREATING) {
+ dout(10) << "local group is still in creating state, issuing a removal"
+ << dendl;
+ remove_local_group();
+ return;
+ } else if (m_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_DISABLING) {
+ dout(10) << "local group mirroring is in disabling state" << dendl;
+
+ finish(-ERESTART);
+ return;
+ }
+
+ if (m_mirror_group.mirror_image_mode != cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT) {
+ derr << "unsupported mirror mode "
+ << m_mirror_group.mirror_image_mode << " "
+ << "for group " << m_global_group_id << dendl;
+ finish(-EOPNOTSUPP);
+ return;
+ }
+
+ list_group_images();
+}
+
+template <typename I>
+void PrepareLocalGroupRequest<I>::list_group_images() {
+ dout(10) << dendl;
+
+ librados::ObjectReadOperation op;
+ cls::rbd::GroupImageSpec start_after;
+
+ m_out_bl.clear();
+
+ if (!m_images.empty()) {
+ start_after = m_images.rbegin()->spec;
+ }
+
+ librbd::cls_client::group_image_list_start(&op, start_after, MAX_RETURN);
+ auto comp = create_rados_callback<
+ PrepareLocalGroupRequest<I>,
+ &PrepareLocalGroupRequest<I>::handle_list_group_images>(this);
+ int r = m_io_ctx.aio_operate(
+ librbd::util::group_header_name(m_local_group_id), comp, &op, &m_out_bl);
+ ceph_assert(r == 0);
+ comp->release();
+}
+
+template <typename I>
+void PrepareLocalGroupRequest<I>::handle_list_group_images(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ std::vector<cls::rbd::GroupImageStatus> images;
+ if (r == 0) {
+ auto iter = m_out_bl.cbegin();
+ r = librbd::cls_client::group_image_list_finish(&iter, &images);
+ }
+
+ if (r < 0) {
+ dout(10) << "error listing local group: " << cpp_strerror(r) << dendl;
+ finish(r);
+ return;
+ }
+
+ m_images.insert(m_images.end(), images.begin(), images.end());
+
+ if (images.size() == MAX_RETURN) {
+ list_group_images();
+ return;
+ }
+
+ dout(10) << "number of images: " << m_images.size() << dendl;
+ get_mirror_images();
+}
+
+template <typename I>
+void PrepareLocalGroupRequest<I>::get_mirror_images() {
+ dout(10) << dendl;
+
+ if (m_images.empty()) {
+ dout(10) << "local_group_id=" << m_local_group_id << ", "
+ << "local_promotion_state=" << m_promotion_state
+ << dendl;
+
+ (*m_state_builder)->local_group_id = m_local_group_id;
+ (*m_state_builder)->local_promotion_state = m_promotion_state;
+ (*m_state_builder)->group_name = *m_local_group_name; //FIXME
+ (*m_state_builder)->local_images.insert(m_local_images.begin(),
+ m_local_images.end());
+ finish(0);
+ return;
+ }
+
+ auto &spec = m_images.front().spec;
+
+ dout(10) << "pool_id: " << spec.pool_id
+ << ", image_id " << spec.image_id << dendl;
+
+ // FIXME: Currently images must be in the same pool as the group
+ librados::ObjectReadOperation op;
+ librbd::cls_client::mirror_image_get_start(&op, spec.image_id);
+
+ m_out_bl.clear();
+ auto comp = create_rados_callback<
+ PrepareLocalGroupRequest<I>,
+ &PrepareLocalGroupRequest<I>::handle_get_mirror_images>(this);
+
+ int r = m_io_ctx.aio_operate(RBD_MIRRORING, comp, &op, &m_out_bl);
+ ceph_assert(r == 0);
+ comp->release();
+}
+
+template <typename I>
+void PrepareLocalGroupRequest<I>::handle_get_mirror_images(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ auto &spec = m_images.front().spec;
+ cls::rbd::MirrorImage mirror_image;
+
+ if (r == 0) {
+ auto iter = m_out_bl.cbegin();
+ r = librbd::cls_client::mirror_image_get_finish(&iter, &mirror_image);
+ }
+
+ if (r < 0) {
+ derr << "error getting local mirror image: " << cpp_strerror(r) << dendl;
+ finish(r);
+ return;
+ }
+ m_local_images.insert({mirror_image.global_image_id,
+ {spec.pool_id, spec.image_id}});
+
+ m_images.pop_front();
+ get_mirror_images();
+}
+
+template <typename I>
+void PrepareLocalGroupRequest<I>::remove_local_group() {
+ dout(10) << "group_name: " << *m_local_group_name
+ << " , group_id: " << m_local_group_id << dendl;
+
+ auto ctx = create_context_callback<
+ PrepareLocalGroupRequest,
+ &PrepareLocalGroupRequest<I>::handle_remove_local_group>(this);
+
+ auto req = RemoveLocalGroupRequest<I>::create(m_io_ctx, m_global_group_id,
+ false, m_work_queue, ctx);
+ req->send();
+}
+
+template <typename I>
+void PrepareLocalGroupRequest<I>::handle_remove_local_group(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ if (r < 0 && r != -ENOENT) {
+ derr << "error removing local group: " << cpp_strerror(r) << dendl;
+ finish(r);
+ return;
+ }
+
+ finish(-ENOENT);
+}
+
+template <typename I>
+void PrepareLocalGroupRequest<I>::finish(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ m_on_finish->complete(r);
+ delete this;
+}
+
+} // namespace group_replayer
+} // namespace mirror
+} // namespace rbd
+
+template class rbd::mirror::group_replayer::PrepareLocalGroupRequest<librbd::ImageCtx>;
+
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef RBD_MIRROR_GROUP_REPLAYER_PREPARE_LOCAL_GROUP_REQUEST_H
+#define RBD_MIRROR_GROUP_REPLAYER_PREPARE_LOCAL_GROUP_REQUEST_H
+
+#include "include/buffer.h"
+#include "include/rados/librados_fwd.hpp"
+#include "cls/rbd/cls_rbd_types.h"
+#include "librbd/mirror/Types.h"
+#include <string>
+
+struct Context;
+
+namespace librbd {
+struct ImageCtx;
+namespace asio { struct ContextWQ; }
+} // namespace librbd
+
+namespace rbd {
+namespace mirror {
+namespace group_replayer {
+
+template <typename> class GroupStateBuilder;
+
+template <typename ImageCtxT = librbd::ImageCtx>
+class PrepareLocalGroupRequest {
+public:
+ static PrepareLocalGroupRequest *create(
+ librados::IoCtx &io_ctx,
+ const std::string &global_group_id,
+ std::string *local_group_name,
+ GroupStateBuilder<ImageCtxT>** state_builder,
+ librbd::asio::ContextWQ *work_queue,
+ Context *on_finish) {
+ return new PrepareLocalGroupRequest(io_ctx, global_group_id,
+ local_group_name, state_builder,
+ work_queue, on_finish);
+ }
+
+ PrepareLocalGroupRequest(
+ librados::IoCtx &io_ctx,
+ const std::string &global_group_id,
+ std::string *local_group_name,
+ GroupStateBuilder<ImageCtxT>** state_builder,
+ librbd::asio::ContextWQ *work_queue,
+ Context *on_finish)
+ : m_io_ctx(io_ctx), m_global_group_id(global_group_id),
+ m_local_group_name(local_group_name), m_state_builder(state_builder),
+ m_work_queue(work_queue), m_on_finish(on_finish) {
+ }
+
+ void send();
+
+private:
+ /**
+ * @verbatim
+ *
+ * <start>
+ * |
+ * v
+ * GET_LOCAL_GROUP_ID
+ * |
+ * v
+ * GET_LOCAL_GROUP_NAME
+ * |
+ * v
+ * GET_MIRROR_INFO ---------------
+ * | |
+ * v (if the group mirror state is CREATING)
+ * LIST_GROUP_IMAGES |
+ * | |
+ * v v
+ * GET_MIRROR_IMAGES REMOVE_LOCAL_GROUP
+ * | |
+ * v |
+ * <finish> <--------------------
+ *
+ * @endverbatim
+ */
+
+ librados::IoCtx &m_io_ctx;
+ std::string m_global_group_id;
+ std::string *m_local_group_name;
+ GroupStateBuilder<ImageCtxT>** m_state_builder;
+ librbd::asio::ContextWQ *m_work_queue;
+ Context *m_on_finish;
+
+ bufferlist m_out_bl;
+ std::string m_local_group_id;
+ cls::rbd::MirrorGroup m_mirror_group;
+ librbd::mirror::PromotionState m_promotion_state;
+ std::list<cls::rbd::GroupImageStatus> m_images;
+ std::map<std::string /*global-id*/, std::pair<int64_t /*pool_id*/, std::string /*image_id*/>> m_local_images;
+
+ void get_local_group_id();
+ void handle_get_local_group_id(int r);
+
+ void get_local_group_name();
+ void handle_get_local_group_name(int r);
+
+ void get_mirror_info();
+ void handle_get_mirror_info(int r);
+
+ void list_group_images();
+ void handle_list_group_images(int r);
+
+ void get_mirror_images();
+ void handle_get_mirror_images(int r);
+
+ void remove_local_group();
+ void handle_remove_local_group(int r);
+
+ void finish(int r);
+
+};
+
+} // namespace group_replayer
+} // namespace mirror
+} // namespace rbd
+
+extern template class rbd::mirror::group_replayer::PrepareLocalGroupRequest<librbd::ImageCtx>;
+
+#endif // RBD_MIRROR_GROUP_REPLAYER_PREPARE_LOCAL_GROUP_REQUEST_H
+
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "tools/rbd_mirror/group_replayer/PrepareRemoteGroupRequest.h"
+#include "tools/rbd_mirror/group_replayer/GroupStateBuilder.h"
+#include "include/rados/librados.hpp"
+#include "cls/rbd/cls_rbd_client.h"
+#include "common/debug.h"
+#include "common/errno.h"
+#include "librbd/ImageCtx.h"
+#include "librbd/Utils.h"
+#include "librbd/mirror/GroupGetInfoRequest.h"
+#include "tools/rbd_mirror/Threads.h"
+
+#define dout_context g_ceph_context
+#define dout_subsys ceph_subsys_rbd_mirror
+#undef dout_prefix
+#define dout_prefix *_dout << "rbd::mirror::group_replayer::" \
+ << "PrepareRemoteGroupRequest: " << this << " " \
+ << __func__ << ": "
+
+namespace {
+ static const uint32_t MAX_RETURN = 1024;
+} // anonymous namespace
+
+namespace rbd {
+namespace mirror {
+namespace group_replayer {
+
+using librbd::util::create_context_callback;
+using librbd::util::create_rados_callback;
+
+template <typename I>
+void PrepareRemoteGroupRequest<I>::send() {
+ dout(10) << "global_group_id: " << m_global_group_id << dendl;
+ get_remote_group_id();
+}
+
+template <typename I>
+void PrepareRemoteGroupRequest<I>::get_remote_group_id() {
+ dout(10) << dendl;
+
+ librados::ObjectReadOperation op;
+ librbd::cls_client::mirror_group_get_group_id_start(&op, m_global_group_id);
+ m_out_bl.clear();
+ auto comp = create_rados_callback<
+ PrepareRemoteGroupRequest<I>,
+ &PrepareRemoteGroupRequest<I>::handle_get_remote_group_id>(this);
+
+ int r = m_io_ctx.aio_operate(RBD_MIRRORING, comp, &op, &m_out_bl);
+ ceph_assert(r == 0);
+ comp->release();
+}
+
+template <typename I>
+void PrepareRemoteGroupRequest<I>::handle_get_remote_group_id(int r) {
+ dout(10) << "r=" << r << ", global_image_id: " << m_global_group_id << dendl;
+
+ if (r == 0) {
+ auto iter = m_out_bl.cbegin();
+ r = librbd::cls_client::mirror_group_get_group_id_finish(
+ &iter, &m_remote_group_id);
+ }
+
+ if (r < 0) {
+ derr << "error getting remote group id: " << cpp_strerror(r) << dendl;
+ finish(r);
+ return;
+ }
+
+ get_remote_group_name();
+}
+
+template <typename I>
+void PrepareRemoteGroupRequest<I>::get_remote_group_name() {
+ dout(10) << dendl;
+
+ librados::ObjectReadOperation op;
+ librbd::cls_client::dir_get_name_start(&op, m_remote_group_id);
+ m_out_bl.clear();
+ auto comp = create_rados_callback<
+ PrepareRemoteGroupRequest<I>,
+ &PrepareRemoteGroupRequest<I>::handle_get_remote_group_name>(this);
+
+ int r = m_io_ctx.aio_operate(RBD_GROUP_DIRECTORY, comp, &op,
+ &m_out_bl);
+ ceph_assert(r == 0);
+ comp->release();
+}
+
+template <typename I>
+void PrepareRemoteGroupRequest<I>::handle_get_remote_group_name(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ *m_remote_group_name = "";
+ if (r == 0) {
+ auto iter = m_out_bl.cbegin();
+ r = librbd::cls_client::dir_get_name_finish(&iter, m_remote_group_name);
+ }
+
+ if (r < 0) {
+ derr << "error getting remote group name: " << cpp_strerror(r) << dendl;
+ finish(r);
+ return;
+ }
+
+ get_mirror_info();
+}
+
+template <typename I>
+void PrepareRemoteGroupRequest<I>::get_mirror_info() {
+ dout(10) << dendl;
+
+ auto ctx = create_context_callback<
+ PrepareRemoteGroupRequest<I>,
+ &PrepareRemoteGroupRequest<I>::handle_get_mirror_info>(this);
+ auto req = librbd::mirror::GroupGetInfoRequest<I>::create(
+ m_io_ctx, "", m_remote_group_id, &m_mirror_group,
+ &m_promotion_state, ctx);
+ req->send();
+}
+
+template <typename I>
+void PrepareRemoteGroupRequest<I>::handle_get_mirror_info(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ if (r < 0) {
+ if (r == -ENOENT) {
+ dout(10) << "group " << m_global_group_id << " is not mirrored" << dendl;
+ } else {
+ derr << "failed to retrieve remote mirror group info: " << cpp_strerror(r)
+ << dendl;
+ }
+ finish(r);
+ return;
+ }
+
+ if (m_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_DISABLING) {
+ dout(10) << "remote group mirroring is being disabled" << dendl;
+ finish(-ERESTART);
+ return;
+ }
+
+ if (m_mirror_group.mirror_image_mode !=
+ cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT) {
+ derr << "unsupported mirror group mode "
+ << m_mirror_group.mirror_image_mode << " "
+ << "for group " << m_global_group_id << dendl;
+ finish(-EOPNOTSUPP);
+ return;
+ }
+
+ dout(10) << "remote_group_id=" << m_remote_group_id << ", "
+ << "remote_promotion_state=" << m_promotion_state
+ << dendl;
+
+ list_group_images();
+}
+
+template <typename I>
+void PrepareRemoteGroupRequest<I>::list_group_images() {
+ dout(10) << dendl;
+
+ librados::ObjectReadOperation op;
+ cls::rbd::GroupImageSpec start_after;
+
+ m_out_bl.clear();
+
+ if (!m_images.empty()) {
+ start_after = m_images.rbegin()->spec;
+ }
+
+ librbd::cls_client::group_image_list_start(&op, start_after, MAX_RETURN);
+ auto comp = create_rados_callback<
+ PrepareRemoteGroupRequest<I>,
+ &PrepareRemoteGroupRequest<I>::handle_list_group_images>(this);
+ int r = m_io_ctx.aio_operate(
+ librbd::util::group_header_name(m_remote_group_id), comp, &op, &m_out_bl);
+ ceph_assert(r == 0);
+ comp->release();
+}
+
+template <typename I>
+void PrepareRemoteGroupRequest<I>::handle_list_group_images(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ std::vector<cls::rbd::GroupImageStatus> images;
+ if (r == 0) {
+ auto iter = m_out_bl.cbegin();
+ r = librbd::cls_client::group_image_list_finish(&iter, &images);
+ }
+
+ if (r < 0) {
+ derr << "error listing remote group: " << cpp_strerror(r) << dendl;
+ finish(r);
+ return;
+ }
+
+ m_images.insert(m_images.end(), images.begin(), images.end());
+
+ if (images.size() == MAX_RETURN) {
+ list_group_images();
+ return;
+ }
+ dout(10) << "number of images: " << m_images.size() << dendl;
+ get_mirror_images();
+}
+
+template <typename I>
+void PrepareRemoteGroupRequest<I>::get_mirror_images() {
+ dout(10) << dendl;
+
+ if (m_images.empty()) {
+ finish(0);
+ return;
+ }
+
+ auto &spec = m_images.front().spec;
+
+ dout(10) << "pool_id: " << spec.pool_id
+ << ", image_id " << spec.image_id << dendl;
+
+ // Currently the group and its images must belong to the same pool.
+
+ librados::ObjectReadOperation op;
+ librbd::cls_client::mirror_image_get_start(&op, spec.image_id);
+ m_out_bl.clear();
+ auto comp = create_rados_callback<
+ PrepareRemoteGroupRequest<I>,
+ &PrepareRemoteGroupRequest<I>::handle_get_mirror_images>(this);
+
+ int r = m_io_ctx.aio_operate(RBD_MIRRORING, comp, &op, &m_out_bl);
+ ceph_assert(r == 0);
+ comp->release();
+}
+
+template <typename I>
+void PrepareRemoteGroupRequest<I>::handle_get_mirror_images(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ auto &spec = m_images.front().spec;
+ cls::rbd::MirrorImage mirror_image;
+
+ if (r == 0) {
+ auto iter = m_out_bl.cbegin();
+ r = librbd::cls_client::mirror_image_get_finish(&iter, &mirror_image);
+ }
+
+ if (r < 0) {
+ derr << "error getting local mirror image: " << cpp_strerror(r) << dendl;
+ finish(r);
+ return;
+ }
+
+ m_remote_images.insert({spec.pool_id, mirror_image.global_image_id});
+
+ m_images.pop_front();
+ get_mirror_images();
+}
+
+template <typename I>
+void PrepareRemoteGroupRequest<I>::finish(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ if (r == 0) {
+ (*m_state_builder)->remote_group_id = m_remote_group_id;
+ (*m_state_builder)->group_name = *m_remote_group_name;
+ (*m_state_builder)->remote_promotion_state = m_promotion_state;
+ (*m_state_builder)->remote_images.insert(m_remote_images.begin(),
+ m_remote_images.end());
+ }
+
+ m_on_finish->complete(r);
+ delete this;
+}
+
+} // namespace group_replayer
+} // namespace mirror
+} // namespace rbd
+
+template class rbd::mirror::group_replayer::PrepareRemoteGroupRequest<librbd::ImageCtx>;
+
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef RBD_MIRROR_GROUP_REPLAYER_PREPARE_REMOTE_GROUP_REQUEST_H
+#define RBD_MIRROR_GROUP_REPLAYER_PREPARE_REMOTE_GROUP_REQUEST_H
+
+#include "include/buffer.h"
+#include "include/rados/librados_fwd.hpp"
+#include "cls/rbd/cls_rbd_types.h"
+#include "librbd/mirror/Types.h"
+#include "tools/rbd_mirror/group_replayer/Types.h"
+#include <string>
+
+struct Context;
+
+namespace librbd {
+struct ImageCtx;
+namespace asio { struct ContextWQ; }
+} // namespace librbd
+
+namespace rbd {
+namespace mirror {
+namespace group_replayer {
+
+template <typename> class GroupStateBuilder;
+
+template <typename ImageCtxT = librbd::ImageCtx>
+class PrepareRemoteGroupRequest {
+public:
+ static PrepareRemoteGroupRequest *create(
+ librados::IoCtx &io_ctx,
+ const std::string &global_group_id,
+ std::string *remote_group_name,
+ GroupStateBuilder<ImageCtxT>** state_builder,
+ Context *on_finish) {
+ return new PrepareRemoteGroupRequest(io_ctx, global_group_id,
+ remote_group_name, state_builder,
+ on_finish);
+ }
+
+ PrepareRemoteGroupRequest(
+ librados::IoCtx &io_ctx,
+ const std::string &global_group_id,
+ std::string *remote_group_name,
+ GroupStateBuilder<ImageCtxT>** state_builder,
+ Context *on_finish)
+ : m_io_ctx(io_ctx), m_global_group_id(global_group_id),
+ m_remote_group_name(remote_group_name), m_state_builder(state_builder),
+ m_on_finish(on_finish) {
+ }
+
+ void send();
+
+private:
+ /**
+ * @verbatim
+ *
+ * <start>
+ * |
+ * v
+ * GET_REMOTE_GROUP_ID
+ * |
+ * v
+ * GET_REMOTE_GROUP_NAME
+ * |
+ * v
+ * GET_MIRROR_INFO
+ * |
+ * |
+ * v
+ * LIST_GROUP_IMAGES
+ * |
+ * v
+ * GET_MIRROR_IMAGES
+ * |
+ * v
+ * <finish>
+ *
+ * @endverbatim
+ */
+
+ librados::IoCtx &m_io_ctx;
+ std::string m_global_group_id;
+ std::string *m_remote_group_name; // FIXME: Not required?
+ GroupStateBuilder<ImageCtxT>** m_state_builder;
+ Context *m_on_finish;
+
+ bufferlist m_out_bl;
+ std::string m_remote_group_id;
+ cls::rbd::MirrorGroup m_mirror_group;
+ librbd::mirror::PromotionState m_promotion_state;
+
+ std::list<cls::rbd::GroupImageStatus> m_images;
+ std::set<GlobalImageId> m_remote_images;
+
+ void get_remote_group_id();
+ void handle_get_remote_group_id(int r);
+
+ void get_remote_group_name();
+ void handle_get_remote_group_name(int r);
+
+ void get_mirror_info();
+ void handle_get_mirror_info(int r);
+
+ void list_group_images();
+ void handle_list_group_images(int r);
+
+ void get_mirror_images();
+ void handle_get_mirror_images(int r);
+
+ void finish(int r);
+
+};
+
+} // namespace group_replayer
+} // namespace mirror
+} // namespace rbd
+
+extern template class rbd::mirror::group_replayer::PrepareRemoteGroupRequest<librbd::ImageCtx>;
+
+#endif // RBD_MIRROR_GROUP_REPLAYER_PREPARE_REMOTE_GROUP_REQUEST_H
+
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "RemoveLocalGroupRequest.h"
+#include "include/rados/librados.hpp"
+#include "common/debug.h"
+#include "common/dout.h"
+#include "common/errno.h"
+#include "cls/rbd/cls_rbd_client.h"
+#include "cls/rbd/cls_rbd_types.h"
+#include "librbd/ImageCtx.h"
+#include "librbd/MirroringWatcher.h"
+#include "librbd/Utils.h"
+#include "librbd/group/RemoveImageRequest.h"
+#include "librbd/mirror/GroupGetInfoRequest.h"
+#include "tools/rbd_mirror/image_deleter/TrashMoveRequest.h"
+
+#define dout_context g_ceph_context
+#define dout_subsys ceph_subsys_rbd_mirror
+#undef dout_prefix
+#define dout_prefix *_dout << "rbd::mirror::group_replayer::" \
+ << "RemoveLocalGroupRequest: " << this << " " \
+ << __func__ << ": "
+namespace {
+ static const uint32_t MAX_RETURN = 1024;
+} // anonymous namespace
+
+namespace rbd {
+namespace mirror {
+namespace group_replayer {
+
+using librbd::util::create_context_callback;
+using librbd::util::create_rados_callback;
+
+template <typename I>
+void RemoveLocalGroupRequest<I>::send() {
+ get_local_group_id();
+}
+
+template <typename I>
+void RemoveLocalGroupRequest<I>::get_local_group_id() {
+ dout(10) << dendl;
+
+ librados::ObjectReadOperation op;
+ librbd::cls_client::mirror_group_get_group_id_start(&op, m_global_group_id);
+
+ m_out_bl.clear();
+ auto comp = create_rados_callback<
+ RemoveLocalGroupRequest<I>,
+ &RemoveLocalGroupRequest<I>::handle_get_local_group_id>(this);
+
+ int r = m_io_ctx.aio_operate(RBD_MIRRORING, comp, &op, &m_out_bl);
+ ceph_assert(r == 0);
+ comp->release();
+}
+
+template <typename I>
+void RemoveLocalGroupRequest<I>::handle_get_local_group_id(int r) {
+ dout(10) << "r=" << r << ", global_group_id: " << m_global_group_id << dendl;
+
+ if (r == 0) {
+ auto iter = m_out_bl.cbegin();
+ r = librbd::cls_client::mirror_group_get_group_id_finish(
+ &iter, &m_group_id);
+ }
+
+ if (r < 0) {
+ if (r != -ENOENT) {
+ derr << "error getting local group id: " << cpp_strerror(r) << dendl;
+ }
+ finish(r);
+ return;
+ }
+
+ get_local_group_name();
+}
+
+template <typename I>
+void RemoveLocalGroupRequest<I>::get_local_group_name() {
+ dout(10) << dendl;
+
+ librados::ObjectReadOperation op;
+ librbd::cls_client::dir_get_name_start(&op, m_group_id);
+
+ m_out_bl.clear();
+ auto comp = create_rados_callback<
+ RemoveLocalGroupRequest<I>,
+ &RemoveLocalGroupRequest<I>::handle_get_local_group_name>(this);
+
+ int r = m_io_ctx.aio_operate(RBD_GROUP_DIRECTORY, comp, &op,
+ &m_out_bl);
+ ceph_assert(r == 0);
+ comp->release();
+}
+
+template <typename I>
+void RemoveLocalGroupRequest<I>::handle_get_local_group_name(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ m_group_name = "";
+ if (r == 0) {
+ auto iter = m_out_bl.cbegin();
+ r = librbd::cls_client::dir_get_name_finish(&iter, &m_group_name);
+ }
+
+ if (r == -ENOENT) {
+ // proceed - we should have a mirror group record if we got this far
+ dout(10) << "local group does not exist for id " << m_group_id
+ << dendl;
+ } else if (r < 0) {
+ derr << "error getting local group name: " << cpp_strerror(r) << dendl;
+ finish(r);
+ return;
+ }
+
+ get_mirror_group();
+}
+
+template <typename I>
+void RemoveLocalGroupRequest<I>::get_mirror_group() {
+ dout(10) << dendl;
+
+ auto ctx = create_context_callback<
+ RemoveLocalGroupRequest<I>,
+ &RemoveLocalGroupRequest<I>::handle_get_mirror_group>(this);
+ auto req = librbd::mirror::GroupGetInfoRequest<I>::create(
+ m_io_ctx, "", m_group_id, &m_mirror_group, &m_promotion_state,
+ ctx);
+ req->send();
+}
+
+template <typename I>
+void RemoveLocalGroupRequest<I>::handle_get_mirror_group(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ if (r < 0) {
+ if (r == -ENOENT) {
+ dout(10) << "group " << m_global_group_id << " is not mirrored" << dendl;
+ } else {
+ derr << "error retrieving mirror info for group "
+ << m_global_group_id << ": " << cpp_strerror(r) << dendl;
+ }
+ finish(r);
+ return;
+ }
+
+ if (m_promotion_state == librbd::mirror::PROMOTION_STATE_PRIMARY) {
+ dout(10) << "group " << m_global_group_id << " is local primary" << dendl;
+ finish(-EPERM);
+ return;
+ } else if (m_promotion_state == librbd::mirror::PROMOTION_STATE_ORPHAN &&
+ !m_resync) {
+ dout(10) << "group " << m_global_group_id << " is orphaned" << dendl;
+ finish(-EPERM);
+ return;
+ }
+
+ if (m_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED) {
+ m_notify_watcher = true;
+ }
+
+ disable_mirror_group();
+}
+
+template <typename I>
+void RemoveLocalGroupRequest<I>::disable_mirror_group() {
+ dout(10) << dendl;
+
+ // need to send 'disabling' since the cls methods will fail if we aren't
+ // in that state
+ m_mirror_group.state= cls::rbd::MIRROR_GROUP_STATE_DISABLING;
+
+ librados::ObjectWriteOperation op;
+ librbd::cls_client::mirror_group_set(&op, m_group_id,
+ m_mirror_group);
+
+ auto aio_comp = create_rados_callback<
+ RemoveLocalGroupRequest<I>,
+ &RemoveLocalGroupRequest<I>::handle_disable_mirror_group>(this);
+ int r = m_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op);
+ ceph_assert(r == 0);
+ aio_comp->release();
+}
+
+template <typename I>
+void RemoveLocalGroupRequest<I>::handle_disable_mirror_group(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ if (r < 0) {
+ derr << "failed to disable mirror group " << m_global_group_id << ": "
+ << cpp_strerror(r) << dendl;
+ finish(r);
+ return;
+ }
+
+ list_group_images();
+}
+
+template <typename I>
+void RemoveLocalGroupRequest<I>::list_group_images() {
+ dout(10) << dendl;
+
+ librados::ObjectReadOperation op;
+ cls::rbd::GroupImageSpec start_after;
+
+ m_out_bl.clear();
+
+ if (!m_images.empty()) {
+ start_after = m_images.rbegin()->spec;
+ }
+
+ librbd::cls_client::group_image_list_start(&op, start_after, MAX_RETURN);
+ auto comp = create_rados_callback<
+ RemoveLocalGroupRequest<I>,
+ &RemoveLocalGroupRequest<I>::handle_list_group_images>(this);
+ int r = m_io_ctx.aio_operate(
+ librbd::util::group_header_name(m_group_id), comp, &op, &m_out_bl);
+ ceph_assert(r == 0);
+ comp->release();
+}
+
+template <typename I>
+void RemoveLocalGroupRequest<I>::handle_list_group_images(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ std::vector<cls::rbd::GroupImageStatus> images;
+ if (r == 0) {
+ auto iter = m_out_bl.cbegin();
+ r = librbd::cls_client::group_image_list_finish(&iter, &images);
+ }
+
+ // -ENOENT == group creation failed in the middle
+ if (r < 0 && r != -ENOENT) {
+ dout(10) << "error listing local group: " << cpp_strerror(r) << dendl;
+ finish(r);
+ return;
+ }
+
+ m_images.insert(m_images.end(), images.begin(), images.end());
+
+ if (images.size() == MAX_RETURN) {
+ list_group_images();
+ return;
+ }
+ m_num_images = m_images.size();
+ dout(10) << "number of images: " << m_images.size() << dendl;
+ get_mirror_images();
+}
+
+template <typename I>
+void RemoveLocalGroupRequest<I>::get_mirror_images() {
+ dout(10) << dendl;
+
+ if (m_images.empty()) {
+ remove_image_from_group();
+ return;
+ }
+
+ auto &spec = m_images.front().spec;
+
+ dout(10) << "pool_id: " << spec.pool_id
+ << ", image_id " << spec.image_id << dendl;
+
+ // FIXME: Currently images must be in the same pool as the group
+ librados::ObjectReadOperation op;
+ librbd::cls_client::mirror_image_get_start(&op, spec.image_id);
+
+ m_out_bl.clear();
+ auto comp = create_rados_callback<
+ RemoveLocalGroupRequest<I>,
+ &RemoveLocalGroupRequest<I>::handle_get_mirror_images>(this);
+
+ int r = m_io_ctx.aio_operate(RBD_MIRRORING, comp, &op, &m_out_bl);
+ ceph_assert(r == 0);
+ comp->release();
+}
+
+template <typename I>
+void RemoveLocalGroupRequest<I>::handle_get_mirror_images(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ auto &spec = m_images.front().spec;
+ cls::rbd::MirrorImage mirror_image;
+
+ if (r == 0) {
+ auto iter = m_out_bl.cbegin();
+ r = librbd::cls_client::mirror_image_get_finish(&iter, &mirror_image);
+ }
+
+ if (r < 0) {
+ derr << "error getting local mirror image: " << cpp_strerror(r) << dendl;
+ finish(r);
+ return;
+ } else {
+ m_trash_images.insert({mirror_image.global_image_id,
+ {spec.pool_id, spec.image_id}});
+ }
+
+ m_images.pop_front();
+ get_mirror_images();
+}
+
+
+template <typename I>
+void RemoveLocalGroupRequest<I>::remove_image_from_group() {
+
+ if (m_trash_images.empty()) {
+ remove_local_group();
+ return;
+ }
+
+ auto &[pool_id, image_id] = m_trash_images.begin()->second;
+ dout(10) << "global_image_id=" << m_trash_images.begin()->first
+ << " ,image_id=" << image_id
+ << " ,pool_id=" << pool_id
+ << " ,group_id=" << m_group_id << dendl;
+
+ auto ctx = create_context_callback<
+ RemoveLocalGroupRequest,
+ &RemoveLocalGroupRequest<I>::handle_remove_image_from_group>(this);
+
+ auto req = librbd::group::RemoveImageRequest<I>::create(
+ m_io_ctx, m_group_id, m_io_ctx, image_id, ctx);
+ req->send();
+}
+
+template <typename I>
+void RemoveLocalGroupRequest<I>::handle_remove_image_from_group(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ if (r < 0 && r != -ENOENT) {
+ finish(r);
+ }
+
+ move_image_to_trash();
+}
+
+template <typename I>
+void RemoveLocalGroupRequest<I>::move_image_to_trash() {
+ ceph_assert(!m_trash_images.empty());
+
+ auto &global_image_id = m_trash_images.begin()->first;
+ dout(10) << "global image id=" << global_image_id << dendl;
+
+ auto ctx = create_context_callback<
+ RemoveLocalGroupRequest,
+ &RemoveLocalGroupRequest<I>::handle_move_image_to_trash>(this);
+
+ auto req = image_deleter::TrashMoveRequest<I>::create(
+ m_io_ctx, global_image_id, m_resync,
+ m_work_queue, ctx);
+ req->send();
+}
+
+template <typename I>
+void RemoveLocalGroupRequest<I>::handle_move_image_to_trash(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ if (r < 0 && r != -ENOENT) {
+ finish(r);
+ }
+
+ m_trash_images.erase(m_trash_images.begin());
+ remove_image_from_group();
+}
+
+template <typename I>
+void RemoveLocalGroupRequest<I>::remove_local_group() {
+ dout(10) << "group_name=" << m_group_name
+ << " ,group_id=" << m_group_id << dendl;
+
+ librados::ObjectWriteOperation op;
+ op.remove();
+ auto comp = create_rados_callback<
+ RemoveLocalGroupRequest<I>,
+ &RemoveLocalGroupRequest<I>::handle_remove_local_group>(this);
+
+ int r = m_io_ctx.aio_operate(
+ librbd::util::group_header_name(m_group_id), comp, &op);
+ ceph_assert(r == 0);
+ comp->release();
+}
+
+template <typename I>
+void RemoveLocalGroupRequest<I>::handle_remove_local_group(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ if (r < 0 && r != -ENOENT) {
+ derr << "error removing local group: " << cpp_strerror(r) << dendl;
+ finish(r);
+ return;
+ }
+
+ remove_local_group_id();
+}
+
+template <typename I>
+void RemoveLocalGroupRequest<I>::remove_local_group_id() {
+ dout(10) << dendl;
+
+ librados::ObjectWriteOperation op;
+ librbd::cls_client::group_dir_remove(&op, m_group_name, m_group_id);
+
+ auto comp = create_rados_callback<
+ RemoveLocalGroupRequest<I>,
+ &RemoveLocalGroupRequest<I>::handle_remove_local_group_id>(this);
+
+ int r = m_io_ctx.aio_operate(RBD_GROUP_DIRECTORY, comp, &op);
+ ceph_assert(r == 0);
+ comp->release();
+}
+
+template <typename I>
+void RemoveLocalGroupRequest<I>::handle_remove_local_group_id(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ if (r < 0 && r != -ENOENT) {
+ derr << "error removing local group id: " << cpp_strerror(r) << dendl;
+ finish(r);
+ return;
+ }
+
+ remove_mirror_group();
+}
+
+template <typename I>
+void RemoveLocalGroupRequest<I>::remove_mirror_group() {
+ dout(10) << dendl;
+
+ librados::ObjectWriteOperation op;
+ librbd::cls_client::mirror_group_remove(&op, m_group_id);
+
+ auto aio_comp = create_rados_callback<
+ RemoveLocalGroupRequest<I>,
+ &RemoveLocalGroupRequest<I>::handle_remove_mirror_group>(this);
+ int r = m_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op);
+ ceph_assert(r == 0);
+ aio_comp->release();
+}
+
+template <typename I>
+void RemoveLocalGroupRequest<I>::handle_remove_mirror_group(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ if (r < 0) {
+ derr << "failed to remove mirror group " << m_global_group_id << ": "
+ << cpp_strerror(r) << dendl;
+ finish(r);
+ return;
+ }
+
+//FIXME: Should the mirroring watcher be notified?
+ notify_mirroring_watcher();
+}
+
+template <typename I>
+void RemoveLocalGroupRequest<I>::notify_mirroring_watcher() {
+ if (!m_notify_watcher) {
+ finish(0);
+ return;
+ }
+
+ dout(10) << dendl;
+
+ auto ctx = create_context_callback<
+ RemoveLocalGroupRequest<I>,
+ &RemoveLocalGroupRequest<I>::handle_notify_mirroring_watcher>(this);
+
+ librbd::MirroringWatcher<I>::notify_group_updated(
+ m_io_ctx, cls::rbd::MIRROR_GROUP_STATE_DISABLED, m_group_id,
+ m_global_group_id, m_num_images, ctx);
+}
+
+template <typename I>
+void RemoveLocalGroupRequest<I>::handle_notify_mirroring_watcher(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ if (r < 0) {
+ derr << "failed to notify mirror group update: " << cpp_strerror(r)
+ << dendl;
+ }
+
+ finish(0);
+}
+
+
+template <typename I>
+void RemoveLocalGroupRequest<I>::finish(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ m_on_finish->complete(r);
+ delete this;
+}
+
+} // namespace group_replayer
+} // namespace mirror
+} // namespace rbd
+
+template class rbd::mirror::group_replayer::RemoveLocalGroupRequest<librbd::ImageCtx>;
+
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef RBD_MIRROR_GROUP_REPLAYER_REMOVE_LOCAL_GROUP_REQUEST_H
+#define RBD_MIRROR_GROUP_REPLAYER_REMOVE_LOCAL_GROUP_REQUEST_H
+
+#include "include/buffer.h"
+#include "include/rados/librados_fwd.hpp"
+#include "cls/rbd/cls_rbd_types.h"
+#include "librbd/mirror/Types.h"
+#include <string>
+
+struct Context;
+
+namespace librbd {
+struct ImageCtx;
+namespace asio { struct ContextWQ; }
+} // namespace librbd
+
+namespace rbd {
+namespace mirror {
+namespace group_replayer {
+
+template <typename> class GroupStateBuilder;
+
+
+template <typename ImageCtxT = librbd::ImageCtx>
+class RemoveLocalGroupRequest {
+public:
+
+ static RemoveLocalGroupRequest *create(
+ librados::IoCtx &io_ctx,
+ const std::string &global_group_id,
+ bool resync,
+ librbd::asio::ContextWQ *work_queue,
+ Context *on_finish) {
+ return new RemoveLocalGroupRequest(io_ctx, global_group_id,
+ resync, work_queue, on_finish);
+ }
+
+ RemoveLocalGroupRequest(
+ librados::IoCtx &io_ctx,
+ const std::string &global_group_id,
+ bool resync,
+ librbd::asio::ContextWQ *work_queue,
+ Context *on_finish)
+ : m_io_ctx(io_ctx), m_global_group_id(global_group_id),
+ m_resync(resync), m_work_queue(work_queue), m_on_finish(on_finish) {
+ }
+
+ void send();
+
+private:
+ /**
+ * @verbatim
+ *
+ * <start>
+ * |
+ * v
+ * GET_GROUP_ID
+ * |
+ * v
+ * GET_GROUP_NAME
+ * |
+ * v
+ * GET_MIRROR_GROUP
+ * |
+ * v
+ * DISABLE_MIRROR_GROUP
+ * |
+ * v
+ * REMOVE_IMAGE_FROM_GROUP <--
+ * | |
+ * v |
+ * IMAGE_TRASH_MOVE ----------
+ * |
+ * v
+ * REMOVE_LOCAL_GROUP
+ * |
+ * v
+ * REMOVE_LOCAL_GROUP_ID
+ * |
+ * v
+ * REMOVE_MIRROR_GROUP
+ * |
+ * v
+ * NOTIFY_MIRRORING_WATCHER
+ * |
+ * v
+ * <finish>
+ *
+ * @endverbatim
+ */
+
+ librados::IoCtx &m_io_ctx;
+ std::string m_global_group_id;
+ bool m_resync;
+ librbd::asio::ContextWQ *m_work_queue;
+ Context *m_on_finish;
+
+ std::string m_group_id;
+ std::string m_group_name;
+ uint64_t m_num_images;
+
+ bufferlist m_out_bl;
+ std::list<cls::rbd::GroupImageStatus> m_images;
+
+ cls::rbd::MirrorGroup m_mirror_group;
+ librbd::mirror::PromotionState m_promotion_state;
+ std::map<std::string /*global_image_id*/, std::pair<int64_t/*pool_id*/, std::string /*image_id*/>> m_trash_images;
+
+ bool m_notify_watcher = false;
+
+ void get_local_group_id();
+ void handle_get_local_group_id(int r);
+
+ void get_local_group_name();
+ void handle_get_local_group_name(int r);
+
+ void get_mirror_group();
+ void handle_get_mirror_group(int r);
+
+ void disable_mirror_group();
+ void handle_disable_mirror_group(int r);
+
+ void list_group_images();
+ void handle_list_group_images(int r);
+
+ void get_mirror_images();
+ void handle_get_mirror_images(int r);
+
+ void remove_image_from_group();
+ void handle_remove_image_from_group(int r);
+
+ void move_image_to_trash();
+ void handle_move_image_to_trash(int r);
+
+ void remove_local_group();
+ void handle_remove_local_group(int r);
+
+ void remove_local_group_id();
+ void handle_remove_local_group_id(int r);
+
+ void remove_mirror_group();
+ void handle_remove_mirror_group(int r);
+
+ void notify_mirroring_watcher();
+ void handle_notify_mirroring_watcher(int r);
+
+ void finish(int r);
+
+};
+
+} // namespace group_replayer
+} // namespace mirror
+} // namespace rbd
+
+extern template class rbd::mirror::group_replayer::RemoveLocalGroupRequest<librbd::ImageCtx>;
+
+#endif // RBD_MIRROR_GROUP_REPLAYER_REMOVE_LOCAL_GROUP_REQUEST_H
+
// vim: ts=8 sw=2 smarttab
#include "Replayer.h"
+#include "GroupMirrorStateUpdateRequest.h"
#include "common/Cond.h"
#include "common/debug.h"
#include "common/errno.h"
}
template <typename I>
-void Replayer<I>::notify_group_listener_stop() {
+void Replayer<I>::handle_replay_complete(int r, const std::string &desc) {
+ dout(10) << "r=" << r << ", desc=" << desc << dendl;
+
+ std::unique_lock locker{m_lock};
+ if (m_error_code == 0) {
+ m_error_code = r;
+ m_error_description = desc;
+ }
+
+ if (m_state != STATE_REPLAYING && m_state != STATE_IDLE) {
+ return;
+ }
+
+ m_stop_requested = true;
+ m_state = STATE_COMPLETE;
+ notify_group_listener();
+}
+
+template <typename I>
+void Replayer<I>::notify_group_listener() {
dout(10) << dendl;
Context *ctx = new LambdaContext([this](int) {
- m_local_group_ctx->listener->stop();
+ m_local_group_ctx->listener->handle_notification();
+ m_in_flight_op_tracker.finish_op();
});
+ m_in_flight_op_tracker.start_op();
m_threads->work_queue->queue(ctx, 0);
}
on_finish->complete(0);
+ m_update_group_state = true;
load_local_group_snapshots();
}
if (m_stop_requested) {
return;
} else if (is_rename_requested()) {
- m_stop_requested = true;
- dout(10) << "remote group rename requested" << dendl;
- // send stop for Group Replayer
- notify_group_listener_stop();
+ dout(10) << "remote group renamed" << dendl;
+ locker.unlock();
+ handle_replay_complete(0, "remote group renamed");
return;
}
}
if (r < 0) {
derr << "error listing local mirror group snapshots: " << cpp_strerror(r)
<< dendl;
- notify_group_listener_stop();
+ locker.unlock();
+ handle_replay_complete(r, "failed to list local group snapshots");
return;
}
if (ns->state != cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY) {
break;
}
- m_state = STATE_COMPLETE;
- notify_group_listener_stop();
+ locker.unlock();
+ handle_replay_complete(0, "local group is primary");
return;
}
if (r < 0) { // may be remote group is deleted?
derr << "error listing remote mirror group snapshots: " << cpp_strerror(r)
<< dendl;
- notify_group_listener_stop();
+ handle_replay_complete(r, "failed to list remote group snapshots");
return;
} else if (is_resync_requested()) {
dout(10) << "local group resync requested" << dendl;
&last_remote_snap->snapshot_namespace);
if (last_remote_snap_ns &&
last_remote_snap_ns->state == cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY) {
- m_stop_requested = true;
- // send stop for Group Replayer
- notify_group_listener_stop();
+ handle_replay_complete(0, "resync requested");
return;
}
- dout(10) << "turns out remote is not primary, we cannot resync, will retry later"
- << dendl;
+ dout(10) << "cannot resync as remote is not primary" << dendl;
}
if (!m_local_group_snaps.empty()) {
last_local_snap_ns->state == cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY_DEMOTED &&
!m_remote_group_snaps.empty()) {
if (last_local_snap->id == last_remote_snap->id) {
- m_stop_requested = true;
- notify_group_listener_stop();
+ handle_replay_complete(-EREMOTEIO, "remote group demoted");
+ return;
+ }
+ } else if (last_local_snap_ns &&
+ last_local_snap_ns->state == cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED) {
+ bool split_brain = true;
+ for (auto it = m_remote_group_snaps.begin();
+ it != m_remote_group_snaps.end(); it++) {
+ auto ns = std::get_if<cls::rbd::GroupSnapshotNamespaceMirror>(
+ &it->snapshot_namespace);
+ if (ns == nullptr ||
+ it->id != last_local_snap->id) {
+ continue;
+ }
+ if (ns->state == cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY_DEMOTED) {
+ split_brain = false;
+ break;
+ }
+ }
+ if (split_brain) {
+ handle_replay_complete(-EEXIST, "split-brain");
return;
}
}
locker.unlock();
if (m_stop_requested) {
// stop group replayer
- notify_group_listener_stop();
+ handle_replay_complete(0, "");
return;
}
schedule_load_group_snapshots();
snap_ns->state == cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY ?
cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY :
cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY_DEMOTED;
- C_SaferCond *ctx = new C_SaferCond;
- create_mirror_snapshot(&snap, snap_state, locker, ctx);
- ctx->wait();
+
+ C_SaferCond create_ctx;
+ create_mirror_snapshot(&snap, snap_state, locker, &create_ctx);
+ int r = create_ctx.wait();
+
+ if (r == 0 && m_update_group_state) {
+ // Set the mirror group state to enabled after the first non-primary
+ // mirror snapshot is created
+ C_SaferCond update_ctx;
+ auto req = GroupMirrorStateUpdateRequest<I>::create(m_local_io_ctx,
+ m_local_group_id,
+ m_image_replayers->size(),
+ &update_ctx);
+ req->send();
+ r = update_ctx.wait();
+ if (r < 0) {
+ // failed to set group state
+ handle_replay_complete(r, "failed to set group state to enabled");
+ return;
+ }
+ m_update_group_state = false;
+ }
+ if (r == 0) {
+
+ // if m_replayer in the ImageReplayer is null this cannot be forwarded.
+ // May be we should retry this setting in the validate_image_snaps_sync_complete().
+ // Same for image_replayer->prune_snapshot(); setting actually!!!!
+ set_image_replayer_limits("", &snap);
+ }
} else if (snap_type == cls::rbd::GROUP_SNAPSHOT_NAMESPACE_TYPE_USER) {
bool found = false;
auto next_remote_snap = m_remote_group_snaps.end();
mirror_peer_uuids.insert(peer.uuid);
}
}
+
cls::rbd::GroupSnapshot local_snap =
{group_snap_id,
cls::rbd::GroupSnapshotNamespaceMirror{
}
m_local_group_snaps.erase(local_snap);
}
- } else {
- // if m_replayer in the ImageReplayer is null this cannot be forwarded.
- // May be we should retry this setting in the validate_image_snaps_sync_complete().
- // Same for image_replayer->prune_snapshot(); setting actually!!!!
- set_image_replayer_limits("", snap);
}
on_finish->complete(r);
bool get_replay_status(std::string* description);
+ int get_error_code() const {
+ std::unique_lock locker(m_lock);
+ return m_error_code;
+ }
+
+ std::string get_error_description() const {
+ std::unique_lock locker(m_lock);
+ return m_error_description;
+ }
+
+
private:
enum State {
STATE_INIT,
std::vector<cls::rbd::GroupSnapshot> m_local_group_snaps;
std::vector<cls::rbd::GroupSnapshot> m_remote_group_snaps;
+ bool m_update_group_state = true;
+
Context* m_load_snapshots_task = nullptr;
Context* m_on_shutdown = nullptr;
AsyncOpTracker m_in_flight_op_tracker;
+ int m_error_code = 0;
+ std::string m_error_description;
+
bool m_stop_requested = false;
bool m_retry_validate_snap = false;
void handle_schedule_load_group_snapshots(int r);
void cancel_load_group_snapshots();
- void notify_group_listener_stop();
+ void handle_replay_complete(int r, const std::string& desc);
+ void notify_group_listener();
+
bool is_resync_requested();
bool is_rename_requested();
Context *on_finish);
void handle_regular_snapshot_complete(
int r, const std::string &group_snap_id, Context *on_finish);
+
};
} // namespace group_replayer
HEALTH_STATE_ERROR
};
+typedef std::pair<int64_t /*pool_id*/, std::string /*global_image_id*/> GlobalImageId;
+
} // namespace group_replayer
} // namespace mirror
} // namespace rbd
projects / ceph.git / commitdiff