template <typename I>
void GetInfoRequest<I>::get_mirror_image() {
- ldout(m_cct, 20) << dendl;
+ ldout(m_cct, 20) << "image_id: " << m_image_id << dendl;
librados::ObjectReadOperation op;
cls_client::mirror_image_get_start(&op, m_image_id);
struct MockReplayerListener : public image_replayer::ReplayerListener {
MOCK_METHOD0(handle_notification, void());
+ MOCK_METHOD5(create_mirror_snapshot_start,
+ void(const cls::rbd::MirrorSnapshotNamespace &,
+ int64_t *, std::string *, std::string *, Context *));
+ MOCK_METHOD3(create_mirror_snapshot_finish, void(const std::string &,
+ uint64_t, Context *));
};
} // anonymous namespace
struct MockReplayerListener : public image_replayer::ReplayerListener {
MOCK_METHOD0(handle_notification, void());
+ MOCK_METHOD5(create_mirror_snapshot_start,
+ void(const cls::rbd::MirrorSnapshotNamespace &,
+ int64_t *, std::string *, std::string *, Context *));
+ MOCK_METHOD3(create_mirror_snapshot_finish, void(const std::string &,
+ uint64_t, Context *));
};
} // anonymous namespace
#include "tools/rbd_mirror/image_replayer/Utils.h"
#include "GroupReplayer.h"
+#include <algorithm>
+
#define dout_context g_ceph_context
#define dout_subsys ceph_subsys_rbd_mirror
#undef dout_prefix
namespace mirror {
using librbd::util::create_context_callback;
+using librbd::util::create_rados_callback;
using librbd::util::unique_lock_name;
namespace {
-std::string calc_ind_mirror_snap_name(uint64_t pool_id,
- const std::string &group_id,
- const std::string &snap_id)
-{
- std::stringstream ind_snap_name_stream;
- ind_snap_name_stream << ".mirror." << std::hex << pool_id << "_"
- << group_id << "_" << snap_id;
- return ind_snap_name_stream.str();
-}
-
template <typename I>
class GroupReplayerAdminSocketCommand {
public:
template <typename I>
bool GroupReplayer<I>::needs_restart() const {
- std::lock_guard locker{m_lock};
+ dout(10) << dendl;
+ std::lock_guard locker{m_lock};
if (m_state != STATE_REPLAYING) {
return false;
}
for (auto &[_, image_replayer] : m_image_replayers) {
if (image_replayer->is_stopped()) {
+ dout(10) << "image replayer is in stopped state, needs restart" << dendl;
return true;
}
}
{
std::lock_guard locker{m_lock};
if (!is_stopped_()) {
- derr << "already running" << dendl;
+ derr << "already running: " << m_state << dendl;
r = -EINVAL;
} else if (m_manual_stop && !manual) {
dout(5) << "stopped manually, ignoring start without manual flag"
m_state = STATE_STARTING;
m_last_r = 0;
m_state_desc.clear();
+ m_local_group_snaps.clear();
+ m_remote_group_snaps.clear();
+ m_image_replayers.clear();
+ m_image_replayer_index.clear();
+ m_get_remote_group_snap_ret_vals.clear();
m_manual_stop = false;
ceph_assert(m_on_start_finish == nullptr);
std::swap(m_on_start_finish, on_finish);
<< ", restart=" << restart << dendl;
group_replayer::BootstrapRequest<I> *bootstrap_request = nullptr;
+ std::map<std::string, std::map<ImageReplayer<I> *, Context *>> create_snap_requests;
bool shut_down_replay = false;
bool running = true;
{
m_restart_requested = true;
}
+ dout(10) << "state: " << m_state << ", m_stop_requested: " << m_stop_requested << dendl;
if (!is_running_()) {
+ dout(10) << "replayers not running" << dendl;
running = false;
if (!restart && m_restart_requested) {
dout(10) << "canceling restart" << dendl;
m_restart_requested = false;
}
} else {
+ dout(10) << "replayers still running" << dendl;
if (!is_stopped_()) {
if (m_state == STATE_STARTING) {
dout(10) << "canceling start" << dendl;
bootstrap_request = m_bootstrap_request;
bootstrap_request->get();
}
- } else {
- dout(10) << "interrupting replay" << dendl;
shut_down_replay = true;
- m_state = STATE_STOPPING;
+ } else {
+ dout(10) << "interrupting replay" << dendl;
+ shut_down_replay = true;
+ for (auto it = m_create_snap_requests.begin();
+ it != m_create_snap_requests.end(); ) {
+ auto &remote_group_snap_id = it->first;
+ auto &requests = it->second;
+ if (m_get_remote_group_snap_ret_vals.count(remote_group_snap_id) == 0) {
+ dout(20) << "getting remote group snap for "
+ << remote_group_snap_id << " is still in-progress"
+ << dendl;
+ shut_down_replay = false;
+ } else if (m_pending_snap_create.count(remote_group_snap_id) > 0) {
+ dout(20) << "group snap create for " << remote_group_snap_id
+ << " is still in-progress" << dendl;
+ shut_down_replay = false;
+ } else {
+ create_snap_requests[remote_group_snap_id] = requests;
+ it = m_create_snap_requests.erase(it);
+ continue;
+ }
+ it++;
+ }
}
+ m_state = STATE_STOPPING;
ceph_assert(m_on_stop_finish == nullptr);
std::swap(m_on_stop_finish, on_finish);
bootstrap_request->put();
}
- if (!running) {
- dout(20) << "not running" << dendl;
- if (on_finish) {
- on_finish->complete(-EINVAL);
+ for (auto &[_, requests] : create_snap_requests) {
+ for (auto &[_, on_finish] : requests) {
+ on_finish->complete(-ESTALE);
}
- return;
}
if (shut_down_replay) {
}
on_finish->complete(0);
}
+
+ if (!running && shut_down_replay) {
+ dout(20) << "not running" << dendl;
+ if (on_finish) {
+ on_finish->complete(-EINVAL);
+ }
+ }
}
template <typename I>
void GroupReplayer<I>::restart(Context *on_finish) {
+ dout(10) << dendl;
{
std::lock_guard locker{m_lock};
m_restart_requested = true;
+ m_on_start_finish = nullptr;
}
auto ctx = new LambdaContext(
m_threads, m_local_io_ctx, m_remote_group_peer.io_ctx, m_global_group_id,
m_local_mirror_uuid, m_instance_watcher, m_local_status_updater,
m_remote_group_peer.mirror_status_updater, m_cache_manager_handler,
- m_pool_meta_cache, &m_local_group_ctx, &m_image_replayers, ctx);
+ m_pool_meta_cache, &m_remote_group_id, &m_local_group_ctx,
+ &m_image_replayers, &m_image_replayer_index, ctx);
request->get();
m_bootstrap_request = request;
dout(10) << "r=" << r << dendl;
{
std::lock_guard locker{m_lock};
- m_bootstrap_request->put();
- m_bootstrap_request = nullptr;
+ if (m_state == STATE_STOPPING || m_state == STATE_STOPPED) {
+ dout(10) << "stop prevailed" <<dendl;
+ return;
+ }
+ if (m_bootstrap_request != nullptr) {
+ m_bootstrap_request->put();
+ m_bootstrap_request = nullptr;
+ }
+ m_local_group_ctx.listener = &m_listener;
+ if (!m_local_group_ctx.name.empty()) {
+ m_local_group_name = m_local_group_ctx.name;
+ }
}
+ reregister_admin_socket_hook();
+
if (finish_start_if_interrupted()) {
return;
} else if (r == -ENOENT) {
return;
}
- {
- std::lock_guard locker{m_lock};
- m_local_group_name = m_local_group_ctx.name;
- }
- reregister_admin_socket_hook();
-
start_image_replayers();
}
template <typename I>
void GroupReplayer<I>::handle_start_image_replayers(int r) {
dout(10) << "r=" << r << dendl;
+ {
+ std::lock_guard locker{m_lock};
+ if (m_state == STATE_STOPPING || m_state == STATE_STOPPED) {
+ dout(10) << "stop prevailed" <<dendl;
+ return;
+ }
+ }
if (finish_start_if_interrupted()) {
return;
m_state = STATE_STOPPED;
std::swap(on_finish, m_on_stop_finish);
+ m_image_replayer_index.clear();
+
for (auto &[_, image_replayer] : m_image_replayers) {
delete image_replayer;
}
template <typename I>
void GroupReplayer<I>::create_mirror_snapshot_start(
- const std::string &remote_group_snap_id, ImageReplayer<I> *image_replayer,
- int64_t *local_group_pool_id, std::string *local_group_id,
- std::string *local_group_snap_id, Context *on_finish) {
- dout(20) << remote_group_snap_id << " " << image_replayer << dendl;
+ const cls::rbd::MirrorSnapshotNamespace &remote_group_snap_ns,
+ ImageReplayer<I> *image_replayer, int64_t *local_group_pool_id,
+ std::string *local_group_id, std::string *local_group_snap_id,
+ Context *on_finish) {
+ dout(20) << remote_group_snap_ns << " " << image_replayer << dendl;
+
+ ceph_assert(remote_group_snap_ns.is_primary());
+ int r = 0;
std::unique_lock locker{m_lock};
- ceph_assert(m_pending_snap_create == false);
+ auto &remote_group_snap_id = remote_group_snap_ns.group_snap_id;
+ if (m_local_group_snaps.find(remote_group_snap_id) != m_local_group_snaps.end() &&
+ m_local_group_snaps[remote_group_snap_id].state == cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE) {
+ dout(20) << "group snapshot: " << remote_group_snap_id << " already exists"
+ << dendl;
+ r = -EEXIST;
+ } else if (m_state == STATE_STARTING) {
+ derr << "group replayer is not ready yet, m_state: " << m_state << dendl;
+ r = -EAGAIN;
+ } else if (m_state != STATE_REPLAYING) {
+ derr << "group replayer is not in replaying state, m_state: "
+ << m_state << dendl;
+ r = -ESTALE;
+ }
- if (m_state != STATE_REPLAYING) {
- derr << "not in replaying state" << dendl;
+ if (r != 0) {
locker.unlock();
- on_finish->complete(-ESTALE);
+ on_finish->complete(r);
return;
}
- if (m_remote_group_snap_id.empty()) {
- ceph_assert(m_create_snap_requests.empty());
- m_remote_group_snap_id = remote_group_snap_id;
+ auto requests_it = m_create_snap_requests.find(remote_group_snap_id);
+
+ if (requests_it == m_create_snap_requests.end()) {
+ ceph_assert(m_local_group_snaps.count(remote_group_snap_id) == 0);
+
+ requests_it = m_create_snap_requests.insert(
+ {remote_group_snap_id, {}}).first;
+
+ auto snap_state =
+ remote_group_snap_ns.state == cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY ?
+ cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY :
+ cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY_DEMOTED;
+
+ m_local_group_snaps[remote_group_snap_id] =
+ {remote_group_snap_id,
+ cls::rbd::MirrorGroupSnapshotNamespace{
+ snap_state, {}, m_remote_group_peer.uuid, remote_group_snap_id},
+ {}, cls::rbd::GROUP_SNAPSHOT_STATE_INCOMPLETE};
- // XXXMG: make the same name as the primary snapshot has
- m_group_snap = {librbd::util::generate_uuid(m_local_io_ctx),
- cls::rbd::MirrorGroupSnapshotNamespace{
- cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY,
- {}, m_remote_group_peer.uuid, m_remote_group_snap_id},
- calc_ind_mirror_snap_name(
- m_remote_group_peer.io_ctx.get_id(), m_remote_group_id,
- m_remote_group_snap_id),
- cls::rbd::GROUP_SNAPSHOT_STATE_INCOMPLETE};
+ librados::ObjectReadOperation op;
+ librbd::cls_client::group_snap_get_by_id_start(&op, remote_group_snap_id);
+ auto ctx = new C_GetRemoteGroupSnap(this, remote_group_snap_id);
+ auto comp = create_rados_callback(ctx);
+
+ r = m_remote_group_peer.io_ctx.aio_operate(
+ librbd::util::group_header_name(m_remote_group_id), comp, &op,
+ &ctx->bl);
+ ceph_assert(r == 0);
+ comp->release();
}
- ceph_assert(m_create_snap_requests.count(image_replayer) == 0);
- m_create_snap_requests[image_replayer] = on_finish;
+ ceph_assert(requests_it->second.count(image_replayer) == 0);
+ requests_it->second[image_replayer] = on_finish;
+
+ ceph_assert(m_local_group_snaps.count(remote_group_snap_id) > 0);
+ auto &local_group_snap = m_local_group_snaps[remote_group_snap_id];
+
+ local_group_snap.snaps.emplace_back(image_replayer->get_local_pool_id(),
+ image_replayer->get_local_image_id(),
+ CEPH_NOSNAP);
+ ceph_assert(!local_group_snap.snaps.back().image_id.empty());
*local_group_pool_id = m_local_io_ctx.get_id();
*local_group_id = m_local_group_ctx.group_id;
- *local_group_snap_id = m_group_snap.id;
-
- if (m_remote_group_snap_id != remote_group_snap_id) {
- dout(15) << "request snap_id " << remote_group_snap_id
- << " does not match us " << m_remote_group_snap_id
- << " -- restarting" << dendl;
- auto create_snap_requests = m_create_snap_requests;
- m_create_snap_requests.clear();
+ *local_group_snap_id = m_local_group_snaps[remote_group_snap_id].id;
+
+ if (m_get_remote_group_snap_ret_vals.count(remote_group_snap_id) == 0) {
+ dout(20) << "getting remote group snap is still in-progress" << dendl;
locker.unlock();
- for (auto &[_, on_finish] : create_snap_requests) {
- on_finish->complete(-EAGAIN);
- }
return;
}
- // XXXMG: m_image_replayers.size() will not always work
- if (m_create_snap_requests.size() < m_image_replayers.size()) {
- return;
- }
+ maybe_create_mirror_snapshot(locker, remote_group_snap_id);
+}
+
+template <typename I>
+void GroupReplayer<I>::maybe_create_mirror_snapshot(
+ std::unique_lock<ceph::mutex>& locker,
+ const std::string &remote_group_snap_id) {
+ ceph_assert(ceph_mutex_is_locked_by_me(m_lock));
+
+ dout(20) << remote_group_snap_id << dendl;
- m_pending_snap_create = true;
+ auto &remote_group_snap = m_remote_group_snaps[remote_group_snap_id];
+ ceph_assert(!remote_group_snap.id.empty());
- for (auto &[io_ctx, image_replayer] : m_image_replayers) {
- m_group_snap.snaps.emplace_back(io_ctx.get_id(),
- image_replayer->get_local_image_id(),
- CEPH_NOSNAP);
- ceph_assert(!m_group_snap.snaps.back().image_id.empty());
+ ceph_assert(m_create_snap_requests.count(remote_group_snap_id) > 0);
+ auto &create_snap_requests = m_create_snap_requests[remote_group_snap_id];
+
+ for (auto &s : remote_group_snap.snaps) {
+ auto it = m_image_replayer_index.find({s.pool, s.image_id});
+ if (it == m_image_replayer_index.end()) {
+ continue;
+ }
+
+ if (create_snap_requests.count(it->second) == 0) {
+ dout(20) << "waiting for create_mirror_snapshot_start "
+ << remote_group_snap_id << " from " << it->second << dendl;
+ locker.unlock();
+ return;
+ }
}
- dout(20) << m_group_snap.id << " " << m_group_snap.name << dendl;
+ ceph_assert(m_local_group_snaps.count(remote_group_snap_id) > 0);
+ auto &local_group_snap = m_local_group_snaps[remote_group_snap_id];
+
+ dout(20) << local_group_snap.id << " " << local_group_snap.name << dendl;
+
+ bool inserted = m_pending_snap_create.insert(remote_group_snap_id).second;
+ ceph_assert(inserted);
+
+ ceph_assert(m_get_remote_group_snap_ret_vals.count(remote_group_snap_id) > 0);
+ int r = m_get_remote_group_snap_ret_vals[remote_group_snap_id];
+ m_get_remote_group_snap_ret_vals.erase(remote_group_snap_id);
+ if (r < 0) {
+ locker.unlock();
+ handle_create_mirror_snapshot_start(remote_group_snap_id, r);
+ return;
+ }
librados::ObjectWriteOperation op;
- librbd::cls_client::group_snap_set(&op, m_group_snap);
- auto comp = create_rados_callback<
- GroupReplayer<I>,
- &GroupReplayer<I>::handle_create_mirror_snapshot_start>(this);
+ librbd::cls_client::group_snap_set(&op, local_group_snap);
+ auto comp = create_rados_callback(
+ new LambdaContext([this, remote_group_snap_id](int r) {
+ handle_create_mirror_snapshot_start(remote_group_snap_id, r);
+ }));
- int r = m_local_io_ctx.aio_operate(
+ r = m_local_io_ctx.aio_operate(
librbd::util::group_header_name(m_local_group_ctx.group_id), comp, &op);
+ locker.unlock();
ceph_assert(r == 0);
comp->release();
}
template <typename I>
-void GroupReplayer<I>::handle_create_mirror_snapshot_start(int r) {
- dout(20) << m_remote_group_snap_id << " r=" << r << dendl;
+void GroupReplayer<I>::handle_get_remote_group_snapshot(
+ const std::string &remote_group_snap_id, bufferlist &out_bl, int r) {
+ dout(20) << remote_group_snap_id << " r=" << r << dendl;
+
+ auto &remote_group_snap = m_remote_group_snaps[remote_group_snap_id];
std::unique_lock locker{m_lock};
- ceph_assert(m_pending_snap_create == true);
+ if (r == 0) {
+ auto iter = out_bl.cbegin();
+ r = librbd::cls_client::group_snap_get_by_id_finish(
+ &iter, &remote_group_snap);
+ }
+
+ if (r < 0) {
+ derr << "failed to get remote group snapshot: " << cpp_strerror(r) << dendl;
+ } else {
+ m_local_group_snaps[remote_group_snap_id].name = remote_group_snap.name;
+ }
- auto create_snap_requests = m_create_snap_requests;
- m_create_snap_requests.clear();
+ if (m_state == STATE_STOPPING) {
+ dout(20) << "interrupted" << dendl;
+ m_local_group_snaps.erase(remote_group_snap_id);
+ m_remote_group_snaps.erase(remote_group_snap_id);
+ auto create_snap_requests = m_create_snap_requests[remote_group_snap_id];
+ m_create_snap_requests.erase(remote_group_snap_id);
+ bool shut_down_replay = m_pending_snap_create.empty() &&
+ m_create_snap_requests.empty();
+ locker.unlock();
+ for (auto &[_, on_finish] : create_snap_requests) {
+ on_finish->complete(r);
+ }
+ if (shut_down_replay) {
+ stop_image_replayers();
+ }
+ return;
+ }
+
+ m_get_remote_group_snap_ret_vals[remote_group_snap_id] = r;
+
+ maybe_create_mirror_snapshot(locker, remote_group_snap_id);
+}
+
+template <typename I>
+void GroupReplayer<I>::handle_create_mirror_snapshot_start(
+ const std::string &remote_group_snap_id, int r) {
+ dout(20) << remote_group_snap_id << " r=" << r << dendl;
+
+ std::unique_lock locker{m_lock};
+ ceph_assert(m_pending_snap_create.count(remote_group_snap_id) > 0);
+
+ ceph_assert(m_create_snap_requests.count(remote_group_snap_id) > 0);
+ auto create_snap_requests = m_create_snap_requests[remote_group_snap_id];
+ m_create_snap_requests.erase(remote_group_snap_id);
+
+ bool shut_down_replay = false;
+ if (r == -EEXIST) {
+ dout(20) << "group snapshot: " << remote_group_snap_id << " already exists"
+ << dendl;
+ r = 0;
+ } else if (r < 0) {
+ m_pending_snap_create.erase(remote_group_snap_id);
+ m_remote_group_snaps.erase(remote_group_snap_id);
+ m_local_group_snaps.erase(remote_group_snap_id);
+ shut_down_replay = m_state == STATE_STOPPING && !m_restart_requested &&
+ m_pending_snap_create.empty() &&
+ m_create_snap_requests.empty();
+ }
locker.unlock();
for (auto &[_, on_finish] : create_snap_requests) {
on_finish->complete(r);
}
+ if (shut_down_replay) {
+ stop_image_replayers();
+ }
}
template <typename I>
<< snap_id << dendl;
std::lock_guard locker{m_lock};
- ceph_assert(m_pending_snap_create == true);
- ceph_assert(m_state == STATE_REPLAYING || m_state == STATE_STOPPING);
- ceph_assert(m_remote_group_snap_id == remote_group_snap_id);
- ceph_assert(m_create_snap_requests.count(image_replayer) == 0);
+ if (m_local_group_snaps.find(remote_group_snap_id) != m_local_group_snaps.end() &&
+ m_local_group_snaps[remote_group_snap_id].state == cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE) {
+ on_finish->complete(-EEXIST);
+ return;
+ }
- m_create_snap_requests[image_replayer] = on_finish;
+ ceph_assert(m_pending_snap_create.count(remote_group_snap_id) > 0);
+
+ auto &create_snap_requests = m_create_snap_requests[remote_group_snap_id];
+
+ ceph_assert(create_snap_requests.count(image_replayer) == 0);
+ create_snap_requests[image_replayer] = on_finish;
auto pool = image_replayer->get_local_pool_id();
auto image_id = image_replayer->get_local_image_id();
+ auto &local_group_snap = m_local_group_snaps[remote_group_snap_id];
auto it = std::find_if(
- m_group_snap.snaps.begin(), m_group_snap.snaps.end(),
+ local_group_snap.snaps.begin(), local_group_snap.snaps.end(),
[&pool, &image_id](const cls::rbd::ImageSnapshotSpec &s) {
return pool == s.pool && image_id == s.image_id;
});
- ceph_assert(it != m_group_snap.snaps.end());
+ ceph_assert(it != local_group_snap.snaps.end());
it->snap_id = snap_id;
- if (m_create_snap_requests.size() < m_image_replayers.size()) {
+ if (create_snap_requests.size() < local_group_snap.snaps.size()) {
return;
}
- m_group_snap.state = cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE;
+ local_group_snap.state = cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE;
- dout(20) << m_group_snap.id << " " << m_group_snap.name << dendl;
+ dout(20) << local_group_snap.id << " " << local_group_snap.name << dendl;
librados::ObjectWriteOperation op;
- librbd::cls_client::group_snap_set(&op, m_group_snap);
- auto comp = create_rados_callback<
- GroupReplayer<I>,
- &GroupReplayer<I>::handle_create_mirror_snapshot_finish>(this);
+ librbd::cls_client::group_snap_set(&op, local_group_snap);
+ auto comp = create_rados_callback(
+ new LambdaContext([this, remote_group_snap_id](int r) {
+ handle_create_mirror_snapshot_finish(remote_group_snap_id, r);
+ }));
int r = m_local_io_ctx.aio_operate(
librbd::util::group_header_name(m_local_group_ctx.group_id), comp, &op);
}
template <typename I>
-void GroupReplayer<I>::handle_create_mirror_snapshot_finish(int r) {
- dout(20) << m_remote_group_snap_id << " r=" << r << dendl;
+void GroupReplayer<I>::handle_create_mirror_snapshot_finish(
+ const std::string &remote_group_snap_id, int r) {
+ dout(20) << remote_group_snap_id << " r=" << r << dendl;
std::unique_lock locker{m_lock};
- ceph_assert(m_pending_snap_create == true);
- m_pending_snap_create = false;
- m_remote_group_snap_id.clear();
- bool stopping = (m_state == STATE_STOPPING);
- auto create_snap_requests = m_create_snap_requests;
- m_create_snap_requests.clear();
+ auto count = m_pending_snap_create.erase(remote_group_snap_id);
+ ceph_assert(count > 0);
+
+ auto create_snap_requests = m_create_snap_requests[remote_group_snap_id];
+ m_create_snap_requests.erase(remote_group_snap_id);
+ bool shut_down_replay = m_state == STATE_STOPPING && !m_restart_requested &&
+ m_pending_snap_create.empty() &&
+ m_create_snap_requests.empty();
locker.unlock();
for (auto &[_, on_finish] : create_snap_requests) {
on_finish->complete(r);
}
- if (stopping) {
+ if (shut_down_replay) {
stop_image_replayers();
}
}
STATE_STOPPED,
};
+ struct Listener : public GroupCtx::Listener {
+ GroupReplayer *group_replayer;
+
+ Listener(GroupReplayer *group_replayer) : group_replayer(group_replayer) {
+ }
+
+ void create_mirror_snapshot_start(
+ const cls::rbd::MirrorSnapshotNamespace &remote_group_snap_ns,
+ void *arg, int64_t *local_group_pool_id, std::string *local_group_id,
+ std::string *local_group_snap_id, Context *on_finish) override {
+ group_replayer->create_mirror_snapshot_start(
+ remote_group_snap_ns, static_cast<ImageReplayer<ImageCtxT> *>(arg),
+ local_group_pool_id, local_group_id, local_group_snap_id, on_finish);
+ }
+
+ void create_mirror_snapshot_finish(const std::string &remote_group_snap_id,
+ void *arg, uint64_t snap_id,
+ Context *on_finish) override {
+ group_replayer->create_mirror_snapshot_finish(
+ remote_group_snap_id, static_cast<ImageReplayer<ImageCtxT> *>(arg),
+ snap_id, on_finish);
+ }
+ };
+
+ struct C_GetRemoteGroupSnap : public Context {
+ GroupReplayer *group_replayer;
+ std::string group_snap_id;
+ bufferlist bl;
+
+ C_GetRemoteGroupSnap(GroupReplayer *group_replayer,
+ const std::string &group_snap_id)
+ : group_replayer(group_replayer), group_snap_id(group_snap_id) {
+ }
+
+ void finish(int r) override {
+ group_replayer->handle_get_remote_group_snapshot(group_snap_id, bl, r);
+ }
+ };
+
librados::IoCtx &m_local_io_ctx;
std::string m_local_mirror_uuid;
std::string m_global_group_id;
GroupCtx m_local_group_ctx;
Peers m_peers;
Peer<ImageCtxT> m_remote_group_peer;
+ std::string m_remote_group_id;
mutable ceph::mutex m_lock;
State m_state = STATE_STOPPED;
group_replayer::BootstrapRequest<ImageCtxT> *m_bootstrap_request = nullptr;
std::list<std::pair<librados::IoCtx, ImageReplayer<ImageCtxT> *>> m_image_replayers;
+ Listener m_listener = {this};
+ std::map<std::pair<int64_t, std::string>, ImageReplayer<ImageCtxT> *> m_image_replayer_index;
+ std::map<std::string, cls::rbd::GroupSnapshot> m_local_group_snaps;
+ std::map<std::string, cls::rbd::GroupSnapshot> m_remote_group_snaps;
+ std::map<std::string, int> m_get_remote_group_snap_ret_vals;
+ std::map<std::string, std::map<ImageReplayer<ImageCtxT> *, Context *>> m_create_snap_requests;
+ std::set<std::string> m_pending_snap_create;
+
static std::string state_to_string(const State &state) {
switch (state) {
case STATE_STARTING:
void set_mirror_group_status_update(cls::rbd::MirrorGroupStatusState state,
const std::string &desc);
+
+ void create_mirror_snapshot_start(
+ const cls::rbd::MirrorSnapshotNamespace &remote_group_snap_ns,
+ ImageReplayer<ImageCtxT> *image_replayer, int64_t *local_group_pool_id,
+ std::string *local_group_id, std::string *local_group_snap_id,
+ Context *on_finish);
+ void handle_create_mirror_snapshot_start(
+ const std::string &remote_group_snap_id, int r);
+ void handle_get_remote_group_snapshot(
+ const std::string &remote_group_snap_id, bufferlist &out_bl, int r);
+ void maybe_create_mirror_snapshot(
+ std::unique_lock<ceph::mutex>& locker,
+ const std::string &remote_group_snap_id);
+
+ void create_mirror_snapshot_finish(
+ const std::string &remote_group_snap_id,
+ ImageReplayer<ImageCtxT> *image_replayer,
+ uint64_t snap_id, Context *on_finish);
+ void handle_create_mirror_snapshot_finish(
+ const std::string &remote_group_snap_id, int r);
};
} // namespace mirror
#include "Threads.h"
#include "tools/rbd_mirror/image_replayer/BootstrapRequest.h"
#include "tools/rbd_mirror/image_replayer/ReplayerListener.h"
-#include "tools/rbd_mirror/image_replayer/StateBuilder.h"
#include "tools/rbd_mirror/image_replayer/Utils.h"
#include "tools/rbd_mirror/image_replayer/journal/Replayer.h"
#include "tools/rbd_mirror/image_replayer/journal/StateBuilder.h"
struct ImageReplayer<I>::ReplayerListener
: public image_replayer::ReplayerListener {
ImageReplayer<I>* image_replayer;
+ GroupCtx *local_group_ctx;
- ReplayerListener(ImageReplayer<I>* image_replayer)
- : image_replayer(image_replayer) {
+ ReplayerListener(ImageReplayer<I>* image_replayer, GroupCtx *local_group_ctx)
+ : image_replayer(image_replayer), local_group_ctx(local_group_ctx) {
}
void handle_notification() override {
image_replayer->handle_replayer_notification();
}
+
+ void create_mirror_snapshot_start(
+ const cls::rbd::MirrorSnapshotNamespace &remote_group_snap_ns,
+ int64_t *local_group_pool_id, std::string *local_group_id,
+ std::string *local_group_snap_id, Context *on_finish) override {
+ if (local_group_ctx == nullptr) {
+ on_finish->complete(0);
+ return;
+ }
+
+ local_group_ctx->listener->create_mirror_snapshot_start(
+ remote_group_snap_ns, image_replayer, local_group_pool_id,
+ local_group_id, local_group_snap_id, on_finish);
+ }
+
+ void create_mirror_snapshot_finish(const std::string &remote_group_snap_id,
+ uint64_t snap_id,
+ Context *on_finish) override {
+ if (local_group_ctx == nullptr) {
+ on_finish->complete(0);
+ return;
+ }
+
+ local_group_ctx->listener->create_mirror_snapshot_finish(
+ remote_group_snap_id, image_replayer, snap_id, on_finish);
+ }
};
template <typename I>
m_lock(ceph::make_mutex("rbd::mirror::ImageReplayer " +
stringify(local_io_ctx.get_id()) + " " + global_image_id)),
m_progress_cxt(this),
- m_replayer_listener(new ReplayerListener(this))
-{
+ m_replayer_listener(new ReplayerListener(this, local_group_ctx)) {
// Register asok commands using a temporary "remote_pool_name/global_image_id"
// name. When the image name becomes known on start the asok commands will be
// re-registered using "remote_pool_name/remote_image_name" name.
#include "cls/rbd/cls_rbd_types.h"
#include "ProgressContext.h"
#include "tools/rbd_mirror/Types.h"
+#include "tools/rbd_mirror/image_replayer/StateBuilder.h"
#include "tools/rbd_mirror/image_replayer/Types.h"
#include <boost/optional.hpp>
#include <string>
inline const std::string& get_global_image_id() const {
return m_global_image_id;
}
+ inline const std::string& get_local_image_id() const {
+ return m_state_builder->local_image_id;
+ }
void start(Context *on_finish, bool manual = false, bool restart = false);
void stop(Context *on_finish, bool manual = false, bool restart = false);
void MirrorStatusWatcher<I>::handle_notify(uint64_t notify_id, uint64_t handle,
uint64_t notifier_id,
bufferlist &bl) {
- dout(20) << dendl;
+ dout(10) << "notify_id=" << notify_id << ", handle=" << handle
+ << ", notifier_id=" << notifier_id << dendl;
bufferlist out;
acknowledge_notify(notify_id, handle, out);
#include "include/rados/librados.hpp"
#include "include/rbd/librbd.hpp"
+class Context;
+
+namespace cls { namespace rbd { struct MirrorSnapshotNamespace; } }
+
namespace rbd {
namespace mirror {
std::ostream& operator<<(std::ostream& os, const PeerSpec &peer);
struct GroupCtx {
+ struct Listener {
+ virtual ~Listener() {
+ }
+
+ virtual void create_mirror_snapshot_start(
+ const cls::rbd::MirrorSnapshotNamespace &remote_group_snap_ns,
+ void *arg, int64_t *local_group_pool_id, std::string *local_group_id,
+ std::string *local_group_snap_id, Context *on_finish) = 0;
+ virtual void create_mirror_snapshot_finish(
+ const std::string &remote_group_snap_id, void *arg, uint64_t snap_id,
+ Context *on_finish) = 0;
+ };
+
std::string name;
std::string group_id;
std::string global_group_id;
bool primary = false;
mutable librados::IoCtx io_ctx;
+ Listener *listener = nullptr;
GroupCtx() {
}
#define dout_subsys ceph_subsys_rbd_mirror
#undef dout_prefix
#define dout_prefix *_dout << "rbd::mirror::group_replayer::" \
- << "BootstrapRequest: " << this << " " \
+ << "BootstrapRequest: " << " " \
<< __func__ << ": "
namespace rbd {
static const uint32_t MAX_RETURN = 1024;
+bool is_demoted_snap_exists(
+ const std::vector<cls::rbd::GroupSnapshot> &snaps) {
+ for (auto it = snaps.rbegin(); it != snaps.rend(); it++) {
+ auto ns = std::get_if<cls::rbd::MirrorGroupSnapshotNamespace>(
+ &it->snapshot_namespace);
+ if (ns != nullptr) {
+ if (ns->is_demoted()) {
+ return true;
+ }
+ }
+ }
+ return false;
+}
+
int get_last_mirror_snapshot_state(
const std::vector<cls::rbd::GroupSnapshot> &snaps,
cls::rbd::MirrorSnapshotState *state) {
MirrorStatusUpdater<I> *remote_status_updater,
journal::CacheManagerHandler *cache_manager_handler,
PoolMetaCache *pool_meta_cache,
+ std::string *remote_group_id,
GroupCtx *local_group_ctx,
std::list<std::pair<librados::IoCtx, ImageReplayer<I> *>> *image_replayers,
+ std::map<std::pair<int64_t, std::string>, ImageReplayer<I> *> *image_replayer_index,
Context* on_finish)
: CancelableRequest("rbd::mirror::group_replayer::BootstrapRequest",
reinterpret_cast<CephContext*>(local_io_ctx.cct()),
m_remote_status_updater(remote_status_updater),
m_cache_manager_handler(cache_manager_handler),
m_pool_meta_cache(pool_meta_cache),
+ m_remote_group_id(remote_group_id),
m_local_group_ctx(local_group_ctx),
m_image_replayers(image_replayers),
+ m_image_replayer_index(image_replayer_index),
m_on_finish(on_finish) {
dout(10) << "global_group_id=" << m_global_group_id << dendl;
}
if (r == 0) {
auto iter = m_out_bl.cbegin();
r = librbd::cls_client::mirror_group_get_group_id_finish(
- &iter, &m_remote_group_id);
+ &iter, m_remote_group_id);
}
if (r < 0) {
dout(10) << dendl;
librados::ObjectReadOperation op;
- librbd::cls_client::dir_get_name_start(&op, m_remote_group_id);
+ librbd::cls_client::dir_get_name_start(&op, *m_remote_group_id);
m_out_bl.clear();
auto comp = create_rados_callback<
BootstrapRequest<I>,
dout(10) << dendl;
librados::ObjectReadOperation op;
- librbd::cls_client::mirror_group_get_start(&op, m_remote_group_id);
+ librbd::cls_client::mirror_group_get_start(&op, *m_remote_group_id);
m_out_bl.clear();
auto comp = create_rados_callback<
BootstrapRequest<I>,
&BootstrapRequest<I>::handle_list_remote_group>(this);
m_out_bl.clear();
int r = m_remote_io_ctx.aio_operate(
- librbd::util::group_header_name(m_remote_group_id), comp, &op, &m_out_bl);
+ librbd::util::group_header_name(*m_remote_group_id), comp, &op,
+ &m_out_bl);
ceph_assert(r == 0);
comp->release();
}
return;
}
- m_remote_images.insert({spec.pool_id, mirror_image.global_image_id});
+ m_remote_images[{spec.pool_id, mirror_image.global_image_id}] = spec.image_id;
m_images.pop_front();
r = get_last_mirror_snapshot_state(m_local_group_snaps, &state);
if (r == -ENOENT) {
derr << "failed to find local mirror group snapshot" << dendl;
- finish(-EINVAL);
- return;
+ } else {
+ if (state == cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED) {
+ // if local snapshot is primary demoted, check if there is demote snapshot
+ // in remote, if not then split brain
+ if (!is_demoted_snap_exists(m_remote_group_snaps)) {
+ finish(-EEXIST);
+ return;
+ }
+ }
}
- ceph_assert(r == 0);
m_local_mirror_group_primary = (state == cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY);
}
if (m_local_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED &&
m_local_mirror_group_primary) {
derr << "both remote and local groups are primary" << dendl;
- finish(-EEXIST); // split-brain
- return;
}
} else if (m_local_mirror_group.state != cls::rbd::MIRROR_GROUP_STATE_ENABLED ||
!m_local_mirror_group_primary) {
}
m_local_mirror_group.state = cls::rbd::MIRROR_GROUP_STATE_DISABLED;
-
- if (m_remote_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED &&
- m_remote_mirror_group_primary) {
+ if (r != -ENOENT && (m_remote_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED &&
+ m_remote_mirror_group_primary)) {
create_local_mirror_group();
} else {
remove_local_group();
int r = 0;
if (m_remote_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED &&
m_remote_mirror_group_primary) {
- for (auto &[remote_pool_id, global_image_id] : m_remote_images) {
+ for (auto &[p, remote_image_id] : m_remote_images) {
+ auto &remote_pool_id = p.first;
+ auto &global_image_id = p.second;
+
m_image_replayers->emplace_back(librados::IoCtx(), nullptr);
auto &local_io_ctx = m_image_replayers->back().first;
auto &image_replayer = m_image_replayers->back().second;
// TODO only a single peer is currently supported
image_replayer->add_peer({local_pool_meta.mirror_uuid, remote_io_ctx,
remote_pool_meta, m_remote_status_updater});
+
+ (*m_image_replayer_index)[{remote_pool_id, remote_image_id}] = image_replayer;
}
} else if (m_local_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED &&
m_local_mirror_group_primary) {
MirrorStatusUpdater<ImageCtxT> *remote_status_updater,
journal::CacheManagerHandler *cache_manager_handler,
PoolMetaCache *pool_meta_cache,
+ std::string *remote_group_id,
GroupCtx *local_group_ctx,
std::list<std::pair<librados::IoCtx, ImageReplayer<ImageCtxT> *>> *image_replayers,
+ std::map<std::pair<int64_t, std::string>, ImageReplayer<ImageCtxT> *> *image_replayer_index,
Context *on_finish) {
return new BootstrapRequest(
threads, local_io_ctx, remote_io_ctx, global_group_id, local_mirror_uuid,
instance_watcher, local_status_updater, remote_status_updater,
- cache_manager_handler, pool_meta_cache, local_group_ctx, image_replayers,
- on_finish);
+ cache_manager_handler, pool_meta_cache, remote_group_id, local_group_ctx,
+ image_replayers, image_replayer_index, on_finish);
}
BootstrapRequest(
MirrorStatusUpdater<ImageCtxT> *remote_status_updater,
journal::CacheManagerHandler *cache_manager_handler,
PoolMetaCache *pool_meta_cache,
+ std::string *remote_group_id,
GroupCtx *local_group_ctx,
std::list<std::pair<librados::IoCtx, ImageReplayer<ImageCtxT> *>> *image_replayers,
+ std::map<std::pair<int64_t, std::string>, ImageReplayer<ImageCtxT> *> *image_replayer_index,
Context* on_finish);
void send() override;
MirrorStatusUpdater<ImageCtxT> *m_remote_status_updater;
journal::CacheManagerHandler *m_cache_manager_handler;
PoolMetaCache *m_pool_meta_cache;
+ std::string *m_remote_group_id;
GroupCtx *m_local_group_ctx;
std::list<std::pair<librados::IoCtx, ImageReplayer<ImageCtxT> *>> *m_image_replayers;
+ std::map<std::pair<int64_t, std::string>, ImageReplayer<ImageCtxT> *> *m_image_replayer_index;
Context *m_on_finish;
std::atomic<bool> m_canceled = false;
std::string m_group_name;
- std::string m_remote_group_id;
std::string m_local_group_id;
bool m_local_group_id_by_name = false;
cls::rbd::MirrorGroup m_remote_mirror_group;
std::list<cls::rbd::GroupImageStatus> m_images;
librados::IoCtx m_image_io_ctx;
- std::set<GlobalImageId> m_remote_images;
+ std::map<GlobalImageId, std::string> m_remote_images;
std::set<GlobalImageId> m_local_images;
std::map<GlobalImageId, std::string> m_local_trash_images;
m_do_resync(do_resync),
m_lock(ceph::make_mutex(unique_lock_name("BootstrapRequest::m_lock",
this))) {
- dout(10) << dendl;
+ dout(10) << "global_image_id: " << m_global_image_id << dendl;
}
template <typename I>
template <typename I>
void PrepareLocalImageRequest<I>::send() {
- dout(10) << dendl;
+ dout(10) << "global_image_id: " << m_global_image_id << dendl;
get_local_image_id();
}
template <typename I>
void PrepareLocalImageRequest<I>::handle_get_local_image_id(int r) {
- dout(10) << "r=" << r << ", "
- << "local_image_id=" << m_local_image_id << dendl;
+ dout(10) << "r=" << r << ", global_image_id: " << m_global_image_id
+ << ", local_image_id=" << m_local_image_id << dendl;
if (r < 0) {
finish(r);
template <typename I>
void PrepareLocalImageRequest<I>::handle_get_mirror_info(int r) {
- dout(10) << ": r=" << r << dendl;
+ dout(10) << "r=" << r << dendl;
if (r < 0) {
derr << "failed to retrieve local mirror image info: " << cpp_strerror(r)
return;
}
- if (m_mirror_image.state == cls::rbd::MIRROR_IMAGE_STATE_CREATING) {
+ if (m_mirror_image.state == cls::rbd::MIRROR_IMAGE_STATE_CREATING &&
+ !m_mirror_image.group_spec.is_valid()) {
dout(5) << "local image is still in creating state, issuing a removal"
<< dendl;
move_to_trash();
virtual ~ReplayerListener() {}
virtual void handle_notification() = 0;
+
+ virtual void create_mirror_snapshot_start(
+ const cls::rbd::MirrorSnapshotNamespace &remote_group_snap_ns,
+ int64_t *local_group_pool_id, std::string *local_group_id,
+ std::string *local_group_snap_id, Context *on_finish) = 0;
+ virtual void create_mirror_snapshot_finish(
+ const std::string &remote_group_snap_id, uint64_t snap_id,
+ Context *on_finish) = 0;
};
} // namespace image_replayer
return;
}
- create_non_primary_snapshot();
+ create_group_snap_start();
}
template <typename I>
request_sync();
}
+template <typename I>
+void Replayer<I>::create_group_snap_start() {
+
+ if (!m_remote_mirror_snap_ns.group_spec.is_valid() ||
+ m_remote_mirror_snap_ns.group_snap_id.empty()) {
+ create_non_primary_snapshot();
+ return;
+ }
+
+ dout(10) << dendl;
+
+ auto ctx = create_context_callback<
+ Replayer<I>, &Replayer<I>::handle_create_group_snap_start>(this);
+
+ m_replayer_listener->create_mirror_snapshot_start(
+ m_remote_mirror_snap_ns, &m_local_group_pool_id,
+ &m_local_group_id, &m_local_group_snap_id, ctx);
+}
+
+template <typename I>
+void Replayer<I>::handle_create_group_snap_start(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ if (r < 0 && r != -EEXIST) {
+ if (r == -EAGAIN) {
+ dout(15) << "restarting replayer" << dendl;
+ load_local_image_meta();
+ } else if (r == -ESTALE) {
+ dout(15) << "waiting for shut down" << dendl;
+ handle_replay_complete(r, "waiting for shut down");
+ } else {
+ derr << "failed to create group snapshot: " << cpp_strerror(r) << dendl;
+ handle_replay_complete(r, "failed to create group snapshot");
+ }
+ return;
+ }
+
+ create_non_primary_snapshot();
+}
+
template <typename I>
void Replayer<I>::create_non_primary_snapshot() {
auto local_image_ctx = m_state_builder->local_image_ctx;
auto req = librbd::mirror::snapshot::CreateNonPrimaryRequest<I>::create(
local_image_ctx, m_remote_mirror_snap_ns.is_demoted(),
m_state_builder->remote_mirror_uuid, m_remote_snap_id_end,
- m_local_mirror_snap_ns.snap_seqs, -1, {}, {}, m_image_state,
- &m_local_snap_id_end, ctx);
+ m_local_mirror_snap_ns.snap_seqs, m_local_group_pool_id, m_local_group_id,
+ m_local_group_snap_id, m_image_state, &m_local_snap_id_end, ctx);
req->send();
}
dout(15) << "local_snap_id_end=" << m_local_snap_id_end << dendl;
+ create_group_snap_finish();
+}
+
+template <typename I>
+void Replayer<I>::create_group_snap_finish() {
+ if (!m_remote_mirror_snap_ns.group_spec.is_valid() ||
+ m_remote_mirror_snap_ns.group_snap_id.empty()) {
+ update_mirror_image_state();
+ return;
+ }
+
+ dout(10) << dendl;
+
+ auto ctx = create_context_callback<
+ Replayer<I>, &Replayer<I>::handle_create_group_snap_finish>(this);
+
+ m_replayer_listener->create_mirror_snapshot_finish(
+ m_remote_mirror_snap_ns.group_snap_id, m_local_snap_id_end, ctx);
+}
+
+template <typename I>
+void Replayer<I>::handle_create_group_snap_finish(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ if (r < 0 && r != -EEXIST) {
+ derr << "failed to create group snapshot: " << cpp_strerror(r) << dendl;
+ handle_replay_complete(r, "failed to create group snapshot");
+ return;
+ }
+
update_mirror_image_state();
}
* | v | |
* | GET_REMOTE_IMAGE_STATE | |
* | | | |
+ * | v (skip if no group) | |
+ * | CREATE_GROUP_SNAP_START | |
+ * | | | |
* | v | |
* | CREATE_NON_PRIMARY_SNAPSHOT | |
* | | | |
+ * | v (skip if no group) | |
+ * | CREATE_GROUP_SNAP_FINISH | |
+ * | | | |
* | v (skip if not needed)| |
* | UPDATE_MIRROR_IMAGE_STATE | |
* | | | |
uint64_t m_remote_snap_id_end = CEPH_NOSNAP;
cls::rbd::MirrorSnapshotNamespace m_remote_mirror_snap_ns;
+ int64_t m_local_group_pool_id = -1;
+ std::string m_local_group_id;
+ std::string m_local_group_snap_id;
+
librbd::mirror::snapshot::ImageState m_image_state;
DeepCopyHandler* m_deep_copy_handler = nullptr;
void get_local_image_state();
void handle_get_local_image_state(int r);
+ void create_group_snap_start();
+ void handle_create_group_snap_start(int r);
+
void create_non_primary_snapshot();
void handle_create_non_primary_snapshot(int r);
+ void create_group_snap_finish();
+ void handle_create_group_snap_finish(int r);
+
void update_mirror_image_state();
void handle_update_mirror_image_state(int r);
projects / ceph.git / commitdiff