if (m_snap_seqs.find(src_snap_id) == m_snap_seqs.end()) {
// the source snapshot is not in our mapping table, ...
- if (std::holds_alternative<cls::rbd::UserSnapshotNamespace>(snap_namespace)) {
+ if (std::holds_alternative<cls::rbd::UserSnapshotNamespace>(snap_namespace) ||
+ std::holds_alternative<cls::rbd::GroupImageSnapshotNamespace>(snap_namespace)) {
// ... create it since it's a user snapshot
break;
} else if (src_snap_id == m_src_snap_id_end) {
auto snap_it = m_dst_image_ctx->snap_ids.find(
{cls::rbd::UserSnapshotNamespace(), m_snap_name});
+ if (snap_it == m_dst_image_ctx->snap_ids.end()) {
+ snap_it = m_dst_image_ctx->snap_ids.find(
+ {cls::rbd::GroupImageSnapshotNamespace(), m_snap_name});
+ }
ceph_assert(snap_it != m_dst_image_ctx->snap_ids.end());
librados::snap_t dst_snap_id = snap_it->second;
#include "include/stringify.h"
#include "common/Formatter.h"
#include "common/admin_socket.h"
+#include "common/Cond.h"
#include "common/debug.h"
#include "common/errno.h"
#include "librbd/ImageCtx.h"
#include "librbd/Utils.h"
#include "librbd/asio/ContextWQ.h"
+#include "librbd/group/ListSnapshotsRequest.h"
#include "tools/rbd_mirror/ImageReplayer.h"
#include "tools/rbd_mirror/MirrorStatusUpdater.h"
#include "tools/rbd_mirror/Threads.h"
m_last_r = 0;
m_state_desc.clear();
m_local_group_snaps.clear();
- m_remote_group_snaps.clear();
m_image_replayers.clear();
m_image_replayer_index.clear();
m_get_remote_group_snap_ret_vals.clear();
}
} else {
dout(10) << "replayers still running" << dendl;
- if (!is_stopped_()) {
+ if (!is_stopped_() || m_state == STATE_STOPPING) {
if (m_state == STATE_STARTING) {
dout(10) << "canceling start" << dendl;
if (m_bootstrap_request != nullptr) {
m_local_mirror_uuid, m_instance_watcher, m_local_status_updater,
m_remote_group_peer.mirror_status_updater, m_cache_manager_handler,
m_pool_meta_cache, &m_local_group_id, &m_remote_group_id,
- &m_local_group_ctx, &m_image_replayers, &m_image_replayer_index, ctx);
+ &m_local_group_snaps, &m_local_group_ctx, &m_image_replayers,
+ &m_image_replayer_index, ctx);
request->get();
m_bootstrap_request = request;
}
}
+template <typename I>
+void GroupReplayer<I>::create_regular_group_snapshot(
+ const std::string &remote_group_snap_name,
+ const std::string &remote_group_snap_id,
+ std::vector<cls::rbd::GroupImageStatus> *local_images,
+ Context *on_finish) {
+ // each image will have one snapshot specific to group snap, and so for each
+ // image get a ImageSnapshotSpec and prepare a vector
+ // for image :: <images in that group> {
+ // * get snap whos name has group snap_id for that we can list snaps and
+ // filter with remote_group_snap_id
+ // * get its { pool_id, snap_id, image_id }
+ // }
+ // finally write to the object
+ dout(10) << dendl;
+ librados::ObjectWriteOperation op;
+ cls::rbd::GroupSnapshot group_snap{
+ remote_group_snap_id, // keeping it same as remote group snap id
+ cls::rbd::UserGroupSnapshotNamespace{},
+ remote_group_snap_name,
+ cls::rbd::GROUP_SNAPSHOT_STATE_INCOMPLETE};
+ librbd::cls_client::group_snap_set(&op, group_snap);
+ if (m_local_group_snaps.find(group_snap.id) == m_local_group_snaps.end()) {
+ m_local_group_snaps.insert(make_pair(group_snap.id, group_snap));
+ }
+
+ std::vector<cls::rbd::ImageSnapshotSpec> local_image_snap_specs;
+ local_image_snap_specs = std::vector<cls::rbd::ImageSnapshotSpec>(
+ local_images->size(), cls::rbd::ImageSnapshotSpec());
+ for (auto& image : *local_images) {
+ std::string image_header_oid = librbd::util::header_name(
+ image.spec.image_id);
+ ::SnapContext snapc;
+ int r = librbd::cls_client::get_snapcontext(&m_local_io_ctx,
+ image_header_oid, &snapc);
+ if (r < 0) {
+ derr << "get snap context failed: " << cpp_strerror(r) << dendl;
+ on_finish->complete(r);
+ return;
+ }
+
+ auto image_snap_name = ".group." + std::to_string(image.spec.pool_id) +
+ "_" + m_remote_group_id + "_" + remote_group_snap_id;
+ // stored in reverse order
+ for (auto snap_id : snapc.snaps) {
+ cls::rbd::SnapshotInfo snap_info;
+ r = librbd::cls_client::snapshot_get(&m_local_io_ctx, image_header_oid,
+ snap_id, &snap_info);
+ if (r < 0) {
+ derr << "failed getting snap info for snap id: " << snap_id
+ << ", : " << cpp_strerror(r) << dendl;
+ on_finish->complete(r);
+ return;
+ }
+
+ // extract { pool_id, snap_id, image_id }
+ if (snap_info.name == image_snap_name) {
+ cls::rbd::ImageSnapshotSpec snap_spec;
+ snap_spec.pool = image.spec.pool_id;
+ snap_spec.image_id = image.spec.image_id;
+ snap_spec.snap_id = snap_info.id;
+
+ local_image_snap_specs.push_back(snap_spec);
+ }
+ }
+ }
+
+ group_snap.snaps = local_image_snap_specs;
+ group_snap.state = cls::rbd::GROUP_SNAPSHOT_STATE_COMPLETE;
+ librbd::cls_client::group_snap_set(&op, group_snap);
+ m_local_group_snaps[group_snap.id] = group_snap;
+
+ auto comp = create_rados_callback(
+ new LambdaContext([this, on_finish](int r) {
+ handle_create_regular_group_snapshot(r, on_finish);
+ }));
+ int r = m_local_io_ctx.aio_operate(
+ librbd::util::group_header_name(m_local_group_ctx.group_id), comp, &op);
+ ceph_assert(r == 0);
+ comp->release();
+}
+
+template <typename I>
+void GroupReplayer<I>::handle_create_regular_group_snapshot(
+ int r, Context *on_finish) {
+ dout(10) << "r=" << r << dendl;
+
+ if (r < 0) {
+ derr << "error creating local non-primary group snapshot: "
+ << cpp_strerror(r) << dendl;
+ }
+
+ on_finish->complete(0);
+}
+
+template <typename I>
+void GroupReplayer<I>::list_remote_group_snapshots(Context *on_finish) {
+ dout(10) << dendl;
+
+ remote_group_snaps.clear();
+ auto ctx = new LambdaContext(
+ [this, on_finish] (int r) {
+ handle_list_remote_group_snapshots(r, on_finish);
+ });
+
+ auto req = librbd::group::ListSnapshotsRequest<I>::create(
+ m_remote_group_peer.io_ctx, m_remote_group_id, &remote_group_snaps, ctx);
+ req->send();
+}
+
+template <typename I>
+void GroupReplayer<I>::handle_list_remote_group_snapshots(int r,
+ Context *on_finish) {
+ dout(10) << "r=" << r << dendl;
+ std::unique_lock locker{m_lock};
+
+ if (r < 0) {
+ derr << "error listing remote mirror group snapshots: " << cpp_strerror(r)
+ << dendl;
+ on_finish->complete(r);
+ return;
+ }
+
+ m_remote_group_snaps.clear();
+ for (auto it : remote_group_snaps) {
+ dout(10) << "found remote group snap id: " << it.id << dendl;
+ m_remote_group_snaps.insert(make_pair(it.id, it));
+ }
+
+ std::vector<cls::rbd::GroupImageStatus> local_images;
+ std::vector<C_SaferCond*> on_finishes;
+ for (auto it = m_remote_group_snaps.begin(); it != m_remote_group_snaps.end(); ++it) {
+ auto snap_type = cls::rbd::get_group_snap_namespace_type(
+ it->second.snapshot_namespace);
+ if (snap_type == cls::rbd::GROUP_SNAPSHOT_NAMESPACE_TYPE_USER) {
+ dout(10) << "found user snap, snap name: " << it->second.name
+ << ", remote group snap id: " << it->second.id << dendl;
+ if (local_images.empty()) {
+ r = local_group_image_list_by_id(&local_images);
+ if (r < 0) {
+ locker.unlock();
+ on_finish->complete(r);
+ return;
+ }
+ }
+ if (m_local_group_snaps.find(it->second.id) == m_local_group_snaps.end()) {
+ C_SaferCond* ctx = new C_SaferCond;
+ create_regular_group_snapshot(it->second.name,
+ it->second.id, &local_images, ctx);
+ on_finishes.push_back(ctx);
+ }
+ }
+ }
+
+ for (auto &finish : on_finishes) {
+ finish->wait();
+ }
+
+ locker.unlock();
+ on_finish->complete(0);
+}
+
+template <typename I>
+int GroupReplayer<I>::local_group_image_list_by_id(
+ std::vector<cls::rbd::GroupImageStatus> *image_ids) {
+ std::string group_header_oid = librbd::util::group_header_name(
+ m_local_group_ctx.group_id);
+
+ dout(10) << "listing images in local group id " << group_header_oid << dendl;
+ image_ids->clear();
+
+ int r = 0;
+ const int max_read = 1024;
+ cls::rbd::GroupImageSpec start_last;
+ do {
+ std::vector<cls::rbd::GroupImageStatus> image_ids_page;
+
+ r = librbd::cls_client::group_image_list(&m_local_io_ctx, group_header_oid,
+ start_last, max_read,
+ &image_ids_page);
+
+ if (r < 0) {
+ derr << "error reading image list from local group: "
+ << cpp_strerror(-r) << dendl;
+ return r;
+ }
+ image_ids->insert(image_ids->end(), image_ids_page.begin(),
+ image_ids_page.end());
+
+ if (image_ids_page.size() > 0)
+ start_last = image_ids_page.rbegin()->spec;
+
+ r = image_ids_page.size();
+ } while (r == max_read);
+
+ return 0;
+}
+
template <typename I>
void GroupReplayer<I>::create_mirror_snapshot_start(
const cls::rbd::MirrorSnapshotNamespace &remote_group_snap_ns,
if (m_state == STATE_STOPPING) {
dout(20) << "interrupted" << dendl;
m_local_group_snaps.erase(remote_group_snap_id);
- m_remote_group_snaps.erase(remote_group_snap_id);
auto create_snap_requests = m_create_snap_requests[remote_group_snap_id];
m_create_snap_requests.erase(remote_group_snap_id);
bool shut_down_replay = m_pending_snap_create.empty() &&
r = 0;
} else if (r < 0) {
m_pending_snap_create.erase(remote_group_snap_id);
- m_remote_group_snaps.erase(remote_group_snap_id);
m_local_group_snaps.erase(remote_group_snap_id);
shut_down_replay = m_state == STATE_STOPPING && !m_restart_requested &&
m_pending_snap_create.empty() &&
Listener(GroupReplayer *group_replayer) : group_replayer(group_replayer) {
}
+ void list_remote_group_snapshots(Context *on_finish) override {
+ group_replayer->list_remote_group_snapshots(on_finish);
+ }
+
void create_mirror_snapshot_start(
const cls::rbd::MirrorSnapshotNamespace &remote_group_snap_ns,
void *arg, int64_t *local_group_pool_id, std::string *local_group_id,
std::map<std::pair<int64_t, std::string>, ImageReplayer<ImageCtxT> *> m_image_replayer_index;
std::map<std::string, cls::rbd::GroupSnapshot> m_local_group_snaps;
std::map<std::string, cls::rbd::GroupSnapshot> m_remote_group_snaps;
+ std::vector<cls::rbd::GroupSnapshot> remote_group_snaps;
std::map<std::string, int> m_get_remote_group_snap_ret_vals;
std::map<std::string, std::map<ImageReplayer<ImageCtxT> *, Context *>> m_create_snap_requests;
std::set<std::string> m_pending_snap_create;
void set_mirror_group_status_update(cls::rbd::MirrorGroupStatusState state,
const std::string &desc);
+ void create_regular_group_snapshot(const std::string &remote_snap_name,
+ const std::string &remote_snap_id,
+ std::vector<cls::rbd::GroupImageStatus> *local_images,
+ Context *on_finish);
+ void handle_create_regular_group_snapshot(int r, Context *on_finish);
+ void list_remote_group_snapshots(Context *on_finish);
+ void handle_list_remote_group_snapshots(int r, Context *on_finish);
+ int local_group_image_list_by_id(std::vector<cls::rbd::GroupImageStatus> *image_ids);
void create_mirror_snapshot_start(
const cls::rbd::MirrorSnapshotNamespace &remote_group_snap_ns,
ImageReplayer<ImageCtxT> *image_replayer, int64_t *local_group_pool_id,
image_replayer->handle_replayer_notification();
}
+ void list_remote_group_snapshots(Context *on_finish) override {
+ if (local_group_ctx == nullptr) {
+ on_finish->complete(0);
+ return;
+ }
+
+ local_group_ctx->listener->list_remote_group_snapshots(on_finish);
+ }
+
void create_mirror_snapshot_start(
const cls::rbd::MirrorSnapshotNamespace &remote_group_snap_ns,
int64_t *local_group_pool_id, std::string *local_group_id,
virtual ~Listener() {
}
+ virtual void list_remote_group_snapshots(Context *on_finish) = 0;
+
virtual void create_mirror_snapshot_start(
const cls::rbd::MirrorSnapshotNamespace &remote_group_snap_ns,
void *arg, int64_t *local_group_pool_id, std::string *local_group_id,
PoolMetaCache *pool_meta_cache,
std::string *local_group_id,
std::string *remote_group_id,
+ std::map<std::string, cls::rbd::GroupSnapshot> *local_group_snaps,
GroupCtx *local_group_ctx,
std::list<std::pair<librados::IoCtx, ImageReplayer<I> *>> *image_replayers,
std::map<std::pair<int64_t, std::string>, ImageReplayer<I> *> *image_replayer_index,
m_pool_meta_cache(pool_meta_cache),
m_local_group_id(local_group_id),
m_remote_group_id(remote_group_id),
+ m_local_group_snaps(local_group_snaps),
m_local_group_ctx(local_group_ctx),
m_image_replayers(image_replayers),
m_image_replayer_index(image_replayer_index),
&BootstrapRequest<I>::handle_list_remote_group_snapshots>(this);
auto req = librbd::group::ListSnapshotsRequest<I>::create(m_remote_io_ctx,
- *m_remote_group_id, true, true, &m_remote_group_snaps, ctx);
+ *m_remote_group_id, true, true, &remote_group_snaps, ctx);
req->send();
}
if (m_remote_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED) {
cls::rbd::MirrorSnapshotState state;
- r = get_last_mirror_snapshot_state(m_remote_group_snaps, &state);
+ r = get_last_mirror_snapshot_state(remote_group_snaps, &state);
if (r == -ENOENT) {
derr << "failed to find remote mirror group snapshot" << dendl;
finish(-EINVAL);
&BootstrapRequest<I>::handle_list_local_group_snapshots>(this);
auto req = librbd::group::ListSnapshotsRequest<I>::create(m_local_io_ctx,
- *m_local_group_id, true, true, &m_local_group_snaps, ctx);
+ *m_local_group_id, true, true, &local_group_snaps, ctx);
req->send();
}
return;
}
+ for (auto it : local_group_snaps) {
+ m_local_group_snaps->insert(make_pair(it.id, it));
+ }
+
if (m_local_mirror_group.state == cls::rbd::MIRROR_GROUP_STATE_ENABLED) {
cls::rbd::MirrorSnapshotState state;
- r = get_last_mirror_snapshot_state(m_local_group_snaps, &state);
+ r = get_last_mirror_snapshot_state(local_group_snaps, &state);
if (r == -ENOENT) {
derr << "failed to find local mirror group snapshot" << dendl;
} else {
if (state == cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED) {
// if local snapshot is primary demoted, check if there is demote snapshot
// in remote, if not then split brain
- if (!is_demoted_snap_exists(m_remote_group_snaps)) {
+ if (!is_demoted_snap_exists(remote_group_snaps)) {
finish(-EEXIST);
return;
}
PoolMetaCache *pool_meta_cache,
std::string *local_group_id,
std::string *remote_group_id,
+ std::map<std::string, cls::rbd::GroupSnapshot> *local_group_snaps,
GroupCtx *local_group_ctx,
std::list<std::pair<librados::IoCtx, ImageReplayer<ImageCtxT> *>> *image_replayers,
std::map<std::pair<int64_t, std::string>, ImageReplayer<ImageCtxT> *> *image_replayer_index,
threads, local_io_ctx, remote_io_ctx, global_group_id, local_mirror_uuid,
instance_watcher, local_status_updater, remote_status_updater,
cache_manager_handler, pool_meta_cache, local_group_id, remote_group_id,
- local_group_ctx, image_replayers, image_replayer_index, on_finish);
+ local_group_snaps, local_group_ctx, image_replayers, image_replayer_index,
+ on_finish);
}
BootstrapRequest(
PoolMetaCache *pool_meta_cache,
std::string *local_group_id,
std::string *remote_group_id,
+ std::map<std::string, cls::rbd::GroupSnapshot> *local_group_snaps,
GroupCtx *local_group_ctx,
std::list<std::pair<librados::IoCtx, ImageReplayer<ImageCtxT> *>> *image_replayers,
std::map<std::pair<int64_t, std::string>, ImageReplayer<ImageCtxT> *> *image_replayer_index,
PoolMetaCache *m_pool_meta_cache;
std::string *m_local_group_id;
std::string *m_remote_group_id;
+ std::map<std::string, cls::rbd::GroupSnapshot> *m_local_group_snaps;
GroupCtx *m_local_group_ctx;
std::list<std::pair<librados::IoCtx, ImageReplayer<ImageCtxT> *>> *m_image_replayers;
std::map<std::pair<int64_t, std::string>, ImageReplayer<ImageCtxT> *> *m_image_replayer_index;
bool m_local_group_id_by_name = false;
cls::rbd::MirrorGroup m_remote_mirror_group;
cls::rbd::MirrorGroup m_local_mirror_group;
- std::vector<cls::rbd::GroupSnapshot> m_remote_group_snaps;
- std::vector<cls::rbd::GroupSnapshot> m_local_group_snaps;
+ std::vector<cls::rbd::GroupSnapshot> remote_group_snaps;
+ std::vector<cls::rbd::GroupSnapshot> local_group_snaps;
bool m_remote_mirror_group_primary = false;
bool m_local_mirror_group_primary = false;
std::list<cls::rbd::GroupImageStatus> m_images;
virtual ~ReplayerListener() {}
virtual void handle_notification() = 0;
+ virtual void list_remote_group_snapshots(Context *on_finish) = 0;
virtual void create_mirror_snapshot_start(
const cls::rbd::MirrorSnapshotNamespace &remote_group_snap_ns,
return;
}
- create_group_snap_start();
+ refresh_remote_group_snapshot_list();
}
template <typename I>
}
template <typename I>
-void Replayer<I>::create_group_snap_start() {
-
+void Replayer<I>::refresh_remote_group_snapshot_list() {
if (!m_remote_mirror_snap_ns.group_spec.is_valid() ||
m_remote_mirror_snap_ns.group_snap_id.empty()) {
create_non_primary_snapshot();
dout(10) << dendl;
+ auto ctx = new LambdaContext(
+ [this](int r) {
+ handle_refresh_remote_group_snapshot_list(r);
+ });
+
+ m_replayer_listener->list_remote_group_snapshots(ctx);
+}
+
+template <typename I>
+void Replayer<I>::handle_refresh_remote_group_snapshot_list(int r) {
+ dout(10) << "r=" << r << dendl;
+
+ if (r < 0) {
+ dout(15) << "restarting replayer" << dendl;
+ load_local_image_meta();
+ return;
+ }
+
+ create_group_snap_start();
+}
+
+template <typename I>
+void Replayer<I>::create_group_snap_start() {
+ dout(10) << dendl;
+
auto ctx = create_context_callback<
Replayer<I>, &Replayer<I>::handle_create_group_snap_start>(this);
if (r < 0 && r != -EEXIST) {
if (r == -EAGAIN) {
- dout(15) << "restarting replayer" << dendl;
- load_local_image_meta();
+ auto ctx = new LambdaContext(
+ [this](int r) {
+ // retry after 1 sec
+ refresh_remote_group_snapshot_list();
+ });
+ std::lock_guard timer_locker{m_threads->timer_lock};
+ m_threads->timer->add_event_after(1, ctx);
} else if (r == -ESTALE) {
dout(15) << "waiting for shut down" << dendl;
handle_replay_complete(r, "waiting for shut down");
void get_local_image_state();
void handle_get_local_image_state(int r);
+ void refresh_remote_group_snapshot_list();
+ void handle_refresh_remote_group_snapshot_list(int r);
+
void create_group_snap_start();
void handle_create_group_snap_start(int r);
projects / ceph.git / commitdiff