From 9a943dbea359cd1e2bed5887f98a1644104d34d2 Mon Sep 17 00:00:00 2001 From: Jason Dillaman Date: Mon, 7 Mar 2016 17:33:49 -0500 Subject: [PATCH] rbd-mirror: sync snapshots between remote-local images during bootstrap Signed-off-by: Jason Dillaman --- src/tools/Makefile-client.am | 6 +- .../image_sync/SnapshotCopyRequest.cc | 239 ++++++++++++++++++ .../image_sync/SnapshotCopyRequest.h | 112 ++++++++ 3 files changed, 355 insertions(+), 2 deletions(-) create mode 100644 src/tools/rbd_mirror/image_sync/SnapshotCopyRequest.cc create mode 100644 src/tools/rbd_mirror/image_sync/SnapshotCopyRequest.h diff --git a/src/tools/Makefile-client.am b/src/tools/Makefile-client.am index ed6da4e560f..dbd3cf05199 100644 --- a/src/tools/Makefile-client.am +++ b/src/tools/Makefile-client.am @@ -88,7 +88,8 @@ librbd_mirror_internal_la_SOURCES = \ tools/rbd_mirror/Replayer.cc \ tools/rbd_mirror/Threads.cc \ tools/rbd_mirror/types.cc \ - tools/rbd_mirror/image_sync/ObjectCopyRequest.cc + tools/rbd_mirror/image_sync/ObjectCopyRequest.cc \ + tools/rbd_mirror/image_sync/SnapshotCopyRequest.cc noinst_LTLIBRARIES += librbd_mirror_internal.la noinst_HEADERS += \ tools/rbd_mirror/ClusterWatcher.h \ @@ -98,7 +99,8 @@ noinst_HEADERS += \ tools/rbd_mirror/Replayer.h \ tools/rbd_mirror/Threads.h \ tools/rbd_mirror/types.h \ - tools/rbd_mirror/image_sync/ObjectCopyRequest.h + tools/rbd_mirror/image_sync/ObjectCopyRequest.h \ + tools/rbd_mirror/image_sync/SnapshotCopyRequest.h rbd_mirror_SOURCES = \ tools/rbd_mirror/main.cc diff --git a/src/tools/rbd_mirror/image_sync/SnapshotCopyRequest.cc b/src/tools/rbd_mirror/image_sync/SnapshotCopyRequest.cc new file mode 100644 index 00000000000..b53c0e1c551 --- /dev/null +++ b/src/tools/rbd_mirror/image_sync/SnapshotCopyRequest.cc @@ -0,0 +1,239 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "SnapshotCopyRequest.h" +#include "common/errno.h" +#include "journal/Journaler.h" +#include "librbd/Operations.h" +#include "librbd/Utils.h" +#include "librbd/journal/Types.h" + +#define dout_subsys ceph_subsys_rbd_mirror +#undef dout_prefix +#define dout_prefix *_dout << "rbd::mirror::image_sync::SnapshotCopyRequest: " \ + << this << " " << __func__ + +namespace rbd { +namespace mirror { +namespace image_sync { + +namespace { + +template +const std::string &get_snapshot_name(I *image_ctx, librados::snap_t snap_id) { + auto snap_it = std::find_if(image_ctx->snap_ids.begin(), + image_ctx->snap_ids.end(), + [snap_id]( + const std::pair &pair) { + return pair.second == snap_id; + }); + assert(snap_it != image_ctx->snap_ids.end()); + return snap_it->first; +} + +} // anonymous namespace + +using librbd::util::create_context_callback; + + + +template +SnapshotCopyRequest::SnapshotCopyRequest(I *local_image_ctx, + I *remote_image_ctx, + SnapMap *snap_map, + Journaler *journaler, + librbd::journal::MirrorPeerClientMeta *meta, + Context *on_finish) + : m_local_image_ctx(local_image_ctx), m_remote_image_ctx(remote_image_ctx), + m_snap_map(snap_map), m_journaler(journaler), m_client_meta(meta), + m_on_finish(on_finish), m_snap_seqs(meta->snap_seqs) { + m_snap_map->clear(); + + // snap ids ordered from oldest to newest + m_remote_snap_ids.insert(remote_image_ctx->snaps.begin(), + remote_image_ctx->snaps.end()); + m_local_snap_ids.insert(local_image_ctx->snaps.begin(), + local_image_ctx->snaps.end()); +} + +template +void SnapshotCopyRequest::send() { + send_snap_remove(); +} + +template +void SnapshotCopyRequest::send_snap_remove() { + librados::snap_t local_snap_id = CEPH_NOSNAP; + while (local_snap_id == CEPH_NOSNAP && !m_local_snap_ids.empty()) { + librados::snap_t snap_id = *m_local_snap_ids.begin(); + + // if local snapshot id isn't in our mapping table, delete it + // we match by id since snapshots can be renamed + if (std::find_if(m_snap_seqs.begin(), m_snap_seqs.end(), + [snap_id](const SnapSeqs::value_type& pair) { + return pair.second == snap_id; }) == m_snap_seqs.end()) { + local_snap_id = snap_id; + m_local_snap_ids.erase(m_local_snap_ids.begin()); + } + } + + if (local_snap_id == CEPH_NOSNAP && m_local_snap_ids.empty()) { + // no local snapshots to delete + send_snap_create(); + return; + } + + m_snap_name = get_snapshot_name(m_local_image_ctx, local_snap_id); + + CephContext *cct = m_local_image_ctx->cct; + ldout(cct, 20) << ": " + << "snap_name=" << m_snap_name << ", " + << "snap_id=" << local_snap_id << dendl; + + Context *ctx = create_context_callback< + SnapshotCopyRequest, &SnapshotCopyRequest::handle_snap_remove>( + this); + RWLock::RLocker owner_locker(m_local_image_ctx->owner_lock); + m_local_image_ctx->operations->snap_remove(m_snap_name.c_str(), ctx); +} + +template +void SnapshotCopyRequest::handle_snap_remove(int r) { + CephContext *cct = m_local_image_ctx->cct; + ldout(cct, 20) << ": r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to remove snapshot '" << m_snap_name << "': " + << cpp_strerror(r) << dendl; + finish(r); + return; + } + + send_snap_remove(); +} + +template +void SnapshotCopyRequest::send_snap_create() { + librados::snap_t remote_snap_id = CEPH_NOSNAP; + while (remote_snap_id == CEPH_NOSNAP && !m_remote_snap_ids.empty()) { + librados::snap_t snap_id = *m_remote_snap_ids.begin(); + if (m_snap_seqs.find(snap_id) == m_snap_seqs.end()) { + // missing remote -> local mapping + remote_snap_id = snap_id; + } else { + // already have remote -> local mapping + m_remote_snap_ids.erase(m_remote_snap_ids.begin()); + } + } + + if (remote_snap_id == CEPH_NOSNAP && m_remote_snap_ids.empty()) { + // no local snapshots to create + send_update_client(); + return; + } + + m_snap_name = get_snapshot_name(m_remote_image_ctx, remote_snap_id); + + CephContext *cct = m_local_image_ctx->cct; + ldout(cct, 20) << ": " + << "snap_name=" << m_snap_name << ", " + << "snap_id=" << remote_snap_id << dendl; + + Context *ctx = create_context_callback< + SnapshotCopyRequest, &SnapshotCopyRequest::handle_snap_create>( + this); + RWLock::RLocker owner_locker(m_local_image_ctx->owner_lock); + m_local_image_ctx->operations->snap_create(m_snap_name.c_str(), ctx, 0U); +} + +template +void SnapshotCopyRequest::handle_snap_create(int r) { + CephContext *cct = m_local_image_ctx->cct; + ldout(cct, 20) << ": r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to create snapshot '" << m_snap_name << "': " + << cpp_strerror(r) << dendl; + finish(r); + return; + } + + assert(!m_remote_snap_ids.empty()); + librados::snap_t remote_snap_id = *m_remote_snap_ids.begin(); + m_remote_snap_ids.erase(m_remote_snap_ids.begin()); + + auto snap_it = m_local_image_ctx->snap_ids.find(m_snap_name); + assert(snap_it != m_local_image_ctx->snap_ids.end()); + librados::snap_t local_snap_id = snap_it->second; + + ldout(cct, 20) << ": mapping remote snap id " << remote_snap_id << " to " + << local_snap_id << dendl; + m_snap_seqs[remote_snap_id] = local_snap_id; + + send_snap_create(); +} + +template +void SnapshotCopyRequest::send_update_client() { + CephContext *cct = m_local_image_ctx->cct; + ldout(cct, 20) << dendl; + + compute_snap_map(); + + librbd::journal::MirrorPeerClientMeta client_meta(*m_client_meta); + client_meta.snap_seqs = m_snap_seqs; + + librbd::journal::ClientData client_data(client_meta); + bufferlist data_bl; + ::encode(client_data, data_bl); + + Context *ctx = create_context_callback< + SnapshotCopyRequest, &SnapshotCopyRequest::handle_update_client>( + this); + m_journaler->update_client(data_bl, ctx); +} + +template +void SnapshotCopyRequest::handle_update_client(int r) { + CephContext *cct = m_local_image_ctx->cct; + ldout(cct, 20) << ": r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to update client data: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + m_client_meta->snap_seqs = m_snap_seqs; + + finish(0); +} + +template +void SnapshotCopyRequest::finish(int r) { + CephContext *cct = m_local_image_ctx->cct; + ldout(cct, 20) << ": r=" << r << dendl; + + if (r >= 0) { + m_client_meta->snap_seqs = m_snap_seqs; + } + + m_on_finish->complete(r); + delete this; +} + +template +void SnapshotCopyRequest::compute_snap_map() { + SnapIds local_snap_ids; + for (auto &pair : m_snap_seqs) { + local_snap_ids.reserve(1 + local_snap_ids.size()); + local_snap_ids.insert(local_snap_ids.begin(), pair.second); + m_snap_map->insert(std::make_pair(pair.first, local_snap_ids)); + } +} + +} // namespace image_sync +} // namespace mirror +} // namespace rbd + +template class rbd::mirror::image_sync::SnapshotCopyRequest; diff --git a/src/tools/rbd_mirror/image_sync/SnapshotCopyRequest.h b/src/tools/rbd_mirror/image_sync/SnapshotCopyRequest.h new file mode 100644 index 00000000000..44368f23d56 --- /dev/null +++ b/src/tools/rbd_mirror/image_sync/SnapshotCopyRequest.h @@ -0,0 +1,112 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef RBD_MIRROR_IMAGE_SYNC_SNAPSHOT_COPY_REQUEST_H +#define RBD_MIRROR_IMAGE_SYNC_SNAPSHOT_COPY_REQUEST_H + +#include "include/int_types.h" +#include "include/rados/librados.hpp" +#include "common/snap_types.h" +#include "librbd/ImageCtx.h" +#include "librbd/Journal.h" +#include +#include +#include +#include + +class Context; +namespace journal { class Journaler; } +namespace librbd { namespace journal { struct MirrorPeerClientMeta; } } + +namespace rbd { +namespace mirror { +namespace image_sync { + +template +class SnapshotCopyRequest { +public: + typedef librbd::journal::TypeTraits TypeTraits; + typedef typename TypeTraits::Journaler Journaler; + + typedef std::vector SnapIds; + typedef std::map SnapMap; + + static SnapshotCopyRequest* create(ImageCtxT *local_image_ctx, + ImageCtxT *remote_image_ctx, + SnapMap *snap_map, Journaler *journaler, + librbd::journal::MirrorPeerClientMeta *client_meta, + Context *on_finish) { + return new SnapshotCopyRequest(local_image_ctx, remote_image_ctx, + snap_map, journaler, client_meta, on_finish); + } + + SnapshotCopyRequest(ImageCtxT *local_image_ctx, ImageCtxT *remote_image_ctx, + SnapMap *snap_map, Journaler *journaler, + librbd::journal::MirrorPeerClientMeta *client_meta, + Context *on_finish); + + void send(); + +private: + /** + * @verbatim + * + * + * | + * | /-------\ + * | | | + * v v | (repeat as needed) + * REMOVE_SNAP <--/ + * | + * | /-------\ + * | | | + * v v | (repeat as needed) + * CREATE_SNAP <--/ + * | + * v + * UPDATE_CLIENT + * | + * v + * + * + * @endverbatim + */ + + typedef std::set SnapIdSet; + typedef std::map SnapSeqs; + + ImageCtxT *m_local_image_ctx; + ImageCtxT *m_remote_image_ctx; + SnapMap *m_snap_map; + Journaler *m_journaler; + librbd::journal::MirrorPeerClientMeta *m_client_meta; + Context *m_on_finish; + + SnapIdSet m_local_snap_ids; + SnapIdSet m_remote_snap_ids; + SnapSeqs m_snap_seqs; + + std::string m_snap_name; + + void send_snap_remove(); + void handle_snap_remove(int r); + + void send_snap_create(); + void handle_snap_create(int r); + + void send_update_client(); + void handle_update_client(int r); + + void finish(int r); + + void compute_snap_map(); + +}; + +} // namespace image_sync +} // namespace mirror +} // namespace rbd + +extern template class rbd::mirror::image_sync::SnapshotCopyRequest; + +#endif // RBD_MIRROR_IMAGE_SYNC_SNAPSHOT_COPY_REQUEST_H -- 2.39.5