From 47181120c844a630918766b323f30bee51f68f2c Mon Sep 17 00:00:00 2001 From: Jason Dillaman Date: Fri, 11 Mar 2016 10:35:03 -0500 Subject: [PATCH] rbd-mirror: new image sync point create/prune state machines These are responsible for creating/removing snapshots in the remote image and updating the client registration to account for the snapshots. Signed-off-by: Jason Dillaman --- src/tools/Makefile-client.am | 8 +- .../image_sync/SyncPointCreateRequest.cc | 162 ++++++++++++++ .../image_sync/SyncPointCreateRequest.h | 90 ++++++++ .../image_sync/SyncPointPruneRequest.cc | 202 ++++++++++++++++++ .../image_sync/SyncPointPruneRequest.h | 94 ++++++++ 5 files changed, 554 insertions(+), 2 deletions(-) create mode 100644 src/tools/rbd_mirror/image_sync/SyncPointCreateRequest.cc create mode 100644 src/tools/rbd_mirror/image_sync/SyncPointCreateRequest.h create mode 100644 src/tools/rbd_mirror/image_sync/SyncPointPruneRequest.cc create mode 100644 src/tools/rbd_mirror/image_sync/SyncPointPruneRequest.h diff --git a/src/tools/Makefile-client.am b/src/tools/Makefile-client.am index dbd3cf05199..91302e51359 100644 --- a/src/tools/Makefile-client.am +++ b/src/tools/Makefile-client.am @@ -89,7 +89,9 @@ librbd_mirror_internal_la_SOURCES = \ tools/rbd_mirror/Threads.cc \ tools/rbd_mirror/types.cc \ tools/rbd_mirror/image_sync/ObjectCopyRequest.cc \ - tools/rbd_mirror/image_sync/SnapshotCopyRequest.cc + tools/rbd_mirror/image_sync/SnapshotCopyRequest.cc \ + tools/rbd_mirror/image_sync/SyncPointCreateRequest.cc \ + tools/rbd_mirror/image_sync/SyncPointPruneRequest.cc noinst_LTLIBRARIES += librbd_mirror_internal.la noinst_HEADERS += \ tools/rbd_mirror/ClusterWatcher.h \ @@ -100,7 +102,9 @@ noinst_HEADERS += \ tools/rbd_mirror/Threads.h \ tools/rbd_mirror/types.h \ tools/rbd_mirror/image_sync/ObjectCopyRequest.h \ - tools/rbd_mirror/image_sync/SnapshotCopyRequest.h + tools/rbd_mirror/image_sync/SnapshotCopyRequest.h \ + tools/rbd_mirror/image_sync/SyncPointCreateRequest.h \ + tools/rbd_mirror/image_sync/SyncPointPruneRequest.h rbd_mirror_SOURCES = \ tools/rbd_mirror/main.cc diff --git a/src/tools/rbd_mirror/image_sync/SyncPointCreateRequest.cc b/src/tools/rbd_mirror/image_sync/SyncPointCreateRequest.cc new file mode 100644 index 00000000000..74460498d69 --- /dev/null +++ b/src/tools/rbd_mirror/image_sync/SyncPointCreateRequest.cc @@ -0,0 +1,162 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "SyncPointCreateRequest.h" +#include "include/uuid.h" +#include "common/errno.h" +#include "journal/Journaler.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "librbd/Operations.h" +#include "librbd/Utils.h" + +#define dout_subsys ceph_subsys_rbd_mirror +#undef dout_prefix +#define dout_prefix *_dout << "rbd::mirror::image_sync::SyncPointCreateRequest: " \ + << this << " " << __func__ + +namespace rbd { +namespace mirror { +namespace image_sync { + +namespace { + +static const std::string SNAP_NAME_PREFIX(".rbd-mirror"); + +} // anonymous namespace + +using librbd::util::create_context_callback; + +template +SyncPointCreateRequest::SyncPointCreateRequest(I *remote_image_ctx, + const std::string &mirror_uuid, + Journaler *journaler, + MirrorPeerClientMeta *client_meta, + Context *on_finish) + : m_remote_image_ctx(remote_image_ctx), m_mirror_uuid(mirror_uuid), + m_journaler(journaler), m_client_meta(client_meta), m_on_finish(on_finish), + m_client_meta_copy(*client_meta) { + assert(m_client_meta->sync_points.size() < 2); + + // initialize the updated client meta with the new sync point + m_client_meta_copy.sync_points.emplace_back(); + if (m_client_meta_copy.sync_points.size() > 1) { + m_client_meta_copy.sync_points.back().from_snap_name = + m_client_meta_copy.sync_points.front().snap_name; + } +} + +template +void SyncPointCreateRequest::send() { + send_update_client(); +} + +template +void SyncPointCreateRequest::send_update_client() { + uuid_d uuid_gen; + uuid_gen.generate_random(); + + MirrorPeerSyncPoint &sync_point = m_client_meta_copy.sync_points.back(); + sync_point.snap_name = SNAP_NAME_PREFIX + "." + m_mirror_uuid + "." + + uuid_gen.to_string(); + + CephContext *cct = m_remote_image_ctx->cct; + ldout(cct, 20) << ": sync_point=" << sync_point << dendl; + + bufferlist client_data_bl; + librbd::journal::ClientData client_data(m_client_meta_copy); + ::encode(client_data, client_data_bl); + + Context *ctx = create_context_callback< + SyncPointCreateRequest, &SyncPointCreateRequest::handle_update_client>( + this); + m_journaler->update_client(client_data_bl, ctx); +} + +template +void SyncPointCreateRequest::handle_update_client(int r) { + CephContext *cct = m_remote_image_ctx->cct; + ldout(cct, 20) << ": r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to update client data: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + // update provided meta structure to reflect reality + *m_client_meta = m_client_meta_copy; + + send_refresh_image(); +} + +template +void SyncPointCreateRequest::send_refresh_image() { + CephContext *cct = m_remote_image_ctx->cct; + ldout(cct, 20) << dendl; + + Context *ctx = create_context_callback< + SyncPointCreateRequest, &SyncPointCreateRequest::handle_refresh_image>( + this); + m_remote_image_ctx->state->refresh(ctx); +} + +template +void SyncPointCreateRequest::handle_refresh_image(int r) { + CephContext *cct = m_remote_image_ctx->cct; + ldout(cct, 20) << ": r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "remote image refresh failed: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + send_create_snap(); +} + +template +void SyncPointCreateRequest::send_create_snap() { + CephContext *cct = m_remote_image_ctx->cct; + ldout(cct, 20) << dendl; + + MirrorPeerSyncPoint &sync_point = m_client_meta_copy.sync_points.back(); + + Context *ctx = create_context_callback< + SyncPointCreateRequest, &SyncPointCreateRequest::handle_create_snap>( + this); + m_remote_image_ctx->operations->snap_create( + sync_point.snap_name.c_str(), ctx); +} + +template +void SyncPointCreateRequest::handle_create_snap(int r) { + CephContext *cct = m_remote_image_ctx->cct; + ldout(cct, 20) << ": r=" << r << dendl; + + if (r == -EEXIST) { + send_update_client(); + return; + } else if (r < 0) { + lderr(cct) << "failed to create snapshot: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + finish(0); +} + +template +void SyncPointCreateRequest::finish(int r) { + CephContext *cct = m_remote_image_ctx->cct; + ldout(cct, 20) << ": r=" << r << dendl; + + m_on_finish->complete(r); + delete this; +} + +} // namespace image_sync +} // namespace mirror +} // namespace rbd + +template class rbd::mirror::image_sync::SyncPointCreateRequest; diff --git a/src/tools/rbd_mirror/image_sync/SyncPointCreateRequest.h b/src/tools/rbd_mirror/image_sync/SyncPointCreateRequest.h new file mode 100644 index 00000000000..ce09eda0db7 --- /dev/null +++ b/src/tools/rbd_mirror/image_sync/SyncPointCreateRequest.h @@ -0,0 +1,90 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef RBD_MIRROR_IMAGE_SYNC_SYNC_POINT_CREATE_REQUEST_H +#define RBD_MIRROR_IMAGE_SYNC_SYNC_POINT_CREATE_REQUEST_H + +#include "librbd/Journal.h" +#include "librbd/journal/Types.h" +#include + +class Context; +namespace journal { class Journaler; } +namespace librbd { class ImageCtx; } +namespace librbd { namespace journal { struct MirrorPeerClientMeta; } } + +namespace rbd { +namespace mirror { +namespace image_sync { + +template +class SyncPointCreateRequest { +public: + typedef librbd::journal::TypeTraits TypeTraits; + typedef typename TypeTraits::Journaler Journaler; + typedef librbd::journal::MirrorPeerClientMeta MirrorPeerClientMeta; + typedef librbd::journal::MirrorPeerSyncPoint MirrorPeerSyncPoint; + + static SyncPointCreateRequest* create(ImageCtxT *remote_image_ctx, + const std::string &mirror_uuid, + Journaler *journaler, + MirrorPeerClientMeta *client_meta, + Context *on_finish) { + return new SyncPointCreateRequest(remote_image_ctx, mirror_uuid, journaler, + client_meta, on_finish); + } + + SyncPointCreateRequest(ImageCtxT *remote_image_ctx, + const std::string &mirror_uuid, Journaler *journaler, + MirrorPeerClientMeta *client_meta, Context *on_finish); + + void send(); + +private: + /** + * @verbatim + * + * + * | + * v + * UPDATE_CLIENT < . . + * | . + * v . + * REFRESH_IMAGE . + * | . (repeat on EEXIST) + * v . + * CREATE_SNAP . . . . + * | + * v + * + * + * @endverbatim + */ + + ImageCtxT *m_remote_image_ctx; + std::string m_mirror_uuid; + Journaler *m_journaler; + MirrorPeerClientMeta *m_client_meta; + Context *m_on_finish; + + MirrorPeerClientMeta m_client_meta_copy; + + void send_update_client(); + void handle_update_client(int r); + + void send_refresh_image(); + void handle_refresh_image(int r); + + void send_create_snap(); + void handle_create_snap(int r); + + void finish(int r); +}; + +} // namespace image_sync +} // namespace mirror +} // namespace rbd + +extern template class rbd::mirror::image_sync::SyncPointCreateRequest; + +#endif // RBD_MIRROR_IMAGE_SYNC_SYNC_POINT_CREATE_REQUEST_H diff --git a/src/tools/rbd_mirror/image_sync/SyncPointPruneRequest.cc b/src/tools/rbd_mirror/image_sync/SyncPointPruneRequest.cc new file mode 100644 index 00000000000..6c653e4a473 --- /dev/null +++ b/src/tools/rbd_mirror/image_sync/SyncPointPruneRequest.cc @@ -0,0 +1,202 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "SyncPointPruneRequest.h" +#include "common/errno.h" +#include "journal/Journaler.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "librbd/Operations.h" +#include "librbd/Utils.h" +#include + +#define dout_subsys ceph_subsys_rbd_mirror +#undef dout_prefix +#define dout_prefix *_dout << "rbd::mirror::image_sync::SyncPointPruneRequest: " \ + << this << " " << __func__ +namespace rbd { +namespace mirror { +namespace image_sync { + +using librbd::util::create_context_callback; + +template +SyncPointPruneRequest::SyncPointPruneRequest(I *remote_image_ctx, + bool sync_complete, + Journaler *journaler, + MirrorPeerClientMeta *client_meta, + Context *on_finish) + : m_remote_image_ctx(remote_image_ctx), m_sync_complete(sync_complete), + m_journaler(journaler), m_client_meta(client_meta), m_on_finish(on_finish), + m_client_meta_copy(*client_meta) { +} + +template +void SyncPointPruneRequest::send() { + if (m_client_meta->sync_points.empty()) { + send_remove_snap(); + return; + } + + if (m_sync_complete) { + // if sync is complete, we can remove the master sync point + auto it = m_client_meta_copy.sync_points.begin(); + MirrorPeerSyncPoint &sync_point = *it; + + ++it; + if (it == m_client_meta_copy.sync_points.end() || + it->from_snap_name != sync_point.snap_name) { + m_snap_names.push_back(sync_point.snap_name); + } + + if (!sync_point.from_snap_name.empty()) { + m_snap_names.push_back(sync_point.from_snap_name); + } + } else { + // if we have more than one sync point, trim the extras off + std::set snap_names; + for (auto it = m_client_meta_copy.sync_points.rbegin(); + it != m_client_meta_copy.sync_points.rend(); ++it) { + MirrorPeerSyncPoint &sync_point = + m_client_meta_copy.sync_points.back(); + if (&sync_point == &m_client_meta_copy.sync_points.front()) { + break; + } + + if (snap_names.count(sync_point.snap_name) == 0) { + snap_names.insert(sync_point.snap_name); + m_snap_names.push_back(sync_point.snap_name); + } + + MirrorPeerSyncPoint &front_sync_point = + m_client_meta_copy.sync_points.front(); + if (!sync_point.from_snap_name.empty() && + snap_names.count(sync_point.from_snap_name) == 0 && + sync_point.from_snap_name != front_sync_point.snap_name) { + snap_names.insert(sync_point.from_snap_name); + m_snap_names.push_back(sync_point.from_snap_name); + } + } + } + + send_remove_snap(); +} + +template +void SyncPointPruneRequest::send_remove_snap() { + if (m_snap_names.empty()) { + send_refresh_image(); + return; + } + + std::string snap_name = m_snap_names.front(); + + CephContext *cct = m_remote_image_ctx->cct; + ldout(cct, 20) << ": snap_name=" << snap_name << dendl; + + Context *ctx = create_context_callback< + SyncPointPruneRequest, &SyncPointPruneRequest::handle_remove_snap>( + this); + m_remote_image_ctx->operations->snap_remove(snap_name.c_str(), ctx); +} + +template +void SyncPointPruneRequest::handle_remove_snap(int r) { + CephContext *cct = m_remote_image_ctx->cct; + ldout(cct, 20) << ": r=" << r << dendl; + + assert(!m_snap_names.empty()); + std::string snap_name = m_snap_names.front(); + m_snap_names.pop_front(); + + if (r == -ENOENT) { + r = 0; + } + if (r < 0) { + lderr(cct) << "failed to remove snapshot '" << snap_name << "': " + << cpp_strerror(r) << dendl; + finish(r); + return; + } + + send_remove_snap(); +} + +template +void SyncPointPruneRequest::send_refresh_image() { + CephContext *cct = m_remote_image_ctx->cct; + ldout(cct, 20) << dendl; + + Context *ctx = create_context_callback< + SyncPointPruneRequest, &SyncPointPruneRequest::handle_refresh_image>( + this); + m_remote_image_ctx->state->refresh(ctx); +} + +template +void SyncPointPruneRequest::handle_refresh_image(int r) { + CephContext *cct = m_remote_image_ctx->cct; + ldout(cct, 20) << ": r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "remote image refresh failed: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + send_update_client(); +} + +template +void SyncPointPruneRequest::send_update_client() { + CephContext *cct = m_remote_image_ctx->cct; + ldout(cct, 20) << dendl; + + if (m_sync_complete) { + m_client_meta_copy.sync_points.pop_front(); + } else { + while (m_client_meta_copy.sync_points.size() > 1) { + m_client_meta_copy.sync_points.pop_back(); + } + } + + bufferlist client_data_bl; + librbd::journal::ClientData client_data(m_client_meta_copy); + ::encode(client_data, client_data_bl); + + Context *ctx = create_context_callback< + SyncPointPruneRequest, &SyncPointPruneRequest::handle_update_client>( + this); + m_journaler->update_client(client_data_bl, ctx); +} + +template +void SyncPointPruneRequest::handle_update_client(int r) { + CephContext *cct = m_remote_image_ctx->cct; + ldout(cct, 20) << ": r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to update client data: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + // update provided meta structure to reflect reality + *m_client_meta = m_client_meta_copy; + finish(0); +} + +template +void SyncPointPruneRequest::finish(int r) { + CephContext *cct = m_remote_image_ctx->cct; + ldout(cct, 20) << ": r=" << r << dendl; + + m_on_finish->complete(r); + delete this; +} + +} // namespace image_sync +} // namespace mirror +} // namespace rbd + +template class rbd::mirror::image_sync::SyncPointPruneRequest; diff --git a/src/tools/rbd_mirror/image_sync/SyncPointPruneRequest.h b/src/tools/rbd_mirror/image_sync/SyncPointPruneRequest.h new file mode 100644 index 00000000000..b643fbb20e7 --- /dev/null +++ b/src/tools/rbd_mirror/image_sync/SyncPointPruneRequest.h @@ -0,0 +1,94 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef RBD_MIRROR_IMAGE_SYNC_SYNC_POINT_PRUNE_REQUEST_H +#define RBD_MIRROR_IMAGE_SYNC_SYNC_POINT_PRUNE_REQUEST_H + +#include "librbd/Journal.h" +#include "librbd/journal/Types.h" +#include +#include + +class Context; +namespace journal { class Journaler; } +namespace librbd { class ImageCtx; } +namespace librbd { namespace journal { struct MirrorPeerClientMeta; } } + +namespace rbd { +namespace mirror { +namespace image_sync { + +template +class SyncPointPruneRequest { +public: + typedef librbd::journal::TypeTraits TypeTraits; + typedef typename TypeTraits::Journaler Journaler; + typedef librbd::journal::MirrorPeerClientMeta MirrorPeerClientMeta; + typedef librbd::journal::MirrorPeerSyncPoint MirrorPeerSyncPoint; + + static SyncPointPruneRequest* create(ImageCtxT *remote_image_ctx, + bool sync_complete, + Journaler *journaler, + MirrorPeerClientMeta *client_meta, + Context *on_finish) { + return new SyncPointPruneRequest(remote_image_ctx, sync_complete, journaler, + client_meta, on_finish); + } + + SyncPointPruneRequest(ImageCtxT *remote_image_ctx, bool sync_complete, + Journaler *journaler, MirrorPeerClientMeta *client_meta, + Context *on_finish); + + void send(); + +private: + /** + * @verbatim + * + * + * | + * | . . . . . + * | . . + * v v . (repeat if from snap + * REMOVE_SNAP . . . unused by other sync) + * | + * v + * REFRESH_IMAGE + * | + * v + * UPDATE_CLIENT + * | + * v + * + * + * @endverbatim + */ + + ImageCtxT *m_remote_image_ctx; + bool m_sync_complete; + Journaler *m_journaler; + MirrorPeerClientMeta *m_client_meta; + Context *m_on_finish; + + MirrorPeerClientMeta m_client_meta_copy; + std::list m_snap_names; + + void send_remove_snap(); + void handle_remove_snap(int r); + + void send_refresh_image(); + void handle_refresh_image(int r); + + void send_update_client(); + void handle_update_client(int r); + + void finish(int r); +}; + +} // namespace image_sync +} // namespace mirror +} // namespace rbd + +extern template class rbd::mirror::image_sync::SyncPointPruneRequest; + +#endif // RBD_MIRROR_IMAGE_SYNC_SYNC_POINT_PRUNE_REQUEST_H -- 2.39.5