From: Jason Dillaman Date: Sat, 12 Mar 2016 02:24:17 +0000 (-0500) Subject: rbd-mirror: image sync state machine X-Git-Tag: v10.1.0~104^2~4 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=cacb48b1b47fc2cac0f7dd1f221df51d000b7c54;p=ceph-ci.git rbd-mirror: image sync state machine Signed-off-by: Jason Dillaman --- diff --git a/src/tools/Makefile-client.am b/src/tools/Makefile-client.am index e04ba46cc8b..099832db03a 100644 --- a/src/tools/Makefile-client.am +++ b/src/tools/Makefile-client.am @@ -83,6 +83,7 @@ endif # LINUX librbd_mirror_internal_la_SOURCES = \ tools/rbd_mirror/ClusterWatcher.cc \ tools/rbd_mirror/ImageReplayer.cc \ + tools/rbd_mirror/ImageSync.cc \ tools/rbd_mirror/Mirror.cc \ tools/rbd_mirror/PoolWatcher.cc \ tools/rbd_mirror/Replayer.cc \ @@ -97,6 +98,7 @@ noinst_LTLIBRARIES += librbd_mirror_internal.la noinst_HEADERS += \ tools/rbd_mirror/ClusterWatcher.h \ tools/rbd_mirror/ImageReplayer.h \ + tools/rbd_mirror/ImageSync.h \ tools/rbd_mirror/Mirror.h \ tools/rbd_mirror/PoolWatcher.h \ tools/rbd_mirror/Replayer.h \ diff --git a/src/tools/rbd_mirror/ImageSync.cc b/src/tools/rbd_mirror/ImageSync.cc new file mode 100644 index 00000000000..6f16da69633 --- /dev/null +++ b/src/tools/rbd_mirror/ImageSync.cc @@ -0,0 +1,243 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "ImageSync.h" +#include "common/errno.h" +#include "journal/Journaler.h" +#include "librbd/ImageCtx.h" +#include "librbd/Utils.h" +#include "librbd/journal/Types.h" +#include "tools/rbd_mirror/image_sync/ImageCopyRequest.h" +#include "tools/rbd_mirror/image_sync/SnapshotCopyRequest.h" +#include "tools/rbd_mirror/image_sync/SyncPointCreateRequest.h" +#include "tools/rbd_mirror/image_sync/SyncPointPruneRequest.h" + +#define dout_subsys ceph_subsys_rbd_mirror +#undef dout_prefix +#define dout_prefix *_dout << "rbd::mirror::ImageSync: " \ + << this << " " << __func__ + +namespace rbd { +namespace mirror { + +using namespace image_sync; +using librbd::util::create_context_callback; +using librbd::util::unique_lock_name; + +template +ImageSync::ImageSync(I *local_image_ctx, I *remote_image_ctx, + SafeTimer *timer, Mutex *timer_lock, + const std::string &mirror_uuid, Journaler *journaler, + MirrorPeerClientMeta *client_meta, Context *on_finish) + : m_local_image_ctx(local_image_ctx), m_remote_image_ctx(remote_image_ctx), + m_timer(timer), m_timer_lock(timer_lock), m_mirror_uuid(mirror_uuid), + m_journaler(journaler), m_client_meta(client_meta), m_on_finish(on_finish), + m_lock(unique_lock_name("ImageSync::m_lock", this)) { +} + +template +void ImageSync::start() { + send_prune_catch_up_sync_point(); +} + +template +void ImageSync::cancel() { + Mutex::Locker locker(m_lock); + + CephContext *cct = m_local_image_ctx->cct; + ldout(cct, 20) << dendl; + + m_canceled = true; + if (m_image_copy_request != nullptr) { + m_image_copy_request->cancel(); + } +} + +template +void ImageSync::send_prune_catch_up_sync_point() { + if (m_client_meta->sync_points.size() <= 1) { + send_create_sync_point(); + return; + } + + CephContext *cct = m_local_image_ctx->cct; + ldout(cct, 20) << dendl; + + Context *ctx = create_context_callback< + ImageSync, &ImageSync::handle_prune_catch_up_sync_point>(this); + SyncPointPruneRequest *request = SyncPointPruneRequest::create( + m_remote_image_ctx, false, m_journaler, m_client_meta, ctx); + request->send(); +} + +template +void ImageSync::handle_prune_catch_up_sync_point(int r) { + CephContext *cct = m_local_image_ctx->cct; + ldout(cct, 20) << ": r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to prune catch-up sync point: " + << cpp_strerror(r) << dendl; + finish(r); + return; + } + + send_create_sync_point(); +} + +template +void ImageSync::send_create_sync_point() { + // TODO: when support for disconnecting laggy clients is added, + // re-connect and create catch-up sync point + if (m_client_meta->sync_points.size() > 0) { + send_copy_snapshots(); + return; + } + + CephContext *cct = m_local_image_ctx->cct; + ldout(cct, 20) << dendl; + + Context *ctx = create_context_callback< + ImageSync, &ImageSync::handle_create_sync_point>(this); + SyncPointCreateRequest *request = SyncPointCreateRequest::create( + m_remote_image_ctx, m_mirror_uuid, m_journaler, m_client_meta, ctx); + request->send(); +} + +template +void ImageSync::handle_create_sync_point(int r) { + CephContext *cct = m_local_image_ctx->cct; + ldout(cct, 20) << ": r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to create sync point: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + send_copy_snapshots(); +} + +template +void ImageSync::send_copy_snapshots() { + CephContext *cct = m_local_image_ctx->cct; + ldout(cct, 20) << dendl; + + Context *ctx = create_context_callback< + ImageSync, &ImageSync::handle_copy_snapshots>(this); + SnapshotCopyRequest *request = SnapshotCopyRequest::create( + m_local_image_ctx, m_remote_image_ctx, &m_snap_map, m_journaler, + m_client_meta, ctx); + request->send(); +} + +template +void ImageSync::handle_copy_snapshots(int r) { + CephContext *cct = m_local_image_ctx->cct; + ldout(cct, 20) << ": r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to copy snapshot metadata: " + << cpp_strerror(r) << dendl; + finish(r); + return; + } + + send_copy_image(); +} + +template +void ImageSync::send_copy_image() { + m_lock.Lock(); + if (m_canceled) { + m_lock.Unlock(); + finish(-ECANCELED); + return; + } + + CephContext *cct = m_local_image_ctx->cct; + ldout(cct, 20) << dendl; + + Context *ctx = create_context_callback< + ImageSync, &ImageSync::handle_copy_image>(this); + m_image_copy_request = ImageCopyRequest::create( + m_local_image_ctx, m_remote_image_ctx, m_timer, m_timer_lock, + m_journaler, m_client_meta, &m_client_meta->sync_points.front(), + ctx); + m_lock.Unlock(); + + m_image_copy_request->send(); +} + +template +void ImageSync::handle_copy_image(int r) { + { + Mutex::Locker locker(m_lock); + m_image_copy_request = nullptr; + if (r == 0 && m_canceled) { + r = -ECANCELED; + } + } + + CephContext *cct = m_local_image_ctx->cct; + ldout(cct, 20) << ": r=" << r << dendl; + + if (r == -ECANCELED) { + ldout(cct, 10) << "image copy canceled" << dendl; + finish(r); + return; + } else if (r < 0) { + lderr(cct) << "failed to copy image: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + send_prune_sync_points(); +} + +template +void ImageSync::send_prune_sync_points() { + CephContext *cct = m_local_image_ctx->cct; + ldout(cct, 20) << dendl; + + Context *ctx = create_context_callback< + ImageSync, &ImageSync::handle_prune_sync_points>(this); + SyncPointPruneRequest *request = SyncPointPruneRequest::create( + m_remote_image_ctx, true, m_journaler, m_client_meta, ctx); + request->send(); +} + +template +void ImageSync::handle_prune_sync_points(int r) { + CephContext *cct = m_local_image_ctx->cct; + ldout(cct, 20) << ": r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to prune sync point: " + << cpp_strerror(r) << dendl; + finish(r); + return; + } + + if (!m_client_meta->sync_points.empty()) { + send_copy_image(); + return; + } + + finish(0); +} + +template +void ImageSync::finish(int r) { + CephContext *cct = m_local_image_ctx->cct; + ldout(cct, 20) << ": r=" << r << dendl; + + m_on_finish->complete(r); + delete this; +} + +} // namespace mirror +} // namespace rbd + +template class rbd::mirror::ImageSync; diff --git a/src/tools/rbd_mirror/ImageSync.h b/src/tools/rbd_mirror/ImageSync.h new file mode 100644 index 00000000000..b34214180c0 --- /dev/null +++ b/src/tools/rbd_mirror/ImageSync.h @@ -0,0 +1,109 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef RBD_MIRROR_IMAGE_SYNC_H +#define RBD_MIRROR_IMAGE_SYNC_H + +#include "include/int_types.h" +#include "librbd/Journal.h" +#include "common/Mutex.h" +#include +#include + +class Context; +class Mutex; +class SafeTimer; +namespace journal { class Journaler; } +namespace librbd { class ImageCtx; } +namespace librbd { namespace journal { struct MirrorPeerClientMeta; } } + +namespace rbd { +namespace mirror { + +namespace image_sync { template class ImageCopyRequest; } + +template +class ImageSync { +public: + typedef librbd::journal::TypeTraits TypeTraits; + typedef typename TypeTraits::Journaler Journaler; + typedef librbd::journal::MirrorPeerClientMeta MirrorPeerClientMeta; + + ImageSync(ImageCtxT *local_image_ctx, ImageCtxT *remote_image_ctx, + SafeTimer *timer, Mutex *timer_lock, const std::string &mirror_uuid, + Journaler *journaler, MirrorPeerClientMeta *client_meta, + Context *on_finish); + + void start(); + void cancel(); + +private: + /** + * @verbatim + * + * + * | + * v + * PRUNE_CATCH_UP_SYNC_POINT + * | + * v + * CREATE_SYNC_POINT (skip if already exists and + * | not disconnected) + * v + * COPY_SNAPSHOTS + * | + * v + * COPY_IMAGE . . . . . . + * | . + * v . + * PRUNE_SYNC_POINTS . (image sync canceled) + * | . + * v . + * < . . . . . . + * + * @endverbatim + */ + + typedef std::vector SnapIds; + typedef std::map SnapMap; + + ImageCtxT *m_local_image_ctx; + ImageCtxT *m_remote_image_ctx; + SafeTimer *m_timer; + Mutex *m_timer_lock; + std::string m_mirror_uuid; + Journaler *m_journaler; + MirrorPeerClientMeta *m_client_meta; + Context *m_on_finish; + + SnapMap m_snap_map; + + Mutex m_lock; + bool m_canceled = false; + + image_sync::ImageCopyRequest *m_image_copy_request; + + void send_prune_catch_up_sync_point(); + void handle_prune_catch_up_sync_point(int r); + + void send_create_sync_point(); + void handle_create_sync_point(int r); + + void send_copy_snapshots(); + void handle_copy_snapshots(int r); + + void send_copy_image(); + void handle_copy_image(int r); + + void send_prune_sync_points(); + void handle_prune_sync_points(int r); + + void finish(int r); +}; + +} // namespace mirror +} // namespace rbd + +extern template class rbd::mirror::ImageSync; + +#endif // RBD_MIRROR_IMAGE_SYNC_H