From e530fbf22fed68f8cd3353a6470b34690b6c6474 Mon Sep 17 00:00:00 2001 From: Jason Dillaman Date: Fri, 11 Mar 2016 15:45:09 -0500 Subject: [PATCH] rbd-mirror: state machine to iterate and copy all image objects Signed-off-by: Jason Dillaman --- src/librbd/ImageCtx.cc | 7 + src/librbd/ImageCtx.h | 1 + src/test/librbd/mock/MockImageCtx.h | 1 + src/tools/Makefile-client.am | 2 + .../rbd_mirror/image_sync/ImageCopyRequest.cc | 287 ++++++++++++++++++ .../rbd_mirror/image_sync/ImageCopyRequest.h | 119 ++++++++ 6 files changed, 417 insertions(+) create mode 100644 src/tools/rbd_mirror/image_sync/ImageCopyRequest.cc create mode 100644 src/tools/rbd_mirror/image_sync/ImageCopyRequest.h diff --git a/src/librbd/ImageCtx.cc b/src/librbd/ImageCtx.cc index 30ec670e0b592..0f9b5a5d40d9b 100644 --- a/src/librbd/ImageCtx.cc +++ b/src/librbd/ImageCtx.cc @@ -26,6 +26,7 @@ #include "librbd/operation/ResizeRequest.h" #include "librbd/Utils.h" +#include "osdc/Striper.h" #include #define dout_subsys ceph_subsys_rbd @@ -541,6 +542,12 @@ struct C_InvalidateCache : public Context { return 0; } + uint64_t ImageCtx::get_object_count(snap_t in_snap_id) const { + assert(snap_lock.is_locked()); + uint64_t image_size = get_image_size(in_snap_id); + return Striper::get_num_objects(layout, image_size); + } + bool ImageCtx::test_features(uint64_t features) const { RWLock::RLocker l(snap_lock); diff --git a/src/librbd/ImageCtx.h b/src/librbd/ImageCtx.h index 409addf7a1b4b..a302a1467c787 100644 --- a/src/librbd/ImageCtx.h +++ b/src/librbd/ImageCtx.h @@ -228,6 +228,7 @@ namespace librbd { uint8_t protection_status, uint64_t flags); void rm_snap(std::string in_snap_name, librados::snap_t id); uint64_t get_image_size(librados::snap_t in_snap_id) const; + uint64_t get_object_count(librados::snap_t in_snap_id) const; bool test_features(uint64_t test_features) const; bool test_features(uint64_t test_features, const RWLock &in_snap_lock) const; diff --git a/src/test/librbd/mock/MockImageCtx.h b/src/test/librbd/mock/MockImageCtx.h index b6e8e9e7362d1..60d5fbbf996f7 100644 --- a/src/test/librbd/mock/MockImageCtx.h +++ b/src/test/librbd/mock/MockImageCtx.h @@ -113,6 +113,7 @@ struct MockImageCtx { MOCK_CONST_METHOD1(get_object_name, std::string(uint64_t)); MOCK_CONST_METHOD0(get_current_size, uint64_t()); MOCK_CONST_METHOD1(get_image_size, uint64_t(librados::snap_t)); + MOCK_CONST_METHOD1(get_object_count, uint64_t(librados::snap_t)); MOCK_CONST_METHOD1(get_snap_id, librados::snap_t(std::string in_snap_name)); MOCK_CONST_METHOD1(get_snap_info, const SnapInfo*(librados::snap_t)); MOCK_CONST_METHOD2(get_parent_spec, int(librados::snap_t in_snap_id, diff --git a/src/tools/Makefile-client.am b/src/tools/Makefile-client.am index 91302e5135984..e04ba46cc8bdd 100644 --- a/src/tools/Makefile-client.am +++ b/src/tools/Makefile-client.am @@ -88,6 +88,7 @@ librbd_mirror_internal_la_SOURCES = \ tools/rbd_mirror/Replayer.cc \ tools/rbd_mirror/Threads.cc \ tools/rbd_mirror/types.cc \ + tools/rbd_mirror/image_sync/ImageCopyRequest.cc \ tools/rbd_mirror/image_sync/ObjectCopyRequest.cc \ tools/rbd_mirror/image_sync/SnapshotCopyRequest.cc \ tools/rbd_mirror/image_sync/SyncPointCreateRequest.cc \ @@ -101,6 +102,7 @@ noinst_HEADERS += \ tools/rbd_mirror/Replayer.h \ tools/rbd_mirror/Threads.h \ tools/rbd_mirror/types.h \ + tools/rbd_mirror/image_sync/ImageCopyRequest.h \ tools/rbd_mirror/image_sync/ObjectCopyRequest.h \ tools/rbd_mirror/image_sync/SnapshotCopyRequest.h \ tools/rbd_mirror/image_sync/SyncPointCreateRequest.h \ diff --git a/src/tools/rbd_mirror/image_sync/ImageCopyRequest.cc b/src/tools/rbd_mirror/image_sync/ImageCopyRequest.cc new file mode 100644 index 0000000000000..aedf2f5321d0f --- /dev/null +++ b/src/tools/rbd_mirror/image_sync/ImageCopyRequest.cc @@ -0,0 +1,287 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "ImageCopyRequest.h" +#include "ObjectCopyRequest.h" +#include "common/errno.h" +#include "journal/Journaler.h" +#include "librbd/Utils.h" + +#define dout_subsys ceph_subsys_rbd_mirror +#undef dout_prefix +#define dout_prefix *_dout << "rbd::mirror::image_sync::ImageCopyRequest: " \ + << this << " " << __func__ + +namespace rbd { +namespace mirror { +namespace image_sync { + +using librbd::util::create_context_callback; +using librbd::util::unique_lock_name; + +template +ImageCopyRequest::ImageCopyRequest(I *local_image_ctx, I *remote_image_ctx, + SafeTimer *timer, Mutex *timer_lock, + Journaler *journaler, + MirrorPeerClientMeta *client_meta, + MirrorPeerSyncPoint *sync_point, + Context *on_finish) + : m_local_image_ctx(local_image_ctx), m_remote_image_ctx(remote_image_ctx), + m_timer(timer), m_timer_lock(timer_lock), m_journaler(journaler), + m_client_meta(client_meta), m_sync_point(sync_point), + m_on_finish(on_finish), + m_lock(unique_lock_name("ImageCopyRequest::m_lock", this)), + m_client_meta_copy(*client_meta) { + assert(!m_client_meta_copy.sync_points.empty()); + assert(!m_client_meta_copy.snap_seqs.empty()); +} + +template +void ImageCopyRequest::send() { + int r = compute_snap_map(); + if (r < 0) { + finish(r); + return; + } + + send_update_max_object_count(); +} + +template +void ImageCopyRequest::cancel() { + Mutex::Locker locker(m_lock); + + CephContext *cct = m_local_image_ctx->cct; + ldout(cct, 20) << dendl; + m_canceled = true; +} + +template +void ImageCopyRequest::send_update_max_object_count() { + uint64_t max_objects = m_client_meta->sync_object_count; + { + RWLock::RLocker snap_locker(m_remote_image_ctx->snap_lock); + max_objects = std::max(max_objects, + m_remote_image_ctx->get_object_count(CEPH_NOSNAP)); + for (auto snap_id : m_remote_image_ctx->snaps) { + max_objects = std::max(max_objects, + m_remote_image_ctx->get_object_count(snap_id)); + } + } + + if (max_objects == m_client_meta->sync_object_count) { + send_object_copies(); + return; + } + + CephContext *cct = m_local_image_ctx->cct; + ldout(cct, 20) << ": sync_object_count=" << max_objects << dendl; + + m_client_meta_copy = *m_client_meta; + m_client_meta_copy.sync_object_count = max_objects; + + bufferlist client_data_bl; + librbd::journal::ClientData client_data(m_client_meta_copy); + ::encode(client_data, client_data_bl); + + Context *ctx = create_context_callback< + ImageCopyRequest, &ImageCopyRequest::handle_update_max_object_count>( + this); + m_journaler->update_client(client_data_bl, ctx); +} + +template +void ImageCopyRequest::handle_update_max_object_count(int r) { + CephContext *cct = m_local_image_ctx->cct; + ldout(cct, 20) << ": r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to update client data: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + // update provided meta structure to reflect reality + m_client_meta->sync_object_count = m_client_meta_copy.sync_object_count; + m_object_no = 0; + if (m_sync_point->object_number) { + m_object_no = *m_sync_point->object_number + 1; + } + m_end_object_no = m_client_meta_copy.sync_object_count; + + send_object_copies(); +} + +template +void ImageCopyRequest::send_object_copies() { + CephContext *cct = m_local_image_ctx->cct; + bool complete; + { + Mutex::Locker locker(m_lock); + for (int i = 0; i < cct->_conf->rbd_concurrent_management_ops; ++i) { + send_next_object_copy(); + if (m_ret_val < 0 && m_current_ops == 0) { + break; + } + } + complete = (m_current_ops == 0); + } + if (complete) { + send_flush_sync_point(); + } +} + +template +void ImageCopyRequest::send_next_object_copy() { + assert(m_lock.is_locked()); + if (m_canceled) { + return; + } else if (m_ret_val < 0 || m_object_no >= m_end_object_no) { + return; + } + + uint64_t ono = m_object_no++; + + CephContext *cct = m_local_image_ctx->cct; + ldout(cct, 20) << ": object_num=" << ono << dendl; + + ++m_current_ops; + + Context *ctx = create_context_callback< + ImageCopyRequest, &ImageCopyRequest::handle_object_copy>(this); + ObjectCopyRequest *req = ObjectCopyRequest::create( + m_local_image_ctx, m_remote_image_ctx, &m_snap_map, ono, ctx); + req->send(); +} + +template +void ImageCopyRequest::handle_object_copy(int r) { + CephContext *cct = m_local_image_ctx->cct; + ldout(cct, 20) << ": r=" << r << dendl; + + bool complete; + { + Mutex::Locker locker(m_lock); + assert(m_current_ops > 0); + --m_current_ops; + + if (r < 0) { + lderr(cct) << "object copy failed: " << cpp_strerror(r) << dendl; + if (m_ret_val == 0) { + m_ret_val = r; + } + } + + send_next_object_copy(); + complete = (m_current_ops == 0); + } + + if (complete) { + send_flush_sync_point(); + } +} + +template +void ImageCopyRequest::send_flush_sync_point() { + if (m_ret_val < 0) { + finish(m_ret_val); + return; + } + + m_client_meta_copy = *m_client_meta; + if (m_object_no > 0) { + m_sync_point->object_number = m_object_no - 1; + } else { + m_sync_point->object_number = boost::none; + } + + CephContext *cct = m_local_image_ctx->cct; + ldout(cct, 20) << ": sync_point=" << *m_sync_point << dendl; + + bufferlist client_data_bl; + librbd::journal::ClientData client_data(m_client_meta_copy); + ::encode(client_data, client_data_bl); + + Context *ctx = create_context_callback< + ImageCopyRequest, &ImageCopyRequest::handle_flush_sync_point>( + this); + m_journaler->update_client(client_data_bl, ctx); +} + +template +void ImageCopyRequest::handle_flush_sync_point(int r) { + CephContext *cct = m_local_image_ctx->cct; + ldout(cct, 20) << ": r=" << r << dendl; + + if (r < 0) { + *m_client_meta = m_client_meta_copy; + + lderr(cct) << "failed to update client data: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + finish(0); +} + +template +void ImageCopyRequest::finish(int r) { + CephContext *cct = m_local_image_ctx->cct; + ldout(cct, 20) << ": r=" << r << dendl; + + m_on_finish->complete(r); + delete this; +} + +template +int ImageCopyRequest::compute_snap_map() { + CephContext *cct = m_local_image_ctx->cct; + + librados::snap_t snap_id_start = 0; + librados::snap_t snap_id_end; + { + RWLock::RLocker snap_locker(m_remote_image_ctx->snap_lock); + snap_id_end = m_remote_image_ctx->get_snap_id(m_sync_point->snap_name); + if (snap_id_end == CEPH_NOSNAP) { + lderr(cct) << "failed to locate snapshot: " + << m_sync_point->snap_name << dendl; + return -ENOENT; + } + + if (!m_sync_point->from_snap_name.empty()) { + snap_id_start = m_remote_image_ctx->get_snap_id( + m_sync_point->from_snap_name); + if (snap_id_start == CEPH_NOSNAP) { + lderr(cct) << "failed to locate from snapshot: " + << m_sync_point->from_snap_name << dendl; + return -ENOENT; + } + } + } + + SnapIds snap_ids; + for (auto it = m_client_meta->snap_seqs.begin(); + it != m_client_meta->snap_seqs.end(); ++it) { + snap_ids.insert(snap_ids.begin(), it->second); + if (it->first < snap_id_start) { + continue; + } else if (it->first > snap_id_end) { + break; + } + + m_snap_map[it->first] = snap_ids; + } + + if (m_snap_map.empty()) { + lderr(cct) << "failed to map snapshots within boundary" << dendl; + return -EINVAL; + } + + return 0; +} + +} // namespace image_sync +} // namespace mirror +} // namespace rbd + +template class rbd::mirror::image_sync::ImageCopyRequest; diff --git a/src/tools/rbd_mirror/image_sync/ImageCopyRequest.h b/src/tools/rbd_mirror/image_sync/ImageCopyRequest.h new file mode 100644 index 0000000000000..f2d1396f20bd6 --- /dev/null +++ b/src/tools/rbd_mirror/image_sync/ImageCopyRequest.h @@ -0,0 +1,119 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef RBD_MIRROR_IMAGE_SYNC_IMAGE_COPY_REQUEST_H +#define RBD_MIRROR_IMAGE_SYNC_IMAGE_COPY_REQUEST_H + +#include "include/int_types.h" +#include "include/rados/librados.hpp" +#include "common/Mutex.h" +#include "librbd/Journal.h" +#include "librbd/journal/Types.h" +#include +#include + +class Context; +namespace journal { class Journaler; } +namespace librbd { struct ImageCtx; } + +namespace rbd { +namespace mirror { +namespace image_sync { + +template +class ImageCopyRequest { +public: + typedef std::vector SnapIds; + typedef std::map SnapMap; + typedef librbd::journal::TypeTraits TypeTraits; + typedef typename TypeTraits::Journaler Journaler; + typedef librbd::journal::MirrorPeerSyncPoint MirrorPeerSyncPoint; + typedef librbd::journal::MirrorPeerClientMeta MirrorPeerClientMeta; + + static ImageCopyRequest* create(ImageCtxT *local_image_ctx, + ImageCtxT *remote_image_ctx, + SafeTimer *timer, Mutex *timer_lock, + Journaler *journaler, + MirrorPeerClientMeta *client_meta, + MirrorPeerSyncPoint *sync_point, + Context *on_finish) { + return new ImageCopyRequest(local_image_ctx, remote_image_ctx, timer, + timer_lock, journaler, client_meta, sync_point, + on_finish); + } + + ImageCopyRequest(ImageCtxT *local_image_ctx, ImageCtxT *remote_image_ctx, + SafeTimer *timer, Mutex *timer_lock, Journaler *journaler, + MirrorPeerClientMeta *client_meta, + MirrorPeerSyncPoint *sync_point, Context *on_finish); + + void send(); + void cancel(); + +private: + /** + * @verbatim + * + * + * | + * v + * UPDATE_MAX_OBJECT_COUNT + * | + * | . . . . . + * | . . (parallel execution of + * v v . multiple objects at once) + * COPY_OBJECT . . + * | + * v + * FLUSH_SYNC_POINT + * | + * v + * + * + * @endverbatim + */ + + ImageCtxT *m_local_image_ctx; + ImageCtxT *m_remote_image_ctx; + SafeTimer *m_timer; + Mutex *m_timer_lock; + Journaler *m_journaler; + MirrorPeerClientMeta *m_client_meta; + MirrorPeerSyncPoint *m_sync_point; + Context *m_on_finish; + + SnapMap m_snap_map; + + Mutex m_lock; + bool m_canceled = false; + + uint64_t m_object_no = 0; + uint64_t m_end_object_no; + uint64_t m_current_ops = 0; + int m_ret_val = 0; + + MirrorPeerClientMeta m_client_meta_copy; + + void send_update_max_object_count(); + void handle_update_max_object_count(int r); + + void send_object_copies(); + void send_next_object_copy(); + void handle_object_copy(int r); + + void send_flush_sync_point(); + void handle_flush_sync_point(int r); + + void finish(int r); + + int compute_snap_map(); + +}; + +} // namespace image_sync +} // namespace mirror +} // namespace rbd + +extern template class rbd::mirror::image_sync::ImageCopyRequest; + +#endif // RBD_MIRROR_IMAGE_SYNC_IMAGE_COPY_REQUEST_H -- 2.39.5