From: Jason Dillaman Date: Sun, 13 Mar 2016 03:11:42 +0000 (-0500) Subject: rbd-mirror: update object map while performing image sync X-Git-Tag: v10.1.0~104^2~1 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=0ef801f563eb418da10231e7cd22983f5a384794;p=ceph.git rbd-mirror: update object map while performing image sync Signed-off-by: Jason Dillaman --- diff --git a/src/tools/rbd_mirror/ImageSync.cc b/src/tools/rbd_mirror/ImageSync.cc index 6f16da696338..5bb145779472 100644 --- a/src/tools/rbd_mirror/ImageSync.cc +++ b/src/tools/rbd_mirror/ImageSync.cc @@ -5,6 +5,7 @@ #include "common/errno.h" #include "journal/Journaler.h" #include "librbd/ImageCtx.h" +#include "librbd/ObjectMap.h" #include "librbd/Utils.h" #include "librbd/journal/Types.h" #include "tools/rbd_mirror/image_sync/ImageCopyRequest.h" @@ -193,6 +194,72 @@ void ImageSync::handle_copy_image(int r) { return; } + send_copy_object_map(); +} + +template +void ImageSync::send_copy_object_map() { + m_local_image_ctx->snap_lock.get_read(); + if (!m_local_image_ctx->test_features(RBD_FEATURE_OBJECT_MAP, + m_local_image_ctx->snap_lock)) { + m_local_image_ctx->snap_lock.put_read(); + send_prune_sync_points(); + return; + } + + assert(m_local_image_ctx->object_map != nullptr); + + assert(!m_client_meta->sync_points.empty()); + librbd::journal::MirrorPeerSyncPoint &sync_point = + m_client_meta->sync_points.front(); + auto snap_id_it = m_local_image_ctx->snap_ids.find(sync_point.snap_name); + assert(snap_id_it != m_local_image_ctx->snap_ids.end()); + librados::snap_t snap_id = snap_id_it->second; + + CephContext *cct = m_local_image_ctx->cct; + ldout(cct, 20) << ": snap_id=" << snap_id << ", " + << "snap_name=" << sync_point.snap_name << dendl; + + // rollback the object map (copy snapshot object map to HEAD) + RWLock::WLocker object_map_locker(m_local_image_ctx->object_map_lock); + Context *ctx = create_context_callback< + ImageSync, &ImageSync::handle_copy_object_map>(this); + m_local_image_ctx->object_map->rollback(snap_id, ctx); + m_local_image_ctx->snap_lock.put_read(); +} + +template +void ImageSync::handle_copy_object_map(int r) { + CephContext *cct = m_local_image_ctx->cct; + ldout(cct, 20) << dendl; + + assert(r == 0); + send_refresh_object_map(); +} + +template +void ImageSync::send_refresh_object_map() { + CephContext *cct = m_local_image_ctx->cct; + ldout(cct, 20) << dendl; + + Context *ctx = create_context_callback< + ImageSync, &ImageSync::handle_refresh_object_map>(this); + m_object_map = m_local_image_ctx->create_object_map(CEPH_NOSNAP); + m_object_map->open(ctx); +} + +template +void ImageSync::handle_refresh_object_map(int r) { + CephContext *cct = m_local_image_ctx->cct; + ldout(cct, 20) << dendl; + + assert(r == 0); + { + RWLock::WLocker snap_locker(m_local_image_ctx->snap_lock); + std::swap(m_local_image_ctx->object_map, m_object_map); + } + delete m_object_map; + send_prune_sync_points(); } diff --git a/src/tools/rbd_mirror/ImageSync.h b/src/tools/rbd_mirror/ImageSync.h index b34214180c01..175cb7804128 100644 --- a/src/tools/rbd_mirror/ImageSync.h +++ b/src/tools/rbd_mirror/ImageSync.h @@ -5,6 +5,7 @@ #define RBD_MIRROR_IMAGE_SYNC_H #include "include/int_types.h" +#include "librbd/ImageCtx.h" #include "librbd/Journal.h" #include "common/Mutex.h" #include @@ -14,7 +15,6 @@ class Context; class Mutex; class SafeTimer; namespace journal { class Journaler; } -namespace librbd { class ImageCtx; } namespace librbd { namespace journal { struct MirrorPeerClientMeta; } } namespace rbd { @@ -53,13 +53,19 @@ private: * COPY_SNAPSHOTS * | * v - * COPY_IMAGE . . . . . . - * | . - * v . - * PRUNE_SYNC_POINTS . (image sync canceled) - * | . - * v . - * < . . . . . . + * COPY_IMAGE . . . . . . . . . . . . . . + * | . + * v . + * COPY_OBJECT_MAP (skip if object . + * | map disabled) . + * v . + * REFRESH_OBJECT_MAP (skip if object . + * | map disabled) . + * v + * PRUNE_SYNC_POINTS . (image sync canceled) + * | . + * v . + * < . . . . . . . . . . . . . . * * @endverbatim */ @@ -82,6 +88,7 @@ private: bool m_canceled = false; image_sync::ImageCopyRequest *m_image_copy_request; + decltype(ImageCtxT::object_map) m_object_map = nullptr; void send_prune_catch_up_sync_point(); void handle_prune_catch_up_sync_point(int r); @@ -95,6 +102,12 @@ private: void send_copy_image(); void handle_copy_image(int r); + void send_copy_object_map(); + void handle_copy_object_map(int r); + + void send_refresh_object_map(); + void handle_refresh_object_map(int r); + void send_prune_sync_points(); void handle_prune_sync_points(int r); diff --git a/src/tools/rbd_mirror/image_sync/ObjectCopyRequest.cc b/src/tools/rbd_mirror/image_sync/ObjectCopyRequest.cc index eb669345c6ea..c49beed2f101 100644 --- a/src/tools/rbd_mirror/image_sync/ObjectCopyRequest.cc +++ b/src/tools/rbd_mirror/image_sync/ObjectCopyRequest.cc @@ -3,6 +3,7 @@ #include "ObjectCopyRequest.h" #include "librados/snap_set_diff.h" +#include "librbd/ObjectMap.h" #include "librbd/Utils.h" #include "common/errno.h" @@ -15,6 +16,7 @@ namespace rbd { namespace mirror { namespace image_sync { +using librbd::util::create_context_callback; using librbd::util::create_rados_ack_callback; using librbd::util::create_rados_safe_callback; @@ -218,7 +220,55 @@ void ObjectCopyRequest::handle_write_object(int r) { return; } - finish(r); + send_update_object_map(); +} + +template +void ObjectCopyRequest::send_update_object_map() { + m_local_image_ctx->snap_lock.get_read(); + if (!m_local_image_ctx->test_features(RBD_FEATURE_OBJECT_MAP, + m_local_image_ctx->snap_lock) || + m_snap_object_states.empty()) { + m_local_image_ctx->snap_lock.put_read(); + finish(0); + return; + } + + assert(m_local_image_ctx->object_map != nullptr); + + auto snap_object_state = *m_snap_object_states.begin(); + m_snap_object_states.erase(m_snap_object_states.begin()); + + CephContext *cct = m_local_image_ctx->cct; + ldout(cct, 20) << ": " + << "snap_id=" << snap_object_state.first << ", " + << "object_state=" << static_cast( + snap_object_state.second) + << dendl; + + RWLock::WLocker object_map_locker(m_local_image_ctx->object_map_lock); + Context *ctx = create_context_callback< + ObjectCopyRequest, &ObjectCopyRequest::handle_update_object_map>( + this); + m_local_image_ctx->object_map->aio_update(snap_object_state.first, + m_object_number, + m_object_number + 1, + snap_object_state.second, + boost::none, ctx); + m_local_image_ctx->snap_lock.put_read(); +} + +template +void ObjectCopyRequest::handle_update_object_map(int r) { + CephContext *cct = m_local_image_ctx->cct; + ldout(cct, 20) << ": r=" << r << dendl; + + assert(r == 0); + if (!m_snap_object_states.empty()) { + send_update_object_map(); + return; + } + finish(0); } template @@ -256,6 +306,18 @@ void ObjectCopyRequest::compute_diffs() { ldout(cct, 20) << ": clearing truncate diff: " << trunc << dendl; } + // prepare the object map state + { + RWLock::RLocker snap_locker(m_local_image_ctx->snap_lock); + uint8_t object_state = OBJECT_EXISTS; + if (m_local_image_ctx->test_features(RBD_FEATURE_FAST_DIFF, + m_local_image_ctx->snap_lock) && + diff.empty() && end_size == prev_end_size) { + object_state = OBJECT_EXISTS_CLEAN; + } + m_snap_object_states[end_snap_id] = object_state; + } + // object write/zero, or truncate for (auto it = diff.begin(); it != diff.end(); ++it) { ldout(cct, 20) << ": read/write op: " << it.get_start() << "~" @@ -270,11 +332,14 @@ void ObjectCopyRequest::compute_diffs() { m_snap_sync_ops[start_snap_id].emplace_back(SYNC_OP_TYPE_TRUNC, end_size, 0U, bufferlist()); } - } else if (prev_exists) { - // object remove - ldout(cct, 20) << ": remove op" << dendl; - m_snap_sync_ops[start_snap_id].emplace_back(SYNC_OP_TYPE_REMOVE, 0U, 0U, - bufferlist()); + } else { + m_snap_object_states[end_snap_id] = OBJECT_NONEXISTENT; + if (prev_exists) { + // object remove + ldout(cct, 20) << ": remove op" << dendl; + m_snap_sync_ops[start_snap_id].emplace_back(SYNC_OP_TYPE_REMOVE, 0U, 0U, + bufferlist()); + } } prev_end_size = end_size; diff --git a/src/tools/rbd_mirror/image_sync/ObjectCopyRequest.h b/src/tools/rbd_mirror/image_sync/ObjectCopyRequest.h index 64d8e18cf018..83fe16c52c92 100644 --- a/src/tools/rbd_mirror/image_sync/ObjectCopyRequest.h +++ b/src/tools/rbd_mirror/image_sync/ObjectCopyRequest.h @@ -58,10 +58,16 @@ private: * LIST_SNAPS * | * v - * READ_OBJECT <----\ - * | | (repeat for each snapshot) - * v | - * WRITE_OBJECT ----/ + * READ_OBJECT <--------\ + * | | (repeat for each snapshot) + * v | + * WRITE_OBJECT --------/ + * | + * | /-----------\ + * | | | (repeat for each snapshot) + * v v | + * UPDATE_OBJECT_MAP ---/ (skip if object + * | map disabled) * | * v * @@ -78,6 +84,7 @@ private: typedef std::tuple SyncOp; typedef std::list SyncOps; typedef std::map SnapSyncOps; + typedef std::map SnapObjectStates; ImageCtxT *m_local_image_ctx; ImageCtxT *m_remote_image_ctx; @@ -94,6 +101,7 @@ private: int m_snap_ret; SnapSyncOps m_snap_sync_ops; + SnapObjectStates m_snap_object_states; void send_list_snaps(); void handle_list_snaps(int r); @@ -104,6 +112,9 @@ private: void send_write_object(); void handle_write_object(int r); + void send_update_object_map(); + void handle_update_object_map(int r); + void compute_diffs(); void finish(int r);