From 330f2a7bb94fff4555280e7fc5ad85dbdcfaaaea Mon Sep 17 00:00:00 2001 From: Jason Dillaman Date: Tue, 10 Mar 2020 16:02:20 -0400 Subject: [PATCH] librbd: helper state machine for computing diffs between object-maps This is a port of the existing synchronous logic from the diff iterate API but converted to be asynchrous. The goal is to re-use this logic when performing a deep-copy image sync to skip any objects that have not been updated. Signed-off-by: Jason Dillaman --- src/librbd/CMakeLists.txt | 1 + src/librbd/object_map/DiffRequest.cc | 210 +++++++++++++++++++++++++++ src/librbd/object_map/DiffRequest.h | 87 +++++++++++ src/librbd/object_map/Types.h | 19 +++ 4 files changed, 317 insertions(+) create mode 100644 src/librbd/object_map/DiffRequest.cc create mode 100644 src/librbd/object_map/DiffRequest.h create mode 100644 src/librbd/object_map/Types.h diff --git a/src/librbd/CMakeLists.txt b/src/librbd/CMakeLists.txt index 9eba807f75b92..bcdc8af8e2683 100644 --- a/src/librbd/CMakeLists.txt +++ b/src/librbd/CMakeLists.txt @@ -121,6 +121,7 @@ set(librbd_internal_srcs mirror/snapshot/Utils.cc mirror/snapshot/WriteImageStateRequest.cc object_map/CreateRequest.cc + object_map/DiffRequest.cc object_map/InvalidateRequest.cc object_map/LockRequest.cc object_map/RefreshRequest.cc diff --git a/src/librbd/object_map/DiffRequest.cc b/src/librbd/object_map/DiffRequest.cc new file mode 100644 index 0000000000000..bd279de902390 --- /dev/null +++ b/src/librbd/object_map/DiffRequest.cc @@ -0,0 +1,210 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/object_map/DiffRequest.h" +#include "common/debug.h" +#include "common/errno.h" +#include "librbd/ImageCtx.h" +#include "librbd/ObjectMap.h" +#include "librbd/Utils.h" +#include "osdc/Striper.h" +#include + +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "librbd::object_map::DiffRequest: " \ + << this << " " << __func__ << ": " + +namespace librbd { +namespace object_map { + +using util::create_rados_callback; + +template +void DiffRequest::send() { + std::shared_lock image_locker{m_image_ctx->image_lock}; + + m_diff_from_start = (m_snap_id_start == 0); + if (m_snap_id_start == 0) { + if (!m_image_ctx->snap_info.empty()) { + m_snap_id_start = m_image_ctx->snap_info.begin()->first; + } else { + m_snap_id_start = CEPH_NOSNAP; + } + } + + m_object_diff_state->clear(); + m_current_snap_id = m_snap_id_start; + m_next_snap_id = m_snap_id_end; + + load_object_map(&image_locker); +} + +template +void DiffRequest::load_object_map( + std::shared_lock* image_locker) { + ceph_assert(ceph_mutex_is_locked(m_image_ctx->image_lock)); + + auto cct = m_image_ctx->cct; + ldout(cct, 10) << dendl; + + m_current_size = m_image_ctx->size; + if (m_current_snap_id != CEPH_NOSNAP) { + auto snap_it = m_image_ctx->snap_info.find(m_current_snap_id); + ceph_assert(snap_it != m_image_ctx->snap_info.end()); + m_current_size = snap_it->second.size; + + ++snap_it; + if (snap_it != m_image_ctx->snap_info.end()) { + m_next_snap_id = snap_it->first; + } else { + m_next_snap_id = CEPH_NOSNAP; + } + } + + if ((m_image_ctx->features & RBD_FEATURE_FAST_DIFF) != 0) { + image_locker->unlock(); + + ldout(cct, 10) << "fast-diff feature not enabled" << dendl; + finish(-EINVAL); + return; + } + + uint64_t flags = 0; + int r = m_image_ctx->get_flags(m_current_snap_id, &flags); + if (r < 0) { + image_locker->unlock(); + + lderr(cct) << "failed to retrieve image flags: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + if ((flags & RBD_FLAG_FAST_DIFF_INVALID) != 0) { + image_locker->unlock(); + + ldout(cct, 1) << "cannot perform fast diff on invalid object map" + << dendl; + finish(-EINVAL); + return; + } + + std::string oid(ObjectMap<>::object_map_name(m_image_ctx->id, + m_current_snap_id)); + + librados::ObjectReadOperation op; + cls_client::object_map_load_start(&op); + + m_out_bl.clear(); + auto aio_comp = create_rados_callback< + DiffRequest, &DiffRequest::handle_load_object_map>(this); + r = m_image_ctx->md_ctx.aio_operate(oid, aio_comp, &op, &m_out_bl); + ceph_assert(r == 0); + aio_comp->release(); +} + +template +void DiffRequest::handle_load_object_map(int r) { + auto cct = m_image_ctx->cct; + ldout(cct, 10) << "r=" << r << dendl; + + if (r == 0) { + auto bl_it = m_out_bl.cbegin(); + r = cls_client::object_map_load_finish(&bl_it, &m_object_map); + } + + std::string oid(ObjectMap<>::object_map_name(m_image_ctx->id, + m_current_snap_id)); + if (r < 0) { + lderr(cct) << "failed to load object map: " << oid << dendl; + finish(r); + return; + } + ldout(cct, 20) << "loaded object map " << oid << dendl; + + uint64_t num_objs = Striper::get_num_objects(m_image_ctx->layout, + m_current_size); + if (m_object_map.size() < num_objs) { + ldout(cct, 1) << "object map too small: " + << m_object_map.size() << " < " << num_objs << dendl; + finish(-EINVAL); + return; + } else { + m_object_map.resize(num_objs); + } + + if (m_object_diff_state->size() < num_objs) { + // the diff state should be the largest of all snapshots in the set + m_object_diff_state->resize(num_objs); + } + if (m_object_map.size() < m_object_diff_state->size()) { + // the image was shrunk so expanding the object map will flag end objects + // as non-existent and they will be compared against the previous object + // map + m_object_map.resize(m_object_diff_state->size()); + } + + uint64_t overlap = std::min(m_object_map.size(), m_prev_object_map.size()); + auto it = m_object_map.begin(); + auto overlap_end_it = it + overlap; + auto pre_it = m_prev_object_map.begin(); + auto diff_it = m_object_diff_state->begin(); + uint64_t i = 0; + for (; it != overlap_end_it; ++it, ++pre_it, ++diff_it, ++i) { + ldout(cct, 20) << "object state: " << i << " " + << static_cast(*pre_it) + << "->" << static_cast(*it) << dendl; + if (*it == OBJECT_NONEXISTENT) { + if (*pre_it != OBJECT_NONEXISTENT) { + *diff_it = DIFF_STATE_HOLE; + } + } else if (*it == OBJECT_EXISTS || + (*pre_it != *it && + !(*pre_it == OBJECT_EXISTS && + *it == OBJECT_EXISTS_CLEAN))) { + *diff_it = DIFF_STATE_UPDATED; + } + } + ldout(cct, 20) << "computed overlap diffs" << dendl; + + auto end_it = m_object_map.end(); + if (m_object_map.size() > m_prev_object_map.size() && + (m_diff_from_start || m_prev_object_map_valid)) { + for (; it != end_it; ++it,++diff_it, ++i) { + ldout(cct, 20) << "object state: " << i << " " + << "->" << static_cast(*it) << dendl; + if (*it == OBJECT_NONEXISTENT) { + *diff_it = DIFF_STATE_NONE; + } else { + *diff_it = DIFF_STATE_UPDATED; + } + } + } + ldout(cct, 20) << "computed resize diffs" << dendl; + + if (m_current_snap_id == m_next_snap_id || m_next_snap_id > m_snap_id_end) { + finish(0); + return; + } + + m_current_snap_id = m_next_snap_id; + m_prev_object_map = m_object_map; + m_prev_object_map_valid = true; + + std::shared_lock image_locker{m_image_ctx->image_lock}; + load_object_map(&image_locker); +} + +template +void DiffRequest::finish(int r) { + auto cct = m_image_ctx->cct; + ldout(cct, 10) << "r=" << r << dendl; + + m_on_finish->complete(r); + delete this; +} + +} // namespace object_map +} // namespace librbd + +template class librbd::object_map::DiffRequest; diff --git a/src/librbd/object_map/DiffRequest.h b/src/librbd/object_map/DiffRequest.h new file mode 100644 index 0000000000000..e80a02d99e6f6 --- /dev/null +++ b/src/librbd/object_map/DiffRequest.h @@ -0,0 +1,87 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_OBJECT_MAP_DIFF_REQUEST_H +#define CEPH_LIBRBD_OBJECT_MAP_DIFF_REQUEST_H + +#include "include/int_types.h" +#include "common/bit_vector.hpp" +#include "common/ceph_mutex.h" +#include "librbd/object_map/Types.h" + +struct Context; + +namespace librbd { + +struct ImageCtx; + +namespace object_map { + +template +class DiffRequest { +public: + static DiffRequest* create(ImageCtxT* image_ctx, uint64_t snap_id_start, + uint64_t snap_id_end, + BitVector<2>* object_diff_state, + Context* on_finish) { + return new DiffRequest(image_ctx, snap_id_start, snap_id_end, + object_diff_state, on_finish); + } + + DiffRequest(ImageCtxT* image_ctx, uint64_t snap_id_start, + uint64_t snap_id_end, BitVector<2>* object_diff_state, + Context* on_finish) + : m_image_ctx(image_ctx), m_snap_id_start(snap_id_start), + m_snap_id_end(snap_id_end), m_object_diff_state(object_diff_state), + m_on_finish(on_finish) { + } + + void send(); + +private: + /** + * @verbatim + * + * + * | + * | /---------\ + * | | | + * v v | + * LOAD_OBJECT_MAP ---/ + * | + * v + * + * + * @endverbatim + */ + ImageCtxT* m_image_ctx; + uint64_t m_snap_id_start; + uint64_t m_snap_id_end; + BitVector<2>* m_object_diff_state; + Context* m_on_finish; + + bool m_diff_from_start = false; + uint64_t m_current_snap_id = 0; + uint64_t m_next_snap_id = 0; + + uint64_t m_current_size = 0; + + BitVector<2> m_object_map; + BitVector<2> m_prev_object_map; + bool m_prev_object_map_valid = false; + + bufferlist m_out_bl; + + void load_object_map(std::shared_lock* image_locker); + void handle_load_object_map(int r); + + void finish(int r); + +}; + +} // namespace object_map +} // namespace librbd + +extern template class librbd::object_map::DiffRequest; + +#endif // CEPH_LIBRBD_OBJECT_MAP_DIFF_REQUEST_H diff --git a/src/librbd/object_map/Types.h b/src/librbd/object_map/Types.h new file mode 100644 index 0000000000000..b3741929d771b --- /dev/null +++ b/src/librbd/object_map/Types.h @@ -0,0 +1,19 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_OBJECT_MAP_TYPES_H +#define CEPH_LIBRBD_OBJECT_MAP_TYPES_H + +namespace librbd { +namespace object_map { + +enum DiffState { + DIFF_STATE_NONE = 0, + DIFF_STATE_UPDATED = 1, + DIFF_STATE_HOLE = 2 +}; + +} // namespace object_map +} // namespace librbd + +#endif // CEPH_LIBRBD_OBJECT_MAP_TYPES_H -- 2.47.3