librbd: helper state machine for computing diffs between object-maps

This is a port of the existing synchronous logic from the diff iterate
API but converted to be asynchrous. The goal is to re-use this logic
when performing a deep-copy image sync to skip any objects that have
not been updated.

Signed-off-by: Jason Dillaman <dillaman@redhat.com>

diff --git a/src/librbd/CMakeLists.txt b/src/librbd/CMakeLists.txt
index 9eba807f75b92c1171032536f1ac8cf2d54aaae6..bcdc8af8e26839bf5b6851e236f88af3fc0fabb0 100644 (file)
--- a/src/librbd/CMakeLists.txt
+++ b/src/librbd/CMakeLists.txt
@@ -121,6 +121,7 @@ set(librbd_internal_srcs
   mirror/snapshot/Utils.cc
   mirror/snapshot/WriteImageStateRequest.cc
   object_map/CreateRequest.cc
+  object_map/DiffRequest.cc
   object_map/InvalidateRequest.cc
   object_map/LockRequest.cc
   object_map/RefreshRequest.cc

diff --git a/src/librbd/object_map/DiffRequest.cc b/src/librbd/object_map/DiffRequest.cc
new file mode 100644 (file)
index 0000000..bd279de
--- /dev/null
+++ b/src/librbd/object_map/DiffRequest.cc
@@ -0,0 +1,210 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "librbd/object_map/DiffRequest.h"
+#include "common/debug.h"
+#include "common/errno.h"
+#include "librbd/ImageCtx.h"
+#include "librbd/ObjectMap.h"
+#include "librbd/Utils.h"
+#include "osdc/Striper.h"
+#include <string>
+
+#define dout_subsys ceph_subsys_rbd
+#undef dout_prefix
+#define dout_prefix *_dout << "librbd::object_map::DiffRequest: " \
+                           << this << " " << __func__ << ": "
+
+namespace librbd {
+namespace object_map {
+
+using util::create_rados_callback;
+
+template <typename I>
+void DiffRequest<I>::send() {
+  std::shared_lock image_locker{m_image_ctx->image_lock};
+
+  m_diff_from_start = (m_snap_id_start == 0);
+  if (m_snap_id_start == 0) {
+    if (!m_image_ctx->snap_info.empty()) {
+      m_snap_id_start = m_image_ctx->snap_info.begin()->first;
+    } else {
+      m_snap_id_start = CEPH_NOSNAP;
+    }
+  }
+
+  m_object_diff_state->clear();
+  m_current_snap_id = m_snap_id_start;
+  m_next_snap_id = m_snap_id_end;
+
+  load_object_map(&image_locker);
+}
+
+template <typename I>
+void DiffRequest<I>::load_object_map(
+    std::shared_lock<ceph::shared_mutex>* image_locker) {
+  ceph_assert(ceph_mutex_is_locked(m_image_ctx->image_lock));
+
+  auto cct = m_image_ctx->cct;
+  ldout(cct, 10) << dendl;
+
+  m_current_size = m_image_ctx->size;
+  if (m_current_snap_id != CEPH_NOSNAP) {
+    auto snap_it = m_image_ctx->snap_info.find(m_current_snap_id);
+    ceph_assert(snap_it != m_image_ctx->snap_info.end());
+    m_current_size = snap_it->second.size;
+
+    ++snap_it;
+    if (snap_it != m_image_ctx->snap_info.end()) {
+      m_next_snap_id = snap_it->first;
+    } else {
+      m_next_snap_id = CEPH_NOSNAP;
+    }
+  }
+
+  if ((m_image_ctx->features & RBD_FEATURE_FAST_DIFF) != 0) {
+    image_locker->unlock();
+
+    ldout(cct, 10) << "fast-diff feature not enabled" << dendl;
+    finish(-EINVAL);
+    return;
+  }
+
+  uint64_t flags = 0;
+  int r = m_image_ctx->get_flags(m_current_snap_id, &flags);
+  if (r < 0) {
+    image_locker->unlock();
+
+    lderr(cct) << "failed to retrieve image flags: " << cpp_strerror(r)
+               << dendl;
+    finish(r);
+    return;
+  }
+  if ((flags & RBD_FLAG_FAST_DIFF_INVALID) != 0) {
+    image_locker->unlock();
+
+    ldout(cct, 1) << "cannot perform fast diff on invalid object map"
+                  << dendl;
+    finish(-EINVAL);
+    return;
+  }
+
+  std::string oid(ObjectMap<>::object_map_name(m_image_ctx->id,
+                                               m_current_snap_id));
+
+  librados::ObjectReadOperation op;
+  cls_client::object_map_load_start(&op);
+
+  m_out_bl.clear();
+  auto aio_comp = create_rados_callback<
+    DiffRequest<I>, &DiffRequest<I>::handle_load_object_map>(this);
+  r = m_image_ctx->md_ctx.aio_operate(oid, aio_comp, &op, &m_out_bl);
+  ceph_assert(r == 0);
+  aio_comp->release();
+}
+
+template <typename I>
+void DiffRequest<I>::handle_load_object_map(int r) {
+  auto cct = m_image_ctx->cct;
+  ldout(cct, 10) << "r=" << r << dendl;
+
+  if (r == 0) {
+    auto bl_it = m_out_bl.cbegin();
+    r = cls_client::object_map_load_finish(&bl_it, &m_object_map);
+  }
+
+  std::string oid(ObjectMap<>::object_map_name(m_image_ctx->id,
+                                               m_current_snap_id));
+  if (r < 0) {
+    lderr(cct) << "failed to load object map: " << oid << dendl;
+    finish(r);
+    return;
+  }
+  ldout(cct, 20) << "loaded object map " << oid << dendl;
+
+  uint64_t num_objs = Striper::get_num_objects(m_image_ctx->layout,
+                                               m_current_size);
+  if (m_object_map.size() < num_objs) {
+    ldout(cct, 1) << "object map too small: "
+                  << m_object_map.size() << " < " << num_objs << dendl;
+    finish(-EINVAL);
+    return;
+  } else {
+    m_object_map.resize(num_objs);
+  }
+
+  if (m_object_diff_state->size() < num_objs) {
+    // the diff state should be the largest of all snapshots in the set
+    m_object_diff_state->resize(num_objs);
+  }
+  if (m_object_map.size() < m_object_diff_state->size()) {
+    // the image was shrunk so expanding the object map will flag end objects
+    // as non-existent and they will be compared against the previous object
+    // map
+    m_object_map.resize(m_object_diff_state->size());
+  }
+
+  uint64_t overlap = std::min(m_object_map.size(), m_prev_object_map.size());
+  auto it = m_object_map.begin();
+  auto overlap_end_it = it + overlap;
+  auto pre_it = m_prev_object_map.begin();
+  auto diff_it = m_object_diff_state->begin();
+  uint64_t i = 0;
+  for (; it != overlap_end_it; ++it, ++pre_it, ++diff_it, ++i) {
+    ldout(cct, 20) << "object state: " << i << " "
+                   << static_cast<uint32_t>(*pre_it)
+                   << "->" << static_cast<uint32_t>(*it) << dendl;
+    if (*it == OBJECT_NONEXISTENT) {
+      if (*pre_it != OBJECT_NONEXISTENT) {
+        *diff_it = DIFF_STATE_HOLE;
+      }
+    } else if (*it == OBJECT_EXISTS ||
+               (*pre_it != *it &&
+                !(*pre_it == OBJECT_EXISTS &&
+                  *it == OBJECT_EXISTS_CLEAN))) {
+      *diff_it = DIFF_STATE_UPDATED;
+    }
+  }
+  ldout(cct, 20) << "computed overlap diffs" << dendl;
+
+  auto end_it = m_object_map.end();
+  if (m_object_map.size() > m_prev_object_map.size() &&
+      (m_diff_from_start || m_prev_object_map_valid)) {
+    for (; it != end_it; ++it,++diff_it, ++i) {
+      ldout(cct, 20) << "object state: " << i << " "
+                     << "->" << static_cast<uint32_t>(*it) << dendl;
+      if (*it == OBJECT_NONEXISTENT) {
+        *diff_it = DIFF_STATE_NONE;
+      } else {
+        *diff_it = DIFF_STATE_UPDATED;
+      }
+    }
+  }
+  ldout(cct, 20) << "computed resize diffs" << dendl;
+
+  if (m_current_snap_id == m_next_snap_id || m_next_snap_id > m_snap_id_end) {
+    finish(0);
+    return;
+  }
+
+  m_current_snap_id = m_next_snap_id;
+  m_prev_object_map = m_object_map;
+  m_prev_object_map_valid = true;
+
+  std::shared_lock image_locker{m_image_ctx->image_lock};
+  load_object_map(&image_locker);
+}
+
+template <typename I>
+void DiffRequest<I>::finish(int r) {
+  auto cct = m_image_ctx->cct;
+  ldout(cct, 10) << "r=" << r << dendl;
+
+  m_on_finish->complete(r);
+  delete this;
+}
+
+} // namespace object_map
+} // namespace librbd
+
+template class librbd::object_map::DiffRequest<librbd::ImageCtx>;

diff --git a/src/librbd/object_map/DiffRequest.h b/src/librbd/object_map/DiffRequest.h
new file mode 100644 (file)
index 0000000..e80a02d
--- /dev/null
+++ b/src/librbd/object_map/DiffRequest.h
@@ -0,0 +1,87 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CEPH_LIBRBD_OBJECT_MAP_DIFF_REQUEST_H
+#define CEPH_LIBRBD_OBJECT_MAP_DIFF_REQUEST_H
+
+#include "include/int_types.h"
+#include "common/bit_vector.hpp"
+#include "common/ceph_mutex.h"
+#include "librbd/object_map/Types.h"
+
+struct Context;
+
+namespace librbd {
+
+struct ImageCtx;
+
+namespace object_map {
+
+template <typename ImageCtxT>
+class DiffRequest {
+public:
+  static DiffRequest* create(ImageCtxT* image_ctx, uint64_t snap_id_start,
+                             uint64_t snap_id_end,
+                             BitVector<2>* object_diff_state,
+                             Context* on_finish) {
+    return new DiffRequest(image_ctx, snap_id_start, snap_id_end,
+                           object_diff_state, on_finish);
+  }
+
+  DiffRequest(ImageCtxT* image_ctx, uint64_t snap_id_start,
+              uint64_t snap_id_end, BitVector<2>* object_diff_state,
+              Context* on_finish)
+    : m_image_ctx(image_ctx), m_snap_id_start(snap_id_start),
+      m_snap_id_end(snap_id_end), m_object_diff_state(object_diff_state),
+      m_on_finish(on_finish) {
+  }
+
+  void send();
+
+private:
+  /**
+   * @verbatim
+   *
+   * <start>
+   *    |
+   *    |     /---------\
+   *    |     |         |
+   *    v     v         |
+   * LOAD_OBJECT_MAP ---/
+   *    |
+   *    v
+   * <finish>
+   *
+   * @endverbatim
+   */
+  ImageCtxT* m_image_ctx;
+  uint64_t m_snap_id_start;
+  uint64_t m_snap_id_end;
+  BitVector<2>* m_object_diff_state;
+  Context* m_on_finish;
+
+  bool m_diff_from_start = false;
+  uint64_t m_current_snap_id = 0;
+  uint64_t m_next_snap_id = 0;
+
+  uint64_t m_current_size = 0;
+
+  BitVector<2> m_object_map;
+  BitVector<2> m_prev_object_map;
+  bool m_prev_object_map_valid = false;
+
+  bufferlist m_out_bl;
+
+  void load_object_map(std::shared_lock<ceph::shared_mutex>* image_locker);
+  void handle_load_object_map(int r);
+
+  void finish(int r);
+
+};
+
+} // namespace object_map
+} // namespace librbd
+
+extern template class librbd::object_map::DiffRequest<librbd::ImageCtx>;
+
+#endif // CEPH_LIBRBD_OBJECT_MAP_DIFF_REQUEST_H

diff --git a/src/librbd/object_map/Types.h b/src/librbd/object_map/Types.h
new file mode 100644 (file)
index 0000000..b374192
--- /dev/null
+++ b/src/librbd/object_map/Types.h
@@ -0,0 +1,19 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CEPH_LIBRBD_OBJECT_MAP_TYPES_H
+#define CEPH_LIBRBD_OBJECT_MAP_TYPES_H
+
+namespace librbd {
+namespace object_map {
+
+enum DiffState {
+  DIFF_STATE_NONE    = 0,
+  DIFF_STATE_UPDATED = 1,
+  DIFF_STATE_HOLE    = 2
+};
+
+} // namespace object_map
+} // namespace librbd
+
+#endif // CEPH_LIBRBD_OBJECT_MAP_TYPES_H