rbd-mirror: image sync state machine

Signed-off-by: Jason Dillaman <dillaman@redhat.com>

diff --git a/src/tools/Makefile-client.am b/src/tools/Makefile-client.am
index e04ba46cc8bdd1b3d4d6daed05977f7a2c647ddd..099832db03a1cd6f946c5b3d160b3cbfefe89980 100644 (file)
--- a/src/tools/Makefile-client.am
+++ b/src/tools/Makefile-client.am
@@ -83,6 +83,7 @@ endif # LINUX
 librbd_mirror_internal_la_SOURCES = \
        tools/rbd_mirror/ClusterWatcher.cc \
        tools/rbd_mirror/ImageReplayer.cc \
+       tools/rbd_mirror/ImageSync.cc \
        tools/rbd_mirror/Mirror.cc \
        tools/rbd_mirror/PoolWatcher.cc \
        tools/rbd_mirror/Replayer.cc \
@@ -97,6 +98,7 @@ noinst_LTLIBRARIES += librbd_mirror_internal.la
 noinst_HEADERS += \
        tools/rbd_mirror/ClusterWatcher.h \
        tools/rbd_mirror/ImageReplayer.h \
+       tools/rbd_mirror/ImageSync.h \
        tools/rbd_mirror/Mirror.h \
        tools/rbd_mirror/PoolWatcher.h \
        tools/rbd_mirror/Replayer.h \

diff --git a/src/tools/rbd_mirror/ImageSync.cc b/src/tools/rbd_mirror/ImageSync.cc
new file mode 100644 (file)
index 0000000..6f16da6
--- /dev/null
+++ b/src/tools/rbd_mirror/ImageSync.cc
@@ -0,0 +1,243 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "ImageSync.h"
+#include "common/errno.h"
+#include "journal/Journaler.h"
+#include "librbd/ImageCtx.h"
+#include "librbd/Utils.h"
+#include "librbd/journal/Types.h"
+#include "tools/rbd_mirror/image_sync/ImageCopyRequest.h"
+#include "tools/rbd_mirror/image_sync/SnapshotCopyRequest.h"
+#include "tools/rbd_mirror/image_sync/SyncPointCreateRequest.h"
+#include "tools/rbd_mirror/image_sync/SyncPointPruneRequest.h"
+
+#define dout_subsys ceph_subsys_rbd_mirror
+#undef dout_prefix
+#define dout_prefix *_dout << "rbd::mirror::ImageSync: " \
+                           << this << " " << __func__
+
+namespace rbd {
+namespace mirror {
+
+using namespace image_sync;
+using librbd::util::create_context_callback;
+using librbd::util::unique_lock_name;
+
+template <typename I>
+ImageSync<I>::ImageSync(I *local_image_ctx, I *remote_image_ctx,
+                        SafeTimer *timer, Mutex *timer_lock,
+                        const std::string &mirror_uuid, Journaler *journaler,
+                        MirrorPeerClientMeta *client_meta, Context *on_finish)
+  : m_local_image_ctx(local_image_ctx), m_remote_image_ctx(remote_image_ctx),
+    m_timer(timer), m_timer_lock(timer_lock), m_mirror_uuid(mirror_uuid),
+    m_journaler(journaler), m_client_meta(client_meta), m_on_finish(on_finish),
+    m_lock(unique_lock_name("ImageSync::m_lock", this)) {
+}
+
+template <typename I>
+void ImageSync<I>::start() {
+  send_prune_catch_up_sync_point();
+}
+
+template <typename I>
+void ImageSync<I>::cancel() {
+  Mutex::Locker locker(m_lock);
+
+  CephContext *cct = m_local_image_ctx->cct;
+  ldout(cct, 20) << dendl;
+
+  m_canceled = true;
+  if (m_image_copy_request != nullptr) {
+    m_image_copy_request->cancel();
+  }
+}
+
+template <typename I>
+void ImageSync<I>::send_prune_catch_up_sync_point() {
+  if (m_client_meta->sync_points.size() <= 1) {
+    send_create_sync_point();
+    return;
+  }
+
+  CephContext *cct = m_local_image_ctx->cct;
+  ldout(cct, 20) << dendl;
+
+  Context *ctx = create_context_callback<
+    ImageSync<I>, &ImageSync<I>::handle_prune_catch_up_sync_point>(this);
+  SyncPointPruneRequest<I> *request = SyncPointPruneRequest<I>::create(
+    m_remote_image_ctx, false, m_journaler, m_client_meta, ctx);
+  request->send();
+}
+
+template <typename I>
+void ImageSync<I>::handle_prune_catch_up_sync_point(int r) {
+  CephContext *cct = m_local_image_ctx->cct;
+  ldout(cct, 20) << ": r=" << r << dendl;
+
+  if (r < 0) {
+    lderr(cct) << "failed to prune catch-up sync point: "
+               << cpp_strerror(r) << dendl;
+    finish(r);
+    return;
+  }
+
+  send_create_sync_point();
+}
+
+template <typename I>
+void ImageSync<I>::send_create_sync_point() {
+  // TODO: when support for disconnecting laggy clients is added,
+  //       re-connect and create catch-up sync point
+  if (m_client_meta->sync_points.size() > 0) {
+    send_copy_snapshots();
+    return;
+  }
+
+  CephContext *cct = m_local_image_ctx->cct;
+  ldout(cct, 20) << dendl;
+
+  Context *ctx = create_context_callback<
+    ImageSync<I>, &ImageSync<I>::handle_create_sync_point>(this);
+  SyncPointCreateRequest<I> *request = SyncPointCreateRequest<I>::create(
+    m_remote_image_ctx, m_mirror_uuid, m_journaler, m_client_meta, ctx);
+  request->send();
+}
+
+template <typename I>
+void ImageSync<I>::handle_create_sync_point(int r) {
+  CephContext *cct = m_local_image_ctx->cct;
+  ldout(cct, 20) << ": r=" << r << dendl;
+
+  if (r < 0) {
+    lderr(cct) << "failed to create sync point: " << cpp_strerror(r)
+               << dendl;
+    finish(r);
+    return;
+  }
+
+  send_copy_snapshots();
+}
+
+template <typename I>
+void ImageSync<I>::send_copy_snapshots() {
+  CephContext *cct = m_local_image_ctx->cct;
+  ldout(cct, 20) << dendl;
+
+  Context *ctx = create_context_callback<
+    ImageSync<I>, &ImageSync<I>::handle_copy_snapshots>(this);
+  SnapshotCopyRequest<I> *request = SnapshotCopyRequest<I>::create(
+    m_local_image_ctx, m_remote_image_ctx, &m_snap_map, m_journaler,
+    m_client_meta, ctx);
+  request->send();
+}
+
+template <typename I>
+void ImageSync<I>::handle_copy_snapshots(int r) {
+  CephContext *cct = m_local_image_ctx->cct;
+  ldout(cct, 20) << ": r=" << r << dendl;
+
+  if (r < 0) {
+    lderr(cct) << "failed to copy snapshot metadata: "
+               << cpp_strerror(r) << dendl;
+    finish(r);
+    return;
+  }
+
+  send_copy_image();
+}
+
+template <typename I>
+void ImageSync<I>::send_copy_image() {
+  m_lock.Lock();
+  if (m_canceled) {
+    m_lock.Unlock();
+    finish(-ECANCELED);
+    return;
+  }
+
+  CephContext *cct = m_local_image_ctx->cct;
+  ldout(cct, 20) << dendl;
+
+  Context *ctx = create_context_callback<
+    ImageSync<I>, &ImageSync<I>::handle_copy_image>(this);
+  m_image_copy_request = ImageCopyRequest<I>::create(
+    m_local_image_ctx, m_remote_image_ctx, m_timer, m_timer_lock,
+    m_journaler, m_client_meta, &m_client_meta->sync_points.front(),
+    ctx);
+  m_lock.Unlock();
+
+  m_image_copy_request->send();
+}
+
+template <typename I>
+void ImageSync<I>::handle_copy_image(int r) {
+  {
+    Mutex::Locker locker(m_lock);
+    m_image_copy_request = nullptr;
+    if (r == 0 && m_canceled) {
+      r = -ECANCELED;
+    }
+  }
+
+  CephContext *cct = m_local_image_ctx->cct;
+  ldout(cct, 20) << ": r=" << r << dendl;
+
+  if (r == -ECANCELED) {
+    ldout(cct, 10) << "image copy canceled" << dendl;
+    finish(r);
+    return;
+  } else if (r < 0) {
+    lderr(cct) << "failed to copy image: " << cpp_strerror(r) << dendl;
+    finish(r);
+    return;
+  }
+
+  send_prune_sync_points();
+}
+
+template <typename I>
+void ImageSync<I>::send_prune_sync_points() {
+  CephContext *cct = m_local_image_ctx->cct;
+  ldout(cct, 20) << dendl;
+
+  Context *ctx = create_context_callback<
+    ImageSync<I>, &ImageSync<I>::handle_prune_sync_points>(this);
+  SyncPointPruneRequest<I> *request = SyncPointPruneRequest<I>::create(
+    m_remote_image_ctx, true, m_journaler, m_client_meta, ctx);
+  request->send();
+}
+
+template <typename I>
+void ImageSync<I>::handle_prune_sync_points(int r) {
+  CephContext *cct = m_local_image_ctx->cct;
+  ldout(cct, 20) << ": r=" << r << dendl;
+
+  if (r < 0) {
+    lderr(cct) << "failed to prune sync point: "
+               << cpp_strerror(r) << dendl;
+    finish(r);
+    return;
+  }
+
+  if (!m_client_meta->sync_points.empty()) {
+    send_copy_image();
+    return;
+  }
+
+  finish(0);
+}
+
+template <typename I>
+void ImageSync<I>::finish(int r) {
+  CephContext *cct = m_local_image_ctx->cct;
+  ldout(cct, 20) << ": r=" << r << dendl;
+
+  m_on_finish->complete(r);
+  delete this;
+}
+
+} // namespace mirror
+} // namespace rbd
+
+template class rbd::mirror::ImageSync<librbd::ImageCtx>;

diff --git a/src/tools/rbd_mirror/ImageSync.h b/src/tools/rbd_mirror/ImageSync.h
new file mode 100644 (file)
index 0000000..b342141
--- /dev/null
+++ b/src/tools/rbd_mirror/ImageSync.h
@@ -0,0 +1,109 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef RBD_MIRROR_IMAGE_SYNC_H
+#define RBD_MIRROR_IMAGE_SYNC_H
+
+#include "include/int_types.h"
+#include "librbd/Journal.h"
+#include "common/Mutex.h"
+#include <map>
+#include <vector>
+
+class Context;
+class Mutex;
+class SafeTimer;
+namespace journal { class Journaler; }
+namespace librbd { class ImageCtx; }
+namespace librbd { namespace journal { struct MirrorPeerClientMeta; } }
+
+namespace rbd {
+namespace mirror {
+
+namespace image_sync { template <typename> class ImageCopyRequest; }
+
+template <typename ImageCtxT = librbd::ImageCtx>
+class ImageSync {
+public:
+  typedef librbd::journal::TypeTraits<ImageCtxT> TypeTraits;
+  typedef typename TypeTraits::Journaler Journaler;
+  typedef librbd::journal::MirrorPeerClientMeta MirrorPeerClientMeta;
+
+  ImageSync(ImageCtxT *local_image_ctx, ImageCtxT *remote_image_ctx,
+            SafeTimer *timer, Mutex *timer_lock, const std::string &mirror_uuid,
+            Journaler *journaler, MirrorPeerClientMeta *client_meta,
+            Context *on_finish);
+
+  void start();
+  void cancel();
+
+private:
+  /**
+   * @verbatim
+   *
+   * <start>
+   *    |
+   *    v
+   * PRUNE_CATCH_UP_SYNC_POINT
+   *    |
+   *    v
+   * CREATE_SYNC_POINT (skip if already exists and
+   *    |               not disconnected)
+   *    v
+   * COPY_SNAPSHOTS
+   *    |
+   *    v
+   * COPY_IMAGE . . . . . .
+   *    |                 .
+   *    v                 .
+   * PRUNE_SYNC_POINTS    . (image sync canceled)
+   *    |                 .
+   *    v                 .
+   * <finish> < . . . . . .
+   *
+   * @endverbatim
+   */
+
+  typedef std::vector<librados::snap_t> SnapIds;
+  typedef std::map<librados::snap_t, SnapIds> SnapMap;
+
+  ImageCtxT *m_local_image_ctx;
+  ImageCtxT *m_remote_image_ctx;
+  SafeTimer *m_timer;
+  Mutex *m_timer_lock;
+  std::string m_mirror_uuid;
+  Journaler *m_journaler;
+  MirrorPeerClientMeta *m_client_meta;
+  Context *m_on_finish;
+
+  SnapMap m_snap_map;
+
+  Mutex m_lock;
+  bool m_canceled = false;
+
+  image_sync::ImageCopyRequest<ImageCtxT> *m_image_copy_request;
+
+  void send_prune_catch_up_sync_point();
+  void handle_prune_catch_up_sync_point(int r);
+
+  void send_create_sync_point();
+  void handle_create_sync_point(int r);
+
+  void send_copy_snapshots();
+  void handle_copy_snapshots(int r);
+
+  void send_copy_image();
+  void handle_copy_image(int r);
+
+  void send_prune_sync_points();
+  void handle_prune_sync_points(int r);
+
+  void finish(int r);
+};
+
+} // namespace mirror
+} // namespace rbd
+
+extern template class rbd::mirror::ImageSync<librbd::ImageCtx>;
+
+#endif // RBD_MIRROR_IMAGE_SYNC_H