rbd-mirror: eliminate context switch from close image state machine

Signed-off-by: Jason Dillaman <dillaman@redhat.com>

diff --git a/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc b/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc
index c6dee1859325fa6e03583a3209176b25f78240bb..cc1bfc55dc37cd4059f397f49b15b18de711b0d9 100644 (file)
--- a/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc
+++ b/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc
@@ -101,7 +101,6 @@ struct CloseImageRequest<librbd::MockTestImageCtx> {
   Context *on_finish = nullptr;
 
   static CloseImageRequest* create(librbd::MockTestImageCtx **image_ctx,
-                                   ContextWQ *work_queue, bool destroy_only,
                                    Context *on_finish) {
     assert(s_instance != nullptr);
     s_instance->image_ctx = image_ctx;
@@ -184,8 +183,7 @@ struct OpenImageRequest<librbd::MockTestImageCtx> {
   static OpenImageRequest* create(librados::IoCtx &io_ctx,
                                   librbd::MockTestImageCtx **image_ctx,
                                   const std::string &image_id,
-                                  bool read_only, ContextWQ *work_queue,
-                                  Context *on_finish) {
+                                  bool read_only, Context *on_finish) {
     assert(s_instance != nullptr);
     s_instance->image_ctx = image_ctx;
     s_instance->on_finish = on_finish;

diff --git a/src/test/rbd_mirror/image_replayer/test_mock_CreateImageRequest.cc b/src/test/rbd_mirror/image_replayer/test_mock_CreateImageRequest.cc
index 297fd6bf7adf76d6e70995b32c2441baa1df85a4..455362d30d137e7b583c2b47a43a54b6d512ba13 100644 (file)
--- a/src/test/rbd_mirror/image_replayer/test_mock_CreateImageRequest.cc
+++ b/src/test/rbd_mirror/image_replayer/test_mock_CreateImageRequest.cc
@@ -118,7 +118,6 @@ struct CloseImageRequest<librbd::MockTestImageCtx> {
   Context *on_finish = nullptr;
 
   static CloseImageRequest* create(librbd::MockTestImageCtx **image_ctx,
-                                   ContextWQ *work_queue, bool destroy_only,
                                    Context *on_finish) {
     assert(s_instance != nullptr);
     s_instance->construct(*image_ctx);
@@ -147,8 +146,7 @@ struct OpenImageRequest<librbd::MockTestImageCtx> {
   static OpenImageRequest* create(librados::IoCtx &io_ctx,
                                   librbd::MockTestImageCtx **image_ctx,
                                   const std::string &image_id,
-                                  bool read_only, ContextWQ *work_queue,
-                                  Context *on_finish) {
+                                  bool read_only, Context *on_finish) {
     assert(s_instance != nullptr);
     s_instance->image_ctx = image_ctx;
     s_instance->on_finish = on_finish;

diff --git a/src/test/rbd_mirror/test_mock_ImageReplayer.cc b/src/test/rbd_mirror/test_mock_ImageReplayer.cc
index ee3e466a6d3cec3adc9f650bf1da5719c9643557..ead5d102efa083e9b7f1a089e96ea37b6e3ed5ea 100644 (file)
--- a/src/test/rbd_mirror/test_mock_ImageReplayer.cc
+++ b/src/test/rbd_mirror/test_mock_ImageReplayer.cc
@@ -147,7 +147,6 @@ struct CloseImageRequest<librbd::MockTestImageCtx> {
   Context *on_finish = nullptr;
 
   static CloseImageRequest* create(librbd::MockTestImageCtx **image_ctx,
-                                   ContextWQ *work_queue, bool destroy_only,
                                    Context *on_finish) {
     assert(s_instance != nullptr);
     s_instance->image_ctx = image_ctx;

diff --git a/src/tools/rbd_mirror/ImageReplayer.cc b/src/tools/rbd_mirror/ImageReplayer.cc
index 0ef8560d7f979bfc94f5b1587de909a4f0ae80b3..bd5036b53cecd7fba174e212f98d06982364eebf 100644 (file)
--- a/src/tools/rbd_mirror/ImageReplayer.cc
+++ b/src/tools/rbd_mirror/ImageReplayer.cc
@@ -1423,7 +1423,7 @@ void ImageReplayer<I>::shut_down(int r) {
   if (m_local_image_ctx) {
     ctx = new FunctionContext([this, ctx](int r) {
       CloseImageRequest<I> *request = CloseImageRequest<I>::create(
-        &m_local_image_ctx, m_threads->work_queue, false, ctx);
+        &m_local_image_ctx, ctx);
       request->send();
     });
   }

diff --git a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc
index bd820c208f96905915dba99db327615d7f90e217..c1d805af4c26c7b76f8cb06b43e050f6a09eeaf2 100644 (file)
--- a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc
+++ b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc
@@ -253,7 +253,7 @@ void BootstrapRequest<I>::open_remote_image() {
       this);
   OpenImageRequest<I> *request = OpenImageRequest<I>::create(
     m_remote_io_ctx, &m_remote_image_ctx, m_remote_image_id, false,
-    m_work_queue, ctx);
+    ctx);
   request->send();
 }
 
@@ -729,7 +729,7 @@ void BootstrapRequest<I>::close_local_image() {
     BootstrapRequest<I>, &BootstrapRequest<I>::handle_close_local_image>(
       this);
   CloseImageRequest<I> *request = CloseImageRequest<I>::create(
-    m_local_image_ctx, m_work_queue, false, ctx);
+    m_local_image_ctx, ctx);
   request->send();
 }
 
@@ -755,7 +755,7 @@ void BootstrapRequest<I>::close_remote_image() {
     BootstrapRequest<I>, &BootstrapRequest<I>::handle_close_remote_image>(
       this);
   CloseImageRequest<I> *request = CloseImageRequest<I>::create(
-    &m_remote_image_ctx, m_work_queue, false, ctx);
+    &m_remote_image_ctx, ctx);
   request->send();
 }
 

diff --git a/src/tools/rbd_mirror/image_replayer/CloseImageRequest.cc b/src/tools/rbd_mirror/image_replayer/CloseImageRequest.cc
index d7ea6085edbe2559ada25510f2058cdb221b10ad..234fded69ee85a53fcc6cd6d6e9ff67a83952f55 100644 (file)
--- a/src/tools/rbd_mirror/image_replayer/CloseImageRequest.cc
+++ b/src/tools/rbd_mirror/image_replayer/CloseImageRequest.cc
@@ -22,10 +22,8 @@ namespace image_replayer {
 using librbd::util::create_context_callback;
 
 template <typename I>
-CloseImageRequest<I>::CloseImageRequest(I **image_ctx, ContextWQ *work_queue,
-                                        bool destroy_only, Context *on_finish)
-  : m_image_ctx(image_ctx), m_work_queue(work_queue),
-    m_destroy_only(destroy_only), m_on_finish(on_finish) {
+CloseImageRequest<I>::CloseImageRequest(I **image_ctx, Context *on_finish)
+  : m_image_ctx(image_ctx), m_on_finish(on_finish) {
 }
 
 template <typename I>
@@ -35,11 +33,6 @@ void CloseImageRequest<I>::send() {
 
 template <typename I>
 void CloseImageRequest<I>::close_image() {
-  if (m_destroy_only) {
-    switch_thread_context();
-    return;
-  }
-
   dout(20) << dendl;
 
   Context *ctx = create_context_callback<
@@ -56,26 +49,6 @@ void CloseImageRequest<I>::handle_close_image(int r) {
          << dendl;
   }
 
-  switch_thread_context();
-}
-
-template <typename I>
-void CloseImageRequest<I>::switch_thread_context() {
-  dout(20) << dendl;
-
-  // swap the librbd thread context for the rbd-mirror thread context
-  Context *ctx = create_context_callback<
-    CloseImageRequest<I>, &CloseImageRequest<I>::handle_switch_thread_context>(
-      this);
-  m_work_queue->queue(ctx, 0);
-}
-
-template <typename I>
-void CloseImageRequest<I>::handle_switch_thread_context(int r) {
-  dout(20) << dendl;
-
-  assert(r == 0);
-
   delete *m_image_ctx;
   *m_image_ctx = nullptr;
 

diff --git a/src/tools/rbd_mirror/image_replayer/CloseImageRequest.h b/src/tools/rbd_mirror/image_replayer/CloseImageRequest.h
index dddad47220e2d64a050dff6e73b3ca785f9f8a2d..02481369d000c77203739923c7515e208cb58905 100644 (file)
--- a/src/tools/rbd_mirror/image_replayer/CloseImageRequest.h
+++ b/src/tools/rbd_mirror/image_replayer/CloseImageRequest.h
@@ -9,7 +9,6 @@
 #include <string>
 
 class Context;
-class ContextWQ;
 namespace librbd { class ImageCtx; }
 
 namespace rbd {
@@ -19,14 +18,11 @@ namespace image_replayer {
 template <typename ImageCtxT = librbd::ImageCtx>
 class CloseImageRequest {
 public:
-  static CloseImageRequest* create(ImageCtxT **image_ctx, ContextWQ *work_queue,
-                                   bool destroy_only, Context *on_finish) {
-    return new CloseImageRequest(image_ctx, work_queue, destroy_only,
-                                 on_finish);
+  static CloseImageRequest* create(ImageCtxT **image_ctx, Context *on_finish) {
+    return new CloseImageRequest(image_ctx, on_finish);
   }
 
-  CloseImageRequest(ImageCtxT **image_ctx, ContextWQ *work_queue,
-                    bool destroy_only, Context *on_finish);
+  CloseImageRequest(ImageCtxT **image_ctx, Context *on_finish);
 
   void send();
 
@@ -37,10 +33,7 @@ private:
    * <start>
    *    |
    *    v
-   * CLOSE_IMAGE (skip if not needed)
-   *    |
-   *    v
-   * SWITCH_CONTEXT
+   * CLOSE_IMAGE
    *    |
    *    v
    * <finish>
@@ -48,15 +41,10 @@ private:
    * @endverbatim
    */
   ImageCtxT **m_image_ctx;
-  ContextWQ *m_work_queue;
-  bool m_destroy_only;
   Context *m_on_finish;
 
   void close_image();
   void handle_close_image(int r);
-
-  void switch_thread_context();
-  void handle_switch_thread_context(int r);
 };
 
 } // namespace image_replayer

diff --git a/src/tools/rbd_mirror/image_replayer/CreateImageRequest.cc b/src/tools/rbd_mirror/image_replayer/CreateImageRequest.cc
index 88d856c467f729bea61df2bafdee3aee2d6a7e12..6097388c82d415fb524de49e562e99e6dd8ed3f6 100644 (file)
--- a/src/tools/rbd_mirror/image_replayer/CreateImageRequest.cc
+++ b/src/tools/rbd_mirror/image_replayer/CreateImageRequest.cc
@@ -193,7 +193,7 @@ void CreateImageRequest<I>::open_remote_parent_image() {
     &CreateImageRequest<I>::handle_open_remote_parent_image>(this);
   OpenImageRequest<I> *request = OpenImageRequest<I>::create(
     m_remote_parent_io_ctx, &m_remote_parent_image_ctx,
-    m_remote_parent_spec.image_id, true, m_work_queue, ctx);
+    m_remote_parent_spec.image_id, true, ctx);
   request->send();
 }
 
@@ -218,8 +218,8 @@ void CreateImageRequest<I>::open_local_parent_image() {
     CreateImageRequest<I>,
     &CreateImageRequest<I>::handle_open_local_parent_image>(this);
   OpenImageRequest<I> *request = OpenImageRequest<I>::create(
-    m_local_parent_io_ctx, &m_local_parent_image_ctx, m_local_parent_spec.image_id,
-    true, m_work_queue, ctx);
+    m_local_parent_io_ctx, &m_local_parent_image_ctx,
+    m_local_parent_spec.image_id, true, ctx);
   request->send();
 }
 
@@ -320,7 +320,7 @@ void CreateImageRequest<I>::close_local_parent_image() {
     CreateImageRequest<I>,
     &CreateImageRequest<I>::handle_close_local_parent_image>(this);
   CloseImageRequest<I> *request = CloseImageRequest<I>::create(
-    &m_local_parent_image_ctx, m_work_queue, false, ctx);
+    &m_local_parent_image_ctx, ctx);
   request->send();
 }
 
@@ -342,7 +342,7 @@ void CreateImageRequest<I>::close_remote_parent_image() {
     CreateImageRequest<I>,
     &CreateImageRequest<I>::handle_close_remote_parent_image>(this);
   CloseImageRequest<I> *request = CloseImageRequest<I>::create(
-    &m_remote_parent_image_ctx, m_work_queue, false, ctx);
+    &m_remote_parent_image_ctx, ctx);
   request->send();
 }
 

diff --git a/src/tools/rbd_mirror/image_replayer/OpenImageRequest.cc b/src/tools/rbd_mirror/image_replayer/OpenImageRequest.cc
index 3ffdaad6ea6bf805f7e3602eb650e6e144835475..efba012731b04b23c8aa40b99bb231edc57032ec 100644 (file)
--- a/src/tools/rbd_mirror/image_replayer/OpenImageRequest.cc
+++ b/src/tools/rbd_mirror/image_replayer/OpenImageRequest.cc
@@ -2,9 +2,7 @@
 // vim: ts=8 sw=2 smarttab
 
 #include "OpenImageRequest.h"
-#include "CloseImageRequest.h"
 #include "common/errno.h"
-#include "common/WorkQueue.h"
 #include "librbd/ImageCtx.h"
 #include "librbd/ImageState.h"
 #include "librbd/Utils.h"
@@ -25,10 +23,9 @@ using librbd::util::create_context_callback;
 template <typename I>
 OpenImageRequest<I>::OpenImageRequest(librados::IoCtx &io_ctx, I **image_ctx,
                                       const std::string &image_id,
-                                      bool read_only, ContextWQ *work_queue,
-                                      Context *on_finish)
+                                      bool read_only, Context *on_finish)
   : m_io_ctx(io_ctx), m_image_ctx(image_ctx), m_image_id(image_id),
-    m_read_only(read_only), m_work_queue(work_queue), m_on_finish(on_finish) {
+    m_read_only(read_only), m_on_finish(on_finish) {
 }
 
 template <typename I>
@@ -55,35 +52,11 @@ void OpenImageRequest<I>::handle_open_image(int r) {
   if (r < 0) {
     derr << ": failed to open image '" << m_image_id << "': "
          << cpp_strerror(r) << dendl;
-    send_close_image(r);
-    return;
+    (*m_image_ctx)->destroy();
+    *m_image_ctx = nullptr;
   }
 
-  finish(0);
-}
-
-template <typename I>
-void OpenImageRequest<I>::send_close_image(int r) {
-  dout(20) << dendl;
-
-  if (m_ret_val == 0 && r < 0) {
-    m_ret_val = r;
-  }
-
-  Context *ctx = create_context_callback<
-    OpenImageRequest<I>, &OpenImageRequest<I>::handle_close_image>(
-      this);
-  CloseImageRequest<I> *request = CloseImageRequest<I>::create(
-    m_image_ctx, m_work_queue, true, ctx);
-  request->send();
-}
-
-template <typename I>
-void OpenImageRequest<I>::handle_close_image(int r) {
-  dout(20) << dendl;
-
-  assert(r == 0);
-  finish(m_ret_val);
+  finish(r);
 }
 
 template <typename I>

diff --git a/src/tools/rbd_mirror/image_replayer/OpenImageRequest.h b/src/tools/rbd_mirror/image_replayer/OpenImageRequest.h
index d9407c052b167d63ffb818b3f06e395ae3767626..01ab31171950bdfaacc28a3879949f9db505d952 100644 (file)
--- a/src/tools/rbd_mirror/image_replayer/OpenImageRequest.h
+++ b/src/tools/rbd_mirror/image_replayer/OpenImageRequest.h
@@ -9,7 +9,6 @@
 #include <string>
 
 class Context;
-class ContextWQ;
 namespace librbd { class ImageCtx; }
 
 namespace rbd {
@@ -22,15 +21,14 @@ public:
   static OpenImageRequest* create(librados::IoCtx &io_ctx,
                                   ImageCtxT **image_ctx,
                                   const std::string &image_id,
-                                  bool read_only, ContextWQ *work_queue,
-                                  Context *on_finish) {
+                                  bool read_only, Context *on_finish) {
     return new OpenImageRequest(io_ctx, image_ctx, image_id, read_only,
-                                work_queue, on_finish);
+                                on_finish);
   }
 
   OpenImageRequest(librados::IoCtx &io_ctx, ImageCtxT **image_ctx,
                    const std::string &image_id, bool read_only,
-                   ContextWQ *m_work_queue, Context *on_finish);
+                   Context *on_finish);
 
   void send();
 
@@ -41,10 +39,10 @@ private:
    * <start>
    *    |
    *    v
-   * OPEN_IMAGE * * * * * * * *
-   *    |                     *
-   *    v                     v
-   * <finish> <---------- CLOSE_IMAGE
+   * OPEN_IMAGE
+   *    |
+   *    v
+   * <finish>
    *
    * @endverbatim
    */
@@ -52,11 +50,8 @@ private:
   ImageCtxT **m_image_ctx;
   std::string m_image_id;
   bool m_read_only;
-  ContextWQ *m_work_queue;
   Context *m_on_finish;
 
-  int m_ret_val = 0;
-
   void send_open_image();
   void handle_open_image(int r);
 

diff --git a/src/tools/rbd_mirror/image_replayer/OpenLocalImageRequest.cc b/src/tools/rbd_mirror/image_replayer/OpenLocalImageRequest.cc
index cf37048aa5db366b2360bcad9cff8950e4e7eaa5..5f67c41cfa43028cabb23cec74b38db75e699ed2 100644 (file)
--- a/src/tools/rbd_mirror/image_replayer/OpenLocalImageRequest.cc
+++ b/src/tools/rbd_mirror/image_replayer/OpenLocalImageRequest.cc
@@ -112,7 +112,9 @@ void OpenLocalImageRequest<I>::handle_open_image(int r) {
   if (r < 0) {
     derr << ": failed to open image '" << m_local_image_id << "': "
          << cpp_strerror(r) << dendl;
-    send_close_image(true, r);
+    (*m_local_image_ctx)->destroy();
+    *m_local_image_ctx = nullptr;
+    finish(r);
     return;
   }
 
@@ -138,7 +140,7 @@ void OpenLocalImageRequest<I>::handle_is_primary(int r) {
   if (r < 0) {
     derr << ": error querying local image primary status: " << cpp_strerror(r)
          << dendl;
-    send_close_image(false, r);
+    send_close_image(r);
     return;
   }
 
@@ -146,7 +148,7 @@ void OpenLocalImageRequest<I>::handle_is_primary(int r) {
   // we aren't going to mirror peer data into this image anyway
   if (m_primary) {
     dout(10) << ": local image is primary -- skipping image replay" << dendl;
-    send_close_image(false, -EREMOTEIO);
+    send_close_image(-EREMOTEIO);
     return;
   }
 
@@ -160,7 +162,7 @@ void OpenLocalImageRequest<I>::send_lock_image() {
   RWLock::RLocker owner_locker((*m_local_image_ctx)->owner_lock);
   if ((*m_local_image_ctx)->exclusive_lock == nullptr) {
     derr << ": image does not support exclusive lock" << dendl;
-    send_close_image(false, -EINVAL);
+    send_close_image(-EINVAL);
     return;
   }
 
@@ -181,7 +183,7 @@ void OpenLocalImageRequest<I>::handle_lock_image(int r) {
   if (r < 0) {
     derr << ": failed to lock image '" << m_local_image_id << "': "
        << cpp_strerror(r) << dendl;
-    send_close_image(false, r);
+    send_close_image(r);
     return;
   }
 
@@ -190,7 +192,7 @@ void OpenLocalImageRequest<I>::handle_lock_image(int r) {
     if ((*m_local_image_ctx)->exclusive_lock == nullptr ||
        !(*m_local_image_ctx)->exclusive_lock->is_lock_owner()) {
       derr << ": image is not locked" << dendl;
-      send_close_image(false, -EBUSY);
+      send_close_image(-EBUSY);
       return;
     }
   }
@@ -199,7 +201,7 @@ void OpenLocalImageRequest<I>::handle_lock_image(int r) {
 }
 
 template <typename I>
-void OpenLocalImageRequest<I>::send_close_image(bool destroy_only, int r) {
+void OpenLocalImageRequest<I>::send_close_image(int r) {
   dout(20) << dendl;
 
   if (m_ret_val == 0 && r < 0) {
@@ -210,7 +212,7 @@ void OpenLocalImageRequest<I>::send_close_image(bool destroy_only, int r) {
     OpenLocalImageRequest<I>, &OpenLocalImageRequest<I>::handle_close_image>(
       this);
   CloseImageRequest<I> *request = CloseImageRequest<I>::create(
-    m_local_image_ctx, m_work_queue, destroy_only, ctx);
+    m_local_image_ctx, ctx);
   request->send();
 }
 

diff --git a/src/tools/rbd_mirror/image_replayer/OpenLocalImageRequest.h b/src/tools/rbd_mirror/image_replayer/OpenLocalImageRequest.h
index 2a1bbb2a8e2e83540341395c67abcda965643517..570728dd762bd4a9b6b97d5d6128e2804fe8582f 100644 (file)
--- a/src/tools/rbd_mirror/image_replayer/OpenLocalImageRequest.h
+++ b/src/tools/rbd_mirror/image_replayer/OpenLocalImageRequest.h
@@ -78,7 +78,7 @@ private:
   void send_lock_image();
   void handle_lock_image(int r);
 
-  void send_close_image(bool destroy_only, int r);
+  void send_close_image(int r);
   void handle_close_image(int r);
 
   void finish(int r);