librbd: init and shut down image IO dispatch layer

Signed-off-by: Jason Dillaman <dillaman@redhat.com>

diff --git a/src/librbd/ImageCtx.cc b/src/librbd/ImageCtx.cc
index 1a583aeeb51bb04e76c821fa9131a04a0e0dae30..d5f5e5516af4a0f40bf8451cf08c1801591d485f 100644 (file)
--- a/src/librbd/ImageCtx.cc
+++ b/src/librbd/ImageCtx.cc
@@ -28,6 +28,7 @@
 #include "librbd/exclusive_lock/StandardPolicy.h"
 #include "librbd/io/AioCompletion.h"
 #include "librbd/io/AsyncOperation.h"
+#include "librbd/io/ImageDispatcher.h"
 #include "librbd/io/ImageRequestWQ.h"
 #include "librbd/io/ObjectDispatcher.h"
 #include "librbd/journal/StandardPolicy.h"
@@ -141,6 +142,7 @@ public:
       this, "librbd::io_work_queue",
       cct->_conf.get_val<uint64_t>("rbd_op_thread_timeout"),
       thread_pool);
+    io_image_dispatcher = new io::ImageDispatcher<ImageCtx>(this);
     io_object_dispatcher = new io::ObjectDispatcher<ImageCtx>(this);
 
     if (cct->_conf.get_val<bool>("rbd_auto_exclusive_lock_until_manual_request")) {
@@ -176,6 +178,7 @@ public:
     io_work_queue->drain();
 
     delete io_object_dispatcher;
+    delete io_image_dispatcher;
 
     delete journal_policy;
     delete exclusive_lock_policy;

diff --git a/src/librbd/ImageCtx.h b/src/librbd/ImageCtx.h
index 74feaafd34af39b55937b34a066f40d758608295..ef930f57c7812d3462728affaa4432187f394c52 100644 (file)
--- a/src/librbd/ImageCtx.h
+++ b/src/librbd/ImageCtx.h
@@ -59,6 +59,7 @@ namespace librbd {
   class AsyncOperation;
   template <typename> class CopyupRequest;
   template <typename> class ImageRequestWQ;
+  struct ImageDispatcherInterface;
   struct ObjectDispatcherInterface;
   }
   namespace journal { struct Policy; }
@@ -181,6 +182,7 @@ namespace librbd {
     xlist<operation::ResizeRequest<ImageCtx>*> resize_reqs;
 
     io::ImageRequestWQ<ImageCtx> *io_work_queue;
+    io::ImageDispatcherInterface *io_image_dispatcher = nullptr;
     io::ObjectDispatcherInterface *io_object_dispatcher = nullptr;
 
     ContextWQ *op_work_queue;

diff --git a/src/librbd/image/CloseRequest.cc b/src/librbd/image/CloseRequest.cc
index 998459c2cde09b52e9cc1d86e26251b575123061..5dba010b4befed6b50d645b3fda6ac35afea0e8c 100644 (file)
--- a/src/librbd/image/CloseRequest.cc
+++ b/src/librbd/image/CloseRequest.cc
@@ -11,6 +11,7 @@
 #include "librbd/ObjectMap.h"
 #include "librbd/Utils.h"
 #include "librbd/io/AioCompletion.h"
+#include "librbd/io/ImageDispatcher.h"
 #include "librbd/io/ImageDispatchSpec.h"
 #include "librbd/io/ImageRequestWQ.h"
 #include "librbd/io/ObjectDispatcherInterface.h"
@@ -251,6 +252,29 @@ void CloseRequest<I>::handle_shut_down_object_dispatcher(int r) {
     lderr(cct) << "failed to shut down object dispatcher: "
                << cpp_strerror(r) << dendl;
   }
+  send_shut_down_image_dispatcher();
+}
+
+template <typename I>
+void CloseRequest<I>::send_shut_down_image_dispatcher() {
+  CephContext *cct = m_image_ctx->cct;
+  ldout(cct, 10) << this << " " << __func__ << dendl;
+
+  m_image_ctx->io_image_dispatcher->shut_down(create_context_callback<
+    CloseRequest<I>,
+    &CloseRequest<I>::handle_shut_down_image_dispatcher>(this));
+}
+
+template <typename I>
+void CloseRequest<I>::handle_shut_down_image_dispatcher(int r) {
+  CephContext *cct = m_image_ctx->cct;
+  ldout(cct, 10) << this << " " << __func__ << ": r=" << r << dendl;
+
+  save_result(r);
+  if (r < 0) {
+    lderr(cct) << "failed to shut down image dispatcher: "
+               << cpp_strerror(r) << dendl;
+  }
   send_flush_op_work_queue();
 }
 

diff --git a/src/librbd/image/CloseRequest.h b/src/librbd/image/CloseRequest.h
index fd101beedb331c2fb679068d01da4f17a7334b81..95c02bb5de9778d22ba46b18e8637e114d325305 100644 (file)
--- a/src/librbd/image/CloseRequest.h
+++ b/src/librbd/image/CloseRequest.h
@@ -53,6 +53,9 @@ private:
    * SHUT_DOWN_OBJECT_DISPATCHER
    *    |
    *    v
+   * SHUT_DOWN_IMAGE_DISPATCHER
+   *    |
+   *    v
    * FLUSH_OP_WORK_QUEUE
    *    |
    *    v (skip if no parent)
@@ -100,6 +103,9 @@ private:
   void send_shut_down_object_dispatcher();
   void handle_shut_down_object_dispatcher(int r);
 
+  void send_shut_down_image_dispatcher();
+  void handle_shut_down_image_dispatcher(int r);
+
   void send_flush_op_work_queue();
   void handle_flush_op_work_queue(int r);
 

diff --git a/src/librbd/io/ImageDispatchSpec.h b/src/librbd/io/ImageDispatchSpec.h
index 90eddb616d13c0f50387c6f0622458fb57a3bf57..f26a297e20727911908342b6d03ae54f8760c1e5 100644 (file)
--- a/src/librbd/io/ImageDispatchSpec.h
+++ b/src/librbd/io/ImageDispatchSpec.h
@@ -215,10 +215,11 @@ private:
                     AioCompletion* aio_comp, Extents&& image_extents,
                     Request&& request, int op_flags,
                     const ZTracer::Trace& parent_trace, uint64_t tid)
-    : dispatcher_ctx(this), dispatch_layer(image_dispatch_layer),
-      aio_comp(aio_comp), image_extents(std::move(image_extents)),
-      request(std::move(request)), op_flags(op_flags),
-      parent_trace(parent_trace), tid(tid), m_image_ctx(image_ctx) {
+    : dispatcher_ctx(this), image_dispatcher(image_ctx.io_image_dispatcher),
+      dispatch_layer(image_dispatch_layer), aio_comp(aio_comp),
+      image_extents(std::move(image_extents)), request(std::move(request)),
+      op_flags(op_flags), parent_trace(parent_trace), tid(tid),
+      m_image_ctx(image_ctx) {
     aio_comp->get();
   }
 

diff --git a/src/test/librbd/mock/MockImageCtx.h b/src/test/librbd/mock/MockImageCtx.h
index dcd620abd76c7d0b15ca113b57446910d9f226b1..1a35b2933f178cd9591942aed02e5d049b53270b 100644 (file)
--- a/src/test/librbd/mock/MockImageCtx.h
+++ b/src/test/librbd/mock/MockImageCtx.h
@@ -13,6 +13,7 @@
 #include "test/librbd/mock/MockObjectMap.h"
 #include "test/librbd/mock/MockOperations.h"
 #include "test/librbd/mock/MockReadahead.h"
+#include "test/librbd/mock/io/MockImageDispatcher.h"
 #include "test/librbd/mock/io/MockImageRequestWQ.h"
 #include "test/librbd/mock/io/MockObjectDispatcher.h"
 #include "common/RWLock.h"
@@ -84,6 +85,7 @@ struct MockImageCtx {
       group_spec(image_ctx.group_spec),
       layout(image_ctx.layout),
       io_work_queue(new io::MockImageRequestWQ()),
+      io_image_dispatcher(new io::MockImageDispatcher()),
       io_object_dispatcher(new io::MockObjectDispatcher()),
       op_work_queue(new MockContextWQ()),
       readahead_max_bytes(image_ctx.readahead_max_bytes),
@@ -126,6 +128,7 @@ struct MockImageCtx {
     delete image_watcher;
     delete op_work_queue;
     delete io_work_queue;
+    delete io_image_dispatcher;
     delete io_object_dispatcher;
   }
 
@@ -282,6 +285,7 @@ struct MockImageCtx {
   std::map<uint64_t, io::CopyupRequest<MockImageCtx>*> copyup_list;
 
   io::MockImageRequestWQ *io_work_queue;
+  io::MockImageDispatcher *io_image_dispatcher;
   io::MockObjectDispatcher *io_object_dispatcher;
   MockContextWQ *op_work_queue;
 

diff --git a/src/test/librbd/mock/io/MockImageDispatcher.h b/src/test/librbd/mock/io/MockImageDispatcher.h
new file mode 100644 (file)
index 0000000..5eefde7
--- /dev/null
+++ b/src/test/librbd/mock/io/MockImageDispatcher.h
@@ -0,0 +1,33 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CEPH_TEST_LIBRBD_MOCK_IO_IMAGE_DISPATCHER_H
+#define CEPH_TEST_LIBRBD_MOCK_IO_IMAGE_DISPATCHER_H
+
+#include "gmock/gmock.h"
+#include "include/Context.h"
+#include "librbd/io/ImageDispatcher.h"
+#include "librbd/io/ImageDispatchSpec.h"
+#include "librbd/io/Types.h"
+
+class Context;
+
+namespace librbd {
+namespace io {
+
+struct ImageDispatchInterface;
+
+struct MockImageDispatcher : public ImageDispatcherInterface {
+public:
+  MOCK_METHOD1(shut_down, void(Context*));
+
+  MOCK_METHOD1(register_dispatch, void(ImageDispatchInterface*));
+  MOCK_METHOD2(shut_down_dispatch, void(ImageDispatchLayer, Context*));
+
+  MOCK_METHOD1(send, void(ImageDispatchSpec<>*));
+};
+
+} // namespace io
+} // namespace librbd
+
+#endif // CEPH_TEST_LIBRBD_MOCK_IO_IMAGE_DISPATCHER_H