librbd: ensure in-flight read-ahead + copy-on-read ops are finalized

Now that the image dispatcher isn't flushing AsyncOperations with each
flush request, we need to ensure that we flush all background ops prior
to fully closing the image.

Signed-off-by: Jason Dillaman <dillaman@redhat.com>

diff --git a/src/librbd/io/ImageDispatcher.cc b/src/librbd/io/ImageDispatcher.cc
index c1016ea9e46b2c471226ab6b7dc3a41a2e118ed4..090cddb3186f18c10c6c724e9d2350b30014d6ad 100644 (file)
--- a/src/librbd/io/ImageDispatcher.cc
+++ b/src/librbd/io/ImageDispatcher.cc
@@ -8,6 +8,7 @@
 #include "common/WorkQueue.h"
 #include "librbd/ImageCtx.h"
 #include "librbd/Utils.h"
+#include "librbd/io/AsyncOperation.h"
 #include "librbd/io/ImageDispatch.h"
 #include "librbd/io/ImageDispatchInterface.h"
 #include "librbd/io/ImageDispatchSpec.h"
@@ -117,6 +118,25 @@ ImageDispatcher<I>::ImageDispatcher(I* image_ctx)
   this->register_dispatch(refresh_image_dispatch);
 }
 
+template <typename I>
+void ImageDispatcher<I>::shut_down(Context* on_finish) {
+  // TODO ensure all IOs are executed via a dispatcher
+  // ensure read-ahead / copy-on-read ops are finished since they are
+  // currently outside dispatcher tracking
+  auto async_op = new AsyncOperation();
+
+  on_finish = new LambdaContext([this, async_op, on_finish](int r) {
+      async_op->finish_op();
+      delete async_op;
+      on_finish->complete(0);
+    });
+  on_finish = new LambdaContext([this, async_op, on_finish](int r) {
+      async_op->start_op(*this->m_image_ctx);
+      async_op->flush(on_finish);
+    });
+  Dispatcher<I, ImageDispatcherInterface>::shut_down(on_finish);
+}
+
 template <typename I>
 void ImageDispatcher<I>::apply_qos_schedule_tick_min(uint64_t tick) {
   m_qos_image_dispatch->apply_qos_schedule_tick_min(tick);

diff --git a/src/librbd/io/ImageDispatcher.h b/src/librbd/io/ImageDispatcher.h
index 7628cd38d16ef1ce3937c0e55cffeab838485571..9c5850e520a52cc3fdead680dadff3fc478b4521 100644 (file)
--- a/src/librbd/io/ImageDispatcher.h
+++ b/src/librbd/io/ImageDispatcher.h
@@ -30,6 +30,8 @@ class ImageDispatcher : public Dispatcher<ImageCtxT, ImageDispatcherInterface> {
 public:
   ImageDispatcher(ImageCtxT* image_ctx);
 
+  void shut_down(Context* on_finish) override;
+
   void apply_qos_schedule_tick_min(uint64_t tick) override;
   void apply_qos_limit(uint64_t flag, uint64_t limit, uint64_t burst) override;
 

diff --git a/src/test/librbd/io/test_mock_CopyupRequest.cc b/src/test/librbd/io/test_mock_CopyupRequest.cc
index 5d87ae6f6c2f4e5dcd370ba64472c99b2fa757ea..6bb9294bec8d5f3b1c4741b2f4b099e7e19e77f3 100644 (file)
--- a/src/test/librbd/io/test_mock_CopyupRequest.cc
+++ b/src/test/librbd/io/test_mock_CopyupRequest.cc
@@ -26,6 +26,10 @@ struct MockTestImageCtx : public MockImageCtx {
     : MockImageCtx(image_ctx) {
     parent = mock_parent_image_ctx;
   }
+  ~MockTestImageCtx() override {
+    // copyups need to complete prior to attempting to delete this object
+    wait_for_async_ops();
+  }
 
   std::map<uint64_t, librbd::io::CopyupRequest<librbd::MockTestImageCtx>*> copyup_list;
 };

diff --git a/src/test/librbd/mock/MockImageCtx.cc b/src/test/librbd/mock/MockImageCtx.cc
index dd8612ec900c22aed540db7c430c83bd04f607b7..9a1c5430daa3f206ac6f429d913ef8f43f221198 100644 (file)
--- a/src/test/librbd/mock/MockImageCtx.cc
+++ b/src/test/librbd/mock/MockImageCtx.cc
@@ -3,6 +3,7 @@
 
 #include "test/librbd/mock/MockImageCtx.h"
 #include "test/librbd/mock/MockSafeTimer.h"
+#include "librbd/io/AsyncOperation.h"
 
 static MockSafeTimer *s_timer;
 static ceph::mutex *s_timer_lock;
@@ -23,4 +24,15 @@ void MockImageCtx::get_timer_instance(CephContext *cct, MockSafeTimer **timer,
   *timer_lock = s_timer_lock;
 }
 
+void MockImageCtx::wait_for_async_ops() {
+  io::AsyncOperation async_op;
+  async_op.start_op(*image_ctx);
+
+  C_SaferCond ctx;
+  async_op.flush(&ctx);
+  ctx.wait();
+
+  async_op.finish_op();
+}
+
 } // namespace librbd

diff --git a/src/test/librbd/mock/MockImageCtx.h b/src/test/librbd/mock/MockImageCtx.h
index b86a87cb13ac62dfb9d7d658febb0b62233d8230..0d12b391dca1f01275cc65f7d18a8a691c2b9a93 100644 (file)
--- a/src/test/librbd/mock/MockImageCtx.h
+++ b/src/test/librbd/mock/MockImageCtx.h
@@ -116,8 +116,9 @@ struct MockImageCtx {
     }
   }
 
-  ~MockImageCtx() {
+  virtual ~MockImageCtx() {
     wait_for_async_requests();
+    wait_for_async_ops();
     image_ctx->md_ctx.aio_flush();
     image_ctx->data_ctx.aio_flush();
     image_ctx->op_work_queue->drain();
@@ -129,6 +130,7 @@ struct MockImageCtx {
     delete io_object_dispatcher;
   }
 
+  void wait_for_async_ops();
   void wait_for_async_requests() {
     async_ops_lock.lock();
     if (async_requests.empty()) {