From: Jason Dillaman <dillaman@redhat.com>
Date: Fri, 8 May 2020 15:36:32 +0000 (-0400)
Subject: librbd: ensure in-flight read-ahead + copy-on-read ops are finalized
X-Git-Tag: v16.1.0~2318^2~4
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=724262987014723576238f10a6ff5ec31af18eba;p=ceph.git

librbd: ensure in-flight read-ahead + copy-on-read ops are finalized

Now that the image dispatcher isn't flushing AsyncOperations with each
flush request, we need to ensure that we flush all background ops prior
to fully closing the image.

Signed-off-by: Jason Dillaman <dillaman@redhat.com>
---

diff --git a/src/librbd/io/ImageDispatcher.cc b/src/librbd/io/ImageDispatcher.cc
index c1016ea9e46..090cddb3186 100644
--- a/src/librbd/io/ImageDispatcher.cc
+++ b/src/librbd/io/ImageDispatcher.cc
@@ -8,6 +8,7 @@
 #include "common/WorkQueue.h"
 #include "librbd/ImageCtx.h"
 #include "librbd/Utils.h"
+#include "librbd/io/AsyncOperation.h"
 #include "librbd/io/ImageDispatch.h"
 #include "librbd/io/ImageDispatchInterface.h"
 #include "librbd/io/ImageDispatchSpec.h"
@@ -117,6 +118,25 @@ ImageDispatcher<I>::ImageDispatcher(I* image_ctx)
   this->register_dispatch(refresh_image_dispatch);
 }
 
+template <typename I>
+void ImageDispatcher<I>::shut_down(Context* on_finish) {
+  // TODO ensure all IOs are executed via a dispatcher
+  // ensure read-ahead / copy-on-read ops are finished since they are
+  // currently outside dispatcher tracking
+  auto async_op = new AsyncOperation();
+
+  on_finish = new LambdaContext([this, async_op, on_finish](int r) {
+      async_op->finish_op();
+      delete async_op;
+      on_finish->complete(0);
+    });
+  on_finish = new LambdaContext([this, async_op, on_finish](int r) {
+      async_op->start_op(*this->m_image_ctx);
+      async_op->flush(on_finish);
+    });
+  Dispatcher<I, ImageDispatcherInterface>::shut_down(on_finish);
+}
+
 template <typename I>
 void ImageDispatcher<I>::apply_qos_schedule_tick_min(uint64_t tick) {
   m_qos_image_dispatch->apply_qos_schedule_tick_min(tick);
diff --git a/src/librbd/io/ImageDispatcher.h b/src/librbd/io/ImageDispatcher.h
index 7628cd38d16..9c5850e520a 100644
--- a/src/librbd/io/ImageDispatcher.h
+++ b/src/librbd/io/ImageDispatcher.h
@@ -30,6 +30,8 @@ class ImageDispatcher : public Dispatcher<ImageCtxT, ImageDispatcherInterface> {
 public:
   ImageDispatcher(ImageCtxT* image_ctx);
 
+  void shut_down(Context* on_finish) override;
+
   void apply_qos_schedule_tick_min(uint64_t tick) override;
   void apply_qos_limit(uint64_t flag, uint64_t limit, uint64_t burst) override;
 
diff --git a/src/test/librbd/io/test_mock_CopyupRequest.cc b/src/test/librbd/io/test_mock_CopyupRequest.cc
index 5d87ae6f6c2..6bb9294bec8 100644
--- a/src/test/librbd/io/test_mock_CopyupRequest.cc
+++ b/src/test/librbd/io/test_mock_CopyupRequest.cc
@@ -26,6 +26,10 @@ struct MockTestImageCtx : public MockImageCtx {
     : MockImageCtx(image_ctx) {
     parent = mock_parent_image_ctx;
   }
+  ~MockTestImageCtx() override {
+    // copyups need to complete prior to attempting to delete this object
+    wait_for_async_ops();
+  }
 
   std::map<uint64_t, librbd::io::CopyupRequest<librbd::MockTestImageCtx>*> copyup_list;
 };
diff --git a/src/test/librbd/mock/MockImageCtx.cc b/src/test/librbd/mock/MockImageCtx.cc
index dd8612ec900..9a1c5430daa 100644
--- a/src/test/librbd/mock/MockImageCtx.cc
+++ b/src/test/librbd/mock/MockImageCtx.cc
@@ -3,6 +3,7 @@
 
 #include "test/librbd/mock/MockImageCtx.h"
 #include "test/librbd/mock/MockSafeTimer.h"
+#include "librbd/io/AsyncOperation.h"
 
 static MockSafeTimer *s_timer;
 static ceph::mutex *s_timer_lock;
@@ -23,4 +24,15 @@ void MockImageCtx::get_timer_instance(CephContext *cct, MockSafeTimer **timer,
   *timer_lock = s_timer_lock;
 }
 
+void MockImageCtx::wait_for_async_ops() {
+  io::AsyncOperation async_op;
+  async_op.start_op(*image_ctx);
+
+  C_SaferCond ctx;
+  async_op.flush(&ctx);
+  ctx.wait();
+
+  async_op.finish_op();
+}
+
 } // namespace librbd
diff --git a/src/test/librbd/mock/MockImageCtx.h b/src/test/librbd/mock/MockImageCtx.h
index b86a87cb13a..0d12b391dca 100644
--- a/src/test/librbd/mock/MockImageCtx.h
+++ b/src/test/librbd/mock/MockImageCtx.h
@@ -116,8 +116,9 @@ struct MockImageCtx {
     }
   }
 
-  ~MockImageCtx() {
+  virtual ~MockImageCtx() {
     wait_for_async_requests();
+    wait_for_async_ops();
     image_ctx->md_ctx.aio_flush();
     image_ctx->data_ctx.aio_flush();
     image_ctx->op_work_queue->drain();
@@ -129,6 +130,7 @@ struct MockImageCtx {
     delete io_object_dispatcher;
   }
 
+  void wait_for_async_ops();
   void wait_for_async_requests() {
     async_ops_lock.lock();
     if (async_requests.empty()) {