From 724262987014723576238f10a6ff5ec31af18eba Mon Sep 17 00:00:00 2001
From: Jason Dillaman <dillaman@redhat.com>
Date: Fri, 8 May 2020 11:36:32 -0400
Subject: [PATCH] librbd: ensure in-flight read-ahead + copy-on-read ops are
 finalized

Now that the image dispatcher isn't flushing AsyncOperations with each
flush request, we need to ensure that we flush all background ops prior
to fully closing the image.

Signed-off-by: Jason Dillaman <dillaman@redhat.com>
---
 src/librbd/io/ImageDispatcher.cc              | 20 +++++++++++++++++++
 src/librbd/io/ImageDispatcher.h               |  2 ++
 src/test/librbd/io/test_mock_CopyupRequest.cc |  4 ++++
 src/test/librbd/mock/MockImageCtx.cc          | 12 +++++++++++
 src/test/librbd/mock/MockImageCtx.h           |  4 +++-
 5 files changed, 41 insertions(+), 1 deletion(-)

diff --git a/src/librbd/io/ImageDispatcher.cc b/src/librbd/io/ImageDispatcher.cc
index c1016ea9e46b2..090cddb3186f1 100644
--- a/src/librbd/io/ImageDispatcher.cc
+++ b/src/librbd/io/ImageDispatcher.cc
@@ -8,6 +8,7 @@
 #include "common/WorkQueue.h"
 #include "librbd/ImageCtx.h"
 #include "librbd/Utils.h"
+#include "librbd/io/AsyncOperation.h"
 #include "librbd/io/ImageDispatch.h"
 #include "librbd/io/ImageDispatchInterface.h"
 #include "librbd/io/ImageDispatchSpec.h"
@@ -117,6 +118,25 @@ ImageDispatcher<I>::ImageDispatcher(I* image_ctx)
   this->register_dispatch(refresh_image_dispatch);
 }
 
+template <typename I>
+void ImageDispatcher<I>::shut_down(Context* on_finish) {
+  // TODO ensure all IOs are executed via a dispatcher
+  // ensure read-ahead / copy-on-read ops are finished since they are
+  // currently outside dispatcher tracking
+  auto async_op = new AsyncOperation();
+
+  on_finish = new LambdaContext([this, async_op, on_finish](int r) {
+      async_op->finish_op();
+      delete async_op;
+      on_finish->complete(0);
+    });
+  on_finish = new LambdaContext([this, async_op, on_finish](int r) {
+      async_op->start_op(*this->m_image_ctx);
+      async_op->flush(on_finish);
+    });
+  Dispatcher<I, ImageDispatcherInterface>::shut_down(on_finish);
+}
+
 template <typename I>
 void ImageDispatcher<I>::apply_qos_schedule_tick_min(uint64_t tick) {
   m_qos_image_dispatch->apply_qos_schedule_tick_min(tick);
diff --git a/src/librbd/io/ImageDispatcher.h b/src/librbd/io/ImageDispatcher.h
index 7628cd38d16ef..9c5850e520a52 100644
--- a/src/librbd/io/ImageDispatcher.h
+++ b/src/librbd/io/ImageDispatcher.h
@@ -30,6 +30,8 @@ class ImageDispatcher : public Dispatcher<ImageCtxT, ImageDispatcherInterface> {
 public:
   ImageDispatcher(ImageCtxT* image_ctx);
 
+  void shut_down(Context* on_finish) override;
+
   void apply_qos_schedule_tick_min(uint64_t tick) override;
   void apply_qos_limit(uint64_t flag, uint64_t limit, uint64_t burst) override;
 
diff --git a/src/test/librbd/io/test_mock_CopyupRequest.cc b/src/test/librbd/io/test_mock_CopyupRequest.cc
index 5d87ae6f6c2f4..6bb9294bec8d5 100644
--- a/src/test/librbd/io/test_mock_CopyupRequest.cc
+++ b/src/test/librbd/io/test_mock_CopyupRequest.cc
@@ -26,6 +26,10 @@ struct MockTestImageCtx : public MockImageCtx {
     : MockImageCtx(image_ctx) {
     parent = mock_parent_image_ctx;
   }
+  ~MockTestImageCtx() override {
+    // copyups need to complete prior to attempting to delete this object
+    wait_for_async_ops();
+  }
 
   std::map<uint64_t, librbd::io::CopyupRequest<librbd::MockTestImageCtx>*> copyup_list;
 };
diff --git a/src/test/librbd/mock/MockImageCtx.cc b/src/test/librbd/mock/MockImageCtx.cc
index dd8612ec900c2..9a1c5430daa3f 100644
--- a/src/test/librbd/mock/MockImageCtx.cc
+++ b/src/test/librbd/mock/MockImageCtx.cc
@@ -3,6 +3,7 @@
 
 #include "test/librbd/mock/MockImageCtx.h"
 #include "test/librbd/mock/MockSafeTimer.h"
+#include "librbd/io/AsyncOperation.h"
 
 static MockSafeTimer *s_timer;
 static ceph::mutex *s_timer_lock;
@@ -23,4 +24,15 @@ void MockImageCtx::get_timer_instance(CephContext *cct, MockSafeTimer **timer,
   *timer_lock = s_timer_lock;
 }
 
+void MockImageCtx::wait_for_async_ops() {
+  io::AsyncOperation async_op;
+  async_op.start_op(*image_ctx);
+
+  C_SaferCond ctx;
+  async_op.flush(&ctx);
+  ctx.wait();
+
+  async_op.finish_op();
+}
+
 } // namespace librbd
diff --git a/src/test/librbd/mock/MockImageCtx.h b/src/test/librbd/mock/MockImageCtx.h
index b86a87cb13ac6..0d12b391dca1f 100644
--- a/src/test/librbd/mock/MockImageCtx.h
+++ b/src/test/librbd/mock/MockImageCtx.h
@@ -116,8 +116,9 @@ struct MockImageCtx {
     }
   }
 
-  ~MockImageCtx() {
+  virtual ~MockImageCtx() {
     wait_for_async_requests();
+    wait_for_async_ops();
     image_ctx->md_ctx.aio_flush();
     image_ctx->data_ctx.aio_flush();
     image_ctx->op_work_queue->drain();
@@ -129,6 +130,7 @@ struct MockImageCtx {
     delete io_object_dispatcher;
   }
 
+  void wait_for_async_ops();
   void wait_for_async_requests() {
     async_ops_lock.lock();
     if (async_requests.empty()) {
-- 
2.39.5