librbd: track async exclusive-lock dependent operations

Signed-off-by: Jason Dillaman <dillaman@redhat.com>

diff --git a/src/librbd/ExclusiveLock.cc b/src/librbd/ExclusiveLock.cc
index 5d942980dfbcdd8c09a4aa6a432a35f6f8651738..f35b90f1df5ab67206d5d5e730489ab24a7575a5 100644 (file)
--- a/src/librbd/ExclusiveLock.cc
+++ b/src/librbd/ExclusiveLock.cc
@@ -2,6 +2,7 @@
 // vim: ts=8 sw=2 smarttab
 
 #include "librbd/ExclusiveLock.h"
+#include "librbd/ImageCtx.h"
 #include "librbd/ImageWatcher.h"
 #include "librbd/ImageState.h"
 #include "librbd/exclusive_lock/PreAcquireRequest.h"
@@ -53,12 +54,15 @@ bool ExclusiveLock<I>::accept_requests(int *ret_val) const {
 template <typename I>
 bool ExclusiveLock<I>::accept_ops() const {
   Mutex::Locker locker(ML<I>::m_lock);
-  bool accept_ops = (!ML<I>::is_state_shutdown() &&
-                     (ML<I>::is_state_locked() ||
-                      ML<I>::is_state_post_acquiring()));
+  bool accept = accept_ops(ML<I>::m_lock);
+  ldout(m_image_ctx.cct, 20) << "=" << accept << dendl;
+  return accept;
+}
 
-  ldout(m_image_ctx.cct, 20) << "=" << accept_ops << dendl;
-  return accept_ops;
+template <typename I>
+bool ExclusiveLock<I>::accept_ops(const Mutex &lock) const {
+  return (!ML<I>::is_state_shutdown() &&
+          (ML<I>::is_state_locked() || ML<I>::is_state_post_acquiring()));
 }
 
 template <typename I>
@@ -124,6 +128,21 @@ void ExclusiveLock<I>::handle_peer_notification(int r) {
   ML<I>::execute_next_action();
 }
 
+template <typename I>
+Context *ExclusiveLock<I>::start_op() {
+  assert(m_image_ctx.owner_lock.is_locked());
+  Mutex::Locker locker(ML<I>::m_lock);
+
+  if (!accept_ops(ML<I>::m_lock)) {
+    return nullptr;
+  }
+
+  m_async_op_tracker.start_op();
+  return new FunctionContext([this](int r) {
+      m_async_op_tracker.finish_op();
+    });
+}
+
 template <typename I>
 void ExclusiveLock<I>::handle_init_complete(uint64_t features) {
   ldout(m_image_ctx.cct, 10) << "features=" << features << dendl;
@@ -264,7 +283,7 @@ void ExclusiveLock<I>::pre_release_lock_handler(bool shutting_down,
   Mutex::Locker locker(ML<I>::m_lock);
 
   PreReleaseRequest<I> *req = PreReleaseRequest<I>::create(
-    m_image_ctx, shutting_down, on_finish);
+    m_image_ctx, shutting_down, m_async_op_tracker, on_finish);
   m_image_ctx.op_work_queue->queue(new FunctionContext([req](int r) {
     req->send();
   }));

diff --git a/src/librbd/ExclusiveLock.h b/src/librbd/ExclusiveLock.h
index 102501f5fbfce5401b2cbd457af650c40188fca4..0aaad091ea786c74fe4be16edb8497297d2ac652 100644 (file)
--- a/src/librbd/ExclusiveLock.h
+++ b/src/librbd/ExclusiveLock.h
@@ -5,7 +5,7 @@
 #define CEPH_LIBRBD_EXCLUSIVE_LOCK_H
 
 #include "librbd/ManagedLock.h"
-#include "librbd/ImageCtx.h"
+#include "common/AsyncOpTracker.h"
 
 namespace librbd {
 
@@ -29,6 +29,8 @@ public:
 
   void handle_peer_notification(int r);
 
+  Context *start_op();
+
 protected:
   void shutdown_handler(int r, Context *on_finish) override;
   void pre_acquire_lock_handler(Context *on_finish) override;
@@ -84,11 +86,15 @@ private:
   ImageCtxT& m_image_ctx;
   Context *m_pre_post_callback = nullptr;
 
+  AsyncOpTracker m_async_op_tracker;
+
   uint32_t m_request_blocked_count = 0;
   int m_request_blocked_ret_val = 0;
 
   int m_acquire_lock_peer_ret_val = 0;
 
+  bool accept_ops(const Mutex &lock) const;
+
   void handle_init_complete(uint64_t features);
   void handle_post_acquiring_lock(int r);
   void handle_post_acquired_lock(int r);

diff --git a/src/librbd/exclusive_lock/PreReleaseRequest.cc b/src/librbd/exclusive_lock/PreReleaseRequest.cc
index ee29a98584f587ceb7391aff6c2f2c12e8973a19..5a37acc96a53301a06e39c4a97e19d18a38166a7 100644 (file)
--- a/src/librbd/exclusive_lock/PreReleaseRequest.cc
+++ b/src/librbd/exclusive_lock/PreReleaseRequest.cc
@@ -2,6 +2,7 @@
 // vim: ts=8 sw=2 smarttab
 
 #include "librbd/exclusive_lock/PreReleaseRequest.h"
+#include "common/AsyncOpTracker.h"
 #include "common/dout.h"
 #include "common/errno.h"
 #include "librbd/ExclusiveLock.h"
@@ -24,19 +25,20 @@ using util::create_async_context_callback;
 using util::create_context_callback;
 
 template <typename I>
-PreReleaseRequest<I>* PreReleaseRequest<I>::create(I &image_ctx,
-                                                   bool shutting_down,
-                                                   Context *on_finish) {
-  return new PreReleaseRequest(image_ctx, shutting_down, on_finish);
+PreReleaseRequest<I>* PreReleaseRequest<I>::create(
+    I &image_ctx, bool shutting_down, AsyncOpTracker &async_op_tracker,
+    Context *on_finish) {
+  return new PreReleaseRequest(image_ctx, shutting_down, async_op_tracker,
+                               on_finish);
 }
 
 template <typename I>
 PreReleaseRequest<I>::PreReleaseRequest(I &image_ctx, bool shutting_down,
+                                        AsyncOpTracker &async_op_tracker,
                                         Context *on_finish)
-  : m_image_ctx(image_ctx),
-    m_on_finish(create_async_context_callback(image_ctx, on_finish)),
-    m_shutting_down(shutting_down), m_error_result(0), m_object_map(nullptr),
-    m_journal(nullptr) {
+  : m_image_ctx(image_ctx), m_shutting_down(shutting_down),
+    m_async_op_tracker(async_op_tracker),
+    m_on_finish(create_async_context_callback(image_ctx, on_finish)) {
 }
 
 template <typename I>
@@ -131,6 +133,24 @@ void PreReleaseRequest<I>::handle_block_writes(int r) {
     return;
   }
 
+  send_wait_for_ops();
+}
+
+template <typename I>
+void PreReleaseRequest<I>::send_wait_for_ops() {
+  CephContext *cct = m_image_ctx.cct;
+  ldout(cct, 10) << dendl;
+
+  Context *ctx = create_context_callback<
+    PreReleaseRequest<I>, &PreReleaseRequest<I>::handle_wait_for_ops>(this);
+  m_async_op_tracker.wait_for_ops(ctx);
+}
+
+template <typename I>
+void PreReleaseRequest<I>::handle_wait_for_ops(int r) {
+  CephContext *cct = m_image_ctx.cct;
+  ldout(cct, 10) << dendl;
+
   send_invalidate_cache(false);
 }
 

diff --git a/src/librbd/exclusive_lock/PreReleaseRequest.h b/src/librbd/exclusive_lock/PreReleaseRequest.h
index 98e3a1e9dac73cc467910603ff0baa2aa8c888de..34ac07ab26c859353d8476dccc8d55cba4fb184e 100644 (file)
--- a/src/librbd/exclusive_lock/PreReleaseRequest.h
+++ b/src/librbd/exclusive_lock/PreReleaseRequest.h
@@ -7,6 +7,7 @@
 #include "librbd/ImageCtx.h"
 #include <string>
 
+class AsyncOpTracker;
 class Context;
 
 namespace librbd {
@@ -19,6 +20,7 @@ template <typename ImageCtxT = ImageCtx>
 class PreReleaseRequest {
 public:
   static PreReleaseRequest* create(ImageCtxT &image_ctx, bool shutting_down,
+                                   AsyncOpTracker &async_op_tracker,
                                    Context *on_finish);
 
   ~PreReleaseRequest();
@@ -40,6 +42,9 @@ private:
    * BLOCK_WRITES
    *    |
    *    v
+   * WAIT_FOR_OPS
+   *    |
+   *    v
    * INVALIDATE_CACHE
    *    |
    *    v
@@ -58,16 +63,17 @@ private:
    */
 
   PreReleaseRequest(ImageCtxT &image_ctx, bool shutting_down,
-                    Context *on_finish);
+                    AsyncOpTracker &async_op_tracker, Context *on_finish);
 
   ImageCtxT &m_image_ctx;
-  Context *m_on_finish;
   bool m_shutting_down;
+  AsyncOpTracker &m_async_op_tracker;
+  Context *m_on_finish;
 
-  int m_error_result;
+  int m_error_result = 0;
 
-  decltype(m_image_ctx.object_map) m_object_map;
-  decltype(m_image_ctx.journal) m_journal;
+  decltype(m_image_ctx.object_map) m_object_map = nullptr;
+  decltype(m_image_ctx.journal) m_journal = nullptr;
 
   void send_prepare_lock();
   void handle_prepare_lock(int r);
@@ -78,6 +84,9 @@ private:
   void send_block_writes();
   void handle_block_writes(int r);
 
+  void send_wait_for_ops();
+  void handle_wait_for_ops(int r);
+
   void send_invalidate_cache(bool purge_on_error);
   void handle_invalidate_cache(int r);
 

diff --git a/src/test/librbd/exclusive_lock/test_mock_PreReleaseRequest.cc b/src/test/librbd/exclusive_lock/test_mock_PreReleaseRequest.cc
index 72249d186c30d6e7946e7b44e445e1aa2cd83d63..99de2551212a501990a6ecc8a3943bcd3cea5fac 100644 (file)
--- a/src/test/librbd/exclusive_lock/test_mock_PreReleaseRequest.cc
+++ b/src/test/librbd/exclusive_lock/test_mock_PreReleaseRequest.cc
@@ -7,6 +7,7 @@
 #include "test/librbd/mock/MockJournal.h"
 #include "test/librbd/mock/MockObjectMap.h"
 #include "test/librados_test_stub/MockTestMemIoCtxImpl.h"
+#include "common/AsyncOpTracker.h"
 #include "librbd/exclusive_lock/PreReleaseRequest.h"
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
@@ -118,6 +119,7 @@ public:
     EXPECT_CALL(*mock_image_ctx.state, handle_prepare_lock_complete());
   }
 
+  AsyncOpTracker m_async_op_tracker;
 };
 
 TEST_F(TestMockExclusiveLockPreReleaseRequest, Success) {
@@ -149,7 +151,7 @@ TEST_F(TestMockExclusiveLockPreReleaseRequest, Success) {
 
   C_SaferCond ctx;
   MockPreReleaseRequest *req = MockPreReleaseRequest::create(
-    mock_image_ctx, false, &ctx);
+    mock_image_ctx, false, m_async_op_tracker, &ctx);
   req->send();
   ASSERT_EQ(0, ctx.wait());
 }
@@ -179,7 +181,7 @@ TEST_F(TestMockExclusiveLockPreReleaseRequest, SuccessJournalDisabled) {
 
   C_SaferCond ctx;
   MockPreReleaseRequest *req = MockPreReleaseRequest::create(
-    mock_image_ctx, false, &ctx);
+    mock_image_ctx, false, m_async_op_tracker, &ctx);
   req->send();
   ASSERT_EQ(0, ctx.wait());
 }
@@ -203,7 +205,7 @@ TEST_F(TestMockExclusiveLockPreReleaseRequest, SuccessObjectMapDisabled) {
   C_SaferCond release_ctx;
   C_SaferCond ctx;
   MockPreReleaseRequest *req = MockPreReleaseRequest::create(
-    mock_image_ctx, true, &ctx);
+    mock_image_ctx, true, m_async_op_tracker, &ctx);
   req->send();
   ASSERT_EQ(0, ctx.wait());
 }
@@ -239,7 +241,7 @@ TEST_F(TestMockExclusiveLockPreReleaseRequest, Blacklisted) {
 
   C_SaferCond ctx;
   MockPreReleaseRequest *req = MockPreReleaseRequest::create(
-    mock_image_ctx, false, &ctx);
+    mock_image_ctx, false, m_async_op_tracker, &ctx);
   req->send();
   ASSERT_EQ(0, ctx.wait());
 }
@@ -261,7 +263,7 @@ TEST_F(TestMockExclusiveLockPreReleaseRequest, BlockWritesError) {
 
   C_SaferCond ctx;
   MockPreReleaseRequest *req = MockPreReleaseRequest::create(
-    mock_image_ctx, true, &ctx);
+    mock_image_ctx, true, m_async_op_tracker, &ctx);
   req->send();
   ASSERT_EQ(-EINVAL, ctx.wait());
 }
@@ -284,7 +286,7 @@ TEST_F(TestMockExclusiveLockPreReleaseRequest, UnlockError) {
 
   C_SaferCond ctx;
   MockPreReleaseRequest *req = MockPreReleaseRequest::create(
-    mock_image_ctx, true, &ctx);
+    mock_image_ctx, true, m_async_op_tracker, &ctx);
   req->send();
   ASSERT_EQ(0, ctx.wait());
 }

diff --git a/src/test/librbd/mock/MockExclusiveLock.h b/src/test/librbd/mock/MockExclusiveLock.h
index 58ae19edd10b8a76f0d09b94de7fbf516587cb47..b73ab02c709ef6c8300eeab0738d081e982126a9 100644 (file)
--- a/src/test/librbd/mock/MockExclusiveLock.h
+++ b/src/test/librbd/mock/MockExclusiveLock.h
@@ -29,6 +29,8 @@ struct MockExclusiveLock {
 
   MOCK_METHOD0(accept_requests, bool());
   MOCK_METHOD0(accept_ops, bool());
+
+  MOCK_METHOD0(start_op, Context*());
 };
 
 } // namespace librbd

diff --git a/src/test/librbd/test_mock_ExclusiveLock.cc b/src/test/librbd/test_mock_ExclusiveLock.cc
index eeb62f63ea5d2a5370b20a971b09f1cec865720e..2a3388bdb39a04bc94b63113161a7de78cbd56d8 100644 (file)
--- a/src/test/librbd/test_mock_ExclusiveLock.cc
+++ b/src/test/librbd/test_mock_ExclusiveLock.cc
@@ -126,7 +126,7 @@ template <>
 struct PreReleaseRequest<MockExclusiveLockImageCtx> : public BaseRequest<PreReleaseRequest<MockExclusiveLockImageCtx> > {
   static PreReleaseRequest<MockExclusiveLockImageCtx> *create(
       MockExclusiveLockImageCtx &image_ctx, bool shutting_down,
-      Context *on_finish) {
+      AsyncOpTracker &async_op_tracker, Context *on_finish) {
     return BaseRequest::create(image_ctx, nullptr, on_finish);
   }
   MOCK_METHOD0(send, void());