From: Jason Dillaman Date: Wed, 31 May 2017 01:38:14 +0000 (-0400) Subject: librbd: track async exclusive-lock dependent operations X-Git-Tag: ses5-milestone6~8^2~5^2~1 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=f51fb1626b851eab73baa7f77f1dc99044c9c53b;p=ceph.git librbd: track async exclusive-lock dependent operations Signed-off-by: Jason Dillaman --- diff --git a/src/librbd/ExclusiveLock.cc b/src/librbd/ExclusiveLock.cc index 5d942980dfbc..f35b90f1df5a 100644 --- a/src/librbd/ExclusiveLock.cc +++ b/src/librbd/ExclusiveLock.cc @@ -2,6 +2,7 @@ // vim: ts=8 sw=2 smarttab #include "librbd/ExclusiveLock.h" +#include "librbd/ImageCtx.h" #include "librbd/ImageWatcher.h" #include "librbd/ImageState.h" #include "librbd/exclusive_lock/PreAcquireRequest.h" @@ -53,12 +54,15 @@ bool ExclusiveLock::accept_requests(int *ret_val) const { template bool ExclusiveLock::accept_ops() const { Mutex::Locker locker(ML::m_lock); - bool accept_ops = (!ML::is_state_shutdown() && - (ML::is_state_locked() || - ML::is_state_post_acquiring())); + bool accept = accept_ops(ML::m_lock); + ldout(m_image_ctx.cct, 20) << "=" << accept << dendl; + return accept; +} - ldout(m_image_ctx.cct, 20) << "=" << accept_ops << dendl; - return accept_ops; +template +bool ExclusiveLock::accept_ops(const Mutex &lock) const { + return (!ML::is_state_shutdown() && + (ML::is_state_locked() || ML::is_state_post_acquiring())); } template @@ -124,6 +128,21 @@ void ExclusiveLock::handle_peer_notification(int r) { ML::execute_next_action(); } +template +Context *ExclusiveLock::start_op() { + assert(m_image_ctx.owner_lock.is_locked()); + Mutex::Locker locker(ML::m_lock); + + if (!accept_ops(ML::m_lock)) { + return nullptr; + } + + m_async_op_tracker.start_op(); + return new FunctionContext([this](int r) { + m_async_op_tracker.finish_op(); + }); +} + template void ExclusiveLock::handle_init_complete(uint64_t features) { ldout(m_image_ctx.cct, 10) << "features=" << features << dendl; @@ -264,7 +283,7 @@ void ExclusiveLock::pre_release_lock_handler(bool shutting_down, Mutex::Locker locker(ML::m_lock); PreReleaseRequest *req = PreReleaseRequest::create( - m_image_ctx, shutting_down, on_finish); + m_image_ctx, shutting_down, m_async_op_tracker, on_finish); m_image_ctx.op_work_queue->queue(new FunctionContext([req](int r) { req->send(); })); diff --git a/src/librbd/ExclusiveLock.h b/src/librbd/ExclusiveLock.h index 102501f5fbfc..0aaad091ea78 100644 --- a/src/librbd/ExclusiveLock.h +++ b/src/librbd/ExclusiveLock.h @@ -5,7 +5,7 @@ #define CEPH_LIBRBD_EXCLUSIVE_LOCK_H #include "librbd/ManagedLock.h" -#include "librbd/ImageCtx.h" +#include "common/AsyncOpTracker.h" namespace librbd { @@ -29,6 +29,8 @@ public: void handle_peer_notification(int r); + Context *start_op(); + protected: void shutdown_handler(int r, Context *on_finish) override; void pre_acquire_lock_handler(Context *on_finish) override; @@ -84,11 +86,15 @@ private: ImageCtxT& m_image_ctx; Context *m_pre_post_callback = nullptr; + AsyncOpTracker m_async_op_tracker; + uint32_t m_request_blocked_count = 0; int m_request_blocked_ret_val = 0; int m_acquire_lock_peer_ret_val = 0; + bool accept_ops(const Mutex &lock) const; + void handle_init_complete(uint64_t features); void handle_post_acquiring_lock(int r); void handle_post_acquired_lock(int r); diff --git a/src/librbd/exclusive_lock/PreReleaseRequest.cc b/src/librbd/exclusive_lock/PreReleaseRequest.cc index ee29a98584f5..5a37acc96a53 100644 --- a/src/librbd/exclusive_lock/PreReleaseRequest.cc +++ b/src/librbd/exclusive_lock/PreReleaseRequest.cc @@ -2,6 +2,7 @@ // vim: ts=8 sw=2 smarttab #include "librbd/exclusive_lock/PreReleaseRequest.h" +#include "common/AsyncOpTracker.h" #include "common/dout.h" #include "common/errno.h" #include "librbd/ExclusiveLock.h" @@ -24,19 +25,20 @@ using util::create_async_context_callback; using util::create_context_callback; template -PreReleaseRequest* PreReleaseRequest::create(I &image_ctx, - bool shutting_down, - Context *on_finish) { - return new PreReleaseRequest(image_ctx, shutting_down, on_finish); +PreReleaseRequest* PreReleaseRequest::create( + I &image_ctx, bool shutting_down, AsyncOpTracker &async_op_tracker, + Context *on_finish) { + return new PreReleaseRequest(image_ctx, shutting_down, async_op_tracker, + on_finish); } template PreReleaseRequest::PreReleaseRequest(I &image_ctx, bool shutting_down, + AsyncOpTracker &async_op_tracker, Context *on_finish) - : m_image_ctx(image_ctx), - m_on_finish(create_async_context_callback(image_ctx, on_finish)), - m_shutting_down(shutting_down), m_error_result(0), m_object_map(nullptr), - m_journal(nullptr) { + : m_image_ctx(image_ctx), m_shutting_down(shutting_down), + m_async_op_tracker(async_op_tracker), + m_on_finish(create_async_context_callback(image_ctx, on_finish)) { } template @@ -131,6 +133,24 @@ void PreReleaseRequest::handle_block_writes(int r) { return; } + send_wait_for_ops(); +} + +template +void PreReleaseRequest::send_wait_for_ops() { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 10) << dendl; + + Context *ctx = create_context_callback< + PreReleaseRequest, &PreReleaseRequest::handle_wait_for_ops>(this); + m_async_op_tracker.wait_for_ops(ctx); +} + +template +void PreReleaseRequest::handle_wait_for_ops(int r) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 10) << dendl; + send_invalidate_cache(false); } diff --git a/src/librbd/exclusive_lock/PreReleaseRequest.h b/src/librbd/exclusive_lock/PreReleaseRequest.h index 98e3a1e9dac7..34ac07ab26c8 100644 --- a/src/librbd/exclusive_lock/PreReleaseRequest.h +++ b/src/librbd/exclusive_lock/PreReleaseRequest.h @@ -7,6 +7,7 @@ #include "librbd/ImageCtx.h" #include +class AsyncOpTracker; class Context; namespace librbd { @@ -19,6 +20,7 @@ template class PreReleaseRequest { public: static PreReleaseRequest* create(ImageCtxT &image_ctx, bool shutting_down, + AsyncOpTracker &async_op_tracker, Context *on_finish); ~PreReleaseRequest(); @@ -40,6 +42,9 @@ private: * BLOCK_WRITES * | * v + * WAIT_FOR_OPS + * | + * v * INVALIDATE_CACHE * | * v @@ -58,16 +63,17 @@ private: */ PreReleaseRequest(ImageCtxT &image_ctx, bool shutting_down, - Context *on_finish); + AsyncOpTracker &async_op_tracker, Context *on_finish); ImageCtxT &m_image_ctx; - Context *m_on_finish; bool m_shutting_down; + AsyncOpTracker &m_async_op_tracker; + Context *m_on_finish; - int m_error_result; + int m_error_result = 0; - decltype(m_image_ctx.object_map) m_object_map; - decltype(m_image_ctx.journal) m_journal; + decltype(m_image_ctx.object_map) m_object_map = nullptr; + decltype(m_image_ctx.journal) m_journal = nullptr; void send_prepare_lock(); void handle_prepare_lock(int r); @@ -78,6 +84,9 @@ private: void send_block_writes(); void handle_block_writes(int r); + void send_wait_for_ops(); + void handle_wait_for_ops(int r); + void send_invalidate_cache(bool purge_on_error); void handle_invalidate_cache(int r); diff --git a/src/test/librbd/exclusive_lock/test_mock_PreReleaseRequest.cc b/src/test/librbd/exclusive_lock/test_mock_PreReleaseRequest.cc index 72249d186c30..99de2551212a 100644 --- a/src/test/librbd/exclusive_lock/test_mock_PreReleaseRequest.cc +++ b/src/test/librbd/exclusive_lock/test_mock_PreReleaseRequest.cc @@ -7,6 +7,7 @@ #include "test/librbd/mock/MockJournal.h" #include "test/librbd/mock/MockObjectMap.h" #include "test/librados_test_stub/MockTestMemIoCtxImpl.h" +#include "common/AsyncOpTracker.h" #include "librbd/exclusive_lock/PreReleaseRequest.h" #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -118,6 +119,7 @@ public: EXPECT_CALL(*mock_image_ctx.state, handle_prepare_lock_complete()); } + AsyncOpTracker m_async_op_tracker; }; TEST_F(TestMockExclusiveLockPreReleaseRequest, Success) { @@ -149,7 +151,7 @@ TEST_F(TestMockExclusiveLockPreReleaseRequest, Success) { C_SaferCond ctx; MockPreReleaseRequest *req = MockPreReleaseRequest::create( - mock_image_ctx, false, &ctx); + mock_image_ctx, false, m_async_op_tracker, &ctx); req->send(); ASSERT_EQ(0, ctx.wait()); } @@ -179,7 +181,7 @@ TEST_F(TestMockExclusiveLockPreReleaseRequest, SuccessJournalDisabled) { C_SaferCond ctx; MockPreReleaseRequest *req = MockPreReleaseRequest::create( - mock_image_ctx, false, &ctx); + mock_image_ctx, false, m_async_op_tracker, &ctx); req->send(); ASSERT_EQ(0, ctx.wait()); } @@ -203,7 +205,7 @@ TEST_F(TestMockExclusiveLockPreReleaseRequest, SuccessObjectMapDisabled) { C_SaferCond release_ctx; C_SaferCond ctx; MockPreReleaseRequest *req = MockPreReleaseRequest::create( - mock_image_ctx, true, &ctx); + mock_image_ctx, true, m_async_op_tracker, &ctx); req->send(); ASSERT_EQ(0, ctx.wait()); } @@ -239,7 +241,7 @@ TEST_F(TestMockExclusiveLockPreReleaseRequest, Blacklisted) { C_SaferCond ctx; MockPreReleaseRequest *req = MockPreReleaseRequest::create( - mock_image_ctx, false, &ctx); + mock_image_ctx, false, m_async_op_tracker, &ctx); req->send(); ASSERT_EQ(0, ctx.wait()); } @@ -261,7 +263,7 @@ TEST_F(TestMockExclusiveLockPreReleaseRequest, BlockWritesError) { C_SaferCond ctx; MockPreReleaseRequest *req = MockPreReleaseRequest::create( - mock_image_ctx, true, &ctx); + mock_image_ctx, true, m_async_op_tracker, &ctx); req->send(); ASSERT_EQ(-EINVAL, ctx.wait()); } @@ -284,7 +286,7 @@ TEST_F(TestMockExclusiveLockPreReleaseRequest, UnlockError) { C_SaferCond ctx; MockPreReleaseRequest *req = MockPreReleaseRequest::create( - mock_image_ctx, true, &ctx); + mock_image_ctx, true, m_async_op_tracker, &ctx); req->send(); ASSERT_EQ(0, ctx.wait()); } diff --git a/src/test/librbd/mock/MockExclusiveLock.h b/src/test/librbd/mock/MockExclusiveLock.h index 58ae19edd10b..b73ab02c709e 100644 --- a/src/test/librbd/mock/MockExclusiveLock.h +++ b/src/test/librbd/mock/MockExclusiveLock.h @@ -29,6 +29,8 @@ struct MockExclusiveLock { MOCK_METHOD0(accept_requests, bool()); MOCK_METHOD0(accept_ops, bool()); + + MOCK_METHOD0(start_op, Context*()); }; } // namespace librbd diff --git a/src/test/librbd/test_mock_ExclusiveLock.cc b/src/test/librbd/test_mock_ExclusiveLock.cc index eeb62f63ea5d..2a3388bdb39a 100644 --- a/src/test/librbd/test_mock_ExclusiveLock.cc +++ b/src/test/librbd/test_mock_ExclusiveLock.cc @@ -126,7 +126,7 @@ template <> struct PreReleaseRequest : public BaseRequest > { static PreReleaseRequest *create( MockExclusiveLockImageCtx &image_ctx, bool shutting_down, - Context *on_finish) { + AsyncOpTracker &async_op_tracker, Context *on_finish) { return BaseRequest::create(image_ctx, nullptr, on_finish); } MOCK_METHOD0(send, void());