From ec84d0e75614fc2435003ae6e9c84cbd2217cb19 Mon Sep 17 00:00:00 2001 From: Jason Dillaman Date: Wed, 27 May 2020 19:13:31 -0400 Subject: [PATCH] librbd: removed block write logic from queue image dispatch layer It now has its own layer later in the stack. Signed-off-by: Jason Dillaman --- src/librbd/io/QueueImageDispatch.cc | 145 ++-------------------------- src/librbd/io/QueueImageDispatch.h | 30 +----- 2 files changed, 11 insertions(+), 164 deletions(-) diff --git a/src/librbd/io/QueueImageDispatch.cc b/src/librbd/io/QueueImageDispatch.cc index 93bb56a372c..0114fefa098 100644 --- a/src/librbd/io/QueueImageDispatch.cc +++ b/src/librbd/io/QueueImageDispatch.cc @@ -20,10 +20,7 @@ namespace io { template QueueImageDispatch::QueueImageDispatch(I* image_ctx) - : m_image_ctx(image_ctx), - m_lock(ceph::make_shared_mutex( - util::unique_lock_name("librbd::io::QueueImageDispatch::m_lock", - this))) { + : m_image_ctx(image_ctx) { auto cct = m_image_ctx->cct; ldout(cct, 5) << "ictx=" << image_ctx << dendl; } @@ -33,85 +30,6 @@ void QueueImageDispatch::shut_down(Context* on_finish) { on_finish->complete(0); } -template -int QueueImageDispatch::block_writes() { - C_SaferCond cond_ctx; - block_writes(&cond_ctx); - return cond_ctx.wait(); -} - -template -void QueueImageDispatch::block_writes(Context *on_blocked) { - ceph_assert(ceph_mutex_is_locked(m_image_ctx->owner_lock)); - auto cct = m_image_ctx->cct; - - // ensure onwer lock is not held after block_writes completes - on_blocked = util::create_async_context_callback( - *m_image_ctx, on_blocked); - - { - std::unique_lock locker{m_lock}; - ++m_write_blockers; - ldout(cct, 5) << m_image_ctx << ", " - << "num=" << m_write_blockers << dendl; - if (!m_write_blocker_contexts.empty() || !m_in_flight_write_tids.empty()) { - ldout(cct, 5) << "waiting for in-flight writes to complete: " - << "write_tids=" << m_in_flight_write_tids << dendl; - m_write_blocker_contexts.push_back(on_blocked); - return; - } - } - - // ensure that all in-flight IO is flushed - flush_image(on_blocked); -}; - -template -void QueueImageDispatch::unblock_writes() { - auto cct = m_image_ctx->cct; - - Contexts waiter_contexts; - Contexts dispatch_contexts; - { - std::unique_lock locker{m_lock}; - ceph_assert(m_write_blockers > 0); - --m_write_blockers; - - ldout(cct, 5) << m_image_ctx << ", " - << "num=" << m_write_blockers << dendl; - if (m_write_blockers == 0) { - std::swap(waiter_contexts, m_unblocked_write_waiter_contexts); - std::swap(dispatch_contexts, m_on_dispatches); - } - } - - for (auto ctx : waiter_contexts) { - ctx->complete(0); - } - - for (auto ctx : dispatch_contexts) { - ctx->complete(0); - } -} - -template -void QueueImageDispatch::wait_on_writes_unblocked(Context *on_unblocked) { - ceph_assert(ceph_mutex_is_locked(m_image_ctx->owner_lock)); - auto cct = m_image_ctx->cct; - - { - std::unique_lock locker{m_lock}; - ldout(cct, 20) << m_image_ctx << ", " - << "write_blockers=" << m_write_blockers << dendl; - if (!m_unblocked_write_waiter_contexts.empty() || m_write_blockers > 0) { - m_unblocked_write_waiter_contexts.push_back(on_unblocked); - return; - } - } - - on_unblocked->complete(0); -} - template bool QueueImageDispatch::read( AioCompletion* aio_comp, Extents &&image_extents, ReadResult &&read_result, @@ -121,7 +39,7 @@ bool QueueImageDispatch::read( auto cct = m_image_ctx->cct; ldout(cct, 20) << "tid=" << tid << dendl; - return enqueue(true, tid, dispatch_result, on_dispatched); + return enqueue(dispatch_result, on_dispatched); } template @@ -133,7 +51,7 @@ bool QueueImageDispatch::write( auto cct = m_image_ctx->cct; ldout(cct, 20) << "tid=" << tid << dendl; - return enqueue(false, tid, dispatch_result, on_dispatched); + return enqueue(dispatch_result, on_dispatched); } template @@ -145,7 +63,7 @@ bool QueueImageDispatch::discard( auto cct = m_image_ctx->cct; ldout(cct, 20) << "tid=" << tid << dendl; - return enqueue(false, tid, dispatch_result, on_dispatched); + return enqueue(dispatch_result, on_dispatched); } template @@ -157,7 +75,7 @@ bool QueueImageDispatch::write_same( auto cct = m_image_ctx->cct; ldout(cct, 20) << "tid=" << tid << dendl; - return enqueue(false, tid, dispatch_result, on_dispatched); + return enqueue(dispatch_result, on_dispatched); } template @@ -170,7 +88,7 @@ bool QueueImageDispatch::compare_and_write( auto cct = m_image_ctx->cct; ldout(cct, 20) << "tid=" << tid << dendl; - return enqueue(false, tid, dispatch_result, on_dispatched); + return enqueue(dispatch_result, on_dispatched); } template @@ -186,49 +104,12 @@ bool QueueImageDispatch::flush( return false; } - return enqueue(false, tid, dispatch_result, on_dispatched); -} - -template -void QueueImageDispatch::handle_finished(int r, uint64_t tid) { - auto cct = m_image_ctx->cct; - ldout(cct, 20) << "r=" << r << ", tid=" << tid << dendl; - - std::unique_lock locker{m_lock}; - auto it = m_in_flight_write_tids.find(tid); - if (it == m_in_flight_write_tids.end()) { - // assumed to be a read op - return; - } - m_in_flight_write_tids.erase(it); - - Contexts write_blocker_contexts; - if (m_in_flight_write_tids.empty()) { - std::swap(write_blocker_contexts, m_write_blocker_contexts); - } - locker.unlock(); - - for (auto ctx : write_blocker_contexts) { - ctx->complete(0); - } + return enqueue(dispatch_result, on_dispatched); } template bool QueueImageDispatch::enqueue( - bool read_op, uint64_t tid, DispatchResult* dispatch_result, - Context* on_dispatched) { - std::unique_lock locker{m_lock}; - if (!read_op) { - if (m_write_blockers > 0 || !m_on_dispatches.empty()) { - *dispatch_result = DISPATCH_RESULT_RESTART; - m_on_dispatches.push_back(on_dispatched); - return true; - } - - m_in_flight_write_tids.insert(tid); - } - locker.unlock(); - + DispatchResult* dispatch_result, Context* on_dispatched) { if (!m_image_ctx->non_blocking_aio) { return false; } @@ -238,16 +119,6 @@ bool QueueImageDispatch::enqueue( return true; } -template -void QueueImageDispatch::flush_image(Context* on_finish) { - auto aio_comp = AioCompletion::create_and_start( - on_finish, util::get_image_ctx(m_image_ctx), librbd::io::AIO_TYPE_FLUSH); - auto req = ImageDispatchSpec::create_flush( - *m_image_ctx, IMAGE_DISPATCH_LAYER_QUEUE, aio_comp, FLUSH_SOURCE_INTERNAL, - {}); - req->send(); -} - } // namespace io } // namespace librbd diff --git a/src/librbd/io/QueueImageDispatch.h b/src/librbd/io/QueueImageDispatch.h index bd48f7f8fea..03d5068d759 100644 --- a/src/librbd/io/QueueImageDispatch.h +++ b/src/librbd/io/QueueImageDispatch.h @@ -7,7 +7,6 @@ #include "librbd/io/ImageDispatchInterface.h" #include "include/int_types.h" #include "include/buffer.h" -#include "common/ceph_mutex.h" #include "common/zipkin_trace.h" #include "common/Throttle.h" #include "librbd/io/ReadResult.h" @@ -36,17 +35,6 @@ public: void shut_down(Context* on_finish) override; - int block_writes(); - void block_writes(Context *on_blocked); - void unblock_writes(); - - inline bool writes_blocked() const { - std::shared_lock locker{m_lock}; - return (m_write_blockers > 0); - } - - void wait_on_writes_unblocked(Context *on_unblocked); - bool read( AioCompletion* aio_comp, Extents &&image_extents, ReadResult &&read_result, int op_flags, @@ -81,25 +69,13 @@ public: std::atomic* image_dispatch_flags, DispatchResult* dispatch_result, Context* on_dispatched) override; - void handle_finished(int r, uint64_t tid) override; + void handle_finished(int r, uint64_t tid) override { + } private: - typedef std::list Contexts; - typedef std::set Tids; - ImageCtxT* m_image_ctx; - mutable ceph::shared_mutex m_lock; - Contexts m_on_dispatches; - Tids m_in_flight_write_tids; - - uint32_t m_write_blockers = 0; - Contexts m_write_blocker_contexts; - Contexts m_unblocked_write_waiter_contexts; - - bool enqueue(bool read_op, uint64_t tid, DispatchResult* dispatch_result, - Context* on_dispatched); - void flush_image(Context* on_blocked); + bool enqueue(DispatchResult* dispatch_result, Context* on_dispatched); }; -- 2.39.5