From 8ef37b0259bd09c1c47c267918bc28330bae281f Mon Sep 17 00:00:00 2001 From: Jason Dillaman Date: Thu, 10 Dec 2015 18:39:28 -0500 Subject: [PATCH] librbd: IO ops will now asynchronously refresh the image if needed This avoids blocking the AIO path with the previous synchronous refresh path. Signed-off-by: Jason Dillaman --- src/librbd/AioImageRequest.cc | 11 ++++----- src/librbd/AioImageRequest.h | 1 + src/librbd/AioImageRequestWQ.cc | 44 ++++++++++++++++++++++++++------- src/librbd/AioImageRequestWQ.h | 20 ++++++++++++++- 4 files changed, 60 insertions(+), 16 deletions(-) diff --git a/src/librbd/AioImageRequest.cc b/src/librbd/AioImageRequest.cc index c4d677aab65..f0c3228119c 100644 --- a/src/librbd/AioImageRequest.cc +++ b/src/librbd/AioImageRequest.cc @@ -117,15 +117,14 @@ void AioImageRequest::send() { << "completion=" << m_aio_comp << dendl; m_aio_comp->get(); - int r = m_image_ctx.state->refresh_if_required(m_image_ctx.owner_lock); - if (r < 0) { - m_aio_comp->fail(cct, r); - return; - } - send_request(); } +void AioImageRequest::fail(int r) { + m_aio_comp->get(); + m_aio_comp->fail(m_image_ctx.cct, r); +} + void AioImageRead::send_request() { CephContext *cct = m_image_ctx.cct; diff --git a/src/librbd/AioImageRequest.h b/src/librbd/AioImageRequest.h index de14bfd783c..6ee6d64322b 100644 --- a/src/librbd/AioImageRequest.h +++ b/src/librbd/AioImageRequest.h @@ -40,6 +40,7 @@ public: } void send(); + void fail(int r); protected: typedef std::list AioObjectRequests; diff --git a/src/librbd/AioImageRequestWQ.cc b/src/librbd/AioImageRequestWQ.cc index f57eae97bea..9871bb0535e 100644 --- a/src/librbd/AioImageRequestWQ.cc +++ b/src/librbd/AioImageRequestWQ.cc @@ -7,6 +7,7 @@ #include "librbd/AioImageRequest.h" #include "librbd/ExclusiveLock.h" #include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" #include "librbd/internal.h" #include "librbd/Utils.h" @@ -22,7 +23,8 @@ AioImageRequestWQ::AioImageRequestWQ(ImageCtx *image_ctx, const string &name, m_image_ctx(*image_ctx), m_lock(util::unique_lock_name("AioImageRequestWQ::m_lock", this)), m_write_blockers(0), m_in_progress_writes(0), m_queued_writes(0), - m_in_flight_ops(0), m_shutdown(false), m_on_shutdown(nullptr) { + m_in_flight_ops(0), m_refresh_in_progress(false), + m_shutdown(false), m_on_shutdown(nullptr) { CephContext *cct = m_image_ctx.cct; ldout(cct, 5) << this << " " << ": ictx=" << image_ctx << dendl; } @@ -261,23 +263,32 @@ void AioImageRequestWQ::unblock_writes() { void *AioImageRequestWQ::_void_dequeue() { AioImageRequest *peek_item = front(); - if (peek_item == NULL) { + if (peek_item == NULL || m_refresh_in_progress) { return NULL; } - { - if (peek_item->is_write_op()) { - RWLock::RLocker locker(m_lock); - if (m_write_blockers > 0) { - return NULL; - } - m_in_progress_writes.inc(); + if (peek_item->is_write_op()) { + RWLock::RLocker locker(m_lock); + if (m_write_blockers > 0) { + return NULL; } + m_in_progress_writes.inc(); } AioImageRequest *item = reinterpret_cast( ThreadPool::PointerWQ::_void_dequeue()); assert(peek_item == item); + + if (m_image_ctx.state->is_refresh_required()) { + ldout(m_image_ctx.cct, 15) << "image refresh required: delaying IO " << item + << dendl; + m_refresh_in_progress = true; + + get_pool_lock().Unlock(); + m_image_ctx.state->refresh(new C_RefreshFinish(this, item)); + get_pool_lock().Lock(); + return NULL; + } return item; } @@ -380,6 +391,21 @@ void AioImageRequestWQ::queue(AioImageRequest *req) { } } +void AioImageRequestWQ::handle_refreshed(int r, AioImageRequest *req) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 15) << "resuming IO after image refresh: r=" << r << ", " + << "req=" << req << dendl; + if (r < 0) { + req->fail(r); + } else { + process(req); + process_finish(); + + m_refresh_in_progress = false; + signal(); + } +} + void AioImageRequestWQ::handle_blocked_writes(int r) { Contexts contexts; { diff --git a/src/librbd/AioImageRequestWQ.h b/src/librbd/AioImageRequestWQ.h index 0eeb716bc99..c84548955a3 100644 --- a/src/librbd/AioImageRequestWQ.h +++ b/src/librbd/AioImageRequestWQ.h @@ -6,8 +6,10 @@ #include "include/Context.h" #include "include/atomic.h" -#include "common/WorkQueue.h" +#include "common/Cond.h" #include "common/RWLock.h" +#include "common/WorkQueue.h" +#include namespace librbd { @@ -57,6 +59,19 @@ protected: private: typedef std::list Contexts; + struct C_RefreshFinish : public Context { + AioImageRequestWQ *aio_work_queue; + AioImageRequest *aio_image_request; + + C_RefreshFinish(AioImageRequestWQ *aio_work_queue, + AioImageRequest *aio_image_request) + : aio_work_queue(aio_work_queue), aio_image_request(aio_image_request) { + } + virtual void finish(int r) override { + aio_work_queue->handle_refreshed(r, aio_image_request); + } + }; + struct C_BlockedWrites : public Context { AioImageRequestWQ *aio_work_queue; C_BlockedWrites(AioImageRequestWQ *_aio_work_queue) @@ -76,6 +91,8 @@ private: atomic_t m_queued_writes; atomic_t m_in_flight_ops; + bool m_refresh_in_progress; + bool m_shutdown; Context *m_on_shutdown; @@ -86,6 +103,7 @@ private: bool is_lock_required() const; void queue(AioImageRequest *req); + void handle_refreshed(int r, AioImageRequest *req); void handle_blocked_writes(int r); }; -- 2.39.5