From 0024677dc8adfd610418ca47599dd95d3a5a6612 Mon Sep 17 00:00:00 2001 From: Jason Dillaman Date: Thu, 30 Apr 2015 13:40:16 -0400 Subject: [PATCH] librbd: complete cache read in a new thread context The ObjectCacher complete the read callback while still holding the cache lock. This introduces lock ordering issues which are resolved by queuing the completion to execute in a clean (unlocked) context. Signed-off-by: Jason Dillaman --- src/librbd/AioCompletion.cc | 11 +++++++++++ src/librbd/AioCompletion.h | 8 ++++++-- src/librbd/internal.cc | 2 +- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/librbd/AioCompletion.cc b/src/librbd/AioCompletion.cc index 58616d16c269..40dbc0aa0813 100644 --- a/src/librbd/AioCompletion.cc +++ b/src/librbd/AioCompletion.cc @@ -171,6 +171,17 @@ namespace librbd { m_completion->complete_request(m_cct, r); } + void C_CacheRead::complete(int r) { + if (!m_enqueued) { + // cache_lock creates a lock ordering issue -- so re-execute this context + // outside the cache_lock + m_enqueued = true; + m_image_ctx.op_work_queue->queue(this, r); + return; + } + Context::complete(r); + } + void C_CacheRead::finish(int r) { m_req->complete(r); diff --git a/src/librbd/AioCompletion.h b/src/librbd/AioCompletion.h index ddf0a14761c5..362f0e6be596 100644 --- a/src/librbd/AioCompletion.h +++ b/src/librbd/AioCompletion.h @@ -183,11 +183,15 @@ namespace librbd { class C_CacheRead : public Context { public: - explicit C_CacheRead(AioRead *req) : m_req(req) {} - virtual ~C_CacheRead() {} + explicit C_CacheRead(ImageCtx *ictx, AioRead *req) + : m_image_ctx(*ictx), m_req(req), m_enqueued(false) {} + virtual void complete(int r); + protected: virtual void finish(int r); private: + ImageCtx &m_image_ctx; AioRead *m_req; + bool m_enqueued; }; } diff --git a/src/librbd/internal.cc b/src/librbd/internal.cc index 96e9a554ef77..9add9828c46f 100644 --- a/src/librbd/internal.cc +++ b/src/librbd/internal.cc @@ -4105,7 +4105,7 @@ reprotect_and_return_err: c->add_request(); if (ictx->object_cacher) { - C_CacheRead *cache_comp = new C_CacheRead(req); + C_CacheRead *cache_comp = new C_CacheRead(ictx, req); ictx->aio_read_from_cache(q->oid, q->objectno, &req->data(), q->length, q->offset, cache_comp, op_flags); -- 2.47.3