librbd: complete cache read in a new thread context

The ObjectCacher complete the read callback while still holding
the cache lock.  This introduces lock ordering issues which are
resolved by queuing the completion to execute in a clean (unlocked)
context.

Signed-off-by: Jason Dillaman <dillaman@redhat.com>
(cherry picked from commit 0024677dc8adfd610418ca47599dd95d3a5a6612)

diff --git a/src/librbd/AioCompletion.cc b/src/librbd/AioCompletion.cc
index 2663e74c160853c8c6eeef0c432594f217963499..d126efd040a793ac94bf2d2cda6e45431aa703f6 100644 (file)
--- a/src/librbd/AioCompletion.cc
+++ b/src/librbd/AioCompletion.cc
@@ -6,6 +6,7 @@
 #include "common/ceph_context.h"
 #include "common/dout.h"
 #include "common/errno.h"
+#include "common/WorkQueue.h"
 
 #include "librbd/AioRequest.h"
 #include "librbd/internal.h"
@@ -171,6 +172,17 @@ namespace librbd {
     m_completion->complete_request(m_cct, r);
   }
 
+  void C_CacheRead::complete(int r) {
+    if (!m_enqueued) {
+      // cache_lock creates a lock ordering issue -- so re-execute this context
+      // outside the cache_lock
+      m_enqueued = true;
+      m_image_ctx.op_work_queue->queue(this, r);
+      return;
+    }
+    Context::complete(r);
+  }
+
   void C_CacheRead::finish(int r)
   {
     m_req->complete(r);

diff --git a/src/librbd/AioCompletion.h b/src/librbd/AioCompletion.h
index bd527b130d6f54605911e5b74b177680c408202d..991c22517d081d6ef7a9d4bb6cb57d6f66990e0b 100644 (file)
--- a/src/librbd/AioCompletion.h
+++ b/src/librbd/AioCompletion.h
@@ -183,11 +183,15 @@ namespace librbd {
 
   class C_CacheRead : public Context {
   public:
-    explicit C_CacheRead(AioRead *req) : m_req(req) {}
-    virtual ~C_CacheRead() {}
+    explicit C_CacheRead(ImageCtx *ictx, AioRead *req)
+      : m_image_ctx(*ictx), m_req(req), m_enqueued(false) {}
+    virtual void complete(int r);
+  protected:
     virtual void finish(int r);
   private:
+    ImageCtx &m_image_ctx;
     AioRead *m_req;
+    bool m_enqueued;
   };
 }
 

diff --git a/src/librbd/internal.cc b/src/librbd/internal.cc
index be088bd1a151173c7f86b3ec8cb1c2a8011cdf2b..2a82aa648b7279779720f03ef4ace734546ca39b 100644 (file)
--- a/src/librbd/internal.cc
+++ b/src/librbd/internal.cc
@@ -3634,7 +3634,7 @@ reprotect_and_return_err:
        c->add_request();
 
        if (ictx->object_cacher) {
-         C_CacheRead *cache_comp = new C_CacheRead(req);
+         C_CacheRead *cache_comp = new C_CacheRead(ictx, req);
          ictx->aio_read_from_cache(q->oid, q->objectno, &req->data(),
                                    q->length, q->offset,
                                    cache_comp, op_flags);