librbd: complete cache read in a new thread context

The ObjectCacher complete the read callback while still holding
the cache lock.  This introduces lock ordering issues which are
resolved by queuing the completion to execute in a clean (unlocked)
context.

Signed-off-by: Jason Dillaman <dillaman@redhat.com>

diff --git a/src/librbd/AioCompletion.cc b/src/librbd/AioCompletion.cc
index 58616d16c2694a9f16a02a2b078041750b1d93bd..40dbc0aa08133eb5e5963ed025e2da28c8244994 100644 (file)
--- a/src/librbd/AioCompletion.cc
+++ b/src/librbd/AioCompletion.cc
@@ -171,6 +171,17 @@ namespace librbd {
     m_completion->complete_request(m_cct, r);
   }
 
+  void C_CacheRead::complete(int r) {
+    if (!m_enqueued) {
+      // cache_lock creates a lock ordering issue -- so re-execute this context
+      // outside the cache_lock
+      m_enqueued = true;
+      m_image_ctx.op_work_queue->queue(this, r);
+      return;
+    }
+    Context::complete(r);
+  }
+
   void C_CacheRead::finish(int r)
   {
     m_req->complete(r);

diff --git a/src/librbd/AioCompletion.h b/src/librbd/AioCompletion.h
index ddf0a14761c5efa5730984d26863adf692cdff08..362f0e6be59675803b6a8f7b7b055658620de58b 100644 (file)
--- a/src/librbd/AioCompletion.h
+++ b/src/librbd/AioCompletion.h
@@ -183,11 +183,15 @@ namespace librbd {
 
   class C_CacheRead : public Context {
   public:
-    explicit C_CacheRead(AioRead *req) : m_req(req) {}
-    virtual ~C_CacheRead() {}
+    explicit C_CacheRead(ImageCtx *ictx, AioRead *req)
+      : m_image_ctx(*ictx), m_req(req), m_enqueued(false) {}
+    virtual void complete(int r);
+  protected:
     virtual void finish(int r);
   private:
+    ImageCtx &m_image_ctx;
     AioRead *m_req;
+    bool m_enqueued;
   };
 }
 

diff --git a/src/librbd/internal.cc b/src/librbd/internal.cc
index 96e9a554ef77e4f20216a0d4bd6e5e497dcf5376..9add9828c46fdcf235a6ed5d5d845c28a15c95e1 100644 (file)
--- a/src/librbd/internal.cc
+++ b/src/librbd/internal.cc
@@ -4105,7 +4105,7 @@ reprotect_and_return_err:
        c->add_request();
 
        if (ictx->object_cacher) {
-         C_CacheRead *cache_comp = new C_CacheRead(req);
+         C_CacheRead *cache_comp = new C_CacheRead(ictx, req);
          ictx->aio_read_from_cache(q->oid, q->objectno, &req->data(),
                                    q->length, q->offset,
                                    cache_comp, op_flags);