librbd: IO ops will now asynchronously refresh the image if needed

This avoids blocking the AIO path with the previous synchronous
refresh path.

Signed-off-by: Jason Dillaman <dillaman@redhat.com>

diff --git a/src/librbd/AioImageRequest.cc b/src/librbd/AioImageRequest.cc
index c4d677aab652d2aba506ad56a5c2ffa98ecc5842..f0c3228119c6b614590285e841e4b967531db3a7 100644 (file)
--- a/src/librbd/AioImageRequest.cc
+++ b/src/librbd/AioImageRequest.cc
@@ -117,15 +117,14 @@ void AioImageRequest::send() {
                  << "completion=" << m_aio_comp <<  dendl;
 
   m_aio_comp->get();
-  int r = m_image_ctx.state->refresh_if_required(m_image_ctx.owner_lock);
-  if (r < 0) {
-    m_aio_comp->fail(cct, r);
-    return;
-  }
-
   send_request();
 }
 
+void AioImageRequest::fail(int r) {
+  m_aio_comp->get();
+  m_aio_comp->fail(m_image_ctx.cct, r);
+}
+
 void AioImageRead::send_request() {
   CephContext *cct = m_image_ctx.cct;
 

diff --git a/src/librbd/AioImageRequest.h b/src/librbd/AioImageRequest.h
index de14bfd783c2c6b6dd6ac24a008457cb7c849852..6ee6d64322ba5d60eb0a3103f7a183451cfc0bda 100644 (file)
--- a/src/librbd/AioImageRequest.h
+++ b/src/librbd/AioImageRequest.h
@@ -40,6 +40,7 @@ public:
   }
 
   void send();
+  void fail(int r);
 
 protected:
   typedef std::list<AioObjectRequest *> AioObjectRequests;

diff --git a/src/librbd/AioImageRequestWQ.cc b/src/librbd/AioImageRequestWQ.cc
index f57eae97bea578ad00a66788c2b6b31fca4d097d..9871bb0535ef4c1b7f25531dc4f55a1ad5634bcf 100644 (file)
--- a/src/librbd/AioImageRequestWQ.cc
+++ b/src/librbd/AioImageRequestWQ.cc
@@ -7,6 +7,7 @@
 #include "librbd/AioImageRequest.h"
 #include "librbd/ExclusiveLock.h"
 #include "librbd/ImageCtx.h"
+#include "librbd/ImageState.h"
 #include "librbd/internal.h"
 #include "librbd/Utils.h"
 
@@ -22,7 +23,8 @@ AioImageRequestWQ::AioImageRequestWQ(ImageCtx *image_ctx, const string &name,
     m_image_ctx(*image_ctx),
     m_lock(util::unique_lock_name("AioImageRequestWQ::m_lock", this)),
     m_write_blockers(0), m_in_progress_writes(0), m_queued_writes(0),
-    m_in_flight_ops(0), m_shutdown(false), m_on_shutdown(nullptr) {
+    m_in_flight_ops(0), m_refresh_in_progress(false),
+    m_shutdown(false), m_on_shutdown(nullptr) {
   CephContext *cct = m_image_ctx.cct;
   ldout(cct, 5) << this << " " << ": ictx=" << image_ctx << dendl;
 }
@@ -261,23 +263,32 @@ void AioImageRequestWQ::unblock_writes() {
 
 void *AioImageRequestWQ::_void_dequeue() {
   AioImageRequest *peek_item = front();
-  if (peek_item == NULL) {
+  if (peek_item == NULL || m_refresh_in_progress) {
     return NULL;
   }
 
-  {
-    if (peek_item->is_write_op()) {
-      RWLock::RLocker locker(m_lock);
-      if (m_write_blockers > 0) {
-        return NULL;
-      }
-      m_in_progress_writes.inc();
+  if (peek_item->is_write_op()) {
+    RWLock::RLocker locker(m_lock);
+    if (m_write_blockers > 0) {
+      return NULL;
     }
+    m_in_progress_writes.inc();
   }
 
   AioImageRequest *item = reinterpret_cast<AioImageRequest *>(
     ThreadPool::PointerWQ<AioImageRequest>::_void_dequeue());
   assert(peek_item == item);
+
+  if (m_image_ctx.state->is_refresh_required()) {
+    ldout(m_image_ctx.cct, 15) << "image refresh required: delaying IO " << item
+                               << dendl;
+    m_refresh_in_progress = true;
+
+    get_pool_lock().Unlock();
+    m_image_ctx.state->refresh(new C_RefreshFinish(this, item));
+    get_pool_lock().Lock();
+    return NULL;
+  }
   return item;
 }
 
@@ -380,6 +391,21 @@ void AioImageRequestWQ::queue(AioImageRequest *req) {
   }
 }
 
+void AioImageRequestWQ::handle_refreshed(int r, AioImageRequest *req) {
+  CephContext *cct = m_image_ctx.cct;
+  ldout(cct, 15) << "resuming IO after image refresh: r=" << r << ", "
+                 << "req=" << req << dendl;
+  if (r < 0) {
+    req->fail(r);
+  } else {
+    process(req);
+    process_finish();
+
+    m_refresh_in_progress = false;
+    signal();
+  }
+}
+
 void AioImageRequestWQ::handle_blocked_writes(int r) {
   Contexts contexts;
   {

diff --git a/src/librbd/AioImageRequestWQ.h b/src/librbd/AioImageRequestWQ.h
index 0eeb716bc992d4c4e2ea10481762e36d8fd2073c..c84548955a338106349f3fc16b90762a3402ca8f 100644 (file)
--- a/src/librbd/AioImageRequestWQ.h
+++ b/src/librbd/AioImageRequestWQ.h
@@ -6,8 +6,10 @@
 
 #include "include/Context.h"
 #include "include/atomic.h"
-#include "common/WorkQueue.h"
+#include "common/Cond.h"
 #include "common/RWLock.h"
+#include "common/WorkQueue.h"
+#include <list>
 
 namespace librbd {
 
@@ -57,6 +59,19 @@ protected:
 private:
   typedef std::list<Context *> Contexts;
 
+  struct C_RefreshFinish : public Context {
+    AioImageRequestWQ *aio_work_queue;
+    AioImageRequest *aio_image_request;
+
+    C_RefreshFinish(AioImageRequestWQ *aio_work_queue,
+                    AioImageRequest *aio_image_request)
+      : aio_work_queue(aio_work_queue), aio_image_request(aio_image_request) {
+    }
+    virtual void finish(int r) override {
+      aio_work_queue->handle_refreshed(r, aio_image_request);
+    }
+  };
+
   struct C_BlockedWrites : public Context {
     AioImageRequestWQ *aio_work_queue;
     C_BlockedWrites(AioImageRequestWQ *_aio_work_queue)
@@ -76,6 +91,8 @@ private:
   atomic_t m_queued_writes;
   atomic_t m_in_flight_ops;
 
+  bool m_refresh_in_progress;
+
   bool m_shutdown;
   Context *m_on_shutdown;
 
@@ -86,6 +103,7 @@ private:
   bool is_lock_required() const;
   void queue(AioImageRequest *req);
 
+  void handle_refreshed(int r, AioImageRequest *req);
   void handle_blocked_writes(int r);
 };