From c9ff6cf1945b248b38e83484aa4a22c206da846b Mon Sep 17 00:00:00 2001
From: Jason Dillaman <dillaman@redhat.com>
Date: Tue, 2 Apr 2019 15:33:00 -0400
Subject: [PATCH] librbd: empty IO requests should complete in new context

This will avoid the possibility of a lock being held by the caller
and having the IO immediately complete back to the calling state
machine.

Signed-off-by: Jason Dillaman <dillaman@redhat.com>
---
 src/librbd/io/AioCompletion.cc | 16 +++++++++++-----
 src/librbd/io/AioCompletion.h  | 18 +-----------------
 src/librbd/io/ImageRequest.cc  |  2 +-
 src/librbd/journal/Replay.cc   |  2 +-
 4 files changed, 14 insertions(+), 24 deletions(-)

diff --git a/src/librbd/io/AioCompletion.cc b/src/librbd/io/AioCompletion.cc
index 661cd57255b..c106ee11a78 100644
--- a/src/librbd/io/AioCompletion.cc
+++ b/src/librbd/io/AioCompletion.cc
@@ -145,11 +145,17 @@ void AioCompletion::set_request_count(uint32_t count) {
 
   ldout(cct, 20) << "pending=" << count << dendl;
   ceph_assert(pending_count == 0);
-  pending_count = count;
-  lock.Unlock();
 
-  // if no pending requests, completion will fire now
-  unblock();
+  if (count > 0) {
+    pending_count = count;
+    lock.Unlock();
+  } else {
+    pending_count = 1;
+    lock.Unlock();
+
+    // ensure completion fires in clean lock context
+    ictx->op_work_queue->queue(new C_AioRequest(this), 0);
+  }
 }
 
 void AioCompletion::complete_request(ssize_t r)
@@ -169,7 +175,7 @@ void AioCompletion::complete_request(ssize_t r)
 
   ldout(cct, 20) << "cb=" << complete_cb << ", "
                  << "pending=" << pending_count << dendl;
-  if (!count && blockers == 0) {
+  if (!count) {
     finalize(rval);
     complete();
   }
diff --git a/src/librbd/io/AioCompletion.h b/src/librbd/io/AioCompletion.h
index ef0a3c8999b..ffc8129c7e6 100644
--- a/src/librbd/io/AioCompletion.h
+++ b/src/librbd/io/AioCompletion.h
@@ -50,7 +50,6 @@ struct AioCompletion {
   void *complete_arg;
   rbd_completion_t rbd_comp;
   uint32_t pending_count;   ///< number of requests
-  uint32_t blockers;
   int ref;
   bool released;
   ImageCtx *ictx;
@@ -106,8 +105,7 @@ struct AioCompletion {
   AioCompletion() : lock("AioCompletion::lock", true, false),
                     state(AIO_STATE_PENDING), rval(0), complete_cb(NULL),
                     complete_arg(NULL), rbd_comp(NULL),
-                    pending_count(0), blockers(1),
-                    ref(1), released(false), ictx(NULL),
+                    pending_count(0), ref(1), released(false), ictx(NULL),
                     aio_type(AIO_TYPE_NONE), m_xlist_item(this),
                     event_notify(false) {
   }
@@ -188,20 +186,6 @@ struct AioCompletion {
     }
   }
 
-  void block() {
-    Mutex::Locker l(lock);
-    ++blockers;
-  }
-  void unblock() {
-    Mutex::Locker l(lock);
-    ceph_assert(blockers > 0);
-    --blockers;
-    if (pending_count == 0 && blockers == 0) {
-      finalize(rval);
-      complete();
-    }
-  }
-
   void set_event_notify(bool s) {
     Mutex::Locker l(lock);
     event_notify = s;
diff --git a/src/librbd/io/ImageRequest.cc b/src/librbd/io/ImageRequest.cc
index a86720621c6..5a74fabf0e9 100644
--- a/src/librbd/io/ImageRequest.cc
+++ b/src/librbd/io/ImageRequest.cc
@@ -395,7 +395,7 @@ void AbstractImageWriteRequest<I>::send_request() {
     send_object_requests(object_extents, snapc, journal_tid);
   } else {
     // no IO to perform -- fire completion
-    aio_comp->unblock();
+    aio_comp->set_request_count(0);
   }
 
   update_stats(clip_len);
diff --git a/src/librbd/journal/Replay.cc b/src/librbd/journal/Replay.cc
index 25dd61203f7..f96153e7edb 100644
--- a/src/librbd/journal/Replay.cc
+++ b/src/librbd/journal/Replay.cc
@@ -1176,7 +1176,7 @@ bool Replay<I>::clipped_io(uint64_t image_offset, io::AioCompletion *aio_comp) {
     // it wouldn't have been recorded in the journal
     ldout(cct, 5) << ": no-op IO event beyond image size" << dendl;
     aio_comp->get();
-    aio_comp->unblock();
+    aio_comp->set_request_count(0);
     aio_comp->put();
     return true;
   }
-- 
2.39.5