From: Jason Dillaman <dillaman@redhat.com>
Date: Mon, 13 Jul 2020 17:45:44 +0000 (-0400)
Subject: librbd: fix race condition with AIO completion callbacks
X-Git-Tag: v15.2.5~61^2~2
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=ed44fbe893c702f894c075573fa20c465070220b;p=ceph.git

librbd: fix race condition with AIO completion callbacks

Now that librbd utilizes multiple threads for the IO path, it's
possible for a race condition to occur if a client app is waiting
on a completion to fire and the actual invokation of the
completion.

Signed-off-by: Jason Dillaman <dillaman@redhat.com>
(cherry picked from commit 50694f790245ca90a3b8a644da7b128a7a148cc6)

Conflicts:
	src/librbd/io/AioCompletion.cc: trivial resolution
---

diff --git a/src/librbd/io/AioCompletion.cc b/src/librbd/io/AioCompletion.cc
index afd4a30d7688..2ddefdaccb96 100644
--- a/src/librbd/io/AioCompletion.cc
+++ b/src/librbd/io/AioCompletion.cc
@@ -105,15 +105,11 @@ void AioCompletion::complete() {
     } else {
       complete_cb(rbd_comp, complete_arg);
       complete_event_socket();
+      notify_callbacks_complete();
     }
   } else {
     complete_event_socket();
-  }
-  state = AIO_STATE_COMPLETE;
-
-  {
-    std::unique_lock<std::mutex> locker(lock);
-    cond.notify_all();
+    notify_callbacks_complete();
   }
 
   // note: possible for image to be closed after op marked finished
@@ -260,6 +256,7 @@ void AioCompletion::complete_external_callback() {
     while (ictx->external_callback_completions.pop(aio_comp)) {
       aio_comp->complete_cb(aio_comp->rbd_comp, aio_comp->complete_arg);
       aio_comp->complete_event_socket();
+      aio_comp->notify_callbacks_complete();
     }
 
     ictx->external_callback_in_progress.store(false);
@@ -278,5 +275,12 @@ void AioCompletion::complete_event_socket() {
   }
 }
 
+void AioCompletion::notify_callbacks_complete() {
+  state = AIO_STATE_COMPLETE;
+
+  std::unique_lock<std::mutex> locker(lock);
+  cond.notify_all();
+}
+
 } // namespace io
 } // namespace librbd
diff --git a/src/librbd/io/AioCompletion.h b/src/librbd/io/AioCompletion.h
index 9df547cd2413..ecb3bc66d2cb 100644
--- a/src/librbd/io/AioCompletion.h
+++ b/src/librbd/io/AioCompletion.h
@@ -179,7 +179,7 @@ private:
   void queue_complete();
   void complete_external_callback();
   void complete_event_socket();
-
+  void notify_callbacks_complete();
 };
 
 class C_AioRequest : public Context {