From 50694f790245ca90a3b8a644da7b128a7a148cc6 Mon Sep 17 00:00:00 2001
From: Jason Dillaman <dillaman@redhat.com>
Date: Mon, 13 Jul 2020 13:45:44 -0400
Subject: [PATCH] librbd: fix race condition with AIO completion callbacks

Now that librbd utilizes multiple threads for the IO path, it's
possible for a race condition to occur if a client app is waiting
on a completion to fire and the actual invokation of the
completion.

Signed-off-by: Jason Dillaman <dillaman@redhat.com>
---
 src/librbd/io/AioCompletion.cc | 16 ++++++++++------
 src/librbd/io/AioCompletion.h  |  2 +-
 2 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/src/librbd/io/AioCompletion.cc b/src/librbd/io/AioCompletion.cc
index 9e1d1e18fe7a3..cad103364244c 100644
--- a/src/librbd/io/AioCompletion.cc
+++ b/src/librbd/io/AioCompletion.cc
@@ -107,15 +107,11 @@ void AioCompletion::complete() {
     } else {
       complete_cb(rbd_comp, complete_arg);
       complete_event_socket();
+      notify_callbacks_complete();
     }
   } else {
     complete_event_socket();
-  }
-  state = AIO_STATE_COMPLETE;
-
-  {
-    std::unique_lock<std::mutex> locker(lock);
-    cond.notify_all();
+    notify_callbacks_complete();
   }
 
   if (image_dispatcher_ctx != nullptr) {
@@ -263,6 +259,7 @@ void AioCompletion::complete_external_callback() {
   boost::asio::dispatch(ictx->asio_engine->get_api_strand(), [this]() {
       complete_cb(rbd_comp, complete_arg);
       complete_event_socket();
+      notify_callbacks_complete();
       put();
     });
 }
@@ -274,5 +271,12 @@ void AioCompletion::complete_event_socket() {
   }
 }
 
+void AioCompletion::notify_callbacks_complete() {
+  state = AIO_STATE_COMPLETE;
+
+  std::unique_lock<std::mutex> locker(lock);
+  cond.notify_all();
+}
+
 } // namespace io
 } // namespace librbd
diff --git a/src/librbd/io/AioCompletion.h b/src/librbd/io/AioCompletion.h
index 43ef324659778..4ae93fe36d209 100644
--- a/src/librbd/io/AioCompletion.h
+++ b/src/librbd/io/AioCompletion.h
@@ -181,7 +181,7 @@ private:
   void queue_complete();
   void complete_external_callback();
   void complete_event_socket();
-
+  void notify_callbacks_complete();
 };
 
 class C_AioRequest : public Context {
-- 
2.39.5