librbd: fix race condition with AIO completion callbacks

Now that librbd utilizes multiple threads for the IO path, it's
possible for a race condition to occur if a client app is waiting
on a completion to fire and the actual invokation of the
completion.

Signed-off-by: Jason Dillaman <dillaman@redhat.com>

diff --git a/src/librbd/io/AioCompletion.cc b/src/librbd/io/AioCompletion.cc
index 9e1d1e18fe7a3ac8af5bbd86a7b522c01c1c1808..cad103364244c1a1f4a970a86c679dee4d3c2f72 100644 (file)
--- a/src/librbd/io/AioCompletion.cc
+++ b/src/librbd/io/AioCompletion.cc
@@ -107,15 +107,11 @@ void AioCompletion::complete() {
     } else {
       complete_cb(rbd_comp, complete_arg);
       complete_event_socket();
+      notify_callbacks_complete();
     }
   } else {
     complete_event_socket();
-  }
-  state = AIO_STATE_COMPLETE;
-
-  {
-    std::unique_lock<std::mutex> locker(lock);
-    cond.notify_all();
+    notify_callbacks_complete();
   }
 
   if (image_dispatcher_ctx != nullptr) {
@@ -263,6 +259,7 @@ void AioCompletion::complete_external_callback() {
   boost::asio::dispatch(ictx->asio_engine->get_api_strand(), [this]() {
       complete_cb(rbd_comp, complete_arg);
       complete_event_socket();
+      notify_callbacks_complete();
       put();
     });
 }
@@ -274,5 +271,12 @@ void AioCompletion::complete_event_socket() {
   }
 }
 
+void AioCompletion::notify_callbacks_complete() {
+  state = AIO_STATE_COMPLETE;
+
+  std::unique_lock<std::mutex> locker(lock);
+  cond.notify_all();
+}
+
 } // namespace io
 } // namespace librbd

diff --git a/src/librbd/io/AioCompletion.h b/src/librbd/io/AioCompletion.h
index 43ef32465977877c2dd1a2c3967ab162fbd2a6ac..4ae93fe36d209426e6c4fbc13238b272f13aeb28 100644 (file)
--- a/src/librbd/io/AioCompletion.h
+++ b/src/librbd/io/AioCompletion.h
@@ -181,7 +181,7 @@ private:
   void queue_complete();
   void complete_external_callback();
   void complete_event_socket();
-
+  void notify_callbacks_complete();
 };
 
 class C_AioRequest : public Context {