From 5583499d8e32db774ea877858573ea41c8dfd9c8 Mon Sep 17 00:00:00 2001
From: Jason Dillaman <dillaman@redhat.com>
Date: Mon, 16 Mar 2020 13:17:17 -0400
Subject: [PATCH] librbd: defer event socket completion until after callback
 issued

A change post-Nautilus fixed an issue where multiple threads could
concurrently invoke callbacks to librbd clients. However, it also
introduced the potential that a callback hasn't yet fired by the
time the event socket is completed. This resulted in a crash of
fio under high-throughput testing since it expected both a callback
and the event socket completion, in that order.

Signed-off-by: Jason Dillaman <dillaman@redhat.com>
---
 src/librbd/io/AioCompletion.cc | 16 +++++++++++-----
 src/librbd/io/AioCompletion.h  |  1 +
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/src/librbd/io/AioCompletion.cc b/src/librbd/io/AioCompletion.cc
index 603c1f18fb1a..afd4a30d7688 100644
--- a/src/librbd/io/AioCompletion.cc
+++ b/src/librbd/io/AioCompletion.cc
@@ -104,12 +104,10 @@ void AioCompletion::complete() {
       complete_external_callback();
     } else {
       complete_cb(rbd_comp, complete_arg);
+      complete_event_socket();
     }
-  }
-
-  if (ictx != nullptr && event_notify && ictx->event_socket.is_valid()) {
-    ictx->event_socket_completions.push(this);
-    ictx->event_socket.notify();
+  } else {
+    complete_event_socket();
   }
   state = AIO_STATE_COMPLETE;
 
@@ -261,6 +259,7 @@ void AioCompletion::complete_external_callback() {
     AioCompletion* aio_comp;
     while (ictx->external_callback_completions.pop(aio_comp)) {
       aio_comp->complete_cb(aio_comp->rbd_comp, aio_comp->complete_arg);
+      aio_comp->complete_event_socket();
     }
 
     ictx->external_callback_in_progress.store(false);
@@ -272,5 +271,12 @@ void AioCompletion::complete_external_callback() {
   }
 }
 
+void AioCompletion::complete_event_socket() {
+  if (ictx != nullptr && event_notify && ictx->event_socket.is_valid()) {
+    ictx->event_socket_completions.push(this);
+    ictx->event_socket.notify();
+  }
+}
+
 } // namespace io
 } // namespace librbd
diff --git a/src/librbd/io/AioCompletion.h b/src/librbd/io/AioCompletion.h
index cdaaa6a32b50..9df547cd2413 100644
--- a/src/librbd/io/AioCompletion.h
+++ b/src/librbd/io/AioCompletion.h
@@ -178,6 +178,7 @@ struct AioCompletion {
 private:
   void queue_complete();
   void complete_external_callback();
+  void complete_event_socket();
 
 };
 
-- 
2.47.3