From f07cd30733a674c3577bfa1853bd722851783b97 Mon Sep 17 00:00:00 2001
From: Max Kellermann <max.kellermann@ionos.com>
Date: Mon, 7 Oct 2024 18:57:59 +0200
Subject: [PATCH] msg/async/ProtocolV[12]: unlock the `write_lock` before doing
 I/O

Reduce lock time/contention.

Signed-off-by: Max Kellermann <max.kellermann@ionos.com>
---
 src/msg/async/ProtocolV1.cc | 14 ++++++++++++--
 src/msg/async/ProtocolV2.cc | 14 ++++++++++++--
 2 files changed, 24 insertions(+), 4 deletions(-)

diff --git a/src/msg/async/ProtocolV1.cc b/src/msg/async/ProtocolV1.cc
index b14de7b1e56..996f5a9465d 100644
--- a/src/msg/async/ProtocolV1.cc
+++ b/src/msg/async/ProtocolV1.cc
@@ -223,7 +223,7 @@ void ProtocolV1::send_message(Message *m) {
     is_prepared = true;
   }
 
-  std::lock_guard<std::mutex> l(connection->write_lock);
+  std::unique_lock l{connection->write_lock};
   // "features" changes will change the payload encoding
   if (can_fast_prepare &&
       (can_write == WriteStatus::NOWRITE || connection->get_features() != f)) {
@@ -246,6 +246,11 @@ void ProtocolV1::send_message(Message *m) {
                    << dendl;
     if (can_write != WriteStatus::REPLACING && !write_in_progress) {
       write_in_progress = true;
+
+      /* unlock the mutex now because dispatch_event_external() may
+         block waiting for another mutex */
+      l.unlock();
+
       connection->center->dispatch_event_external(connection->write_handler);
     }
   }
@@ -271,9 +276,14 @@ void ProtocolV1::prepare_send_message(uint64_t features, Message *m,
 
 void ProtocolV1::send_keepalive() {
   ldout(cct, 10) << __func__ << dendl;
-  std::lock_guard<std::mutex> l(connection->write_lock);
+  std::unique_lock l{connection->write_lock};
   if (can_write != WriteStatus::CLOSED) {
     keepalive = true;
+
+    /* unlock the mutex now because dispatch_event_external() may
+       block waiting for another mutex */
+    l.unlock();
+
     connection->center->dispatch_event_external(connection->write_handler);
   }
 }
diff --git a/src/msg/async/ProtocolV2.cc b/src/msg/async/ProtocolV2.cc
index 6d44d6c783f..0ca8afcad25 100644
--- a/src/msg/async/ProtocolV2.cc
+++ b/src/msg/async/ProtocolV2.cc
@@ -439,7 +439,7 @@ void ProtocolV2::send_message(Message *m) {
     is_prepared = false;
   }
 
-  std::lock_guard<std::mutex> l(connection->write_lock);
+  std::unique_lock l{connection->write_lock};
   // "features" changes will change the payload encoding
   if (can_fast_prepare && (!can_write || connection->get_features() != f)) {
     // ensure the correctness of message encoding
@@ -463,6 +463,11 @@ void ProtocolV2::send_message(Message *m) {
                    << dendl;
     if (((!replacing && can_write) || state == STANDBY) && !write_in_progress) {
       write_in_progress = true;
+
+      /* unlock the mutex now because dispatch_event_external() may
+         block waiting for another mutex */
+      l.unlock();
+
       connection->center->dispatch_event_external(connection->write_handler);
     }
   }
@@ -470,9 +475,14 @@ void ProtocolV2::send_message(Message *m) {
 
 void ProtocolV2::send_keepalive() {
   ldout(cct, 10) << __func__ << dendl;
-  std::lock_guard<std::mutex> l(connection->write_lock);
+  std::unique_lock l{connection->write_lock};
   if (state != CLOSED) {
     keepalive = true;
+
+    /* unlock the mutex now because dispatch_event_external() may
+       block waiting for another mutex */
+    l.unlock();
+
     connection->center->dispatch_event_external(connection->write_handler);
   }
 }
-- 
2.39.5