From 2092003ca27e8857a610d305a8f630521ed702cd Mon Sep 17 00:00:00 2001
From: Yingxin Cheng <yingxin.cheng@intel.com>
Date: Fri, 13 Mar 2020 16:58:31 +0800
Subject: [PATCH] crimson/net: allow mark_down() inside ms_handle_reset()

Although it is not necessary to mark_down the connection in its
ms_handle_reset() event, but it can be more convenient to allow it.
And Heartbeat already encounters this assertion failure.

So move the assertion to close_clean() which will help identify problems
if we happen to make ms_handle_reset() wait for messenger shutdown.

Signed-off-by: Yingxin Cheng <yingxin.cheng@intel.com>
---
 src/crimson/net/Protocol.cc | 7 ++-----
 src/crimson/net/Protocol.h  | 3 +++
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/crimson/net/Protocol.cc b/src/crimson/net/Protocol.cc
index 8ae7bb1d3af08..7eb952609ed00 100644
--- a/src/crimson/net/Protocol.cc
+++ b/src/crimson/net/Protocol.cc
@@ -45,7 +45,6 @@ void Protocol::close(bool dispatch_reset,
 {
   if (closed) {
     // already closing
-    assert(close_ready.valid());
     return;
   }
 
@@ -60,10 +59,8 @@ void Protocol::close(bool dispatch_reset,
 #endif
     };
 
-  // close_ready become valid only after state is state_t::closing
-  assert(!close_ready.valid());
-
   // atomic operations
+  closed = true;
   trigger_close();
   if (f_accept_new) {
     (*f_accept_new)();
@@ -71,7 +68,6 @@ void Protocol::close(bool dispatch_reset,
   if (socket) {
     socket->shutdown();
   }
-  closed = true;
   set_write_state(write_state_t::drop);
   auto gate_closed = pending_dispatch.close();
   auto reset_dispatched = seastar::futurize_apply([this, dispatch_reset] {
@@ -86,6 +82,7 @@ void Protocol::close(bool dispatch_reset,
   });
 
   // asynchronous operations
+  assert(!close_ready.valid());
   close_ready = seastar::when_all_succeed(
     std::move(gate_closed).finally([this] {
       if (socket) {
diff --git a/src/crimson/net/Protocol.h b/src/crimson/net/Protocol.h
index c5f8a5b728627..f70f4b0b291c3 100644
--- a/src/crimson/net/Protocol.h
+++ b/src/crimson/net/Protocol.h
@@ -34,6 +34,9 @@ class Protocol {
   void close(bool dispatch_reset, std::optional<std::function<void()>> f_accept_new=std::nullopt);
   seastar::future<> close_clean(bool dispatch_reset) {
     close(dispatch_reset);
+    // it can happen if close_clean() is called inside Dispatcher::ms_handle_reset()
+    // which will otherwise result in deadlock
+    assert(close_ready.valid());
     return close_ready.get_future();
   }
 
-- 
2.39.5