From 6f48da58c382af2c022ee35db9e6de2dbc6949f5 Mon Sep 17 00:00:00 2001
From: Yingxin Cheng <yingxin.cheng@intel.com>
Date: Fri, 16 Jun 2023 15:35:22 +0800
Subject: [PATCH] crimson/net: dispatch the initial acceptance from the socket
 core

It's meaningless to dispatch the initial acceptance always from the
msgr core.

Signed-off-by: Yingxin Cheng <yingxin.cheng@intel.com>
(cherry picked from commit 1e8a39f359d8481bb3038b461063c87384bf3ee8)
---
 src/crimson/net/ProtocolV2.cc |  9 +++++++++
 src/crimson/net/io_handler.cc | 26 ++++++++++++++++++++++++++
 src/crimson/net/io_handler.h  |  5 +++++
 3 files changed, 40 insertions(+)

diff --git a/src/crimson/net/ProtocolV2.cc b/src/crimson/net/ProtocolV2.cc
index 5a9e0c21c5ada..f6a235d4af3fe 100644
--- a/src/crimson/net/ProtocolV2.cc
+++ b/src/crimson/net/ProtocolV2.cc
@@ -186,6 +186,15 @@ void ProtocolV2::start_accept(SocketFRef&& new_socket,
   logger().info("{} ProtocolV2::start_accept(): target_addr={}", conn, _peer_addr);
   messenger.accept_conn(
     seastar::static_pointer_cast<SocketConnection>(conn.shared_from_this()));
+
+  auto cc_seq = crosscore.prepare_submit();
+  gate.dispatch_in_background("set_accepted_sid", conn, [this, cc_seq] {
+    return io_handler.set_accepted_sid(
+        cc_seq,
+        frame_assembler->get_socket_shard_id(),
+        seastar::make_foreign(conn.shared_from_this()));
+  });
+
   execute_accepting();
 }
 
diff --git a/src/crimson/net/io_handler.cc b/src/crimson/net/io_handler.cc
index adcb8611148b9..b138ed0f26e0e 100644
--- a/src/crimson/net/io_handler.cc
+++ b/src/crimson/net/io_handler.cc
@@ -705,6 +705,32 @@ IOHandler::to_new_sid(
   });
 }
 
+seastar::future<> IOHandler::set_accepted_sid(
+    crosscore_t::seq_t cc_seq,
+    seastar::shard_id sid,
+    ConnectionFRef conn_fref)
+{
+  assert(seastar::this_shard_id() == get_shard_id());
+  assert(get_io_state() == io_state_t::none);
+  ceph_assert_always(conn_ref);
+  conn_ref.reset();
+  assert(maybe_prv_shard_states == nullptr);
+  shard_states.reset();
+  shard_states = shard_states_t::create(sid, io_state_t::none);
+  return seastar::smp::submit_to(sid,
+      [this, cc_seq, conn_fref=std::move(conn_fref)]() mutable {
+    // must be the first to proceed
+    ceph_assert_always(crosscore.proceed_or_wait(cc_seq));
+
+    logger().debug("{} set accepted sid", conn);
+    ceph_assert_always(seastar::this_shard_id() == get_shard_id());
+    ceph_assert_always(get_io_state() == io_state_t::none);
+    assert(maybe_prv_shard_states == nullptr);
+    ceph_assert_always(!conn_ref);
+    conn_ref = make_local_shared_foreign(std::move(conn_fref));
+  });
+}
+
 void IOHandler::dispatch_reset(bool is_replace)
 {
   ceph_assert_always(get_io_state() == io_state_t::drop);
diff --git a/src/crimson/net/io_handler.h b/src/crimson/net/io_handler.h
index 108386fc7ae40..0175d2c522c98 100644
--- a/src/crimson/net/io_handler.h
+++ b/src/crimson/net/io_handler.h
@@ -218,6 +218,11 @@ public:
   };
   void print_io_stat(std::ostream &out) const;
 
+  seastar::future<> set_accepted_sid(
+      crosscore_t::seq_t cc_seq,
+      seastar::shard_id sid,
+      ConnectionFRef conn_fref);
+
   /*
    * may be called cross-core
    */
-- 
2.39.5