From: Yingxin Cheng <yingxin.cheng@intel.com>
Date: Tue, 4 Jul 2023 02:24:22 +0000 (+0800)
Subject: crimson/osd: avoid unnecessary foreign copy in ms_dispatch()
X-Git-Tag: v18.2.1~105^2~1
X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=7d95ac020fa08d320982e7a88fa4ede73ec20937;p=ceph.git

crimson/osd: avoid unnecessary foreign copy in ms_dispatch()

Foreign-copy also implies that requests may be out-of-order, which is
incorrect in the data path.

Signed-off-by: Yingxin Cheng <yingxin.cheng@intel.com>
(cherry picked from commit 436706c4612bada876682df967cd40187805e823)
---

diff --git a/src/crimson/osd/osd.cc b/src/crimson/osd/osd.cc
index fcc77548e50c0..f9e00924dcd57 100644
--- a/src/crimson/osd/osd.cc
+++ b/src/crimson/osd/osd.cc
@@ -777,9 +777,7 @@ OSD::ms_dispatch(crimson::net::ConnectionRef conn, MessageRef m)
       case MSG_OSD_PG_UPDATE_LOG_MISSING:
       case MSG_OSD_PG_UPDATE_LOG_MISSING_REPLY:
       {
-        return conn.get_foreign().then([this, m = std::move(m)](auto f_conn) {
-          return shard_dispatchers.local().ms_dispatch(std::move(f_conn), std::move(m));
-        });
+        return shard_dispatchers.local().ms_dispatch(conn, std::move(m));
       }
       default:
       {
@@ -793,7 +791,7 @@ OSD::ms_dispatch(crimson::net::ConnectionRef conn, MessageRef m)
 
 seastar::future<>
 OSD::ShardDispatcher::ms_dispatch(
-  crimson::net::ConnectionFRef f_conn,
+   crimson::net::ConnectionRef conn,
    MessageRef m)
 {
   if (seastar::this_shard_id() != PRIMARY_CORE) {
@@ -801,14 +799,18 @@ OSD::ShardDispatcher::ms_dispatch(
     case CEPH_MSG_OSD_MAP:
     case MSG_COMMAND:
     case MSG_OSD_MARK_ME_DOWN:
-      return container().invoke_on(PRIMARY_CORE,
-      [f_conn = std::move(f_conn), m = std::move(m)]
-      (auto& local_dispatcher) mutable {
-        return local_dispatcher.ms_dispatch(std::move(f_conn), std::move(m));
+      // FIXME: order is not guaranteed in this path
+      return conn.get_foreign(
+      ).then([this, m=std::move(m)](auto f_conn) {
+        return container().invoke_on(PRIMARY_CORE,
+            [f_conn=std::move(f_conn), m=std::move(m)]
+            (auto& local_dispatcher) mutable {
+          auto conn = make_local_shared_foreign(std::move(f_conn));
+          return local_dispatcher.ms_dispatch(conn, std::move(m));
+        });
       });
     }
   }
-  crimson::net::ConnectionRef conn = make_local_shared_foreign(std::move(f_conn));
 
   switch (m->get_type()) {
   case CEPH_MSG_OSD_MAP:
diff --git a/src/crimson/osd/osd.h b/src/crimson/osd/osd.h
index d9b6f1dc71f20..9e7694836d1ad 100644
--- a/src/crimson/osd/osd.h
+++ b/src/crimson/osd/osd.h
@@ -80,8 +80,7 @@ public:
     ~ShardDispatcher() = default;
 
     // Dispatcher methods
-    seastar::future<> ms_dispatch(crimson::net::ConnectionFRef,
-                                                 MessageRef);
+    seastar::future<> ms_dispatch(crimson::net::ConnectionRef, MessageRef);
 
   private:
     bool require_mon_peer(crimson::net::Connection *conn, Ref<Message> m);