From: Samuel Just <sjust@redhat.com>
Date: Thu, 8 Sep 2022 18:40:43 +0000 (+0000)
Subject: crimson/net/Connection: let send and keepalive be called from foreign cores
X-Git-Tag: v18.1.0~1115^2~8
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=f152424631a4993bacfb46a8abf35efa705f0259;p=ceph.git

crimson/net/Connection: let send and keepalive be called from foreign cores

Signed-off-by: Samuel Just <sjust@redhat.com>
---

diff --git a/src/crimson/net/Connection.h b/src/crimson/net/Connection.h
index 95d7d807fa0f..b2d2b236f2f6 100644
--- a/src/crimson/net/Connection.h
+++ b/src/crimson/net/Connection.h
@@ -28,6 +28,14 @@ class Interceptor;
 
 using seq_num_t = uint64_t;
 
+/**
+ * Connection
+ *
+ * Abstraction for messenger connections.
+ *
+ * Except when otherwise specified, methods must be invoked from the core on which
+ * the connection originates.
+ */
 class Connection : public seastar::enable_shared_from_this<Connection> {
   entity_name_t peer_name = {0, entity_name_t::NEW};
 
@@ -125,11 +133,22 @@ class Connection : public seastar::enable_shared_from_this<Connection> {
   virtual bool peer_wins() const = 0;
 #endif
 
-  /// send a message over a connection that has completed its handshake
+  /**
+   * send
+   *
+   * Send a message over a connection that has completed its handshake.
+   * May be invoked from any core.
+   */
   virtual seastar::future<> send(MessageURef msg) = 0;
 
-  /// send a keepalive message over a connection that has completed its
-  /// handshake
+  /**
+   * keepalive
+   *
+   * Send a keepalive message over a connection that has completed its
+   * handshake.
+   *
+   * May be invoked from any core.
+   */
   virtual seastar::future<> keepalive() = 0;
 
   // close the connection and cancel any any pending futures from read/send,
diff --git a/src/crimson/net/SocketConnection.cc b/src/crimson/net/SocketConnection.cc
index 2fb2f1f1ec8f..65ab399156bc 100644
--- a/src/crimson/net/SocketConnection.cc
+++ b/src/crimson/net/SocketConnection.cc
@@ -27,7 +27,8 @@ using crimson::common::local_conf;
 
 SocketConnection::SocketConnection(SocketMessenger& messenger,
                                    ChainedDispatchers& dispatchers)
-  : messenger(messenger),
+  : core(messenger.shard_id()),
+    messenger(messenger),
     protocol(std::make_unique<ProtocolV2>(dispatchers, *this, messenger))
 {
 #ifdef UNIT_TESTS_BUILT
@@ -72,14 +73,20 @@ bool SocketConnection::peer_wins() const
 
 seastar::future<> SocketConnection::send(MessageURef msg)
 {
-  assert(seastar::this_shard_id() == shard_id());
-  return protocol->send(std::move(msg));
+  return seastar::smp::submit_to(
+    shard_id(),
+    [this, msg=std::move(msg)]() mutable {
+      return protocol->send(std::move(msg));
+    });
 }
 
 seastar::future<> SocketConnection::keepalive()
 {
-  assert(seastar::this_shard_id() == shard_id());
-  return protocol->keepalive();
+  return seastar::smp::submit_to(
+    shard_id(),
+    [this] {
+      return protocol->keepalive();
+    });
 }
 
 void SocketConnection::mark_down()
@@ -128,7 +135,7 @@ SocketConnection::close_clean(bool dispatch_reset)
 }
 
 seastar::shard_id SocketConnection::shard_id() const {
-  return messenger.shard_id();
+  return core;
 }
 
 seastar::socket_address SocketConnection::get_local_address() const {
diff --git a/src/crimson/net/SocketConnection.h b/src/crimson/net/SocketConnection.h
index 4aad9d65747c..b26f77dd7141 100644
--- a/src/crimson/net/SocketConnection.h
+++ b/src/crimson/net/SocketConnection.h
@@ -29,6 +29,7 @@ class SocketConnection;
 using SocketConnectionRef = seastar::shared_ptr<SocketConnection>;
 
 class SocketConnection : public Connection {
+  const seastar::shard_id core;
   SocketMessenger& messenger;
   std::unique_ptr<Protocol> protocol;