This change introduces the shared memory communication (SMC-D) for the cluster network.

SMC-D is faster than ethernet in IBM Z LPARs and/or VMs (zVM or KVM).

Fixes: https://tracker.ceph.com/issues/66702
Signed-off-by: Aliaksei Makarau <aliaksei.makarau@ibm.com>
(cherry picked from commit e65af75a67b445bf7014842e9d9b3cfbae1e464b)

diff --git a/src/common/options/global.yaml.in b/src/common/options/global.yaml.in
index 99cadcdf35e3bce430c3c3c23dc9e0a48c80e804..947b25113f8e669a773c098085ae749bc3d54779 100644 (file)
--- a/src/common/options/global.yaml.in
+++ b/src/common/options/global.yaml.in
@@ -920,7 +920,7 @@ options:
   level: advanced
   desc: Messenger implementation to use for network communication
   fmt_desc: Transport type used by Async Messenger. Can be ``async+posix``,
-    ``async+dpdk`` or ``async+rdma``. Posix uses standard TCP/IP networking and is
+    ``async+dpdk``, ``async+rdma``, or ``async+smc``. Posix uses standard TCP/IP networking and is
     default. Other transports may be experimental and support may be limited.
   default: async+posix
   flags:

diff --git a/src/msg/async/AsyncMessenger.cc b/src/msg/async/AsyncMessenger.cc
index 3a8f9e0c1894cfc67c7a6baffbd5f94da3081d44..00d7b07cc83a1b15be5a7e0d306f61838eca25c4 100644 (file)
--- a/src/msg/async/AsyncMessenger.cc
+++ b/src/msg/async/AsyncMessenger.cc
@@ -374,6 +374,8 @@ AsyncMessenger::AsyncMessenger(CephContext *cct, entity_name_t name,
     transport_type = "rdma";
   else if (type.find("dpdk") != std::string::npos)
     transport_type = "dpdk";
+  else if (type.find("smc") != std::string::npos)
+    transport_type = "smc";
 
   auto single = &cct->lookup_or_create_singleton_object<StackSingleton>(
     "AsyncMessenger::NetworkStack::" + transport_type, true, cct);

diff --git a/src/msg/async/PosixStack.cc b/src/msg/async/PosixStack.cc
index 373bed7dec3e3825d36a34a0daecb15014fe9875..602323b7bc36779a722793b8d782f99d7fae2a20 100644 (file)
--- a/src/msg/async/PosixStack.cc
+++ b/src/msg/async/PosixStack.cc
@@ -335,7 +335,7 @@ int PosixWorker::connect(const entity_addr_t &addr, const SocketOptions &opts, C
   return 0;
 }
 
-PosixNetworkStack::PosixNetworkStack(CephContext *c)
-    : NetworkStack(c)
+PosixNetworkStack::PosixNetworkStack(CephContext *c, bool try_smc)
+    : NetworkStack(c), try_smc(try_smc)
 {
 }

diff --git a/src/msg/async/PosixStack.h b/src/msg/async/PosixStack.h
index a4ff65b28f466e5d2ec65c4cc3e3cad445d65a02..5a4cfd2b57057a574a71749f836d0c2561410c8a 100644 (file)
--- a/src/msg/async/PosixStack.h
+++ b/src/msg/async/PosixStack.h
@@ -28,8 +28,8 @@ class PosixWorker : public Worker {
   ceph::NetHandler net;
   void initialize() override;
  public:
-  PosixWorker(CephContext *c, unsigned i)
-      : Worker(c, i), net(c) {}
+  PosixWorker(CephContext *c, unsigned i, bool try_smc)
+      : Worker(c, i), net(c, try_smc) {}
   int listen(entity_addr_t &sa,
             unsigned addr_slot,
             const SocketOptions &opt,
@@ -39,13 +39,14 @@ class PosixWorker : public Worker {
 
 class PosixNetworkStack : public NetworkStack {
   std::vector<std::thread> threads;
+  bool try_smc;
 
   virtual Worker* create_worker(CephContext *c, unsigned worker_id) override {
-    return new PosixWorker(c, worker_id);
+    return new PosixWorker(c, worker_id, try_smc);
   }
 
  public:
-  explicit PosixNetworkStack(CephContext *c);
+  explicit PosixNetworkStack(CephContext *c, bool try_smc);
 
   void spawn_worker(std::function<void ()> &&func) override {
     threads.emplace_back(std::move(func));

diff --git a/src/msg/async/Stack.cc b/src/msg/async/Stack.cc
index 4ef660df1a856b2ea5dc2bfeac91ea34e55070be..6b7d033834e32f6e98f100a1927ac8e5357c5033 100644 (file)
--- a/src/msg/async/Stack.cc
+++ b/src/msg/async/Stack.cc
@@ -65,7 +65,9 @@ std::shared_ptr<NetworkStack> NetworkStack::create(CephContext *c,
   std::shared_ptr<NetworkStack> stack = nullptr;
 
   if (t == "posix")
-    stack.reset(new PosixNetworkStack(c));
+    stack.reset(new PosixNetworkStack(c, false));
+  else if (t == "smc")
+    stack.reset(new PosixNetworkStack(c, true));
 #ifdef HAVE_RDMA
   else if (t == "rdma")
     stack.reset(new RDMAStack(c));

diff --git a/src/msg/async/net_handler.cc b/src/msg/async/net_handler.cc
index 862c834bc8d7965b8b60863939e41ef795333428..d420fe0bf60029ba968af40900f2b47684275473 100644 (file)
--- a/src/msg/async/net_handler.cc
+++ b/src/msg/async/net_handler.cc
@@ -31,14 +31,33 @@
 #undef dout_prefix
 #define dout_prefix *_dout << "NetHandler "
 
+#ifndef SMCPROTO_SMC
+  #define SMCPROTO_SMC           0       /* SMC protocol, IPv4 */
+  #define SMCPROTO_SMC6          1       /* SMC protocol, IPv6 */
+#endif
+
 namespace ceph{
 
 int NetHandler::create_socket(int domain, bool reuse_addr)
 {
   int s;
   int r = 0;
+  int protocol = IPPROTO_TCP;
+
+#if defined(AF_SMC)
+  if (this->try_smc) {
+    /* check if socket is eligible for AF_SMC */
+    if (domain == AF_INET || domain == AF_INET6) {
+      if (domain == AF_INET)
+        protocol = SMCPROTO_SMC;
+      else /* AF_INET6 */
+        protocol = SMCPROTO_SMC6;
+      domain = AF_SMC;
+    }
+  }
+#endif
 
-  if ((s = socket_cloexec(domain, SOCK_STREAM, 0)) == -1) {
+  if ((s = socket_cloexec(domain, SOCK_STREAM, protocol)) == -1) {
     r = ceph_sock_errno();
     lderr(cct) << __func__ << " couldn't create socket " << cpp_strerror(r) << dendl;
     return -r;

diff --git a/src/msg/async/net_handler.h b/src/msg/async/net_handler.h
index 1904237724fe73345269883ce233e7e2e912169d..afb44563f1254e85e6bb528453fe471d5819fdc4 100644 (file)
--- a/src/msg/async/net_handler.h
+++ b/src/msg/async/net_handler.h
@@ -23,9 +23,11 @@ namespace ceph {
     int generic_connect(const entity_addr_t& addr, const entity_addr_t& bind_addr, bool nonblock);
 
     CephContext *cct;
+    bool try_smc;
    public:
     int create_socket(int domain, bool reuse_addr=false);
-    explicit NetHandler(CephContext *c): cct(c) {}
+    explicit NetHandler(CephContext *c, bool try_smc=false): cct(c), try_smc(try_smc) {
+    }
     int set_nonblock(int sd);
     int set_socket_options(int sd, bool nodelay, int size);
     int connect(const entity_addr_t &addr, const entity_addr_t& bind_addr);