crimson/net: use boost::container::small_vector in ChainedDispatchers

Signed-off-by: Yingxin Cheng <yingxin.cheng@intel.com>

diff --git a/src/crimson/net/Fwd.h b/src/crimson/net/Fwd.h
index 77818704a85dcd895f07bee71432bcd4d93962af..e10120571f3353fc8fe05403377c387152db13e5 100644 (file)
--- a/src/crimson/net/Fwd.h
+++ b/src/crimson/net/Fwd.h
@@ -14,6 +14,7 @@
 
 #pragma once
 
+#include <boost/container/small_vector.hpp>
 #include <seastar/core/future.hh>
 #include <seastar/core/future-util.hh>
 #include <seastar/core/shared_ptr.hh>
@@ -38,7 +39,10 @@ using stop_t = seastar::stop_iteration;
 class Connection;
 using ConnectionRef = seastar::shared_ptr<Connection>;
 
+class Dispatcher;
 class ChainedDispatchers;
+constexpr std::size_t NUM_DISPATCHERS = 4u;
+using dispatchers_t = boost::container::small_vector<Dispatcher*, NUM_DISPATCHERS>;
 
 class Messenger;
 using MessengerRef = seastar::shared_ptr<Messenger>;

diff --git a/src/crimson/net/Messenger.h b/src/crimson/net/Messenger.h
index 9133cdd0356f4c7bfb16d48c452ce011c2ffb808..2b39fbf63a668527e1250e5eb798654f151869df 100644 (file)
--- a/src/crimson/net/Messenger.h
+++ b/src/crimson/net/Messenger.h
@@ -14,8 +14,6 @@
 
 #pragma once
 
-#include <list>
-
 #include "Fwd.h"
 #include "crimson/common/throttle.h"
 #include "msg/Message.h"
@@ -34,8 +32,6 @@ namespace crimson::net {
 class Interceptor;
 #endif
 
-class Dispatcher;
-
 using Throttle = crimson::common::Throttle;
 using SocketPolicy = ceph::net::Policy<Throttle>;
 
@@ -77,13 +73,7 @@ public:
                                        uint32_t min_port, uint32_t max_port) = 0;
 
   /// start the messenger
-  virtual seastar::future<> start(const std::list<Dispatcher*>&) = 0;
-
-  seastar::future<> start(Dispatcher& dispatcher) {
-    std::list<Dispatcher*> dispatchers;
-    dispatchers.push_back(&dispatcher);
-    return start(dispatchers);
-  }
+  virtual seastar::future<> start(const dispatchers_t&) = 0;
 
   /// either return an existing connection to the peer,
   /// or a new pending connection

diff --git a/src/crimson/net/SocketMessenger.cc b/src/crimson/net/SocketMessenger.cc
index f1ad9bde2e5eaf9eba7b3896f723a152ee490efd..db9421e79e28a09aff172b7c95037201bafe93d5 100644 (file)
--- a/src/crimson/net/SocketMessenger.cc
+++ b/src/crimson/net/SocketMessenger.cc
@@ -122,7 +122,8 @@ SocketMessenger::try_bind(const entity_addrvec_t& addrs,
   });
 }
 
-seastar::future<> SocketMessenger::start(const std::list<Dispatcher*>& _dispatchers) {
+seastar::future<> SocketMessenger::start(
+    const dispatchers_t& _dispatchers) {
   assert(seastar::this_shard_id() == master_sid);
 
   dispatchers.assign(_dispatchers);

diff --git a/src/crimson/net/SocketMessenger.h b/src/crimson/net/SocketMessenger.h
index cca955f3a30486ac5e77ea58dc822f31ac397298..44c1d3c21375fd27087fcd5d1d417d2d518c1120 100644 (file)
--- a/src/crimson/net/SocketMessenger.h
+++ b/src/crimson/net/SocketMessenger.h
@@ -14,7 +14,6 @@
 
 #pragma once
 
-#include <list>
 #include <map>
 #include <set>
 #include <vector>
@@ -66,7 +65,7 @@ class SocketMessenger final : public Messenger {
   bind_ertr::future<> try_bind(const entity_addrvec_t& addr,
                                uint32_t min_port, uint32_t max_port) override;
 
-  seastar::future<> start(const std::list<Dispatcher*>& dispatchers) override;
+  seastar::future<> start(const dispatchers_t& dispatchers) override;
 
   ConnectionRef connect(const entity_addr_t& peer_addr,
                         const entity_name_t& peer_name) override;

diff --git a/src/crimson/net/chained_dispatchers.h b/src/crimson/net/chained_dispatchers.h
index 531fe906369c234591c57a89bca38d46288609a0..712b0894b9fd257518f125be2f45e65c9acfa575 100644 (file)
--- a/src/crimson/net/chained_dispatchers.h
+++ b/src/crimson/net/chained_dispatchers.h
@@ -3,8 +3,6 @@
 
 #pragma once
 
-#include <list>
-
 #include "Fwd.h"
 #include "crimson/common/log.h"
 
@@ -14,7 +12,7 @@ class Dispatcher;
 
 class ChainedDispatchers {
 public:
-  void assign(const std::list<Dispatcher*> _dispatchers) {
+  void assign(const dispatchers_t& _dispatchers) {
     assert(empty());
     assert(!_dispatchers.empty());
     dispatchers = _dispatchers;
@@ -32,7 +30,7 @@ public:
   void ms_handle_remote_reset(crimson::net::ConnectionRef conn);
 
  private:
-  std::list<Dispatcher*> dispatchers;
+  dispatchers_t dispatchers;
 };
 
 }

diff --git a/src/crimson/osd/heartbeat.cc b/src/crimson/osd/heartbeat.cc
index 23014cf7f937cb265cba1e0e5e3f4b8be8f22cc7..3f4fff3aac5c93504a47e43b3e93b18e3cd80458 100644 (file)
--- a/src/crimson/osd/heartbeat.cc
+++ b/src/crimson/osd/heartbeat.cc
@@ -75,7 +75,7 @@ Heartbeat::start_messenger(crimson::net::Messenger& msgr,
                        local_conf()->ms_bind_port_min,
                        local_conf()->ms_bind_port_max)
   .safe_then([this, &msgr]() mutable {
-    return msgr.start(*this);
+    return msgr.start({this});
   }, crimson::net::Messenger::bind_ertr::all_same_way(
       [] (const std::error_code& e) {
     logger().error("heartbeat messenger try_bind(): address range is unavailable.");

diff --git a/src/crimson/osd/osd.cc b/src/crimson/osd/osd.cc
index e8f5143ed28fec2267fd1e45b4086705c98d1ce3..43ccd812c0cbda7bad21bcb7714f23ee6ba1a788 100644 (file)
--- a/src/crimson/osd/osd.cc
+++ b/src/crimson/osd/osd.cc
@@ -265,10 +265,7 @@ seastar::future<> OSD::start()
     cluster_msgr->set_policy(entity_name_t::TYPE_CLIENT,
                              SocketPolicy::stateless_server(0));
 
-    std::list<Dispatcher*> dispatchers;
-    dispatchers.push_front(mgrc.get());
-    dispatchers.push_front(monc.get());
-    dispatchers.push_front(this);
+    crimson::net::dispatchers_t dispatchers{this, monc.get(), mgrc.get()};
     return seastar::when_all_succeed(
       cluster_msgr->try_bind(pick_addresses(CEPH_PICK_ADDRESS_CLUSTER),
                              local_conf()->ms_bind_port_min,

diff --git a/src/test/crimson/test_messenger.cc b/src/test/crimson/test_messenger.cc
index 5afb6f55f3257a698a5a2993f7493d655298d2db..aed86c3ab14bfb9ebcdf5450691bb961a4231a1a 100644 (file)
--- a/src/test/crimson/test_messenger.cc
+++ b/src/test/crimson/test_messenger.cc
@@ -77,7 +77,7 @@ static seastar::future<> test_echo(unsigned rounds,
         msgr->set_auth_client(&dummy_auth);
         msgr->set_auth_server(&dummy_auth);
         return msgr->bind(entity_addrvec_t{addr}).safe_then([this] {
-          return msgr->start(*this);
+          return msgr->start({this});
         }, crimson::net::Messenger::bind_ertr::all_same_way(
             [addr] (const std::error_code& e) {
           logger().error("test_echo(): "
@@ -152,7 +152,7 @@ static seastar::future<> test_echo(unsigned rounds,
         msgr->set_default_policy(crimson::net::SocketPolicy::lossy_client(0));
         msgr->set_auth_client(&dummy_auth);
         msgr->set_auth_server(&dummy_auth);
-        return msgr->start(*this);
+        return msgr->start({this});
       }
 
       seastar::future<> shutdown() {
@@ -304,7 +304,7 @@ static seastar::future<> test_concurrent_dispatch(bool v2)
         msgr->set_auth_client(&dummy_auth);
         msgr->set_auth_server(&dummy_auth);
         return msgr->bind(entity_addrvec_t{addr}).safe_then([this] {
-          return msgr->start(*this);
+          return msgr->start({this});
         }, crimson::net::Messenger::bind_ertr::all_same_way(
             [addr] (const std::error_code& e) {
           logger().error("test_concurrent_dispatch(): "
@@ -331,7 +331,7 @@ static seastar::future<> test_concurrent_dispatch(bool v2)
         msgr->set_default_policy(crimson::net::SocketPolicy::lossy_client(0));
         msgr->set_auth_client(&dummy_auth);
         msgr->set_auth_server(&dummy_auth);
-        return msgr->start(*this);
+        return msgr->start({this});
       }
     };
   };
@@ -394,7 +394,7 @@ seastar::future<> test_preemptive_shutdown(bool v2) {
         msgr->set_auth_client(&dummy_auth);
         msgr->set_auth_server(&dummy_auth);
         return msgr->bind(entity_addrvec_t{addr}).safe_then([this] {
-          return msgr->start(*this);
+          return msgr->start({this});
         }, crimson::net::Messenger::bind_ertr::all_same_way(
             [addr] (const std::error_code& e) {
           logger().error("test_preemptive_shutdown(): "
@@ -432,7 +432,7 @@ seastar::future<> test_preemptive_shutdown(bool v2) {
         msgr->set_default_policy(crimson::net::SocketPolicy::lossy_client(0));
         msgr->set_auth_client(&dummy_auth);
         msgr->set_auth_server(&dummy_auth);
-        return msgr->start(*this);
+        return msgr->start({this});
       }
       void send_pings(const entity_addr_t& addr) {
         auto conn = msgr->connect(addr, entity_name_t::TYPE_OSD);
@@ -927,7 +927,7 @@ class FailoverSuite : public Dispatcher {
     test_msgr->set_auth_server(&dummy_auth);
     test_msgr->interceptor = &interceptor;
     return test_msgr->bind(entity_addrvec_t{addr}).safe_then([this] {
-      return test_msgr->start(*this);
+      return test_msgr->start({this});
     }, Messenger::bind_ertr::all_same_way([addr] (const std::error_code& e) {
       logger().error("FailoverSuite: "
                      "there is another instance running at {}", addr);
@@ -1276,7 +1276,7 @@ class FailoverTest : public Dispatcher {
     cmd_msgr->set_default_policy(SocketPolicy::lossy_client(0));
     cmd_msgr->set_auth_client(&dummy_auth);
     cmd_msgr->set_auth_server(&dummy_auth);
-    return cmd_msgr->start(*this).then([this, cmd_peer_addr] {
+    return cmd_msgr->start({this}).then([this, cmd_peer_addr] {
       logger().info("CmdCli connect to CmdSrv({}) ...", cmd_peer_addr);
       cmd_conn = cmd_msgr->connect(cmd_peer_addr, entity_name_t::TYPE_OSD);
       return pingpong();
@@ -1436,7 +1436,7 @@ class FailoverSuitePeer : public Dispatcher {
     peer_msgr->set_auth_client(&dummy_auth);
     peer_msgr->set_auth_server(&dummy_auth);
     return peer_msgr->bind(entity_addrvec_t{addr}).safe_then([this] {
-      return peer_msgr->start(*this);
+      return peer_msgr->start({this});
     }, Messenger::bind_ertr::all_same_way([addr] (const std::error_code& e) {
       logger().error("FailoverSuitePeer: "
                      "there is another instance running at {}", addr);
@@ -1620,7 +1620,7 @@ class FailoverTestPeer : public Dispatcher {
     cmd_msgr->set_auth_client(&dummy_auth);
     cmd_msgr->set_auth_server(&dummy_auth);
     return cmd_msgr->bind(entity_addrvec_t{cmd_peer_addr}).safe_then([this] {
-      return cmd_msgr->start(*this);
+      return cmd_msgr->start({this});
     }, Messenger::bind_ertr::all_same_way([cmd_peer_addr] (const std::error_code& e) {
       logger().error("FailoverTestPeer: "
                      "there is another instance running at {}", cmd_peer_addr);

diff --git a/src/tools/crimson/perf_crimson_msgr.cc b/src/tools/crimson/perf_crimson_msgr.cc
index e20a570292f1cedf853db6f6c75b969a8d7c0bed..d53c619e9f7ec211f23c3ad7c4ceba0ecb914082 100644 (file)
--- a/src/tools/crimson/perf_crimson_msgr.cc
+++ b/src/tools/crimson/perf_crimson_msgr.cc
@@ -182,7 +182,7 @@ static seastar::future<> run(
             msgr->set_crc_data();
           }
           return msgr->bind(entity_addrvec_t{addr}).safe_then([this] {
-            return msgr->start(*this);
+            return msgr->start({this});
           }, crimson::net::Messenger::bind_ertr::all_same_way(
               [addr] (const std::error_code& e) {
             logger().error("Server: "
@@ -348,7 +348,7 @@ static seastar::future<> run(
               client.msgr->set_crc_header();
               client.msgr->set_crc_data();
             }
-            return client.msgr->start(client);
+            return client.msgr->start({&client});
           }
           return seastar::now();
         });