From a5a77b518705123f51e15c4d9e9bb83c7969288d Mon Sep 17 00:00:00 2001 From: Yingxin Cheng Date: Wed, 2 Dec 2020 10:34:52 +0800 Subject: [PATCH] crimson/net: use boost::container::small_vector in ChainedDispatchers Signed-off-by: Yingxin Cheng --- src/crimson/net/Fwd.h | 4 ++++ src/crimson/net/Messenger.h | 12 +----------- src/crimson/net/SocketMessenger.cc | 3 ++- src/crimson/net/SocketMessenger.h | 3 +-- src/crimson/net/chained_dispatchers.h | 6 ++---- src/crimson/osd/heartbeat.cc | 2 +- src/crimson/osd/osd.cc | 5 +---- src/test/crimson/test_messenger.cc | 20 ++++++++++---------- src/tools/crimson/perf_crimson_msgr.cc | 4 ++-- 9 files changed, 24 insertions(+), 35 deletions(-) diff --git a/src/crimson/net/Fwd.h b/src/crimson/net/Fwd.h index 77818704a85..e10120571f3 100644 --- a/src/crimson/net/Fwd.h +++ b/src/crimson/net/Fwd.h @@ -14,6 +14,7 @@ #pragma once +#include #include #include #include @@ -38,7 +39,10 @@ using stop_t = seastar::stop_iteration; class Connection; using ConnectionRef = seastar::shared_ptr; +class Dispatcher; class ChainedDispatchers; +constexpr std::size_t NUM_DISPATCHERS = 4u; +using dispatchers_t = boost::container::small_vector; class Messenger; using MessengerRef = seastar::shared_ptr; diff --git a/src/crimson/net/Messenger.h b/src/crimson/net/Messenger.h index 9133cdd0356..2b39fbf63a6 100644 --- a/src/crimson/net/Messenger.h +++ b/src/crimson/net/Messenger.h @@ -14,8 +14,6 @@ #pragma once -#include - #include "Fwd.h" #include "crimson/common/throttle.h" #include "msg/Message.h" @@ -34,8 +32,6 @@ namespace crimson::net { class Interceptor; #endif -class Dispatcher; - using Throttle = crimson::common::Throttle; using SocketPolicy = ceph::net::Policy; @@ -77,13 +73,7 @@ public: uint32_t min_port, uint32_t max_port) = 0; /// start the messenger - virtual seastar::future<> start(const std::list&) = 0; - - seastar::future<> start(Dispatcher& dispatcher) { - std::list dispatchers; - dispatchers.push_back(&dispatcher); - return start(dispatchers); - } + virtual seastar::future<> start(const dispatchers_t&) = 0; /// either return an existing connection to the peer, /// or a new pending connection diff --git a/src/crimson/net/SocketMessenger.cc b/src/crimson/net/SocketMessenger.cc index f1ad9bde2e5..db9421e79e2 100644 --- a/src/crimson/net/SocketMessenger.cc +++ b/src/crimson/net/SocketMessenger.cc @@ -122,7 +122,8 @@ SocketMessenger::try_bind(const entity_addrvec_t& addrs, }); } -seastar::future<> SocketMessenger::start(const std::list& _dispatchers) { +seastar::future<> SocketMessenger::start( + const dispatchers_t& _dispatchers) { assert(seastar::this_shard_id() == master_sid); dispatchers.assign(_dispatchers); diff --git a/src/crimson/net/SocketMessenger.h b/src/crimson/net/SocketMessenger.h index cca955f3a30..44c1d3c2137 100644 --- a/src/crimson/net/SocketMessenger.h +++ b/src/crimson/net/SocketMessenger.h @@ -14,7 +14,6 @@ #pragma once -#include #include #include #include @@ -66,7 +65,7 @@ class SocketMessenger final : public Messenger { bind_ertr::future<> try_bind(const entity_addrvec_t& addr, uint32_t min_port, uint32_t max_port) override; - seastar::future<> start(const std::list& dispatchers) override; + seastar::future<> start(const dispatchers_t& dispatchers) override; ConnectionRef connect(const entity_addr_t& peer_addr, const entity_name_t& peer_name) override; diff --git a/src/crimson/net/chained_dispatchers.h b/src/crimson/net/chained_dispatchers.h index 531fe906369..712b0894b9f 100644 --- a/src/crimson/net/chained_dispatchers.h +++ b/src/crimson/net/chained_dispatchers.h @@ -3,8 +3,6 @@ #pragma once -#include - #include "Fwd.h" #include "crimson/common/log.h" @@ -14,7 +12,7 @@ class Dispatcher; class ChainedDispatchers { public: - void assign(const std::list _dispatchers) { + void assign(const dispatchers_t& _dispatchers) { assert(empty()); assert(!_dispatchers.empty()); dispatchers = _dispatchers; @@ -32,7 +30,7 @@ public: void ms_handle_remote_reset(crimson::net::ConnectionRef conn); private: - std::list dispatchers; + dispatchers_t dispatchers; }; } diff --git a/src/crimson/osd/heartbeat.cc b/src/crimson/osd/heartbeat.cc index 23014cf7f93..3f4fff3aac5 100644 --- a/src/crimson/osd/heartbeat.cc +++ b/src/crimson/osd/heartbeat.cc @@ -75,7 +75,7 @@ Heartbeat::start_messenger(crimson::net::Messenger& msgr, local_conf()->ms_bind_port_min, local_conf()->ms_bind_port_max) .safe_then([this, &msgr]() mutable { - return msgr.start(*this); + return msgr.start({this}); }, crimson::net::Messenger::bind_ertr::all_same_way( [] (const std::error_code& e) { logger().error("heartbeat messenger try_bind(): address range is unavailable."); diff --git a/src/crimson/osd/osd.cc b/src/crimson/osd/osd.cc index e8f5143ed28..43ccd812c0c 100644 --- a/src/crimson/osd/osd.cc +++ b/src/crimson/osd/osd.cc @@ -265,10 +265,7 @@ seastar::future<> OSD::start() cluster_msgr->set_policy(entity_name_t::TYPE_CLIENT, SocketPolicy::stateless_server(0)); - std::list dispatchers; - dispatchers.push_front(mgrc.get()); - dispatchers.push_front(monc.get()); - dispatchers.push_front(this); + crimson::net::dispatchers_t dispatchers{this, monc.get(), mgrc.get()}; return seastar::when_all_succeed( cluster_msgr->try_bind(pick_addresses(CEPH_PICK_ADDRESS_CLUSTER), local_conf()->ms_bind_port_min, diff --git a/src/test/crimson/test_messenger.cc b/src/test/crimson/test_messenger.cc index 5afb6f55f32..aed86c3ab14 100644 --- a/src/test/crimson/test_messenger.cc +++ b/src/test/crimson/test_messenger.cc @@ -77,7 +77,7 @@ static seastar::future<> test_echo(unsigned rounds, msgr->set_auth_client(&dummy_auth); msgr->set_auth_server(&dummy_auth); return msgr->bind(entity_addrvec_t{addr}).safe_then([this] { - return msgr->start(*this); + return msgr->start({this}); }, crimson::net::Messenger::bind_ertr::all_same_way( [addr] (const std::error_code& e) { logger().error("test_echo(): " @@ -152,7 +152,7 @@ static seastar::future<> test_echo(unsigned rounds, msgr->set_default_policy(crimson::net::SocketPolicy::lossy_client(0)); msgr->set_auth_client(&dummy_auth); msgr->set_auth_server(&dummy_auth); - return msgr->start(*this); + return msgr->start({this}); } seastar::future<> shutdown() { @@ -304,7 +304,7 @@ static seastar::future<> test_concurrent_dispatch(bool v2) msgr->set_auth_client(&dummy_auth); msgr->set_auth_server(&dummy_auth); return msgr->bind(entity_addrvec_t{addr}).safe_then([this] { - return msgr->start(*this); + return msgr->start({this}); }, crimson::net::Messenger::bind_ertr::all_same_way( [addr] (const std::error_code& e) { logger().error("test_concurrent_dispatch(): " @@ -331,7 +331,7 @@ static seastar::future<> test_concurrent_dispatch(bool v2) msgr->set_default_policy(crimson::net::SocketPolicy::lossy_client(0)); msgr->set_auth_client(&dummy_auth); msgr->set_auth_server(&dummy_auth); - return msgr->start(*this); + return msgr->start({this}); } }; }; @@ -394,7 +394,7 @@ seastar::future<> test_preemptive_shutdown(bool v2) { msgr->set_auth_client(&dummy_auth); msgr->set_auth_server(&dummy_auth); return msgr->bind(entity_addrvec_t{addr}).safe_then([this] { - return msgr->start(*this); + return msgr->start({this}); }, crimson::net::Messenger::bind_ertr::all_same_way( [addr] (const std::error_code& e) { logger().error("test_preemptive_shutdown(): " @@ -432,7 +432,7 @@ seastar::future<> test_preemptive_shutdown(bool v2) { msgr->set_default_policy(crimson::net::SocketPolicy::lossy_client(0)); msgr->set_auth_client(&dummy_auth); msgr->set_auth_server(&dummy_auth); - return msgr->start(*this); + return msgr->start({this}); } void send_pings(const entity_addr_t& addr) { auto conn = msgr->connect(addr, entity_name_t::TYPE_OSD); @@ -927,7 +927,7 @@ class FailoverSuite : public Dispatcher { test_msgr->set_auth_server(&dummy_auth); test_msgr->interceptor = &interceptor; return test_msgr->bind(entity_addrvec_t{addr}).safe_then([this] { - return test_msgr->start(*this); + return test_msgr->start({this}); }, Messenger::bind_ertr::all_same_way([addr] (const std::error_code& e) { logger().error("FailoverSuite: " "there is another instance running at {}", addr); @@ -1276,7 +1276,7 @@ class FailoverTest : public Dispatcher { cmd_msgr->set_default_policy(SocketPolicy::lossy_client(0)); cmd_msgr->set_auth_client(&dummy_auth); cmd_msgr->set_auth_server(&dummy_auth); - return cmd_msgr->start(*this).then([this, cmd_peer_addr] { + return cmd_msgr->start({this}).then([this, cmd_peer_addr] { logger().info("CmdCli connect to CmdSrv({}) ...", cmd_peer_addr); cmd_conn = cmd_msgr->connect(cmd_peer_addr, entity_name_t::TYPE_OSD); return pingpong(); @@ -1436,7 +1436,7 @@ class FailoverSuitePeer : public Dispatcher { peer_msgr->set_auth_client(&dummy_auth); peer_msgr->set_auth_server(&dummy_auth); return peer_msgr->bind(entity_addrvec_t{addr}).safe_then([this] { - return peer_msgr->start(*this); + return peer_msgr->start({this}); }, Messenger::bind_ertr::all_same_way([addr] (const std::error_code& e) { logger().error("FailoverSuitePeer: " "there is another instance running at {}", addr); @@ -1620,7 +1620,7 @@ class FailoverTestPeer : public Dispatcher { cmd_msgr->set_auth_client(&dummy_auth); cmd_msgr->set_auth_server(&dummy_auth); return cmd_msgr->bind(entity_addrvec_t{cmd_peer_addr}).safe_then([this] { - return cmd_msgr->start(*this); + return cmd_msgr->start({this}); }, Messenger::bind_ertr::all_same_way([cmd_peer_addr] (const std::error_code& e) { logger().error("FailoverTestPeer: " "there is another instance running at {}", cmd_peer_addr); diff --git a/src/tools/crimson/perf_crimson_msgr.cc b/src/tools/crimson/perf_crimson_msgr.cc index e20a570292f..d53c619e9f7 100644 --- a/src/tools/crimson/perf_crimson_msgr.cc +++ b/src/tools/crimson/perf_crimson_msgr.cc @@ -182,7 +182,7 @@ static seastar::future<> run( msgr->set_crc_data(); } return msgr->bind(entity_addrvec_t{addr}).safe_then([this] { - return msgr->start(*this); + return msgr->start({this}); }, crimson::net::Messenger::bind_ertr::all_same_way( [addr] (const std::error_code& e) { logger().error("Server: " @@ -348,7 +348,7 @@ static seastar::future<> run( client.msgr->set_crc_header(); client.msgr->set_crc_data(); } - return client.msgr->start(client); + return client.msgr->start({&client}); } return seastar::now(); }); -- 2.39.5