From 6a01988894c11d54d3f5c6eb0d8078da0a8836cc Mon Sep 17 00:00:00 2001 From: Yingxin Cheng Date: Wed, 31 May 2023 14:11:34 +0800 Subject: [PATCH] crimson/net: expose is_fixed_cpu configuration at Messenger level Signed-off-by: Yingxin Cheng --- src/crimson/net/Messenger.cc | 6 ++- src/crimson/net/Messenger.h | 3 +- src/crimson/net/SocketMessenger.cc | 6 ++- src/crimson/net/SocketMessenger.h | 4 +- src/crimson/osd/main.cc | 3 +- .../osd/main_config_bootstrap_helpers.cc | 3 +- src/crimson/tools/perf_crimson_msgr.cc | 8 +++- src/test/crimson/test_alien_echo.cc | 4 +- src/test/crimson/test_messenger.cc | 39 ++++++++++++++----- src/test/crimson/test_messenger_thrash.cc | 6 ++- src/test/crimson/test_monc.cc | 2 +- 11 files changed, 59 insertions(+), 25 deletions(-) diff --git a/src/crimson/net/Messenger.cc b/src/crimson/net/Messenger.cc index aab476f7a4b02..9068453838812 100644 --- a/src/crimson/net/Messenger.cc +++ b/src/crimson/net/Messenger.cc @@ -9,9 +9,11 @@ namespace crimson::net { MessengerRef Messenger::create(const entity_name_t& name, const std::string& lname, - const uint64_t nonce) + uint64_t nonce, + bool is_fixed_cpu) { - return seastar::make_shared(name, lname, nonce); + return seastar::make_shared( + name, lname, nonce, is_fixed_cpu); } } // namespace crimson::net diff --git a/src/crimson/net/Messenger.h b/src/crimson/net/Messenger.h index e2fba32578b79..c06655d85b852 100644 --- a/src/crimson/net/Messenger.h +++ b/src/crimson/net/Messenger.h @@ -108,7 +108,8 @@ public: static MessengerRef create(const entity_name_t& name, const std::string& lname, - const uint64_t nonce); + uint64_t nonce, + bool is_fixed_cpu); #ifdef UNIT_TESTS_BUILT virtual void set_interceptor(Interceptor *) = 0; diff --git a/src/crimson/net/SocketMessenger.cc b/src/crimson/net/SocketMessenger.cc index 11cec9b974fff..019c879d66ea3 100644 --- a/src/crimson/net/SocketMessenger.cc +++ b/src/crimson/net/SocketMessenger.cc @@ -34,10 +34,12 @@ namespace crimson::net { SocketMessenger::SocketMessenger(const entity_name_t& myname, const std::string& logic_name, - uint32_t nonce) + uint32_t nonce, + bool is_fixed_cpu) : sid{seastar::this_shard_id()}, logic_name{logic_name}, nonce{nonce}, + is_fixed_cpu{is_fixed_cpu}, my_name{myname} {} @@ -92,7 +94,7 @@ SocketMessenger::do_listen(const entity_addrvec_t& addrs) set_myaddrs(addrs); return seastar::futurize_invoke([this] { if (!listener) { - return ShardedServerSocket::create(true + return ShardedServerSocket::create(is_fixed_cpu ).then([this] (auto _listener) { listener = _listener; }); diff --git a/src/crimson/net/SocketMessenger.h b/src/crimson/net/SocketMessenger.h index 36d814379a611..8d696bf885f27 100644 --- a/src/crimson/net/SocketMessenger.h +++ b/src/crimson/net/SocketMessenger.h @@ -36,7 +36,8 @@ class SocketMessenger final : public Messenger { public: SocketMessenger(const entity_name_t& myname, const std::string& logic_name, - uint32_t nonce); + uint32_t nonce, + bool is_fixed_cpu); ~SocketMessenger() override; @@ -164,6 +165,7 @@ private: // Distinguish messengers with meaningful names for debugging const std::string logic_name; const uint32_t nonce; + const bool is_fixed_cpu; entity_name_t my_name; entity_addrvec_t my_addrs; diff --git a/src/crimson/osd/main.cc b/src/crimson/osd/main.cc index 234259f0e6d23..1e817415d03b9 100644 --- a/src/crimson/osd/main.cc +++ b/src/crimson/osd/main.cc @@ -192,7 +192,8 @@ int main(int argc, const char* argv[]) make_pair(std::ref(hb_back_msgr), "hb_back"s)}) { msgr = crimson::net::Messenger::create(entity_name_t::OSD(whoami), name, - nonce); + nonce, + true); } auto store = crimson::os::FuturizedStore::create( local_conf().get_val("osd_objectstore"), diff --git a/src/crimson/osd/main_config_bootstrap_helpers.cc b/src/crimson/osd/main_config_bootstrap_helpers.cc index 0777822b9a0e6..807fd1591aed5 100644 --- a/src/crimson/osd/main_config_bootstrap_helpers.cc +++ b/src/crimson/osd/main_config_bootstrap_helpers.cc @@ -55,7 +55,8 @@ seastar::future<> populate_config_from_mon() auto auth_handler = std::make_unique(); auto msgr = crimson::net::Messenger::create(entity_name_t::CLIENT(), "temp_mon_client", - get_nonce()); + get_nonce(), + true); crimson::mon::Client monc{*msgr, *auth_handler}; msgr->set_auth_client(&monc); msgr->start({&monc}).get(); diff --git a/src/crimson/tools/perf_crimson_msgr.cc b/src/crimson/tools/perf_crimson_msgr.cc index 2d45dfaaff2f7..eaedaf0cd86c4 100644 --- a/src/crimson/tools/perf_crimson_msgr.cc +++ b/src/crimson/tools/perf_crimson_msgr.cc @@ -174,7 +174,9 @@ static seastar::future<> run( seastar::future<> init(const entity_addr_t& addr) { return seastar::smp::submit_to(msgr_sid, [addr, this] { // server msgr is always with nonce 0 - msgr = crimson::net::Messenger::create(entity_name_t::OSD(msgr_sid), lname, 0); + msgr = crimson::net::Messenger::create( + entity_name_t::OSD(msgr_sid), + lname, 0, true); msgr->set_default_policy(crimson::net::SocketPolicy::stateless_server(0)); msgr->set_auth_client(&dummy_auth); msgr->set_auth_server(&dummy_auth); @@ -339,7 +341,9 @@ static seastar::future<> run( seastar::future<> init() { return container().invoke_on_all([] (auto& client) { if (client.is_active()) { - client.msgr = crimson::net::Messenger::create(entity_name_t::OSD(client.sid), client.lname, client.sid); + client.msgr = crimson::net::Messenger::create( + entity_name_t::OSD(client.sid), + client.lname, client.sid, true); client.msgr->set_default_policy(crimson::net::SocketPolicy::lossy_client(0)); client.msgr->set_auth_client(&client.dummy_auth); client.msgr->set_auth_server(&client.dummy_auth); diff --git a/src/test/crimson/test_alien_echo.cc b/src/test/crimson/test_alien_echo.cc index e75d3d03c5969..8bef5e6517b03 100644 --- a/src/test/crimson/test_alien_echo.cc +++ b/src/test/crimson/test_alien_echo.cc @@ -166,7 +166,7 @@ seastar_echo(const entity_addr_t addr, echo_role role, unsigned count) if (role == echo_role::as_server) { return seastar::do_with( seastar_pingpong::Server{crimson::net::Messenger::create( - entity_name_t::OSD(0), "server", addr.get_nonce())}, + entity_name_t::OSD(0), "server", addr.get_nonce(), true)}, [addr, count](auto& server) mutable { std::cout << "server listening at " << addr << std::endl; // bind the server @@ -193,7 +193,7 @@ seastar_echo(const entity_addr_t addr, echo_role role, unsigned count) } else { return seastar::do_with( seastar_pingpong::Client{crimson::net::Messenger::create( - entity_name_t::OSD(1), "client", addr.get_nonce())}, + entity_name_t::OSD(1), "client", addr.get_nonce(), true)}, [addr, count](auto& client) { std::cout << "client sending to " << addr << std::endl; client.msgr->set_default_policy(crimson::net::SocketPolicy::lossy_client(0)); diff --git a/src/test/crimson/test_messenger.cc b/src/test/crimson/test_messenger.cc index 8af521d48c80e..60b5439b228a8 100644 --- a/src/test/crimson/test_messenger.cc +++ b/src/test/crimson/test_messenger.cc @@ -75,7 +75,8 @@ static seastar::future<> test_echo(unsigned rounds, const std::string& lname, const uint64_t nonce, const entity_addr_t& addr) { - msgr = crimson::net::Messenger::create(name, lname, nonce); + msgr = crimson::net::Messenger::create( + name, lname, nonce, true); msgr->set_default_policy(crimson::net::SocketPolicy::stateless_server(0)); msgr->set_auth_client(&dummy_auth); msgr->set_auth_server(&dummy_auth); @@ -154,7 +155,8 @@ static seastar::future<> test_echo(unsigned rounds, seastar::future<> init(const entity_name_t& name, const std::string& lname, const uint64_t nonce) { - msgr = crimson::net::Messenger::create(name, lname, nonce); + msgr = crimson::net::Messenger::create( + name, lname, nonce, true); msgr->set_default_policy(crimson::net::SocketPolicy::lossy_client(0)); msgr->set_auth_client(&dummy_auth); msgr->set_auth_server(&dummy_auth); @@ -303,7 +305,8 @@ static seastar::future<> test_concurrent_dispatch() const std::string& lname, const uint64_t nonce, const entity_addr_t& addr) { - msgr = crimson::net::Messenger::create(name, lname, nonce); + msgr = crimson::net::Messenger::create( + name, lname, nonce, true); msgr->set_default_policy(crimson::net::SocketPolicy::stateless_server(0)); msgr->set_auth_client(&dummy_auth); msgr->set_auth_server(&dummy_auth); @@ -331,7 +334,8 @@ static seastar::future<> test_concurrent_dispatch() seastar::future<> init(const entity_name_t& name, const std::string& lname, const uint64_t nonce) { - msgr = crimson::net::Messenger::create(name, lname, nonce); + msgr = crimson::net::Messenger::create( + name, lname, nonce, true); msgr->set_default_policy(crimson::net::SocketPolicy::lossy_client(0)); msgr->set_auth_client(&dummy_auth); msgr->set_auth_server(&dummy_auth); @@ -392,7 +396,8 @@ seastar::future<> test_preemptive_shutdown() { const std::string& lname, const uint64_t nonce, const entity_addr_t& addr) { - msgr = crimson::net::Messenger::create(name, lname, nonce); + msgr = crimson::net::Messenger::create( + name, lname, nonce, true); msgr->set_default_policy(crimson::net::SocketPolicy::stateless_server(0)); msgr->set_auth_client(&dummy_auth); msgr->set_auth_server(&dummy_auth); @@ -431,7 +436,8 @@ seastar::future<> test_preemptive_shutdown() { seastar::future<> init(const entity_name_t& name, const std::string& lname, const uint64_t nonce) { - msgr = crimson::net::Messenger::create(name, lname, nonce); + msgr = crimson::net::Messenger::create( + name, lname, nonce, true); msgr->set_default_policy(crimson::net::SocketPolicy::lossy_client(0)); msgr->set_auth_client(&dummy_auth); msgr->set_auth_server(&dummy_auth); @@ -1120,7 +1126,11 @@ class FailoverSuite : public Dispatcher { entity_addr_t test_peer_addr, const TestInterceptor& interceptor) { auto suite = std::make_unique( - Messenger::create(entity_name_t::OSD(TEST_OSD), "Test", TEST_NONCE), + Messenger::create( + entity_name_t::OSD(TEST_OSD), + "Test", + TEST_NONCE, + true), test_peer_addr, interceptor); return suite->init(test_addr, test_policy ).then([suite = std::move(suite)] () mutable { @@ -1334,7 +1344,11 @@ class FailoverTest : public Dispatcher { entity_addr_t cmd_peer_addr, entity_addr_t test_peer_addr) { auto test = seastar::make_lw_shared( - Messenger::create(entity_name_t::OSD(CMD_CLI_OSD), "CmdCli", CMD_CLI_NONCE), + Messenger::create( + entity_name_t::OSD(CMD_CLI_OSD), + "CmdCli", + CMD_CLI_NONCE, + true), test_addr, test_peer_addr); return test->init(cmd_peer_addr).then([test] { logger().info("CmdCli ready"); @@ -1554,7 +1568,8 @@ class FailoverSuitePeer : public Dispatcher { Messenger::create( entity_name_t::OSD(TEST_PEER_OSD), "TestPeer", - TEST_PEER_NONCE), + TEST_PEER_NONCE, + true), op_callback ); return suite->init(test_peer_addr, policy @@ -1679,7 +1694,11 @@ class FailoverTestPeer : public Dispatcher { static seastar::future> create(entity_addr_t cmd_peer_addr, entity_addr_t test_peer_addr) { auto test_peer = std::make_unique( - Messenger::create(entity_name_t::OSD(CMD_SRV_OSD), "CmdSrv", CMD_SRV_NONCE), + Messenger::create( + entity_name_t::OSD(CMD_SRV_OSD), + "CmdSrv", + CMD_SRV_NONCE, + true), test_peer_addr); return test_peer->init(cmd_peer_addr ).then([test_peer = std::move(test_peer)] () mutable { diff --git a/src/test/crimson/test_messenger_thrash.cc b/src/test/crimson/test_messenger_thrash.cc index 7be81f831c8da..3b41fe16e7ded 100644 --- a/src/test/crimson/test_messenger_thrash.cc +++ b/src/test/crimson/test_messenger_thrash.cc @@ -354,7 +354,8 @@ class SyntheticWorkload { const uint64_t nonce, const entity_addr_t& addr) { crimson::net::MessengerRef msgr = - crimson::net::Messenger::create(name, lname, nonce); + crimson::net::Messenger::create( + name, lname, nonce, true); msgr->set_default_policy(server_policy); msgr->set_auth_client(&dummy_auth); msgr->set_auth_server(&dummy_auth); @@ -375,7 +376,8 @@ class SyntheticWorkload { const std::string& lname, const uint64_t nonce) { crimson::net::MessengerRef msgr = - crimson::net::Messenger::create(name, lname, nonce); + crimson::net::Messenger::create( + name, lname, nonce, true); msgr->set_default_policy(client_policy); msgr->set_auth_client(&dummy_auth); msgr->set_auth_server(&dummy_auth); diff --git a/src/test/crimson/test_monc.cc b/src/test/crimson/test_monc.cc index 8723d20c6f060..e60df45257688 100644 --- a/src/test/crimson/test_monc.cc +++ b/src/test/crimson/test_monc.cc @@ -41,7 +41,7 @@ static seastar::future<> test_monc() }).then([] { return crimson::common::sharded_perf_coll().start(); }).then([]() mutable { - auto msgr = crimson::net::Messenger::create(entity_name_t::OSD(0), "monc", 0); + auto msgr = crimson::net::Messenger::create(entity_name_t::OSD(0), "monc", 0, true); return seastar::do_with(MonClient{*msgr, dummy_handler}, [msgr](auto& monc) mutable { return msgr->start({&monc}).then([&monc] { -- 2.39.5