From ad0bbc8f78a632b1008f624ffb6090f86731a0e0 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 28 Feb 2020 15:11:37 -0600 Subject: [PATCH] msg: add get_{pid,random}_nonce() helpers In cases where we normally use a pid for a nonce, fall back to a random value when the pid == 1 (i.e., we're in a container). For the cases where we use a random value, use the helper. Signed-off-by: Sage Weil --- src/ceph_mds.cc | 5 ++--- src/ceph_mon.cc | 6 ++++-- src/ceph_osd.cc | 15 ++++++++------- src/crimson/osd/main.cc | 18 ++++++++++++++++-- src/mgr/DaemonServer.cc | 3 ++- src/mgr/MgrStandby.cc | 2 +- src/msg/Messenger.cc | 17 ++++++++++++++++- src/msg/Messenger.h | 3 +++ 8 files changed, 52 insertions(+), 17 deletions(-) diff --git a/src/ceph_mds.cc b/src/ceph_mds.cc index 41adf75f13104..4d960dc2e7f27 100644 --- a/src/ceph_mds.cc +++ b/src/ceph_mds.cc @@ -163,12 +163,11 @@ int main(int argc, const char **argv) common_init_finish(g_ceph_context); global_init_chdir(g_ceph_context); - auto nonce = ceph::util::generate_random_number(); - std::string public_msgr_type = g_conf()->ms_public_type.empty() ? g_conf().get_val("ms_type") : g_conf()->ms_public_type; Messenger *msgr = Messenger::create(g_ceph_context, public_msgr_type, entity_name_t::MDS(-1), "mds", - nonce, Messenger::HAS_MANY_CONNECTIONS); + Messenger::get_random_nonce(), + Messenger::HAS_MANY_CONNECTIONS); if (!msgr) forker.exit(1); msgr->set_cluster_protocol(CEPH_MDS_PROTOCOL); diff --git a/src/ceph_mon.cc b/src/ceph_mon.cc index 3b90deecb4481..3463110d1ded3 100644 --- a/src/ceph_mon.cc +++ b/src/ceph_mon.cc @@ -778,7 +778,8 @@ int main(int argc, const char **argv) std::string public_msgr_type = g_conf()->ms_public_type.empty() ? g_conf().get_val("ms_type") : g_conf()->ms_public_type; Messenger *msgr = Messenger::create(g_ceph_context, public_msgr_type, entity_name_t::MON(rank), "mon", - 0, Messenger::HAS_MANY_CONNECTIONS); + 0, // zero nonce + Messenger::HAS_MANY_CONNECTIONS); if (!msgr) exit(1); msgr->set_cluster_protocol(CEPH_MON_PROTOCOL); @@ -829,7 +830,8 @@ int main(int argc, const char **argv) Messenger *mgr_msgr = Messenger::create(g_ceph_context, public_msgr_type, entity_name_t::MON(rank), "mon-mgrc", - getpid(), 0); + Messenger::get_pid_nonce(), + 0); if (!mgr_msgr) { derr << "unable to create mgr_msgr" << dendl; prefork.exit(1); diff --git a/src/ceph_osd.cc b/src/ceph_osd.cc index 1bdb36b8369aa..1e9516aa30d62 100644 --- a/src/ceph_osd.cc +++ b/src/ceph_osd.cc @@ -524,31 +524,32 @@ flushjournal_out: public_msg_type = public_msg_type.empty() ? msg_type : public_msg_type; cluster_msg_type = cluster_msg_type.empty() ? msg_type : cluster_msg_type; + uint64_t nonce = Messenger::get_pid_nonce(); Messenger *ms_public = Messenger::create(g_ceph_context, public_msg_type, entity_name_t::OSD(whoami), "client", - getpid(), + nonce, Messenger::HAS_HEAVY_TRAFFIC | Messenger::HAS_MANY_CONNECTIONS); Messenger *ms_cluster = Messenger::create(g_ceph_context, cluster_msg_type, entity_name_t::OSD(whoami), "cluster", - getpid(), + nonce, Messenger::HAS_HEAVY_TRAFFIC | Messenger::HAS_MANY_CONNECTIONS); Messenger *ms_hb_back_client = Messenger::create(g_ceph_context, cluster_msg_type, entity_name_t::OSD(whoami), "hb_back_client", - getpid(), Messenger::HEARTBEAT); + nonce, Messenger::HEARTBEAT); Messenger *ms_hb_front_client = Messenger::create(g_ceph_context, public_msg_type, entity_name_t::OSD(whoami), "hb_front_client", - getpid(), Messenger::HEARTBEAT); + nonce, Messenger::HEARTBEAT); Messenger *ms_hb_back_server = Messenger::create(g_ceph_context, cluster_msg_type, entity_name_t::OSD(whoami), "hb_back_server", - getpid(), Messenger::HEARTBEAT); + nonce, Messenger::HEARTBEAT); Messenger *ms_hb_front_server = Messenger::create(g_ceph_context, public_msg_type, entity_name_t::OSD(whoami), "hb_front_server", - getpid(), Messenger::HEARTBEAT); + nonce, Messenger::HEARTBEAT); Messenger *ms_objecter = Messenger::create(g_ceph_context, public_msg_type, entity_name_t::OSD(whoami), "ms_objecter", - getpid(), 0); + nonce, 0); if (!ms_public || !ms_cluster || !ms_hb_front_client || !ms_hb_back_client || !ms_hb_back_server || !ms_hb_front_server || !ms_objecter) forker.exit(1); ms_cluster->set_cluster_protocol(CEPH_OSD_PROTOCOL); diff --git a/src/crimson/osd/main.cc b/src/crimson/osd/main.cc index 370d1e8dfbd60..d734fad46a9f3 100644 --- a/src/crimson/osd/main.cc +++ b/src/crimson/osd/main.cc @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -105,6 +106,18 @@ seastar::future<> make_keyring() }); } +uint64_t get_nonce() +{ + if (auto pid = getpid(); pid != 1) { + return pid; + } else { + // we're running in a container; use a random number instead! + std::random_device rd; + std::default_random_engine rng{rd()}; + return std::uniform_int_distribution{}(rng); + } +} + int main(int argc, char* argv[]) { seastar::app_template app; @@ -153,14 +166,15 @@ int main(int argc, char* argv[]) local_conf().parse_argv(ceph_args).get(); pidfile_write(local_conf()->pid_file); const int whoami = std::stoi(local_conf()->name.get_id()); - const auto nonce = static_cast(getpid()); + const auto nonce = get_nonce(); crimson::net::MessengerRef cluster_msgr, client_msgr; crimson::net::MessengerRef hb_front_msgr, hb_back_msgr; for (auto [msgr, name] : {make_pair(std::ref(cluster_msgr), "cluster"s), make_pair(std::ref(client_msgr), "client"s), make_pair(std::ref(hb_front_msgr), "hb_front"s), make_pair(std::ref(hb_back_msgr), "hb_back"s)}) { - msgr = crimson::net::Messenger::create(entity_name_t::OSD(whoami), name, nonce); + msgr = crimson::net::Messenger::create(entity_name_t::OSD(whoami), name, + nonce); if (local_conf()->ms_crc_data) { msgr->set_crc_data(); } diff --git a/src/mgr/DaemonServer.cc b/src/mgr/DaemonServer.cc index 06573211ba3c2..95d164767fcb7 100644 --- a/src/mgr/DaemonServer.cc +++ b/src/mgr/DaemonServer.cc @@ -108,7 +108,8 @@ int DaemonServer::init(uint64_t gid, entity_addrvec_t client_addrs) msgr = Messenger::create(g_ceph_context, public_msgr_type, entity_name_t::MGR(gid), "mgr", - getpid(), 0); + Messenger::get_pid_nonce(), + 0); msgr->set_default_policy(Messenger::Policy::stateless_server(0)); msgr->set_auth_client(monc); diff --git a/src/mgr/MgrStandby.cc b/src/mgr/MgrStandby.cc index 16c82b72e88d8..296494ece14ca 100644 --- a/src/mgr/MgrStandby.cc +++ b/src/mgr/MgrStandby.cc @@ -44,7 +44,7 @@ MgrStandby::MgrStandby(int argc, const char **argv) : cct->_conf.get_val("ms_type"), entity_name_t::MGR(), "mgr", - getpid(), + Messenger::get_pid_nonce(), 0)), objecter{g_ceph_context, client_messenger.get(), &monc, NULL, 0, 0}, client{client_messenger.get(), &monc, &objecter}, diff --git a/src/msg/Messenger.cc b/src/msg/Messenger.cc index b975006a1dd7a..4e2ea96ebe20b 100644 --- a/src/msg/Messenger.cc +++ b/src/msg/Messenger.cc @@ -13,11 +13,26 @@ Messenger *Messenger::create_client_messenger(CephContext *cct, string lname) { std::string public_msgr_type = cct->_conf->ms_public_type.empty() ? cct->_conf.get_val("ms_type") : cct->_conf->ms_public_type; - auto nonce = ceph::util::generate_random_number(); + auto nonce = get_random_nonce(); return Messenger::create(cct, public_msgr_type, entity_name_t::CLIENT(), std::move(lname), nonce, 0); } +uint64_t Messenger::get_pid_nonce() +{ + uint64_t nonce = getpid(); + if (nonce == 1) { + // we're running in a container; use a random number instead! + nonce = ceph::util::generate_random_number(); + } + return nonce; +} + +uint64_t Messenger::get_random_nonce() +{ + return ceph::util::generate_random_number(); +} + Messenger *Messenger::create(CephContext *cct, const string &type, entity_name_t name, string lname, uint64_t nonce, uint64_t cflags) diff --git a/src/msg/Messenger.h b/src/msg/Messenger.h index 22f5fe387cd48..c00374c114d2d 100644 --- a/src/msg/Messenger.h +++ b/src/msg/Messenger.h @@ -153,6 +153,9 @@ public: uint64_t nonce, uint64_t cflags); + static uint64_t get_random_nonce(); + static uint64_t get_pid_nonce(); + /** * create a new messenger * -- 2.39.5