msg: add get_{pid,random}_nonce() helpers

In cases where we normally use a pid for a nonce, fall back to a random
value when the pid == 1 (i.e., we're in a container).  For the cases where
we use a random value, use the helper.

Signed-off-by: Sage Weil <sage@redhat.com>

diff --git a/src/ceph_mds.cc b/src/ceph_mds.cc
index 41adf75f131047b8ad38e7221d021717d4d501be..4d960dc2e7f2733fceb54ade343cea5c5a533fcc 100644 (file)
--- a/src/ceph_mds.cc
+++ b/src/ceph_mds.cc
@@ -163,12 +163,11 @@ int main(int argc, const char **argv)
   common_init_finish(g_ceph_context);
   global_init_chdir(g_ceph_context);
 
-  auto nonce = ceph::util::generate_random_number<uint64_t>();
-
   std::string public_msgr_type = g_conf()->ms_public_type.empty() ? g_conf().get_val<std::string>("ms_type") : g_conf()->ms_public_type;
   Messenger *msgr = Messenger::create(g_ceph_context, public_msgr_type,
                                      entity_name_t::MDS(-1), "mds",
-                                     nonce, Messenger::HAS_MANY_CONNECTIONS);
+                                     Messenger::get_random_nonce(),
+                                     Messenger::HAS_MANY_CONNECTIONS);
   if (!msgr)
     forker.exit(1);
   msgr->set_cluster_protocol(CEPH_MDS_PROTOCOL);

diff --git a/src/ceph_mon.cc b/src/ceph_mon.cc
index 3b90deecb448163dbf1201e1447fe98d6b5e08e7..3463110d1ded3083e64dc3fa616a400c6d039fa6 100644 (file)
--- a/src/ceph_mon.cc
+++ b/src/ceph_mon.cc
@@ -778,7 +778,8 @@ int main(int argc, const char **argv)
   std::string public_msgr_type = g_conf()->ms_public_type.empty() ? g_conf().get_val<std::string>("ms_type") : g_conf()->ms_public_type;
   Messenger *msgr = Messenger::create(g_ceph_context, public_msgr_type,
                                      entity_name_t::MON(rank), "mon",
-                                     0, Messenger::HAS_MANY_CONNECTIONS);
+                                     0,  // zero nonce
+                                     Messenger::HAS_MANY_CONNECTIONS);
   if (!msgr)
     exit(1);
   msgr->set_cluster_protocol(CEPH_MON_PROTOCOL);
@@ -829,7 +830,8 @@ int main(int argc, const char **argv)
 
   Messenger *mgr_msgr = Messenger::create(g_ceph_context, public_msgr_type,
                                          entity_name_t::MON(rank), "mon-mgrc",
-                                         getpid(), 0);
+                                         Messenger::get_pid_nonce(),
+                                         0);
   if (!mgr_msgr) {
     derr << "unable to create mgr_msgr" << dendl;
     prefork.exit(1);

diff --git a/src/ceph_osd.cc b/src/ceph_osd.cc
index 1bdb36b8369aa5afaa84b7c4de4de932e3d55785..1e9516aa30d6250215c4a67eecd883fca33b5fe3 100644 (file)
--- a/src/ceph_osd.cc
+++ b/src/ceph_osd.cc
@@ -524,31 +524,32 @@ flushjournal_out:
 
   public_msg_type = public_msg_type.empty() ? msg_type : public_msg_type;
   cluster_msg_type = cluster_msg_type.empty() ? msg_type : cluster_msg_type;
+  uint64_t nonce = Messenger::get_pid_nonce();
   Messenger *ms_public = Messenger::create(g_ceph_context, public_msg_type,
                                           entity_name_t::OSD(whoami), "client",
-                                          getpid(),
+                                          nonce,
                                           Messenger::HAS_HEAVY_TRAFFIC |
                                           Messenger::HAS_MANY_CONNECTIONS);
   Messenger *ms_cluster = Messenger::create(g_ceph_context, cluster_msg_type,
                                            entity_name_t::OSD(whoami), "cluster",
-                                           getpid(),
+                                           nonce,
                                            Messenger::HAS_HEAVY_TRAFFIC |
                                            Messenger::HAS_MANY_CONNECTIONS);
   Messenger *ms_hb_back_client = Messenger::create(g_ceph_context, cluster_msg_type,
                                             entity_name_t::OSD(whoami), "hb_back_client",
-                                            getpid(), Messenger::HEARTBEAT);
+                                            nonce, Messenger::HEARTBEAT);
   Messenger *ms_hb_front_client = Messenger::create(g_ceph_context, public_msg_type,
                                             entity_name_t::OSD(whoami), "hb_front_client",
-                                            getpid(), Messenger::HEARTBEAT);
+                                            nonce, Messenger::HEARTBEAT);
   Messenger *ms_hb_back_server = Messenger::create(g_ceph_context, cluster_msg_type,
                                                   entity_name_t::OSD(whoami), "hb_back_server",
-                                                  getpid(), Messenger::HEARTBEAT);
+                                                  nonce, Messenger::HEARTBEAT);
   Messenger *ms_hb_front_server = Messenger::create(g_ceph_context, public_msg_type,
                                                    entity_name_t::OSD(whoami), "hb_front_server",
-                                                   getpid(), Messenger::HEARTBEAT);
+                                                   nonce, Messenger::HEARTBEAT);
   Messenger *ms_objecter = Messenger::create(g_ceph_context, public_msg_type,
                                             entity_name_t::OSD(whoami), "ms_objecter",
-                                            getpid(), 0);
+                                            nonce, 0);
   if (!ms_public || !ms_cluster || !ms_hb_front_client || !ms_hb_back_client || !ms_hb_back_server || !ms_hb_front_server || !ms_objecter)
     forker.exit(1);
   ms_cluster->set_cluster_protocol(CEPH_OSD_PROTOCOL);

diff --git a/src/crimson/osd/main.cc b/src/crimson/osd/main.cc
index 370d1e8dfbd60d89dac35fbc268a425ae3ea1c9d..d734fad46a9f3450d641aa25759b5b24e3a0f367 100644 (file)
--- a/src/crimson/osd/main.cc
+++ b/src/crimson/osd/main.cc
@@ -5,6 +5,7 @@
 #include <unistd.h>
 
 #include <iostream>
+#include <random>
 
 #include <seastar/core/app-template.hh>
 #include <seastar/core/print.hh>
@@ -105,6 +106,18 @@ seastar::future<> make_keyring()
   });
 }
 
+uint64_t get_nonce()
+{
+  if (auto pid = getpid(); pid != 1) {
+    return pid;
+  } else {
+    // we're running in a container; use a random number instead!
+    std::random_device rd;
+    std::default_random_engine rng{rd()};
+    return std::uniform_int_distribution<uint64_t>{}(rng);
+  }
+}
+
 int main(int argc, char* argv[])
 {
   seastar::app_template app;
@@ -153,14 +166,15 @@ int main(int argc, char* argv[])
         local_conf().parse_argv(ceph_args).get();
         pidfile_write(local_conf()->pid_file);
         const int whoami = std::stoi(local_conf()->name.get_id());
-        const auto nonce = static_cast<uint32_t>(getpid());
+        const auto nonce = get_nonce();
         crimson::net::MessengerRef cluster_msgr, client_msgr;
         crimson::net::MessengerRef hb_front_msgr, hb_back_msgr;
         for (auto [msgr, name] : {make_pair(std::ref(cluster_msgr), "cluster"s),
                                   make_pair(std::ref(client_msgr), "client"s),
                                   make_pair(std::ref(hb_front_msgr), "hb_front"s),
                                   make_pair(std::ref(hb_back_msgr), "hb_back"s)}) {
-          msgr = crimson::net::Messenger::create(entity_name_t::OSD(whoami), name, nonce);
+          msgr = crimson::net::Messenger::create(entity_name_t::OSD(whoami), name,
+                                                 nonce);
           if (local_conf()->ms_crc_data) {
             msgr->set_crc_data();
           }

diff --git a/src/mgr/DaemonServer.cc b/src/mgr/DaemonServer.cc
index 06573211ba3c28c9cc21a7453af4bad2ec7a6e50..95d164767fcb7519568f02400d14fb06f33340d3 100644 (file)
--- a/src/mgr/DaemonServer.cc
+++ b/src/mgr/DaemonServer.cc
@@ -108,7 +108,8 @@ int DaemonServer::init(uint64_t gid, entity_addrvec_t client_addrs)
   msgr = Messenger::create(g_ceph_context, public_msgr_type,
                           entity_name_t::MGR(gid),
                           "mgr",
-                          getpid(), 0);
+                          Messenger::get_pid_nonce(),
+                          0);
   msgr->set_default_policy(Messenger::Policy::stateless_server(0));
 
   msgr->set_auth_client(monc);

diff --git a/src/mgr/MgrStandby.cc b/src/mgr/MgrStandby.cc
index 16c82b72e88d8548e8c6669943a0173bd0642684..296494ece14cad86084fcc888bb30c2ab27c681a 100644 (file)
--- a/src/mgr/MgrStandby.cc
+++ b/src/mgr/MgrStandby.cc
@@ -44,7 +44,7 @@ MgrStandby::MgrStandby(int argc, const char **argv) :
                     cct->_conf.get_val<std::string>("ms_type"),
                     entity_name_t::MGR(),
                     "mgr",
-                    getpid(),
+                    Messenger::get_pid_nonce(),
                     0)),
   objecter{g_ceph_context, client_messenger.get(), &monc, NULL, 0, 0},
   client{client_messenger.get(), &monc, &objecter},

diff --git a/src/msg/Messenger.cc b/src/msg/Messenger.cc
index b975006a1dd7a42b5aa5f2b9ecbe9d53dfcb0df4..4e2ea96ebe20b16cb8326c7a0b73c3438d811af5 100644 (file)
--- a/src/msg/Messenger.cc
+++ b/src/msg/Messenger.cc
@@ -13,11 +13,26 @@
 Messenger *Messenger::create_client_messenger(CephContext *cct, string lname)
 {
   std::string public_msgr_type = cct->_conf->ms_public_type.empty() ? cct->_conf.get_val<std::string>("ms_type") : cct->_conf->ms_public_type;
-  auto nonce = ceph::util::generate_random_number<uint64_t>();
+  auto nonce = get_random_nonce();
   return Messenger::create(cct, public_msgr_type, entity_name_t::CLIENT(),
                           std::move(lname), nonce, 0);
 }
 
+uint64_t Messenger::get_pid_nonce()
+{
+  uint64_t nonce = getpid();
+  if (nonce == 1) {
+    // we're running in a container; use a random number instead!
+    nonce = ceph::util::generate_random_number<uint64_t>();
+  }
+  return nonce;
+}
+
+uint64_t Messenger::get_random_nonce()
+{
+  return ceph::util::generate_random_number<uint64_t>();
+}
+
 Messenger *Messenger::create(CephContext *cct, const string &type,
                             entity_name_t name, string lname,
                             uint64_t nonce, uint64_t cflags)

diff --git a/src/msg/Messenger.h b/src/msg/Messenger.h
index 22f5fe387cd4812556a6ac2606343ed65e40ad1d..c00374c114d2d5ef02e270bd75fea9ca69d6ebf6 100644 (file)
--- a/src/msg/Messenger.h
+++ b/src/msg/Messenger.h
@@ -153,6 +153,9 @@ public:
                            uint64_t nonce,
                           uint64_t cflags);
 
+  static uint64_t get_random_nonce();
+  static uint64_t get_pid_nonce();
+
   /**
    * create a new messenger
    *