msgr: let user explicitly set nonce

There will be problems if two messengers use the same entity_addr_t because
they are on the same ip and choose the same nonce (e.g., because they are
in the same process).  Let the caller sort this out in whatever way it
finds most appropriate.

For libceph, librados, and csyn, all N million to the pid.

Fixes: #877
Signed-off-by: Sage Weil <sage.weil@dreamhost.com>

diff --git a/src/cfuse.cc b/src/cfuse.cc
index 41341ea352513f6799daf3e0f76c63ec3a561f45..100cbf333a33e0e3eac4dd37f53600816b7b919b 100644 (file)
--- a/src/cfuse.cc
+++ b/src/cfuse.cc
@@ -118,7 +118,7 @@ int main(int argc, const char **argv, const char *envp[]) {
 
     cout << "cfuse[" << getpid() << "]: starting ceph client" << std::endl;
 
-    messenger->start(false); // Do not daemonize here
+    messenger->start(false, getpid()); // Do not daemonize here
 
     // start client
     client->init();

diff --git a/src/cmds.cc b/src/cmds.cc
index f920cd14391bc32319c6e03839db9d245d401724..459613a646738a8ec66196dacde52f14ec2d6723 100644 (file)
--- a/src/cmds.cc
+++ b/src/cmds.cc
@@ -119,7 +119,7 @@ int main(int argc, const char **argv)
     return -1;
 
   SimpleMessenger *messenger = new SimpleMessenger();
-  messenger->bind();
+  messenger->bind(getpid());
   if (dump_journal >= 0) {
     Dumper *journal_dumper = new Dumper(messenger, &mc);
     journal_dumper->init(dump_journal);

diff --git a/src/cmon.cc b/src/cmon.cc
index 56edc862c6022ec5104e7fd34cb5b223665c6654..96ed18deb7f4d17b0fd629b439f36ba1d7dfde7c 100644 (file)
--- a/src/cmon.cc
+++ b/src/cmon.cc
@@ -206,8 +206,7 @@ int main(int argc, const char **argv)
        << " mon_data " << g_conf.mon_data
        << " fsid " << monmap.get_fsid()
        << std::endl;
-  g_conf.public_addr = monmap.get_addr(g_conf.name->get_id());
-  err = messenger->bind();
+  err = messenger->bind(monmap.get_addr(g_conf.name->get_id()), 0);
   if (err < 0)
     return 1;
 

diff --git a/src/cosd.cc b/src/cosd.cc
index 02b3bff5168d42267d1c0917923fbbce7a551a94..a7f0c8a8fc7e89df1769fdddba651a819bf492e8 100644 (file)
--- a/src/cosd.cc
+++ b/src/cosd.cc
@@ -208,21 +208,21 @@ int main(int argc, const char **argv)
   SimpleMessenger *messenger_hb = new SimpleMessenger();
 
   if (client_addr_set)
-    client_messenger->bind(g_conf.public_addr);
+    client_messenger->bind(g_conf.public_addr, getpid());
   else
-    client_messenger->bind();
+    client_messenger->bind(getpid());
 
   entity_addr_t hb_addr;  // hb should bind to same ip ad cluster_addr (if specified)
 
   if (cluster_addr_set) {
-    cluster_messenger->bind(g_conf.cluster_addr);
+    cluster_messenger->bind(g_conf.cluster_addr, getpid());
     hb_addr = g_conf.cluster_addr;
     hb_addr.set_port(0);
   } else {
-    cluster_messenger->bind();
+    cluster_messenger->bind(getpid());
   }
 
-  messenger_hb->bind(hb_addr);
+  messenger_hb->bind(hb_addr, getpid());
 
   cout << "starting osd" << whoami
        << " at " << client_messenger->get_ms_addr() 

diff --git a/src/csyn.cc b/src/csyn.cc
index e18d9d2276961ce5201d85c3ccd68765086bd87a..fff06349ad6494c6e23b15327d6686d521c5d01e 100644 (file)
--- a/src/csyn.cc
+++ b/src/csyn.cc
@@ -67,7 +67,7 @@ int main(int argc, const char **argv, char *envp[])
   for (int i=0; i<g_conf.num_client; i++) {
     messengers[i] = new SimpleMessenger();
     messengers[i]->register_entity(entity_name_t(entity_name_t::TYPE_CLIENT,-1));
-    messengers[i]->bind();
+    messengers[i]->bind(i * 1000000 + getpid());
     mclients[i] = new MonClient();
     mclients[i]->build_initial_monmap();
     Client *client = new Client(messengers[i], mclients[i]);

diff --git a/src/libceph.cc b/src/libceph.cc
index c722f9e9701b566a27c834b543bfd2112b13da00..c56cd93e328a2311f7d9439721c6919c751f40b3 100644 (file)
--- a/src/libceph.cc
+++ b/src/libceph.cc
@@ -35,6 +35,7 @@ static int client_mount = 0;
 static Client *client = NULL;
 static MonClient *monclient = NULL;
 static SimpleMessenger *messenger = NULL;
+static int instance = 0;
 
 extern "C" int ceph_initialize(int argc, const char **argv)
 {
@@ -61,7 +62,8 @@ extern "C" int ceph_initialize(int argc, const char **argv)
     //at last the client
     client = new Client(messenger, monclient);
 
-    messenger->start(false); // do not daemonize
+    uint64_t nonce = (uint64_t)++instance * 1000000ull + (uint64_t)getpid();
+    messenger->start(false, nonce); // do not daemonize
 
     client->init();
   }

diff --git a/src/librados.cc b/src/librados.cc
index 3e5452dd5d4694897e14766edf243d364b394b5e..8e9d3e9d53c1778f8e850db0b96babc7bf61ad54 100644 (file)
--- a/src/librados.cc
+++ b/src/librados.cc
@@ -51,6 +51,9 @@ using namespace std;
 #define dout_prefix *_dout << "librados: "
 
 
+static atomic_t rados_instance;
+
+
 /*
  * Structure of this file
  *
@@ -568,7 +571,11 @@ connect()
 
   messenger->add_dispatcher_head(this);
 
-  messenger->start(false); // do not daemonize
+  uint64_t nonce;
+  rados_instance.inc();
+  nonce = getpid() + (1000000 * (uint64_t)rados_instance.read());
+
+  messenger->start(false, nonce); // do not daemonize
   messenger->add_dispatcher_head(this);
 
   dout(1) << "setting wanted keys" << dendl;

diff --git a/src/mds/Dumper.cc b/src/mds/Dumper.cc
index 3b0902a4008edf900212ed22db8149b1ece5f322..79d346de3c3de67bd4899d5b06733a7f52929474 100644 (file)
--- a/src/mds/Dumper.cc
+++ b/src/mds/Dumper.cc
@@ -56,7 +56,7 @@ void Dumper::init(int rank)
 
   messenger->register_entity(entity_name_t::CLIENT());
   messenger->add_dispatcher_head(this);
-  messenger->start(false); // do not daemonize
+  messenger->start(false, getpid()); // do not daemonize
 
   monc->set_want_keys(CEPH_ENTITY_TYPE_MON|CEPH_ENTITY_TYPE_OSD|CEPH_ENTITY_TYPE_MDS);
   monc->set_messenger(messenger);

diff --git a/src/mds/Resetter.cc b/src/mds/Resetter.cc
index 8cc17704f0641add28739ba386bec0b8e4c9241c..3ee0e074a9802c47682a6352a9b74714655a080d 100644 (file)
--- a/src/mds/Resetter.cc
+++ b/src/mds/Resetter.cc
@@ -50,7 +50,7 @@ void Resetter::init(int rank)
 
   messenger->register_entity(entity_name_t::CLIENT());
   messenger->add_dispatcher_head(this);
-  messenger->start(true);
+  messenger->start(true, getpid());
 
   monc->set_want_keys(CEPH_ENTITY_TYPE_MON|CEPH_ENTITY_TYPE_OSD|CEPH_ENTITY_TYPE_MDS);
   monc->set_messenger(messenger);

diff --git a/src/mon/MonClient.cc b/src/mon/MonClient.cc
index 2946ef2ac9a86712d47f8c3fea0a29e8b4608957..189207d50ce75cc015f9fd8d8e70dc55f2acc9a9 100644 (file)
--- a/src/mon/MonClient.cc
+++ b/src/mon/MonClient.cc
@@ -173,7 +173,7 @@ int MonClient::get_monmap_privately()
     messenger = smessenger = new SimpleMessenger();
     smessenger->register_entity(entity_name_t::CLIENT(-1));
     messenger->add_dispatcher_head(this);
-    smessenger->start(false);  // do not daemonize!
+    smessenger->start(false, getpid());  // do not daemonize!
     temp_msgr = true; 
   }
   

diff --git a/src/msg/SimpleMessenger.cc b/src/msg/SimpleMessenger.cc
index 357ba57d1a7e7b7f1f293f0c5c67bf5e73588d15..467e5a05623a3b0f009189ae92aa5430788cfeb3 100644 (file)
--- a/src/msg/SimpleMessenger.cc
+++ b/src/msg/SimpleMessenger.cc
@@ -52,7 +52,7 @@ static ostream& _prefix(SimpleMessenger *messenger) {
  * Accepter
  */
 
-int SimpleMessenger::Accepter::bind(int64_t force_nonce, entity_addr_t &bind_addr, int avoid_port1, int avoid_port2)
+int SimpleMessenger::Accepter::bind(uint64_t nonce, entity_addr_t &bind_addr, int avoid_port1, int avoid_port2)
 {
   // bind to a socket
   dout(10) << "accepter.bind" << dendl;
@@ -145,10 +145,7 @@ int SimpleMessenger::Accepter::bind(int64_t force_nonce, entity_addr_t &bind_add
 
   if (messenger->ms_addr.get_port() == 0) {
     messenger->ms_addr = listen_addr;
-    if (force_nonce >= 0)
-      messenger->ms_addr.nonce = force_nonce;
-    else
-      messenger->ms_addr.nonce = getpid(); // FIXME: pid might not be best choice here.
+    messenger->ms_addr.nonce = nonce;
   }
 
   messenger->init_local_pipe();
@@ -2298,7 +2295,7 @@ void SimpleMessenger::queue_reap(Pipe *pipe)
 
 
 
-int SimpleMessenger::bind(entity_addr_t &bind_addr, int64_t force_nonce)
+int SimpleMessenger::bind(entity_addr_t bind_addr, int64_t nonce)
 {
   lock.Lock();
   if (started) {
@@ -2310,7 +2307,7 @@ int SimpleMessenger::bind(entity_addr_t &bind_addr, int64_t force_nonce)
   lock.Unlock();
 
   // bind to a socket
-  return accepter.bind(force_nonce, bind_addr);
+  return accepter.bind(nonce, bind_addr);
 }
 
 int SimpleMessenger::rebind(int avoid_port)
@@ -2406,7 +2403,7 @@ int SimpleMessenger::write_pid_file(int pid)
   return 0;
 }
 
-int SimpleMessenger::start(bool daemonize)
+int SimpleMessenger::start(bool daemonize, uint64_t nonce)
 {
   // register at least one entity, first!
   assert(my_type >= 0); 
@@ -2417,9 +2414,10 @@ int SimpleMessenger::start(bool daemonize)
     lock.Unlock();
     return 0;
   }
-
-  if (!did_bind)
-    ms_addr.nonce = getpid();
+  
+  if (!did_bind) {
+    ms_addr.nonce = nonce;
+  }
 
   dout(1) << "messenger.start" << dendl;
   started = true;

diff --git a/src/msg/SimpleMessenger.h b/src/msg/SimpleMessenger.h
index 4c57ecc95892ed3322109cfb188a843c829d27e8..a488406c79b8734360239c2110c316c772bb807b 100644 (file)
--- a/src/msg/SimpleMessenger.h
+++ b/src/msg/SimpleMessenger.h
@@ -104,7 +104,7 @@ private:
     
     void *entry();
     void stop();
-    int bind(int64_t force_nonce, entity_addr_t &bind_addr, int avoid_port1=0, int avoid_port2=0);
+    int bind(uint64_t nonce, entity_addr_t &bind_addr, int avoid_port1=0, int avoid_port2=0);
     int rebind(int avoid_port);
     int start();
   } accepter;
@@ -560,9 +560,15 @@ public:
 
   //void set_listen_addr(tcpaddr_t& a);
 
-  int bind(entity_addr_t& bind_addr, int64_t force_nonce = -1);
-  int bind(int64_t force_nonce = -1) { return bind(g_conf.public_addr, force_nonce); }
-  int start(bool daemonize);
+  int bind(entity_addr_t bind_addr, int64_t nonce);
+  int bind(uint64_t nonce) {
+    return bind(g_conf.public_addr, nonce);
+  }
+  int start(bool daemonize, uint64_t nonce);  // if we didn't bind
+  int start(bool daemonize) {                 // if we did
+    assert(did_bind);
+    start(daemonize, 0);
+  }
   void wait();
 
   int write_pid_file(int pid);

diff --git a/src/tools/common.cc b/src/tools/common.cc
index 6579843ced7f6a378100813c6b9a62bd5b6b87ed..04d82b5d9f074b62117c640e1d450cb959bce10d 100644 (file)
--- a/src/tools/common.cc
+++ b/src/tools/common.cc
@@ -553,7 +553,7 @@ int ceph_tool_common_init(ceph_tool_mode_t mode)
   // start up network
   messenger = new SimpleMessenger();
   messenger->register_entity(entity_name_t::CLIENT());
-  messenger->start(false); // do not daemonize
+  messenger->start(false, getpid()); // do not daemonize
   messenger->add_dispatcher_head(&dispatcher);
 
   g.lock.Lock();