msgr: take an arbitrary set of ports to avoid binding to

We used to only need to avoid 2 ports; now we need 3.  Make it a set so we
don't have this problem later.

Signed-off-by: Sage Weil <sage@inktank.com>

diff --git a/src/msg/Accepter.cc b/src/msg/Accepter.cc
index 90c68df6cf3de3936709bfd1c5f6b6afa763464b..4d13be8fdca35cc94706da8c31ed3b9f98e98926 100644 (file)
--- a/src/msg/Accepter.cc
+++ b/src/msg/Accepter.cc
@@ -37,7 +37,7 @@
  * Accepter
  */
 
-int Accepter::bind(const entity_addr_t &bind_addr, int avoid_port1, int avoid_port2)
+int Accepter::bind(const entity_addr_t &bind_addr, const set<int>& avoid_ports)
 {
   const md_config_t *conf = msgr->cct->_conf;
   // bind to a socket
@@ -92,7 +92,7 @@ int Accepter::bind(const entity_addr_t &bind_addr, int avoid_port1, int avoid_po
   } else {
     // try a range of ports
     for (int port = msgr->cct->_conf->ms_bind_port_min; port <= msgr->cct->_conf->ms_bind_port_max; port++) {
-      if (port == avoid_port1 || port == avoid_port2)
+      if (avoid_ports.count(port))
        continue;
       listen_addr.set_port(port);
       rc = ::bind(listen_sd, (struct sockaddr *) &listen_addr.ss_addr(), listen_addr.addr_size());
@@ -151,9 +151,9 @@ int Accepter::bind(const entity_addr_t &bind_addr, int avoid_port1, int avoid_po
   return 0;
 }
 
-int Accepter::rebind(int avoid_port)
+int Accepter::rebind(const set<int>& avoid_ports)
 {
-  ldout(msgr->cct,1) << "accepter.rebind avoid " << avoid_port << dendl;
+  ldout(msgr->cct,1) << "accepter.rebind avoid " << avoid_ports << dendl;
   
   stop();
 
@@ -161,11 +161,12 @@ int Accepter::rebind(int avoid_port)
   msgr->unlearn_addr();
 
   entity_addr_t addr = msgr->get_myaddr();
-  int old_port = addr.get_port();
+  set<int> new_avoid = avoid_ports;
+  new_avoid.insert(addr.get_port());
   addr.set_port(0);
 
-  ldout(msgr->cct,10) << " will try " << addr << dendl;
-  int r = bind(addr, old_port, avoid_port);
+  ldout(msgr->cct,10) << " will try " << addr << " and avoid ports " << new_avoid << dendl;
+  int r = bind(addr, new_avoid);
   if (r == 0)
     start();
   return r;

diff --git a/src/msg/Accepter.h b/src/msg/Accepter.h
index 07d766b32cddf70341ef1b5b50cc5bc76bd28e3f..4b1421f9e112259c02d2916beb8ac09df2f9f853 100644 (file)
--- a/src/msg/Accepter.h
+++ b/src/msg/Accepter.h
@@ -35,8 +35,8 @@ public:
     
   void *entry();
   void stop();
-  int bind(const entity_addr_t &bind_addr, int avoid_port1=0, int avoid_port2=0);
-  int rebind(int avoid_port);
+  int bind(const entity_addr_t &bind_addr, const set<int>& avoid_ports);
+  int rebind(const set<int>& avoid_port);
   int start();
 };
 

diff --git a/src/msg/Messenger.h b/src/msg/Messenger.h
index ca80dd1c5be3e4984e057976e9547cdee95b05f3..13d34611e19cc127b451cde70d5b22f66b9ffd44 100644 (file)
--- a/src/msg/Messenger.h
+++ b/src/msg/Messenger.h
@@ -341,7 +341,7 @@ public:
    *
    * @param avoid_port An additional port to avoid binding to.
    */
-  virtual int rebind(int avoid_port) { return -EOPNOTSUPP; }
+  virtual int rebind(const set<int>& avoid_ports) { return -EOPNOTSUPP; }
   /**
    * @} // Configuration
    */

diff --git a/src/msg/SimpleMessenger.cc b/src/msg/SimpleMessenger.cc
index 46e51dcf9f271addd213fbdb12f4ecd428d9b435..426fe237b76040b7680af46634b90831cd79f445 100644 (file)
--- a/src/msg/SimpleMessenger.cc
+++ b/src/msg/SimpleMessenger.cc
@@ -262,18 +262,19 @@ int SimpleMessenger::bind(const entity_addr_t &bind_addr)
   lock.Unlock();
 
   // bind to a socket
-  int r = accepter.bind(bind_addr);
+  set<int> avoid_ports;
+  int r = accepter.bind(bind_addr, avoid_ports);
   if (r >= 0)
     did_bind = true;
   return r;
 }
 
-int SimpleMessenger::rebind(int avoid_port)
+int SimpleMessenger::rebind(const set<int>& avoid_ports)
 {
-  ldout(cct,1) << "rebind avoid " << avoid_port << dendl;
+  ldout(cct,1) << "rebind avoid " << avoid_ports << dendl;
   mark_down_all();
   assert(did_bind);
-  return accepter.rebind(avoid_port);
+  return accepter.rebind(avoid_ports);
 }
 
 int SimpleMessenger::start()

diff --git a/src/msg/SimpleMessenger.h b/src/msg/SimpleMessenger.h
index 6be1a0a95392cb5ea1325ca686dc2dcb8e86952f..0d54d174965ca21f9c8ce365ec6c73029604fc23 100644 (file)
--- a/src/msg/SimpleMessenger.h
+++ b/src/msg/SimpleMessenger.h
@@ -197,7 +197,7 @@ public:
    *
    * @param avoid_port An additional port to avoid binding to.
    */
-  int rebind(int avoid_port);
+  int rebind(const set<int>& avoid_ports);
   /** @} Configuration functions */
 
   /**

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index b5a3a5ea309b2b69218ce51263964442c425a47e..5b51b44f24d0d0da31d4c023a2b1326c7f6b9434 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -4464,18 +4464,20 @@ void OSD::handle_osd_map(MOSDMap *m)
        do_restart = true;
        bind_epoch = osdmap->get_epoch();
 
-       int cport = cluster_messenger->get_myaddr().get_port();
-       int hbport = hb_back_server_messenger->get_myaddr().get_port();
+       set<int> avoid_ports;
+       avoid_ports.insert(cluster_messenger->get_myaddr().get_port());
+       avoid_ports.insert(hb_back_server_messenger->get_myaddr().get_port());
+       avoid_ports.insert(hb_front_server_messenger->get_myaddr().get_port());
 
-       int r = cluster_messenger->rebind(hbport);
+       int r = cluster_messenger->rebind(avoid_ports);
        if (r != 0)
          do_shutdown = true;  // FIXME: do_restart?
 
-       r = hb_back_server_messenger->rebind(cport);
+       r = hb_back_server_messenger->rebind(avoid_ports);
        if (r != 0)
          do_shutdown = true;  // FIXME: do_restart?
 
-       r = hb_front_server_messenger->rebind(hbport);
+       r = hb_front_server_messenger->rebind(avoid_ports);
        if (r != 0)
          do_shutdown = true;  // FIXME: do_restart?