From: Sage Weil <sage@inktank.com>
Date: Wed, 22 May 2013 00:20:45 +0000 (-0700)
Subject: msgr: take an arbitrary set of ports to avoid binding to
X-Git-Tag: v0.64~61^2~2
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=28851424bf8290258a3dcb121d6f10d8b299054a;p=ceph.git

msgr: take an arbitrary set of ports to avoid binding to

We used to only need to avoid 2 ports; now we need 3.  Make it a set so we
don't have this problem later.

Signed-off-by: Sage Weil <sage@inktank.com>
---

diff --git a/src/msg/Accepter.cc b/src/msg/Accepter.cc
index 90c68df6cf3d..4d13be8fdca3 100644
--- a/src/msg/Accepter.cc
+++ b/src/msg/Accepter.cc
@@ -37,7 +37,7 @@
  * Accepter
  */
 
-int Accepter::bind(const entity_addr_t &bind_addr, int avoid_port1, int avoid_port2)
+int Accepter::bind(const entity_addr_t &bind_addr, const set<int>& avoid_ports)
 {
   const md_config_t *conf = msgr->cct->_conf;
   // bind to a socket
@@ -92,7 +92,7 @@ int Accepter::bind(const entity_addr_t &bind_addr, int avoid_port1, int avoid_po
   } else {
     // try a range of ports
     for (int port = msgr->cct->_conf->ms_bind_port_min; port <= msgr->cct->_conf->ms_bind_port_max; port++) {
-      if (port == avoid_port1 || port == avoid_port2)
+      if (avoid_ports.count(port))
 	continue;
       listen_addr.set_port(port);
       rc = ::bind(listen_sd, (struct sockaddr *) &listen_addr.ss_addr(), listen_addr.addr_size());
@@ -151,9 +151,9 @@ int Accepter::bind(const entity_addr_t &bind_addr, int avoid_port1, int avoid_po
   return 0;
 }
 
-int Accepter::rebind(int avoid_port)
+int Accepter::rebind(const set<int>& avoid_ports)
 {
-  ldout(msgr->cct,1) << "accepter.rebind avoid " << avoid_port << dendl;
+  ldout(msgr->cct,1) << "accepter.rebind avoid " << avoid_ports << dendl;
   
   stop();
 
@@ -161,11 +161,12 @@ int Accepter::rebind(int avoid_port)
   msgr->unlearn_addr();
 
   entity_addr_t addr = msgr->get_myaddr();
-  int old_port = addr.get_port();
+  set<int> new_avoid = avoid_ports;
+  new_avoid.insert(addr.get_port());
   addr.set_port(0);
 
-  ldout(msgr->cct,10) << " will try " << addr << dendl;
-  int r = bind(addr, old_port, avoid_port);
+  ldout(msgr->cct,10) << " will try " << addr << " and avoid ports " << new_avoid << dendl;
+  int r = bind(addr, new_avoid);
   if (r == 0)
     start();
   return r;
diff --git a/src/msg/Accepter.h b/src/msg/Accepter.h
index 07d766b32cdd..4b1421f9e112 100644
--- a/src/msg/Accepter.h
+++ b/src/msg/Accepter.h
@@ -35,8 +35,8 @@ public:
     
   void *entry();
   void stop();
-  int bind(const entity_addr_t &bind_addr, int avoid_port1=0, int avoid_port2=0);
-  int rebind(int avoid_port);
+  int bind(const entity_addr_t &bind_addr, const set<int>& avoid_ports);
+  int rebind(const set<int>& avoid_port);
   int start();
 };
 
diff --git a/src/msg/Messenger.h b/src/msg/Messenger.h
index ca80dd1c5be3..13d34611e19c 100644
--- a/src/msg/Messenger.h
+++ b/src/msg/Messenger.h
@@ -341,7 +341,7 @@ public:
    *
    * @param avoid_port An additional port to avoid binding to.
    */
-  virtual int rebind(int avoid_port) { return -EOPNOTSUPP; }
+  virtual int rebind(const set<int>& avoid_ports) { return -EOPNOTSUPP; }
   /**
    * @} // Configuration
    */
diff --git a/src/msg/SimpleMessenger.cc b/src/msg/SimpleMessenger.cc
index 46e51dcf9f27..426fe237b760 100644
--- a/src/msg/SimpleMessenger.cc
+++ b/src/msg/SimpleMessenger.cc
@@ -262,18 +262,19 @@ int SimpleMessenger::bind(const entity_addr_t &bind_addr)
   lock.Unlock();
 
   // bind to a socket
-  int r = accepter.bind(bind_addr);
+  set<int> avoid_ports;
+  int r = accepter.bind(bind_addr, avoid_ports);
   if (r >= 0)
     did_bind = true;
   return r;
 }
 
-int SimpleMessenger::rebind(int avoid_port)
+int SimpleMessenger::rebind(const set<int>& avoid_ports)
 {
-  ldout(cct,1) << "rebind avoid " << avoid_port << dendl;
+  ldout(cct,1) << "rebind avoid " << avoid_ports << dendl;
   mark_down_all();
   assert(did_bind);
-  return accepter.rebind(avoid_port);
+  return accepter.rebind(avoid_ports);
 }
 
 int SimpleMessenger::start()
diff --git a/src/msg/SimpleMessenger.h b/src/msg/SimpleMessenger.h
index 6be1a0a95392..0d54d174965c 100644
--- a/src/msg/SimpleMessenger.h
+++ b/src/msg/SimpleMessenger.h
@@ -197,7 +197,7 @@ public:
    *
    * @param avoid_port An additional port to avoid binding to.
    */
-  int rebind(int avoid_port);
+  int rebind(const set<int>& avoid_ports);
   /** @} Configuration functions */
 
   /**
diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index b5a3a5ea309b..5b51b44f24d0 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -4464,18 +4464,20 @@ void OSD::handle_osd_map(MOSDMap *m)
 	do_restart = true;
 	bind_epoch = osdmap->get_epoch();
 
-	int cport = cluster_messenger->get_myaddr().get_port();
-	int hbport = hb_back_server_messenger->get_myaddr().get_port();
+	set<int> avoid_ports;
+	avoid_ports.insert(cluster_messenger->get_myaddr().get_port());
+	avoid_ports.insert(hb_back_server_messenger->get_myaddr().get_port());
+	avoid_ports.insert(hb_front_server_messenger->get_myaddr().get_port());
 
-	int r = cluster_messenger->rebind(hbport);
+	int r = cluster_messenger->rebind(avoid_ports);
 	if (r != 0)
 	  do_shutdown = true;  // FIXME: do_restart?
 
-	r = hb_back_server_messenger->rebind(cport);
+	r = hb_back_server_messenger->rebind(avoid_ports);
 	if (r != 0)
 	  do_shutdown = true;  // FIXME: do_restart?
 
-	r = hb_front_server_messenger->rebind(hbport);
+	r = hb_front_server_messenger->rebind(avoid_ports);
 	if (r != 0)
 	  do_shutdown = true;  // FIXME: do_restart?