From: Greg Farnum <greg@inktank.com>
Date: Fri, 4 Apr 2014 21:32:37 +0000 (-0700)
Subject: OSD: do not take the pre_publish_lock in connection utility functions
X-Git-Tag: v0.81~57^2~7
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=fccf1c7010e902f3db8ad9e0548b8b72727dc8ac;p=ceph.git

OSD: do not take the pre_publish_lock in connection utility functions

They loop back around for local connections and deadlock, so we use the
map reservation mechanism instead.
TODO: actually that issue is out of date, do we still want this change?

Signed-off-by: Greg Farnum <greg@inktank.com>
---

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index fc910ed98e0..6846aff3e0a 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -3966,51 +3966,55 @@ void OSD::send_alive()
 
 void OSDService::send_message_osd_cluster(int peer, Message *m, epoch_t from_epoch)
 {
-  Mutex::Locker l(pre_publish_lock);
-
+  OSDMapRef next_map = get_nextmap_reserved();
   // service map is always newer/newest
-  assert(from_epoch <= next_osdmap->get_epoch());
+  assert(from_epoch <= next_map->get_epoch());
 
-  if (next_osdmap->is_down(peer) ||
-      next_osdmap->get_info(peer).up_from > from_epoch) {
+  if (next_map->is_down(peer) ||
+      next_map->get_info(peer).up_from > from_epoch) {
     m->put();
+    release_map(next_map);
     return;
   }
-  const entity_inst_t& peer_inst = next_osdmap->get_cluster_inst(peer);
+  const entity_inst_t& peer_inst = next_map->get_cluster_inst(peer);
   Connection *peer_con = osd->cluster_messenger->get_connection(peer_inst).get();
-  osd->_share_map_outgoing(peer, peer_con, next_osdmap);
+  osd->_share_map_outgoing(peer, peer_con, next_map);
   osd->cluster_messenger->send_message(m, peer_inst);
+  release_map(next_map);
 }
 
 ConnectionRef OSDService::get_con_osd_cluster(int peer, epoch_t from_epoch)
 {
-  Mutex::Locker l(pre_publish_lock);
-
+  OSDMapRef next_map = get_nextmap_reserved();
   // service map is always newer/newest
-  assert(from_epoch <= next_osdmap->get_epoch());
+  assert(from_epoch <= next_map->get_epoch());
 
-  if (next_osdmap->is_down(peer) ||
-      next_osdmap->get_info(peer).up_from > from_epoch) {
+  if (next_map->is_down(peer) ||
+      next_map->get_info(peer).up_from > from_epoch) {
+    release_map(next_map);
     return NULL;
   }
-  return osd->cluster_messenger->get_connection(next_osdmap->get_cluster_inst(peer));
+  ConnectionRef con = osd->cluster_messenger->get_connection(next_map->get_cluster_inst(peer));
+  release_map(next_map);
+  return con;
 }
 
 pair<ConnectionRef,ConnectionRef> OSDService::get_con_osd_hb(int peer, epoch_t from_epoch)
 {
-  Mutex::Locker l(pre_publish_lock);
-
+  OSDMapRef next_map = get_nextmap_reserved();
   // service map is always newer/newest
-  assert(from_epoch <= next_osdmap->get_epoch());
+  assert(from_epoch <= next_map->get_epoch());
 
   pair<ConnectionRef,ConnectionRef> ret;
-  if (next_osdmap->is_down(peer) ||
-      next_osdmap->get_info(peer).up_from > from_epoch) {
+  if (next_map->is_down(peer) ||
+      next_map->get_info(peer).up_from > from_epoch) {
+    release_map(next_map);
     return ret;
   }
-  ret.first = osd->hbclient_messenger->get_connection(next_osdmap->get_hb_back_inst(peer));
-  if (next_osdmap->get_hb_front_addr(peer) != entity_addr_t())
-    ret.second = osd->hbclient_messenger->get_connection(next_osdmap->get_hb_front_inst(peer));
+  ret.first = osd->hbclient_messenger->get_connection(next_map->get_hb_back_inst(peer));
+  if (next_map->get_hb_front_addr(peer) != entity_addr_t())
+    ret.second = osd->hbclient_messenger->get_connection(next_map->get_hb_front_inst(peer));
+  release_map(next_map);
   return ret;
 }