OSD: shard heartbeat_lock

heartbeat_need_update must be protected independently in order to avoid
a loop with the pg_map_lock and the PG::_lock.

Signed-off-by: Greg Farnum <greg@inktank.com>

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 4fb27e7610a29231e49a888f9592bd83a559ef16..3b7dfc5fda7d2641a63a90654b1a53634b3b3372 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -923,7 +923,8 @@ OSD::OSD(CephContext *cct_, ObjectStore *store_,
   paused_recovery(false),
   session_waiting_for_map_lock("OSD::session_waiting_for_map_lock"),
   heartbeat_lock("OSD::heartbeat_lock"),
-  heartbeat_stop(false), heartbeat_need_update(true), heartbeat_epoch(0),
+  heartbeat_stop(false), heartbeat_update_lock("OSD::heartbeat_update_lock"),
+  heartbeat_need_update(true), heartbeat_epoch(0),
   hbclient_messenger(hb_clientm),
   hb_front_server_messenger(hb_front_serverm),
   hb_back_server_messenger(hb_back_serverm),
@@ -2798,11 +2799,10 @@ void OSD::_remove_heartbeat_peer(int n)
 
 void OSD::need_heartbeat_peer_update()
 {
-  Mutex::Locker l(heartbeat_lock);
   if (is_stopping())
     return;
   dout(20) << "need_heartbeat_peer_update" << dendl;
-  heartbeat_need_update = true;
+  heartbeat_set_peers_need_update();
 }
 
 void OSD::maybe_update_heartbeat_peers()
@@ -2813,12 +2813,12 @@ void OSD::maybe_update_heartbeat_peers()
     utime_t now = ceph_clock_now(cct);
     if (last_heartbeat_resample == utime_t()) {
       last_heartbeat_resample = now;
-      heartbeat_need_update = true;
-    } else if (!heartbeat_need_update) {
+      heartbeat_set_peers_need_update();
+    } else if (!heartbeat_peers_need_update()) {
       utime_t dur = now - last_heartbeat_resample;
       if (dur > cct->_conf->osd_heartbeat_grace) {
        dout(10) << "maybe_update_heartbeat_peers forcing update after " << dur << " seconds" << dendl;
-       heartbeat_need_update = true;
+       heartbeat_set_peers_need_update();
        last_heartbeat_resample = now;
        reset_heartbeat_peers();   // we want *new* peers!
       }
@@ -2826,7 +2826,7 @@ void OSD::maybe_update_heartbeat_peers()
   }
 
   Mutex::Locker l(heartbeat_lock);
-  if (!heartbeat_need_update)
+  if (!heartbeat_peers_need_update())
     return;
   heartbeat_need_update = false;
 

diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index f0cb41952605c6a67a3f1bc3362e4c61449be9e8..c62b14b0f68982ed44290f5e917ba69f60c05c61 100644 (file)
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -1074,6 +1074,7 @@ private:
   map<int, int> debug_heartbeat_drops_remaining;
   Cond heartbeat_cond;
   bool heartbeat_stop;
+  Mutex heartbeat_update_lock; // orders under heartbeat_lock
   bool heartbeat_need_update;   ///< true if we need to refresh our heartbeat peers
   epoch_t heartbeat_epoch;      ///< last epoch we updated our heartbeat peers
   map<int,HeartbeatInfo> heartbeat_peers;  ///< map of osd id to HeartbeatInfo
@@ -1088,6 +1089,14 @@ private:
   bool heartbeat_reset(Connection *con);
   void maybe_update_heartbeat_peers();
   void reset_heartbeat_peers();
+  bool heartbeat_peers_need_update() {
+    Mutex::Locker l(heartbeat_update_lock);
+    return heartbeat_need_update;
+  }
+  void heartbeat_set_peers_need_update() {
+    Mutex::Locker l(heartbeat_update_lock);
+    heartbeat_need_update = true;
+  }
   void heartbeat();
   void heartbeat_check();
   void heartbeat_entry();