From 0234e3c84842d40d823248e70575d2bb269f7a95 Mon Sep 17 00:00:00 2001
From: Sage Weil <sage@newdream.net>
Date: Fri, 14 Nov 2008 20:50:35 -0800
Subject: [PATCH] osd: introduce map_lock RWLock, take read lock during
 heartbeat

This prevents a race between handle_osd_map updating the map while
heartbeat() is using it to ping peers.

Currently we take a write ref over the entirely to handle_osd_map; we may
be able to push that down a bit.
---
 src/osd/OSD.cc | 10 ++++++++--
 src/osd/OSD.h  |  2 ++
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index cec7888a91ea2..a1d11b2194be5 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -271,6 +271,7 @@ OSD::OSD(int id, Messenger *m, MonMap *mm, const char *dev) :
   iat_averager(g_conf.osd_flash_crowd_iat_alpha),
   finished_lock("OSD::finished_lock"),
   osdmap(NULL),
+  map_lock("OSD::map_lock"),
   map_cache_lock("OSD::map_cache_lock"),
   snap_trimmer_lock("OSD::snap_trimmer_lock"),
   snap_trimmer_thread(this),
@@ -956,6 +957,8 @@ void OSD::heartbeat()
 
   //load_calc.set_size(stat_ops);
   
+  map_lock.get_read();
+
   // send heartbeats
   for (set<int>::iterator i = heartbeat_to.begin();
        i != heartbeat_to.end();
@@ -1031,6 +1034,7 @@ void OSD::heartbeat()
   remove_list.clear();
   remove_list_lock.Unlock();
 
+  map_lock.put_read();
 
   // schedule next!  randomly.
   float wait = .5 + ((float)(rand() % 10)/10.0) * (float)g_conf.osd_heartbeat_interval;
@@ -1494,7 +1498,8 @@ void OSD::handle_osd_map(MOSDMap *m)
 
   wait_for_no_ops();
   pause_recovery_thread();
-  
+  map_lock.get_write();
+
   assert(osd_lock.is_locked());
 
   ObjectStore::Transaction t;
@@ -1684,8 +1689,9 @@ void OSD::handle_osd_map(MOSDMap *m)
   store->apply_transaction(t);
   store->sync();
 
+  map_lock.put_write();
   unpause_recovery_thread();
-  
+
   //if (osdmap->get_epoch() == 1) store->sync();     // in case of early death, blah
 
   delete m;
diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index aa8ed2a37d378..cb3ba10938803 100644
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -18,6 +18,7 @@
 #include "msg/Dispatcher.h"
 
 #include "common/Mutex.h"
+#include "common/RWLock.h"
 #include "common/ThreadPool.h"
 #include "common/Timer.h"
 
@@ -249,6 +250,7 @@ private:
 
   // -- osd map --
   OSDMap         *osdmap;
+  RWLock          map_lock;
   list<Message*>  waiting_for_osdmap;
 
   hash_map<entity_name_t, epoch_t>  peer_map_epoch;  // FIXME types
-- 
2.39.5