a few osd/pg locking fixes, cleanup

git-svn-id: https://ceph.svn.sf.net/svnroot/ceph@1524 29311d96-e01e-0410-9327-a35deaab8ce9

diff --git a/trunk/ceph/osd/OSD.cc b/trunk/ceph/osd/OSD.cc
index 4dd8afe86fd6689d3a356bfb48eec8abc1f2d783..f7aa65a29b93b8d16e601e11b711a477750f0bcd 100644 (file)
--- a/trunk/ceph/osd/OSD.cc
+++ b/trunk/ceph/osd/OSD.cc
@@ -2160,6 +2160,12 @@ void OSD::dequeue_op(PG *pg)
     dout(10) << "dequeue_op " << *op << " pg " << *pg
             << ", " << (pending_ops-1) << " more pending"
             << dendl;
+
+    // share map?
+    //  do this preemptively while we hold osd_lock and pg->lock
+    //  to avoid lock ordering issues later.
+    for (unsigned i=1; i<pg->acting.size(); i++) 
+      _share_map_outgoing( osdmap->get_inst(pg->acting[i]) ); 
   }
   osd_lock.Unlock();
 

diff --git a/trunk/ceph/osd/OSD.h b/trunk/ceph/osd/OSD.h
index dc8bc92fb8964de9f7d155f203eba82c39fcd7f2..86f2950884c12a778ff91b768d8ebd6649656d58 100644 (file)
--- a/trunk/ceph/osd/OSD.h
+++ b/trunk/ceph/osd/OSD.h
@@ -53,6 +53,7 @@ public:
   //current implementation is moving averges.
   class LoadCalculator {
   private:
+    Mutex lock;
     deque<double> m_Data ;
     unsigned m_Size ;
     double  m_Total ;
@@ -61,6 +62,8 @@ public:
     LoadCalculator( unsigned size ) : m_Size(0), m_Total(0) { }
 
     void add( double element ) {
+      Mutex::Locker locker(lock);
+
       // add item
       m_Data.push_back(element);
       m_Total += element;
@@ -73,6 +76,8 @@ public:
     }
     
     double get_average() {
+      Mutex::Locker locker(lock);
+
       if (m_Data.empty())
        return -1;
       return m_Total / (double)m_Data.size();
@@ -87,6 +92,7 @@ public:
       iat_data() : last_req_stamp(0), average_iat(0) {}
     };
   private:
+    mutable Mutex lock;
     double alpha;
     hash_map<object_t, iat_data> iat_map;
 
@@ -94,6 +100,7 @@ public:
     IATAverager(double a) : alpha(a) {}
     
     void add_sample(object_t oid, double now) {
+      Mutex::Locker locker(lock);
       iat_data &r = iat_map[oid];
       double iat = now - r.last_req_stamp;
       r.last_req_stamp = now;
@@ -101,16 +108,19 @@ public:
     }
     
     bool have(object_t oid) const {
+      Mutex::Locker locker(lock);
       return iat_map.count(oid);
     }
 
     double get_average_iat(object_t oid) const {
+      Mutex::Locker locker(lock);
       hash_map<object_t, iat_data>::const_iterator p = iat_map.find(oid);
       assert(p != iat_map.end());
       return p->second.average_iat;
     }
 
     bool is_flash_crowd_candidate(object_t oid) const {
+      Mutex::Locker locker(lock);
       return get_average_iat(oid) <= g_conf.osd_flash_crowd_iat_threshold;
     }
   };

diff --git a/trunk/ceph/osd/PG.h b/trunk/ceph/osd/PG.h
index 2ef620806b18cecd3ec557f796d1f6466990fb7d..545c42807c182020c7e9bae0853b02e3d48204e8 100644 (file)
--- a/trunk/ceph/osd/PG.h
+++ b/trunk/ceph/osd/PG.h
@@ -396,26 +396,26 @@ protected:
 
 public:
   void lock() {
-    cout << this << " " << info.pgid << " lock" << endl;
+    //cout << this << " " << info.pgid << " lock" << endl;
     _lock.Lock();
   }
   void unlock() {
-    cout << this << " " << info.pgid << " unlock" << endl;
+    //cout << this << " " << info.pgid << " unlock" << endl;
     _lock.Unlock();
   }
   void get() {
-    cout << this << " " << info.pgid << " get " << ref << endl;
+    //cout << this << " " << info.pgid << " get " << ref << endl;
     assert(_lock.is_locked());
     ++ref; 
   }
   void put() { 
-    cout << this << " " << info.pgid << " put " << ref << endl;
+    //cout << this << " " << info.pgid << " put " << ref << endl;
     assert(_lock.is_locked());
     --ref; 
     assert(ref > 0);  // last put must be a put_unlock.
   }
   void put_unlock() { 
-    cout << this << " " << info.pgid << " put_unlock " << ref << endl;
+    //cout << this << " " << info.pgid << " put_unlock " << ref << endl;
     assert(_lock.is_locked());
     --ref; 
     _lock.Unlock();

diff --git a/trunk/ceph/osd/ReplicatedPG.cc b/trunk/ceph/osd/ReplicatedPG.cc
index c802c00f1ac83519cf975fe9cdd89bf7b1e2818d..ea6d123195e43df2dba1ac87e4d610fac87b8003 100644 (file)
--- a/trunk/ceph/osd/ReplicatedPG.cc
+++ b/trunk/ceph/osd/ReplicatedPG.cc
@@ -1084,16 +1084,6 @@ void ReplicatedPG::op_modify(MOSDOp *op)
   }
 
 
-  // share latest osd map with rest of pg?
-  osd->osd_lock.Lock();
-  {
-    for (unsigned i=1; i<acting.size(); i++) {
-      osd->_share_map_outgoing( osd->osdmap->get_inst(acting[i]) ); 
-    }
-  }
-  osd->osd_lock.Unlock();
-  
-
   // dup op?
   if (is_dup(op->get_reqid())) {
     dout(-3) << "op_modify " << opname << " dup op " << op->get_reqid()