osd: track recovery sources independently of missing list

Fixes pull() to choose an osd that isn't down.

diff --git a/src/osd/PG.cc b/src/osd/PG.cc
index 86bfc6f0b862c182637e991512b156c2d40f6e50..6433d3cfa943a27c4d54ac123632deb5c80a8690 100644 (file)
--- a/src/osd/PG.cc
+++ b/src/osd/PG.cc
@@ -432,7 +432,7 @@ void PG::proc_replica_missing(Log &olog, Missing &omissing, int fromosd)
     else if (need <= olog.top) {
       dout(10) << "proc_missing " << p->first << " " << need
                << " is on osd" << fromosd << dendl;
-      missing.loc[p->first] = fromosd;
+      missing_loc[p->first].insert(fromosd);
     } else {
       dout(10) << "proc_missing " << p->first << " " << need
                << " > olog.top " << olog.top << ", also missing on osd" << fromosd
@@ -777,7 +777,8 @@ void PG::clear_primary_state()
   peer_missing.clear();
 
   finish_sync_event = 0;  // so that _finish_recvoery doesn't go off in another thread
-  
+
+  missing_loc.clear();
   log.reset_recovery();
 
   stat_object_temp_rd.clear();
@@ -946,8 +947,8 @@ void PG::peer(ObjectStore::Transaction& t,
 
   
   // -- ok.  and have i located all pg contents?
-  if (missing.num_lost() > 0) {
-    dout(10) << "there are still " << missing.num_lost() << " lost objects" << dendl;
+  if (missing_loc.size()) {
+    dout(10) << "there are still " << missing_loc.size() << " lost objects" << dendl;
 
     // *****
     // FIXME: i don't think this actually accomplishes anything!
@@ -973,13 +974,13 @@ void PG::peer(ObjectStore::Transaction& t,
     }
     
     if (!waiting) {
-      dout(10) << missing.num_lost() << " objects are still lost, waiting+hoping for a notify from someone else!" << dendl;
+      dout(10) << missing_loc.size() << " objects are still lost, waiting+hoping for a notify from someone else!" << dendl;
     }
     return;
   }
 
   // sanity check
-  assert(missing.num_lost() == 0);
+  assert(missing_loc.empty());
   assert(info.last_complete >= log.bottom || log.backlog);
 
   // -- do need to notify the monitor?

diff --git a/src/osd/PG.h b/src/osd/PG.h
index 3a1a73c5a559b47631f28a1e9bb9ee367cb18ae3..29251e5ca4e4b93b916ab2afb13ae33d8b67a883 100644 (file)
--- a/src/osd/PG.h
+++ b/src/osd/PG.h
@@ -418,9 +418,6 @@ public:
     map<object_t, item> missing;         // oid -> (need v, have v)
     map<eversion_t, object_t> rmissing;  // v -> oid
 
-    map<object_t, int>       loc;       // where i think i can get them.
-
-    int num_lost() const { return missing.size() - loc.size(); }
     int num_missing() const { return missing.size(); }
 
     bool is_missing(object_t oid) {
@@ -457,24 +454,20 @@ public:
       if (missing.count(oid) && missing[oid].need < when) {
         rmissing.erase(missing[oid].need);
         missing.erase(oid);
-        loc.erase(oid);
-      }        
+      }
     }
     void got(object_t oid, eversion_t v) {
       assert(missing.count(oid));
       assert(missing[oid].need <= v);
-      loc.erase(oid);
       rmissing.erase(missing[oid].need);
       missing.erase(oid);
     }
 
     void encode(bufferlist &bl) const {
       ::encode(missing, bl);
-      ::encode(loc, bl);
     }
     void decode(bufferlist::iterator &bl) {
       ::decode(missing, bl);
-      ::decode(loc, bl);
 
       for (map<object_t,item>::iterator it = missing.begin();
            it != missing.end();
@@ -541,6 +534,8 @@ public:
   IndexedLog  log;
   OndiskLog   ondisklog;
   Missing     missing;
+  map<object_t, set<int> > missing_loc;
+  
   set<snapid_t> snap_collections;
 
   xlist<PG*>::item recovery_item;
@@ -868,8 +863,11 @@ inline ostream& operator<<(ostream& out, const PG& pg)
   //out << " (" << pg.log.bottom << "," << pg.log.top << "]";
   if (pg.missing.num_missing())
     out << " m=" << pg.missing.num_missing();
-  if (pg.is_primary() && pg.missing.num_lost())
-    out << " l=" << pg.missing.num_lost();
+  if (pg.is_primary()) {
+    int lost = pg.missing.num_missing() - pg.missing_loc.size();
+    if (lost)
+      out << " l=" << lost;
+  }
   if (pg.info.dead_snaps.size())
     out << " dead=" << pg.info.dead_snaps;
   out << "]";

diff --git a/src/osd/ReplicatedPG.cc b/src/osd/ReplicatedPG.cc
index 323595ed2d85017a20142106459c2e1bc2101b39..fe5c54b8b1db704643193bf5feff52ac8a36edda 100644 (file)
--- a/src/osd/ReplicatedPG.cc
+++ b/src/osd/ReplicatedPG.cc
@@ -1737,19 +1737,27 @@ void ReplicatedPG::calc_clone_subsets(SnapSet& snapset, pobject_t poid,
  */
 bool ReplicatedPG::pull(pobject_t poid)
 {
-  assert(missing.loc.count(poid.oid));
   eversion_t v = missing.missing[poid.oid].need;
-  int fromosd = missing.loc[poid.oid];
+
+  int fromosd = -1;
+  assert(missing_loc.count(poid.oid));
+  for (set<int>::iterator p = missing_loc[poid.oid].begin();
+       p != missing_loc[poid.oid].end();
+       p++) {
+    if (osd->osdmap->is_up(fromosd)) {
+      fromosd = *p;
+      break;
+    }
+  }
   
   dout(7) << "pull " << poid
           << " v " << v 
-          << " from osd" << fromosd
-          << dendl;
+         << " on osds " << missing_loc[poid.oid]
+         << " from osd" << fromosd
+         << dendl;
 
-  if (!osd->osdmap->is_up(fromosd)) {
-    dout(7) << " osd" << fromosd << " is down" << dendl;
+  if (fromosd < 0)
     return false;
-  }
 
   map<pobject_t, interval_set<__u64> > clone_subsets;
   interval_set<__u64> data_subset;
@@ -2144,7 +2152,7 @@ void ReplicatedPG::sub_op_push(MOSDSubOp *op)
     pulling.erase(poid.oid);
 
   missing.got(poid.oid, v);
-
+  missing_loc.erase(poid.oid);
 
   // raise last_complete?
   assert(log.complete_to != log.log.end());
@@ -2322,7 +2330,7 @@ void ReplicatedPG::on_role_change()
 void ReplicatedPG::cancel_recovery()
 {
   // forget about where missing items are, or anything we're pulling
-  missing.loc.clear();
+  missing_loc.clear();
   osd->num_pulling -= pulling.size();
   pulling.clear();
   pushing.clear();
@@ -2416,6 +2424,7 @@ int ReplicatedPG::recover_primary(int max)
                      latest->snaps);
          osd->store->apply_transaction(t);
          missing.got(latest->oid, latest->version);
+         missing_loc.erase(latest->oid);
          continue;
        }
       }