osd: fix divergent backfill targets

During peering, a previous backfill target may have a slightly newer
last_update than the other options, but it will not be chosen because it
is incomplete.  That caused a failed assert during activate() (#1983).

To fix, we remove the bad assert, and then fix merge_log() so that the
replica/backfill target will trim its divergent entries when it gets the
activation MLogRec.  We also fix the handling of MInfoRec, as that can
trigger the same analogous condition.

Signed-off-by: Sage Weil <sage.weil@dreamhost.com>

diff --git a/src/osd/PG.cc b/src/osd/PG.cc
index ae3bf1405b033bf662b6bbd9b18672183a89d5d7..c35c14685c203829057c9ababea4f9984747a718 100644 (file)
--- a/src/osd/PG.cc
+++ b/src/osd/PG.cc
@@ -361,6 +361,48 @@ bool PG::merge_old_entry(ObjectStore::Transaction& t, Log::Entry& oe)
   return false;
 }
 
+/**
+ * rewind divergent entries at the head of the log
+ *
+ * This rewinds entries off the head of our log that are divergent.
+ * This is used by replicas during activation.
+ *
+ * @param t transaction
+ * @param newhead new head to rewind to
+ */
+void PG::rewind_divergent_log(ObjectStore::Transaction& t, eversion_t newhead)
+{
+  dout(10) << "rewind_divergent_log truncate divergent future " << newhead << dendl;
+  assert(newhead > log.tail);
+
+  list<Log::Entry>::iterator p = log.log.end();
+  list<Log::Entry> divergent;
+  while (true) {
+    if (p == log.log.begin()) {
+      // yikes, the whole thing is divergent!
+      divergent.swap(log.log);
+      break;
+    }
+    --p;
+    if (p->version == newhead) {
+      ++p;
+      divergent.splice(divergent.begin(), log.log, p, log.log.end());
+      break;
+    }
+    assert(p->version > newhead);
+    dout(10) << "rewind_divergent_log future divergent " << *p << dendl;
+    log.unindex(*p);
+  }
+
+  log.head = newhead;
+  info.last_update = newhead;
+  if (info.last_complete > newhead)
+    info.last_complete == newhead;
+
+  for (list<Log::Entry>::iterator d = divergent.begin(); d != divergent.end(); d++)
+    merge_old_entry(t, *d);
+}
+
 void PG::merge_log(ObjectStore::Transaction& t,
                   Info &oinfo, Log &olog, int fromosd)
 {
@@ -408,7 +450,12 @@ void PG::merge_log(ObjectStore::Transaction& t,
     info.log_tail = log.tail = olog.tail;
     changed = true;
   }
-    
+
+  // do we have divergent entries to throw out?
+  if (olog.head < log.head) {
+    rewind_divergent_log(t, olog.head);
+  }
+
   // extend on head?
   if (olog.head > log.head) {
     dout(10) << "merge_log extending head to " << olog.head << dendl;
@@ -467,7 +514,6 @@ void PG::merge_log(ObjectStore::Transaction& t,
                   olog.log, from, to);
     log.index();   
 
-      
     info.last_update = log.head = olog.head;
     if (oinfo.stats.reported < info.stats.reported)   // make sure reported always increases
       oinfo.stats.reported = info.stats.reported;
@@ -1316,7 +1362,6 @@ void PG::activate(ObjectStore::Transaction& t, list<Context*>& tfin,
        assert(log.tail <= pi.last_update);
        m = new MOSDPGLog(get_osdmap()->get_epoch(), info);
        // send new stuff to append to replicas log
-       assert(info.last_update > pi.last_update);
        m->log.copy_after(log, pi.last_update);
       }
 
@@ -4137,6 +4182,12 @@ boost::statechart::result PG::RecoveryState::Stray::react(const MInfoRec& infoev
   PG *pg = context< RecoveryMachine >().pg;
   dout(10) << "got info from osd." << infoevt.from << " " << infoevt.info << dendl;
 
+  if (pg->info.last_update > infoevt.info.last_update) {
+    // rewind divergent log entries
+    pg->rewind_divergent_log(*context< RecoveryMachine >().get_cur_transaction(),
+                            infoevt.info.last_update);
+  }
+  
   assert(infoevt.info.last_update == pg->info.last_update);
   assert(pg->log.tail <= pg->info.last_complete);
   assert(pg->log.head == pg->info.last_update);

diff --git a/src/osd/PG.h b/src/osd/PG.h
index 1619e1ee8f00bba33ffe1ef90aea7bb23c8dbbea..526f0aae0696887ede61e668dfd7f687d11af8aa 100644 (file)
--- a/src/osd/PG.h
+++ b/src/osd/PG.h
@@ -1555,6 +1555,8 @@ public:
   bool proc_replica_info(int from, Info &info);
   bool merge_old_entry(ObjectStore::Transaction& t, Log::Entry& oe);
   void merge_log(ObjectStore::Transaction& t, Info &oinfo, Log &olog, int from);
+  void rewind_divergent_log(ObjectStore::Transaction& t, eversion_t newhead);
+
   bool search_for_missing(const Info &oinfo, const Missing *omissing,
                          int fromosd);