osd: revamp forgetting lost objects

The old forget lost objects rewrote history in the PG log, which is asking
for all kinds of trouble.  Instead, add new logs events to indicate that
an object is LOST (deleted) or LOST_REVERTed (reverted to an older
version).

The LOST_REVERT case means we may need to recover the old version from
another node and rewrite the version number.  This isn't implemented yet;
for now we just assert.

Signed-off-by: Sage Weil <sage@newdream.net>

diff --git a/src/config.cc b/src/config.cc
index d30d5c84c4a510a0295f1d8a4c0029a429f9a7b9..c2c028e5c8f9cabb7a0e49f59139c0fb75732666 100644 (file)
--- a/src/config.cc
+++ b/src/config.cc
@@ -496,6 +496,7 @@ static struct config_option config_optionsp[] = {
        OPTION(osd_recovery_delay_start, 0, OPT_FLOAT, 15),
        OPTION(osd_recovery_max_active, 0, OPT_INT, 5),
        OPTION(osd_recovery_max_chunk, 0, OPT_LONGLONG, 1<<20),  // max size of push chunk
+       OPTION(osd_recovery_forget_lost_objects, 0, OPT_BOOL, false),   // off for now
        OPTION(osd_auto_weight, 0, OPT_BOOL, false),
        OPTION(osd_class_error_timeout, 0, OPT_DOUBLE, 60.0),  // seconds
        OPTION(osd_class_timeout, 0, OPT_DOUBLE, 60*60.0), // seconds

diff --git a/src/config.h b/src/config.h
index 9ce53f2c6ea9b4265381d0f74269e7e762986d86..013301a9e65948e15e1c4519fc186c36287d46bd 100644 (file)
--- a/src/config.h
+++ b/src/config.h
@@ -370,6 +370,8 @@ struct md_config_t {
   int osd_recovery_max_active;
   uint64_t osd_recovery_max_chunk;
 
+  bool osd_recovery_forget_lost_objects;
+
   bool osd_auto_weight;
 
   double osd_class_error_timeout;

diff --git a/src/osd/PG.cc b/src/osd/PG.cc
index db9721a0dbaf1c5806315ffb55a8bb7ab9f272a2..218ec26db980132dc3fe9f7b70bfe72d117bee8e 100644 (file)
--- a/src/osd/PG.cc
+++ b/src/osd/PG.cc
@@ -1448,8 +1448,15 @@ void PG::check_for_lost_objects()
    * means any lost objects are also (permanently) lost.
    */
   dout(10) << " prior_set_down " << prior_set_down << dendl;
-  if (prior_set_down.empty())
+  if (prior_set_down.empty()) {
+    if (g_conf.osd_recovery_forget_lost_objects) {
+      dout(10) << " nobody in the prior_set is down; give up" << dendl;
+      forget_lost_objects();
+    } else {
+      dout(10) << " nobody in the prior_set is down, but not giving up (osd_recovery_forget_lost_objects=false)" << dendl;
+    }
     return;
+  }
   
   bool all_lost = true;
   for (set<int>::iterator q = prior_set_down.begin();
@@ -1470,102 +1477,88 @@ void PG::check_for_lost_objects()
 
   if (all_lost) {
     dout(10) << " all prior_set_down osds " << prior_set_down << " are lost" << dendl;
+    forget_lost_objects();
+    //assert(0);
+  }
+}
 
-    dout(10) << "log before:\n";
-    log.print(*_dout);
-    *_dout << dendl;
+void PG::forget_lost_objects()
+{
+  dout(10) << "forget_lost_objects" << dendl;
 
-    eversion_t oldest_lost = info.last_update;
-    int lost = 0;
-    
-    map<sobject_t,Missing::item>::iterator p = missing.missing.begin();
-    while (p != missing.missing.end()) {
-      if (missing_loc.count(p->first) == 0) {
-       dout(10) << "    " << p->first << " " << p->second.need << " is permanently lost" << dendl;
-       
-       eversion_t v = p->second.need;
+  dout(30) << "log before:\n";
+  log.print(*_dout);
+  *_dout << dendl;
 
-       list<Log::Entry>::iterator lp = log.find_entry(v);
-       eversion_t prior = lp->prior_version;
-       dout(10) << " log entry is " << *lp << dendl;
+  eversion_t oldest_lost = info.last_update;
 
-       if (prior != eversion_t() &&
-           prior != p->second.have) {
-         dout(10) << " prior version is " << prior << ", but we only have " << p->second.have << dendl;
-         p++;
-         continue;
-       }
+  eversion_t v = info.last_update;
+  v.epoch = osd->osdmap->get_epoch();
+  v.version++;
+
+  utime_t mtime = g_clock.now();
+    
+  map<sobject_t,Missing::item>::iterator p = missing.missing.begin();
+  while (p != missing.missing.end()) {
+    if (missing_loc.count(p->first) == 0) {
+     
+      Log::Entry e(Log::Entry::LOST, p->first, v, p->second.need, osd_reqid_t(), mtime);
+      
+      if (p->second.have == eversion_t()) {
+       dout(10) << "    " << p->first << " " << p->second.need << " is permanently lost" << dendl;
 
        stringstream ss;
        ss << "lost object " << p->first << " " << v << " in " << info.pgid;
        osd->logclient.log(LOG_ERROR, ss);
 
-       // remove from our log
-       eversion_t prev_entry = log.tail;
-       if (lp != log.log.begin()) {
-         lp--;
-         prev_entry = lp->version;
-         lp++;
-       }
-       dout(10) << " prior object version " << prior << ", prev_entry " << prev_entry << dendl;
-
-       if (v == info.last_update)
-         log.head = info.last_update = prev_entry;
-       if (v == info.last_complete)
-         info.last_complete = prev_entry;
-       log.log.erase(lp);
-       lost++;
-
-       if (prev_entry < oldest_lost)
-         oldest_lost = prev_entry;
+       e.op = Log::Entry::LOST;
 
        // remove from peer_missing
        for (map<int,Missing>::iterator q = peer_missing.begin();
             q != peer_missing.end();
             q++) {
-         if (q->second.missing.count(p->first) &&
-             q->second.missing[p->first].need == v) {
-           if (q->second.missing[p->first].have == prior) {
-             dout(10) << "  peer osd" << q->first << " no longer missing at all" << dendl;
-             q->second.missing.erase(p->first);
-             q->second.rmissing.erase(v);
-           } else {
-             dout(10) << "  peer osd" << q->first << " now missing prior " << prior << dendl;
-             q->second.missing[p->first].need = prior;
-             q->second.missing[p->first].have = eversion_t(); // a lie!
-           }
+         if (q->second.missing.count(p->first)) {
+           dout(10) << "  peer osd" << q->first << " no longer missing " << p->first << " at all" << dendl;
+           q->second.rm(p->first, v);
          }
-         if (peer_info[q->first].last_update == v)
-           peer_info[q->first].last_update = prev_entry;
-         if (peer_info[q->first].last_complete == v)
-           peer_info[q->first].last_complete = prev_entry;
-
-         dout(10) << " osd" << q->first << " info now " << peer_info[q->first] << dendl;
-         dout(10) << " osd" << q->first << " missing now " << peer_missing[q->first] << dendl;
-         dout(20) << " osd" << q->first << " missing now " << peer_missing[q->first].missing << dendl;
        }
+
+       // remove from my missing
+       missing.rmissing.erase(p->second.need);
+       missing.missing.erase(p++);
+      } else {
+       dout(10) << "    " << p->first << " " << p->second.need << " is permanently lost, reverting to "
+                << p->second.have << dendl;
+
+       stringstream ss;
+       ss << "lost object " << p->first << " " << p->second.need
+          << " reverted to " << p->second.have << " in " << info.pgid;
+       osd->logclient.log(LOG_ERROR, ss);
        
+       e.op = Log::Entry::LOST_REVERT;
+
+       assert(0 == "not implemented yet");  // FIXME ****
+
        // remove from my missing
-       missing.rmissing.erase(v);
+       missing.rmissing.erase(p->second.need);
        missing.missing.erase(p++);
-       continue;
       }
-      p++;
-    }
 
-    if (lost) {
-      dout(10) << lost << " objects lost, reindexing log" << dendl;
-      log.index();
-    }
+      log.add(e);
+      info.last_update = v;
+      v.version++;
 
-    dout(10) << "log after:\n";
-    log.print(*_dout);
-    *_dout << dendl;
-    dout(10) << "missing now " << missing << dendl;
-    dout(20) << "missing now " << missing.missing << dendl;
+      dout(10) << "  " << e << dendl;
 
-    //assert(0);
+    } else
+      p++;
   }
+  
+  dout(30) << "log after:\n";
+  log.print(*_dout);
+  *_dout << dendl;
+  dout(10) << "missing now " << missing << dendl;
+  dout(20) << "missing now " << missing.missing << dendl;
 }
 
 void PG::activate(ObjectStore::Transaction& t, list<Context*>& tfin,

diff --git a/src/osd/PG.h b/src/osd/PG.h
index 9268aa92721128a7f685c04db960754793c7d529..80c3ffc771fb8bf6d660c9705dd7314db6e7fd8b 100644 (file)
--- a/src/osd/PG.h
+++ b/src/osd/PG.h
@@ -238,11 +238,12 @@ public:
     /** Entry
      */
     struct Entry {
-      const static int LOST = 0;
+      const static int LOST = 0;        // lost new version, now deleted
       const static int MODIFY = 1;
       const static int CLONE = 2;  
       const static int DELETE = 3;
       const static int BACKLOG = 4;  // event invented by generate_backlog
+      const static int LOST_REVERT = 5; // lost new version, reverted to old
 
       __s32      op;   // write, zero, trunc, remove
       sobject_t  soid;
@@ -264,10 +265,12 @@ public:
       bool is_clone() const { return op == CLONE; }
       bool is_modify() const { return op == MODIFY; }
       bool is_backlog() const { return op == BACKLOG; }
-      bool is_update() const { return is_clone() || is_modify() || is_backlog(); }
+      bool is_lost() const { return op == LOST; }
+      bool is_lost_revert() const { return op == LOST_REVERT; }
+      bool is_update() const { return is_clone() || is_modify() || is_backlog() || is_lost_revert(); }
 
       bool reqid_is_indexed() const {
-       return reqid != osd_reqid_t() && op != BACKLOG && op != CLONE;
+       return reqid != osd_reqid_t() && op != BACKLOG && op != CLONE && op != LOST && op != LOST_REVERT;
       }
 
       void encode(bufferlist &bl) const {
@@ -855,6 +858,7 @@ public:
   void search_for_missing(Log &olog, Missing &omissing, int fromosd);
 
   void check_for_lost_objects();
+  void forget_lost_objects();
   
   bool build_backlog_map(map<eversion_t,Log::Entry>& omap);
   void assemble_backlog(map<eversion_t,Log::Entry>& omap);
@@ -1052,7 +1056,9 @@ inline ostream& operator<<(ostream& out, const PG::Log::Entry& e)
                 (e.is_clone() ? " c ":
                  (e.is_modify() ? " m ":
                   (e.is_backlog() ? " b ":
-                   " ? "))))
+                   (e.is_lost() ? " L ":
+                    (e.is_lost_revert() ? " R " :
+                     " ? "))))))
              << e.soid << " by " << e.reqid << " " << e.mtime;
 }