OSD: tolerate holes in stored maps

We may have holes in stored maps during init_splits_between
and advance_pg.  In either case, we should simply skip the
missing maps.

Fixes: #5677
Signed-off-by: Samuel Just <sam.just@inktank.com>
Reviewed-by: Sage Weil <sage@inktank.com>
(cherry picked from commit 6951d2345a5d837c3b14103bd4d8f5ee4407c937)

Conflicts:

	src/osd/OSD.cc

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 3dbbe392bc09b45c5709a9c4498025cfb52f686c..0581f76c57812bfe7dbe2c1a7ee41a6752654e79 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -285,11 +285,13 @@ void OSDService::init_splits_between(pg_t pgid,
     // Ok, a split happened, so we need to walk the osdmaps
     set<pg_t> new_pgs; // pgs to scan on each map
     new_pgs.insert(pgid);
+    OSDMapRef curmap(get_map(frommap->get_epoch()));
     for (epoch_t e = frommap->get_epoch() + 1;
         e <= tomap->get_epoch();
         ++e) {
-      OSDMapRef curmap(get_map(e-1));
-      OSDMapRef nextmap(get_map(e));
+      OSDMapRef nextmap(try_get_map(e));
+      if (!nextmap)
+       continue;
       set<pg_t> even_newer_pgs; // pgs added in this loop
       for (set<pg_t>::iterator i = new_pgs.begin(); i != new_pgs.end(); ++i) {
        set<pg_t> split_pgs;
@@ -301,7 +303,9 @@ void OSDService::init_splits_between(pg_t pgid,
        }
       }
       new_pgs.insert(even_newer_pgs.begin(), even_newer_pgs.end());
+      curmap = nextmap;
     }
+    assert(curmap == tomap); // we must have had both frommap and tomap
   }
 }
 
@@ -4496,7 +4500,9 @@ void OSD::advance_pg(
   for (;
        next_epoch <= osd_epoch;
        ++next_epoch) {
-    OSDMapRef nextmap = get_map(next_epoch);
+    OSDMapRef nextmap = service.try_get_map(next_epoch);
+    if (!nextmap)
+      continue;
 
     vector<int> newup, newacting;
     nextmap->pg_to_up_acting_osds(pg->info.pgid, newup, newacting);
@@ -4831,7 +4837,7 @@ OSDMapRef OSDService::_add_map(OSDMap *o)
   return l;
 }
 
-OSDMapRef OSDService::get_map(epoch_t epoch)
+OSDMapRef OSDService::try_get_map(epoch_t epoch)
 {
   Mutex::Locker l(map_cache_lock);
   OSDMapRef retval = map_cache.lookup(epoch);
@@ -4844,7 +4850,10 @@ OSDMapRef OSDService::get_map(epoch_t epoch)
   if (epoch > 0) {
     dout(20) << "get_map " << epoch << " - loading and decoding " << map << dendl;
     bufferlist bl;
-    assert(_get_map_bl(epoch, bl));
+    if (!_get_map_bl(epoch, bl)) {
+      delete map;
+      return OSDMapRef();
+    }
     map->decode(bl);
   } else {
     dout(20) << "get_map " << epoch << " - return initial " << map << dendl;

diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index 2db9d3b8c44df25c1807aa3d3ca208b92aa842c1..ff2af85d227baef2bb0526361d8e974e7c87632e 100644 (file)
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -347,7 +347,12 @@ public:
   SimpleLRU<epoch_t, bufferlist> map_bl_cache;
   SimpleLRU<epoch_t, bufferlist> map_bl_inc_cache;
 
-  OSDMapRef get_map(epoch_t e);
+  OSDMapRef try_get_map(epoch_t e);
+  OSDMapRef get_map(epoch_t e) {
+    OSDMapRef ret(try_get_map(e));
+    assert(ret);
+    return ret;
+  }
   OSDMapRef add_map(OSDMap *o) {
     Mutex::Locker l(map_cache_lock);
     return _add_map(o);

diff --git a/src/osd/PG.cc b/src/osd/PG.cc
index 3b06d89be4b3279a0a71217f89c35b5c106f49f0..fa8e1a88bec7190e9d2ff3034cad4ad35878d597 100644 (file)
--- a/src/osd/PG.cc
+++ b/src/osd/PG.cc
@@ -5516,7 +5516,6 @@ void PG::handle_advance_map(OSDMapRef osdmap, OSDMapRef lastmap,
                            vector<int>& newup, vector<int>& newacting,
                            RecoveryCtx *rctx)
 {
-  assert(osdmap->get_epoch() == (lastmap->get_epoch() + 1));
   assert(lastmap->get_epoch() == osdmap_ref->get_epoch());
   assert(lastmap == osdmap_ref);
   dout(10) << "handle_advance_map " << newup << "/" << newacting << dendl;

diff --git a/src/osd/osd_types.h b/src/osd/osd_types.h
index a7a10307d1fefe5337126912830f197b6cf7b0ba..78d4b9fff5d49325739efcb24a78b6876bc405fe 100644 (file)
--- a/src/osd/osd_types.h
+++ b/src/osd/osd_types.h
@@ -1021,6 +1021,13 @@ struct pg_history_t {
   epoch_t last_epoch_clean;    // lower bound on last epoch the PG was completely clean.
   epoch_t last_epoch_split;    // as parent
   
+  /**
+   * In the event of a map discontinuity, same_*_since may reflect the first
+   * map the osd has seen in the new map sequence rather than the actual start
+   * of the interval.  This is ok since a discontinuity at epoch e means there
+   * must have been a clean interval between e and now and that we cannot be
+   * in the active set during the interval containing e.
+   */
   epoch_t same_up_since;       // same acting set since
   epoch_t same_interval_since;   // same acting AND up set since
   epoch_t same_primary_since;  // same primary at least back through this epoch.