osd/PG: MissingLoc: track location counts by upset and other

For every object in missing_loc, calculate how many known locations are in
the upset and all others, and call that loc_count_t.  Keep
a map of loc_count_t to the number of missing objects with that
distribution.

Signed-off-by: Sage Weil <sage@redhat.com>
Signed-off-by: David Zafman <dzafman@redhat.com>

diff --git a/src/osd/PG.cc b/src/osd/PG.cc
index ee65daf2a3ae7d98db40955489100119813eaf5a..f6a6292c7f088e8cda8d303f3dfbaa1a8c9f7ff4 100644 (file)
--- a/src/osd/PG.cc
+++ b/src/osd/PG.cc
@@ -588,6 +588,9 @@ bool PG::search_for_missing(
   return found_missing;
 }
 
+
+// MissingLoc
+
 bool PG::MissingLoc::readable_with_acting(
   const hobject_t &hoid,
   const set<pg_shard_t> &acting) const {
@@ -626,7 +629,16 @@ void PG::MissingLoc::add_batch_sources_info(
     }
     if (i->second.is_delete())
       continue;
+
+    auto p = missing_loc.find(i->first);
+    if (p == missing_loc.end()) {
+      p = missing_loc.emplace(i->first, set<pg_shard_t>()).first;
+    } else {
+      _dec_count(p->second);
+    }
     missing_loc[i->first].insert(sources.begin(), sources.end());
+    _inc_count(p->second);
+
     if (!sources_updated) {
       missing_loc_sources.insert(sources.begin(), sources.end());
       sources_updated = true;
@@ -695,7 +707,17 @@ bool PG::MissingLoc::add_source_info(
     ldout(pg->cct, 10) << "search_for_missing " << soid << " " << need
                       << " is on osd." << fromosd << dendl;
 
-    missing_loc[soid].insert(fromosd);
+    {
+      auto p = missing_loc.find(soid);
+      if (p == missing_loc.end()) {
+       p = missing_loc.emplace(soid, set<pg_shard_t>()).first;
+      } else {
+       _dec_count(p->second);
+      }
+      p->second.insert(fromosd);
+      _inc_count(p->second);
+    }
+
     if (!sources_updated) {
       missing_loc_sources.insert(fromosd);
       sources_updated = true;
@@ -708,6 +730,55 @@ bool PG::MissingLoc::add_source_info(
   return found_missing;
 }
 
+void PG::MissingLoc::check_recovery_sources(const OSDMapRef& osdmap)
+{
+  set<pg_shard_t> now_down;
+  for (set<pg_shard_t>::iterator p = missing_loc_sources.begin();
+       p != missing_loc_sources.end();
+       ) {
+    if (osdmap->is_up(p->osd)) {
+      ++p;
+      continue;
+    }
+    ldout(pg->cct, 10) << __func__ << " source osd." << *p << " now down" << dendl;
+    now_down.insert(*p);
+    missing_loc_sources.erase(p++);
+  }
+
+  if (now_down.empty()) {
+    ldout(pg->cct, 10) << __func__ << " no source osds (" << missing_loc_sources << ") went down" << dendl;
+  } else {
+    ldout(pg->cct, 10) << __func__ << " sources osds " << now_down << " now down, remaining sources are "
+                      << missing_loc_sources << dendl;
+    
+    // filter missing_loc
+    map<hobject_t, set<pg_shard_t>>::iterator p = missing_loc.begin();
+    while (p != missing_loc.end()) {
+      set<pg_shard_t>::iterator q = p->second.begin();
+      bool changed = false;
+      while (q != p->second.end()) {
+       if (now_down.count(*q)) {
+         if (!changed) {
+           changed = true;
+           _dec_count(p->second);
+         }
+         p->second.erase(q++);
+       } else {
+         ++q;
+       }
+      }
+      if (p->second.empty()) {
+       missing_loc.erase(p++);
+      } else {
+       if (changed) {
+         _inc_count(p->second);
+       }
+       ++p;
+      }
+    }
+  }
+}
+  
 void PG::discover_all_missing(map<int, map<spg_t,pg_query_t> > &query_map)
 {
   auto &missing = pg_log.get_missing();
@@ -1838,6 +1909,7 @@ void PG::activate(ObjectStore::Transaction& t,
          complete_shards.insert(*i);
       }
     }
+
     // If necessary, create might_have_unfound to help us find our unfound objects.
     // NOTE: It's important that we build might_have_unfound before trimming the
     // past intervals.
@@ -5808,6 +5880,9 @@ ostream& operator<<(ostream& out, const PG& pg)
        out << " u=" << unfound;
     }
   }
+  if (!pg.is_clean()) {
+    out << " mbc=" << pg.missing_loc.get_missing_by_count();
+  }
   if (!pg.snap_trimq.empty()) {
     out << " trimq=";
     // only show a count if the set is large

diff --git a/src/osd/PG.h b/src/osd/PG.h
index a167e75bbe0f66d7bc1951f5802d162683c86012..ca89dcff038a545edc7a686811bd63507afbb0a6 100644 (file)
--- a/src/osd/PG.h
+++ b/src/osd/PG.h
@@ -24,6 +24,7 @@
 #include <boost/statechart/event_base.hpp>
 #include <boost/scoped_ptr.hpp>
 #include <boost/circular_buffer.hpp>
+#include <boost/container/flat_set.hpp>
 #include "include/memory.h"
 #include "include/mempool.h"
 
@@ -602,17 +603,72 @@ protected:
   }
   ghobject_t    pgmeta_oid;
 
+  // ------------------
+  // MissingLoc
+  
   class MissingLoc {
+  public:
+    // a loc_count indicates how many locations we know in each of
+    // these distinct sets
+    struct loc_count_t {
+      int up = 0;        //< up
+      int other = 0;    //< other
+
+      friend bool operator<(const loc_count_t& l,
+                           const loc_count_t& r) {
+       return (l.up < r.up ||
+               (l.up == r.up &&
+                  (l.other < r.other)));
+      }
+      friend ostream& operator<<(ostream& out, const loc_count_t& l) {
+       assert(l.up >= 0);
+       assert(l.other >= 0);
+       return out << "(" << l.up << "+" << l.other << ")";
+      }
+    };
+
+
+  private:
+
+    loc_count_t _get_count(const set<pg_shard_t>& shards) {
+      loc_count_t r;
+      for (auto s : shards) {
+        if (pg->upset.count(s)) {
+         r.up++;
+       } else {
+         r.other++;
+       }
+      }
+      return r;
+    }
+
     map<hobject_t, pg_missing_item> needs_recovery_map;
     map<hobject_t, set<pg_shard_t> > missing_loc;
     set<pg_shard_t> missing_loc_sources;
+
+    // for every entry in missing_loc, we count how many of each type of shard we have,
+    // and maintain totals here.  The sum of the values for this map will always equal
+    // missing_loc.size().
+    map<loc_count_t,int> missing_by_count;
+
+    void _inc_count(const set<pg_shard_t>& s) {
+      ++missing_by_count[_get_count(s)];
+    }
+    void _dec_count(const set<pg_shard_t>& s) {
+      auto p = missing_by_count.find(_get_count(s));
+      assert(p != missing_by_count.end());
+      if (--p->second == 0) {
+       missing_by_count.erase(p);
+      }
+    }
+
     PG *pg;
     set<pg_shard_t> empty_set;
   public:
     boost::scoped_ptr<IsPGReadablePredicate> is_readable;
     boost::scoped_ptr<IsPGRecoverablePredicate> is_recoverable;
     explicit MissingLoc(PG *pg)
-      : pg(pg) {}
+      : pg(pg) { }
     void set_backend_predicates(
       IsPGReadablePredicate *_is_readable,
       IsPGRecoverablePredicate *_is_recoverable) {
@@ -683,13 +739,26 @@ protected:
       needs_recovery_map.clear();
       missing_loc.clear();
       missing_loc_sources.clear();
+      missing_by_count.clear();
     }
 
     void add_location(const hobject_t &hoid, pg_shard_t location) {
-      missing_loc[hoid].insert(location);
+      auto p = missing_loc.find(hoid);
+      if (p == missing_loc.end()) {
+       p = missing_loc.emplace(hoid, set<pg_shard_t>()).first;
+      } else {
+       _dec_count(p->second);
+      }
+      p->second.insert(location);
+      _inc_count(p->second);
     }
     void remove_location(const hobject_t &hoid, pg_shard_t location) {
-      missing_loc[hoid].erase(location);
+      auto p = missing_loc.find(hoid);
+      if (p != missing_loc.end()) {
+       _dec_count(p->second);
+       p->second.erase(location);
+       _inc_count(p->second);
+      }
     }
     void add_active_missing(const pg_missing_t &missing) {
       for (map<hobject_t, pg_missing_item>::const_iterator i =
@@ -738,7 +807,11 @@ protected:
     /// Call when hoid is no longer missing in acting set
     void recovered(const hobject_t &hoid) {
       needs_recovery_map.erase(hoid);
-      missing_loc.erase(hoid);
+      auto p = missing_loc.find(hoid);
+      if (p != missing_loc.end()) {
+       _dec_count(p->second);
+       missing_loc.erase(p);
+      }
     }
 
     /// Call to update structures for hoid after a change
@@ -788,6 +861,7 @@ protected:
            !i.second.is_missing(hoid))
          mliter->second.insert(i.first);
       }
+      _inc_count(mliter->second);
     }
 
     const set<pg_shard_t> &get_locations(const hobject_t &hoid) const {
@@ -800,6 +874,9 @@ protected:
     const map<hobject_t, pg_missing_item> &get_needs_recovery() const {
       return needs_recovery_map;
     }
+    const map<loc_count_t,int> &get_missing_by_count() const {
+      return missing_by_count;
+    }
   } missing_loc;
   
   PastIntervals past_intervals;

diff --git a/src/osd/PrimaryLogPG.cc b/src/osd/PrimaryLogPG.cc
index 874f61c0bfd52bdfef172916339393f10369b03d..44f0f0a1a02e5ecb30ab2b13663533b8ed205856 100644 (file)
--- a/src/osd/PrimaryLogPG.cc
+++ b/src/osd/PrimaryLogPG.cc
@@ -11829,46 +11829,6 @@ void PrimaryLogPG::check_recovery_sources(const OSDMapRef& osdmap)
   }
 }
 
-void PG::MissingLoc::check_recovery_sources(const OSDMapRef& osdmap)
-{
-  set<pg_shard_t> now_down;
-  for (set<pg_shard_t>::iterator p = missing_loc_sources.begin();
-       p != missing_loc_sources.end();
-       ) {
-    if (osdmap->is_up(p->osd)) {
-      ++p;
-      continue;
-    }
-    ldout(pg->cct, 10) << __func__ << " source osd." << *p << " now down" << dendl;
-    now_down.insert(*p);
-    missing_loc_sources.erase(p++);
-  }
-
-  if (now_down.empty()) {
-    ldout(pg->cct, 10) << __func__ << " no source osds (" << missing_loc_sources << ") went down" << dendl;
-  } else {
-    ldout(pg->cct, 10) << __func__ << " sources osds " << now_down << " now down, remaining sources are "
-                      << missing_loc_sources << dendl;
-    
-    // filter missing_loc
-    map<hobject_t, set<pg_shard_t>>::iterator p = missing_loc.begin();
-    while (p != missing_loc.end()) {
-      set<pg_shard_t>::iterator q = p->second.begin();
-      while (q != p->second.end())
-       if (now_down.count(*q)) {
-         p->second.erase(q++);
-       } else {
-         ++q;
-       }
-      if (p->second.empty())
-       missing_loc.erase(p++);
-      else
-       ++p;
-    }
-  }
-}
-  
-
 bool PrimaryLogPG::start_recovery_ops(
   uint64_t max,
   ThreadPool::TPHandle &handle,