mon/PGMonitor: avoid iterating over all pgs to find stale

Instead of iterating over all pgs when an osd goes down, make a
set of all osds that might have gone down, and only check pgs that
it manages.  This is more efficient, especially for large clusters
with large numbers of OSDs.

Signed-off-by: Sage Weil <sage@redhat.com>

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index dbad270a3b9bfeed8f3f4c190e0ddf50f9afee6b..4eda30098f5b5bb50aba6e26b76a691d83dd05df 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -230,6 +230,7 @@ OPTION(mon_pg_warn_max_per_osd, OPT_INT, 300)  // max # pgs per (in) osd before
 OPTION(mon_pg_warn_max_object_skew, OPT_FLOAT, 10.0) // max skew few average in objects per pg
 OPTION(mon_pg_warn_min_objects, OPT_INT, 10000)  // do not warn below this object #
 OPTION(mon_pg_warn_min_pool_objects, OPT_INT, 1000)  // do not warn on pools below this object #
+OPTION(mon_pg_check_down_all_threshold, OPT_FLOAT, .5) // threshold of down osds after which we check all pgs
 OPTION(mon_cache_target_full_warn_ratio, OPT_FLOAT, .66) // position between pool cache_target_full and max where we start warning
 OPTION(mon_osd_full_ratio, OPT_FLOAT, .95) // what % full makes an OSD "full"
 OPTION(mon_osd_nearfull_ratio, OPT_FLOAT, .85) // what % full makes an OSD near full

diff --git a/src/mon/PGMonitor.cc b/src/mon/PGMonitor.cc
index 77a5af150fb14911389528f1dba821d1fc7bfa1b..53df948af93dffca2a693cfeeccd61949594670c 100644 (file)
--- a/src/mon/PGMonitor.cc
+++ b/src/mon/PGMonitor.cc
@@ -128,7 +128,8 @@ void PGMonitor::tick()
   if (mon->is_leader()) {
     bool propose = false;
     
-    if (need_check_down_pgs && check_down_pgs())
+    if ((need_check_down_pgs || !need_check_down_pg_osds.empty()) &&
+       check_down_pgs())
       propose = true;
     
     if (propose) {
@@ -914,7 +915,7 @@ void PGMonitor::check_osd_map(epoch_t epoch)
         p != inc.new_state.end();
         ++p) {
       if (p->second & CEPH_OSD_UP) {   // true if marked up OR down, but we're too lazy to check which
-       need_check_down_pgs = true;
+       need_check_down_pg_osds.insert(p->first);
 
        // clear out the last_osd_report for this OSD
         map<int, utime_t>::iterator report = last_osd_report.find(p->first);
@@ -951,7 +952,8 @@ void PGMonitor::check_osd_map(epoch_t epoch)
   if (register_new_pgs())
     propose = true;
 
-  if (need_check_down_pgs && check_down_pgs())
+  if ((need_check_down_pgs || !need_check_down_pg_osds.empty()) &&
+      check_down_pgs())
     propose = true;
   
   if (propose)
@@ -1213,6 +1215,21 @@ void PGMonitor::send_pg_creates(int osd, Connection *con)
   last_sent_pg_create[osd] = ceph_clock_now(g_ceph_context);
 }
 
+void PGMonitor::_mark_pg_stale(pg_t pgid, const pg_stat_t& cur_stat)
+{
+  dout(10) << " marking pg " << pgid << " stale" << dendl;
+  map<pg_t,pg_stat_t>::iterator q = pending_inc.pg_stat_updates.find(pgid);
+  pg_stat_t *stat;
+  if (q == pending_inc.pg_stat_updates.end()) {
+    stat = &pending_inc.pg_stat_updates[pgid];
+    *stat = cur_stat;
+  } else {
+    stat = &q->second;
+  }
+  stat->state |= PG_STATE_STALE;
+  stat->last_unstale = ceph_clock_now(g_ceph_context);
+}
+
 bool PGMonitor::check_down_pgs()
 {
   dout(10) << "check_down_pgs" << dendl;
@@ -1220,28 +1237,37 @@ bool PGMonitor::check_down_pgs()
   OSDMap *osdmap = &mon->osdmon()->osdmap;
   bool ret = false;
 
-  for (ceph::unordered_map<pg_t,pg_stat_t>::iterator p = pg_map.pg_stat.begin();
-       p != pg_map.pg_stat.end();
-       ++p) {
-    if ((p->second.state & PG_STATE_STALE) == 0 &&
-       p->second.acting_primary != -1 &&
-       osdmap->is_down(p->second.acting_primary)) {
-      dout(10) << " marking pg " << p->first << " stale with acting " << p->second.acting << dendl;
-
-      map<pg_t,pg_stat_t>::iterator q = pending_inc.pg_stat_updates.find(p->first);
-      pg_stat_t *stat;
-      if (q == pending_inc.pg_stat_updates.end()) {
-       stat = &pending_inc.pg_stat_updates[p->first];
-       *stat = p->second;
-      } else {
-       stat = &q->second;
+  // if a large number of osds changed state, just iterate over the whole
+  // pg map.
+  if (need_check_down_pg_osds.size() > (unsigned)osdmap->get_num_osds() *
+      g_conf->mon_pg_check_down_all_threshold)
+    need_check_down_pgs = true;
+
+  if (need_check_down_pgs) {
+    for (auto p : pg_map.pg_stat) {
+      if ((p.second.state & PG_STATE_STALE) == 0 &&
+         p.second.acting_primary != -1 &&
+         osdmap->is_down(p.second.acting_primary)) {
+       _mark_pg_stale(p.first, p.second);
+       ret = true;
+      }
+    }
+  } else {
+    for (auto osd : need_check_down_pg_osds) {
+      if (osdmap->is_down(osd)) {
+       for (auto pgid : pg_map.pg_by_osd[osd]) {
+         const pg_stat_t &stat = pg_map.pg_stat[pgid];
+         if ((stat.state & PG_STATE_STALE) == 0 &&
+             stat.acting_primary != -1) {
+           _mark_pg_stale(pgid, stat);
+           ret = true;
+         }
+       }
       }
-      stat->state |= PG_STATE_STALE;
-      stat->last_unstale = ceph_clock_now(g_ceph_context);
-      ret = true;
     }
   }
   need_check_down_pgs = false;
+  need_check_down_pg_osds.clear();
 
   return ret;
 }

diff --git a/src/mon/PGMonitor.h b/src/mon/PGMonitor.h
index cb725a67a03c5928885e0525792800dad05a060c..96919b49e3adff2e4a92d0e731e8bf77b59f2ca9 100644 (file)
--- a/src/mon/PGMonitor.h
+++ b/src/mon/PGMonitor.h
@@ -47,6 +47,7 @@ public:
   PGMap pg_map;
 
   bool need_check_down_pgs;
+  set<int> need_check_down_pg_osds;
 
   epoch_t last_map_pg_create_osd_epoch;
 
@@ -131,10 +132,13 @@ private:
    * check pgs for down primary osds
    *
    * clears need_check_down_pgs
+   * clears need_check_down_pg_osds
    *
    * @return true if we updated pending_inc (and should propose)
    */
   bool check_down_pgs();
+  void _mark_pg_stale(pg_t pgid, const pg_stat_t& cur_stat);
+
 
   /**
    * Dump stats from pgs stuck in specified states.