From: Sage Weil Date: Mon, 14 Sep 2015 21:04:23 +0000 (-0400) Subject: mon/PGMonitor: avoid iterating over all pgs to find stale X-Git-Tag: v10.0.1~26^2~65 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=28138c65a6b02ba0dd0e65105303870bb8d2b86e;p=ceph.git mon/PGMonitor: avoid iterating over all pgs to find stale Instead of iterating over all pgs when an osd goes down, make a set of all osds that might have gone down, and only check pgs that it manages. This is more efficient, especially for large clusters with large numbers of OSDs. Signed-off-by: Sage Weil --- diff --git a/src/common/config_opts.h b/src/common/config_opts.h index dbad270a3b9..4eda30098f5 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -230,6 +230,7 @@ OPTION(mon_pg_warn_max_per_osd, OPT_INT, 300) // max # pgs per (in) osd before OPTION(mon_pg_warn_max_object_skew, OPT_FLOAT, 10.0) // max skew few average in objects per pg OPTION(mon_pg_warn_min_objects, OPT_INT, 10000) // do not warn below this object # OPTION(mon_pg_warn_min_pool_objects, OPT_INT, 1000) // do not warn on pools below this object # +OPTION(mon_pg_check_down_all_threshold, OPT_FLOAT, .5) // threshold of down osds after which we check all pgs OPTION(mon_cache_target_full_warn_ratio, OPT_FLOAT, .66) // position between pool cache_target_full and max where we start warning OPTION(mon_osd_full_ratio, OPT_FLOAT, .95) // what % full makes an OSD "full" OPTION(mon_osd_nearfull_ratio, OPT_FLOAT, .85) // what % full makes an OSD near full diff --git a/src/mon/PGMonitor.cc b/src/mon/PGMonitor.cc index 77a5af150fb..53df948af93 100644 --- a/src/mon/PGMonitor.cc +++ b/src/mon/PGMonitor.cc @@ -128,7 +128,8 @@ void PGMonitor::tick() if (mon->is_leader()) { bool propose = false; - if (need_check_down_pgs && check_down_pgs()) + if ((need_check_down_pgs || !need_check_down_pg_osds.empty()) && + check_down_pgs()) propose = true; if (propose) { @@ -914,7 +915,7 @@ void PGMonitor::check_osd_map(epoch_t epoch) p != inc.new_state.end(); ++p) { if (p->second & CEPH_OSD_UP) { // true if marked up OR down, but we're too lazy to check which - need_check_down_pgs = true; + need_check_down_pg_osds.insert(p->first); // clear out the last_osd_report for this OSD map::iterator report = last_osd_report.find(p->first); @@ -951,7 +952,8 @@ void PGMonitor::check_osd_map(epoch_t epoch) if (register_new_pgs()) propose = true; - if (need_check_down_pgs && check_down_pgs()) + if ((need_check_down_pgs || !need_check_down_pg_osds.empty()) && + check_down_pgs()) propose = true; if (propose) @@ -1213,6 +1215,21 @@ void PGMonitor::send_pg_creates(int osd, Connection *con) last_sent_pg_create[osd] = ceph_clock_now(g_ceph_context); } +void PGMonitor::_mark_pg_stale(pg_t pgid, const pg_stat_t& cur_stat) +{ + dout(10) << " marking pg " << pgid << " stale" << dendl; + map::iterator q = pending_inc.pg_stat_updates.find(pgid); + pg_stat_t *stat; + if (q == pending_inc.pg_stat_updates.end()) { + stat = &pending_inc.pg_stat_updates[pgid]; + *stat = cur_stat; + } else { + stat = &q->second; + } + stat->state |= PG_STATE_STALE; + stat->last_unstale = ceph_clock_now(g_ceph_context); +} + bool PGMonitor::check_down_pgs() { dout(10) << "check_down_pgs" << dendl; @@ -1220,28 +1237,37 @@ bool PGMonitor::check_down_pgs() OSDMap *osdmap = &mon->osdmon()->osdmap; bool ret = false; - for (ceph::unordered_map::iterator p = pg_map.pg_stat.begin(); - p != pg_map.pg_stat.end(); - ++p) { - if ((p->second.state & PG_STATE_STALE) == 0 && - p->second.acting_primary != -1 && - osdmap->is_down(p->second.acting_primary)) { - dout(10) << " marking pg " << p->first << " stale with acting " << p->second.acting << dendl; - - map::iterator q = pending_inc.pg_stat_updates.find(p->first); - pg_stat_t *stat; - if (q == pending_inc.pg_stat_updates.end()) { - stat = &pending_inc.pg_stat_updates[p->first]; - *stat = p->second; - } else { - stat = &q->second; + // if a large number of osds changed state, just iterate over the whole + // pg map. + if (need_check_down_pg_osds.size() > (unsigned)osdmap->get_num_osds() * + g_conf->mon_pg_check_down_all_threshold) + need_check_down_pgs = true; + + if (need_check_down_pgs) { + for (auto p : pg_map.pg_stat) { + if ((p.second.state & PG_STATE_STALE) == 0 && + p.second.acting_primary != -1 && + osdmap->is_down(p.second.acting_primary)) { + _mark_pg_stale(p.first, p.second); + ret = true; + } + } + } else { + for (auto osd : need_check_down_pg_osds) { + if (osdmap->is_down(osd)) { + for (auto pgid : pg_map.pg_by_osd[osd]) { + const pg_stat_t &stat = pg_map.pg_stat[pgid]; + if ((stat.state & PG_STATE_STALE) == 0 && + stat.acting_primary != -1) { + _mark_pg_stale(pgid, stat); + ret = true; + } + } } - stat->state |= PG_STATE_STALE; - stat->last_unstale = ceph_clock_now(g_ceph_context); - ret = true; } } need_check_down_pgs = false; + need_check_down_pg_osds.clear(); return ret; } diff --git a/src/mon/PGMonitor.h b/src/mon/PGMonitor.h index cb725a67a03..96919b49e3a 100644 --- a/src/mon/PGMonitor.h +++ b/src/mon/PGMonitor.h @@ -47,6 +47,7 @@ public: PGMap pg_map; bool need_check_down_pgs; + set need_check_down_pg_osds; epoch_t last_map_pg_create_osd_epoch; @@ -131,10 +132,13 @@ private: * check pgs for down primary osds * * clears need_check_down_pgs + * clears need_check_down_pg_osds * * @return true if we updated pending_inc (and should propose) */ bool check_down_pgs(); + void _mark_pg_stale(pg_t pgid, const pg_stat_t& cur_stat); + /** * Dump stats from pgs stuck in specified states.