From: Sage Weil Date: Wed, 5 Mar 2014 18:58:37 +0000 (-0800) Subject: mon: warn when pool nears target max objects/bytes X-Git-Tag: v0.78~83^2 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=0592368070835d02d5ebeb73d31161edfd208d74;p=ceph.git mon: warn when pool nears target max objects/bytes The cache pools will throttle when they reach the target max size, so it is important to make the administrator aware when they approach that point. Unfortunately it is not particularly easy to efficiently keep track of which PGs have hit their limit and use that for reporting. However, it is easy to raise a flag when we start to approach the target for the entire pool, and that sort of early warning is arguably more useful anyway. Trigger the warning based on the target full ratio. Not when we hit the target, but when we are 2/3 between it and completely full. Implements: #7442 Signed-off-by: Sage Weil --- diff --git a/qa/workunits/cephtool/test.sh b/qa/workunits/cephtool/test.sh index 51e7ab4dbbe81..e4efe97c27cb3 100755 --- a/qa/workunits/cephtool/test.sh +++ b/qa/workunits/cephtool/test.sh @@ -95,6 +95,22 @@ ceph osd dump | grep cache3 | grep bloom | grep 'false_positive_probability: 0.0 ceph osd tier remove data cache3 ceph osd pool delete cache3 cache3 --yes-i-really-really-mean-it +# check health check +ceph osd pool create cache4 2 +ceph osd pool set cache4 target_max_objects 5 +ceph osd pool set cache4 target_max_bytes 1000 +for f in `seq 1 5` ; do + rados -p cache4 put foo$f /etc/passwd +done +while ! ceph df | grep cache4 | grep ' 5 ' ; do + echo waiting for pg stats to flush + sleep 2 +done +ceph health | grep WARN | grep cache4 +ceph health detail | grep cache4 | grep 'target max' | grep objects +ceph health detail | grep cache4 | grep 'target max' | grep 'B' +ceph osd pool delete cache4 cache4 --yes-i-really-really-mean-it + # Assumes there are at least 3 MDSes and two OSDs # diff --git a/src/common/config_opts.h b/src/common/config_opts.h index 8bbd58671e04c..fad7204a85ba0 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -166,6 +166,7 @@ OPTION(mon_pg_warn_min_per_osd, OPT_INT, 20) // min # pgs per (in) osd before w OPTION(mon_pg_warn_max_object_skew, OPT_FLOAT, 10.0) // max skew few average in objects per pg OPTION(mon_pg_warn_min_objects, OPT_INT, 10000) // do not warn below this object # OPTION(mon_pg_warn_min_pool_objects, OPT_INT, 1000) // do not warn on pools below this object # +OPTION(mon_cache_target_full_warn_ratio, OPT_FLOAT, .66) // position between pool cache_target_full and max where we start warning OPTION(mon_osd_full_ratio, OPT_FLOAT, .95) // what % full makes an OSD "full" OPTION(mon_osd_nearfull_ratio, OPT_FLOAT, .85) // what % full makes an OSD near full OPTION(mon_globalid_prealloc, OPT_INT, 100) // how many globalids to prealloc diff --git a/src/mon/PGMap.h b/src/mon/PGMap.h index d76b90d355a09..f4e8df5f0ffca 100644 --- a/src/mon/PGMap.h +++ b/src/mon/PGMap.h @@ -205,8 +205,9 @@ public: stamp = s; } - pool_stat_t get_pg_pool_sum_stat(int64_t pool) { - ceph::unordered_map::iterator p = pg_pool_sum.find(pool); + pool_stat_t get_pg_pool_sum_stat(int64_t pool) const { + ceph::unordered_map::const_iterator p = + pg_pool_sum.find(pool); if (p != pg_pool_sum.end()) return p->second; return pool_stat_t(); diff --git a/src/mon/PGMonitor.cc b/src/mon/PGMonitor.cc index 1c64cdc8ec5ae..c93e8557f869a 100644 --- a/src/mon/PGMonitor.cc +++ b/src/mon/PGMonitor.cc @@ -1889,6 +1889,50 @@ void PGMonitor::get_health(list >& summary, check_full_osd_health(summary, detail, pg_map.full_osds, "full", HEALTH_ERR); check_full_osd_health(summary, detail, pg_map.nearfull_osds, "near full", HEALTH_WARN); + // near-target max pools + const map& pools = mon->osdmon()->osdmap.get_pools(); + for (map::const_iterator p = pools.begin(); + p != pools.end(); ++p) { + if ((!p->second.target_max_objects && !p->second.target_max_bytes) || + !pg_map.pg_pool_sum.count(p->first)) + continue; + bool nearfull = false; + const char *name = mon->osdmon()->osdmap.get_pool_name(p->first); + const pool_stat_t& st = pg_map.get_pg_pool_sum_stat(p->first); + uint64_t ratio = p->second.cache_target_full_ratio_micro + + ((1000000 - p->second.cache_target_full_ratio_micro) * + g_conf->mon_cache_target_full_warn_ratio); + if (p->second.target_max_objects && (uint64_t)st.stats.sum.num_objects > + p->second.target_max_objects * ratio / 1000000) { + nearfull = true; + if (detail) { + ostringstream ss; + ss << "cache pool '" << name << "' with " + << si_t(st.stats.sum.num_objects) + << " objects at/near target max " + << si_t(p->second.target_max_objects) << " objects"; + detail->push_back(make_pair(HEALTH_WARN, ss.str())); + } + } + if (p->second.target_max_bytes && (uint64_t)st.stats.sum.num_bytes > + p->second.target_max_bytes * ratio / 1000000) { + nearfull = true; + if (detail) { + ostringstream ss; + ss << "cache pool '" << mon->osdmon()->osdmap.get_pool_name(p->first) + << "' with " << si_t(st.stats.sum.num_bytes) + << "B at/near target max " + << si_t(p->second.target_max_bytes) << "B"; + detail->push_back(make_pair(HEALTH_WARN, ss.str())); + } + } + if (nearfull) { + ostringstream ss; + ss << "'" << name << "' at/near target max"; + summary.push_back(make_pair(HEALTH_WARN, ss.str())); + } + } + // scrub if (pg_map.pg_sum.stats.sum.num_scrub_errors) { ostringstream ss;