From: Sage Weil <sage@inktank.com>
Date: Wed, 5 Mar 2014 18:58:37 +0000 (-0800)
Subject: mon: warn when pool nears target max objects/bytes
X-Git-Tag: v0.78~83^2
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=refs%2Fpull%2F1343%2Fhead;p=ceph.git

mon: warn when pool nears target max objects/bytes

The cache pools will throttle when they reach the target max size, so it
is important to make the administrator aware when they approach that point.
Unfortunately it is not particularly easy to efficiently keep track of
which PGs have hit their limit and use that for reporting.  However, it
is easy to raise a flag when we start to approach the target for the
entire pool, and that sort of early warning is arguably more useful
anyway.

Trigger the warning based on the target full ratio.  Not when we hit the
target, but when we are 2/3 between it and completely full.

Implements: #7442
Signed-off-by: Sage Weil <sage@inktank.com>
---

diff --git a/qa/workunits/cephtool/test.sh b/qa/workunits/cephtool/test.sh
index 51e7ab4dbbe8..e4efe97c27cb 100755
--- a/qa/workunits/cephtool/test.sh
+++ b/qa/workunits/cephtool/test.sh
@@ -95,6 +95,22 @@ ceph osd dump | grep cache3 | grep bloom | grep 'false_positive_probability: 0.0
 ceph osd tier remove data cache3
 ceph osd pool delete cache3 cache3 --yes-i-really-really-mean-it
 
+# check health check
+ceph osd pool create cache4 2
+ceph osd pool set cache4 target_max_objects 5
+ceph osd pool set cache4 target_max_bytes 1000
+for f in `seq 1 5` ; do
+    rados -p cache4 put foo$f /etc/passwd
+done
+while ! ceph df | grep cache4 | grep ' 5 ' ; do
+    echo waiting for pg stats to flush
+    sleep 2
+done
+ceph health | grep WARN | grep cache4
+ceph health detail | grep cache4 | grep 'target max' | grep objects
+ceph health detail | grep cache4 | grep 'target max' | grep 'B'
+ceph osd pool delete cache4 cache4 --yes-i-really-really-mean-it
+
 # Assumes there are at least 3 MDSes and two OSDs
 #
 
diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index 8bbd58671e04..fad7204a85ba 100644
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -166,6 +166,7 @@ OPTION(mon_pg_warn_min_per_osd, OPT_INT, 20)  // min # pgs per (in) osd before w
 OPTION(mon_pg_warn_max_object_skew, OPT_FLOAT, 10.0) // max skew few average in objects per pg
 OPTION(mon_pg_warn_min_objects, OPT_INT, 10000)  // do not warn below this object #
 OPTION(mon_pg_warn_min_pool_objects, OPT_INT, 1000)  // do not warn on pools below this object #
+OPTION(mon_cache_target_full_warn_ratio, OPT_FLOAT, .66) // position between pool cache_target_full and max where we start warning
 OPTION(mon_osd_full_ratio, OPT_FLOAT, .95) // what % full makes an OSD "full"
 OPTION(mon_osd_nearfull_ratio, OPT_FLOAT, .85) // what % full makes an OSD near full
 OPTION(mon_globalid_prealloc, OPT_INT, 100)   // how many globalids to prealloc
diff --git a/src/mon/PGMap.h b/src/mon/PGMap.h
index d76b90d355a0..f4e8df5f0ffc 100644
--- a/src/mon/PGMap.h
+++ b/src/mon/PGMap.h
@@ -205,8 +205,9 @@ public:
     stamp = s;
   }
 
-  pool_stat_t get_pg_pool_sum_stat(int64_t pool) {
-    ceph::unordered_map<int,pool_stat_t>::iterator p = pg_pool_sum.find(pool);
+  pool_stat_t get_pg_pool_sum_stat(int64_t pool) const {
+    ceph::unordered_map<int,pool_stat_t>::const_iterator p =
+      pg_pool_sum.find(pool);
     if (p != pg_pool_sum.end())
       return p->second;
     return pool_stat_t();
diff --git a/src/mon/PGMonitor.cc b/src/mon/PGMonitor.cc
index 1c64cdc8ec5a..c93e8557f869 100644
--- a/src/mon/PGMonitor.cc
+++ b/src/mon/PGMonitor.cc
@@ -1889,6 +1889,50 @@ void PGMonitor::get_health(list<pair<health_status_t,string> >& summary,
   check_full_osd_health(summary, detail, pg_map.full_osds, "full", HEALTH_ERR);
   check_full_osd_health(summary, detail, pg_map.nearfull_osds, "near full", HEALTH_WARN);
 
+  // near-target max pools
+  const map<int64_t,pg_pool_t>& pools = mon->osdmon()->osdmap.get_pools();
+  for (map<int64_t,pg_pool_t>::const_iterator p = pools.begin();
+       p != pools.end(); ++p) {
+    if ((!p->second.target_max_objects && !p->second.target_max_bytes) ||
+	!pg_map.pg_pool_sum.count(p->first))
+      continue;
+    bool nearfull = false;
+    const char *name = mon->osdmon()->osdmap.get_pool_name(p->first);
+    const pool_stat_t& st = pg_map.get_pg_pool_sum_stat(p->first);
+    uint64_t ratio = p->second.cache_target_full_ratio_micro +
+      ((1000000 - p->second.cache_target_full_ratio_micro) *
+       g_conf->mon_cache_target_full_warn_ratio);
+    if (p->second.target_max_objects && (uint64_t)st.stats.sum.num_objects >
+	p->second.target_max_objects * ratio / 1000000) {
+      nearfull = true;
+      if (detail) {
+	ostringstream ss;
+	ss << "cache pool '" << name << "' with "
+	   << si_t(st.stats.sum.num_objects)
+	   << " objects at/near target max "
+	   << si_t(p->second.target_max_objects) << " objects";
+	detail->push_back(make_pair(HEALTH_WARN, ss.str()));
+      }
+    }
+    if (p->second.target_max_bytes && (uint64_t)st.stats.sum.num_bytes >
+	p->second.target_max_bytes * ratio / 1000000) {
+      nearfull = true;
+      if (detail) {
+	ostringstream ss;
+	ss << "cache pool '" << mon->osdmon()->osdmap.get_pool_name(p->first)
+	   << "' with " << si_t(st.stats.sum.num_bytes)
+	   << "B at/near target max "
+	   << si_t(p->second.target_max_bytes) << "B";
+	detail->push_back(make_pair(HEALTH_WARN, ss.str()));
+      }
+    }
+    if (nearfull) {
+      ostringstream ss;
+      ss << "'" << name << "' at/near target max";
+      summary.push_back(make_pair(HEALTH_WARN, ss.str()));
+    }
+  }
+
   // scrub
   if (pg_map.pg_sum.stats.sum.num_scrub_errors) {
     ostringstream ss;