mon: OSDMonitor: add 'osd pool set-quota' command

Signed-off-by: Joao Eduardo Luis <joao.luis@inktank.com>

diff --git a/qa/workunits/rados/test_pool_quota.sh b/qa/workunits/rados/test_pool_quota.sh
new file mode 100755 (executable)
index 0000000..146b677
--- /dev/null
+++ b/qa/workunits/rados/test_pool_quota.sh
@@ -0,0 +1,38 @@
+#!/bin/sh -ex
+
+p=`uuidgen`
+
+# objects
+ceph osd pool create $p 12
+ceph osd pool set-quota $p max_objects 10
+
+for f in `seq 1 10` ; do
+ rados -p $p put obj$f /etc/passwd
+done
+
+sleep 30
+
+rados -p $p put onemore /etc/passwd  && exit 1 || true
+
+ceph osd pool set-quota $p max_objects 100
+sleep 30
+
+rados -p $p put onemore /etc/passwd
+
+# bytes
+ceph osd pool set-quota $p max_bytes 100
+sleep 30
+
+rados -p $p put two /etc/passwd && exit 1 || true
+
+ceph osd pool set-quota $p max_bytes 0
+ceph osd pool set-quota $p max_objects 0
+sleep 30
+
+rados -p $p put three /etc/passwd
+
+# done
+ceph osd pool delete $p $p --yes-i-really-really-mean-it
+
+echo OK
+

diff --git a/src/Makefile.am b/src/Makefile.am
index ac91b5d757a150a1f2aeda876363136568e9aa9f..b27c8bb6b7029a4d6ea066d1886bf388470e5462 100644 (file)
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -1393,7 +1393,8 @@ libmon_a_SOURCES = \
        os/LevelDBStore.cc \
        mon/HealthMonitor.cc \
        mon/DataHealthService.cc \
-       mon/ConfigKeyService.cc
+       mon/ConfigKeyService.cc \
+       common/util.cc
 libmon_a_CXXFLAGS= ${AM_CXXFLAGS}
 noinst_LIBRARIES += libmon.a
 

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index 76c8adaa72a28534e0bbbba57c6185746d5d6b06..700e7252556cfac1bcc2dc22f3fb53baed883841 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -201,6 +201,8 @@ OPTION(mon_client_hunt_interval, OPT_DOUBLE, 3.0)   // try new mon every N secon
 OPTION(mon_client_ping_interval, OPT_DOUBLE, 10.0)  // ping every N seconds
 OPTION(mon_client_max_log_entries_per_message, OPT_INT, 1000)
 OPTION(mon_max_pool_pg_num, OPT_INT, 65536)
+OPTION(mon_pool_quota_warn_threshold, OPT_INT, 0) // percent of quota at which to issue warnings
+OPTION(mon_pool_quota_crit_threshold, OPT_INT, 0) // percent of quota at which to issue errors
 OPTION(client_cache_size, OPT_INT, 16384)
 OPTION(client_cache_mid, OPT_FLOAT, .75)
 OPTION(client_use_random_mds, OPT_BOOL, false)

diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc
index b6cbaa7e97bb9c04cf27bf9755c10c5f01ecb99f..3acfee89b00bcdb21a3e77e2629a724ae00f4b6d 100644 (file)
--- a/src/mon/OSDMonitor.cc
+++ b/src/mon/OSDMonitor.cc
@@ -47,6 +47,7 @@
 #include "include/compat.h"
 #include "include/assert.h"
 #include "include/stringify.h"
+#include "include/util.h"
 
 #define dout_subsys ceph_subsys_mon
 #undef dout_prefix
@@ -1736,6 +1737,9 @@ void OSDMonitor::tick()
 #endif
   // ---------------
 
+  if (update_pools_status())
+    do_propose = true;
+
   update_trim();
 
   if (do_propose ||
@@ -1838,6 +1842,8 @@ void OSDMonitor::get_health(list<pair<health_status_t,string> >& summary,
       if (detail)
        detail->push_back(make_pair(HEALTH_WARN, ss.str()));
     }
+
+    get_pools_health(summary, detail);
   }
 }
 
@@ -2194,6 +2200,148 @@ bool OSDMonitor::preprocess_command(MMonCommand *m)
     return false;
 }
 
+void OSDMonitor::update_pool_flags(int64_t pool_id, uint64_t flags)
+{
+  const pg_pool_t *pool = osdmap.get_pg_pool(pool_id);
+  if (pending_inc.new_pools.count(pool_id) == 0)
+    pending_inc.new_pools[pool_id] = *pool;
+  pending_inc.new_pools[pool_id].flags = flags;
+}
+
+bool OSDMonitor::update_pools_status()
+{
+  if (!mon->pgmon()->is_readable())
+    return false;
+
+  bool ret = false;
+
+  const map<int64_t,pg_pool_t>& pools = osdmap.get_pools();
+  for (map<int64_t,pg_pool_t>::const_iterator it = pools.begin();
+       it != pools.end();
+       ++it) {
+    pool_stat_t& stats = mon->pgmon()->pg_map.pg_pool_sum[it->first];
+    object_stat_sum_t& sum = stats.stats.sum;
+    const pg_pool_t &pool = it->second;
+    const char *pool_name = osdmap.get_pool_name(it->first);
+
+    bool pool_is_full =
+      (pool.quota_max_bytes > 0 && (uint64_t)sum.num_bytes >= pool.quota_max_bytes) ||
+      (pool.quota_max_objects > 0 && (uint64_t)sum.num_objects >= pool.quota_max_objects);
+
+    if (pool.get_flags() & pg_pool_t::FLAG_FULL) {
+      if (pool_is_full)
+        continue;
+
+      mon->clog.info() << "pool '" << pool_name
+                       << "' no longer full; removing FULL flag";
+
+      update_pool_flags(it->first, pool.get_flags() & ~pg_pool_t::FLAG_FULL);
+      ret = true;
+    } else {
+      if (!pool_is_full)
+       continue;
+
+      if ((uint64_t)sum.num_bytes >= pool.quota_max_bytes) {
+        mon->clog.warn() << "pool '" << pool_name << "' is full"
+                         << " (reached quota's max_bytes: "
+                         << si_t(pool.quota_max_bytes) << ")";
+      } else if ((uint64_t)sum.num_objects >= pool.quota_max_objects) {
+        mon->clog.warn() << "pool '" << pool_name << "' is full"
+                         << " (reached quota's max_objects: "
+                         << pool.quota_max_objects << ")";
+      } else {
+        assert(0 == "we shouldn't reach this");
+      }
+      update_pool_flags(it->first, pool.get_flags() | pg_pool_t::FLAG_FULL);
+      ret = true;
+    }
+  }
+  return ret;
+}
+
+void OSDMonitor::get_pools_health(
+    list<pair<health_status_t,string> >& summary,
+    list<pair<health_status_t,string> > *detail) const
+{
+  const map<int64_t,pg_pool_t>& pools = osdmap.get_pools();
+  for (map<int64_t,pg_pool_t>::const_iterator it = pools.begin();
+       it != pools.end(); ++it) {
+    pool_stat_t& stats = mon->pgmon()->pg_map.pg_pool_sum[it->first];
+    object_stat_sum_t& sum = stats.stats.sum;
+    const pg_pool_t &pool = it->second;
+    const char *pool_name = osdmap.get_pool_name(it->first);
+
+    if (pool.get_flags() & pg_pool_t::FLAG_FULL) {
+      // uncomment these asserts if/when we update the FULL flag on pg_stat update
+      //assert((pool.quota_max_objects > 0) || (pool.quota_max_bytes > 0));
+
+      stringstream ss;
+      ss << "pool '" << pool_name << "' is full";
+      summary.push_back(make_pair(HEALTH_WARN, ss.str()));
+      if (detail)
+       detail->push_back(make_pair(HEALTH_WARN, ss.str()));
+    }
+
+    float warn_threshold = g_conf->mon_pool_quota_warn_threshold/100;
+    float crit_threshold = g_conf->mon_pool_quota_crit_threshold/100;
+
+    if (pool.quota_max_objects > 0) {
+      stringstream ss;
+      health_status_t status = HEALTH_OK;
+      if ((uint64_t)sum.num_objects >= pool.quota_max_objects) {
+       // uncomment these asserts if/when we update the FULL flag on pg_stat update
+        //assert(pool.get_flags() & pg_pool_t::FLAG_FULL);
+      } else if (crit_threshold > 0 &&
+                sum.num_objects >= pool.quota_max_objects*crit_threshold) {
+        ss << "pool '" << pool_name
+           << "' has " << sum.num_objects << " objects"
+           << " (max " << pool.quota_max_objects << ")";
+        status = HEALTH_ERR;
+      } else if (warn_threshold > 0 &&
+                sum.num_objects >= pool.quota_max_objects*warn_threshold) {
+        ss << "pool '" << pool_name
+           << "' has " << sum.num_objects << " objects"
+           << " (max " << pool.quota_max_objects << ")";
+        status = HEALTH_WARN;
+      }
+      if (status != HEALTH_OK) {
+        pair<health_status_t,string> s(status, ss.str());
+        summary.push_back(s);
+        if (detail)
+          detail->push_back(s);
+      }
+    }
+
+    if (pool.quota_max_bytes > 0) {
+      health_status_t status = HEALTH_OK;
+      stringstream ss;
+      if ((uint64_t)sum.num_bytes >= pool.quota_max_bytes) {
+       // uncomment these asserts if/when we update the FULL flag on pg_stat update
+       //assert(pool.get_flags() & pg_pool_t::FLAG_FULL);
+      } else if (crit_threshold > 0 &&
+                sum.num_bytes >= pool.quota_max_bytes*crit_threshold) {
+        ss << "pool '" << pool_name
+           << "' has " << si_t(sum.num_bytes) << " bytes"
+           << " (max " << si_t(pool.quota_max_bytes) << ")";
+        status = HEALTH_ERR;
+      } else if (warn_threshold > 0 &&
+                sum.num_bytes >= pool.quota_max_bytes*warn_threshold) {
+        ss << "pool '" << pool_name
+           << "' has " << si_t(sum.num_bytes) << " objects"
+           << " (max " << si_t(pool.quota_max_bytes) << ")";
+        status = HEALTH_WARN;
+      }
+      if (status != HEALTH_OK) {
+        pair<health_status_t,string> s(status, ss.str());
+        summary.push_back(s);
+        if (detail)
+          detail->push_back(s);
+      }
+    }
+  }
+}
+
+
 int OSDMonitor::prepare_new_pool(MPoolOp *m)
 {
   dout(10) << "prepare_new_pool from " << m->get_connection() << dendl;
@@ -3286,11 +3434,48 @@ bool OSDMonitor::prepare_command(MMonCommand *m)
                ss << "crush ruleset " << n << " does not exist";
                err = -ENOENT;
              }
-           } else {
+            } else {
              ss << "unrecognized pool field " << m->cmd[4];
            }
          }
        }
+      } else if (m->cmd[2] == "set-quota" && m->cmd.size() == 6) {
+        int64_t pool_id = osdmap.lookup_pg_pool_name(m->cmd[3].c_str());
+        if (pool_id < 0) {
+          ss << "unrecognized pool '" << m->cmd[3] << "'";
+          err = -ENOENT;
+          goto out;
+        }
+
+        string field = m->cmd[4];
+        if (field != "max_objects" && field != "max_bytes") {
+          ss << "unrecognized field '" << field << "'; max_bytes of max_objects";
+          err = -EINVAL;
+          goto out;
+        }
+
+        stringstream tss;
+        int64_t value = unit_to_bytesize(m->cmd[5], &tss);
+        if (value < 0) {
+          ss << "error parsing value '" << value << "': " << tss.str();
+          err = value;
+          goto out;
+        }
+
+        if (pending_inc.new_pools.count(pool_id) == 0)
+          pending_inc.new_pools[pool_id] = *osdmap.get_pg_pool(pool_id);
+
+        if (field == "max_objects") {
+          pending_inc.new_pools[pool_id].quota_max_objects = value;
+        } else if (field == "max_bytes") {
+          pending_inc.new_pools[pool_id].quota_max_bytes = value;
+        } else {
+          assert(0 == "unrecognized option");
+        }
+        ss << "set-quota " << field << " = " << value << " for pool " << m->cmd[3];
+        rs = ss.str();
+        wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs, get_version()));
+        return true;
       }
       else if (m->cmd[2] == "get") {
        if (m->cmd.size() != 5) {

diff --git a/src/mon/OSDMonitor.h b/src/mon/OSDMonitor.h
index 5c83c6919efbe386082ca4a09432ea785b3e44fc..e9f72298eb65cc7f72782f763d55525ac253e1c1 100644 (file)
--- a/src/mon/OSDMonitor.h
+++ b/src/mon/OSDMonitor.h
@@ -208,7 +208,12 @@ private:
   int prepare_new_pool(string& name, uint64_t auid, int crush_rule,
                        unsigned pg_num, unsigned pgp_num);
   int prepare_new_pool(MPoolOp *m);
-  
+
+  void update_pool_flags(int64_t pool_id, uint64_t flags);
+  bool update_pools_status();
+  void get_pools_health(list<pair<health_status_t,string> >& summary,
+                        list<pair<health_status_t,string> > *detail) const;
+
   bool prepare_set_flag(MMonCommand *m, int flag);
   bool prepare_unset_flag(MMonCommand *m, int flag);
 

diff --git a/src/test/cli/ceph/help.t b/src/test/cli/ceph/help.t
index 6370c036791d8f21cc9afcf890a61c72a84b4623..bafc3b9633f231a896fcfa81c2261cc6d7d1e53c 100644 (file)
--- a/src/test/cli/ceph/help.t
+++ b/src/test/cli/ceph/help.t
@@ -66,6 +66,7 @@
     ceph osd pool delete <pool> [<pool> --yes-i-really-really-mean-it]
     ceph osd pool rename <pool> <new pool name>
     ceph osd pool set <pool> <field> <value>
+    ceph osd pool set-quota <pool> (max_bytes|max_objects) <value>
     ceph osd scrub <osd-id>
     ceph osd deep-scrub <osd-id>
     ceph osd repair <osd-id>

diff --git a/src/tools/ceph.cc b/src/tools/ceph.cc
index f5c95320b912d2e251205fdbb3643274b0d1f03b..ed6a5342807cacf982424f3cfe6602ffa180dae9 100644 (file)
--- a/src/tools/ceph.cc
+++ b/src/tools/ceph.cc
@@ -109,6 +109,7 @@ static void usage()
   cout << "  ceph osd pool delete <pool> [<pool> --yes-i-really-really-mean-it]\n";
   cout << "  ceph osd pool rename <pool> <new pool name>\n";
   cout << "  ceph osd pool set <pool> <field> <value>\n";
+  cout << "  ceph osd pool set-quota <pool> (max_bytes|max_objects) <value>\n";
   cout << "  ceph osd scrub <osd-id>\n";
   cout << "  ceph osd deep-scrub <osd-id>\n";
   cout << "  ceph osd repair <osd-id>\n";