From: Sage Weil Date: Sat, 30 Mar 2013 00:59:35 +0000 (-0700) Subject: mon: OSDMonitor: add 'osd pool set-quota' command X-Git-Tag: v0.62~126^2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=refs%2Fpull%2F174%2Fhead;p=ceph.git mon: OSDMonitor: add 'osd pool set-quota' command Signed-off-by: Joao Eduardo Luis --- diff --git a/qa/workunits/rados/test_pool_quota.sh b/qa/workunits/rados/test_pool_quota.sh new file mode 100755 index 000000000000..146b677a05fc --- /dev/null +++ b/qa/workunits/rados/test_pool_quota.sh @@ -0,0 +1,38 @@ +#!/bin/sh -ex + +p=`uuidgen` + +# objects +ceph osd pool create $p 12 +ceph osd pool set-quota $p max_objects 10 + +for f in `seq 1 10` ; do + rados -p $p put obj$f /etc/passwd +done + +sleep 30 + +rados -p $p put onemore /etc/passwd && exit 1 || true + +ceph osd pool set-quota $p max_objects 100 +sleep 30 + +rados -p $p put onemore /etc/passwd + +# bytes +ceph osd pool set-quota $p max_bytes 100 +sleep 30 + +rados -p $p put two /etc/passwd && exit 1 || true + +ceph osd pool set-quota $p max_bytes 0 +ceph osd pool set-quota $p max_objects 0 +sleep 30 + +rados -p $p put three /etc/passwd + +# done +ceph osd pool delete $p $p --yes-i-really-really-mean-it + +echo OK + diff --git a/src/Makefile.am b/src/Makefile.am index ac91b5d757a1..b27c8bb6b702 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1393,7 +1393,8 @@ libmon_a_SOURCES = \ os/LevelDBStore.cc \ mon/HealthMonitor.cc \ mon/DataHealthService.cc \ - mon/ConfigKeyService.cc + mon/ConfigKeyService.cc \ + common/util.cc libmon_a_CXXFLAGS= ${AM_CXXFLAGS} noinst_LIBRARIES += libmon.a diff --git a/src/common/config_opts.h b/src/common/config_opts.h index 76c8adaa72a2..700e7252556c 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -201,6 +201,8 @@ OPTION(mon_client_hunt_interval, OPT_DOUBLE, 3.0) // try new mon every N secon OPTION(mon_client_ping_interval, OPT_DOUBLE, 10.0) // ping every N seconds OPTION(mon_client_max_log_entries_per_message, OPT_INT, 1000) OPTION(mon_max_pool_pg_num, OPT_INT, 65536) +OPTION(mon_pool_quota_warn_threshold, OPT_INT, 0) // percent of quota at which to issue warnings +OPTION(mon_pool_quota_crit_threshold, OPT_INT, 0) // percent of quota at which to issue errors OPTION(client_cache_size, OPT_INT, 16384) OPTION(client_cache_mid, OPT_FLOAT, .75) OPTION(client_use_random_mds, OPT_BOOL, false) diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc index b6cbaa7e97bb..3acfee89b00b 100644 --- a/src/mon/OSDMonitor.cc +++ b/src/mon/OSDMonitor.cc @@ -47,6 +47,7 @@ #include "include/compat.h" #include "include/assert.h" #include "include/stringify.h" +#include "include/util.h" #define dout_subsys ceph_subsys_mon #undef dout_prefix @@ -1736,6 +1737,9 @@ void OSDMonitor::tick() #endif // --------------- + if (update_pools_status()) + do_propose = true; + update_trim(); if (do_propose || @@ -1838,6 +1842,8 @@ void OSDMonitor::get_health(list >& summary, if (detail) detail->push_back(make_pair(HEALTH_WARN, ss.str())); } + + get_pools_health(summary, detail); } } @@ -2194,6 +2200,148 @@ bool OSDMonitor::preprocess_command(MMonCommand *m) return false; } +void OSDMonitor::update_pool_flags(int64_t pool_id, uint64_t flags) +{ + const pg_pool_t *pool = osdmap.get_pg_pool(pool_id); + if (pending_inc.new_pools.count(pool_id) == 0) + pending_inc.new_pools[pool_id] = *pool; + pending_inc.new_pools[pool_id].flags = flags; +} + +bool OSDMonitor::update_pools_status() +{ + if (!mon->pgmon()->is_readable()) + return false; + + bool ret = false; + + const map& pools = osdmap.get_pools(); + for (map::const_iterator it = pools.begin(); + it != pools.end(); + ++it) { + pool_stat_t& stats = mon->pgmon()->pg_map.pg_pool_sum[it->first]; + object_stat_sum_t& sum = stats.stats.sum; + const pg_pool_t &pool = it->second; + const char *pool_name = osdmap.get_pool_name(it->first); + + bool pool_is_full = + (pool.quota_max_bytes > 0 && (uint64_t)sum.num_bytes >= pool.quota_max_bytes) || + (pool.quota_max_objects > 0 && (uint64_t)sum.num_objects >= pool.quota_max_objects); + + if (pool.get_flags() & pg_pool_t::FLAG_FULL) { + if (pool_is_full) + continue; + + mon->clog.info() << "pool '" << pool_name + << "' no longer full; removing FULL flag"; + + update_pool_flags(it->first, pool.get_flags() & ~pg_pool_t::FLAG_FULL); + ret = true; + } else { + if (!pool_is_full) + continue; + + if ((uint64_t)sum.num_bytes >= pool.quota_max_bytes) { + mon->clog.warn() << "pool '" << pool_name << "' is full" + << " (reached quota's max_bytes: " + << si_t(pool.quota_max_bytes) << ")"; + } else if ((uint64_t)sum.num_objects >= pool.quota_max_objects) { + mon->clog.warn() << "pool '" << pool_name << "' is full" + << " (reached quota's max_objects: " + << pool.quota_max_objects << ")"; + } else { + assert(0 == "we shouldn't reach this"); + } + update_pool_flags(it->first, pool.get_flags() | pg_pool_t::FLAG_FULL); + ret = true; + } + } + return ret; +} + +void OSDMonitor::get_pools_health( + list >& summary, + list > *detail) const +{ + const map& pools = osdmap.get_pools(); + for (map::const_iterator it = pools.begin(); + it != pools.end(); ++it) { + pool_stat_t& stats = mon->pgmon()->pg_map.pg_pool_sum[it->first]; + object_stat_sum_t& sum = stats.stats.sum; + const pg_pool_t &pool = it->second; + const char *pool_name = osdmap.get_pool_name(it->first); + + if (pool.get_flags() & pg_pool_t::FLAG_FULL) { + // uncomment these asserts if/when we update the FULL flag on pg_stat update + //assert((pool.quota_max_objects > 0) || (pool.quota_max_bytes > 0)); + + stringstream ss; + ss << "pool '" << pool_name << "' is full"; + summary.push_back(make_pair(HEALTH_WARN, ss.str())); + if (detail) + detail->push_back(make_pair(HEALTH_WARN, ss.str())); + } + + float warn_threshold = g_conf->mon_pool_quota_warn_threshold/100; + float crit_threshold = g_conf->mon_pool_quota_crit_threshold/100; + + if (pool.quota_max_objects > 0) { + stringstream ss; + health_status_t status = HEALTH_OK; + if ((uint64_t)sum.num_objects >= pool.quota_max_objects) { + // uncomment these asserts if/when we update the FULL flag on pg_stat update + //assert(pool.get_flags() & pg_pool_t::FLAG_FULL); + } else if (crit_threshold > 0 && + sum.num_objects >= pool.quota_max_objects*crit_threshold) { + ss << "pool '" << pool_name + << "' has " << sum.num_objects << " objects" + << " (max " << pool.quota_max_objects << ")"; + status = HEALTH_ERR; + } else if (warn_threshold > 0 && + sum.num_objects >= pool.quota_max_objects*warn_threshold) { + ss << "pool '" << pool_name + << "' has " << sum.num_objects << " objects" + << " (max " << pool.quota_max_objects << ")"; + status = HEALTH_WARN; + } + if (status != HEALTH_OK) { + pair s(status, ss.str()); + summary.push_back(s); + if (detail) + detail->push_back(s); + } + } + + if (pool.quota_max_bytes > 0) { + health_status_t status = HEALTH_OK; + stringstream ss; + if ((uint64_t)sum.num_bytes >= pool.quota_max_bytes) { + // uncomment these asserts if/when we update the FULL flag on pg_stat update + //assert(pool.get_flags() & pg_pool_t::FLAG_FULL); + } else if (crit_threshold > 0 && + sum.num_bytes >= pool.quota_max_bytes*crit_threshold) { + ss << "pool '" << pool_name + << "' has " << si_t(sum.num_bytes) << " bytes" + << " (max " << si_t(pool.quota_max_bytes) << ")"; + status = HEALTH_ERR; + } else if (warn_threshold > 0 && + sum.num_bytes >= pool.quota_max_bytes*warn_threshold) { + ss << "pool '" << pool_name + << "' has " << si_t(sum.num_bytes) << " objects" + << " (max " << si_t(pool.quota_max_bytes) << ")"; + status = HEALTH_WARN; + } + if (status != HEALTH_OK) { + pair s(status, ss.str()); + summary.push_back(s); + if (detail) + detail->push_back(s); + } + } + } +} + + int OSDMonitor::prepare_new_pool(MPoolOp *m) { dout(10) << "prepare_new_pool from " << m->get_connection() << dendl; @@ -3286,11 +3434,48 @@ bool OSDMonitor::prepare_command(MMonCommand *m) ss << "crush ruleset " << n << " does not exist"; err = -ENOENT; } - } else { + } else { ss << "unrecognized pool field " << m->cmd[4]; } } } + } else if (m->cmd[2] == "set-quota" && m->cmd.size() == 6) { + int64_t pool_id = osdmap.lookup_pg_pool_name(m->cmd[3].c_str()); + if (pool_id < 0) { + ss << "unrecognized pool '" << m->cmd[3] << "'"; + err = -ENOENT; + goto out; + } + + string field = m->cmd[4]; + if (field != "max_objects" && field != "max_bytes") { + ss << "unrecognized field '" << field << "'; max_bytes of max_objects"; + err = -EINVAL; + goto out; + } + + stringstream tss; + int64_t value = unit_to_bytesize(m->cmd[5], &tss); + if (value < 0) { + ss << "error parsing value '" << value << "': " << tss.str(); + err = value; + goto out; + } + + if (pending_inc.new_pools.count(pool_id) == 0) + pending_inc.new_pools[pool_id] = *osdmap.get_pg_pool(pool_id); + + if (field == "max_objects") { + pending_inc.new_pools[pool_id].quota_max_objects = value; + } else if (field == "max_bytes") { + pending_inc.new_pools[pool_id].quota_max_bytes = value; + } else { + assert(0 == "unrecognized option"); + } + ss << "set-quota " << field << " = " << value << " for pool " << m->cmd[3]; + rs = ss.str(); + wait_for_finished_proposal(new Monitor::C_Command(mon, m, 0, rs, get_version())); + return true; } else if (m->cmd[2] == "get") { if (m->cmd.size() != 5) { diff --git a/src/mon/OSDMonitor.h b/src/mon/OSDMonitor.h index 5c83c6919efb..e9f72298eb65 100644 --- a/src/mon/OSDMonitor.h +++ b/src/mon/OSDMonitor.h @@ -208,7 +208,12 @@ private: int prepare_new_pool(string& name, uint64_t auid, int crush_rule, unsigned pg_num, unsigned pgp_num); int prepare_new_pool(MPoolOp *m); - + + void update_pool_flags(int64_t pool_id, uint64_t flags); + bool update_pools_status(); + void get_pools_health(list >& summary, + list > *detail) const; + bool prepare_set_flag(MMonCommand *m, int flag); bool prepare_unset_flag(MMonCommand *m, int flag); diff --git a/src/test/cli/ceph/help.t b/src/test/cli/ceph/help.t index 6370c036791d..bafc3b9633f2 100644 --- a/src/test/cli/ceph/help.t +++ b/src/test/cli/ceph/help.t @@ -66,6 +66,7 @@ ceph osd pool delete [ --yes-i-really-really-mean-it] ceph osd pool rename ceph osd pool set + ceph osd pool set-quota (max_bytes|max_objects) ceph osd scrub ceph osd deep-scrub ceph osd repair diff --git a/src/tools/ceph.cc b/src/tools/ceph.cc index f5c95320b912..ed6a5342807c 100644 --- a/src/tools/ceph.cc +++ b/src/tools/ceph.cc @@ -109,6 +109,7 @@ static void usage() cout << " ceph osd pool delete [ --yes-i-really-really-mean-it]\n"; cout << " ceph osd pool rename \n"; cout << " ceph osd pool set \n"; + cout << " ceph osd pool set-quota (max_bytes|max_objects) \n"; cout << " ceph osd scrub \n"; cout << " ceph osd deep-scrub \n"; cout << " ceph osd repair \n";