From 91867b2b74a7f9d3d38fb4714802bab03428be7b Mon Sep 17 00:00:00 2001 From: John Spray Date: Tue, 10 Feb 2015 14:16:34 +0000 Subject: [PATCH] mds: make purge op limit dynamic based on PGs/MDSs Signed-off-by: John Spray --- src/common/config_opts.h | 4 ++- src/mds/MDCache.cc | 4 +++ src/mds/MDCache.h | 1 + src/mds/MDS.cc | 2 ++ src/mds/StrayManager.cc | 55 +++++++++++++++++++++++++++++++++++++--- src/mds/StrayManager.h | 10 +++++++- 6 files changed, 70 insertions(+), 6 deletions(-) diff --git a/src/common/config_opts.h b/src/common/config_opts.h index cbe6174fa1495..382476ad76802 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -456,7 +456,9 @@ OPTION(mds_action_on_write_error, OPT_U32, 1) // 0: ignore; 1: force readonly; 2 // Maximum number of concurrent stray files to purge OPTION(mds_max_purge_files, OPT_U32, 64) // Maximum number of concurrent RADOS ops to issue in purging -OPTION(mds_max_purge_ops, OPT_U32, 256) +OPTION(mds_max_purge_ops, OPT_U32, 8192) +// Maximum number of concurrent RADOS ops to issue in purging, scaled by PG count +OPTION(mds_max_purge_ops_per_pg, OPT_FLOAT, 0.5) // If true, compact leveldb store on mount OPTION(osd_compact_leveldb_on_mount, OPT_BOOL, false) diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc index 76bbf2ad42a3d..8633c28d24e06 100644 --- a/src/mds/MDCache.cc +++ b/src/mds/MDCache.cc @@ -11705,4 +11705,8 @@ void MDCache::maybe_eval_stray(CInode *in, bool delay) { } } +void MDCache::notify_mdsmap_changed() +{ + stray_manager.update_op_limit(); +} diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h index 24873295ba5cc..1141ebef4fab0 100644 --- a/src/mds/MDCache.h +++ b/src/mds/MDCache.h @@ -1083,6 +1083,7 @@ public: void process_delayed_expire(CDir *dir); void discard_delayed_expire(CDir *dir); + void notify_mdsmap_changed(); protected: void dump_cache(const char *fn, Formatter *f); diff --git a/src/mds/MDS.cc b/src/mds/MDS.cc index 2b785c607a41a..4efe6e07a0d7a 100644 --- a/src/mds/MDS.cc +++ b/src/mds/MDS.cc @@ -1800,6 +1800,8 @@ void MDS::handle_mds_map(MMDSMap *m) set_osd_epoch_barrier(osd_epoch); } + mdcache->notify_mdsmap_changed(); + out: beacon.notify_mdsmap(mdsmap); diff --git a/src/mds/StrayManager.cc b/src/mds/StrayManager.cc index 294f5cd5240cc..26bb593e0ea45 100644 --- a/src/mds/StrayManager.cc +++ b/src/mds/StrayManager.cc @@ -422,8 +422,8 @@ bool StrayManager::_consume(CDentry *dn, bool trunc, uint32_t ops_required) // Calculate how much of the ops allowance is available, allowing // for the case where the limit is currently being exceeded. uint32_t ops_avail; - if (ops_in_flight <= g_conf->mds_max_purge_ops) { - ops_avail = g_conf->mds_max_purge_ops - ops_in_flight; + if (ops_in_flight <= max_purge_ops) { + ops_avail = max_purge_ops - ops_in_flight; } else { ops_avail = 0; } @@ -800,8 +800,7 @@ void StrayManager::migrate_stray(CDentry *dn, mds_rank_t to) num_strays(0), num_strays_purging(0), num_strays_delayed(0) { assert(mds != NULL); - - assert(g_conf->mds_max_purge_ops >= g_conf->filer_max_purge_ops); + update_op_limit(); } @@ -909,3 +908,51 @@ void StrayManager::_truncate_stray_logged(CDentry *dn, LogSegment *ls) eval_stray(dn); } + +const char** StrayManager::get_tracked_conf_keys() const +{ + static const char* KEYS[] = { + "mds_max_purge_ops", + NULL + }; + return KEYS; +} + + +// Subscribe to the per-PG ops throttle config event, and do a cool calculation +// of (throttle * pg_num / (mds_num)) +void StrayManager::handle_conf_change(const struct md_config_t *conf, + const std::set &changed) +{ + if (changed.count("mds_max_purge_ops")) { + update_op_limit(); + } +} + + +void StrayManager::update_op_limit() +{ + const OSDMap *osdmap = mds->objecter->get_osdmap_read(); + + // Number of PGs across all data pools + uint64_t pg_count = 0; + const std::set &data_pools = mds->mdsmap->get_data_pools(); + for (std::set::iterator i = data_pools.begin(); + i != data_pools.end(); ++i) { + pg_count += osdmap->get_pg_num(*i); + } + + mds->objecter->put_osdmap_read(); + + uint64_t mds_count = mds->mdsmap->get_max_mds(); + + // Work out a limit based on n_pgs / n_mdss, multiplied by the user's + // preference for how many ops per PG + max_purge_ops = uint64_t(((double)pg_count / (double)mds_count) * g_conf->mds_max_purge_ops_per_pg); + + // User may also specify a hard limit, apply this if so. + if (g_conf->mds_max_purge_ops) { + max_purge_ops = MIN(max_purge_ops, g_conf->mds_max_purge_ops); + } +} + diff --git a/src/mds/StrayManager.h b/src/mds/StrayManager.h index f10e4246cec1c..a1e3bc02bfc0e 100644 --- a/src/mds/StrayManager.h +++ b/src/mds/StrayManager.h @@ -22,7 +22,7 @@ class PerfCounters; class CInode; class CDentry; -class StrayManager +class StrayManager : public md_config_obs_t { protected: class QueuedStray { @@ -48,6 +48,9 @@ class StrayManager uint64_t ops_in_flight; uint64_t files_purging; + // Dynamic op limit per MDS based on PG count + uint64_t max_purge_ops; + // Statistics uint64_t num_strays; uint64_t num_strays_purging; @@ -87,6 +90,11 @@ class StrayManager void notify_stray_created(); void notify_stray_removed(); void abort_queue(); + + void update_op_limit(); + virtual const char** get_tracked_conf_keys() const; + virtual void handle_conf_change(const struct md_config_t *conf, + const std::set &changed); }; #endif // STRAY_MANAGER_H -- 2.39.5