mds: make purge op limit dynamic based on PGs/MDSs

Signed-off-by: John Spray <john.spray@redhat.com>

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index cbe6174fa149549bda71903d095a6ab0f2f1ef80..382476ad76802110ffc3edfee174e9e836b4428c 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -456,7 +456,9 @@ OPTION(mds_action_on_write_error, OPT_U32, 1) // 0: ignore; 1: force readonly; 2
 // Maximum number of concurrent stray files to purge
 OPTION(mds_max_purge_files, OPT_U32, 64)
 // Maximum number of concurrent RADOS ops to issue in purging
-OPTION(mds_max_purge_ops, OPT_U32, 256)
+OPTION(mds_max_purge_ops, OPT_U32, 8192)
+// Maximum number of concurrent RADOS ops to issue in purging, scaled by PG count
+OPTION(mds_max_purge_ops_per_pg, OPT_FLOAT, 0.5)
 
 // If true, compact leveldb store on mount
 OPTION(osd_compact_leveldb_on_mount, OPT_BOOL, false)

diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index 76bbf2ad42a3d64801b0c52993c08737ecc90adc..8633c28d24e06364588b92ce068cb2c9fca5a7ac 100644 (file)
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -11705,4 +11705,8 @@ void MDCache::maybe_eval_stray(CInode *in, bool delay) {
   }
 }
 
+void MDCache::notify_mdsmap_changed()
+{
+  stray_manager.update_op_limit();
+}
 

diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h
index 24873295ba5cc5d8e7d44df626a92b689b2e0cc6..1141ebef4fab07b32beeaf20aca374d9b7c8c834 100644 (file)
--- a/src/mds/MDCache.h
+++ b/src/mds/MDCache.h
@@ -1083,6 +1083,7 @@ public:
   void process_delayed_expire(CDir *dir);
   void discard_delayed_expire(CDir *dir);
 
+  void notify_mdsmap_changed();
 
 protected:
   void dump_cache(const char *fn, Formatter *f);

diff --git a/src/mds/MDS.cc b/src/mds/MDS.cc
index 2b785c607a41a341d43136b24c6402a9d31e2335..4efe6e07a0d7ae211702b5e4256acc45e8820719 100644 (file)
--- a/src/mds/MDS.cc
+++ b/src/mds/MDS.cc
@@ -1800,6 +1800,8 @@ void MDS::handle_mds_map(MMDSMap *m)
     set_osd_epoch_barrier(osd_epoch);
   }
 
+  mdcache->notify_mdsmap_changed();
+
  out:
   beacon.notify_mdsmap(mdsmap);
 

diff --git a/src/mds/StrayManager.cc b/src/mds/StrayManager.cc
index 294f5cd5240cc35274848e3556fa973b7f15cc6e..26bb593e0ea45e0d9397f53443a3cbe17077d00d 100644 (file)
--- a/src/mds/StrayManager.cc
+++ b/src/mds/StrayManager.cc
@@ -422,8 +422,8 @@ bool StrayManager::_consume(CDentry *dn, bool trunc, uint32_t ops_required)
   // Calculate how much of the ops allowance is available, allowing
   // for the case where the limit is currently being exceeded.
   uint32_t ops_avail;
-  if (ops_in_flight <= g_conf->mds_max_purge_ops) {
-    ops_avail = g_conf->mds_max_purge_ops - ops_in_flight;
+  if (ops_in_flight <= max_purge_ops) {
+    ops_avail = max_purge_ops - ops_in_flight;
   } else {
     ops_avail = 0;
   }
@@ -800,8 +800,7 @@ void StrayManager::migrate_stray(CDentry *dn, mds_rank_t to)
     num_strays(0), num_strays_purging(0), num_strays_delayed(0)
 {
   assert(mds != NULL);
-
-  assert(g_conf->mds_max_purge_ops >= g_conf->filer_max_purge_ops);
+  update_op_limit();
 }
 
 
@@ -909,3 +908,51 @@ void StrayManager::_truncate_stray_logged(CDentry *dn, LogSegment *ls)
   eval_stray(dn);
 }
 
+
+const char** StrayManager::get_tracked_conf_keys() const
+{
+  static const char* KEYS[] = {
+    "mds_max_purge_ops",
+    NULL
+  };
+  return KEYS;
+}
+
+
+// Subscribe to the per-PG ops throttle config event, and do a cool calculation
+// of (throttle * pg_num / (mds_num))
+void StrayManager::handle_conf_change(const struct md_config_t *conf,
+                         const std::set <std::string> &changed)
+{
+  if (changed.count("mds_max_purge_ops")) {
+    update_op_limit();
+  }
+}
+
+
+void StrayManager::update_op_limit()
+{
+  const OSDMap *osdmap = mds->objecter->get_osdmap_read();
+
+  // Number of PGs across all data pools
+  uint64_t pg_count = 0;
+  const std::set<int64_t> &data_pools = mds->mdsmap->get_data_pools();
+  for (std::set<int64_t>::iterator i = data_pools.begin();
+       i != data_pools.end(); ++i) {
+    pg_count += osdmap->get_pg_num(*i);
+  }
+
+  mds->objecter->put_osdmap_read();
+
+  uint64_t mds_count = mds->mdsmap->get_max_mds();
+
+  // Work out a limit based on n_pgs / n_mdss, multiplied by the user's
+  // preference for how many ops per PG
+  max_purge_ops = uint64_t(((double)pg_count / (double)mds_count) * g_conf->mds_max_purge_ops_per_pg);
+
+  // User may also specify a hard limit, apply this if so.
+  if (g_conf->mds_max_purge_ops) {
+    max_purge_ops = MIN(max_purge_ops, g_conf->mds_max_purge_ops);
+  }
+}
+

diff --git a/src/mds/StrayManager.h b/src/mds/StrayManager.h
index f10e4246cec1c61f0a1d1da199715803dab4effc..a1e3bc02bfc0ee0b8c5cfa1c091ef0f1e96728a8 100644 (file)
--- a/src/mds/StrayManager.h
+++ b/src/mds/StrayManager.h
@@ -22,7 +22,7 @@ class PerfCounters;
 class CInode;
 class CDentry;
 
-class StrayManager
+class StrayManager : public md_config_obs_t
 {
   protected:
   class QueuedStray {
@@ -48,6 +48,9 @@ class StrayManager
   uint64_t ops_in_flight;
   uint64_t files_purging;
 
+  // Dynamic op limit per MDS based on PG count
+  uint64_t max_purge_ops;
+
   // Statistics
   uint64_t num_strays;
   uint64_t num_strays_purging;
@@ -87,6 +90,11 @@ class StrayManager
   void notify_stray_created();
   void notify_stray_removed();
   void abort_queue();
+
+  void update_op_limit();
+  virtual const char** get_tracked_conf_keys() const;
+  virtual void handle_conf_change(const struct md_config_t *conf,
+                         const std::set <std::string> &changed);
 };
 
 #endif  // STRAY_MANAGER_H