mds: avoid using g_conf().get_val<...>(...) in hot path

g_conf->get_val<>()'s overhead is high

Fixes: http://tracker.ceph.com/issues/24820
Signed-off-by: "Yan, Zheng" <zyan@redhat.com>

diff --git a/src/common/legacy_config_opts.h b/src/common/legacy_config_opts.h
index c4618b038306a283fbbb4302a83e34c080a2a047..733b8ae10c87e1b23f7c7af02ba748d88af0b4d0 100644 (file)
--- a/src/common/legacy_config_opts.h
+++ b/src/common/legacy_config_opts.h
@@ -444,7 +444,6 @@ OPTION(mds_bal_split_rd, OPT_FLOAT)
 OPTION(mds_bal_split_wr, OPT_FLOAT)
 OPTION(mds_bal_split_bits, OPT_INT)
 OPTION(mds_bal_merge_size, OPT_INT)
-OPTION(mds_bal_fragment_interval, OPT_INT)      // seconds
 OPTION(mds_bal_fragment_size_max, OPT_INT) // order of magnitude higher than split size
 OPTION(mds_bal_fragment_fast_factor, OPT_FLOAT) // multiple of size_max that triggers immediate split
 OPTION(mds_bal_idle_threshold, OPT_FLOAT)

diff --git a/src/mds/Locker.cc b/src/mds/Locker.cc
index 905df5378347bbeaec42fe5e350f017c212db9bf..65ece18567389b049f47f9558ee31a661e4c9877 100644 (file)
--- a/src/mds/Locker.cc
+++ b/src/mds/Locker.cc
@@ -18,6 +18,7 @@
 #include "MDCache.h"
 #include "Locker.h"
 #include "MDBalancer.h"
+#include "Migrator.h"
 #include "CInode.h"
 #include "CDir.h"
 #include "CDentry.h"

diff --git a/src/mds/MDBalancer.cc b/src/mds/MDBalancer.cc
index ef2fc961cb28da17985e859cb494bb33fb0ffadf..2adcadb83031b91ddf3fc30e08647fb1ef463541 100644 (file)
--- a/src/mds/MDBalancer.cc
+++ b/src/mds/MDBalancer.cc
@@ -15,9 +15,9 @@
 #include "include/compat.h"
 #include "mdstypes.h"
 
+#include "mon/MonClient.h"
 #include "MDBalancer.h"
 #include "MDSRank.h"
-#include "mon/MonClient.h"
 #include "MDSMap.h"
 #include "CInode.h"
 #include "CDir.h"
@@ -77,6 +77,23 @@ int MDBalancer::proc_message(Message *m)
   return 0;
 }
 
+MDBalancer::MDBalancer(MDSRank *m, Messenger *msgr, MonClient *monc) :
+    mds(m), messenger(msgr), mon_client(monc)
+{
+  bal_fragment_dirs = g_conf().get_val<bool>("mds_bal_fragment_dirs");
+  bal_fragment_interval = g_conf().get_val<int64_t>("mds_bal_fragment_interval");
+}
+
+void MDBalancer::handle_conf_change(const ConfigProxy& conf,
+                                   const std::set <std::string> &changed,
+                                   const MDSMap &mds_map)
+{
+  if (changed.count("mds_bal_fragment_dirs"))
+    bal_fragment_dirs = g_conf().get_val<bool>("mds_bal_fragment_dirs");
+  if (changed.count("mds_bal_fragment_interval"))
+    bal_fragment_interval = g_conf().get_val<int64_t>("mds_bal_fragment_interval");
+}
+
 void MDBalancer::handle_export_pins(void)
 {
   auto &q = mds->mdcache->export_pin_queue;
@@ -148,8 +165,8 @@ void MDBalancer::handle_export_pins(void)
 void MDBalancer::tick()
 {
   static int num_bal_times = g_conf()->mds_bal_max;
-  auto bal_interval = mds->cct->_conf.get_val<int64_t>("mds_bal_interval");
-  auto bal_max_until = mds->cct->_conf.get_val<int64_t>("mds_bal_max_until");
+  auto bal_interval = g_conf().get_val<int64_t>("mds_bal_interval");
+  auto bal_max_until = g_conf().get_val<int64_t>("mds_bal_max_until");
   time now = clock::now();
 
   if (g_conf()->mds_bal_export_pin) {
@@ -281,8 +298,9 @@ int MDBalancer::localize_balancer()
            << " oid=" << oid << " oloc=" << oloc << dendl;
 
   /* timeout: if we waste half our time waiting for RADOS, then abort! */
+  auto bal_interval = g_conf().get_val<int64_t>("mds_bal_interval");
   lock.Lock();
-  int ret_t = cond.WaitInterval(lock, utime_t(mds->cct->_conf.get_val<int64_t>("mds_bal_interval")/2, 0));
+  int ret_t = cond.WaitInterval(lock, utime_t(bal_interval / 2, 0));
   lock.Unlock();
 
   /* success: store the balancer in memory and set the version. */
@@ -506,7 +524,7 @@ void MDBalancer::queue_split(const CDir *dir, bool fast)
     // Set a timer to really do the split: we don't do it immediately
     // so that bursts of ops on a directory have a chance to go through
     // before we freeze it.
-    mds->timer.add_event_after(g_conf()->mds_bal_fragment_interval,
+    mds->timer.add_event_after(bal_fragment_interval,
                                new FunctionContext(callback));
   }
 }
@@ -570,7 +588,7 @@ void MDBalancer::queue_merge(CDir *dir)
   if (merge_pending.count(frag) == 0) {
     dout(20) << __func__ << " enqueued dir " << *dir << dendl;
     merge_pending.insert(frag);
-    mds->timer.add_event_after(g_conf()->mds_bal_fragment_interval,
+    mds->timer.add_event_after(bal_fragment_interval,
         new FunctionContext(callback));
   } else {
     dout(20) << __func__ << " dir already in queue " << *dir << dendl;
@@ -1115,10 +1133,8 @@ void MDBalancer::hit_inode(CInode *in, int type, int who)
 void MDBalancer::maybe_fragment(CDir *dir, bool hot)
 {
   // split/merge
-  if (mds->cct->_conf.get_val<bool>("mds_bal_fragment_dirs") &&
-      g_conf()->mds_bal_fragment_interval > 0 &&
-      !dir->inode->is_base() &&        // not root/base (for now at least)
-      dir->is_auth()) {
+  if (bal_fragment_dirs && bal_fragment_interval > 0 &&
+      dir->is_auth() && !dir->inode->is_base()) {  // not root/base (for now at least)
 
     // split
     if (g_conf()->mds_bal_split_size > 0 && (dir->should_split() || hot)) {

diff --git a/src/mds/MDBalancer.h b/src/mds/MDBalancer.h
index 9a7d7c5128edcd04973a8dbf41d7012bac59dfa2..19182abc556708fc01279af204a298813a02013f 100644 (file)
--- a/src/mds/MDBalancer.h
+++ b/src/mds/MDBalancer.h
@@ -26,6 +26,7 @@ using std::map;
 #include "common/Clock.h"
 #include "common/Cond.h"
 
+class MDSMap;
 class MDSRank;
 class Message;
 class MHeartbeat;
@@ -40,8 +41,11 @@ public:
   using time = ceph::coarse_mono_time;
   friend class C_Bal_SendHeartbeat;
 
-  MDBalancer(MDSRank *m, Messenger *msgr, MonClient *monc) : 
-    mds(m), messenger(msgr), mon_client(monc) { }
+  MDBalancer(MDSRank *m, Messenger *msgr, MonClient *monc);
+
+  void handle_conf_change(const ConfigProxy& conf,
+                          const std::set <std::string> &changed,
+                          const MDSMap &mds_map);
 
   int proc_message(Message *m);
 
@@ -75,6 +79,9 @@ public:
   int dump_loads(Formatter *f);
 
 private:
+  bool bal_fragment_dirs;
+  int64_t bal_fragment_interval;
+
   typedef struct {
     std::map<mds_rank_t, double> targets;
     std::map<mds_rank_t, double> imported;

diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index 974e9f242476655d15a09f57dffb13224e6080a4..5b4fdbe7a08bab362f717f55fbc62910363e23d1 100644 (file)
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -207,7 +207,13 @@ MDCache::MDCache(MDSRank *m, PurgeQueue &purge_queue_) :
   cap_imports_num_opening = 0;
 
   opening_root = open = false;
-  lru.lru_set_midpoint(cache_mid());
+
+  cache_inode_limit = g_conf().get_val<int64_t>("mds_cache_size");
+  cache_memory_limit = g_conf().get_val<uint64_t>("mds_cache_memory_limit");
+  cache_reservation = g_conf().get_val<double>("mds_cache_reservation");
+  cache_health_threshold = g_conf().get_val<double>("mds_health_cache_threshold");
+
+  lru.lru_set_midpoint(g_conf().get_val<double>("mds_cache_mid"));
 
   bottom_lru.lru_set_midpoint(0);
 
@@ -225,11 +231,28 @@ MDCache::~MDCache()
   }
 }
 
+void MDCache::handle_conf_change(const ConfigProxy& conf,
+                                 const std::set <std::string> &changed,
+                                 const MDSMap &mdsmap)
+{
+  if (changed.count("mds_cache_size"))
+    cache_inode_limit = g_conf().get_val<int64_t>("mds_cache_size");
+  if (changed.count("mds_cache_memory_limit"))
+    cache_memory_limit = g_conf().get_val<uint64_t>("mds_cache_memory_limit");
+  if (changed.count("mds_cache_reservation"))
+    cache_reservation = g_conf().get_val<double>("mds_cache_reservation");
+  if (changed.count("mds_health_cache_threshold"))
+    cache_health_threshold = g_conf().get_val<double>("mds_health_cache_threshold");
+  if (changed.count("mds_cache_mid"))
+    lru.lru_set_midpoint(g_conf().get_val<double>("mds_cache_mid"));
 
+  migrator->handle_conf_change(conf, changed, mdsmap);
+  mds->balancer->handle_conf_change(conf, changed, mdsmap);
+}
 
 void MDCache::log_stat()
 {
-  mds->logger->set(l_mds_inode_max, cache_limit_inodes() == 0 ? INT_MAX : cache_limit_inodes());
+  mds->logger->set(l_mds_inode_max, cache_inode_limit ? : INT_MAX);
   mds->logger->set(l_mds_inodes, lru.lru_get_size());
   mds->logger->set(l_mds_inodes_pinned, lru.lru_get_num_pinned());
   mds->logger->set(l_mds_inodes_top, lru.lru_get_top());
@@ -6659,12 +6682,12 @@ void MDCache::trim_lru(uint64_t count, map<mds_rank_t, MCacheExpire*> &expiremap
 bool MDCache::trim(uint64_t count)
 {
   uint64_t used = cache_size();
-  uint64_t limit = cache_limit_memory();
+  uint64_t limit = cache_memory_limit;
   map<mds_rank_t, MCacheExpire*> expiremap;
 
   dout(7) << "trim bytes_used=" << bytes2str(used)
           << " limit=" << bytes2str(limit)
-          << " reservation=" << cache_reservation()
+          << " reservation=" << cache_reservation
           << "% count=" << count << dendl;
 
   // process delayed eval_stray()

diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h
index 3a72b65ffc7903c5af3ade6dcd9e87f23a06e616..0b36e951e820119ad013cd969701327600a53d76 100644 (file)
--- a/src/mds/MDCache.h
+++ b/src/mds/MDCache.h
@@ -150,27 +150,23 @@ class MDCache {
 
   bool exceeded_size_limit;
 
+private:
+  uint64_t cache_inode_limit;
+  uint64_t cache_memory_limit;
+  double cache_reservation;
+  double cache_health_threshold;
+
 public:
-  static uint64_t cache_limit_inodes(void) {
-    return g_conf().get_val<int64_t>("mds_cache_size");
-  }
-  static uint64_t cache_limit_memory(void) {
-    return g_conf().get_val<uint64_t>("mds_cache_memory_limit");
-  }
-  static double cache_reservation(void) {
-    return g_conf().get_val<double>("mds_cache_reservation");
-  }
-  static double cache_mid(void) {
-    return g_conf().get_val<double>("mds_cache_mid");
+  uint64_t cache_limit_inodes(void) {
+    return cache_inode_limit;
   }
-  static double cache_health_threshold(void) {
-    return g_conf().get_val<double>("mds_health_cache_threshold");
+  uint64_t cache_limit_memory(void) {
+    return cache_memory_limit;
   }
   double cache_toofull_ratio(void) const {
-    uint64_t inode_limit = cache_limit_inodes();
-    double inode_reserve = inode_limit*(1.0-cache_reservation());
-    double memory_reserve = cache_limit_memory()*(1.0-cache_reservation());
-    return fmax(0.0, fmax((cache_size()-memory_reserve)/memory_reserve, inode_limit == 0 ? 0.0 : (CInode::count()-inode_reserve)/inode_reserve));
+    double inode_reserve = cache_inode_limit*(1.0-cache_reservation);
+    double memory_reserve = cache_memory_limit*(1.0-cache_reservation);
+    return fmax(0.0, fmax((cache_size()-memory_reserve)/memory_reserve, cache_inode_limit == 0 ? 0.0 : (CInode::count()-inode_reserve)/inode_reserve));
   }
   bool cache_toofull(void) const {
     return cache_toofull_ratio() > 0.0;
@@ -179,8 +175,7 @@ public:
     return mempool::get_pool(mempool::mds_co::id).allocated_bytes();
   }
   bool cache_overfull(void) const {
-    uint64_t inode_limit = cache_limit_inodes();
-    return (inode_limit > 0 && CInode::count() > inode_limit*cache_health_threshold()) || (cache_size() > cache_limit_memory()*cache_health_threshold());
+    return (cache_inode_limit > 0 && CInode::count() > cache_inode_limit*cache_health_threshold) || (cache_size() > cache_memory_limit*cache_health_threshold);
   }
 
   void advance_stray() {
@@ -722,6 +717,9 @@ public:
  public:
   explicit MDCache(MDSRank *m, PurgeQueue &purge_queue_);
   ~MDCache();
+  void handle_conf_change(const ConfigProxy& conf,
+                          const std::set <std::string> &changed,
+                          const MDSMap &mds_map);
   
   // debug
   void log_stat();

diff --git a/src/mds/MDSDaemon.cc b/src/mds/MDSDaemon.cc
index 30ddc93e5c43c1cc92a44d40e79bffb739e872aa..49a2663b5e4714e24c28a12c6a2d93421aa65e5d 100644 (file)
--- a/src/mds/MDSDaemon.cc
+++ b/src/mds/MDSDaemon.cc
@@ -350,14 +350,24 @@ const char** MDSDaemon::get_tracked_conf_keys() const
     "clog_to_syslog",
     "clog_to_syslog_facility",
     "clog_to_syslog_level",
+    "clog_to_graylog",
+    "clog_to_graylog_host",
+    "clog_to_graylog_port",
+    // MDCache
+    "mds_cache_size",
+    "mds_cache_memory_limit",
+    "mds_cache_reservation",
+    "mds_health_cache_threshold",
+    "mds_cache_mid",
+    // MDBalancer
+    "mds_bal_fragment_dirs",
+    "mds_bal_fragment_interval",
     // PurgeQueue
     "mds_max_purge_ops",
     "mds_max_purge_ops_per_pg",
     "mds_max_purge_files",
+    // Migrator
     "mds_inject_migrator_session_race",
-    "clog_to_graylog",
-    "clog_to_graylog_host",
-    "clog_to_graylog_port",
     "host",
     "fsid",
     NULL

diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index 9d7b77dc2134edebd23681e982340cd62e12420a..2c3556df4f26d01448f68ba783074aaacf1b1b0a 100644 (file)
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -29,6 +29,7 @@
 #include "SnapClient.h"
 #include "SnapServer.h"
 #include "MDBalancer.h"
+#include "Migrator.h"
 #include "Locker.h"
 #include "Server.h"
 #include "InoTable.h"

diff --git a/src/mds/MDSRank.h b/src/mds/MDSRank.h
index dab99d5b6fefc94e4ced2281a1c9174e0c315da0..85448072b5d36bc038310421d0aa1bcd5487e016 100644 (file)
--- a/src/mds/MDSRank.h
+++ b/src/mds/MDSRank.h
@@ -29,7 +29,6 @@
 #include "MDSMap.h"
 #include "SessionMap.h"
 #include "MDCache.h"
-#include "Migrator.h"
 #include "MDLog.h"
 #include "PurgeQueue.h"
 #include "osdc/Journaler.h"
@@ -224,7 +223,7 @@ class MDSRank {
     void handle_conf_change(const ConfigProxy& conf,
                             const std::set <std::string> &changed)
     {
-      mdcache->migrator->handle_conf_change(conf, changed, *mdsmap);
+      mdcache->handle_conf_change(conf, changed, *mdsmap);
       purge_queue.handle_conf_change(conf, changed, *mdsmap);
     }