mds: avoid using g_conf->get_val<...>(...) in hot path

g_conf->get_val<>()'s overhead is high

Fixes: http://tracker.ceph.com/issues/24820
Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
(cherry picked from commit d249581df98455ddc6d0ce21faa2db87b21fad85)

 Conflicts:
	src/common/legacy_config_opts.h
	src/mds/MDBalancer.cc
	src/mds/MDBalancer.h
	src/mds/MDCache.h

diff --git a/src/common/legacy_config_opts.h b/src/common/legacy_config_opts.h
index b77e75ea5e9b288e6d5cde743d5015d799ea3e37..ead9cb6557b8cff7b63489fcf08b459c24f5e47e 100644 (file)
--- a/src/common/legacy_config_opts.h
+++ b/src/common/legacy_config_opts.h
@@ -435,8 +435,6 @@ OPTION(mds_bal_split_rd, OPT_FLOAT)
 OPTION(mds_bal_split_wr, OPT_FLOAT)
 OPTION(mds_bal_split_bits, OPT_INT)
 OPTION(mds_bal_merge_size, OPT_INT)
-OPTION(mds_bal_interval, OPT_INT)           // seconds
-OPTION(mds_bal_fragment_interval, OPT_INT)      // seconds
 OPTION(mds_bal_fragment_size_max, OPT_INT) // order of magnitude higher than split size
 OPTION(mds_bal_fragment_fast_factor, OPT_FLOAT) // multiple of size_max that triggers immediate split
 OPTION(mds_bal_idle_threshold, OPT_FLOAT)

diff --git a/src/mds/Locker.cc b/src/mds/Locker.cc
index 09b17694c3b02213737d9343f70276e2c69e411b..699a535a40a5e1898ede05129e742958204febb8 100644 (file)
--- a/src/mds/Locker.cc
+++ b/src/mds/Locker.cc
@@ -18,6 +18,7 @@
 #include "MDCache.h"
 #include "Locker.h"
 #include "MDBalancer.h"
+#include "Migrator.h"
 #include "CInode.h"
 #include "CDir.h"
 #include "CDentry.h"

diff --git a/src/mds/MDBalancer.cc b/src/mds/MDBalancer.cc
index e6bf839b218b0c8976754a17fe231d25e389a25a..4c8c3946c3ad79376ff49b0a97bbce9d0546e5ef 100644 (file)
--- a/src/mds/MDBalancer.cc
+++ b/src/mds/MDBalancer.cc
@@ -15,9 +15,9 @@
 #include "include/compat.h"
 #include "mdstypes.h"
 
+#include "mon/MonClient.h"
 #include "MDBalancer.h"
 #include "MDSRank.h"
-#include "mon/MonClient.h"
 #include "MDSMap.h"
 #include "CInode.h"
 #include "CDir.h"
@@ -78,6 +78,23 @@ int MDBalancer::proc_message(Message *m)
   return 0;
 }
 
+MDBalancer::MDBalancer(MDSRank *m, Messenger *msgr, MonClient *monc) :
+    mds(m), messenger(msgr), mon_client(monc)
+{
+  bal_fragment_dirs = g_conf->get_val<bool>("mds_bal_fragment_dirs");
+  bal_fragment_interval = g_conf->get_val<int64_t>("mds_bal_fragment_interval");
+}
+
+void MDBalancer::handle_conf_change(const struct md_config_t *conf,
+                                   const std::set <std::string> &changed,
+                                   const MDSMap &mds_map)
+{
+  if (changed.count("mds_bal_fragment_dirs"))
+    bal_fragment_dirs = g_conf->get_val<bool>("mds_bal_fragment_dirs");
+  if (changed.count("mds_bal_fragment_interval"))
+    bal_fragment_interval = g_conf->get_val<int64_t>("mds_bal_fragment_interval");
+}
+
 void MDBalancer::handle_export_pins(void)
 {
   auto &q = mds->mdcache->export_pin_queue;
@@ -151,7 +168,8 @@ void MDBalancer::tick()
   static int num_bal_times = g_conf->mds_bal_max;
   static mono_time first = mono_clock::now();
   mono_time now = mono_clock::now();
-  ceph::timespan elapsed = now - first;
+  auto bal_interval = g_conf->get_val<int64_t>("mds_bal_interval");
+  auto bal_max_until = g_conf->get_val<int64_t>("mds_bal_max_until");
 
   if (g_conf->mds_bal_export_pin) {
     handle_export_pins();
@@ -168,14 +186,11 @@ void MDBalancer::tick()
   // because the values from g_conf are also integers.
   // balance?
   if (mds->get_nodeid() == 0 &&
-      g_conf->mds_bal_interval > 0 &&
-      (num_bal_times ||
-       (g_conf->mds_bal_max_until >= 0 &&
-        duration_cast<chrono::seconds>(elapsed).count() >
-          g_conf->mds_bal_max_until)) &&
       mds->is_active() &&
-      duration_cast<chrono::seconds>(now - last_heartbeat).count() >=
-       g_conf->mds_bal_interval) {
+      bal_interval > 0 &&
+      duration_cast<chrono::seconds>(now - last_heartbeat).count() >= bal_interval &&
+      (num_bal_times ||
+       (bal_max_until >= 0 && duration_cast<chrono::seconds>(now - first).count() > bal_max_until))) {
     last_heartbeat = now;
     send_heartbeat();
     num_bal_times--;
@@ -287,11 +302,9 @@ int MDBalancer::localize_balancer()
            << " oid=" << oid << " oloc=" << oloc << dendl;
 
   /* timeout: if we waste half our time waiting for RADOS, then abort! */
-  double t = ceph_clock_now() + g_conf->mds_bal_interval/2;
-  utime_t timeout;
-  timeout.set_from_double(t);
+  auto bal_interval = g_conf->get_val<int64_t>("mds_bal_interval");
   lock.Lock();
-  int ret_t = cond.WaitUntil(lock, timeout);
+  int ret_t = cond.WaitInterval(lock, utime_t(bal_interval / 2, 0));
   lock.Unlock();
 
   /* success: store the balancer in memory and set the version. */
@@ -510,7 +523,7 @@ void MDBalancer::queue_split(const CDir *dir, bool fast)
     // Set a timer to really do the split: we don't do it immediately
     // so that bursts of ops on a directory have a chance to go through
     // before we freeze it.
-    mds->timer.add_event_after(g_conf->mds_bal_fragment_interval,
+    mds->timer.add_event_after(bal_fragment_interval,
                                new FunctionContext(callback));
   }
 }
@@ -574,7 +587,7 @@ void MDBalancer::queue_merge(CDir *dir)
   if (merge_pending.count(frag) == 0) {
     dout(20) << __func__ << " enqueued dir " << *dir << dendl;
     merge_pending.insert(frag);
-    mds->timer.add_event_after(g_conf->mds_bal_fragment_interval,
+    mds->timer.add_event_after(bal_fragment_interval,
         new FunctionContext(callback));
   } else {
     dout(20) << __func__ << " dir already in queue " << *dir << dendl;
@@ -1118,10 +1131,8 @@ void MDBalancer::hit_inode(const utime_t& now, CInode *in, int type, int who)
 void MDBalancer::maybe_fragment(CDir *dir, bool hot)
 {
   // split/merge
-  if (mds->cct->_conf->get_val<bool>("mds_bal_fragment_dirs") &&
-      g_conf->mds_bal_fragment_interval > 0 &&
-      !dir->inode->is_base() &&        // not root/base (for now at least)
-      dir->is_auth()) {
+  if (bal_fragment_dirs && bal_fragment_interval > 0 &&
+      dir->is_auth() && !dir->inode->is_base()) {  // not root/base (for now at least)
 
     // split
     if (g_conf->mds_bal_split_size > 0 && (dir->should_split() || hot)) {

diff --git a/src/mds/MDBalancer.h b/src/mds/MDBalancer.h
index 3c842873aead776e08dbe0ac8000a8b045ddee0b..1f831b202c4ec5dc0ecf75b45d8d8e4f7f3b5c2f 100644 (file)
--- a/src/mds/MDBalancer.h
+++ b/src/mds/MDBalancer.h
@@ -26,6 +26,7 @@ using std::map;
 #include "common/Clock.h"
 #include "common/Cond.h"
 
+class MDSMap;
 class MDSRank;
 class Message;
 class MHeartbeat;
@@ -37,8 +38,11 @@ class MonClient;
 class MDBalancer {
   friend class C_Bal_SendHeartbeat;
 public:
-  MDBalancer(MDSRank *m, Messenger *msgr, MonClient *monc) : 
-    mds(m), messenger(msgr), mon_client(monc) { }
+  MDBalancer(MDSRank *m, Messenger *msgr, MonClient *monc);
+
+  void handle_conf_change(const struct md_config_t *conf,
+                          const std::set <std::string> &changed,
+                          const MDSMap &mds_map);
 
   int proc_message(Message *m);
 
@@ -72,6 +76,9 @@ public:
   int dump_loads(Formatter *f);
 
 private:
+  bool bal_fragment_dirs;
+  int64_t bal_fragment_interval;
+
   typedef struct {
     std::map<mds_rank_t, double> targets;
     std::map<mds_rank_t, double> imported;

diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index 41d77edbdd3d15031ec33c4cbdb2dd53f7f76a0a..cc8813471ec37cf178d05a0d43ede1514436ddd7 100644 (file)
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -207,7 +207,13 @@ MDCache::MDCache(MDSRank *m, PurgeQueue &purge_queue_) :
   cap_imports_num_opening = 0;
 
   opening_root = open = false;
-  lru.lru_set_midpoint(cache_mid());
+
+  cache_inode_limit = g_conf->get_val<int64_t>("mds_cache_size");
+  cache_memory_limit = g_conf->get_val<uint64_t>("mds_cache_memory_limit");
+  cache_reservation = g_conf->get_val<double>("mds_cache_reservation");
+  cache_health_threshold = g_conf->get_val<double>("mds_health_cache_threshold");
+
+  lru.lru_set_midpoint(g_conf->get_val<double>("mds_cache_mid"));
 
   bottom_lru.lru_set_midpoint(0);
 
@@ -225,11 +231,28 @@ MDCache::~MDCache()
   }
 }
 
+void MDCache::handle_conf_change(const struct md_config_t *conf,
+                                 const std::set <std::string> &changed,
+                                 const MDSMap &mdsmap)
+{
+  if (changed.count("mds_cache_size"))
+    cache_inode_limit = g_conf->get_val<int64_t>("mds_cache_size");
+  if (changed.count("mds_cache_memory_limit"))
+    cache_memory_limit = g_conf->get_val<uint64_t>("mds_cache_memory_limit");
+  if (changed.count("mds_cache_reservation"))
+    cache_reservation = g_conf->get_val<double>("mds_cache_reservation");
+  if (changed.count("mds_health_cache_threshold"))
+    cache_health_threshold = g_conf->get_val<double>("mds_health_cache_threshold");
+  if (changed.count("mds_cache_mid"))
+    lru.lru_set_midpoint(g_conf->get_val<double>("mds_cache_mid"));
 
+  migrator->handle_conf_change(conf, changed, mdsmap);
+  mds->balancer->handle_conf_change(conf, changed, mdsmap);
+}
 
 void MDCache::log_stat()
 {
-  mds->logger->set(l_mds_inode_max, cache_limit_inodes() == 0 ? INT_MAX : cache_limit_inodes());
+  mds->logger->set(l_mds_inode_max, cache_inode_limit ? : INT_MAX);
   mds->logger->set(l_mds_inodes, lru.lru_get_size());
   mds->logger->set(l_mds_inodes_pinned, lru.lru_get_num_pinned());
   mds->logger->set(l_mds_inodes_top, lru.lru_get_top());
@@ -6659,12 +6682,12 @@ void MDCache::trim_lru(uint64_t count, map<mds_rank_t, MCacheExpire*> &expiremap
 bool MDCache::trim(uint64_t count)
 {
   uint64_t used = cache_size();
-  uint64_t limit = cache_limit_memory();
+  uint64_t limit = cache_memory_limit;
   map<mds_rank_t, MCacheExpire*> expiremap;
 
   dout(7) << "trim bytes_used=" << bytes2str(used)
           << " limit=" << bytes2str(limit)
-          << " reservation=" << cache_reservation()
+          << " reservation=" << cache_reservation
           << "% count=" << count << dendl;
 
   // process delayed eval_stray()

diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h
index 76d05a09bc58b54016beb0eadc7247924bf54558..300f265bfe7c285dc3ec154360c51d93de6cc56d 100644 (file)
--- a/src/mds/MDCache.h
+++ b/src/mds/MDCache.h
@@ -150,27 +150,23 @@ class MDCache {
 
   bool exceeded_size_limit;
 
+private:
+  uint64_t cache_inode_limit;
+  uint64_t cache_memory_limit;
+  double cache_reservation;
+  double cache_health_threshold;
+
 public:
-  static uint64_t cache_limit_inodes(void) {
-    return g_conf->get_val<int64_t>("mds_cache_size");
-  }
-  static uint64_t cache_limit_memory(void) {
-    return g_conf->get_val<uint64_t>("mds_cache_memory_limit");
-  }
-  static double cache_reservation(void) {
-    return g_conf->get_val<double>("mds_cache_reservation");
-  }
-  static double cache_mid(void) {
-    return g_conf->get_val<double>("mds_cache_mid");
+  uint64_t cache_limit_inodes(void) {
+    return cache_inode_limit;
   }
-  static double cache_health_threshold(void) {
-    return g_conf->get_val<double>("mds_health_cache_threshold");
+  uint64_t cache_limit_memory(void) {
+    return cache_memory_limit;
   }
   double cache_toofull_ratio(void) const {
-    uint64_t inode_limit = cache_limit_inodes();
-    double inode_reserve = inode_limit*(1.0-cache_reservation());
-    double memory_reserve = cache_limit_memory()*(1.0-cache_reservation());
-    return fmax(0.0, fmax((cache_size()-memory_reserve)/memory_reserve, inode_limit == 0 ? 0.0 : (CInode::count()-inode_reserve)/inode_reserve));
+    double inode_reserve = cache_inode_limit*(1.0-cache_reservation);
+    double memory_reserve = cache_memory_limit*(1.0-cache_reservation);
+    return fmax(0.0, fmax((cache_size()-memory_reserve)/memory_reserve, cache_inode_limit == 0 ? 0.0 : (CInode::count()-inode_reserve)/inode_reserve));
   }
   bool cache_toofull(void) const {
     return cache_toofull_ratio() > 0.0;
@@ -179,8 +175,7 @@ public:
     return mempool::get_pool(mempool::mds_co::id).allocated_bytes();
   }
   bool cache_overfull(void) const {
-    uint64_t inode_limit = cache_limit_inodes();
-    return (inode_limit > 0 && CInode::count() > inode_limit*cache_health_threshold()) || (cache_size() > cache_limit_memory()*cache_health_threshold());
+    return (cache_inode_limit > 0 && CInode::count() > cache_inode_limit*cache_health_threshold) || (cache_size() > cache_memory_limit*cache_health_threshold);
   }
 
   void advance_stray() {
@@ -719,6 +714,9 @@ public:
  public:
   explicit MDCache(MDSRank *m, PurgeQueue &purge_queue_);
   ~MDCache();
+  void handle_conf_change(const struct md_config_t *conf,
+                          const std::set <std::string> &changed,
+                          const MDSMap &mds_map);
   
   // debug
   void log_stat();

diff --git a/src/mds/MDSDaemon.cc b/src/mds/MDSDaemon.cc
index 2ef0e4eca5346685f8a21901d09634571f3e13f1..9c60b9a133be60381749bed80990f59662baeca3 100644 (file)
--- a/src/mds/MDSDaemon.cc
+++ b/src/mds/MDSDaemon.cc
@@ -369,14 +369,24 @@ const char** MDSDaemon::get_tracked_conf_keys() const
     "clog_to_syslog",
     "clog_to_syslog_facility",
     "clog_to_syslog_level",
+    "clog_to_graylog",
+    "clog_to_graylog_host",
+    "clog_to_graylog_port",
+    // MDCache
+    "mds_cache_size",
+    "mds_cache_memory_limit",
+    "mds_cache_reservation",
+    "mds_health_cache_threshold",
+    "mds_cache_mid",
+    // MDBalancer
+    "mds_bal_fragment_dirs",
+    "mds_bal_fragment_interval",
     // PurgeQueue
     "mds_max_purge_ops",
     "mds_max_purge_ops_per_pg",
     "mds_max_purge_files",
+    // Migrator
     "mds_inject_migrator_session_race",
-    "clog_to_graylog",
-    "clog_to_graylog_host",
-    "clog_to_graylog_port",
     "host",
     "fsid",
     NULL

diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index 93a0f4839f29af3e398682bfdc7330eb6fdfa28c..0864b6ee90b1430f9c4428794fb007d6685fed45 100644 (file)
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -29,6 +29,7 @@
 #include "SnapClient.h"
 #include "SnapServer.h"
 #include "MDBalancer.h"
+#include "Migrator.h"
 #include "Locker.h"
 #include "Server.h"
 #include "InoTable.h"

diff --git a/src/mds/MDSRank.h b/src/mds/MDSRank.h
index 60a624bebf86e9867b7f3df60d636a9e3ac7272d..96e0b1c8bec451351bbfc4bd515ba5a68b1b5c04 100644 (file)
--- a/src/mds/MDSRank.h
+++ b/src/mds/MDSRank.h
@@ -29,7 +29,6 @@
 #include "MDSMap.h"
 #include "SessionMap.h"
 #include "MDCache.h"
-#include "Migrator.h"
 #include "MDLog.h"
 #include "PurgeQueue.h"
 #include "osdc/Journaler.h"
@@ -224,7 +223,7 @@ class MDSRank {
     void handle_conf_change(const struct md_config_t *conf,
                             const std::set <std::string> &changed)
     {
-      mdcache->migrator->handle_conf_change(conf, changed, *mdsmap);
+      mdcache->handle_conf_change(conf, changed, *mdsmap);
       purge_queue.handle_conf_change(conf, changed, *mdsmap);
     }