os/bluestore: Add config observer for osd memory specific options.

Add config observer to enable changes to the following osd memory specific
options at runtime,
 - osd_memory_target
 - osd_memory_base
 - osd_memory_cache_min
 - osd_memory_expected_fragmentation

Signed-off-by: Sridhar Seshasayee <sseshasa@redhat.com>

diff --git a/src/common/options.cc b/src/common/options.cc
index b58acbc8889b03f3232e49d2dd71a47a43cd30b8..ec8fdd97cb0d139c4dc1c243adb8721fa158fdd9 100644 (file)
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -4159,8 +4159,13 @@ std::vector<Option> get_global_options() {
 
     Option("osd_memory_target", Option::TYPE_SIZE, Option::LEVEL_BASIC)
     .set_default(4_G)
+    .set_min(896_M)
+    .set_flag(Option::FLAG_RUNTIME)
     .add_see_also("bluestore_cache_autotune")
-    .set_description("When tcmalloc and cache autotuning is enabled, try to keep this many bytes mapped in memory."),
+    .add_see_also("osd_memory_cache_min")
+    .add_see_also("osd_memory_base")
+    .set_description("When tcmalloc and cache autotuning is enabled, try to keep this many bytes mapped in memory.")
+    .set_long_description("The minimum value must be at least equal to osd_memory_base + osd_memory_cache_min."),
 
     Option("osd_memory_target_cgroup_limit_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
     .set_default(0.8)
@@ -4171,17 +4176,21 @@ std::vector<Option> get_global_options() {
 
     Option("osd_memory_base", Option::TYPE_SIZE, Option::LEVEL_DEV)
     .set_default(768_M)
+    .set_flag(Option::FLAG_RUNTIME)
     .add_see_also("bluestore_cache_autotune")
     .set_description("When tcmalloc and cache autotuning is enabled, estimate the minimum amount of memory in bytes the OSD will need."),
 
     Option("osd_memory_expected_fragmentation", Option::TYPE_FLOAT, Option::LEVEL_DEV)
     .set_default(0.15)
     .set_min_max(0.0, 1.0)
+    .set_flag(Option::FLAG_RUNTIME)
     .add_see_also("bluestore_cache_autotune")
     .set_description("When tcmalloc and cache autotuning is enabled, estimate the percent of memory fragmentation."),
 
     Option("osd_memory_cache_min", Option::TYPE_SIZE, Option::LEVEL_DEV)
     .set_default(128_M)
+    .set_min(128_M)
+    .set_flag(Option::FLAG_RUNTIME)
     .add_see_also("bluestore_cache_autotune")
     .set_description("When tcmalloc and cache autotuning is enabled, set the minimum amount of memory used for caches."),
 

diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index a936d445be59b14b88be94267836e5aec724b839..51333deddb97cfa309ebdb150d0f035699c35b0a 100644 (file)
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -3734,6 +3734,7 @@ void *BlueStore::MempoolThread::entry()
 {
   std::unique_lock l{lock};
 
+  uint32_t prev_config_change = store->config_changed.load();
   uint64_t base = store->osd_memory_base;
   double fragmentation = store->osd_memory_expected_fragmentation;
   uint64_t target = store->osd_memory_target;
@@ -3763,6 +3764,13 @@ void *BlueStore::MempoolThread::entry()
 
   bool interval_stats_trim = false;
   while (!stop) {
+    // Update pcm cache settings if related configuration was changed
+    uint32_t cur_config_change = store->config_changed.load();
+    if (cur_config_change != prev_config_change) {
+      _update_cache_settings();
+      prev_config_change = cur_config_change;
+    }
+
     // Before we trim, check and see if it's time to rebalance/resize.
     double autotune_interval = store->cache_autotune_interval;
     double resize_interval = store->osd_memory_cache_resize_interval;
@@ -3878,6 +3886,36 @@ void BlueStore::MempoolThread::_resize_shards(bool interval_stats)
   }
 }
 
+void BlueStore::MempoolThread::_update_cache_settings()
+{
+  // Nothing to do if pcm is not used.
+  if (pcm == nullptr) {
+    return;
+  }
+
+  auto cct = store->cct;
+  uint64_t target = store->osd_memory_target;
+  uint64_t base = store->osd_memory_base;
+  uint64_t min = store->osd_memory_cache_min;
+  uint64_t max = min;
+  double fragmentation = store->osd_memory_expected_fragmentation;
+
+  uint64_t ltarget = (1.0 - fragmentation) * target;
+  if (ltarget > base + min) {
+    max = ltarget - base;
+  }
+
+  // set pcm cache levels
+  pcm->set_target_memory(target);
+  pcm->set_min_memory(min);
+  pcm->set_max_memory(max);
+
+  ldout(cct, 5) << __func__  << " updated pcm target: " << target
+                << " pcm min: " << min
+                << " pcm max: " << max
+                << dendl;
+}
+
 // =======================================================
 
 // OmapIteratorImpl
@@ -4143,6 +4181,7 @@ const char **BlueStore::get_tracked_conf_keys() const
     "osd_memory_target_cgroup_limit_ratio",
     "osd_memory_base",
     "osd_memory_cache_min",
+    "osd_memory_expected_fragmentation",
     "bluestore_cache_autotune",
     "bluestore_cache_autotune_interval",
     "bluestore_no_per_pool_stats_tolerance",
@@ -4217,6 +4256,12 @@ void BlueStore::handle_conf_change(const ConfigProxy& conf,
       _set_max_defer_interval();
     }
   }
+  if (changed.count("osd_memory_target") ||
+      changed.count("osd_memory_base") ||
+      changed.count("osd_memory_cache_min") ||
+      changed.count("osd_memory_expected_fragmentation")) {
+    _update_osd_memory_options();
+  }
 }
 
 void BlueStore::_set_compression()
@@ -4321,6 +4366,21 @@ void BlueStore::_set_blob_size()
            << std::dec << dendl;
 }
 
+void BlueStore::_update_osd_memory_options()
+{
+  osd_memory_target = cct->_conf.get_val<Option::size_t>("osd_memory_target");
+  osd_memory_base = cct->_conf.get_val<Option::size_t>("osd_memory_base");
+  osd_memory_expected_fragmentation = cct->_conf.get_val<double>("osd_memory_expected_fragmentation");
+  osd_memory_cache_min = cct->_conf.get_val<Option::size_t>("osd_memory_cache_min");
+  config_changed++;
+  dout(10) << __func__
+           << " osd_memory_target " << osd_memory_target
+           << " osd_memory_base " << osd_memory_base
+           << " osd_memory_expected_fragmentation " << osd_memory_expected_fragmentation
+           << " osd_memory_cache_min " << osd_memory_cache_min
+           << dendl;
+}
+
 int BlueStore::_set_cache_sizes()
 {
   ceph_assert(bdev);

diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h
index c3fe59c5859bdd8b07f05f0282682e1b94bc1fab..9df11838c9d6ee8f01369c75e44fa074f8629733 100644 (file)
--- a/src/os/bluestore/BlueStore.h
+++ b/src/os/bluestore/BlueStore.h
@@ -1899,6 +1899,7 @@ private:
   uint64_t osd_memory_cache_min = 0; ///< Min memory to assign when autotuning cache
   double osd_memory_cache_resize_interval = 0; ///< Time to wait between cache resizing 
   double max_defer_interval = 0; ///< Time to wait between last deferred submit
+  std::atomic<uint32_t> config_changed = {0}; ///< Counter to determine if there is a configuration change.
 
   typedef map<uint64_t, volatile_statfs> osd_pools_map;
 
@@ -2041,6 +2042,7 @@ private:
 
   private:
     void _adjust_cache_settings();
+    void _update_cache_settings();
     void _resize_shards(bool interval_stats);
   } mempool_thread;
 
@@ -2061,6 +2063,7 @@ private:
   void _set_alloc_sizes();
   void _set_blob_size();
   void _set_finisher_num();
+  void _update_osd_memory_options();
 
   int _open_bdev(bool create);
   // Verifies if disk space is enough for reserved + min bluefs