From 91c99b8550bac0b0404ee9fdef51aef9306eb634 Mon Sep 17 00:00:00 2001
From: Sridhar Seshasayee <sseshasa@redhat.com>
Date: Thu, 8 Aug 2019 16:28:01 +0530
Subject: [PATCH] os/bluestore: Add config observer for osd memory specific
 options.

Add config observer to enable changes to the following osd memory specific
options at runtime,
 - osd_memory_target
 - osd_memory_base
 - osd_memory_cache_min
 - osd_memory_expected_fragmentation

Signed-off-by: Sridhar Seshasayee <sseshasa@redhat.com>
(cherry picked from commit 0d3609a2b7a88ae236335df9ea5d9854aa4de7ad)

 Conflicts:
	src/os/bluestore/BlueStore.cc
- removed config changed detection of bluestore_max_defer_interval
  as it is not present in nautilus
	src/os/bluestore/BlueStore.h
- removed declaration of max_defer_interval.
---
 PendingReleaseNotes           | 15 +++++++++
 src/common/options.cc         | 11 ++++++-
 src/os/bluestore/BlueStore.cc | 60 +++++++++++++++++++++++++++++++++++
 src/os/bluestore/BlueStore.h  |  3 ++
 4 files changed, 88 insertions(+), 1 deletion(-)

diff --git a/PendingReleaseNotes b/PendingReleaseNotes
index d4fb7f9810ad..897f41853319 100644
--- a/PendingReleaseNotes
+++ b/PendingReleaseNotes
@@ -116,3 +116,18 @@
 
 * New OSD daemon command dump_scrub_reservations which reveals the
   scrub reservations that are held for local (primary) and remote (replica) PGs.
+
+14.2.6
+------
+
+* The following OSD memory config options related to bluestore cache autotuning can now
+  be configured during runtime:
+
+    - osd_memory_base (default: 768 MB)
+    - osd_memory_cache_min (default: 128 MB)
+    - osd_memory_expected_fragmentation (default: 0.15)
+    - osd_memory_target (default: 4 GB)
+
+  The above options can be set with::
+
+    ceph config set global <option> <value>
diff --git a/src/common/options.cc b/src/common/options.cc
index cc10924dc511..d5b83a833800 100644
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -4187,8 +4187,13 @@ std::vector<Option> get_global_options() {
 
     Option("osd_memory_target", Option::TYPE_SIZE, Option::LEVEL_BASIC)
     .set_default(4_G)
+    .set_min(896_M)
+    .set_flag(Option::FLAG_RUNTIME)
     .add_see_also("bluestore_cache_autotune")
-    .set_description("When tcmalloc and cache autotuning is enabled, try to keep this many bytes mapped in memory."),
+    .add_see_also("osd_memory_cache_min")
+    .add_see_also("osd_memory_base")
+    .set_description("When tcmalloc and cache autotuning is enabled, try to keep this many bytes mapped in memory.")
+    .set_long_description("The minimum value must be at least equal to osd_memory_base + osd_memory_cache_min."),
 
     Option("osd_memory_target_cgroup_limit_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
     .set_default(0.8)
@@ -4199,17 +4204,21 @@ std::vector<Option> get_global_options() {
 
     Option("osd_memory_base", Option::TYPE_SIZE, Option::LEVEL_DEV)
     .set_default(768_M)
+    .set_flag(Option::FLAG_RUNTIME)
     .add_see_also("bluestore_cache_autotune")
     .set_description("When tcmalloc and cache autotuning is enabled, estimate the minimum amount of memory in bytes the OSD will need."),
 
     Option("osd_memory_expected_fragmentation", Option::TYPE_FLOAT, Option::LEVEL_DEV)
     .set_default(0.15)
     .set_min_max(0.0, 1.0)
+    .set_flag(Option::FLAG_RUNTIME)
     .add_see_also("bluestore_cache_autotune")
     .set_description("When tcmalloc and cache autotuning is enabled, estimate the percent of memory fragmentation."),
 
     Option("osd_memory_cache_min", Option::TYPE_SIZE, Option::LEVEL_DEV)
     .set_default(128_M)
+    .set_min(128_M)
+    .set_flag(Option::FLAG_RUNTIME)
     .add_see_also("bluestore_cache_autotune")
     .set_description("When tcmalloc and cache autotuning is enabled, set the minimum amount of memory used for caches."),
 
diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index 563919f1afe3..5a5f7160aefc 100644
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -3553,6 +3553,7 @@ void *BlueStore::MempoolThread::entry()
 {
   std::unique_lock l(lock);
 
+  uint32_t prev_config_change = store->config_changed.load();
   uint64_t base = store->osd_memory_base;
   double fragmentation = store->osd_memory_expected_fragmentation;
   uint64_t target = store->osd_memory_target;
@@ -3581,6 +3582,13 @@ void *BlueStore::MempoolThread::entry()
 
   bool interval_stats_trim = false;
   while (!stop) {
+    // Update pcm cache settings if related configuration was changed
+    uint32_t cur_config_change = store->config_changed.load();
+    if (cur_config_change != prev_config_change) {
+      _update_cache_settings();
+      prev_config_change = cur_config_change;
+    }
+
     // Before we trim, check and see if it's time to rebalance/resize.
     double autotune_interval = store->cache_autotune_interval;
     double resize_interval = store->osd_memory_cache_resize_interval;
@@ -3682,6 +3690,36 @@ void BlueStore::MempoolThread::_trim_shards(bool interval_stats)
   }
 }
 
+void BlueStore::MempoolThread::_update_cache_settings()
+{
+  // Nothing to do if pcm is not used.
+  if (pcm == nullptr) {
+    return;
+  }
+
+  auto cct = store->cct;
+  uint64_t target = store->osd_memory_target;
+  uint64_t base = store->osd_memory_base;
+  uint64_t min = store->osd_memory_cache_min;
+  uint64_t max = min;
+  double fragmentation = store->osd_memory_expected_fragmentation;
+
+  uint64_t ltarget = (1.0 - fragmentation) * target;
+  if (ltarget > base + min) {
+    max = ltarget - base;
+  }
+
+  // set pcm cache levels
+  pcm->set_target_memory(target);
+  pcm->set_min_memory(min);
+  pcm->set_max_memory(max);
+
+  ldout(cct, 5) << __func__  << " updated pcm target: " << target
+                << " pcm min: " << min
+                << " pcm max: " << max
+                << dendl;
+}
+
 // =======================================================
 
 // OmapIteratorImpl
@@ -3943,6 +3981,7 @@ const char **BlueStore::get_tracked_conf_keys() const
     "osd_memory_target_cgroup_limit_ratio",
     "osd_memory_base",
     "osd_memory_cache_min",
+    "osd_memory_expected_fragmentation",
     "bluestore_cache_autotune",
     "bluestore_cache_autotune_interval",
     "bluestore_warn_on_legacy_statfs",
@@ -4005,6 +4044,12 @@ void BlueStore::handle_conf_change(const ConfigProxy& conf,
     throttle_deferred_bytes.reset_max(
       conf->bluestore_throttle_bytes + conf->bluestore_throttle_deferred_bytes);
   }
+  if (changed.count("osd_memory_target") ||
+      changed.count("osd_memory_base") ||
+      changed.count("osd_memory_cache_min") ||
+      changed.count("osd_memory_expected_fragmentation")) {
+    _update_osd_memory_options();
+  }
 }
 
 void BlueStore::_set_compression()
@@ -4109,6 +4154,21 @@ void BlueStore::_set_blob_size()
            << std::dec << dendl;
 }
 
+void BlueStore::_update_osd_memory_options()
+{
+  osd_memory_target = cct->_conf.get_val<Option::size_t>("osd_memory_target");
+  osd_memory_base = cct->_conf.get_val<Option::size_t>("osd_memory_base");
+  osd_memory_expected_fragmentation = cct->_conf.get_val<double>("osd_memory_expected_fragmentation");
+  osd_memory_cache_min = cct->_conf.get_val<Option::size_t>("osd_memory_cache_min");
+  config_changed++;
+  dout(10) << __func__
+           << " osd_memory_target " << osd_memory_target
+           << " osd_memory_base " << osd_memory_base
+           << " osd_memory_expected_fragmentation " << osd_memory_expected_fragmentation
+           << " osd_memory_cache_min " << osd_memory_cache_min
+           << dendl;
+}
+
 int BlueStore::_set_cache_sizes()
 {
   ceph_assert(bdev);
diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h
index 374034ace2d5..97315f0eabf3 100644
--- a/src/os/bluestore/BlueStore.h
+++ b/src/os/bluestore/BlueStore.h
@@ -2033,6 +2033,7 @@ private:
   double osd_memory_expected_fragmentation = 0; ///< expected memory fragmentation
   uint64_t osd_memory_cache_min = 0; ///< Min memory to assign when autotuning cache
   double osd_memory_cache_resize_interval = 0; ///< Time to wait between cache resizing 
+  std::atomic<uint32_t> config_changed = {0}; ///< Counter to determine if there is a configuration change.
 
   typedef map<uint64_t, volatile_statfs> osd_pools_map;
 
@@ -2184,6 +2185,7 @@ private:
         int64_t *mem_avail, 
         const std::list<std::shared_ptr<PriorityCache::PriCache>>& caches, 
         PriorityCache::Priority pri);
+    void _update_cache_settings();
   } mempool_thread;
 
   // --------------------------------------------------------
@@ -2203,6 +2205,7 @@ private:
   void _set_alloc_sizes();
   void _set_blob_size();
   void _set_finisher_num();
+  void _update_osd_memory_options();
 
   int _open_bdev(bool create);
   // Verifies if disk space is enough for reserved + min bluefs
-- 
2.47.3