os/bluestore: Limit stall from evicting onodes

Limits how many onodes can be evicted from OnodeCacheShard in one go.
Added bluestore_cache_meta_drop_limit that controls how fast
onodes can be evicted from cache.

Fixes: https://tracker.ceph.com/issues/73353
Signed-off-by: Adam Kupczyk <akupczyk@ibm.com>

diff --git a/src/common/options/global.yaml.in b/src/common/options/global.yaml.in
index 5e583932b4c9c178f3351c89a7ac4f639e907ffd..2eb4191e0e273f897c33a98433e9c98300f1813f 100644 (file)
--- a/src/common/options/global.yaml.in
+++ b/src/common/options/global.yaml.in
@@ -5070,6 +5070,17 @@ options:
   default: 0.04
   see_also:
   - bluestore_cache_size
+- name: bluestore_cache_meta_evict_limit
+  type: int
+  level: advanced
+  desc: Elements in onode cache shards are evicted when new element is inserted into the cache.
+    In rare cases of cluster inactivity cache can be reduced, but not evicted. Adjusting size at once
+    will cause stall first time cache shard is accessed. The setting limits how many onodes can get evicted in one go.
+    Any value is less than 2 it is treated as request to adjust immediately.
+  default: 10
+  see_also:
+  - bluestore_cache_meta_ratio
+  with_legacy: true
 - name: bluestore_cache_autotune
   type: bool
   level: dev

diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index 211f2f9f903923e77a96f538beb1aa0b8b911cd5..ca5a9c40783b7c9f6f366b91355b6e1a1f7a9b62 100644 (file)
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -2071,7 +2071,7 @@ BlueStore::OnodeRef BlueStore::OnodeSpace::add_onode(const ghobject_t& oid,
   }
   ldout(cache->cct, 20) << __func__ << " " << oid << " " << o << dendl;
   cache->_add(o.get(), 1);
-  cache->_trim();
+  cache->_trim_some();
   return o;
 }
 
@@ -2157,7 +2157,7 @@ void BlueStore::OnodeSpace::rename(
 
   o->oid = new_oid;
   o->key = new_okey;
-  cache->_trim();
+  cache->_trim_some();
 }
 
 bool BlueStore::OnodeSpace::map_any(std::function<bool(Onode*)> f)

diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h
index c2793ae0382e288dcd2fdbc2fb7059ca694e9204..4e5b134c7a012ec5b9dfa89a29c453d64393b6b1 100644 (file)
--- a/src/os/bluestore/BlueStore.h
+++ b/src/os/bluestore/BlueStore.h
@@ -1510,10 +1510,17 @@ public:
       }
       _trim_to(max);
     }
-
+    void _trim_some() {
+      int32_t max_steps = cct->_conf->bluestore_cache_meta_evict_limit;
+      int64_t new_level = max.load();
+      if (max_steps >= 2) {
+        new_level = std::max((int64_t)num.load() - max_steps, new_level);
+      }
+      _trim_to(new_level);
+    }
     void trim() {
       std::lock_guard l(lock);
-      _trim();    
+      _trim();
     }
     void flush() {
       std::lock_guard l(lock);