os/bluestore: differ default cache size for hdd/ssd backends

This is a follow-up change of https://github.com/ceph/ceph/pull/15976
and makes the bluestore cache capacity being self-adaptive for
different backends.

Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index 1500def7ba972a13b1caa3b01f6d3edc64de947d..41707db1c72ab53784c451c224c09c2d41c8a763 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -1163,10 +1163,12 @@ OPTION(bluestore_cache_trim_max_skip_pinned, OPT_U32, 64) // skip this many onod
 OPTION(bluestore_cache_type, OPT_STR, "2q")   // lru, 2q
 OPTION(bluestore_2q_cache_kin_ratio, OPT_DOUBLE, .5)    // kin page slot size / max page slot size
 OPTION(bluestore_2q_cache_kout_ratio, OPT_DOUBLE, .5)   // number of kout page slot / total number of page slot
-OPTION(bluestore_cache_size, OPT_U64, 3*1024*1024*1024)
+OPTION(bluestore_cache_size, OPT_U64, 0)
+OPTION(bluestore_cache_size_hdd, OPT_U64, 1*1024*1024*1024)
+OPTION(bluestore_cache_size_ssd, OPT_U64, 3*1024*1024*1024)
 OPTION(bluestore_cache_meta_ratio, OPT_DOUBLE, .01)
 OPTION(bluestore_cache_kv_ratio, OPT_DOUBLE, .99)
-OPTION(bluestore_cache_kv_max, OPT_U64, 512*1024*1024) // limit the maximum amont of cache for the kv store
+OPTION(bluestore_cache_kv_max, OPT_U64, 512*1024*1024) // limit the maximum amount of cache for the kv store
 OPTION(bluestore_kvbackend, OPT_STR, "rocksdb")
 OPTION(bluestore_allocator, OPT_STR, "bitmap")     // stupid | bitmap
 OPTION(bluestore_freelist_blocks_per_key, OPT_INT, 128)

diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index 263c89aaf600916732642ad6ce0d64bafc4b8db4..19bfe4113e0e954559d0ef0b1c34054404e22d4d 100644 (file)
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -3316,7 +3316,7 @@ void *BlueStore::MempoolThread::entry()
     size_t num_shards = store->cache_shards.size();
     float target_ratio = store->cache_meta_ratio + store->cache_data_ratio;
     // A little sloppy but should be close enough
-    uint64_t shard_target = target_ratio * (store->cct->_conf->bluestore_cache_size / num_shards);
+    uint64_t shard_target = target_ratio * (store->cache_size / num_shards);
 
     for (auto i : store->cache_shards) {
       i->trim(shard_target,
@@ -3689,10 +3689,20 @@ void BlueStore::_set_blob_size()
 
 int BlueStore::_set_cache_sizes()
 {
+  assert(bdev);
+  if (cct->_conf->bluestore_cache_size) {
+    cache_size = cct->_conf->bluestore_cache_size;
+  } else {
+    // choose global cache size based on backend type
+    if (bdev->is_rotational()) {
+      cache_size = cct->_conf->bluestore_cache_size_hdd;
+    } else {
+      cache_size = cct->_conf->bluestore_cache_size_ssd;
+    }
+  }
   cache_meta_ratio = cct->_conf->bluestore_cache_meta_ratio;
   cache_kv_ratio = cct->_conf->bluestore_cache_kv_ratio;
 
-  double cache_size = cct->_conf->bluestore_cache_size;
   double cache_kv_max = cct->_conf->bluestore_cache_kv_max;
   double cache_kv_max_ratio = 0;
 
@@ -3732,7 +3742,8 @@ int BlueStore::_set_cache_sizes()
     // deal with floating point imprecision
     cache_data_ratio = 0;
   }
-  dout(1) << __func__ << " meta " << cache_meta_ratio
+  dout(1) << __func__ << " cache_size " << cache_size
+          << " meta " << cache_meta_ratio
          << " kv " << cache_kv_ratio
          << " data " << cache_data_ratio
          << dendl;
@@ -3914,12 +3925,6 @@ int BlueStore::get_block_device_fsid(CephContext* cct, const string& path,
 
 int BlueStore::_open_path()
 {
-  // initial sanity check
-  int r = _set_cache_sizes();
-  if (r < 0) {
-    return r;
-  }
-
   assert(path_fd < 0);
   path_fd = ::open(path.c_str(), O_DIRECTORY);
   if (path_fd < 0) {
@@ -4093,6 +4098,11 @@ int BlueStore::_open_bdev(bool create)
   block_mask = ~(block_size - 1);
   block_size_order = ctz(block_size);
   assert(block_size == 1u << block_size_order);
+  // and set cache_size based on device type
+  r = _set_cache_sizes();
+  if (r < 0) {
+    goto fail_close;
+  }
   return 0;
 
  fail_close:
@@ -4640,7 +4650,7 @@ int BlueStore::_open_db(bool create)
   FreelistManager::setup_merge_operators(db);
   db->set_merge_operator(PREFIX_STAT, merge_op);
 
-  db->set_cache_size(cct->_conf->bluestore_cache_size * cache_kv_ratio);
+  db->set_cache_size(cache_size * cache_kv_ratio);
 
   if (kv_backend == "rocksdb")
     options = cct->_conf->bluestore_rocksdb_options;

diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h
index 808eaad1a77db4afd2d90805654adbc90d341f40..e60f906b3977ee65775b6f4f931b35d6a2528d90 100644 (file)
--- a/src/os/bluestore/BlueStore.h
+++ b/src/os/bluestore/BlueStore.h
@@ -1917,6 +1917,7 @@ private:
   uint64_t kv_throttle_costs = 0;
 
   // cache trim control
+  uint64_t cache_size = 0;      ///< total cache size
   float cache_meta_ratio = 0;   ///< cache ratio dedicated to metadata
   float cache_kv_ratio = 0;     ///< cache ratio dedicated to kv (e.g., rocksdb)
   float cache_data_ratio = 0;   ///< cache ratio dedicated to object data

diff --git a/src/test/objectstore/store_test.cc b/src/test/objectstore/store_test.cc
index a51da4ab2df25c7c45d9699f97cf826f52bcb7a7..8980aac37c4f1ca875b0b11c79f9f14cbd63579b 100644 (file)
--- a/src/test/objectstore/store_test.cc
+++ b/src/test/objectstore/store_test.cc
@@ -5765,7 +5765,8 @@ TEST_P(StoreTestSpecificAUSize, OnodeSizeTracking) {
   StartDeferred(block_size);
   g_conf->set_val("bluestore_compression_mode", "none");
   g_conf->set_val("bluestore_csum_type", "none");
-  g_conf->set_val("bluestore_cache_size", "400000000");
+  g_conf->set_val("bluestore_cache_size_hdd", "400000000");
+  g_conf->set_val("bluestore_cache_size_ssd", "400000000");
   g_conf->apply_changes(NULL);
 
   ObjectStore::Sequencer osr("test");
@@ -5853,7 +5854,8 @@ TEST_P(StoreTestSpecificAUSize, OnodeSizeTracking) {
     r = apply_transaction(store, &osr, std::move(t));
     ASSERT_EQ(r, 0);
   }
-  g_ceph_context->_conf->set_val("bluestore_cache_size", "4000000");
+  g_ceph_context->_conf->set_val("bluestore_cache_size_hdd", "4000000");
+  g_ceph_context->_conf->set_val("bluestore_cache_size_ssd", "4000000");
   g_conf->set_val("bluestore_compression_mode", "none");
   g_conf->set_val("bluestore_csum_type", "crc32c");
 
@@ -6656,7 +6658,8 @@ int main(int argc, char **argv) {
   g_ceph_context->_conf->set_val("bluestore_max_alloc_size", "196608");
 
   // set small cache sizes so we see trimming during Synthetic tests
-  g_ceph_context->_conf->set_val("bluestore_cache_size", "4000000");
+  g_ceph_context->_conf->set_val("bluestore_cache_size_hdd", "4000000");
+  g_ceph_context->_conf->set_val("bluestore_cache_size_ssd", "4000000");
 
   // very short *_max prealloc so that we fall back to async submits
   g_ceph_context->_conf->set_val("bluestore_blobid_prealloc", "10");