From: Mark Kogan Date: Mon, 15 Sep 2025 13:36:37 +0000 (+0000) Subject: rgw/d4n: refactor unused conf gw_d4n_l1_datacache_size X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=1336e393134dd97b4933d406855b4b35049d986f;p=ceph.git rgw/d4n: refactor unused conf gw_d4n_l1_datacache_size to the amount of disk space to keep free in reserve so that the datacache by default will not fill the drive completely Signed-off-by: Mark Kogan --- diff --git a/doc/radosgw/config-ref.rst b/doc/radosgw/config-ref.rst index 0e2bc9d6f6cb..603f6fa76b37 100644 --- a/doc/radosgw/config-ref.rst +++ b/doc/radosgw/config-ref.rst @@ -326,7 +326,7 @@ below. .. confval:: rgw_d4n_address .. confval:: rgw_d4n_l1_datacache_persistent_path -.. confval:: rgw_d4n_l1_datacache_size +.. confval:: rgw_d4n_l1_datacache_disk_reserve .. confval:: rgw_d4n_l1_evict_cache_on_start .. confval:: rgw_d4n_l1_fadvise .. confval:: rgw_d4n_libaio_aio_threads diff --git a/src/common/options/rgw.yaml.in b/src/common/options/rgw.yaml.in index ffec3bacb248..ee659cb2ac13 100644 --- a/src/common/options/rgw.yaml.in +++ b/src/common/options/rgw.yaml.in @@ -3894,12 +3894,12 @@ options: services: - rgw with_legacy: true -- name: rgw_d4n_l1_datacache_size +- name: rgw_d4n_l1_datacache_disk_reserve type: size level: advanced - desc: maximum size on disk for datacache - long_desc: The local SSD cache uses this option to configure its size in bytes. This - option is not used by the Redis cache backend. + desc: amount of disk space to keep free for datacache + long_desc: The local SSD cache uses this option to determine how much disk space to + reserve. The cache can grow until disk usage reaches (total disk space - reserved space). default: 1_G services: - rgw diff --git a/src/rgw/driver/d4n/rgw_sal_d4n.cc b/src/rgw/driver/d4n/rgw_sal_d4n.cc index 1b118726f1e5..fca255368b55 100644 --- a/src/rgw/driver/d4n/rgw_sal_d4n.cc +++ b/src/rgw/driver/d4n/rgw_sal_d4n.cc @@ -46,7 +46,7 @@ D4NFilterDriver::D4NFilterDriver(Driver* _next, boost::asio::io_context& io_cont partition_info.location = g_conf()->rgw_d4n_l1_datacache_persistent_path; partition_info.name = "d4n"; partition_info.type = "read-cache"; - partition_info.size = g_conf()->rgw_d4n_l1_datacache_size; + partition_info.reserve_size = g_conf()->rgw_d4n_l1_datacache_disk_reserve; cacheDriver = std::make_unique(partition_info, admin); } diff --git a/src/rgw/rgw_cache_driver.h b/src/rgw/rgw_cache_driver.h index 2ca91538fb95..6c02f24d3496 100644 --- a/src/rgw/rgw_cache_driver.h +++ b/src/rgw/rgw_cache_driver.h @@ -32,7 +32,7 @@ struct Partition { std::string name; std::string type; std::string location; - uint64_t size; + uint64_t reserve_size; }; class CacheDriver { diff --git a/src/rgw/rgw_redis_driver.cc b/src/rgw/rgw_redis_driver.cc index 3fb93583b61a..ee71e61cc7fc 100644 --- a/src/rgw/rgw_redis_driver.cc +++ b/src/rgw/rgw_redis_driver.cc @@ -65,6 +65,75 @@ void redis_exec(std::shared_ptr conn, } } +std::optional RedisDriver::resolve_valkey_data_dir(const DoutPrefixProvider* dpp) const +{ + try { + boost::system::error_code ec; + response> resp; + request req; + req.push("CONFIG", "GET", "dir"); + + redis_exec(conn, ec, req, resp, null_yield); + + if (ec) { + ldpp_dout(dpp, 5) << "RedisDriver::" << __func__ + << "(): failed to execute CONFIG GET dir: " << ec.message() + << dendl; + return std::nullopt; + } + + const auto& entries = std::get<0>(resp); + std::clog << "MK| OK " << __FILE__ << " :" << __LINE__ << " | " << __func__ << "(): entries.value().size()=" << entries.value().size() << std::endl; + if (entries.value().size() < 2) { + ldpp_dout(dpp, 5) << "RedisDriver::" << __func__ + << "(): unexpected CONFIG GET dir response size=" << entries.value().size() + << dendl; + return std::nullopt; + } + + const fs::path dir_path(entries.value()[1]); + std::clog << "MK| OK " << __FILE__ << " :" << __LINE__ << " | " << __func__ << "(): dir_path=" << std::quoted(dir_path.string()) << std::endl; + if (dir_path.empty()) { + ldpp_dout(dpp, 5) << "RedisDriver::" << __func__ + << "(): CONFIG GET dir returned empty path" << dendl; + return std::nullopt; + } + + return dir_path; + } catch (const std::exception& e) { + ldpp_dout(dpp, 0) << "RedisDriver::" << __func__ + << "(): exception while resolving data dir: " << e.what() + << dendl; + } + + return std::nullopt; +} + +uint64_t RedisDriver::get_free_space(const DoutPrefixProvider* dpp) +{ + auto data_dir = resolve_valkey_data_dir(dpp); + if (!data_dir) { + ldpp_dout(dpp, 0) << __func__ << "(): ERROR: could not resolve redis data dir" << dendl; + return 0; + } + + const fs::path redis_probe_path = *data_dir; + ldpp_dout(dpp, 20) << __func__ << "(): redis path = " << std::quoted(redis_probe_path.string()) << dendl; + + std::error_code ec; + fs::space_info space = fs::space(redis_probe_path, ec); + if (ec) { + ldpp_dout(dpp, 0) << __func__ << "(): ERROR: unable to stat redis path " + << std::quoted(redis_probe_path.string()) << " : " << ec.message() << dendl; + return 0; + } + + ldpp_dout(dpp, 20) << __func__ << "(): redis partition space.available=" << space.available + << ", partition_info.reserve_size=" << partition_info.reserve_size << dendl; + + return (space.available < partition_info.reserve_size) ? 0 : (space.available - partition_info.reserve_size); +} + int RedisDriver::initialize(const DoutPrefixProvider* dpp) { if (partition_info.location.back() != '/') { diff --git a/src/rgw/rgw_redis_driver.h b/src/rgw/rgw_redis_driver.h index 7889e8fc7d77..0e018d625619 100644 --- a/src/rgw/rgw_redis_driver.h +++ b/src/rgw/rgw_redis_driver.h @@ -1,6 +1,9 @@ #pragma once #include +#include +#include +#include #include #include "common/async/completion.h" @@ -10,6 +13,7 @@ namespace rgw { namespace cache { namespace net = boost::asio; +namespace fs = std::filesystem; using boost::redis::config; using boost::redis::connection; using boost::redis::request; @@ -19,7 +23,7 @@ using boost::redis::ignore_t; class RedisDriver : public CacheDriver { public: RedisDriver(net::io_context& io_context, Partition& _partition_info) : partition_info(_partition_info), - free_space(_partition_info.size), + free_space(0), outstanding_write_size(0) { conn = std::make_shared(boost::asio::make_strand(io_context)); @@ -28,7 +32,7 @@ class RedisDriver : public CacheDriver { /* Partition */ virtual Partition get_current_partition_info(const DoutPrefixProvider* dpp) override { return partition_info; } - virtual uint64_t get_free_space(const DoutPrefixProvider* dpp) override { return free_space; } + virtual uint64_t get_free_space(const DoutPrefixProvider* dpp) override; virtual int initialize(const DoutPrefixProvider* dpp) override; virtual int put(const DoutPrefixProvider* dpp, const std::string& key, const bufferlist& bl, uint64_t len, const rgw::sal::Attrs& attrs, optional_yield y) override; @@ -54,6 +58,8 @@ class RedisDriver : public CacheDriver { uint64_t free_space; uint64_t outstanding_write_size; + std::optional resolve_valkey_data_dir(const DoutPrefixProvider* dpp) const; + struct redis_response { request req; boost::redis::generic_response resp; diff --git a/src/rgw/rgw_ssd_driver.cc b/src/rgw/rgw_ssd_driver.cc index 51b7007d6cee..cb9fe6c85ce6 100644 --- a/src/rgw/rgw_ssd_driver.cc +++ b/src/rgw/rgw_ssd_driver.cc @@ -374,7 +374,7 @@ int SSDDriver::restore_blocks_objects(const DoutPrefixProvider* dpp, ObjectDataC uint64_t SSDDriver::get_free_space(const DoutPrefixProvider* dpp) { efs::space_info space = efs::space(partition_info.location); - return space.available; + return (space.available < partition_info.reserve_size) ? 0 : (space.available - partition_info.reserve_size); } void SSDDriver::set_free_space(const DoutPrefixProvider* dpp, uint64_t free_space) diff --git a/src/test/rgw/test_d4n_policy.cc b/src/test/rgw/test_d4n_policy.cc index f0e29030f88b..80cfaacdfebb 100644 --- a/src/test/rgw/test_d4n_policy.cc +++ b/src/test/rgw/test_d4n_policy.cc @@ -70,7 +70,7 @@ class LFUDAPolicyFixture : public ::testing::Test { }; conn = std::make_shared(net::make_strand(io)); - rgw::cache::Partition partition_info{ .location = "RedisCache", .size = 1000 }; + rgw::cache::Partition partition_info{ .location = "RedisCache", .reserve_size = 1073741824 }; cacheDriver = new rgw::cache::RedisDriver{io, partition_info}; policyDriver = new rgw::d4n::PolicyDriver(conn, cacheDriver, "lfuda", null_yield); dir = new rgw::d4n::BlockDirectory{conn}; diff --git a/src/test/rgw/test_ssd_driver.cc b/src/test/rgw/test_ssd_driver.cc index 01ea4e7be31b..f9e5b72d740f 100644 --- a/src/test/rgw/test_ssd_driver.cc +++ b/src/test/rgw/test_ssd_driver.cc @@ -86,7 +86,7 @@ class Environment : public ::testing::Environment { class SSDDriverFixture: public ::testing::Test { protected: virtual void SetUp() { - rgw::cache::Partition partition_info{.name = "d4n", .type = "read-cache", .location = "rgw_d4n_datacache", .size = 5368709120}; + rgw::cache::Partition partition_info{.name = "d4n", .type = "read-cache", .location = "rgw_d4n_datacache", .reserve_size = 1073741824}; cacheDriver = new rgw::cache::SSDDriver{partition_info, false}; ASSERT_NE(cacheDriver, nullptr);