rgw/d4n: refactor unused conf gw_d4n_l1_datacache_size

to the amount of disk space to keep free in reserve
so that the datacache by default will not fill the drive completely

Signed-off-by: Mark Kogan <mkogan@ibm.com>

diff --git a/doc/radosgw/config-ref.rst b/doc/radosgw/config-ref.rst
index 0e2bc9d6f6cb2841dfc0fa60c8f5283cbf98e7bb..603f6fa76b37a1062d148c9cef255c6e6e148e68 100644 (file)
--- a/doc/radosgw/config-ref.rst
+++ b/doc/radosgw/config-ref.rst
@@ -326,7 +326,7 @@ below.
 
 .. confval:: rgw_d4n_address
 .. confval:: rgw_d4n_l1_datacache_persistent_path
-.. confval:: rgw_d4n_l1_datacache_size
+.. confval:: rgw_d4n_l1_datacache_disk_reserve
 .. confval:: rgw_d4n_l1_evict_cache_on_start
 .. confval:: rgw_d4n_l1_fadvise
 .. confval:: rgw_d4n_libaio_aio_threads

diff --git a/src/common/options/rgw.yaml.in b/src/common/options/rgw.yaml.in
index ffec3bacb2487fd5bf3248946d7d30942b4466f4..ee659cb2ac13e62ee99214df191d4f19776cb891 100644 (file)
--- a/src/common/options/rgw.yaml.in
+++ b/src/common/options/rgw.yaml.in
@@ -3894,12 +3894,12 @@ options:
   services:
   - rgw
   with_legacy: true
-- name: rgw_d4n_l1_datacache_size
+- name: rgw_d4n_l1_datacache_disk_reserve
   type: size
   level: advanced
-  desc: maximum size on disk for datacache
-  long_desc: The local SSD cache uses this option to configure its size in bytes. This 
-    option is not used by the Redis cache backend. 
+  desc: amount of disk space to keep free for datacache
+  long_desc: The local SSD cache uses this option to determine how much disk space to 
+    reserve. The cache can grow until disk usage reaches (total disk space - reserved space).
   default: 1_G
   services:
   - rgw

diff --git a/src/rgw/driver/d4n/rgw_sal_d4n.cc b/src/rgw/driver/d4n/rgw_sal_d4n.cc
index 1b118726f1e5b6be3a8ca210889f6d9505cbca28..fca255368b5518b142be658bcc796fe816a3536c 100644 (file)
--- a/src/rgw/driver/d4n/rgw_sal_d4n.cc
+++ b/src/rgw/driver/d4n/rgw_sal_d4n.cc
@@ -46,7 +46,7 @@ D4NFilterDriver::D4NFilterDriver(Driver* _next, boost::asio::io_context& io_cont
   partition_info.location = g_conf()->rgw_d4n_l1_datacache_persistent_path;
   partition_info.name = "d4n";
   partition_info.type = "read-cache";
-  partition_info.size = g_conf()->rgw_d4n_l1_datacache_size;
+  partition_info.reserve_size = g_conf()->rgw_d4n_l1_datacache_disk_reserve;
   cacheDriver = std::make_unique<rgw::cache::SSDDriver>(partition_info, admin);
 }
 

diff --git a/src/rgw/rgw_cache_driver.h b/src/rgw/rgw_cache_driver.h
index 2ca91538fb95c1dfa8c5b5d0fa9a4cc18a4e64a8..6c02f24d3496101b8b300a879a82f89b75a3fa9e 100644 (file)
--- a/src/rgw/rgw_cache_driver.h
+++ b/src/rgw/rgw_cache_driver.h
@@ -32,7 +32,7 @@ struct Partition {
   std::string name;
   std::string type;
   std::string location;
-  uint64_t size;
+  uint64_t reserve_size;
 };
 
 class CacheDriver {

diff --git a/src/rgw/rgw_redis_driver.cc b/src/rgw/rgw_redis_driver.cc
index 3fb93583b61a1a4c69cd1d67d1d5a7bbb2b398fc..ee71e61cc7fce97ca851ed74514263c774020aa1 100644 (file)
--- a/src/rgw/rgw_redis_driver.cc
+++ b/src/rgw/rgw_redis_driver.cc
@@ -65,6 +65,75 @@ void redis_exec(std::shared_ptr<connection> conn,
   }
 }
 
+std::optional<fs::path> RedisDriver::resolve_valkey_data_dir(const DoutPrefixProvider* dpp) const
+{
+  try {
+    boost::system::error_code ec;
+    response<std::vector<std::string>> resp;
+    request req;
+    req.push("CONFIG", "GET", "dir");
+
+    redis_exec(conn, ec, req, resp, null_yield);
+
+    if (ec) {
+      ldpp_dout(dpp, 5) << "RedisDriver::" << __func__
+                        << "(): failed to execute CONFIG GET dir: " << ec.message()
+                        << dendl;
+      return std::nullopt;
+    }
+
+    const auto& entries = std::get<0>(resp);
+    std::clog << "MK| OK " << __FILE__ << " :" << __LINE__ << " | " << __func__ << "(): entries.value().size()=" << entries.value().size() << std::endl;
+    if (entries.value().size() < 2) {
+      ldpp_dout(dpp, 5) << "RedisDriver::" << __func__
+                        << "(): unexpected CONFIG GET dir response size=" << entries.value().size()
+                        << dendl;
+      return std::nullopt;
+    }
+
+    const fs::path dir_path(entries.value()[1]);
+    std::clog << "MK| OK " << __FILE__ << " :" << __LINE__ << " | " << __func__ << "(): dir_path=" << std::quoted(dir_path.string()) << std::endl;
+    if (dir_path.empty()) {
+      ldpp_dout(dpp, 5) << "RedisDriver::" << __func__
+                        << "(): CONFIG GET dir returned empty path" << dendl;
+      return std::nullopt;
+    }
+
+    return dir_path;
+  } catch (const std::exception& e) {
+    ldpp_dout(dpp, 0) << "RedisDriver::" << __func__
+                      << "(): exception while resolving data dir: " << e.what()
+                      << dendl;
+  }
+
+  return std::nullopt;
+}
+
+uint64_t RedisDriver::get_free_space(const DoutPrefixProvider* dpp)
+{
+  auto data_dir = resolve_valkey_data_dir(dpp);
+  if (!data_dir) {
+    ldpp_dout(dpp, 0) << __func__ << "(): ERROR: could not resolve redis data dir" << dendl;
+    return 0;
+  }
+
+  const fs::path redis_probe_path = *data_dir;
+  ldpp_dout(dpp, 20) << __func__ << "(): redis path = " << std::quoted(redis_probe_path.string()) << dendl;
+
+  std::error_code ec;
+  fs::space_info space = fs::space(redis_probe_path, ec);
+  if (ec) {
+    ldpp_dout(dpp, 0) << __func__ << "(): ERROR: unable to stat redis path "
+                      << std::quoted(redis_probe_path.string()) << " : " << ec.message() << dendl;
+    return 0;
+  }
+
+  ldpp_dout(dpp, 20) << __func__ << "(): redis partition space.available=" << space.available
+                     << ", partition_info.reserve_size=" << partition_info.reserve_size << dendl;
+
+  return (space.available < partition_info.reserve_size) ? 0 : (space.available - partition_info.reserve_size);
+}
+
 int RedisDriver::initialize(const DoutPrefixProvider* dpp)
 {
   if (partition_info.location.back() != '/') {

diff --git a/src/rgw/rgw_redis_driver.h b/src/rgw/rgw_redis_driver.h
index 7889e8fc7d77e391174fee94c49cba951745344a..0e018d62561998f9a4aad78d52a8c32a2e8c6be1 100644 (file)
--- a/src/rgw/rgw_redis_driver.h
+++ b/src/rgw/rgw_redis_driver.h
@@ -1,6 +1,9 @@
 #pragma once
 
 #include <aio.h>
+#include <filesystem>
+#include <optional>
+#include <system_error>
 #include <boost/redis/connection.hpp>
 
 #include "common/async/completion.h"
@@ -10,6 +13,7 @@
 namespace rgw { namespace cache { 
 
 namespace net = boost::asio;
+namespace fs = std::filesystem;
 using boost::redis::config;
 using boost::redis::connection;
 using boost::redis::request;
@@ -19,7 +23,7 @@ using boost::redis::ignore_t;
 class RedisDriver : public CacheDriver {
   public:
     RedisDriver(net::io_context& io_context, Partition& _partition_info) : partition_info(_partition_info),
-                                                                          free_space(_partition_info.size), 
+                                                                          free_space(0), 
                                                                           outstanding_write_size(0)
     {
       conn = std::make_shared<connection>(boost::asio::make_strand(io_context));
@@ -28,7 +32,7 @@ class RedisDriver : public CacheDriver {
 
     /* Partition */
     virtual Partition get_current_partition_info(const DoutPrefixProvider* dpp) override { return partition_info; }
-    virtual uint64_t get_free_space(const DoutPrefixProvider* dpp) override { return free_space; }
+    virtual uint64_t get_free_space(const DoutPrefixProvider* dpp) override;
 
     virtual int initialize(const DoutPrefixProvider* dpp) override;
     virtual int put(const DoutPrefixProvider* dpp, const std::string& key, const bufferlist& bl, uint64_t len, const rgw::sal::Attrs& attrs, optional_yield y) override;
@@ -54,6 +58,8 @@ class RedisDriver : public CacheDriver {
     uint64_t free_space;
     uint64_t outstanding_write_size;
 
+  std::optional<fs::path> resolve_valkey_data_dir(const DoutPrefixProvider* dpp) const;
+
     struct redis_response {
       request req;
       boost::redis::generic_response resp;

diff --git a/src/rgw/rgw_ssd_driver.cc b/src/rgw/rgw_ssd_driver.cc
index 51b7007d6cee4385aaf3376372f1adf0ec18f19d..cb9fe6c85ce641292c0c21fe1b310c9ef6e73c83 100644 (file)
--- a/src/rgw/rgw_ssd_driver.cc
+++ b/src/rgw/rgw_ssd_driver.cc
@@ -374,7 +374,7 @@ int SSDDriver::restore_blocks_objects(const DoutPrefixProvider* dpp, ObjectDataC
 uint64_t SSDDriver::get_free_space(const DoutPrefixProvider* dpp)
 {
     efs::space_info space = efs::space(partition_info.location);
-    return space.available;
+    return (space.available < partition_info.reserve_size) ? 0 : (space.available - partition_info.reserve_size);
 }
 
 void SSDDriver::set_free_space(const DoutPrefixProvider* dpp, uint64_t free_space)

diff --git a/src/test/rgw/test_d4n_policy.cc b/src/test/rgw/test_d4n_policy.cc
index f0e29030f88b1fa5731dc07fede4bc0c1496c962..80cfaacdfebbbfc806be62913899c312b37033e2 100644 (file)
--- a/src/test/rgw/test_d4n_policy.cc
+++ b/src/test/rgw/test_d4n_policy.cc
@@ -70,7 +70,7 @@ class LFUDAPolicyFixture : public ::testing::Test {
       };
 
       conn = std::make_shared<connection>(net::make_strand(io));
-      rgw::cache::Partition partition_info{ .location = "RedisCache", .size = 1000 };
+      rgw::cache::Partition partition_info{ .location = "RedisCache", .reserve_size = 1073741824 };
       cacheDriver = new rgw::cache::RedisDriver{io, partition_info};
       policyDriver = new rgw::d4n::PolicyDriver(conn, cacheDriver, "lfuda", null_yield);
       dir = new rgw::d4n::BlockDirectory{conn};

diff --git a/src/test/rgw/test_ssd_driver.cc b/src/test/rgw/test_ssd_driver.cc
index 01ea4e7be31b8cd6d1a8fd964c726fb6d6f89219..f9e5b72d740f8020db09f12fbcca1e9738862119 100644 (file)
--- a/src/test/rgw/test_ssd_driver.cc
+++ b/src/test/rgw/test_ssd_driver.cc
@@ -86,7 +86,7 @@ class Environment : public ::testing::Environment {
 class SSDDriverFixture: public ::testing::Test {
   protected:
     virtual void SetUp() {
-        rgw::cache::Partition partition_info{.name = "d4n", .type = "read-cache", .location = "rgw_d4n_datacache", .size = 5368709120};
+        rgw::cache::Partition partition_info{.name = "d4n", .type = "read-cache", .location = "rgw_d4n_datacache", .reserve_size = 1073741824};
         cacheDriver = new rgw::cache::SSDDriver{partition_info, false};
 
         ASSERT_NE(cacheDriver, nullptr);