From 71f09c5a340392a00c1818fd94ee645bb4c91a18 Mon Sep 17 00:00:00 2001 From: Samarah Date: Tue, 26 Sep 2023 20:36:31 +0000 Subject: [PATCH] rgw: Add `rgw_local_cache_address` yaml config Signed-off-by: Samarah --- src/common/options/rgw.yaml.in | 10 ++++++++++ src/rgw/driver/d4n/d4n_policy.cc | 10 +++++----- src/rgw/driver/d4n/d4n_policy.h | 13 ++++--------- src/rgw/driver/d4n/rgw_sal_d4n.cc | 9 ++++----- src/rgw/rgw_redis_driver.cc | 8 ++++---- src/rgw/rgw_redis_driver.h | 1 - 6 files changed, 27 insertions(+), 24 deletions(-) diff --git a/src/common/options/rgw.yaml.in b/src/common/options/rgw.yaml.in index 9f87a037ee2..bcce8389a53 100644 --- a/src/common/options/rgw.yaml.in +++ b/src/common/options/rgw.yaml.in @@ -4072,3 +4072,13 @@ options: services: - rgw with_legacy: true +- name: rgw_local_cache_address + type: str + level: advanced + desc: Local CacheDriver address + default: 127.0.0.1:6379 + services: + - rgw + flags: + - startup + with_legacy: true diff --git a/src/rgw/driver/d4n/d4n_policy.cc b/src/rgw/driver/d4n/d4n_policy.cc index 449596b13b0..a7d68f84591 100644 --- a/src/rgw/driver/d4n/d4n_policy.cc +++ b/src/rgw/driver/d4n/d4n_policy.cc @@ -150,7 +150,7 @@ int LFUDAPolicy::get_min_avg_weight(optional_yield y) { if (!std::get<0>(resp).value()) { // Is int_max what we want here? -Sam - if (set_min_avg_weight(0, ""/* local cache location or keep empty? */, y)) { /* Initialize minimum average weight */ + if (set_min_avg_weight(0, dir->cct->_conf->rgw_local_cache_address, y)) { /* Initialize minimum average weight */ return 0; } else { return -1; @@ -276,7 +276,7 @@ uint64_t LFUDAPolicy::eviction(const DoutPrefixProvider* dpp, rgw::cache::CacheD return 0; } - if (victim.hostsList.size() == 1 && victim.hostsList[0] == "127.0.0.1:6379" /* local cache address */) { /* Last copy */ + if (victim.hostsList.size() == 1 && victim.hostsList[0] == dir->cct->_conf->rgw_local_cache_address) { /* Last copy */ if (victim.globalWeight) { it->second->localWeight += victim.globalWeight; if (cacheNode->set_attr(dpp, victim.cacheObj.objName, "localWeight", std::to_string(it->second->localWeight), y) < 0) { @@ -301,16 +301,16 @@ uint64_t LFUDAPolicy::eviction(const DoutPrefixProvider* dpp, rgw::cache::CacheD ldpp_dout(dpp, 10) << "RGW D4N Policy: Block " << victim.cacheObj.objName << " has been evicted." << dendl; - if (cacheNode->del(dpp, victim.cacheObj.objName, y) < 0 && dir->remove_host(&victim, ""/* local cache address */, y) < 0) { + if (cacheNode->del(dpp, victim.cacheObj.objName, y) < 0 && dir->remove_host(&victim, dir->cct->_conf->rgw_local_cache_address, y) < 0) { return 0; } else { uint64_t num_entries = entries_map.size(); if (!avgWeight) { - if (set_min_avg_weight(0, ""/*local cache location*/, y) < 0) // Where else must this be set? -Sam + if (set_min_avg_weight(0, dir->cct->_conf->rgw_local_cache_address, y) < 0) // Where else must this be set? -Sam return 0; } else { - if (set_min_avg_weight(avgWeight - (it->second->localWeight/num_entries), ""/*local cache location*/, y) < 0) { // Where else must this be set? -Sam + if (set_min_avg_weight(avgWeight - (it->second->localWeight/num_entries), dir->cct->_conf->rgw_local_cache_address, y) < 0) { // Where else must this be set? -Sam return 0; } int age = get_age(y); diff --git a/src/rgw/driver/d4n/d4n_policy.h b/src/rgw/driver/d4n/d4n_policy.h index f5ddfcc7222..7b9b08eb730 100644 --- a/src/rgw/driver/d4n/d4n_policy.h +++ b/src/rgw/driver/d4n/d4n_policy.h @@ -30,15 +30,10 @@ class CachePolicy { }; public: - CephContext* cct; - CachePolicy() {} virtual ~CachePolicy() = default; - virtual int init(CephContext *cct, const DoutPrefixProvider* dpp) { - this->cct = cct; - return 0; - } + virtual int init(CephContext *cct, const DoutPrefixProvider* dpp) { return 0; } virtual int exist_key(std::string key, optional_yield y) = 0; virtual int get_block(const DoutPrefixProvider* dpp, CacheBlock* block, rgw::cache::CacheDriver* cacheNode, optional_yield y) = 0; virtual uint64_t eviction(const DoutPrefixProvider* dpp, rgw::cache::CacheDriver* cacheNode, optional_yield y) = 0; @@ -86,11 +81,11 @@ class LFUDAPolicy : public CachePolicy { } virtual int init(CephContext *cct, const DoutPrefixProvider* dpp) { - this->cct = cct; + std::string address = cct->_conf->rgw_local_cache_address; config cfg; - cfg.addr.host = cct->_conf->rgw_d4n_host; // TODO: Replace with cache address - cfg.addr.port = std::to_string(cct->_conf->rgw_d4n_port); + cfg.addr.host = address.substr(0, address.find(":")); + cfg.addr.port = address.substr(address.find(":") + 1, address.length()); if (!cfg.addr.host.length() || !cfg.addr.port.length()) { ldpp_dout(dpp, 10) << "RGW Redis Cache: Redis cache endpoint was not configured correctly" << dendl; diff --git a/src/rgw/driver/d4n/rgw_sal_d4n.cc b/src/rgw/driver/d4n/rgw_sal_d4n.cc index 34496df703b..8da78b28035 100644 --- a/src/rgw/driver/d4n/rgw_sal_d4n.cc +++ b/src/rgw/driver/d4n/rgw_sal_d4n.cc @@ -693,12 +693,12 @@ int D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::handle_data(bufferlist& bl rgw::d4n::CacheBlock block; rgw::d4n::BlockDirectory* blockDir = source->driver->get_block_dir(); block.version = ""; - block.hostsList.push_back("127.0.0.1:6379" /*current cache addr*/); + block.hostsList.push_back(blockDir->cct->_conf->rgw_local_cache_address); block.cacheObj.objName = source->get_key().get_oid(); block.cacheObj.bucketName = source->get_bucket()->get_name(); block.cacheObj.creationTime = 0; block.cacheObj.dirty = false; - block.cacheObj.hostsList.push_back("127.0.0.1:6379" /*current cache addr*/); + block.cacheObj.hostsList.push_back(blockDir->cct->_conf->rgw_local_cache_address); // Is the entire object getting stored in the local cache as well or only blocks? -Sam if (bl.length() > 0 && last_part) { // if bl = bl_rem has data and this is the last part, write it to cache std::string oid = this->oid + "_" + std::to_string(ofs) + "_" + std::to_string(bl_len); @@ -872,13 +872,12 @@ int D4NFilterWriter::complete(size_t accounted_size, const std::string& etag, .bucketName = obj->get_bucket()->get_name(), .creationTime = 0, // TODO: get correct value .dirty = false, - .hostsList = { "127.0.0.1:6379" /*current cache addr*/ } // TODO: fix + .hostsList = { driver->get_block_dir()->cct->_conf->rgw_local_cache_address } }, .blockID = 0, // TODO: get correct version/blockID .version = "", .size = accounted_size, - .hostsList = { driver->get_block_dir()->cct->_conf->rgw_d4n_host + ":" + - std::to_string(driver->get_block_dir()->cct->_conf->rgw_d4n_port) } + .hostsList = { driver->get_block_dir()->cct->_conf->rgw_local_cache_address } }; int setDirReturn = driver->get_block_dir()->set(&block, y); diff --git a/src/rgw/rgw_redis_driver.cc b/src/rgw/rgw_redis_driver.cc index feca1e8d984..90c52b1846b 100644 --- a/src/rgw/rgw_redis_driver.cc +++ b/src/rgw/rgw_redis_driver.cc @@ -98,15 +98,15 @@ std::vector RedisDriver::list_partitions(const DoutPrefixProvider* dp int RedisDriver::initialize(CephContext* cct, const DoutPrefixProvider* dpp) { - this->cct = cct; - if (partition_info.location.back() != '/') { partition_info.location += "/"; } + std::string address = cct->_conf->rgw_local_cache_address; + config cfg; - cfg.addr.host = cct->_conf->rgw_d4n_host; // TODO: Replace with cache address - cfg.addr.port = std::to_string(cct->_conf->rgw_d4n_port); + cfg.addr.host = address.substr(0, address.find(":")); + cfg.addr.port = address.substr(address.find(":") + 1, address.length()); if (!cfg.addr.host.length() || !cfg.addr.port.length()) { ldpp_dout(dpp, 10) << "RGW Redis Cache: Redis cache endpoint was not configured correctly" << dendl; diff --git a/src/rgw/rgw_redis_driver.h b/src/rgw/rgw_redis_driver.h index 954a5b2373b..8624fdbc5f9 100644 --- a/src/rgw/rgw_redis_driver.h +++ b/src/rgw/rgw_redis_driver.h @@ -78,7 +78,6 @@ class RedisDriver : public CacheDriver { Partition partition_info; uint64_t free_space; uint64_t outstanding_write_size; - CephContext* cct; int add_partition_info(Partition& info); int remove_partition_info(Partition& info); -- 2.39.5