From: Samarah Date: Tue, 14 Oct 2025 17:48:48 +0000 (+0000) Subject: rgw/d4n: Add config option for local RGW address X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=ffc0374066978f64a5a9f43767ae0ba0f7eef941;p=ceph.git rgw/d4n: Add config option for local RGW address Signed-off-by: Samarah --- diff --git a/src/common/options/rgw.yaml.in b/src/common/options/rgw.yaml.in index 16dfe3f77801..b331aa1230e8 100644 --- a/src/common/options/rgw.yaml.in +++ b/src/common/options/rgw.yaml.in @@ -4444,6 +4444,18 @@ options: services: - rgw with_legacy: true +- name: rgw_d4n_local_rgw_address + type: str + level: advanced + desc: local RGW address + long_desc: This is the address used to represent the local RGW in a distributed D4N + system that involves remote RGWs. + default: 127.0.0.1:8000 + services: + - rgw + flags: + - startup + with_legacy: true - name: rgw_d4n_l1_datacache_address type: str level: advanced diff --git a/src/rgw/driver/d4n/d4n_policy.cc b/src/rgw/driver/d4n/d4n_policy.cc index 523c58f83729..e6980c184b95 100644 --- a/src/rgw/driver/d4n/d4n_policy.cc +++ b/src/rgw/driver/d4n/d4n_policy.cc @@ -83,7 +83,7 @@ int LFUDAPolicy::init(CephContext* cct, const DoutPrefixProvider* dpp, asio::io_ req.push("MULTI"); req.push("HSET", "lfuda", "minLocalWeights_sum", std::to_string(weightSum), /* New cache node will always have the minimum average weight */ "minLocalWeights_size", std::to_string(entries_map.size()), - "minLocalWeights_address", dpp->get_cct()->_conf->rgw_d4n_l1_datacache_address); + "minLocalWeights_address", dpp->get_cct()->_conf->rgw_d4n_local_rgw_address); req.push("HSETNX", "lfuda", "age", age); /* Only set maximum age if it doesn't exist */ req.push("EXEC"); @@ -176,7 +176,7 @@ int LFUDAPolicy::local_weight_sync(const DoutPrefixProvider* dpp, optional_yield request req; req.push("HSET", "lfuda", "minLocalWeights_sum", std::to_string(weightSum), "minLocalWeights_size", std::to_string(entries_map.size()), - "minLocalWeights_address", dpp->get_cct()->_conf->rgw_d4n_l1_datacache_address); + "minLocalWeights_address", dpp->get_cct()->_conf->rgw_d4n_local_rgw_address); redis_exec(conn, ec, req, resp, y); @@ -197,7 +197,7 @@ int LFUDAPolicy::local_weight_sync(const DoutPrefixProvider* dpp, optional_yield boost::system::error_code ec; response resp; request req; - req.push("HSET", dpp->get_cct()->_conf->rgw_d4n_l1_datacache_address, "avgLocalWeight_sum", std::to_string(weightSum), + req.push("HSET", dpp->get_cct()->_conf->rgw_d4n_local_rgw_address, "avgLocalWeight_sum", std::to_string(weightSum), "avgLocalWeight_size", std::to_string(entries_map.size())); redis_exec(conn, ec, req, resp, y); @@ -346,7 +346,7 @@ int LFUDAPolicy::eviction(const DoutPrefixProvider* dpp, uint64_t size, optional /* the following part takes care of updating the weight (globalWeight) of the block if this is the last copy in a remote setup and is pushed out to a remote cache where space is available */ #if 0 - if (victim->cacheObj.hostsList.size() == 1 && *(victim->cacheObj.hostsList.begin()) == dpp->get_cct()->_conf->rgw_d4n_l1_datacache_address) { /* Last copy */ + if (victim->cacheObj.hostsList.size() == 1 && *(victim->cacheObj.hostsList.begin()) == dpp->get_cct()->_conf->rgw_d4n_local_rgw_address) { /* Last copy */ if (victim->globalWeight) { it->second->localWeight += victim->globalWeight; (*it->second->handle)->localWeight = it->second->localWeight; @@ -380,7 +380,7 @@ int LFUDAPolicy::eviction(const DoutPrefixProvider* dpp, uint64_t size, optional l.unlock(); //Need to get and then update the host atomically in a remote setup - if ((ret = blockDir->remove_host(dpp, victim, dpp->get_cct()->_conf->rgw_d4n_l1_datacache_address, y)) < 0) { + if ((ret = blockDir->remove_host(dpp, victim, dpp->get_cct()->_conf->rgw_d4n_local_rgw_address, y)) < 0) { delete victim; return ret; } diff --git a/src/rgw/driver/d4n/rgw_sal_d4n.cc b/src/rgw/driver/d4n/rgw_sal_d4n.cc index b850f2b2aeb4..3be2ff5e06e7 100644 --- a/src/rgw/driver/d4n/rgw_sal_d4n.cc +++ b/src/rgw/driver/d4n/rgw_sal_d4n.cc @@ -1181,7 +1181,7 @@ int D4NFilterObject::set_head_obj_dir_entry(const DoutPrefixProvider* dpp, std:: .bucketName = this->get_bucket()->get_bucket_id(), .creationTime = std::to_string(ceph::real_clock::to_double(this->get_mtime())), .dirty = dirty, - .hostsList = { dpp->get_cct()->_conf->rgw_d4n_l1_datacache_address }, + .hostsList = { dpp->get_cct()->_conf->rgw_d4n_local_rgw_address }, .etag = etag, .size = this->get_accounted_size(), .user_id = user_id, @@ -1325,7 +1325,7 @@ int D4NFilterObject::set_head_obj_dir_entry(const DoutPrefixProvider* dpp, std:: .display_name = display_name, }; - version_object.hostsList.insert({ dpp->get_cct()->_conf->rgw_d4n_l1_datacache_address }); + version_object.hostsList.insert({ dpp->get_cct()->_conf->rgw_d4n_local_rgw_address }); rgw::d4n::CacheBlock version_block = rgw::d4n::CacheBlock{ .cacheObj = version_object, @@ -1378,7 +1378,7 @@ int D4NFilterObject::set_data_block_dir_entries(const DoutPrefixProvider* dpp, o for (auto& block : blocks) { block.cacheObj.dirty = dirty; - block.cacheObj.hostsList.insert(dpp->get_cct()->_conf->rgw_d4n_l1_datacache_address); + block.cacheObj.hostsList.insert(dpp->get_cct()->_conf->rgw_d4n_local_rgw_address); block.version = version; } if ((ret = blockDir->set(dpp, blocks, y)) < 0) { @@ -1879,7 +1879,7 @@ int D4NFilterObject::D4NFilterReadOp::flush(const DoutPrefixProvider* dpp, rgw:: dest_block.cacheObj.dirty = true; //writing to cache dest_block.blockID = ofs; dest_block.size = len; - dest_block.cacheObj.hostsList.insert(dpp->get_cct()->_conf->rgw_d4n_l1_datacache_address); + dest_block.cacheObj.hostsList.insert(dpp->get_cct()->_conf->rgw_d4n_local_rgw_address); dest_block.version = dest_version; dest_block.cacheObj.dirty = true; std::string key = get_key_in_cache(get_cache_block_prefix(source->dest_object, dest_version), std::to_string(ofs), std::to_string(len)); @@ -2018,10 +2018,10 @@ int D4NFilterObject::D4NFilterReadOp::iterate(const DoutPrefixProvider* dpp, int } break; } //end if block.version != version - auto it = block.cacheObj.hostsList.find(dpp->get_cct()->_conf->rgw_d4n_l1_datacache_address); + auto it = block.cacheObj.hostsList.find(dpp->get_cct()->_conf->rgw_d4n_local_rgw_address); auto hostsListSize = block.cacheObj.hostsList.size(); if (it != block.cacheObj.hostsList.end()) { - if ((r = block_dir->remove_host(dpp, &block, dpp->get_cct()->_conf->rgw_d4n_l1_datacache_address, y)) < 0) { + if ((r = block_dir->remove_host(dpp, &block, dpp->get_cct()->_conf->rgw_d4n_local_rgw_address, y)) < 0) { ldpp_dout(dpp, 10) << "D4NFilterObject::iterate:: " << __func__ << "(): Error: failed to remove incorrect host from block with oid=" << oid_in_cache <<", ret=" << r << dendl; hostsListSize = hostsListSize - 1; } @@ -2183,7 +2183,7 @@ int D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::handle_data(bufferlist& bl D4NFilterObject* d4n_dest_object = dynamic_cast(source->dest_object); std::string dest_version = d4n_dest_object->get_object_version(); dest_prefix = get_cache_block_prefix(source->dest_object, dest_version); - dest_block.cacheObj.hostsList.insert(dpp->get_cct()->_conf->rgw_d4n_l1_datacache_address); + dest_block.cacheObj.hostsList.insert(dpp->get_cct()->_conf->rgw_d4n_local_rgw_address); dest_block.cacheObj.objName = source->dest_object->get_key().get_oid(); dest_block.cacheObj.bucketName = source->dest_object->get_bucket()->get_bucket_id(); //dest_block.cacheObj.creationTime = std::to_string(ceph::real_clock::to_time_t(source->get_mtime())); @@ -2314,7 +2314,7 @@ int D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::handle_data(bufferlist& bl for (auto& block : blocks) { block.cacheObj.dirty = false; - block.cacheObj.hostsList.insert(dpp->get_cct()->_conf->rgw_d4n_l1_datacache_address); + block.cacheObj.hostsList.insert(dpp->get_cct()->_conf->rgw_d4n_local_rgw_address); block.version = version; } if ((ret = blockDir->set(dpp, blocks, *y)) < 0) { diff --git a/src/test/rgw/test_d4n_policy.cc b/src/test/rgw/test_d4n_policy.cc index 80cfaacdfebb..8e4a04906d04 100644 --- a/src/test/rgw/test_d4n_policy.cc +++ b/src/test/rgw/test_d4n_policy.cc @@ -212,7 +212,7 @@ TEST_F(LFUDAPolicyFixture, RemoteGetBlockYield) .bucketName = "testBucket", .creationTime = "", .dirty = false, - .hostsList = { env->redisHost } + .hostsList = { env->cct->_conf->rgw_d4n_local_rgw_address } }, .blockID = 0, .version = "version", @@ -335,8 +335,8 @@ TEST_F(LFUDAPolicyFixture, RedisSyncTest) req.push("HGET", "lfuda", "minLocalWeights_sum"); req.push("HGET", "lfuda", "minLocalWeights_size"); req.push("HGET", "lfuda", "minLocalWeights_address"); - req.push("HGET", "127.0.0.1:6379", "avgLocalWeight_sum"); - req.push("HGET", "127.0.0.1:6379", "avgLocalWeight_size"); + req.push("HGET", "127.0.0.1:8000", "avgLocalWeight_sum"); + req.push("HGET", "127.0.0.1:8000", "avgLocalWeight_size"); req.push("FLUSHALL"); response(resp).value(), "1"); EXPECT_EQ(std::get<1>(resp).value(), "0"); EXPECT_EQ(std::get<2>(resp).value(), "0"); - EXPECT_EQ(std::get<3>(resp).value(), "127.0.0.1:6379"); + EXPECT_EQ(std::get<3>(resp).value(), "127.0.0.1:8000"); EXPECT_EQ(std::get<4>(resp).value(), "0"); EXPECT_EQ(std::get<4>(resp).value(), "0"); conn->cancel();