rgw/d4n: Add config option for local RGW address

Signed-off-by: Samarah <samarah.uriarte@ibm.com>

diff --git a/src/common/options/rgw.yaml.in b/src/common/options/rgw.yaml.in
index 16dfe3f77801608e7a5c7be784fea7d2738d0d99..b331aa1230e8ccc912ba771502872cd07bd41398 100644 (file)
--- a/src/common/options/rgw.yaml.in
+++ b/src/common/options/rgw.yaml.in
@@ -4444,6 +4444,18 @@ options:
   services:
   - rgw
   with_legacy: true
+- name: rgw_d4n_local_rgw_address
+  type: str
+  level: advanced
+  desc: local RGW address 
+  long_desc: This is the address used to represent the local RGW in a distributed D4N
+    system that involves remote RGWs. 
+  default: 127.0.0.1:8000
+  services:
+  - rgw
+  flags:
+  - startup
+  with_legacy: true
 - name: rgw_d4n_l1_datacache_address
   type: str
   level: advanced

diff --git a/src/rgw/driver/d4n/d4n_policy.cc b/src/rgw/driver/d4n/d4n_policy.cc
index 523c58f83729f617a8123280793bf2bc86255b59..e6980c184b95d7243f6bce82bd0fe306528c5061 100644 (file)
--- a/src/rgw/driver/d4n/d4n_policy.cc
+++ b/src/rgw/driver/d4n/d4n_policy.cc
@@ -83,7 +83,7 @@ int LFUDAPolicy::init(CephContext* cct, const DoutPrefixProvider* dpp, asio::io_
     req.push("MULTI");
     req.push("HSET", "lfuda", "minLocalWeights_sum", std::to_string(weightSum), /* New cache node will always have the minimum average weight */
               "minLocalWeights_size", std::to_string(entries_map.size()), 
-              "minLocalWeights_address", dpp->get_cct()->_conf->rgw_d4n_l1_datacache_address);
+              "minLocalWeights_address", dpp->get_cct()->_conf->rgw_d4n_local_rgw_address);
     req.push("HSETNX", "lfuda", "age", age); /* Only set maximum age if it doesn't exist */
     req.push("EXEC");
   
@@ -176,7 +176,7 @@ int LFUDAPolicy::local_weight_sync(const DoutPrefixProvider* dpp, optional_yield
        request req;
        req.push("HSET", "lfuda", "minLocalWeights_sum", std::to_string(weightSum), 
                   "minLocalWeights_size", std::to_string(entries_map.size()), 
-                  "minLocalWeights_address", dpp->get_cct()->_conf->rgw_d4n_l1_datacache_address);
+                  "minLocalWeights_address", dpp->get_cct()->_conf->rgw_d4n_local_rgw_address);
 
        redis_exec(conn, ec, req, resp, y);
 
@@ -197,7 +197,7 @@ int LFUDAPolicy::local_weight_sync(const DoutPrefixProvider* dpp, optional_yield
     boost::system::error_code ec;
     response<ignore_t> resp;
     request req;
-    req.push("HSET", dpp->get_cct()->_conf->rgw_d4n_l1_datacache_address, "avgLocalWeight_sum", std::to_string(weightSum), 
+    req.push("HSET", dpp->get_cct()->_conf->rgw_d4n_local_rgw_address, "avgLocalWeight_sum", std::to_string(weightSum), 
               "avgLocalWeight_size", std::to_string(entries_map.size()));
 
     redis_exec(conn, ec, req, resp, y);
@@ -346,7 +346,7 @@ int LFUDAPolicy::eviction(const DoutPrefixProvider* dpp, uint64_t size, optional
     /* the following part takes care of updating the weight (globalWeight) of the block if this is the last copy in a remote setup
        and is pushed out to a remote cache where space is available */
 #if 0
-    if (victim->cacheObj.hostsList.size() == 1 && *(victim->cacheObj.hostsList.begin()) == dpp->get_cct()->_conf->rgw_d4n_l1_datacache_address) { /* Last copy */
+    if (victim->cacheObj.hostsList.size() == 1 && *(victim->cacheObj.hostsList.begin()) == dpp->get_cct()->_conf->rgw_d4n_local_rgw_address) { /* Last copy */
       if (victim->globalWeight) {
        it->second->localWeight += victim->globalWeight;
         (*it->second->handle)->localWeight = it->second->localWeight;
@@ -380,7 +380,7 @@ int LFUDAPolicy::eviction(const DoutPrefixProvider* dpp, uint64_t size, optional
     l.unlock();
 
     //Need to get and then update the host atomically in a remote setup
-    if ((ret = blockDir->remove_host(dpp, victim, dpp->get_cct()->_conf->rgw_d4n_l1_datacache_address, y)) < 0) {
+    if ((ret = blockDir->remove_host(dpp, victim, dpp->get_cct()->_conf->rgw_d4n_local_rgw_address, y)) < 0) {
       delete victim;
       return ret;
     }

diff --git a/src/rgw/driver/d4n/rgw_sal_d4n.cc b/src/rgw/driver/d4n/rgw_sal_d4n.cc
index b850f2b2aeb46e10cb71663cbbade78a0993edc7..3be2ff5e06e713a300e38a556047cacd5d8a339e 100644 (file)
--- a/src/rgw/driver/d4n/rgw_sal_d4n.cc
+++ b/src/rgw/driver/d4n/rgw_sal_d4n.cc
@@ -1181,7 +1181,7 @@ int D4NFilterObject::set_head_obj_dir_entry(const DoutPrefixProvider* dpp, std::
       .bucketName = this->get_bucket()->get_bucket_id(),
       .creationTime = std::to_string(ceph::real_clock::to_double(this->get_mtime())),
       .dirty = dirty,
-      .hostsList = { dpp->get_cct()->_conf->rgw_d4n_l1_datacache_address },
+      .hostsList = { dpp->get_cct()->_conf->rgw_d4n_local_rgw_address },
       .etag = etag,
       .size = this->get_accounted_size(),
       .user_id = user_id,
@@ -1325,7 +1325,7 @@ int D4NFilterObject::set_head_obj_dir_entry(const DoutPrefixProvider* dpp, std::
     .display_name = display_name,
     };
 
-    version_object.hostsList.insert({ dpp->get_cct()->_conf->rgw_d4n_l1_datacache_address });
+    version_object.hostsList.insert({ dpp->get_cct()->_conf->rgw_d4n_local_rgw_address });
 
     rgw::d4n::CacheBlock version_block = rgw::d4n::CacheBlock{
       .cacheObj = version_object,
@@ -1378,7 +1378,7 @@ int D4NFilterObject::set_data_block_dir_entries(const DoutPrefixProvider* dpp, o
 
   for (auto& block : blocks) {
     block.cacheObj.dirty = dirty;
-    block.cacheObj.hostsList.insert(dpp->get_cct()->_conf->rgw_d4n_l1_datacache_address);
+    block.cacheObj.hostsList.insert(dpp->get_cct()->_conf->rgw_d4n_local_rgw_address);
     block.version = version;
   }
   if ((ret = blockDir->set(dpp, blocks, y)) < 0) {
@@ -1879,7 +1879,7 @@ int D4NFilterObject::D4NFilterReadOp::flush(const DoutPrefixProvider* dpp, rgw::
         dest_block.cacheObj.dirty = true; //writing to cache
         dest_block.blockID = ofs;
         dest_block.size = len;
-        dest_block.cacheObj.hostsList.insert(dpp->get_cct()->_conf->rgw_d4n_l1_datacache_address);
+        dest_block.cacheObj.hostsList.insert(dpp->get_cct()->_conf->rgw_d4n_local_rgw_address);
         dest_block.version = dest_version;
         dest_block.cacheObj.dirty = true;
         std::string key =  get_key_in_cache(get_cache_block_prefix(source->dest_object, dest_version), std::to_string(ofs), std::to_string(len));
@@ -2018,10 +2018,10 @@ int D4NFilterObject::D4NFilterReadOp::iterate(const DoutPrefixProvider* dpp, int
             }
             break;
           } //end if block.version != version
-          auto it = block.cacheObj.hostsList.find(dpp->get_cct()->_conf->rgw_d4n_l1_datacache_address);
+          auto it = block.cacheObj.hostsList.find(dpp->get_cct()->_conf->rgw_d4n_local_rgw_address);
           auto hostsListSize = block.cacheObj.hostsList.size();
           if (it != block.cacheObj.hostsList.end()) {
-            if ((r = block_dir->remove_host(dpp, &block, dpp->get_cct()->_conf->rgw_d4n_l1_datacache_address, y)) < 0) {
+            if ((r = block_dir->remove_host(dpp, &block, dpp->get_cct()->_conf->rgw_d4n_local_rgw_address, y)) < 0) {
               ldpp_dout(dpp, 10) << "D4NFilterObject::iterate:: " << __func__ << "(): Error: failed to remove incorrect host from block with oid=" << oid_in_cache <<", ret=" << r << dendl;
               hostsListSize = hostsListSize - 1;
             }
@@ -2183,7 +2183,7 @@ int D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::handle_data(bufferlist& bl
       D4NFilterObject* d4n_dest_object = dynamic_cast<D4NFilterObject*>(source->dest_object);
       std::string dest_version = d4n_dest_object->get_object_version();
       dest_prefix = get_cache_block_prefix(source->dest_object, dest_version);
-      dest_block.cacheObj.hostsList.insert(dpp->get_cct()->_conf->rgw_d4n_l1_datacache_address);
+      dest_block.cacheObj.hostsList.insert(dpp->get_cct()->_conf->rgw_d4n_local_rgw_address);
       dest_block.cacheObj.objName = source->dest_object->get_key().get_oid();
       dest_block.cacheObj.bucketName = source->dest_object->get_bucket()->get_bucket_id();
       //dest_block.cacheObj.creationTime = std::to_string(ceph::real_clock::to_time_t(source->get_mtime()));
@@ -2314,7 +2314,7 @@ int D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::handle_data(bufferlist& bl
 
       for (auto& block : blocks) {
         block.cacheObj.dirty = false;
-        block.cacheObj.hostsList.insert(dpp->get_cct()->_conf->rgw_d4n_l1_datacache_address);
+        block.cacheObj.hostsList.insert(dpp->get_cct()->_conf->rgw_d4n_local_rgw_address);
         block.version = version;
       }
       if ((ret = blockDir->set(dpp, blocks, *y)) < 0) {

diff --git a/src/test/rgw/test_d4n_policy.cc b/src/test/rgw/test_d4n_policy.cc
index 80cfaacdfebbbfc806be62913899c312b37033e2..8e4a04906d04585f8178dda8301d2ee18323dd4e 100644 (file)
--- a/src/test/rgw/test_d4n_policy.cc
+++ b/src/test/rgw/test_d4n_policy.cc
@@ -212,7 +212,7 @@ TEST_F(LFUDAPolicyFixture, RemoteGetBlockYield)
        .bucketName = "testBucket",
        .creationTime = "",
        .dirty = false,
-       .hostsList = { env->redisHost }
+       .hostsList = { env->cct->_conf->rgw_d4n_local_rgw_address }
       },
       .blockID = 0,
       .version = "version",
@@ -335,8 +335,8 @@ TEST_F(LFUDAPolicyFixture, RedisSyncTest)
     req.push("HGET", "lfuda", "minLocalWeights_sum");
     req.push("HGET", "lfuda", "minLocalWeights_size");
     req.push("HGET", "lfuda", "minLocalWeights_address");
-    req.push("HGET", "127.0.0.1:6379", "avgLocalWeight_sum");
-    req.push("HGET", "127.0.0.1:6379", "avgLocalWeight_size");
+    req.push("HGET", "127.0.0.1:8000", "avgLocalWeight_sum");
+    req.push("HGET", "127.0.0.1:8000", "avgLocalWeight_size");
     req.push("FLUSHALL");
 
     response<std::string, std::string, std::string,
@@ -349,7 +349,7 @@ TEST_F(LFUDAPolicyFixture, RedisSyncTest)
     EXPECT_EQ(std::get<0>(resp).value(), "1");
     EXPECT_EQ(std::get<1>(resp).value(), "0");
     EXPECT_EQ(std::get<2>(resp).value(), "0");
-    EXPECT_EQ(std::get<3>(resp).value(), "127.0.0.1:6379");
+    EXPECT_EQ(std::get<3>(resp).value(), "127.0.0.1:8000");
     EXPECT_EQ(std::get<4>(resp).value(), "0");
     EXPECT_EQ(std::get<4>(resp).value(), "0");
     conn->cancel();