From 15dac98e54005f263f7d99d068078f1cd58c52a5 Mon Sep 17 00:00:00 2001 From: Samarah Date: Thu, 7 Sep 2023 21:36:28 +0000 Subject: [PATCH] d4n/directory: Update directory `exist_key` method to take in block/object; fix endpoint generation; add `remove_host` method Signed-off-by: Samarah --- src/rgw/driver/d4n/d4n_directory.cc | 146 ++++++++++++++++++++-------- src/rgw/driver/d4n/d4n_directory.h | 8 +- src/rgw/driver/d4n/d4n_policy.cc | 5 +- src/rgw/driver/d4n/rgw_sal_d4n.cc | 6 +- 4 files changed, 115 insertions(+), 50 deletions(-) diff --git a/src/rgw/driver/d4n/d4n_directory.cc b/src/rgw/driver/d4n/d4n_directory.cc index edeb69ff3f7e6..f608d6266e0d1 100644 --- a/src/rgw/driver/d4n/d4n_directory.cc +++ b/src/rgw/driver/d4n/d4n_directory.cc @@ -47,16 +47,10 @@ std::string ObjectDirectory::build_index(CacheObj* object) { return object->bucketName + "_" + object->objName; } -int ObjectDirectory::exist_key(std::string key, optional_yield y) { - int result = 0; - std::vector keys; - keys.push_back(key); -#if 0 - if (!client.is_connected()) { - return result; - } -#endif +int ObjectDirectory::exist_key(CacheObj* object, optional_yield y) { + std::string key = build_index(object); response resp; + try { boost::system::error_code ec; request req; @@ -94,7 +88,7 @@ int ObjectDirectory::set(CacheObj* object, optional_yield y) { redisValues.push_back("dirty"); redisValues.push_back(std::to_string(object->dirty)); redisValues.push_back("objHosts"); - redisValues.push_back(endpoint); // Set in filter -Sam + redisValues.push_back(endpoint); try { boost::system::error_code ec; @@ -116,18 +110,8 @@ int ObjectDirectory::set(CacheObj* object, optional_yield y) { int ObjectDirectory::get(CacheObj* object, optional_yield y) { std::string key = build_index(object); -#if 0 - if (!client.is_connected()) { - find_client(&client); - } -#endif - if (exist_key(key, y)) { - std::string key; - std::string objName; - std::string bucketName; - std::string creationTime; - std::string dirty; - std::string hosts; + + if (exist_key(object, y)) { std::vector fields; fields.push_back("objName"); @@ -174,15 +158,10 @@ int ObjectDirectory::get(CacheObj* object, optional_yield y) { int ObjectDirectory::copy(CacheObj* object, std::string copyName, std::string copyBucketName, optional_yield y) { std::string key = build_index(object); - std::vector keys; - keys.push_back(key); - std::string copyKey; -#if 0 - if (!client.is_connected()) { - find_client(&client); - } -#endif - if (exist_key(key, y)) { + auto copyObj = CacheObj{ .objName = copyName, .bucketName = copyBucketName }; + std::string copyKey = build_index(©Obj); + + if (exist_key(object, y)) { try { response resp; @@ -223,7 +202,7 @@ int ObjectDirectory::copy(CacheObj* object, std::string copyName, std::string co int ObjectDirectory::del(CacheObj* object, optional_yield y) { std::string key = build_index(object); - if (exist_key(key, y)) { + if (exist_key(object, y)) { try { boost::system::error_code ec; request req; @@ -248,7 +227,8 @@ std::string BlockDirectory::build_index(CacheBlock* block) { return block->cacheObj.bucketName + "_" + block->cacheObj.objName + "_" + std::to_string(block->blockID); } -int BlockDirectory::exist_key(std::string key, optional_yield y) { +int BlockDirectory::exist_key(CacheBlock* block, optional_yield y) { + std::string key = build_index(block); response resp; try { @@ -274,8 +254,8 @@ void BlockDirectory::shutdown() int BlockDirectory::set(CacheBlock* block, optional_yield y) { std::string key = build_index(block); - /* Every set will be treated as new */ // or maybe, if key exists, simply return? -Sam - std::string endpoint = cct->_conf->rgw_d4n_host + ":" + std::to_string(cct->_conf->rgw_d4n_port); + /* Every set will be treated as new */ + std::string endpoint; std::list redisValues; /* Creating a redisValues of the entry's properties */ @@ -288,6 +268,15 @@ int BlockDirectory::set(CacheBlock* block, optional_yield y) { redisValues.push_back("globalWeight"); redisValues.push_back(std::to_string(block->globalWeight)); redisValues.push_back("blockHosts"); + + for (auto const& host : block->hostsList) { + if (endpoint.empty()) + endpoint = host + "_"; + else + endpoint = endpoint + host + "_"; + } + + endpoint.pop_back(); redisValues.push_back(endpoint); // Set in filter -Sam redisValues.push_back("objName"); @@ -299,6 +288,16 @@ int BlockDirectory::set(CacheBlock* block, optional_yield y) { redisValues.push_back("dirty"); redisValues.push_back(std::to_string(block->cacheObj.dirty)); redisValues.push_back("objHosts"); + + endpoint.clear(); + for (auto const& host : block->cacheObj.hostsList) { + if (endpoint.empty()) + endpoint = host + "_"; + else + endpoint = endpoint + host + "_"; + } + + endpoint.pop_back(); redisValues.push_back(endpoint); // Set in filter -Sam try { @@ -322,7 +321,7 @@ int BlockDirectory::set(CacheBlock* block, optional_yield y) { int BlockDirectory::get(CacheBlock* block, optional_yield y) { std::string key = build_index(block); - if (exist_key(key, y)) { + if (exist_key(block, y)) { std::vector fields; fields.push_back("blockID"); @@ -395,7 +394,7 @@ int BlockDirectory::copy(CacheBlock* block, std::string copyName, std::string co auto copyBlock = CacheBlock{ .cacheObj = { .objName = copyName, .bucketName = copyBucketName }, .blockID = 0 }; std::string copyKey = build_index(©Block); - if (exist_key(key, y)) { + if (exist_key(block, y)) { try { response resp; @@ -436,7 +435,7 @@ int BlockDirectory::copy(CacheBlock* block, std::string copyName, std::string co int BlockDirectory::del(CacheBlock* block, optional_yield y) { std::string key = build_index(block); - if (exist_key(key, y)) { + if (exist_key(block, y)) { try { boost::system::error_code ec; request req; @@ -460,7 +459,7 @@ int BlockDirectory::del(CacheBlock* block, optional_yield y) { int BlockDirectory::update_field(CacheBlock* block, std::string field, std::string value, optional_yield y) { std::string key = build_index(block); - if (exist_key(key, y)) { + if (exist_key(block, y)) { try { /* Ensure field exists */ { @@ -475,7 +474,7 @@ int BlockDirectory::update_field(CacheBlock* block, std::string field, std::stri return -1; } - if (field == "blockHosts" || field == "objHosts") { + if (field == "blockHosts") { // Need one for object hosts? -Sam /* Append rather than overwrite */ boost::system::error_code ec; request req; @@ -514,4 +513,71 @@ int BlockDirectory::update_field(CacheBlock* block, std::string field, std::stri } } +int BlockDirectory::remove_host(CacheBlock* block, std::string delValue, optional_yield y) { + std::string key = build_index(block); + + if (exist_key(block, y)) { + try { + /* Ensure field exists */ + { + boost::system::error_code ec; + request req; + req.push("HEXISTS", key, "blockHosts"); + response resp; + + redis_exec(conn, ec, req, resp, y); + + if (!std::get<0>(resp).value() || (bool)ec) + return -1; + } + + { + boost::system::error_code ec; + request req; + req.push("HGET", key, "blockHosts"); + response resp; + + redis_exec(conn, ec, req, resp, y); + + if (!std::get<0>(resp).value().size() || (bool)ec) + return -1; + + if (std::get<0>(resp).value().find("_") == std::string::npos) /* Last host, delete entirely */ + return del(block, y); + + std::string result = std::get<0>(resp).value(); + auto it = result.find(delValue); + if (it != std::string::npos) + result.erase(result.begin() + it, result.begin() + it + delValue.size()); + else + return -1; + + if (result[0] == '_') + result.erase(0, 1); + + delValue = result; + } + + { + boost::system::error_code ec; + request req; + req.push_range("HSET", key, std::map{{"blockHosts", delValue}}); + response resp; + + redis_exec(conn, ec, req, resp, y); + + if ((bool)ec) { + return -1; + } + + return std::get<0>(resp).value(); + } + } catch(std::exception &e) { + return -1; + } + } else { + return -2; + } +} + } } // namespace rgw::d4n diff --git a/src/rgw/driver/d4n/d4n_directory.h b/src/rgw/driver/d4n/d4n_directory.h index fc10754b06374..8bf618e59eda6 100644 --- a/src/rgw/driver/d4n/d4n_directory.h +++ b/src/rgw/driver/d4n/d4n_directory.h @@ -66,7 +66,7 @@ class ObjectDirectory: public Directory { // weave into write workflow -Sam return 0; } - int exist_key(std::string key, optional_yield y); + int exist_key(CacheObj* object, optional_yield y); void shutdown(); int set(CacheObj* object, optional_yield y); @@ -96,7 +96,7 @@ class BlockDirectory: public Directory { cfg.addr.host = cct->_conf->rgw_d4n_host; cfg.addr.port = std::to_string(cct->_conf->rgw_d4n_port); - if (!cfg.addr.host.length() || !cfg.addr.port.length()) { // add logs to other methods -Sam + if (!cfg.addr.host.length() || !cfg.addr.port.length()) { ldpp_dout(dpp, 10) << "D4N Directory " << __func__ << ": Block directory endpoint was not configured correctly" << dendl; return -EDESTADDRREQ; } @@ -105,8 +105,7 @@ class BlockDirectory: public Directory { return 0; } - - int exist_key(std::string key, optional_yield y); + int exist_key(CacheBlock* block, optional_yield y); void shutdown(); int set(CacheBlock* block, optional_yield y); @@ -114,6 +113,7 @@ class BlockDirectory: public Directory { int copy(CacheBlock* block, std::string copyName, std::string copyBucketName, optional_yield y); int del(CacheBlock* block, optional_yield y); int update_field(CacheBlock* block, std::string field, std::string value, optional_yield y); + int remove_host(CacheBlock* block, std::string value, optional_yield y); private: std::shared_ptr conn; diff --git a/src/rgw/driver/d4n/d4n_policy.cc b/src/rgw/driver/d4n/d4n_policy.cc index 3981fb0ab4164..449596b13b02c 100644 --- a/src/rgw/driver/d4n/d4n_policy.cc +++ b/src/rgw/driver/d4n/d4n_policy.cc @@ -227,8 +227,7 @@ int LFUDAPolicy::get_block(const DoutPrefixProvider* dpp, CacheBlock* block, rgw return -1; } - std::string key; // = dir->build_index(block); - int exists = dir->exist_key(key, y); + int exists = dir->exist_key(block, y); if (exists > 0) { /* Remote copy */ if (dir->get(block, y) < 0) { return -1; @@ -302,7 +301,7 @@ uint64_t LFUDAPolicy::eviction(const DoutPrefixProvider* dpp, rgw::cache::CacheD ldpp_dout(dpp, 10) << "RGW D4N Policy: Block " << victim.cacheObj.objName << " has been evicted." << dendl; - if (cacheNode->del(dpp, victim.cacheObj.objName, y) < 0) {//} && dir->remove_host(&victim, ""/* local cache address */, y) < 0) { + if (cacheNode->del(dpp, victim.cacheObj.objName, y) < 0 && dir->remove_host(&victim, ""/* local cache address */, y) < 0) { return 0; } else { uint64_t num_entries = entries_map.size(); diff --git a/src/rgw/driver/d4n/rgw_sal_d4n.cc b/src/rgw/driver/d4n/rgw_sal_d4n.cc index 6f90f34c1db6a..c78c28d9852f0 100644 --- a/src/rgw/driver/d4n/rgw_sal_d4n.cc +++ b/src/rgw/driver/d4n/rgw_sal_d4n.cc @@ -824,7 +824,7 @@ int D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::handle_data(bufferlist& bl if (filter->get_cache_driver()->put_async(dpp, oid, bl, bl.length(), source->get_attrs()) == 0) { filter->get_policy_driver()->get_cache_policy()->update(dpp, oid, ofs, bl.length(), "", filter->get_cache_driver(), null_yield); /* Store block in directory */ - if (!blockDir->exist_key(oid, null_yield)) { + if (!blockDir->exist_key(&block, null_yield)) { #if 0 int ret = blockDir->set_value(&block); if (ret < 0) { @@ -848,7 +848,7 @@ int D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::handle_data(bufferlist& bl if (filter->get_cache_driver()->put_async(dpp, oid, bl, bl.length(), source->get_attrs()) == 0) { filter->get_policy_driver()->get_cache_policy()->update(dpp, oid, ofs, bl.length(), "", filter->get_cache_driver(), null_yield); /* Store block in directory */ - if (!blockDir->exist_key(oid, null_yield)) { + if (!blockDir->exist_key(&block, null_yield)) { #if 0 int ret = blockDir->set_value(&block); if (ret < 0) { @@ -880,7 +880,7 @@ int D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::handle_data(bufferlist& bl if (filter->get_cache_driver()->put_async(dpp, oid, bl_rem, bl_rem.length(), source->get_attrs()) == 0) { filter->get_policy_driver()->get_cache_policy()->update(dpp, oid, ofs, bl_rem.length(), "", filter->get_cache_driver(), null_yield); /* Store block in directory */ - if (!blockDir->exist_key(oid, null_yield)) { + if (!blockDir->exist_key(&block, null_yield)) { #if 0 int ret = blockDir->set_value(&block); if (ret < 0) { -- 2.39.5