From 0e9f895dec1d00780be1ae38b4892db99f4ec537 Mon Sep 17 00:00:00 2001 From: Samarah Date: Mon, 24 Mar 2025 15:20:54 +0000 Subject: [PATCH] rgw/d4n: squashing commits related to changing creationTime type from time_t to double. 1. rgw/d4n: Change score calculation 2. rgw/d4n: Update creationTime calculation to include decimal value 3. rgw/d4n: changes for converting string value to time_point, when creationTime type changed to double. Co-authored-by: Pritha Srivastava Updated bucket::list to convert from string to time_point Signed-off-by: Samarah Signed-off-by: Pritha Srivastava --- src/rgw/driver/d4n/d4n_directory.cc | 2 +- src/rgw/driver/d4n/d4n_policy.cc | 16 ++++++++-------- src/rgw/driver/d4n/d4n_policy.h | 22 +++++++++++----------- src/rgw/driver/d4n/rgw_sal_d4n.cc | 27 ++++++++++++++------------- 4 files changed, 34 insertions(+), 33 deletions(-) diff --git a/src/rgw/driver/d4n/d4n_directory.cc b/src/rgw/driver/d4n/d4n_directory.cc index 085e61b2ae6..5600f34812a 100644 --- a/src/rgw/driver/d4n/d4n_directory.cc +++ b/src/rgw/driver/d4n/d4n_directory.cc @@ -508,7 +508,7 @@ int ObjectDirectory::update_field(const DoutPrefixProvider* dpp, CacheObj* objec return -EINVAL; } } else if (ret == -ENOENT) { - ldpp_dout(dpp, 0) << "ObjectDirectory::" << __func__ << "(): Object does not exist." << dendl; + ldpp_dout(dpp, 10) << "ObjectDirectory::" << __func__ << "(): Object does not exist." << dendl; } else { ldpp_dout(dpp, 0) << "ObjectDirectory::" << __func__ << "(): ERROR: ret=" << ret << dendl; } diff --git a/src/rgw/driver/d4n/d4n_policy.cc b/src/rgw/driver/d4n/d4n_policy.cc index 18a9e3b29df..6d9c51773ec 100644 --- a/src/rgw/driver/d4n/d4n_policy.cc +++ b/src/rgw/driver/d4n/d4n_policy.cc @@ -49,13 +49,13 @@ static inline void redis_exec(std::shared_ptr conn, } } -int LFUDAPolicy::init(CephContext *cct, const DoutPrefixProvider* dpp, asio::io_context& io_context, rgw::sal::Driver *_driver) { +int LFUDAPolicy::init(CephContext* cct, const DoutPrefixProvider* dpp, asio::io_context& io_context, rgw::sal::Driver* _driver) { response resp; static auto obj_callback = [this]( - const DoutPrefixProvider* dpp, const std::string& key, const std::string& version, bool dirty, uint64_t size, - time_t creationTime, const rgw_user user, const std::string& etag, const std::string& bucket_name, const std::string& bucket_id, + const DoutPrefixProvider* dpp, const std::string& key, const std::string& version, bool deleteMarker, uint64_t size, + double creationTime, const rgw_user user, const std::string& etag, const std::string& bucket_name, const std::string& bucket_id, const rgw_obj_key& obj_key, optional_yield y, std::string& restore_val) { - update_dirty_object(dpp, key, version, dirty, size, creationTime, user, etag, bucket_name, bucket_id, obj_key, RefCount::NOOP, y, restore_val); + update_dirty_object(dpp, key, version, deleteMarker, size, creationTime, user, etag, bucket_name, bucket_id, obj_key, RefCount::NOOP, y, restore_val); }; static auto block_callback = [this]( @@ -310,7 +310,7 @@ CacheBlock* LFUDAPolicy::get_victim_block(const DoutPrefixProvider* dpp, optiona return victim; } -int LFUDAPolicy::exist_key(std::string key) { +int LFUDAPolicy::exist_key(const std::string& key) { const std::lock_guard l(lfuda_lock); if (entries_map.count(key) != 0) { return true; @@ -480,7 +480,7 @@ void LFUDAPolicy::update(const DoutPrefixProvider* dpp, const std::string& key, weightSum += ((localWeight < 0) ? 0 : localWeight); } -void LFUDAPolicy::update_dirty_object(const DoutPrefixProvider* dpp, const std::string& key, const std::string& version, bool deleteMarker, uint64_t size, time_t creationTime, const rgw_user& user, const std::string& etag, const std::string& bucket_name, const std::string& bucket_id, const rgw_obj_key& obj_key, uint8_t op, optional_yield y, std::string& restore_val) +void LFUDAPolicy::update_dirty_object(const DoutPrefixProvider* dpp, const std::string& key, const std::string& version, bool deleteMarker, uint64_t size, double creationTime, const rgw_user& user, const std::string& etag, const std::string& bucket_name, const std::string& bucket_id, const rgw_obj_key& obj_key, uint8_t op, optional_yield y, std::string& restore_val) { using handle_type = boost::heap::fibonacci_heap>>::handle_type; State state{State::INIT}; @@ -969,7 +969,7 @@ void LFUDAPolicy::cleaning(const DoutPrefixProvider* dpp) } //end-while true } -int LRUPolicy::exist_key(std::string key) +int LRUPolicy::exist_key(const std::string& key) { const std::lock_guard l(lru_lock); if (entries_map.count(key) != 0) { @@ -1008,7 +1008,7 @@ void LRUPolicy::update(const DoutPrefixProvider* dpp, const std::string& key, ui entries_map.emplace(key, e); } -void LRUPolicy::update_dirty_object(const DoutPrefixProvider* dpp, const std::string& key, const std::string& version, bool deleteMarker, uint64_t size, time_t creationTime, const rgw_user& user, const std::string& etag, const std::string& bucket_name, const std::string& bucket_id, +void LRUPolicy::update_dirty_object(const DoutPrefixProvider* dpp, const std::string& key, const std::string& version, bool deleteMarker, uint64_t size, double creationTime, const rgw_user& user, const std::string& etag, const std::string& bucket_name, const std::string& bucket_id, const rgw_obj_key& obj_key, uint8_t op, optional_yield y, std::string& restore_val) { const std::lock_guard l(lru_lock); diff --git a/src/rgw/driver/d4n/d4n_policy.h b/src/rgw/driver/d4n/d4n_policy.h index 3677d19ff92..00a157b7789 100644 --- a/src/rgw/driver/d4n/d4n_policy.h +++ b/src/rgw/driver/d4n/d4n_policy.h @@ -54,7 +54,7 @@ class CachePolicy { std::string version; bool delete_marker; uint64_t size; - time_t creationTime; + double creationTime; rgw_user user; std::string etag; std::string bucket_name; @@ -62,7 +62,7 @@ class CachePolicy { rgw_obj_key obj_key; ObjEntry() = default; ObjEntry(const std::string& key, const std::string& version, bool delete_marker, uint64_t size, - time_t creationTime, rgw_user user, const std::string& etag, + double creationTime, const rgw_user& user, const std::string& etag, const std::string& bucket_name, const std::string& bucket_id, const rgw_obj_key& obj_key) : key(key), version(version), delete_marker(delete_marker), size(size), creationTime(creationTime), user(user), etag(etag), bucket_name(bucket_name), bucket_id(bucket_id), obj_key(obj_key) {} @@ -73,12 +73,12 @@ class CachePolicy { virtual ~CachePolicy() = default; virtual int init(CephContext* cct, const DoutPrefixProvider* dpp, asio::io_context& io_context, rgw::sal::Driver* _driver) = 0; - virtual int exist_key(std::string key) = 0; + virtual int exist_key(const std::string& key) = 0; virtual int eviction(const DoutPrefixProvider* dpp, uint64_t size, optional_yield y) = 0; virtual bool update_refcount_if_key_exists(const DoutPrefixProvider* dpp, const std::string& key, uint8_t op, optional_yield y) = 0; virtual void update(const DoutPrefixProvider* dpp, const std::string& key, uint64_t offset, uint64_t len, const std::string& version, bool dirty, uint8_t op, optional_yield y, std::string& restore_val=empty) = 0; virtual void update_dirty_object(const DoutPrefixProvider* dpp, const std::string& key, const std::string& version, bool deleteMarker, uint64_t size, - time_t creationTime, const rgw_user& user, const std::string& etag, const std::string& bucket_name, const std::string& bucket_id, + double creationTime, const rgw_user& user, const std::string& etag, const std::string& bucket_name, const std::string& bucket_id, const rgw_obj_key& obj_key, uint8_t op, optional_yield y, std::string& restore_val=empty) = 0; virtual bool erase(const DoutPrefixProvider* dpp, const std::string& key, optional_yield y) = 0; virtual bool erase_dirty_object(const DoutPrefixProvider* dpp, const std::string& key, optional_yield y) = 0; @@ -132,7 +132,7 @@ class LFUDAPolicy : public CachePolicy { handle_type handle; LFUDAObjEntry(const std::string& key, const std::string& version, bool deleteMarker, uint64_t size, - time_t creationTime, rgw_user user, const std::string& etag, + double creationTime, const rgw_user& user, const std::string& etag, const std::string& bucket_name, const std::string& bucket_id, const rgw_obj_key& obj_key) : ObjEntry(key, version, deleteMarker, size, creationTime, user, etag, bucket_name, bucket_id, obj_key) {} @@ -201,7 +201,7 @@ class LFUDAPolicy : public CachePolicy { } virtual int init(CephContext *cct, const DoutPrefixProvider* dpp, asio::io_context& io_context, rgw::sal::Driver *_driver); - virtual int exist_key(std::string key) override; + virtual int exist_key(const std::string& key) override; virtual int eviction(const DoutPrefixProvider* dpp, uint64_t size, optional_yield y) override; virtual bool update_refcount_if_key_exists(const DoutPrefixProvider* dpp, const std::string& key, uint8_t op, optional_yield y) override; virtual void update(const DoutPrefixProvider* dpp, const std::string& key, uint64_t offset, uint64_t len, const std::string& version, bool dirty, uint8_t op, optional_yield y, std::string& restore_val=empty) override; @@ -209,7 +209,7 @@ class LFUDAPolicy : public CachePolicy { virtual bool _erase(const DoutPrefixProvider* dpp, const std::string& key, optional_yield y); void save_y(optional_yield y) { this->y = y; } virtual void update_dirty_object(const DoutPrefixProvider* dpp, const std::string& key, const std::string& version, bool deleteMarker, uint64_t size, - time_t creationTime, const rgw_user& user, const std::string& etag, const std::string& bucket_name, const std::string& bucket_id, + double creationTime, const rgw_user& user, const std::string& etag, const std::string& bucket_name, const std::string& bucket_id, const rgw_obj_key& obj_key, uint8_t op, optional_yield y, std::string& restore_val=empty) override; virtual bool erase_dirty_object(const DoutPrefixProvider* dpp, const std::string& key, optional_yield y) override; virtual bool invalidate_dirty_object(const DoutPrefixProvider* dpp, const std::string& key) override; @@ -239,13 +239,13 @@ class LRUPolicy : public CachePolicy { LRUPolicy(rgw::cache::CacheDriver* cacheDriver) : cacheDriver{cacheDriver} {} virtual int init(CephContext* cct, const DoutPrefixProvider* dpp, asio::io_context& io_context, rgw::sal::Driver* _driver) { return 0; } - virtual int exist_key(std::string key) override; + virtual int exist_key(const std::string& key) override; virtual int eviction(const DoutPrefixProvider* dpp, uint64_t size, optional_yield y) override; virtual bool update_refcount_if_key_exists(const DoutPrefixProvider* dpp, const std::string& key, uint8_t op, optional_yield y) override { return false; } virtual void update(const DoutPrefixProvider* dpp, const std::string& key, uint64_t offset, uint64_t len, const std::string& version, bool dirty, uint8_t op, optional_yield y, std::string& restore_val=empty) override; - virtual void update_dirty_object(const DoutPrefixProvider* dpp, const std::string& key, const std::string& version, bool deleteMarker, uint64_t size, - time_t creationTime, const rgw_user& user, const std::string& etag, const std::string& bucket_name, const std::string& bucket_id, - const rgw_obj_key& obj_key, uint8_t op, optional_yield y, std::string& restore_val=empty) override; + virtual void update_dirty_object(const DoutPrefixProvider* dpp, const std::string& key, const std::string& version, bool deleteMarker, uint64_t size, + double creationTime, const rgw_user& user, const std::string& etag, const std::string& bucket_name, const std::string& bucket_id, + const rgw_obj_key& obj_key, uint8_t op, optional_yield y, std::string& restore_val=empty) override; virtual bool erase(const DoutPrefixProvider* dpp, const std::string& key, optional_yield y) override; virtual bool erase_dirty_object(const DoutPrefixProvider* dpp, const std::string& key, optional_yield y) override; virtual bool invalidate_dirty_object(const DoutPrefixProvider* dpp, const std::string& key) override { return false; } diff --git a/src/rgw/driver/d4n/rgw_sal_d4n.cc b/src/rgw/driver/d4n/rgw_sal_d4n.cc index cd161b241e4..7ba7ea40618 100644 --- a/src/rgw/driver/d4n/rgw_sal_d4n.cc +++ b/src/rgw/driver/d4n/rgw_sal_d4n.cc @@ -478,11 +478,12 @@ int D4NFilterBucket::list(const DoutPrefixProvider* dpp, ListParams& params, int entry.meta.storage_class = "CACHE"; entry.meta.size = block.cacheObj.size; entry.meta.accounted_size = block.cacheObj.size; - struct std::tm tm; - std::istringstream ss(block.cacheObj.creationTime); - ss >> std::get_time(&tm, "%H:%M:%S"); - std::time_t creationTime = mktime(&tm); - entry.meta.mtime = ceph::real_clock::from_time_t(creationTime); + try { + double time = std::stod(block.cacheObj.creationTime); + entry.meta.mtime = ceph::real_clock::from_double(time); + } catch (const std::exception& e) { + ldpp_dout(dpp, 0) << "D4NFilterBucket::" << __func__ << " Invalid value of time: " << block.cacheObj.creationTime << dendl; + } entry.meta.etag = block.cacheObj.etag; entry.meta.owner = block.cacheObj.user_id; entry.meta.owner_display_name = block.cacheObj.display_name; @@ -740,7 +741,7 @@ int D4NFilterObject::copy_object(const ACLOwner& owner, } else { dest_mtime = real_clock::now(); } - creationTime = ceph::real_clock::to_time_t(dest_mtime); + creationTime = ceph::real_clock::to_double(dest_mtime); dest_object->set_mtime(dest_mtime); dest_object->set_obj_size(this->get_size()); dest_object->set_accounted_size(this->get_accounted_size()); @@ -1062,7 +1063,7 @@ int D4NFilterObject::create_delete_marker(const DoutPrefixProvider* dpp, optiona ldpp_dout(dpp, 0) << "D4NFilterObject::" << __func__ << "(): BlockDirectory set method failed for head object, ret=" << ret << dendl; return ret; } - auto creationTime = ceph::real_clock::to_time_t(this->get_mtime()); + auto creationTime = ceph::real_clock::to_double(this->get_mtime()); ldpp_dout(dpp, 0) << "D4NFilterObject::" << __func__ << "(): key=" << key << dendl; std::string objEtag = ""; driver->get_policy_driver()->get_cache_policy()->update_dirty_object(dpp, key, version, true, this->get_accounted_size(), creationTime, std::get(this->get_bucket()->get_owner()), objEtag, this->get_bucket()->get_name(), this->get_bucket()->get_bucket_id(), this->get_key(), rgw::d4n::RefCount::NOOP, y); @@ -1129,7 +1130,7 @@ int D4NFilterObject::set_head_obj_dir_entry(const DoutPrefixProvider* dpp, std:: rgw::d4n::CacheObj object = rgw::d4n::CacheObj{ .objName = objName, .bucketName = this->get_bucket()->get_bucket_id(), - .creationTime = std::to_string(ceph::real_clock::to_time_t(this->get_mtime())), + .creationTime = std::to_string(ceph::real_clock::to_double(this->get_mtime())), .dirty = dirty, .hostsList = { dpp->get_cct()->_conf->rgw_d4n_l1_datacache_address }, .etag = etag, @@ -1174,8 +1175,8 @@ int D4NFilterObject::set_head_obj_dir_entry(const DoutPrefixProvider* dpp, std:: object_version = this->get_object_version(); } auto mtime = this->get_mtime(); - auto score = ceph::real_clock::to_time_t(mtime); - ldpp_dout(dpp, 10) << "D4NFilterObject::" << __func__ << "(): Score of object name: "<< this->get_name() << " version: " << object_version << " is: " << std::setprecision(std::numeric_limits::max_digits10) << score << ret << dendl; + auto score = ceph::real_clock::to_double(mtime); + ldpp_dout(dpp, 10) << "D4NFilterObject::" << __func__ << "(): Score of object name: "<< this->get_name() << " version: " << object_version << " is: " << score << ret << dendl; rgw::d4n::ObjectDirectory* objDir = this->driver->get_obj_dir(); ret = objDir->zadd(dpp, &object, score, object_version, y); if (ret < 0) { @@ -1253,7 +1254,7 @@ int D4NFilterObject::set_head_obj_dir_entry(const DoutPrefixProvider* dpp, std:: rgw::d4n::CacheObj version_object = rgw::d4n::CacheObj{ .objName = objName, .bucketName = this->get_bucket()->get_bucket_id(), - .creationTime = std::to_string(ceph::real_clock::to_time_t(this->get_mtime())), + .creationTime = std::to_string(ceph::real_clock::to_double(this->get_mtime())), .dirty = dirty, .etag = etag, .size = this->get_accounted_size(), @@ -2262,7 +2263,7 @@ int D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::handle_data(bufferlist& bl block.cacheObj.objName = source->get_key().get_oid(); block.cacheObj.bucketName = source->get_bucket()->get_bucket_id(); std::stringstream s; - block.cacheObj.creationTime = std::to_string(ceph::real_clock::to_time_t(source->get_mtime())); + block.cacheObj.creationTime = std::to_string(ceph::real_clock::to_double(source->get_mtime())); bool dirty = block.cacheObj.dirty = false; //Reading from the backend, data is clean block.version = version; @@ -2984,7 +2985,7 @@ int D4NFilterWriter::complete(size_t accounted_size, const std::string& etag, return ret; } if (dirty) { - auto creationTime = ceph::real_clock::to_time_t(object->get_mtime()); + auto creationTime = ceph::real_clock::to_double(object->get_mtime()); ldpp_dout(dpp, 20) << "D4NFilterWriter::" << __func__ << "(): key=" << key << dendl; ldpp_dout(dpp, 20) << "D4NFilterWriter::" << __func__ << "(): obj->get_key()=" << obj->get_key() << dendl; driver->get_policy_driver()->get_cache_policy()->update_dirty_object(dpp, key, version, false, accounted_size, creationTime, std::get(obj->get_bucket()->get_owner()), objEtag, obj->get_bucket()->get_name(), obj->get_bucket()->get_bucket_id(), obj->get_key(), rgw::d4n::RefCount::NOOP, y); -- 2.39.5