}
}
erase(dpp, key, y);
- LFUDAEntry *e = new LFUDAEntry(key, offset, len, version, dirty, localWeight);
+ LFUDAEntry* e = new LFUDAEntry(key, offset, len, version, dirty, localWeight);
handle_type handle = entries_heap.push(e);
e->set_handle(handle);
entries_map.emplace(key, e);
weightSum += ((localWeight < 0) ? 0 : localWeight);
}
-void LFUDAPolicy::updateObj(const DoutPrefixProvider* dpp, std::string& key, std::string version, bool dirty, uint64_t size, time_t creationTime, const rgw_user user, std::string& etag, const std::string& bucket_name, const rgw_obj_key& obj_key, optional_yield y)
+void LFUDAPolicy::update_dirty_object(const DoutPrefixProvider* dpp, const std::string& key, const std::string& version, bool dirty, uint64_t size, time_t creationTime, const rgw_user& user, std::string& etag, const std::string& bucket_name, const std::string& bucket_id, const rgw_obj_key& obj_key, optional_yield y)
{
using handle_type = boost::heap::fibonacci_heap<LFUDAObjEntry*, boost::heap::compare<ObjectComparator<LFUDAObjEntry>>>::handle_type;
ldpp_dout(dpp, 10) << "LFUDAPolicy::" << __func__ << "(): Before acquiring lock." << dendl;
const std::lock_guard l(lfuda_cleaning_lock);
- LFUDAObjEntry *e = new LFUDAObjEntry{key, version, dirty, size, creationTime, user, etag, bucket_name, obj_key};
+ LFUDAObjEntry* e = new LFUDAObjEntry{key, version, dirty, size, creationTime, user, etag, bucket_name, bucket_id, obj_key};
handle_type handle = object_heap.push(e);
e->set_handle(handle);
o_entries_map.emplace(key, e);
return true;
}
-bool LFUDAPolicy::eraseObj(const DoutPrefixProvider* dpp, const std::string& key, optional_yield y)
+bool LFUDAPolicy::erase_dirty_object(const DoutPrefixProvider* dpp, const std::string& key, optional_yield y)
{
const std::lock_guard l(lfuda_cleaning_lock);
auto p = o_entries_map.find(key);
const int interval = dpp->get_cct()->_conf->rgw_d4n_cache_cleaning_interval;
while(!quit) {
ldpp_dout(dpp, 20) << __func__ << " : " << " Cache cleaning!" << dendl;
- std::string name = "";
- std::string b_name = "";
- std::string key = "";
uint64_t len = 0;
rgw::sal::Attrs obj_attrs;
std::unique_ptr<rgw::sal::User> c_user = driver->get_user(c_rgw_user);
std::unique_ptr<rgw::sal::Bucket> c_bucket;
- rgw_bucket c_rgw_bucket = rgw_bucket(c_rgw_user.tenant, e->bucket_name, "");
+ rgw_bucket c_rgw_bucket = rgw_bucket(c_rgw_user.tenant, e->bucket_name, e->bucket_id);
RGWBucketInfo c_bucketinfo;
c_bucketinfo.bucket = c_rgw_bucket;
int ret = driver->load_bucket(dpp, c_rgw_bucket, &c_bucket, null_yield);
if (ret < 0) {
ldpp_dout(dpp, 10) << __func__ << "(): load_bucket() returned ret=" << ret << dendl;
- //delete the object from the cache and update all internal metadata?
- //TODO: clean up all dirty blocks from cache and update metadata
- eraseObj(dpp, e->key, null_yield);
+ //Remove bucket should be implemented in d4n which will take care of deleting objects belonging to the bucket, and hence we should not reach here
+ erase_dirty_object(dpp, e->key, null_yield);
continue;
}
int op_ret = processor->prepare(null_yield);
if (op_ret < 0) {
ldpp_dout(dpp, 20) << __func__ << "processor->prepare() returned ret=" << op_ret << dendl;
- eraseObj(dpp, e->key, null_yield);
+ erase_dirty_object(dpp, e->key, null_yield);
}
- std::string prefix = e->bucket_name + "_" + e->version + "_" + c_obj->get_name();
+ std::string prefix = e->bucket_id + "_" + e->version + "_" + c_obj->get_name();
off_t lst = e->size;
off_t fst = 0;
off_t ofs = 0;
- rgw::sal::DataProcessor *filter = processor.get();
+ rgw::sal::DataProcessor* filter = processor.get();
std::string head_oid_in_cache = "D_" + prefix;
std::string new_head_oid_in_cache = prefix;
ldpp_dout(dpp, 10) << __func__ << "(): head_oid_in_cache=" << head_oid_in_cache << dendl;
if (op_ret < 0) {
ldpp_dout(dpp, 20) << __func__ << "processor->process() returned ret="
<< op_ret << dendl;
- eraseObj(dpp, e->key, null_yield);
+ erase_dirty_object(dpp, e->key, null_yield);
}
ofs += len;
if (op_ret < 0) {
ldpp_dout(dpp, 20) << __func__ << "processor->complete() returned ret=" << op_ret << dendl;
- eraseObj(dpp, e->key, null_yield);
+ erase_dirty_object(dpp, e->key, null_yield);
}
//invoke update() with dirty flag set to false, to update in-memory metadata for each block
// reset values
this->update(dpp, new_oid_in_cache, 0, 0, e->version, false, y);
rgw::d4n::CacheBlock block;
- block.cacheObj.bucketName = c_obj->get_bucket()->get_name();
+ block.cacheObj.bucketName = c_obj->get_bucket()->get_bucket_id();
block.cacheObj.objName = c_obj->get_key().get_oid();
block.size = cur_len;
block.blockID = fst;
this->update(dpp, new_head_oid_in_cache, 0, 0, e->version, false, y);
rgw::d4n::CacheBlock block;
- block.cacheObj.bucketName = c_obj->get_bucket()->get_name();
+ block.cacheObj.bucketName = c_obj->get_bucket()->get_bucket_id();
block.cacheObj.objName = c_obj->get_name();
block.size = 0;
block.blockID = 0;
ldpp_dout(dpp, 20) << __func__ << "updating dirty flag in block directory for head failed!" << dendl;
}
}
- //In case of distributed cache, we may have to update this for every instance
if (c_obj->have_instance()) {
rgw::d4n::CacheBlock instance_block;
- instance_block.cacheObj.bucketName = c_obj->get_bucket()->get_name();
+ instance_block.cacheObj.bucketName = c_obj->get_bucket()->get_bucket_id();
instance_block.cacheObj.objName = c_obj->get_oid();
instance_block.size = 0;
instance_block.blockID = 0;
ldpp_dout(dpp, 20) << __func__ << "updating dirty flag in block directory for instance block failed!" << dendl;
}
}
-
- op_ret = blockDir->update_field(dpp, &block, "dirty", "false", null_yield);
- if (op_ret < 0) {
- ldpp_dout(dpp, 0) << __func__ << "updating dirty flag in block directory for head failed, ret=" << op_ret << dendl;
- }
-
- //remove entry from map and queue, eraseObj locks correctly
- rgw::d4n::CacheObj obj;
- obj.bucketName = c_obj->get_bucket()->get_name();
- obj.objName = c_obj->get_key().get_oid();
- op_ret = objDir->update_field(dpp, &obj, "dirty", "false", null_yield);
- if (op_ret < 0) {
- ldpp_dout(dpp, 0) << __func__ << "updating dirty flag in object directory failed, ret=" << op_ret << dendl;
- return;
- }
-
- op_ret = blockDir->update_field(dpp, &block, "dirty", "false", null_yield);
- if (op_ret < 0) {
- ldpp_dout(dpp, 0) << __func__ << "updating dirty flag in block directory failed, ret=" << op_ret << dendl;
- return;
- }
-
- eraseObj(dpp, e->key, null_yield);
+ //remove entry from map and queue, erase_dirty_object locks correctly
+ erase_dirty_object(dpp, e->key, null_yield);
} else { //end-if std::difftime(time(NULL), e->creationTime) > interval
std::this_thread::sleep_for(std::chrono::milliseconds(interval)); //TODO:: should this time be optimised?
}
entries_map.emplace(key, e);
}
-void LRUPolicy::updateObj(const DoutPrefixProvider* dpp, std::string& key, std::string version, bool dirty, uint64_t size, time_t creationTime, const rgw_user user, std::string& etag, const std::string& bucket_name, const rgw_obj_key& obj_key, optional_yield y)
+void LRUPolicy::update_dirty_object(const DoutPrefixProvider* dpp, const std::string& key, const std::string& version, bool dirty, uint64_t size, time_t creationTime, const rgw_user& user, std::string& etag, const std::string& bucket_name, const std::string& bucket_id,
+const rgw_obj_key& obj_key, optional_yield y)
{
const std::lock_guard l(lru_lock);
- ObjEntry* e = new ObjEntry(key, version, dirty, size, creationTime, user, etag, bucket_name, obj_key);
+ ObjEntry* e = new ObjEntry(key, version, dirty, size, creationTime, user, etag, bucket_name, bucket_id, obj_key);
o_entries_map.emplace(key, e);
return;
}
return _erase(dpp, key, y);
}
-bool LRUPolicy::eraseObj(const DoutPrefixProvider* dpp, const std::string& key, optional_yield y)
+bool LRUPolicy::erase_dirty_object(const DoutPrefixProvider* dpp, const std::string& key, optional_yield y)
{
const std::lock_guard l(lru_lock);
auto p = o_entries_map.find(key);
rgw_user user;
std::string etag;
std::string bucket_name;
+ std::string bucket_id;
rgw_obj_key obj_key;
ObjEntry() = default;
- ObjEntry(std::string& key, std::string version, bool dirty, uint64_t size,
+ ObjEntry(const std::string& key, const std::string& version, bool dirty, uint64_t size,
time_t creationTime, rgw_user user, std::string& etag,
- const std::string& bucket_name, const rgw_obj_key& obj_key) : key(key), version(version), dirty(dirty), size(size),
+ const std::string& bucket_name, const std::string& bucket_id, const rgw_obj_key& obj_key) : key(key), version(version), dirty(dirty), size(size),
creationTime(creationTime), user(user), etag(etag),
- bucket_name(bucket_name), obj_key(obj_key) {}
+ bucket_name(bucket_name), bucket_id(bucket_id), obj_key(obj_key) {}
};
public:
virtual int exist_key(std::string key) = 0;
virtual int eviction(const DoutPrefixProvider* dpp, uint64_t size, optional_yield y) = 0;
virtual void update(const DoutPrefixProvider* dpp, const std::string& key, uint64_t offset, uint64_t len, const std::string& version, bool dirty, optional_yield y) = 0;
- virtual void updateObj(const DoutPrefixProvider* dpp, std::string& key, std::string version, bool dirty, uint64_t size,
- time_t creationTime, const rgw_user user, std::string& etag, const std::string& bucket_name,
+ virtual void update_dirty_object(const DoutPrefixProvider* dpp, const std::string& key, const std::string& version, bool dirty, uint64_t size,
+ time_t creationTime, const rgw_user& user, std::string& etag, const std::string& bucket_name, const std::string& bucket_id,
const rgw_obj_key& obj_key, optional_yield y) = 0;
virtual bool erase(const DoutPrefixProvider* dpp, const std::string& key, optional_yield y) = 0;
- virtual bool eraseObj(const DoutPrefixProvider* dpp, const std::string& key, optional_yield y) = 0;
+ virtual bool erase_dirty_object(const DoutPrefixProvider* dpp, const std::string& key, optional_yield y) = 0;
virtual void cleaning(const DoutPrefixProvider* dpp) = 0;
};
using handle_type = boost::heap::fibonacci_heap<LFUDAObjEntry*, boost::heap::compare<ObjectComparator<LFUDAObjEntry>>>::handle_type;
handle_type handle;
- LFUDAObjEntry(std::string& key, std::string& version, bool dirty, uint64_t size,
+ LFUDAObjEntry(const std::string& key, const std::string& version, bool dirty, uint64_t size,
time_t creationTime, rgw_user user, std::string& etag,
- const std::string& bucket_name, const rgw_obj_key& obj_key) : ObjEntry(key, version, dirty, size,
- creationTime, user, etag, bucket_name,
- obj_key) {}
+ const std::string& bucket_name, const std::string& bucket_id, const rgw_obj_key& obj_key) : ObjEntry(key, version, dirty, size,
+ creationTime, user, etag, bucket_name, bucket_id, obj_key) {}
void set_handle(handle_type handle_) { handle = handle_; }
};
virtual void update(const DoutPrefixProvider* dpp, const std::string& key, uint64_t offset, uint64_t len, const std::string& version, bool dirty, optional_yield y) override;
virtual bool erase(const DoutPrefixProvider* dpp, const std::string& key, optional_yield y) override;
void save_y(optional_yield y) { this->y = y; }
- virtual void updateObj(const DoutPrefixProvider* dpp, std::string& key, std::string version, bool dirty, uint64_t size,
- time_t creationTime, const rgw_user user, std::string& etag, const std::string& bucket_name,
+ virtual void update_dirty_object(const DoutPrefixProvider* dpp, const std::string& key, const std::string& version, bool dirty, uint64_t size,
+ time_t creationTime, const rgw_user& user, std::string& etag, const std::string& bucket_name, const std::string& bucket_id,
const rgw_obj_key& obj_key, optional_yield y) override;
- virtual bool eraseObj(const DoutPrefixProvider* dpp, const std::string& key, optional_yield y);
+ virtual bool erase_dirty_object(const DoutPrefixProvider* dpp, const std::string& key, optional_yield y);
virtual void cleaning(const DoutPrefixProvider* dpp) override;
LFUDAObjEntry* find_obj_entry(const std::string& key) {
auto it = o_entries_map.find(key);
virtual int exist_key(std::string key) override;
virtual int eviction(const DoutPrefixProvider* dpp, uint64_t size, optional_yield y) override;
virtual void update(const DoutPrefixProvider* dpp, const std::string& key, uint64_t offset, uint64_t len, const std::string& version, bool dirty, optional_yield y) override;
- virtual void updateObj(const DoutPrefixProvider* dpp, std::string& key, std::string version, bool dirty, uint64_t size,
- time_t creationTime, const rgw_user user, std::string& etag, const std::string& bucket_name,
+ virtual void update_dirty_object(const DoutPrefixProvider* dpp, const std::string& key, const std::string& version, bool dirty, uint64_t size,
+ time_t creationTime, const rgw_user& user, std::string& etag, const std::string& bucket_name, const std::string& bucket_id,
const rgw_obj_key& obj_key, optional_yield y) override;
virtual bool erase(const DoutPrefixProvider* dpp, const std::string& key, optional_yield y) override;
- virtual bool eraseObj(const DoutPrefixProvider* dpp, const std::string& key, optional_yield y) override;
+ virtual bool erase_dirty_object(const DoutPrefixProvider* dpp, const std::string& key, optional_yield y) override;
virtual void cleaning(const DoutPrefixProvider* dpp) override {}
};
bufferlist bl_data;
dest_version = d4n_dest_object->get_object_version();
- std::string key = dest_bucket->get_name() + "_" + dest_version + "_" + dest_object->get_name();
+ std::string key = get_cache_block_prefix(dest_object, dest_version, false);
std::string head_oid_in_cache;
if (dirty) {
- head_oid_in_cache = "D_" + key; //same as key, as there is no len or offset attached to head oid in cache
+ head_oid_in_cache = std::format("{}{}","D_", key); //same as key, as there is no len or offset attached to head oid in cache
} else {
head_oid_in_cache = key;
}
return ret;
}
if (dirty) {
- std::string object_key = dest_object->get_bucket()->get_name() + "_" + dest_object->get_oid();
- driver->get_policy_driver()->get_cache_policy()->updateObj(dpp, object_key, dest_version, true, this->get_size(), creationTime, std::get<rgw_user>(dest_object->get_bucket()->get_owner()), *etag, dest_object->get_bucket()->get_name(), dest_object->get_key(), y);
+ std::string object_key = dest_object->get_bucket()->get_bucket_id() + "_" + dest_object->get_oid();
+ driver->get_policy_driver()->get_cache_policy()->update_dirty_object(dpp, object_key, dest_version, true, this->get_size(), creationTime, std::get<rgw_user>(dest_object->get_bucket()->get_owner()), *etag, dest_object->get_bucket()->get_name(), dest_object->get_bucket()->get_bucket_id(), dest_object->get_key(), y);
}
}
}
if (attrs.count(RGW_ATTR_CRYPT_MODE)) {
std::vector<size_t> parts_len;
uint64_t obj_size = this->get_size();
- uint64_t obj_max_req_size = dpp->get_cct()->_conf->rgw_get_obj_max_req_size;
- uint64_t num_parts = (obj_size%obj_max_req_size) == 0 ? obj_size/obj_max_req_size : (obj_size/obj_max_req_size) + 1;
+ uint64_t obj_max_chunk_size = dpp->get_cct()->_conf->rgw_max_chunk_size;
+ uint64_t num_parts = (obj_size%obj_max_chunk_size) == 0 ? obj_size/obj_max_chunk_size : (obj_size/obj_max_chunk_size) + 1;
size_t remainder_size = obj_size;
for (uint64_t part = 0; part < num_parts; part++) {
size_t part_len;
if (part == (num_parts - 1)) { //last part
part_len = remainder_size;
} else {
- part_len = obj_max_req_size;
+ part_len = obj_max_chunk_size;
}
ldpp_dout(dpp, 20) << "D4NFilterObject::" << __func__ << "(): part_num: " << part << " part_len: " << part_len << dendl;
parts_len.emplace_back(part_len);
ldpp_dout(dpp, 10) << "D4NFilterObject::" << __func__ << "(): objName after special Handling: " << objName << dendl;
rgw::d4n::CacheObj object = rgw::d4n::CacheObj{
.objName = objName,
- .bucketName = this->get_bucket()->get_name(),
+ .bucketName = this->get_bucket()->get_bucket_id(),
.dirty = dirty,
.hostsList = { dpp->get_cct()->_conf->rgw_d4n_l1_datacache_address },
};
}
/* In case of a distributed cache - an entry corresponding to each instance will be needed to locate the head block
- this will also be needed for deleting an object from a version enabled bucket.
- and in that case instead of having a separate entry for an object, this entry could be used during object listing. */
+ this will also be needed for deleting an object from a version enabled bucket. */
if (this->have_instance()) {
rgw::d4n::CacheObj version_object = rgw::d4n::CacheObj{
.objName = this->get_oid(),
- .bucketName = this->get_bucket()->get_name(),
+ .bucketName = this->get_bucket()->get_bucket_id(),
.dirty = dirty,
.hostsList = { dpp->get_cct()->_conf->rgw_d4n_l1_datacache_address },
};
}
off_t cur_size = std::min<off_t>(fst + dpp->get_cct()->_conf->rgw_max_chunk_size, lst);
off_t cur_len = cur_size - fst;
- block.cacheObj.bucketName = this->get_bucket()->get_name();
+ block.cacheObj.bucketName = this->get_bucket()->get_bucket_id();
block.cacheObj.objName = this->get_key().get_oid();
block.cacheObj.dirty = dirty;
block.cacheObj.hostsList.insert(dpp->get_cct()->_conf->rgw_d4n_l1_datacache_address);
if (ret == 0) { //new versioned block will have new version, hostsList etc, how about globalWeight?
block = existing_block;
block.version = version;
+ block.cacheObj.dirty = dirty;
}
block.cacheObj.hostsList.insert(dpp->get_cct()->_conf->rgw_d4n_l1_datacache_address);
off_t cur_size = std::min<off_t>(fst + dpp->get_cct()->_conf->rgw_max_chunk_size, lst);
off_t cur_len = cur_size - fst;
- std::string key = get_bucket()->get_name() + "_" + version + "_" + get_name() + "_" + std::to_string(fst) + "_" + std::to_string(cur_len);
+ std::string prefix = get_cache_block_prefix(this, version, false);
+ std::string key = get_key_in_cache(get_cache_block_prefix(this, version, false), std::to_string(fst), std::to_string(cur_len));
std::string key_in_cache;
if (dirty) {
- key_in_cache = "D_" + key;
+ key_in_cache = std::format("{}{}","D_", key);
} else {
key_in_cache = key;
}
rgw::d4n::BlockDirectory* blockDir = this->driver->get_block_dir();
rgw::d4n::CacheObj object = rgw::d4n::CacheObj{
.objName = this->get_oid(), //version-enabled buckets will not have version for latest version, so this will work even when version is not provided in input
- .bucketName = this->get_bucket()->get_name(),
+ .bucketName = this->get_bucket()->get_bucket_id(),
};
rgw::d4n::CacheBlock block = rgw::d4n::CacheBlock{
//uniform name for versioned and non-versioned objects, since input for versioned objects might not contain version
ldpp_dout(dpp, 10) << "D4NFilterObject::" << __func__ << "(): Is block dirty: " << block.cacheObj.dirty << dendl;
- if (block.cacheObj.dirty) {
- head_oid_in_cache = "D_" + get_bucket()->get_name() + "_" + version + "_" + get_name();
- } else {
- head_oid_in_cache = get_bucket()->get_name() + "_" + version + "_" + get_name();
- }
+ head_oid_in_cache = get_cache_block_prefix(this, version, block.cacheObj.dirty);
ldpp_dout(dpp, 10) << "D4NFilterObject::" << __func__ << "(): Fetching attrs from cache for head obj id: " << head_oid_in_cache << dendl;
auto ret = this->driver->get_cache_driver()->get_attrs(dpp, head_oid_in_cache, attrs, y);
if (ret < 0) {
ldpp_dout(dpp, 0) << "D4NFilterObject::" << __func__ << "(): version could not be calculated." << dendl;
}
std::string objName = this->get_name();
- head_oid_in_cache = this->get_bucket()->get_name() + "_" + version + "_" + this->get_name();
+ head_oid_in_cache = get_cache_block_prefix(this, version, false);
if (this->driver->get_policy_driver()->get_cache_policy()->exist_key(head_oid_in_cache) > 0) {
ret = this->driver->get_cache_driver()->set_attrs(dpp, head_oid_in_cache, attrs, y);
} else {
this->source->set_attr_crypt_parts(dpp, y, attrs);
bufferlist bl;
- head_oid_in_cache = source->get_bucket()->get_name() + "_" + version + "_" + source->get_name();
+ head_oid_in_cache = get_cache_block_prefix(source, version, false);
ret = source->driver->get_policy_driver()->get_cache_policy()->eviction(dpp, attrs.size(), y);
if (ret == 0) {
ret = source->driver->get_cache_driver()->put(dpp, head_oid_in_cache, bl, 0, attrs, y);
rgw::d4n::CacheBlock block;
block.cacheObj.objName = source->get_key().get_oid();
- block.cacheObj.bucketName = source->get_bucket()->get_name();
+ block.cacheObj.bucketName = source->get_bucket()->get_bucket_id();
block.blockID = ofs;
block.size = len;
- std::string oid_in_cache = prefix + "_" + std::to_string(ofs) + "_" + std::to_string(len);
+ std::string oid_in_cache = get_key_in_cache(prefix, std::to_string(ofs), std::to_string(len));
if (source->driver->get_block_dir()->get(dpp, &block, y) == 0){
if (block.cacheObj.dirty){
std::string dest_version = d4n_dest_object->get_object_version();
rgw::d4n::CacheBlock dest_block;
dest_block.cacheObj.objName = source->dest_object->get_oid();
- dest_block.cacheObj.bucketName = source->dest_bucket->get_name();
+ dest_block.cacheObj.bucketName = source->dest_bucket->get_bucket_id();
dest_block.cacheObj.dirty = true; //writing to cache
dest_block.blockID = ofs;
dest_block.size = len;
dest_block.cacheObj.hostsList.insert(dpp->get_cct()->_conf->rgw_d4n_l1_datacache_address);
dest_block.version = dest_version;
dest_block.cacheObj.dirty = true;
- std::string key = source->dest_bucket->get_name() + "_" + dest_version + "_" + source->dest_object->get_name() +
- "_" + std::to_string(ofs) + "_" + std::to_string(len);
- std::string dest_oid_in_cache = "D_" + key;
+ std::string prefix = get_cache_block_prefix(source->dest_object, dest_version, false);
+ std::string key = get_key_in_cache(get_cache_block_prefix(source->dest_object, dest_version, false), std::to_string(ofs), std::to_string(len));
+ std::string dest_oid_in_cache = std::format("{}{}","D_", key);
auto ret = source->driver->get_policy_driver()->get_cache_policy()->eviction(dpp, dest_block.size, y);
if (ret == 0) {
rgw::sal::Attrs attrs;
{
const uint64_t window_size = g_conf()->rgw_get_obj_window_size;
std::string version = source->get_object_version();
- std::string prefix;
-
- prefix = source->get_bucket()->get_name() + "_" + version + "_" + source->get_name();
+ std::string prefix = get_cache_block_prefix(source, version, false);
ldpp_dout(dpp, 20) << "D4NFilterObject::iterate:: " << "prefix: " << prefix << dendl;
ldpp_dout(dpp, 20) << "D4NFilterObject::iterate:: " << "oid: " << source->get_key().get_oid() << " ofs: " << ofs << " end: " << end << dendl;
this->cb->set_client_cb(cb, dpp, &y);
source->set_prefix(prefix);
- /* This algorithm stores chunks for ranged requests also in the cache, which might be smaller than obj_max_req_size
- One simplification could be to overwrite the smaller chunks with a bigger chunk of obj_max_req_size, and to serve requests for smaller
- chunks using the larger chunk, but all corner cases need to be considered like the last chunk which might be smaller than obj_max_req_size
- and also ranged requests where a smaller chunk is overwritten by a larger chunk size != obj_max_req_size */
+ /* This algorithm stores chunks for ranged requests also in the cache, which might be smaller than max_chunk_size
+ One simplification could be to overwrite the smaller chunks with a bigger chunk of max_chunk_size, and to serve requests for smaller
+ chunks using the larger chunk, but all corner cases need to be considered like the last chunk which might be smaller than max_chunk_size
+ and also ranged requests where a smaller chunk is overwritten by a larger chunk size != max_chunk_size */
- uint64_t obj_max_req_size = g_conf()->rgw_get_obj_max_req_size;
+ uint64_t max_chunk_size = g_conf()->rgw_max_chunk_size;
uint64_t start_part_num = 0;
- uint64_t part_num = ofs/obj_max_req_size; //part num of ofs wrt start of the object
- uint64_t adjusted_start_ofs = part_num*obj_max_req_size; //in case of ranged request, adjust the start offset to the beginning of a chunk/ part
+ uint64_t part_num = ofs/max_chunk_size; //part num of ofs wrt start of the object
+ uint64_t adjusted_start_ofs = part_num*max_chunk_size; //in case of ranged request, adjust the start offset to the beginning of a chunk/ part
uint64_t diff_ofs = ofs - adjusted_start_ofs; //difference between actual offset and adjusted offset
off_t len = (end - adjusted_start_ofs) + 1;
- uint64_t num_parts = (len%obj_max_req_size) == 0 ? len/obj_max_req_size : (len/obj_max_req_size) + 1; //calculate num parts based on adjusted offset
+ uint64_t num_parts = (len%max_chunk_size) == 0 ? len/max_chunk_size : (len/max_chunk_size) + 1; //calculate num parts based on adjusted offset
//len_to_read is the actual length read from a part/ chunk in cache, while part_len is the length of the chunk/ part in cache
uint64_t cost = 0, len_to_read = 0, part_len = 0;
if ((params.part_num && !source->is_multipart()) || !params.part_num) {
aio = rgw::make_throttle(window_size, y);
- ldpp_dout(dpp, 20) << "D4NFilterObject::iterate:: " << "obj_max_req_size " << obj_max_req_size << " num_parts " << num_parts << dendl;
+ ldpp_dout(dpp, 20) << "D4NFilterObject::iterate:: " << "max_chunk_size " << max_chunk_size << " num_parts " << num_parts << dendl;
this->offset = ofs;
rgw::d4n::CacheBlock block;
block.cacheObj.objName = source->get_key().get_oid();
- block.cacheObj.bucketName = source->get_bucket()->get_name();
+ block.cacheObj.bucketName = source->get_bucket()->get_bucket_id();
do {
uint64_t id = adjusted_start_ofs, read_ofs = 0; //read_ofs is the actual offset to start reading from the current part/ chunk
part_len = len;
cost = len;
} else {
- len_to_read = obj_max_req_size;
- cost = obj_max_req_size;
- part_len = obj_max_req_size;
+ len_to_read = max_chunk_size;
+ cost = max_chunk_size;
+ part_len = max_chunk_size;
}
if (start_part_num == 0) {
len_to_read -= diff_ofs;
block.size = part_len;
ceph::bufferlist bl;
- std::string oid_in_cache = prefix + "_" + std::to_string(adjusted_start_ofs) + "_" + std::to_string(part_len);
+ std::string oid_in_cache = get_key_in_cache(prefix, std::to_string(adjusted_start_ofs), std::to_string(part_len));
ldpp_dout(dpp, 20) << "D4NFilterObject::iterate:: " << __func__ << "(): READ FROM CACHE: oid=" << oid_in_cache << " length to read is: " << len_to_read << " part num: " << start_part_num <<
" read_ofs: " << read_ofs << " part len: " << part_len << dendl;
} else if (ret == -ENOENT) {
block.blockID = adjusted_start_ofs;
uint64_t obj_size = source->get_size(), chunk_size = 0;
- if (obj_size < obj_max_req_size) {
+ if (obj_size < max_chunk_size) {
chunk_size = obj_size;
} else {
- chunk_size = obj_max_req_size;
+ chunk_size = max_chunk_size;
}
block.size = chunk_size;
ldpp_dout(dpp, 20) << "D4NFilterObject::iterate:: " << __func__ << "(): Block with oid=" << oid_in_cache << " found in local cache." << dendl;
if (block.version == version) {
- oid_in_cache = prefix + "_" + std::to_string(adjusted_start_ofs) + "_" + std::to_string(chunk_size);
+ oid_in_cache = get_key_in_cache(prefix, std::to_string(adjusted_start_ofs), std::to_string(chunk_size));
std::string key = oid_in_cache;
//for range requests, for last part, the whole part might exist in the cache
if (it != block.cacheObj.hostsList.end()) { /* Local copy */
ldpp_dout(dpp, 20) << "D4NFilterObject::iterate:: " << __func__ << "(): Block with oid=" << oid_in_cache << " found in local cache." << dendl;
if (block.version == version) {
- oid_in_cache = prefix + "_" + std::to_string(adjusted_start_ofs) + "_" + std::to_string(last_part_size);
+ oid_in_cache = get_key_in_cache(prefix, std::to_string(adjusted_start_ofs), std::to_string(last_part_size));
ldpp_dout(dpp, 20) << "D4NFilterObject::iterate:: " << __func__ << "(): READ FROM CACHE: oid=" << oid_in_cache <<
" length to read is: " << len_to_read << " part num: " << start_part_num << " read_ofs: " << read_ofs << " part len: " << part_len << dendl;
if ((part_len != last_part_size) && source->driver->get_policy_driver()->get_cache_policy()->exist_key(oid_in_cache) > 0) {
ldpp_dout(dpp, 20) << "D4NFilterObject::iterate:: " << __func__ << "(): Info: draining data for oid: " << oid_in_cache << dendl;
return drain(dpp, y);
} else {
- adjusted_start_ofs += obj_max_req_size;
+ adjusted_start_ofs += max_chunk_size;
}
start_part_num += 1;
- len -= obj_max_req_size;
+ len -= max_chunk_size;
} while (start_part_num < num_parts);
}
ldpp_dout(dpp, 20) << "D4NFilterObject::iterate:: " << __func__ << "(): Fetching object from backend store" << dendl;
//calculate the number of blocks read from backend store, and increment the perfcounter using that
if(perfcounter) {
uint64_t len_to_read_from_store = ((end - adjusted_start_ofs) + 1);
- uint64_t num_blocks = (len_to_read_from_store%obj_max_req_size) == 0 ? len_to_read_from_store/obj_max_req_size : (len_to_read_from_store/obj_max_req_size) + 1;
+ uint64_t num_blocks = (len_to_read_from_store%max_chunk_size) == 0 ? len_to_read_from_store/max_chunk_size : (len_to_read_from_store/max_chunk_size) + 1;
perfcounter->inc(l_rgw_d4n_cache_misses, num_blocks);
}
int D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::handle_data(bufferlist& bl, off_t bl_ofs, off_t bl_len)
{
- auto rgw_get_obj_max_req_size = g_conf()->rgw_get_obj_max_req_size;
+ auto rgw_max_chunk_size = g_conf()->rgw_max_chunk_size;
ldpp_dout(dpp, 20) << __func__ << ": bl_ofs is: " << bl_ofs << " bl_len is: " << bl_len << " ofs is: " << ofs << " part_count: " << part_count << dendl;
- if (!last_part && bl.length() <= rgw_get_obj_max_req_size) {
+ if (!last_part && bl.length() <= rgw_max_chunk_size) {
if (client_cb) {
int r = 0;
//ranged request
if (bl_ofs != ofs && part_count == 0) {
- if (ofs < bl_len) { // this can happen in case of multipart where each chunk returned is not always of size rgw_get_obj_max_req_size
+ if (ofs < bl_len) { // this can happen in case of multipart where each chunk returned is not always of size rgw_max_chunk_size
off_t bl_part_len = bl_len - ofs;
ldpp_dout(dpp, 20) << __func__ << ": bl_part_len is: " << bl_part_len << dendl;
bufferlist bl_part;
}
}
- //Accumulating data from backend store into rgw_get_obj_max_req_size sized chunks and then writing to cache
+ //Accumulating data from backend store into rgw_max_chunk_size sized chunks and then writing to cache
if (write_to_cache) {
Attrs attrs; // empty attrs for cache sets
std::string version = source->get_object_version();
rgw::d4n::CacheBlock block, existing_block, dest_block;
rgw::d4n::BlockDirectory* blockDir = source->driver->get_block_dir();
block.cacheObj.objName = source->get_key().get_oid();
- block.cacheObj.bucketName = source->get_bucket()->get_name();
+ block.cacheObj.bucketName = source->get_bucket()->get_bucket_id();
std::stringstream s;
block.cacheObj.creationTime = std::to_string(ceph::real_clock::to_time_t(source->get_mtime()));
bool dirty = block.cacheObj.dirty = false; //Reading from the backend, data is clean
if (source->dest_object && source->dest_bucket) {
D4NFilterObject* d4n_dest_object = dynamic_cast<D4NFilterObject*>(source->dest_object);
std::string dest_version = d4n_dest_object->get_object_version();
- dest_prefix = source->dest_bucket->get_name() + "_" + dest_version + "_" + source->dest_object->get_name();
+ dest_prefix = get_cache_block_prefix(source->dest_object, dest_version, false);
dest_block.cacheObj.hostsList.insert(dpp->get_cct()->_conf->rgw_d4n_l1_datacache_address);
dest_block.cacheObj.objName = source->dest_object->get_key().get_oid();
- dest_block.cacheObj.bucketName = source->dest_object->get_bucket()->get_name();
+ dest_block.cacheObj.bucketName = source->dest_object->get_bucket()->get_bucket_id();
//dest_block.cacheObj.creationTime = std::to_string(ceph::real_clock::to_time_t(source->get_mtime()));
dest_block.cacheObj.dirty = false;
dest_block.version = dest_version;
ldpp_dout(dpp, 20) << __func__ << ": version stored in update method is: " << version << dendl;
if (bl.length() > 0 && last_part) { // if bl = bl_rem has data and this is the last part, write it to cache
- std::string oid = prefix + "_" + std::to_string(adjusted_start_ofs) + "_" + std::to_string(bl_len);
+ std::string oid = get_key_in_cache(prefix, std::to_string(adjusted_start_ofs), std::to_string(bl_len));
if (!filter->get_policy_driver()->get_cache_policy()->exist_key(oid)) {
block.blockID = adjusted_start_ofs;
block.size = bl.length();
if (ret == 0) {
ret = filter->get_cache_driver()->put(dpp, oid, bl, bl.length(), attrs, *y);
if (ret == 0) {
- std::string objEtag = "";
- filter->get_policy_driver()->get_cache_policy()->update(dpp, oid, adjusted_start_ofs, bl.length(), version, dirty, *y);
+ std::string objEtag = "";
+ filter->get_policy_driver()->get_cache_policy()->update(dpp, oid, adjusted_start_ofs, bl.length(), version, dirty, *y);
- /* Store block in directory */
- existing_block.blockID = block.blockID;
- existing_block.size = block.size;
+ /* Store block in directory */
+ existing_block.blockID = block.blockID;
+ existing_block.size = block.size;
- if ((ret = blockDir->get(dpp, &existing_block, *y)) == 0 || ret == -ENOENT) {
+ if ((ret = blockDir->get(dpp, &existing_block, *y)) == 0 || ret == -ENOENT) {
if (ret == 0) { //new versioned block will have new version, hostsList etc, how about globalWeight?
- block = existing_block;
+ block = existing_block;
block.version = version;
}
- block.cacheObj.hostsList.insert(dpp->get_cct()->_conf->rgw_d4n_l1_datacache_address);
+ block.cacheObj.hostsList.insert(dpp->get_cct()->_conf->rgw_d4n_l1_datacache_address);
- if ((ret = blockDir->set(dpp, &block, *y)) < 0)
- ldpp_dout(dpp, 0) << "D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::" << __func__ << "(): BlockDirectory set() method failed, ret=" << ret << dendl;
- } else {
- ldpp_dout(dpp, 0) << "Failed to fetch existing block for: " << existing_block.cacheObj.objName << " blockID: " << existing_block.blockID << " block size: " << existing_block.size << ", ret=" << ret << dendl;
+ if ((ret = blockDir->set(dpp, &block, *y)) < 0)
+ ldpp_dout(dpp, 10) << "D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::" << __func__ << "(): BlockDirectory set() method failed, ret=" << ret << dendl;
+ } else { //end -if blockDir->get
+ ldpp_dout(dpp, 20) << "Failed to fetch existing block for: " << existing_block.cacheObj.objName << " blockID: " << existing_block.blockID << " block size: " << existing_block.size << ", ret=" << ret << dendl;
}
} else {
- ldpp_dout(dpp, 0) << "D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::" << __func__ << "(): put() to cache backend failed, ret=" << ret << dendl;
+ ldpp_dout(dpp, 10) << "D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::" << __func__ << "(): put() to cache backend failed, ret=" << ret << dendl;
}
- }
- }
+ } //end-if ret == 0
+ } //end-if exist_key
if (source->dest_object && source->dest_bucket) {
D4NFilterObject* d4n_dest_object = dynamic_cast<D4NFilterObject*>(source->dest_object);
std::string dest_version = d4n_dest_object->get_object_version();
- std::string dest_oid = dest_prefix + "_" + std::to_string(adjusted_start_ofs) + "_" + std::to_string(bl_len);
+ std::string dest_oid = get_key_in_cache(dest_prefix, std::to_string(adjusted_start_ofs), std::to_string(bl_len));
dest_block.blockID = adjusted_start_ofs;
dest_block.size = bl.length();
auto ret = filter->get_policy_driver()->get_cache_policy()->eviction(dpp, dest_block.size, *y);
if (ret == 0) {
filter->get_policy_driver()->get_cache_policy()->update(dpp, dest_oid, adjusted_start_ofs, bl.length(), dest_version, dirty, *y);
if (ret = blockDir->set(dpp, &dest_block, *y); ret < 0) {
- ldpp_dout(dpp, 20) << "D4N Filter: " << __func__ << " BlockDirectory set failed with ret: " << ret << dendl;
+ ldpp_dout(dpp, 20) << "D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB:: " << __func__ << " BlockDirectory set failed with ret: " << ret << dendl;
}
}
}
}
- } else if (bl.length() == rgw_get_obj_max_req_size && bl_rem.length() == 0) { // if bl is the same size as rgw_get_obj_max_req_size, write it to cache
- std::string oid = prefix + "_" + std::to_string(adjusted_start_ofs) + "_" + std::to_string(bl_len);
+ } else if (bl.length() == rgw_max_chunk_size && bl_rem.length() == 0) { // if bl is the same size as rgw_max_chunk_size, write it to cache
+ std::string oid = get_key_in_cache(prefix, std::to_string(adjusted_start_ofs), std::to_string(bl_len));
block.blockID = adjusted_start_ofs;
block.size = bl.length();
if (!filter->get_policy_driver()->get_cache_policy()->exist_key(oid)) {
filter->get_policy_driver()->get_cache_policy()->update(dpp, oid, adjusted_start_ofs, bl.length(), version, dirty, *y);
/* Store block in directory */
- existing_block.blockID = block.blockID;
- existing_block.size = block.size;
+ existing_block.blockID = block.blockID;
+ existing_block.size = block.size;
- if ((ret = blockDir->get(dpp, &existing_block, *y)) == 0 || ret == -ENOENT) {
+ if ((ret = blockDir->get(dpp, &existing_block, *y)) == 0 || ret == -ENOENT) {
if (ret == 0) { //new versioned block will have new version, hostsList etc, how about globalWeight?
- block = existing_block;
+ block = existing_block;
block.version = version;
}
- block.cacheObj.hostsList.insert(dpp->get_cct()->_conf->rgw_d4n_l1_datacache_address);
+ block.cacheObj.hostsList.insert(dpp->get_cct()->_conf->rgw_d4n_l1_datacache_address);
- if ((ret = blockDir->set(dpp, &block, *y)) < 0)
- ldpp_dout(dpp, 0) << "D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::" << __func__ << "(): BlockDirectory set() method failed, ret=" << ret << dendl;
+ if ((ret = blockDir->set(dpp, &block, *y)) < 0)
+ ldpp_dout(dpp, 10) << "D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::" << __func__ << "(): BlockDirectory set() method failed, ret=" << ret << dendl;
} else {
- ldpp_dout(dpp, 0) << "Failed to fetch existing block for: " << existing_block.cacheObj.objName << " blockID: " << existing_block.blockID << " block size: " << existing_block.size << ", ret=" << ret << dendl;
+ ldpp_dout(dpp, 20) << "D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::Failed to fetch existing block for: " << existing_block.cacheObj.objName << " blockID: " << existing_block.blockID << " block size: " << existing_block.size << ", ret=" << ret << dendl;
}
} else {
- ldpp_dout(dpp, 0) << "D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::" << __func__ << "(): put() to cache backend failed, ret=" << ret << dendl;
+ ldpp_dout(dpp, 10) << "D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::" << __func__ << "(): put() to cache backend failed, ret=" << ret << dendl;
}
}
}
if (source->dest_object && source->dest_bucket) {
D4NFilterObject* d4n_dest_object = dynamic_cast<D4NFilterObject*>(source->dest_object);
std::string dest_version = d4n_dest_object->get_object_version();
- std::string dest_oid = dest_prefix + "_" + std::to_string(adjusted_start_ofs) + "_" + std::to_string(bl_len);
+ std::string dest_oid = get_key_in_cache(dest_prefix, std::to_string(adjusted_start_ofs), std::to_string(bl_len));
dest_block.blockID = adjusted_start_ofs;
dest_block.size = bl.length();
auto ret = filter->get_policy_driver()->get_cache_policy()->eviction(dpp, dest_block.size, *y);
}
}
adjusted_start_ofs += bl_len;
- } else { //copy data from incoming bl to bl_rem till it is rgw_get_obj_max_req_size, and then write it to cache
- uint64_t rem_space = rgw_get_obj_max_req_size - bl_rem.length();
+ } else { //copy data from incoming bl to bl_rem till it is rgw_max_chunk_size, and then write it to cache
+ uint64_t rem_space = rgw_max_chunk_size - bl_rem.length();
uint64_t len_to_copy = rem_space > bl.length() ? bl.length() : rem_space;
bufferlist bl_copy;
bl.splice(0, len_to_copy, &bl_copy);
bl_rem.claim_append(bl_copy);
- if (bl_rem.length() == rgw_get_obj_max_req_size) {
- std::string oid = prefix + "_" + std::to_string(adjusted_start_ofs) + "_" + std::to_string(bl_rem.length());
+ if (bl_rem.length() == rgw_max_chunk_size) {
+ std::string oid = get_key_in_cache(prefix, std::to_string(adjusted_start_ofs), std::to_string(bl_rem.length()));
if (!filter->get_policy_driver()->get_cache_policy()->exist_key(oid)) {
block.blockID = adjusted_start_ofs;
block.size = bl_rem.length();
filter->get_policy_driver()->get_cache_policy()->update(dpp, oid, adjusted_start_ofs, bl_rem.length(), version, dirty, *y);
/* Store block in directory */
- existing_block.blockID = block.blockID;
- existing_block.size = block.size;
+ existing_block.blockID = block.blockID;
+ existing_block.size = block.size;
- if ((ret = blockDir->get(dpp, &existing_block, *y)) == 0 || ret == -ENOENT) {
- if (ret == 0) { //new versioned block will have new version, hostsList etc, how about globalWeight?
- block = existing_block;
- block.version = version;
- }
+ if ((ret = blockDir->get(dpp, &existing_block, *y)) == 0 || ret == -ENOENT) {
+ if (ret == 0) { //new versioned block will have new version, hostsList etc, how about globalWeight?
+ block = existing_block;
+ block.version = version;
+ }
- block.cacheObj.hostsList.insert(dpp->get_cct()->_conf->rgw_d4n_l1_datacache_address);
+ block.cacheObj.hostsList.insert(dpp->get_cct()->_conf->rgw_d4n_l1_datacache_address);
- if ((ret = blockDir->set(dpp, &block, *y)) < 0)
- ldpp_dout(dpp, 0) << "D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::" << __func__ << "(): BlockDirectory set() method failed, ret=" << ret << dendl;
- } else {
- ldpp_dout(dpp, 0) << "Failed to fetch existing block for: " << existing_block.cacheObj.objName << " blockID: " << existing_block.blockID << " block size: " << existing_block.size << ", ret=" << ret << dendl;
+ if ((ret = blockDir->set(dpp, &block, *y)) < 0)
+ ldpp_dout(dpp, 0) << "D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::" << __func__ << "(): BlockDirectory set() method failed, ret=" << ret << dendl;
+ } else {
+ ldpp_dout(dpp, 0) << "Failed to fetch existing block for: " << existing_block.cacheObj.objName << " blockID: " << existing_block.blockID << " block size: " << existing_block.size << ", ret=" << ret << dendl;
}
} else {
ldpp_dout(dpp, 0) << "D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::" << __func__ << "(): put() to cache backend failed, ret=" << ret << dendl;
if (source->dest_object && source->dest_bucket) {
D4NFilterObject* d4n_dest_object = dynamic_cast<D4NFilterObject*>(source->dest_object);
std::string dest_version = d4n_dest_object->get_object_version();
- std::string dest_oid = dest_prefix + "_" + std::to_string(adjusted_start_ofs) + "_" + std::to_string(bl_rem.length());
+ std::string dest_oid = get_key_in_cache(dest_prefix, std::to_string(adjusted_start_ofs), std::to_string(bl_rem.length()));
dest_block.blockID = adjusted_start_ofs;
dest_block.size = bl_rem.length();
auto ret = filter->get_policy_driver()->get_cache_policy()->eviction(dpp, dest_block.size, *y);
int D4NFilterObject::D4NFilterDeleteOp::delete_obj(const DoutPrefixProvider* dpp,
optional_yield y, uint32_t flags)
{
+ next->params = params;
+ auto ret = next->delete_obj(dpp, y, flags);
+ result = next->result;
+ return ret;
// TODO:
// 1. Send delete request to cache nodes with remote copies
// 2. See if we can derive dirty flag from the head block
off_t fst = 0;
do {
- std::string prefix = source->get_bucket()->get_name() + "_" + version + "_" + source->get_name();
+ std::string prefix = get_cache_block_prefix(source, version, false);
if (fst >= lst) {
break;
}
if (block.cacheObj.dirty)
prefix = "D_" + prefix;
- std::string oid_in_cache = prefix + "_" + std::to_string(fst) + "_" + std::to_string(cur_len);
+ std::string oid_in_cache = get_key_in_cache(prefix, std::to_string(fst), std::to_string(cur_len));
if ((ret = blockDir->del(dpp, &block, y)) == 0) {
if ((ret = source->driver->get_cache_driver()->delete_data(dpp, oid_in_cache, y)) == 0) { // Sam: do we want del or delete_data here?
- if (!(ret = source->driver->get_policy_driver()->get_cache_policy()->erase(dpp, policy_prefix + "_" + std::to_string(fst) + "_" + std::to_string(cur_len), y))) {
+ if (!(ret = source->driver->get_policy_driver()->get_cache_policy()->erase(dpp, get_key_in_cache(policy_prefix, std::to_string(fst), std::to_string(cur_len)), y))) {
ldpp_dout(dpp, 0) << "Failed to delete policy entry for: " << source->get_name() << " blockID: " << fst << " block size: " << cur_len << ", ret=" << ret << dendl;
return ret;
}
if (!objDirty) { // object written to backend
return next->delete_obj(dpp, y, flags);
} else {
- std::string object_key = source->get_bucket()->get_name() + "_" + source->get_oid();
- if (!(ret = source->driver->get_policy_driver()->get_cache_policy()->eraseObj(dpp, object_key, y))) {
+ std::string object_key = source->get_bucket()->get_bucket_id() + "_" + source->get_oid();
+ if (!(ret = source->driver->get_policy_driver()->get_cache_policy()->erase_dirty_object(dpp, object_key, y))) {
ldpp_dout(dpp, 0) << "Failed to delete policy object entry for: " << source->get_name() << ", ret=" << ret << dendl;
return -ENOENT;
} else {
std::string version;
if (!object->have_instance()) {
if (object->get_bucket()->versioned() && !object->get_bucket()->versioning_enabled()) { //if versioning is suspended
- version = "null";
- object->set_instance(version);
- } else {
- constexpr uint32_t OBJ_INSTANCE_LEN = 32;
- char buf[OBJ_INSTANCE_LEN + 1];
- gen_rand_alphanumeric_no_underscore(dpp->get_cct(), buf, OBJ_INSTANCE_LEN);
- version = buf; // using gen_rand_alphanumeric_no_underscore for the time being
- ldpp_dout(dpp, 20) << "D4NFilterWriter::" << __func__ << "(): generating version: " << version << dendl;
+ object->set_instance("null");
}
+ constexpr uint32_t OBJ_INSTANCE_LEN = 32;
+ char buf[OBJ_INSTANCE_LEN + 1];
+ gen_rand_alphanumeric_no_underscore(dpp->get_cct(), buf, OBJ_INSTANCE_LEN);
+ version = buf; // using gen_rand_alphanumeric_no_underscore for the time being
+ ldpp_dout(dpp, 20) << "D4NFilterWriter::" << __func__ << "(): generating version: " << version << dendl;
} else {
ldpp_dout(dpp, 20) << "D4NFilterWriter::" << __func__ << "(): version is: " << object->get_instance() << dendl;
version = object->get_instance();
bool dirty = true;
std::string version = object->get_object_version();
- std::string prefix;
+ std::string prefix = get_cache_block_prefix(obj, version, false);
- prefix = obj->get_bucket()->get_name() + "_" + version + "_" + obj->get_name();
int ret = 0;
if (!d4n_writecache) {
}
std::string version = object->get_object_version();
- std::string key = obj->get_bucket()->get_name() + "_" + version + "_" + obj->get_name();
+ std::string key = get_cache_block_prefix(obj, version, false);
bufferlist bl;
std::string head_oid_in_cache;
//same as key, as there is no len or offset attached to head oid in cache
if (dirty) {
- head_oid_in_cache = "D_" + key;
+ head_oid_in_cache = std::format("{}{}","D_", key);;
} else {
head_oid_in_cache = key;
}
}
if (dirty) {
//using object oid here so that version is automatically picked for versioned buckets, and for non-versioned buckets the old version is replaced by the latest version
- std::string object_key = obj->get_bucket()->get_name() + "_" + obj->get_oid();
+ std::string object_key = obj->get_bucket()->get_bucket_id() + "_" + obj->get_oid();
ldpp_dout(dpp, 16) << "D4NFilterWriter::" << __func__ << "(): object_key=" << object_key << dendl;
- driver->get_policy_driver()->get_cache_policy()->updateObj(dpp, object_key, version, dirty, accounted_size, creationTime, std::get<rgw_user>(obj->get_bucket()->get_owner()), objEtag, obj->get_bucket()->get_name(), obj->get_key(), y);
+ driver->get_policy_driver()->get_cache_policy()->update_dirty_object(dpp, object_key, version, dirty, accounted_size, creationTime, std::get<rgw_user>(obj->get_bucket()->get_owner()), objEtag, obj->get_bucket()->get_name(), obj->get_bucket()->get_bucket_id(), obj->get_key(), y);
}
} else { //if get_cache_driver()->put()
ldpp_dout(dpp, 0) << "D4NFilterWriter::" << __func__ << "(): put failed for head_oid_in_cache, ret=" << ret << dendl;
//Cache only the head object for multipart objects
D4NFilterObject* d4n_target_obj = dynamic_cast<D4NFilterObject*>(target_obj);
- std::string head_oid_in_cache;
- rgw::sal::Attrs attrs;
d4n_target_obj->load_obj_state(dpp, y);
- attrs = d4n_target_obj->get_attrs();
+ rgw::sal::Attrs attrs = d4n_target_obj->get_attrs();
d4n_target_obj->set_attrs_from_obj_state(dpp, y, attrs);
bufferlist bl_val;
bool is_multipart = true;
}
bufferlist bl;
- head_oid_in_cache = d4n_target_obj->get_bucket()->get_name() + "_" + version + "_" + d4n_target_obj->get_name();
+ std::string head_oid_in_cache = get_cache_block_prefix(d4n_target_obj, version, false);
// we are evicting data if needed, since the head object will be a part of read cache, as the whole multipart object is written to the backend store
ret = driver->get_policy_driver()->get_cache_policy()->eviction(dpp, attrs.size(), y);
if (ret == 0) {
projects / ceph.git / commitdiff