rgw/d4n: miscellaneous corrections/additions:

1. storing objects in directory using their oid, so that the version
is included.
2. making sure that the head block corresponds to latest
version in the block directory.
3. add a directory entry for head block for every version
in case of a versioned bucket.
4. Populating hostsList correctly for blocks and objects.

Signed-off-by: Pritha Srivastava <prsrivas@redhat.com>

diff --git a/src/rgw/driver/d4n/d4n_policy.cc b/src/rgw/driver/d4n/d4n_policy.cc
index 878245c029dfdb7072e78deed9ae564067bcb5d9..f7b00624a6e153b62ef213f04e432b0c29511849 100644 (file)
--- a/src/rgw/driver/d4n/d4n_policy.cc
+++ b/src/rgw/driver/d4n/d4n_policy.cc
@@ -493,6 +493,7 @@ void LFUDAPolicy::cleaning(const DoutPrefixProvider* dpp)
       }
 
       std::unique_ptr<rgw::sal::Object> c_obj = c_bucket->get_object(e->obj_key);
+      ldpp_dout(dpp, 20) << __func__ << "(): c_obj oid =" << c_obj->get_oid() << dendl;
 
       ACLOwner owner{c_user->get_id(), c_user->get_display_name()};
 
@@ -609,13 +610,37 @@ void LFUDAPolicy::cleaning(const DoutPrefixProvider* dpp)
       block.cacheObj.objName = c_obj->get_name();
       block.size = 0;
       block.blockID = 0;
-      std::string dirty = "false";
-      op_ret = dir->update_field(dpp, &block, "dirty", dirty, null_yield);
-      if (op_ret < 0) {
-          ldpp_dout(dpp, 20) << __func__ << "updating dirty flag in block directory for head failed!" << dendl;
-          return;
+      if (c_obj->have_instance()) {
+        dir->get(dpp, &block, null_yield);
+        if (block.version == c_obj->get_instance()) { //versioned case - update head block entry that has latest version
+          std::string dirty = "false";
+          op_ret = dir->update_field(dpp, &block, "dirty", dirty, null_yield);
+          if (op_ret < 0) {
+              ldpp_dout(dpp, 20) << __func__ << "updating dirty flag in block directory for head failed!" << dendl;
+              //return;
+          }
+        }
+      } else { //non-versioned case
+        std::string dirty = "false";
+        op_ret = dir->update_field(dpp, &block, "dirty", dirty, null_yield);
+        if (op_ret < 0) {
+            ldpp_dout(dpp, 20) << __func__ << "updating dirty flag in block directory for head failed!" << dendl;
+            //return;
+        }
+      }
+      //In case of distributed cache, we may have to update this for every instance
+      if (c_obj->have_instance()) {
+        rgw::d4n::CacheBlock instance_block;
+        instance_block.cacheObj.bucketName = c_obj->get_bucket()->get_name();
+        instance_block.cacheObj.objName = c_obj->get_oid();
+        instance_block.size = 0;
+        instance_block.blockID = 0;
+        op_ret = dir->update_field(dpp, &instance_block, "dirty", "false", null_yield);
+        if (op_ret < 0) {
+            ldpp_dout(dpp, 20) << __func__ << "updating dirty flag in block directory for instance block failed!" << dendl;
+            //return;
+        }
       }
-
       //remove entry from map and queue, eraseObj locks correctly
       eraseObj(dpp, e->key, null_yield);
     } else { //end-if std::difftime(time(NULL), e->creationTime) > interval

diff --git a/src/rgw/driver/d4n/rgw_sal_d4n.cc b/src/rgw/driver/d4n/rgw_sal_d4n.cc
index 9d45888d990838932f4338eede6ba2541ef3d8f6..140b9ef4bc4388eee710cd18e0ea8f133a56b1d0 100644 (file)
--- a/src/rgw/driver/d4n/rgw_sal_d4n.cc
+++ b/src/rgw/driver/d4n/rgw_sal_d4n.cc
@@ -168,11 +168,11 @@ bool D4NFilterObject::get_obj_attrs_from_cache(const DoutPrefixProvider* dpp, op
           auto mtime = ceph::real_clock::from_double(std::stod(attr.second.c_str()));
           this->set_mtime(mtime);
         } else if (attr.first == "user.rgw.object_size") {
-          auto size = std::stoull(attr.second.c_str());
+          auto size = std::stoull(attr.second.to_str());
           ldpp_dout(dpp, 10) << "D4NFilterObject::" << __func__ << "(): setting object_size to: " << size << dendl;
           this->set_obj_size(size);
         } else if (attr.first == "user.rgw.accounted_size") {
-          auto accounted_size = std::stoull(attr.second.c_str());
+          auto accounted_size = std::stoull(attr.second.to_str());
           this->set_accounted_size(accounted_size);
         } else if (attr.first == "user.rgw.epoch") {
           ldpp_dout(dpp, 10) << "D4NFilterObject::" << __func__ << "(): setting epoch." << dendl;
@@ -254,29 +254,57 @@ int D4NFilterObject::calculate_version(const DoutPrefixProvider* dpp, optional_y
   return 0;
 }
 
-int D4NFilterObject::set_head_obj_dir_entry(const DoutPrefixProvider* dpp, optional_yield y, bool dirty)
+int D4NFilterObject::set_head_obj_dir_entry(const DoutPrefixProvider* dpp, optional_yield y, bool is_latest_version, bool dirty)
 {
   ldpp_dout(dpp, 10) << "D4NFilterObject::" << __func__ << "(): object name: " << this->get_name() << " bucket name: " << this->get_bucket()->get_name() << dendl;
+  // entry that contains latest version for versioned and non-versioned objects
+  int ret = 0;
   rgw::d4n::BlockDirectory* blockDir = this->driver->get_block_dir();
-  rgw::d4n::CacheObj object = rgw::d4n::CacheObj{
-    .objName = this->get_name(),
-    .bucketName = this->get_bucket()->get_name(),
-    .dirty = dirty,
-    .hostsList = { driver->get_block_dir()->cct->_conf->rgw_d4n_l1_datacache_address },
-    };
+  if (is_latest_version) {
+    rgw::d4n::CacheObj object = rgw::d4n::CacheObj{
+      .objName = this->get_name(),
+      .bucketName = this->get_bucket()->get_name(),
+      .dirty = dirty,
+      .hostsList = { driver->get_block_dir()->cct->_conf->rgw_d4n_l1_datacache_address },
+      };
 
-  rgw::d4n::CacheBlock block = rgw::d4n::CacheBlock{
-    .cacheObj = object,
-    .blockID = 0,
-    .version = this->get_object_version(),
+    rgw::d4n::CacheBlock block = rgw::d4n::CacheBlock{
+      .cacheObj = object,
+      .blockID = 0,
+      .version = this->get_object_version(),
+      .dirty = dirty,
+      .size = 0,
+      .hostsList = { driver->get_block_dir()->cct->_conf->rgw_d4n_l1_datacache_address },
+      };
+    ret = blockDir->set(dpp, &block, y);
+    if (ret < 0) {
+      ldpp_dout(dpp, 10) << "D4NFilterObject::" << __func__ << "(): BlockDirectory set method failed for head object with ret: " << ret << dendl;
+    }
+  }
+
+  /* In case of a distributed cache - an entry corresponding to each instance will be needed to locate the head block
+     this will also be needed for deleting an object from a version enabled bucket.
+     and in that case instead of having a separate entry for an object, this entry could be used during object listing. */
+  if (this->have_instance()) {
+    rgw::d4n::CacheObj version_object = rgw::d4n::CacheObj{
+    .objName = this->get_oid(),
+    .bucketName = this->get_bucket()->get_name(),
     .dirty = dirty,
-    .size = 0,
     .hostsList = { driver->get_block_dir()->cct->_conf->rgw_d4n_l1_datacache_address },
     };
 
-  auto ret = blockDir->set(dpp, &block, y);
-  if (ret < 0) {
-    ldpp_dout(dpp, 10) << "D4NFilterObject::" << __func__ << "(): BlockDirectory set method failed for head object with ret: " << ret << dendl;
+    rgw::d4n::CacheBlock version_block = rgw::d4n::CacheBlock{
+      .cacheObj = version_object,
+      .blockID = 0,
+      .version = this->get_object_version(),
+      .dirty = dirty,
+      .size = 0,
+      .hostsList = { driver->get_block_dir()->cct->_conf->rgw_d4n_l1_datacache_address },
+      };
+    auto ret = blockDir->set(dpp, &version_block, y);
+    if (ret < 0) {
+      ldpp_dout(dpp, 10) << "D4NFilterObject::" << __func__ << "(): BlockDirectory set method failed for head object with ret: " << ret << dendl;
+    }
   }
 
   return ret;
@@ -286,7 +314,7 @@ bool D4NFilterObject::check_head_exists_in_cache_get_oid(const DoutPrefixProvide
 {
   rgw::d4n::BlockDirectory* blockDir = this->driver->get_block_dir();
   rgw::d4n::CacheObj object = rgw::d4n::CacheObj{
-        .objName = this->get_name(),
+        .objName = this->get_oid(), //version-enabled buckets will not have version for latest version, so this will work even when versio is not provided in input
         .bucketName = this->get_bucket()->get_name(),
         };
 
@@ -300,25 +328,24 @@ bool D4NFilterObject::check_head_exists_in_cache_get_oid(const DoutPrefixProvide
   //if the block corresponding to head object does not exist in directory, implies it is not cached
   if (blockDir->exist_key(dpp, &block, y) && (blockDir->get(dpp, &block, y) == 0)) {
     std::string version;
-    if (have_instance()) {
-      version = get_instance();
-    } else {
-      version = block.version;
-    }
+    version = block.version;
     this->set_object_version(version);
+
+    /* for distributed cache-the blockHostsList can be used to determine if the head block resides on the localhost, then get the block from localhost, whether or not the block is dirty
+       can be determined using the block entry. */
+
     //uniform name for versioned and non-versioned objects, since input for versioned objects might not contain version
-    head_oid_in_cache = get_bucket()->get_name() + "_" + version + "_" + get_name();
+    ldpp_dout(dpp, 10) << "D4NFilterObject::" << __func__ << "(): Is block dirty: " << block.dirty << dendl;
+    if (block.dirty) {
+      head_oid_in_cache = "D_" + get_bucket()->get_name() + "_" + version + "_" + get_name();
+    } else {
+      head_oid_in_cache = get_bucket()->get_name() + "_" + version + "_" + get_name();
+    }
     ldpp_dout(dpp, 10) << "D4NFilterObject::" << __func__ << "(): Fetching attrs from cache for head obj id: " << head_oid_in_cache << dendl;
     auto ret = this->driver->get_cache_driver()->get_attrs(dpp, head_oid_in_cache, attrs, y);
     if (ret < 0) {
-      //check for dirty blocks also
-      head_oid_in_cache = "D_" + get_bucket()->get_name() + "_" + version + "_" + get_name();
-      ldpp_dout(dpp, 10) << "D4NFilterObject::" << __func__ << "(): Fetching attrs from cache for head obj id: " << head_oid_in_cache << dendl;
-      ret = this->driver->get_cache_driver()->get_attrs(dpp, head_oid_in_cache, attrs, y);
-      if (ret < 0) {
-        found_in_cache = false;
-        ldpp_dout(dpp, 10) << "D4NFilterObject::" << __func__ << "(): CacheDriver get_attrs method failed." << dendl;
-      }
+      found_in_cache = false;
+      ldpp_dout(dpp, 10) << "D4NFilterObject::" << __func__ << "(): CacheDriver get_attrs method failed." << dendl;
     }
   } else { //if blockDir->exist_key
     found_in_cache = false;
@@ -329,6 +356,10 @@ bool D4NFilterObject::check_head_exists_in_cache_get_oid(const DoutPrefixProvide
 int D4NFilterObject::get_obj_attrs(optional_yield y, const DoutPrefixProvider* dpp,
                                 rgw_obj* target_obj)
 {
+  bool is_latest_version = true;
+  if (this->have_instance()) {
+    is_latest_version = false;
+  }
   if (!get_obj_attrs_from_cache(dpp, y)) {
     std::string head_oid_in_cache;
     rgw::sal::Attrs attrs;
@@ -345,7 +376,10 @@ int D4NFilterObject::get_obj_attrs(optional_yield y, const DoutPrefixProvider* d
   
     this->load_obj_state(dpp, y);
     this->obj = this->get_obj();
-    ldpp_dout(dpp, 10) << "D4NFilterObject::" << __func__ << "(): this->obj is: " << this->obj.key.name << dendl;
+    if (!this->obj.key.instance.empty()) {
+      this->set_instance(this->obj.key.instance);
+    }
+    ldpp_dout(dpp, 10) << "D4NFilterObject::" << __func__ << "(): this->obj oid is: " << this->obj.key.name << "instance is: " << this->obj.key.instance << dendl;
     attrs = this->get_attrs();
     this->set_obj_state_attrs(dpp, y, attrs);
 
@@ -369,7 +403,7 @@ int D4NFilterObject::get_obj_attrs(optional_yield y, const DoutPrefixProvider* d
     if (ret == 0) {
       ldpp_dout(dpp, 20) << "D4NFilterObject::" << __func__ << " version stored in update method is: " << this->get_object_version() << dendl;
       this->driver->get_policy_driver()->get_cache_policy()->update(dpp, head_oid_in_cache, 0, 0, version, false, y);
-      ret = set_head_obj_dir_entry(dpp, y);
+      ret = set_head_obj_dir_entry(dpp, y, is_latest_version);
       if (ret < 0) {
         ldpp_dout(dpp, 10) << "D4NFilterObject::" << __func__ << "(): BlockDirectory set method failed for head object with ret: " << ret << dendl;
       }
@@ -483,6 +517,11 @@ std::unique_ptr<Object::DeleteOp> D4NFilterObject::get_delete_op()
 
 int D4NFilterObject::D4NFilterReadOp::prepare(optional_yield y, const DoutPrefixProvider* dpp)
 {
+  //set a flag to show that incoming instance has no version specified
+  bool is_latest_version = true;
+  if (source->have_instance()) {
+    is_latest_version = false;
+  }
   if (!source->get_obj_attrs_from_cache(dpp, y)) {
     std::string head_oid_in_cache;
     rgw::sal::Attrs attrs;
@@ -518,10 +557,23 @@ int D4NFilterObject::D4NFilterReadOp::prepare(optional_yield y, const DoutPrefix
       if (ret == 0) {
         ldpp_dout(dpp, 20) << "D4NFilterObject::" << __func__ << " version stored in update method is: " << this->source->get_object_version() << dendl;
         source->driver->get_policy_driver()->get_cache_policy()->update(dpp, head_oid_in_cache, 0, bl.length(), version, false, y);
-        ret = source->set_head_obj_dir_entry(dpp, y);
+        ret = source->set_head_obj_dir_entry(dpp, y, is_latest_version);
         if (ret < 0) {
           ldpp_dout(dpp, 10) << "D4NFilterObject::" << __func__ << "(): BlockDirectory set method failed for head object with ret: " << ret << dendl;
         }
+        //write object to directory.
+        rgw::d4n::CacheObj object = rgw::d4n::CacheObj{
+          .objName = source->get_oid(),
+          .bucketName = source->get_bucket()->get_name(),
+          .creationTime = std::to_string(ceph::real_clock::to_double(source->get_mtime())),
+          .dirty = false,
+          .hostsList = { source->driver->get_block_dir()->cct->_conf->rgw_d4n_l1_datacache_address }
+        };
+        ret = source->driver->get_obj_dir()->set(&object, y);
+        if (ret < 0) {
+          ldpp_dout(dpp, 10) << "D4NFilterObject::" << __func__ << "(): ObjectDirectory set method failed with err: " << ret << dendl;
+          return ret;
+        }
       } else {
         ldpp_dout(dpp, 10) << "D4NFilterObject::" << __func__ << "(): put for head object failed with error: " << ret << dendl;
       }
@@ -906,6 +958,18 @@ int D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::handle_data(bufferlist& bl
                 }
               }
             }
+            //write object to directory.
+            rgw::d4n::CacheObj object = rgw::d4n::CacheObj{
+                .objName = source->get_oid(),
+                .bucketName = source->get_bucket()->get_name(),
+                .dirty = dirty,
+                .hostsList = { source->driver->get_block_dir()->cct->_conf->rgw_d4n_l1_datacache_address }
+            };
+            ret = source->driver->get_obj_dir()->set(&object, *y);
+            if (ret < 0) {
+              ldpp_dout(dpp, 10) << "D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::::" << __func__ << "(): ObjectDirectory set method failed with err: " << ret << dendl;
+              return ret;
+            }
           } else {
            ldpp_dout(dpp, 0) << "D4NFilterObject::D4NFilterReadOp::D4NFilterGetCB::" << __func__ << "(): put() to cache backend failed with error: " << ret << dendl;
           }
@@ -1065,15 +1129,16 @@ int D4NFilterWriter::prepare(optional_yield y)
   if (!object->have_instance()) {
     if (object->get_bucket()->versioned() && !object->get_bucket()->versioning_enabled()) { //if versioning is suspended
       this->version = "null";
-    } else { // this holds true for non-versioned object and for version enabled object with no versionId given as input
+      object->set_instance(this->version);
+    } else {
       constexpr uint32_t OBJ_INSTANCE_LEN = 32;
       char buf[OBJ_INSTANCE_LEN + 1];
       gen_rand_alphanumeric_no_underscore(dpp->get_cct(), buf, OBJ_INSTANCE_LEN);
       this->version = buf; // using gen_rand_alphanumeric_no_underscore for the time being
-      ldpp_dout(dpp, 10) << "D4NFilterObject::D4NFilterWriteOp::" << __func__ << "(): generating version: " << version << dendl;
+      ldpp_dout(dpp, 20) << "D4NFilterObject::D4NFilterWriteOp::" << __func__ << "(): generating version: " << version << dendl;
     }
   } else {
-    ldpp_dout(dpp, 10) << "D4NFilterWriter::" << __func__ << "(): " << "version is: " << object->get_instance() << dendl;
+    ldpp_dout(dpp, 20) << "D4NFilterWriter::" << __func__ << "(): " << "version is: " << object->get_instance() << dendl;
   }
   return 0;
 }
@@ -1100,6 +1165,7 @@ int D4NFilterWriter::process(bufferlist&& data, uint64_t offset)
     block.cacheObj.bucketName = obj->get_bucket()->get_name();
     block.cacheObj.objName = obj->get_key().get_oid();
     block.cacheObj.dirty = dirty;
+    block.cacheObj.hostsList.push_back(blockDir->cct->_conf->rgw_d4n_l1_datacache_address);
     existing_block.cacheObj.objName = block.cacheObj.objName;
     existing_block.cacheObj.bucketName = block.cacheObj.bucketName;
 
@@ -1181,10 +1247,7 @@ int D4NFilterWriter::complete(size_t accounted_size, const std::string& etag,
     if (object->have_instance()) {
       version = obj->get_instance();
     } else {
-      version = this->version; //version for non-versioned objects, using gen_rand_alphanumeric_no_underscore for the time being
-      if (obj->get_bucket()->versioned()) {
-        object->set_instance(version);
-      }
+      version = this->version;
     } 
     std::string key = obj->get_bucket()->get_name() + "_" + version + "_" + obj->get_name();
 
@@ -1210,7 +1273,7 @@ int D4NFilterWriter::complete(size_t accounted_size, const std::string& etag,
       object->set_attrs(attrs);
       ldpp_dout(dpp, 20) << "D4NFilterWriter::" << __func__ << " version stored in update method is: " << version << dendl;
       driver->get_policy_driver()->get_cache_policy()->update(dpp, key, 0, bl.length(), version, dirty, y);
-      ret = object->set_head_obj_dir_entry(dpp, y, true);
+      ret = object->set_head_obj_dir_entry(dpp, y, true, true);
       if (ret < 0) {
         ldpp_dout(dpp, 10) << "D4NFilterWriter::" << __func__ << "(): BlockDirectory set method failed for head object with ret: " << ret << dendl;
         return ret;
@@ -1220,7 +1283,7 @@ int D4NFilterWriter::complete(size_t accounted_size, const std::string& etag,
       //write object to directory.
       hostsList = { driver->get_block_dir()->cct->_conf->rgw_d4n_l1_datacache_address };
       rgw::d4n::CacheObj object = rgw::d4n::CacheObj{
-          .objName = obj->get_name(), 
+          .objName = obj->get_oid(),
           .bucketName = obj->get_bucket()->get_name(),
           .creationTime = std::to_string(creationTime), 
           .dirty = dirty,

diff --git a/src/rgw/driver/d4n/rgw_sal_d4n.h b/src/rgw/driver/d4n/rgw_sal_d4n.h
index ca1c78497bf8fd225a681af26543dc9e13e99663..3f1e3e9f0e390b8fd02b539d15505cb010291139 100644 (file)
--- a/src/rgw/driver/d4n/rgw_sal_d4n.h
+++ b/src/rgw/driver/d4n/rgw_sal_d4n.h
@@ -205,7 +205,7 @@ class D4NFilterObject : public FilterObject {
     bool get_obj_attrs_from_cache(const DoutPrefixProvider* dpp, optional_yield y);
     void set_obj_state_attrs(const DoutPrefixProvider* dpp, optional_yield y, rgw::sal::Attrs& attrs);
     int calculate_version(const DoutPrefixProvider* dpp, optional_yield y, std::string& version);
-    int set_head_obj_dir_entry(const DoutPrefixProvider* dpp, optional_yield y, bool dirty = false);
+    int set_head_obj_dir_entry(const DoutPrefixProvider* dpp, optional_yield y, bool is_latest_version = true, bool dirty = false);
     bool check_head_exists_in_cache_get_oid(const DoutPrefixProvider* dpp, std::string& head_oid_in_cache, rgw::sal::Attrs& attrs, optional_yield y);
 };