rgw: orphan: introduce a detailed mode (off by default)

We currently stat objects that fit in a head as well and also log them, since we
skip head objects anyway in the rados list output this commit avoids logging
these objects if the object size itself is less than the manifest head size.

Additionally we avoid the stat call itself from the list object output when the
object fits within the chunk size. This behaviour can be unset by setting the
detailed mode which can help in older clusters where the head used to have a
different size.

The old behaviour in both the cases can be turned on by setting the detailed
flag which can be passed on from rgw-admin. Avoiding stat calls and not logging
the head objects significantly reduces the IO activity on clusters which have a
huge percentage of objects that fit in a head.

Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
(cherry picked from commit ca12ecbfed0f65f206e8ad05125fac93c5a5ed0f)

diff --git a/src/rgw/rgw_orphan.cc b/src/rgw/rgw_orphan.cc
index a91be60363fbc7217ce90ec68dfa3b04d361e061..5f5a454c9d4966c9f6acc74949d15e7fa9c4a348 100644 (file)
--- a/src/rgw/rgw_orphan.cc
+++ b/src/rgw/rgw_orphan.cc
@@ -185,7 +185,8 @@ int RGWOrphanStore::read_entries(const string& oid, const string& marker, map<st
   return 0;
 }
 
-int RGWOrphanSearch::init(const string& job_name, RGWOrphanSearchInfo *info) {
+int RGWOrphanSearch::init(const string& job_name, RGWOrphanSearchInfo *info, bool detailed_mode)
+{
   int r = orphan_store.init();
   if (r < 0) {
     return r;
@@ -196,6 +197,7 @@ int RGWOrphanSearch::init(const string& job_name, RGWOrphanSearchInfo *info) {
   max_list_bucket_entries = std::max(store->ctx()->_conf->rgw_list_bucket_min_readahead,
                                      MAX_LIST_OBJS_ENTRIES);
 
+  detailed_mode = detailed_mode;
   RGWOrphanSearchState state;
   r = orphan_store.read_job(job_name, state);
   if (r < 0 && r != -ENOENT) {
@@ -437,6 +439,12 @@ int RGWOrphanSearch::handle_stat_result(map<int, list<string> >& oids, RGWRados:
   } else {
     RGWObjManifest& manifest = result.manifest;
 
+    if (!detailed_mode &&
+        manifest.get_obj_size() <= manifest.get_head_size()) {
+      ldout(store->ctx(), 5) << "skipping object as it fits in a head" << dendl;
+      return 0;
+    }
+
     RGWObjManifest::obj_iterator miter;
     for (miter = manifest.obj_begin(); miter != manifest.obj_end(); ++miter) {
       const rgw_raw_obj& loc = miter.get_location().get_raw_obj(store);
@@ -522,6 +530,14 @@ int RGWOrphanSearch::build_linked_oids_for_bucket(const string& bucket_instance_
       }
 
       ldout(store->ctx(), 20) << __func__ << ": entry.key.name=" << entry.key.name << " entry.key.instance=" << entry.key.instance << dendl;
+
+      if (!detailed_mode &&
+          entry.meta.accounted_size <= (uint64_t)store->ctx()->_conf->rgw_max_chunk_size) {
+        ldout(store->ctx(),5) << __func__ << "skipping stat as the object " << entry.key.name
+                              << "fits in a head" << dendl;
+        continue;
+      }
+
       rgw_obj obj(bucket_info.bucket, entry.key);
 
       RGWRados::Object op_target(store, bucket_info, obj_ctx, obj);

diff --git a/src/rgw/rgw_orphan.h b/src/rgw/rgw_orphan.h
index d7fbdfa62958be8255f58df67a20372dbd630aaf..ffe791c563c5ccc9f13f9b64841372fc73e2802c 100644 (file)
--- a/src/rgw/rgw_orphan.h
+++ b/src/rgw/rgw_orphan.h
@@ -163,7 +163,9 @@ class RGWOrphanSearch {
 
   uint16_t max_concurrent_ios;
   uint64_t stale_secs;
-  uint64_t max_list_obj_entries;
+  int64_t max_list_bucket_entries;
+
+  bool detailed_mode;
 
   struct log_iter_info {
     string oid;
@@ -193,7 +195,7 @@ public:
     return orphan_store.write_job(search_info.job_name, state);
   }
 
-  int init(const string& job_name, RGWOrphanSearchInfo *info);
+  int init(const string& job_name, RGWOrphanSearchInfo *info, bool detailed_mode=false);
 
   int create(const string& job_name, int num_shards);