rgw: orphan: introduce a detailed mode (off by default)

We currently stat objects that fit in a head as well and also log them, since we
skip head objects anyway in the rados list output this commit avoids logging
these objects if the object size itself is less than the manifest head size.

Additionally we avoid the stat call itself from the list object output when the
object fits within the chunk size. This behaviour can be unset by setting the
detailed mode which can help in older clusters where the head used to have a
different size.

The old behaviour in both the cases can be turned on by setting the detailed
flag which can be passed on from rgw-admin. Avoiding stat calls and not logging
the head objects significantly reduces the IO activity on clusters which have a
huge percentage of objects that fit in a head.

Signed-off-by: Abhishek Lekshmanan <abhishek@suse.com>
(cherry picked from commit ca12ecbfed0f65f206e8ad05125fac93c5a5ed0f)

diff --git a/src/rgw/rgw_orphan.cc b/src/rgw/rgw_orphan.cc
index 7cdf53a4ab7690a9e45af7515b755e68dcd54342..1e96d41189b97d19d2f6d6b2c7e31c086fc7f854 100644 (file)
--- a/src/rgw/rgw_orphan.cc
+++ b/src/rgw/rgw_orphan.cc
@@ -182,7 +182,8 @@ int RGWOrphanStore::read_entries(const string& oid, const string& marker, map<st
   return 0;
 }
 
-int RGWOrphanSearch::init(const string& job_name, RGWOrphanSearchInfo *info) {
+int RGWOrphanSearch::init(const string& job_name, RGWOrphanSearchInfo *info, bool detailed_mode)
+{
   int r = orphan_store.init();
   if (r < 0) {
     return r;
@@ -193,6 +194,7 @@ int RGWOrphanSearch::init(const string& job_name, RGWOrphanSearchInfo *info) {
   max_list_bucket_entries = std::max(store->ctx()->_conf->rgw_list_bucket_min_readahead,
                                      MAX_LIST_OBJS_ENTRIES);
 
+  detailed_mode = detailed_mode;
   RGWOrphanSearchState state;
   r = orphan_store.read_job(job_name, state);
   if (r < 0 && r != -ENOENT) {
@@ -434,6 +436,12 @@ int RGWOrphanSearch::handle_stat_result(map<int, list<string> >& oids, RGWRados:
   } else {
     RGWObjManifest& manifest = result.manifest;
 
+    if (!detailed_mode &&
+        manifest.get_obj_size() <= manifest.get_head_size()) {
+      ldout(store->ctx(), 5) << "skipping object as it fits in a head" << dendl;
+      return 0;
+    }
+
     RGWObjManifest::obj_iterator miter;
     for (miter = manifest.obj_begin(); miter != manifest.obj_end(); ++miter) {
       const rgw_raw_obj& loc = miter.get_location().get_raw_obj(store);
@@ -518,6 +526,14 @@ int RGWOrphanSearch::build_linked_oids_for_bucket(const string& bucket_instance_
       }
 
       ldout(store->ctx(), 20) << __func__ << ": entry.key.name=" << entry.key.name << " entry.key.instance=" << entry.key.instance << dendl;
+
+      if (!detailed_mode &&
+          entry.meta.accounted_size <= (uint64_t)store->ctx()->_conf->rgw_max_chunk_size) {
+        ldout(store->ctx(),5) << __func__ << "skipping stat as the object " << entry.key.name
+                              << "fits in a head" << dendl;
+        continue;
+      }
+
       rgw_obj obj(bucket_info.bucket, entry.key);
 
       RGWRados::Object op_target(store, bucket_info, obj_ctx, obj);

diff --git a/src/rgw/rgw_orphan.h b/src/rgw/rgw_orphan.h
index 1ef8b837b14db93181f817aa3fbee9dec2c59d20..c80e67985c8285abe01ff2bf7343fb9db3b05246 100644 (file)
--- a/src/rgw/rgw_orphan.h
+++ b/src/rgw/rgw_orphan.h
@@ -162,7 +162,9 @@ class RGWOrphanSearch {
 
   uint16_t max_concurrent_ios;
   uint64_t stale_secs;
-  uint64_t max_list_obj_entries;
+  int64_t max_list_bucket_entries;
+
+  bool detailed_mode;
 
   struct log_iter_info {
     string oid;
@@ -192,7 +194,7 @@ public:
     return orphan_store.write_job(search_info.job_name, state);
   }
 
-  int init(const string& job_name, RGWOrphanSearchInfo *info);
+  int init(const string& job_name, RGWOrphanSearchInfo *info, bool detailed_mode=false);
 
   int create(const string& job_name, int num_shards);