From 5d9eb18bcf0f9ca8863e8bf964990d461fc327e3 Mon Sep 17 00:00:00 2001 From: Abhishek Lekshmanan Date: Wed, 13 Feb 2019 13:45:47 +0100 Subject: [PATCH] rgw: orphan: introduce a detailed mode (off by default) We currently stat objects that fit in a head as well and also log them, since we skip head objects anyway in the rados list output this commit avoids logging these objects if the object size itself is less than the manifest head size. Additionally we avoid the stat call itself from the list object output when the object fits within the chunk size. This behaviour can be unset by setting the detailed mode which can help in older clusters where the head used to have a different size. The old behaviour in both the cases can be turned on by setting the detailed flag which can be passed on from rgw-admin. Avoiding stat calls and not logging the head objects significantly reduces the IO activity on clusters which have a huge percentage of objects that fit in a head. Signed-off-by: Abhishek Lekshmanan (cherry picked from commit ca12ecbfed0f65f206e8ad05125fac93c5a5ed0f) --- src/rgw/rgw_orphan.cc | 18 +++++++++++++++++- src/rgw/rgw_orphan.h | 6 ++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/src/rgw/rgw_orphan.cc b/src/rgw/rgw_orphan.cc index a91be60363fbc..5f5a454c9d496 100644 --- a/src/rgw/rgw_orphan.cc +++ b/src/rgw/rgw_orphan.cc @@ -185,7 +185,8 @@ int RGWOrphanStore::read_entries(const string& oid, const string& marker, mapctx()->_conf->rgw_list_bucket_min_readahead, MAX_LIST_OBJS_ENTRIES); + detailed_mode = detailed_mode; RGWOrphanSearchState state; r = orphan_store.read_job(job_name, state); if (r < 0 && r != -ENOENT) { @@ -437,6 +439,12 @@ int RGWOrphanSearch::handle_stat_result(map >& oids, RGWRados: } else { RGWObjManifest& manifest = result.manifest; + if (!detailed_mode && + manifest.get_obj_size() <= manifest.get_head_size()) { + ldout(store->ctx(), 5) << "skipping object as it fits in a head" << dendl; + return 0; + } + RGWObjManifest::obj_iterator miter; for (miter = manifest.obj_begin(); miter != manifest.obj_end(); ++miter) { const rgw_raw_obj& loc = miter.get_location().get_raw_obj(store); @@ -522,6 +530,14 @@ int RGWOrphanSearch::build_linked_oids_for_bucket(const string& bucket_instance_ } ldout(store->ctx(), 20) << __func__ << ": entry.key.name=" << entry.key.name << " entry.key.instance=" << entry.key.instance << dendl; + + if (!detailed_mode && + entry.meta.accounted_size <= (uint64_t)store->ctx()->_conf->rgw_max_chunk_size) { + ldout(store->ctx(),5) << __func__ << "skipping stat as the object " << entry.key.name + << "fits in a head" << dendl; + continue; + } + rgw_obj obj(bucket_info.bucket, entry.key); RGWRados::Object op_target(store, bucket_info, obj_ctx, obj); diff --git a/src/rgw/rgw_orphan.h b/src/rgw/rgw_orphan.h index d7fbdfa62958b..ffe791c563c5c 100644 --- a/src/rgw/rgw_orphan.h +++ b/src/rgw/rgw_orphan.h @@ -163,7 +163,9 @@ class RGWOrphanSearch { uint16_t max_concurrent_ios; uint64_t stale_secs; - uint64_t max_list_obj_entries; + int64_t max_list_bucket_entries; + + bool detailed_mode; struct log_iter_info { string oid; @@ -193,7 +195,7 @@ public: return orphan_store.write_job(search_info.job_name, state); } - int init(const string& job_name, RGWOrphanSearchInfo *info); + int init(const string& job_name, RGWOrphanSearchInfo *info, bool detailed_mode=false); int create(const string& job_name, int num_shards); -- 2.39.5