From: J. Eric Ivancich Date: Wed, 13 Oct 2021 21:58:55 +0000 (-0400) Subject: rgw: allow force-check filter to pass through the SAL layer X-Git-Tag: v17.1.0~633^2 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=6517d942c7f90a24355c659023ddf62385504c7f;p=ceph-ci.git rgw: allow force-check filter to pass through the SAL layer The zipper updates did not pass a force-check filter from RGWRados::Bucket::List through to RGWRados::cls_bucket_list_ordered. This filter is necessary for the "radosgw-admin bucket check --fix..." functionality. Declares type RGWBucketListNameFilter to encapsualte the filter type. Renames some fields so the two filters can be distinguished. Signed-off-by: J. Eric Ivancich --- diff --git a/src/rgw/rgw_bucket.cc b/src/rgw/rgw_bucket.cc index edc7f802731..ac9656aea82 100644 --- a/src/rgw/rgw_bucket.cc +++ b/src/rgw/rgw_bucket.cc @@ -295,12 +295,13 @@ void check_bad_user_bucket_mapping(rgw::sal::Store* store, rgw::sal::User* user, } while (user_buckets.is_truncated()); } -// note: function type conforms to RGWRados::check_filter_t -bool rgw_bucket_object_check_filter(const string& oid) +// returns true if entry is in the empty namespace. note: function +// type conforms to type RGWBucketListNameFilter +bool rgw_bucket_object_check_filter(const std::string& oid) { - rgw_obj_key key; - string ns; - return rgw_obj_key::oid_to_key_in_ns(oid, &key, ns); + const static std::string empty_ns; + rgw_obj_key key; // thrown away but needed for parsing + return rgw_obj_key::oid_to_key_in_ns(oid, &key, empty_ns); } int rgw_remove_object(const DoutPrefixProvider *dpp, rgw::sal::Store* store, rgw::sal::Bucket* bucket, rgw_obj_key& key) @@ -604,6 +605,7 @@ int RGWBucket::check_object_index(const DoutPrefixProvider *dpp, while (results.is_truncated) { rgw::sal::Bucket::ListParams params; params.marker = results.next_marker; + params.force_check_filter = rgw_bucket_object_check_filter; int r = bucket->list(dpp, params, listing_max_entries, results, y); diff --git a/src/rgw/rgw_bucket.h b/src/rgw/rgw_bucket.h index 6ec09b9f667..0dbc4904f2a 100644 --- a/src/rgw/rgw_bucket.h +++ b/src/rgw/rgw_bucket.h @@ -50,7 +50,7 @@ extern void rgw_parse_url_bucket(const std::string& bucket, std::string &tenant_name, std::string &bucket_name); // this is used as a filter to RGWRados::cls_bucket_list_ordered; it -// conforms to the type declaration of RGWRados::check_filter_t. +// conforms to the type RGWBucketListNameFilter extern bool rgw_bucket_object_check_filter(const std::string& oid); void init_default_bucket_layout(CephContext *cct, rgw::BucketLayout& layout, diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc index 485848d2297..8e19e36bc47 100644 --- a/src/rgw/rgw_lc.cc +++ b/src/rgw/rgw_lc.cc @@ -837,7 +837,7 @@ int RGWLC::handle_multipart_expiration(rgw::sal::Bucket* target, * operating on one shard at a time */ params.allow_unordered = true; params.ns = RGW_OBJ_NS_MULTIPART; - params.filter = &mp_filter; + params.access_list_filter = &mp_filter; auto pf = [&](RGWLC::LCWorker* wk, WorkQ* wq, WorkItem& wi) { auto wt = boost::get>(wi); diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index 9b4f77a8ca9..2c2b0a57ee5 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -110,6 +110,7 @@ static RGWObjCategory main_category = RGWObjCategory::Main; #define RGW_USAGE_OBJ_PREFIX "usage." +// returns true on success, false on failure static bool rgw_get_obj_data_pool(const RGWZoneGroup& zonegroup, const RGWZoneParams& zone_params, const rgw_placement_rule& head_placement_rule, const rgw_obj& obj, rgw_pool *pool) @@ -1866,7 +1867,8 @@ int RGWRados::Bucket::List::list_objects_ordered( &truncated, &cls_filtered, &cur_marker, - y); + y, + params.force_check_filter); if (r < 0) { return r; } @@ -1918,8 +1920,8 @@ int RGWRados::Bucket::List::list_objects_ordered( next_marker = index_key; } - if (params.filter && - ! params.filter->filter(obj.name, index_key.name)) { + if (params.access_list_filter && + ! params.access_list_filter->filter(obj.name, index_key.name)) { continue; } @@ -2168,12 +2170,15 @@ int RGWRados::Bucket::List::list_objects_unordered(const DoutPrefixProvider *dpp continue; } - if (params.filter && !params.filter->filter(obj.name, index_key.name)) + if (params.access_list_filter && + !params.access_list_filter->filter(obj.name, index_key.name)) { continue; + } if (params.prefix.size() && - (0 != obj.name.compare(0, params.prefix.size(), params.prefix))) + (0 != obj.name.compare(0, params.prefix.size(), params.prefix))) { continue; + } if (count >= max) { truncated = true; @@ -2329,6 +2334,7 @@ int RGWRados::create_bucket(const RGWUserInfo& owner, rgw_bucket& bucket, return -ENOENT; } +// returns true on success, false on failure bool RGWRados::get_obj_data_pool(const rgw_placement_rule& placement_rule, const rgw_obj& obj, rgw_pool *pool) { return rgw_get_obj_data_pool(svc.zone->get_zonegroup(), svc.zone->get_zone_params(), placement_rule, obj, pool); @@ -8377,7 +8383,7 @@ int RGWRados::cls_bucket_list_ordered(const DoutPrefixProvider *dpp, bool* cls_filtered, rgw_obj_index_key *last_entry, optional_yield y, - check_filter_t force_check_filter) + RGWBucketListNameFilter force_check_filter) { /* expansion_factor allows the number of entries to read to grow * exponentially; this is used when earlier reads are producing too @@ -8670,7 +8676,7 @@ int RGWRados::cls_bucket_list_unordered(const DoutPrefixProvider *dpp, bool *is_truncated, rgw_obj_index_key *last_entry, optional_yield y, - check_filter_t force_check_filter) { + RGWBucketListNameFilter force_check_filter) { ldpp_dout(dpp, 10) << __func__ << " " << bucket_info.bucket << " start_after=\"" << start_after << "\", prefix=\"" << prefix << @@ -9049,12 +9055,16 @@ int RGWRados::check_disk_state(const DoutPrefixProvider *dpp, list_state.meta.category = main_category; list_state.meta.etag = etag; list_state.meta.content_type = content_type; - if (astate->obj_tag.length() > 0) + + if (astate->obj_tag.length() > 0) { list_state.tag = astate->obj_tag.c_str(); + } + list_state.meta.owner = owner.get_id().to_str(); list_state.meta.owner_display_name = owner.get_display_name(); list_state.exists = true; + cls_rgw_encode_suggestion(CEPH_RGW_UPDATE | suggest_flag, list_state, suggested_updates); return 0; } diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h index 26b603b128b..f335ecc00ba 100644 --- a/src/rgw/rgw_rados.h +++ b/src/rgw/rgw_rados.h @@ -986,13 +986,14 @@ public: rgw_obj_key end_marker; std::string ns; bool enforce_ns; - RGWAccessListFilter *filter; + RGWAccessListFilter* access_list_filter; + RGWBucketListNameFilter force_check_filter; bool list_versions; bool allow_unordered; Params() : enforce_ns(true), - filter(NULL), + access_list_filter(nullptr), list_versions(false), allow_unordered(false) {} @@ -1361,8 +1362,6 @@ public: using ent_map_t = boost::container::flat_map; - using check_filter_t = bool (*)(const std::string&); - int cls_bucket_list_ordered(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info, const int shard_id, @@ -1377,7 +1376,7 @@ public: bool* cls_filtered, rgw_obj_index_key *last_entry, optional_yield y, - check_filter_t force_check_filter = nullptr); + RGWBucketListNameFilter force_check_filter = {}); int cls_bucket_list_unordered(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info, int shard_id, @@ -1389,7 +1388,7 @@ public: bool *is_truncated, rgw_obj_index_key *last_entry, optional_yield y, - check_filter_t = nullptr); + RGWBucketListNameFilter force_check_filter = {}); int cls_bucket_head(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, int shard_id, diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h index 68120239214..7bd96a7bc41 100644 --- a/src/rgw/rgw_sal.h +++ b/src/rgw/rgw_sal.h @@ -38,6 +38,10 @@ class RGWSyncModuleInstance; typedef std::shared_ptr RGWSyncModuleInstanceRef; class RGWCompressionInfo; + +using RGWBucketListNameFilter = std::function; + + namespace rgw { class Aio; namespace IAM { struct Policy; } @@ -378,7 +382,8 @@ class Bucket { rgw_obj_key end_marker; std::string ns; bool enforce_ns{true}; - RGWAccessListFilter* filter{nullptr}; + RGWAccessListFilter* access_list_filter{nullptr}; + RGWBucketListNameFilter force_check_filter; bool list_versions{false}; bool allow_unordered{false}; int shard_id{RGW_NO_SHARD}; diff --git a/src/rgw/rgw_sal_dbstore.cc b/src/rgw/rgw_sal_dbstore.cc index 67cbdad666d..cd048493982 100644 --- a/src/rgw/rgw_sal_dbstore.cc +++ b/src/rgw/rgw_sal_dbstore.cc @@ -349,7 +349,8 @@ namespace rgw::sal { list_op.params.end_marker = params.end_marker; list_op.params.ns = params.ns; list_op.params.enforce_ns = params.enforce_ns; - list_op.params.filter = params.filter; + list_op.params.access_list_filter = params.access_list_filter; + list_op.params.force_check_filter = params.force_check_filter; list_op.params.list_versions = params.list_versions; list_op.params.allow_unordered = params.allow_unordered; diff --git a/src/rgw/rgw_sal_rados.cc b/src/rgw/rgw_sal_rados.cc index 5ce3b2f0009..e36c901a4fa 100644 --- a/src/rgw/rgw_sal_rados.cc +++ b/src/rgw/rgw_sal_rados.cc @@ -736,7 +736,8 @@ int RadosBucket::list(const DoutPrefixProvider* dpp, ListParams& params, int max list_op.params.end_marker = params.end_marker; list_op.params.ns = params.ns; list_op.params.enforce_ns = params.enforce_ns; - list_op.params.filter = params.filter; + list_op.params.access_list_filter = params.access_list_filter; + list_op.params.force_check_filter = params.force_check_filter; list_op.params.list_versions = params.list_versions; list_op.params.allow_unordered = params.allow_unordered; @@ -766,7 +767,7 @@ int RadosBucket::list_multiparts(const DoutPrefixProvider *dpp, params.delim = delim; params.marker = marker; params.ns = RGW_OBJ_NS_MULTIPART; - params.filter = &mp_filter; + params.access_list_filter = &mp_filter; int ret = list(dpp, params, max_uploads, results, null_yield); diff --git a/src/rgw/store/dbstore/common/dbstore.h b/src/rgw/store/dbstore/common/dbstore.h index 3515fbe718b..4619ff55d2a 100644 --- a/src/rgw/store/dbstore/common/dbstore.h +++ b/src/rgw/store/dbstore/common/dbstore.h @@ -1378,13 +1378,14 @@ class DB { rgw_obj_key end_marker; string ns; bool enforce_ns; - RGWAccessListFilter *filter; + RGWAccessListFilter* access_list_filter; + RGWBucketListNameFilter force_check_filter; bool list_versions; - bool allow_unordered; + bool allow_unordered; Params() : enforce_ns(true), - filter(NULL), + access_list_filter(nullptr), list_versions(false), allow_unordered(false) {}