From f17ceada0a39933a7d41efa1fc0d81e48de3def4 Mon Sep 17 00:00:00 2001 From: "J. Eric Ivancich" Date: Fri, 19 Jul 2019 16:10:59 -0400 Subject: [PATCH] rgw: mitigate bucket list with max-entries excessively high When listing a bucket with radosgw-admin, the user can specify the maximum number of entries. That number can be unreasonably large, and can affect the performance and memory availability. For example: radosgw-admin bucket list --bucket mybucket1 --max-entries=10000000 This has the potential for creating large data structures at multiple levels in the the call stack of the radosgw(-admin) process, potentially causing the process to run out of memory. This change limits the maximum number of entries requested in all but the high level code to help mitigate this issue. Signed-off-by: J. Eric Ivancich (cherry picked from commit 300429c9e98a27e17c2a20ade82c6c63ac276c20) Conflicts: src/rgw/rgw_rados.cc - RGWRados::Bucket::List::list_objects_ordered takes an additional argument (unrelated to this change) in master --- src/rgw/rgw_rados.cc | 35 ++++++++++++++++++++--------------- src/rgw/rgw_rados.h | 3 +++ 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index 32db310a0c96e..f2858ac1c85bb 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -5619,6 +5619,15 @@ int RGWRados::Bucket::update_bucket_id(const string& new_bucket_id) } +static inline std::string after_delim(std::string_view delim) +{ + // assert: ! delim.empty() + std::string result{delim.data(), delim.length()}; + result += char(255); + return result; +} + + /** * Get ordered listing of the objects in a bucket. * @@ -5636,19 +5645,10 @@ int RGWRados::Bucket::update_bucket_id(const string& new_bucket_id) * is_truncated: if number of objects in the bucket is bigger than * max, then truncated. */ -static inline std::string after_delim(std::string_view delim) -{ - // assert: ! delim.empty() - std::string result{delim.data(), delim.length()}; - result += char(255); - return result; -} - -int RGWRados::Bucket::List::list_objects_ordered( - int64_t max, - vector *result, - map *common_prefixes, - bool *is_truncated) +int RGWRados::Bucket::List::list_objects_ordered(int64_t max_p, + vector *result, + map *common_prefixes, + bool *is_truncated) { RGWRados *store = target->get_store(); CephContext *cct = store->ctx(); @@ -5656,7 +5656,9 @@ int RGWRados::Bucket::List::list_objects_ordered( int count = 0; bool truncated = true; - int read_ahead = std::max(cct->_conf->rgw_list_bucket_min_readahead,max); + const int64_t max = // protect against memory issues and non-positive vals + std::min(bucket_list_objects_absolute_max, std::max(int64_t(0), max_p)); + int read_ahead = std::max(cct->_conf->rgw_list_bucket_min_readahead, max); result->clear(); @@ -5829,7 +5831,7 @@ done: * is_truncated: if number of objects in the bucket is bigger than max, then * truncated. */ -int RGWRados::Bucket::List::list_objects_unordered(int64_t max, +int RGWRados::Bucket::List::list_objects_unordered(int64_t max_p, vector *result, map *common_prefixes, bool *is_truncated) @@ -5841,6 +5843,9 @@ int RGWRados::Bucket::List::list_objects_unordered(int64_t max, int count = 0; bool truncated = true; + const int64_t max = // protect against memory issues and non-positive vals + std::min(bucket_list_objects_absolute_max, std::max(int64_t(1), max_p)); + // read a few extra in each call to cls_bucket_list_unordered in // case some are filtered out due to namespace matching, versioning, // filtering, etc. diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h index a2cd710a2ebbe..b552101bee936 100644 --- a/src/rgw/rgw_rados.h +++ b/src/rgw/rgw_rados.h @@ -3040,6 +3040,9 @@ public: class List { protected: + // absolute maximum number of objects that + // list_objects_(un)ordered can return + static constexpr int64_t bucket_list_objects_absolute_max = 25000; RGWRados::Bucket *target; rgw_obj_key next_marker; -- 2.39.5