rgw: mitigate bucket list with max-entries excessively high

When listing a bucket with radosgw-admin, the user can specify the
maximum number of entries. That number can be unreasonably large, and
can affect the performance and memory availability. For example:

    radosgw-admin bucket list --bucket mybucket1 --max-entries=10000000

This has the potential for creating large data structures at multiple
levels in the the call stack of the radosgw(-admin) process,
potentially causing the process to run out of memory. This change
limits the maximum number of entries requested in all but the high
level code to help mitigate this issue.

Signed-off-by: J. Eric Ivancich <ivancich@redhat.com>
(cherry picked from commit 300429c9e98a27e17c2a20ade82c6c63ac276c20)

Conflicts:
	src/rgw/rgw_rados.cc
- RGWRados::Bucket::List::list_objects_ordered takes an additional argument
  (unrelated to this change) in master

diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index 32db310a0c96e4735610ef7a14c2c5c743844e8d..f2858ac1c85bb36c895fe5e07e711ec0080cc93d 100644 (file)
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -5619,6 +5619,15 @@ int RGWRados::Bucket::update_bucket_id(const string& new_bucket_id)
 }
 
 
+static inline std::string after_delim(std::string_view delim)
+{
+  // assert: ! delim.empty()
+  std::string result{delim.data(), delim.length()};
+  result += char(255);
+  return result;
+}
+
+
 /**
  * Get ordered listing of the objects in a bucket.
  *
@@ -5636,19 +5645,10 @@ int RGWRados::Bucket::update_bucket_id(const string& new_bucket_id)
  * is_truncated: if number of objects in the bucket is bigger than
  * max, then truncated.
  */
-static inline std::string after_delim(std::string_view delim)
-{
-  // assert: ! delim.empty()
-  std::string result{delim.data(), delim.length()};
-  result += char(255);
-  return result;
-}
-
-int RGWRados::Bucket::List::list_objects_ordered(
-  int64_t max,
-  vector<rgw_bucket_dir_entry> *result,
-  map<string, bool> *common_prefixes,
-  bool *is_truncated)
+int RGWRados::Bucket::List::list_objects_ordered(int64_t max_p,
+                                                 vector<rgw_bucket_dir_entry> *result,
+                                                 map<string, bool> *common_prefixes,
+                                                 bool *is_truncated)
 {
   RGWRados *store = target->get_store();
   CephContext *cct = store->ctx();
@@ -5656,7 +5656,9 @@ int RGWRados::Bucket::List::list_objects_ordered(
 
   int count = 0;
   bool truncated = true;
-  int read_ahead = std::max(cct->_conf->rgw_list_bucket_min_readahead,max);
+  const int64_t max = // protect against memory issues and non-positive vals
+    std::min(bucket_list_objects_absolute_max, std::max(int64_t(0), max_p));
+  int read_ahead = std::max(cct->_conf->rgw_list_bucket_min_readahead, max);
 
   result->clear();
 
@@ -5829,7 +5831,7 @@ done:
  * is_truncated: if number of objects in the bucket is bigger than max, then
  *               truncated.
  */
-int RGWRados::Bucket::List::list_objects_unordered(int64_t max,
+int RGWRados::Bucket::List::list_objects_unordered(int64_t max_p,
                                                   vector<rgw_bucket_dir_entry> *result,
                                                   map<string, bool> *common_prefixes,
                                                   bool *is_truncated)
@@ -5841,6 +5843,9 @@ int RGWRados::Bucket::List::list_objects_unordered(int64_t max,
   int count = 0;
   bool truncated = true;
 
+  const int64_t max = // protect against memory issues and non-positive vals
+    std::min(bucket_list_objects_absolute_max, std::max(int64_t(1), max_p));
+
   // read a few extra in each call to cls_bucket_list_unordered in
   // case some are filtered out due to namespace matching, versioning,
   // filtering, etc.

diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h
index a2cd710a2ebbed27bdb80cbf54ac463288d6bc71..b552101bee9365c8aac681103b46045a002d8416 100644 (file)
--- a/src/rgw/rgw_rados.h
+++ b/src/rgw/rgw_rados.h
@@ -3040,6 +3040,9 @@ public:
 
     class List {
     protected:
+      // absolute maximum number of objects that
+      // list_objects_(un)ordered can return
+      static constexpr int64_t bucket_list_objects_absolute_max = 25000;
 
       RGWRados::Bucket *target;
       rgw_obj_key next_marker;