rgw: mitigate bucket list with max-entries excessively high

When listing a bucket with radosgw-admin, the user can specify the
maximum number of entries. That number can be unreasonably large, and
can affect the performance and memory availability. For example:

    radosgw-admin bucket list --bucket mybucket1 --max-entries=10000000

This has the potential for creating large data structures at multiple
levels in the the call stack of the radosgw(-admin) process,
potentially causing the process to run out of memory. This change
limits the maximum number of entries requested in all but the high
level code to help mitigate this issue.

Signed-off-by: J. Eric Ivancich <ivancich@redhat.com>
(cherry picked from commit 300429c9e98a27e17c2a20ade82c6c63ac276c20)

Conflicts:
	src/rgw/rgw_rados.cc
- in luminous, RGWRados::Bucket::List::list_objects_ordered does not take
  optional_yield parameter
- no std::string_view in C++11

diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index e57d8b83ad75fd062042855abe9b9fa3e876fec4..b0d34d7ef0e1feecc9f7b68d5124c793f7cb8f38 100644 (file)
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -5610,6 +5610,15 @@ int RGWRados::Bucket::update_bucket_id(const string& new_bucket_id)
 }
 
 
+static inline std::string after_delim(std::string delim)
+{
+  // assert: ! delim.empty()
+  std::string result{delim.data(), delim.length()};
+  result += char(255);
+  return result;
+}
+
+
 /**
  * Get ordered listing of the objects in a bucket.
  *
@@ -5625,7 +5634,7 @@ int RGWRados::Bucket::update_bucket_id(const string& new_bucket_id)
  * common_prefixes: if delim is filled in, any matching prefixes are placed here.
  * is_truncated: if number of objects in the bucket is bigger than max, then truncated.
  */
-int RGWRados::Bucket::List::list_objects_ordered(int64_t max,
+int RGWRados::Bucket::List::list_objects_ordered(int64_t max_p,
                                                 vector<rgw_bucket_dir_entry> *result,
                                                 map<string, bool> *common_prefixes,
                                                 bool *is_truncated)
@@ -5636,7 +5645,9 @@ int RGWRados::Bucket::List::list_objects_ordered(int64_t max,
 
   int count = 0;
   bool truncated = true;
-  int read_ahead = std::max(cct->_conf->rgw_list_bucket_min_readahead,max);
+  const int64_t max = // protect against memory issues and non-positive vals
+    std::min(bucket_list_objects_absolute_max, std::max(int64_t(0), max_p));
+  int read_ahead = std::max(cct->_conf->rgw_list_bucket_min_readahead, max);
 
   result->clear();
 
@@ -5813,7 +5824,7 @@ done:
  * is_truncated: if number of objects in the bucket is bigger than max, then
  *               truncated.
  */
-int RGWRados::Bucket::List::list_objects_unordered(int64_t max,
+int RGWRados::Bucket::List::list_objects_unordered(int64_t max_p,
                                                   vector<rgw_bucket_dir_entry> *result,
                                                   map<string, bool> *common_prefixes,
                                                   bool *is_truncated)
@@ -5825,6 +5836,9 @@ int RGWRados::Bucket::List::list_objects_unordered(int64_t max,
   int count = 0;
   bool truncated = true;
 
+  const int64_t max = // protect against memory issues and non-positive vals
+    std::min(bucket_list_objects_absolute_max, std::max(int64_t(1), max_p));
+
   // read a few extra in each call to cls_bucket_list_unordered in
   // case some are filtered out due to namespace matching, versioning,
   // filtering, etc.

diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h
index f19e10d39519e169cd79759666f4bff95b1beadd..aa377a710267671c06b2bd814a17a7246a4f6ad1 100644 (file)
--- a/src/rgw/rgw_rados.h
+++ b/src/rgw/rgw_rados.h
@@ -3005,6 +3005,9 @@ public:
 
     class List {
     protected:
+      // absolute maximum number of objects that
+      // list_objects_(un)ordered can return
+      static constexpr int64_t bucket_list_objects_absolute_max = 25000;
 
       RGWRados::Bucket *target;
       rgw_obj_key next_marker;