rgw: mitigate bucket list with max-entries excessively high

When listing a bucket with radosgw-admin, the user can specify the
maximum number of entries. That number can be unreasonably large, and
can affect the performance and memory availability. For example:

    radosgw-admin bucket list --bucket mybucket1 --max-entries=10000000

This has the potential for creating large data structures at multiple
levels in the the call stack of the radosgw(-admin) process,
potentially causing the process to run out of memory. This change
limits the maximum number of entries requested in all but the high
level code to help mitigate this issue.

Signed-off-by: J. Eric Ivancich <ivancich@redhat.com>

diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index 556aeda9503f656a7d25b531b75283a1d0f87b39..58ed225955c9afba043fac52dab6e6e383b9e7d2 100644 (file)
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -2378,6 +2378,15 @@ int RGWRados::Bucket::update_bucket_id(const string& new_bucket_id)
 }
 
 
+static inline std::string after_delim(std::string_view delim)
+{
+  // assert: ! delim.empty()
+  std::string result{delim.data(), delim.length()};
+  result += char(255);
+  return result;
+}
+
+
 /**
  * Get ordered listing of the objects in a bucket.
  *
@@ -2395,20 +2404,11 @@ int RGWRados::Bucket::update_bucket_id(const string& new_bucket_id)
  * is_truncated: if number of objects in the bucket is bigger than
  * max, then truncated.
  */
-static inline std::string after_delim(std::string_view delim)
-{
-  // assert: ! delim.empty()
-  std::string result{delim.data(), delim.length()};
-  result += char(255);
-  return result;
-}
-
-int RGWRados::Bucket::List::list_objects_ordered(
-  int64_t max,
-  vector<rgw_bucket_dir_entry> *result,
-  map<string, bool> *common_prefixes,
-  bool *is_truncated,
-  optional_yield y)
+int RGWRados::Bucket::List::list_objects_ordered(int64_t max_p,
+                                                vector<rgw_bucket_dir_entry> *result,
+                                                map<string, bool> *common_prefixes,
+                                                bool *is_truncated,
+                                                optional_yield y)
 {
   RGWRados *store = target->get_store();
   CephContext *cct = store->ctx();
@@ -2416,7 +2416,9 @@ int RGWRados::Bucket::List::list_objects_ordered(
 
   int count = 0;
   bool truncated = true;
-  int read_ahead = std::max(cct->_conf->rgw_list_bucket_min_readahead,max);
+  const int64_t max = // protect against memory issues and non-positive vals
+    std::min(bucket_list_objects_absolute_max, std::max(int64_t(0), max_p));
+  int read_ahead = std::max(cct->_conf->rgw_list_bucket_min_readahead, max);
 
   result->clear();
 
@@ -2591,7 +2593,7 @@ done:
  * is_truncated: if number of objects in the bucket is bigger than max, then
  *               truncated.
  */
-int RGWRados::Bucket::List::list_objects_unordered(int64_t max,
+int RGWRados::Bucket::List::list_objects_unordered(int64_t max_p,
                                                   vector<rgw_bucket_dir_entry> *result,
                                                   map<string, bool> *common_prefixes,
                                                   bool *is_truncated,
@@ -2604,6 +2606,9 @@ int RGWRados::Bucket::List::list_objects_unordered(int64_t max,
   int count = 0;
   bool truncated = true;
 
+  const int64_t max = // protect against memory issues and non-positive vals
+    std::min(bucket_list_objects_absolute_max, std::max(int64_t(1), max_p));
+
   // read a few extra in each call to cls_bucket_list_unordered in
   // case some are filtered out due to namespace matching, versioning,
   // filtering, etc.

diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h
index 235dfab0a0954cb4fed2d02fa03836a0b3fdf73b..637d18ff61ce29377fe142e1fb558119efce3cfa 100644 (file)
--- a/src/rgw/rgw_rados.h
+++ b/src/rgw/rgw_rados.h
@@ -1781,6 +1781,9 @@ public:
 
     class List {
     protected:
+      // absolute maximum number of objects that
+      // list_objects_(un)ordered can return
+      static constexpr int64_t bucket_list_objects_absolute_max = 25000;
 
       RGWRados::Bucket *target;
       rgw_obj_key next_marker;