rgw: radosgw-admin should paginate internally when listing bucket

Currently `radosgw-admin bucket list ...`, when listing a bucket, asks
for the value of "--max-entries" internally. To list a large bucket
entirely the user would have to set "--max-entries" to a large value
(e.g., 10000000). Internally this doesn't paginate, so it will try to
produce the entire list at once. This can consume a lot of memory, and
there are known cases where this induces an out-of-memory crash.

So now we'll set a maximum pagination size of 10,000. So even with
large values of "--max-entries" it will still be able to produce the
full listing without stressing memory, because it will ask for at most
10,000 entries at a time.

Signed-off-by: J. Eric Ivancich <ivancich@redhat.com>

diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index b32561867256c57bb894c2bb612001ed15e27dd1..bb57432e786c50ff056a4fb0a71b8bf9ed6f5909 100644 (file)
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -6002,10 +6002,17 @@ int main(int argc, const char **argv)
         return -ret;
       }
       formatter->open_array_section("entries");
-      bool truncated;
+
+      bool truncated = false;
       int count = 0;
-      if (max_entries < 0)
-        max_entries = 1000;
+
+      static constexpr int MAX_PAGINATE_SIZE = 10000;
+      static constexpr int DEFAULT_MAX_ENTRIES = 1000;
+
+      if (max_entries < 0) {
+       max_entries = DEFAULT_MAX_ENTRIES;
+      }
+      const int paginate_size = std::min(max_entries, MAX_PAGINATE_SIZE);
 
       string prefix;
       string delim;
@@ -6025,7 +6032,10 @@ int main(int argc, const char **argv)
       list_op.params.allow_unordered = bool(allow_unordered);
 
       do {
-        ret = list_op.list_objects(max_entries - count, &result, &common_prefixes, &truncated, null_yield);
+        const int remaining = max_entries - count;
+        ret = list_op.list_objects(std::min(remaining, paginate_size),
+                                  &result, &common_prefixes, &truncated,
+                                  null_yield);
         if (ret < 0) {
           cerr << "ERROR: store->list_objects(): " << cpp_strerror(-ret) << std::endl;
           return -ret;
@@ -6033,8 +6043,7 @@ int main(int argc, const char **argv)
 
         count += result.size();
 
-        for (vector<rgw_bucket_dir_entry>::iterator iter = result.begin(); iter != result.end(); ++iter) {
-          rgw_bucket_dir_entry& entry = *iter;
+        for (const auto& entry : result) {
           encode_json("entry", entry, formatter.get());
         }
         formatter->flush(cout);