rgw: add NextMarker param for bucket listing

Partially fixes #8858.

Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
(cherry picked from commit 924686f0b6593deffcd1d4e80ab06b1e7af00dcb)

diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index 2d6b6e8504f2a00da1e8eb429740916b624deec8..16209eaa1081736930141c02db3e0397e90a8dd2 100644 (file)
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -1358,7 +1358,7 @@ int main(int argc, char **argv)
       do {
         list<rgw_bi_log_entry> entries;
         ret = store->list_objects(bucket, max_entries - count, prefix, delim,
-                                  marker, result, common_prefixes, true,
+                                  marker, NULL, result, common_prefixes, true,
                                   ns, false, &truncated, NULL);
         if (ret < 0) {
           cerr << "ERROR: store->list_objects(): " << cpp_strerror(-ret) << std::endl;

diff --git a/src/rgw/rgw_bucket.cc b/src/rgw/rgw_bucket.cc
index 3e57b934fef7768fa06c62472224abb2251730e2..79442e89470885cb8fabe150ea74b63b5e05a369 100644 (file)
--- a/src/rgw/rgw_bucket.cc
+++ b/src/rgw/rgw_bucket.cc
@@ -333,7 +333,7 @@ int rgw_remove_bucket(RGWRados *store, rgw_bucket& bucket, bool delete_children)
 
   if (delete_children) {
     int max = 1000;
-    ret = store->list_objects(bucket, max, prefix, delim, marker,
+    ret = store->list_objects(bucket, max, prefix, delim, marker, NULL,
             objs, common_prefixes,
             false, ns, true, NULL, NULL);
 
@@ -349,7 +349,7 @@ int rgw_remove_bucket(RGWRados *store, rgw_bucket& bucket, bool delete_children)
       }
       objs.clear();
 
-      ret = store->list_objects(bucket, max, prefix, delim, marker, objs, common_prefixes,
+      ret = store->list_objects(bucket, max, prefix, delim, marker, NULL, objs, common_prefixes,
                                 false, ns, true, NULL, NULL);
       if (ret < 0)
         return ret;
@@ -597,7 +597,7 @@ int RGWBucket::check_bad_index_multipart(RGWBucketAdminOpState& op_state,
 
   do {
     vector<RGWObjEnt> result;
-    int r = store->list_objects(bucket, max, prefix, delim, marker,
+    int r = store->list_objects(bucket, max, prefix, delim, marker, NULL,
                                 result, common_prefixes, false,
                                 ns, true,
                                 &is_truncated, NULL);

diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index ac1e11c86ec64fac19fd0627524614ba556095b7..7c56955269eadfecd30278e0cc7a8db743a6a34d 100644 (file)
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -640,7 +640,7 @@ int RGWGetObj::iterate_user_manifest_parts(rgw_bucket& bucket, string& obj_prefi
 
   do {
 #define MAX_LIST_OBJS 100
-    int r = store->list_objects(bucket, MAX_LIST_OBJS, obj_prefix, delim, marker,
+    int r = store->list_objects(bucket, MAX_LIST_OBJS, obj_prefix, delim, marker, NULL,
                                 objs, common_prefixes,
                                 true, no_ns, true, &is_truncated, NULL);
     if (r < 0)
@@ -1016,7 +1016,9 @@ void RGWListBucket::execute()
   if (ret < 0)
     return;
 
-  ret = store->list_objects(s->bucket, max, prefix, delimiter, marker, objs, common_prefixes,
+  string *pnext_marker = (delimiter.empty() ? NULL : &next_marker);
+
+  ret = store->list_objects(s->bucket, max, prefix, delimiter, marker, pnext_marker, objs, common_prefixes,
                                !!(s->prot_flags & RGW_REST_SWIFT), no_ns, true, &is_truncated, NULL);
 }
 
@@ -2520,7 +2522,7 @@ void RGWListBucketMultiparts::execute()
     }
   }
   marker_meta = marker.get_meta();
-  ret = store->list_objects(s->bucket, max_uploads, prefix, delimiter, marker_meta, objs, common_prefixes,
+  ret = store->list_objects(s->bucket, max_uploads, prefix, delimiter, marker_meta, NULL, objs, common_prefixes,
                                !!(s->prot_flags & RGW_REST_SWIFT), mp_ns, true, &is_truncated, &mp_filter);
   if (!objs.empty()) {
     vector<RGWObjEnt>::iterator iter;

diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h
index 2c9ab0d899b1d6e8cb4acb8d0171a3a950fba1f0..911800cd5f4bb7c665802562ad24f92062776687 100644 (file)
--- a/src/rgw/rgw_op.h
+++ b/src/rgw/rgw_op.h
@@ -187,6 +187,7 @@ class RGWListBucket : public RGWOp {
 protected:
   string prefix;
   string marker; 
+  string next_marker; 
   string max_keys;
   string delimiter;
   int max;

diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index 33fc4c436cada57bd1456c3348daa5c8c44dcfbc..6492f25a4ffd188eabbb2027e1b9fcbc572cb816 100644 (file)
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -1714,7 +1714,8 @@ int rgw_policy_from_attrset(CephContext *cct, map<string, bufferlist>& attrset,
  *     here.
  */
 int RGWRados::list_objects(rgw_bucket& bucket, int max, string& prefix, string& delim,
-                          string& marker, vector<RGWObjEnt>& result, map<string, bool>& common_prefixes,
+                          string& marker, string *next_marker, vector<RGWObjEnt>& result,
+                           map<string, bool>& common_prefixes,
                           bool get_content_type, string& ns, bool enforce_ns,
                            bool *is_truncated, RGWAccessListFilter *filter)
 {
@@ -1781,6 +1782,10 @@ int RGWRados::list_objects(rgw_bucket& bucket, int max, string& prefix, string&
         continue;
       }
 
+      if (next_marker) {
+        *next_marker = obj;
+      }
+
       if (filter && !filter->filter(obj, key))
         continue;
 

diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h
index ea5e53429233a2705986aa02ce9af660a1e1a2bc..7fdac57e1be186de3c61c81bdfa5566aecd05539 100644 (file)
--- a/src/rgw/rgw_rados.h
+++ b/src/rgw/rgw_rados.h
@@ -968,8 +968,9 @@ public:
    *     here.
    */
   virtual int list_objects(rgw_bucket& bucket, int max, std::string& prefix, std::string& delim,
-                   std::string& marker, std::vector<RGWObjEnt>& result, map<string, bool>& common_prefixes,
-                  bool get_content_type, string& ns, bool enforce_ns, bool *is_truncated, RGWAccessListFilter *filter);
+                   std::string& marker, std::string *next_marker, std::vector<RGWObjEnt>& result,
+                   map<string, bool>& common_prefixes, bool get_content_type, string& ns, bool enforce_ns,
+                   bool *is_truncated, RGWAccessListFilter *filter);
 
   virtual int create_pool(rgw_bucket& bucket);
 

diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc
index c6eacce29f6ac30c9dab591d621b6b7d17a72fdf..3811f6c1a5c76c84728a75fa65d89ef7248a2580 100644 (file)
--- a/src/rgw/rgw_rest_s3.cc
+++ b/src/rgw/rgw_rest_s3.cc
@@ -254,6 +254,8 @@ void RGWListBucket_ObjStore_S3::send_response()
   if (!prefix.empty())
     s->formatter->dump_string("Prefix", prefix);
   s->formatter->dump_string("Marker", marker);
+  if (is_truncated && !next_marker.empty())
+    s->formatter->dump_string("NextMarker", next_marker);
   s->formatter->dump_int("MaxKeys", max);
   if (!delimiter.empty())
     s->formatter->dump_string("Delimiter", delimiter);