rgw: improve delmited listing of bucket

If found a prefix, calculate a string greater than that so that next
request we can skip to that. This is still not the most efficient way to
do it. It'll be better to push it down to the objclass, but that'll
require a much bigger change.

Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
(cherry picked from commit e6cf618c257f26f97f60a4c1df1d23a14496cab0)

diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index 8875993cb9231366a4f3beab3814be5102b18cda..cb84ab37fafe903441a7818a6683665fa797d635 100644 (file)
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -3,6 +3,7 @@
 #include <sys/types.h>
 
 #include "common/ceph_json.h"
+#include "common/utf8.h"
 
 #include "common/errno.h"
 #include "common/Formatter.h"
@@ -2106,7 +2107,28 @@ int RGWRados::list_objects(rgw_bucket& bucket, int max, string& prefix, string&
   prefix_obj.set_obj(prefix);
   string cur_prefix = prefix_obj.object;
 
+  string bigger_than_delim;
+
+  if (!delim.empty()) {
+    unsigned long val = decode_utf8((unsigned char *)delim.c_str(), delim.size());
+    char buf[delim.size() + 16];
+    int r = encode_utf8(val + 1, (unsigned char *)buf);
+    if (r < 0) {
+      ldout(cct,0) << "ERROR: encode_utf8() failed" << dendl;
+      return -EINVAL;
+    }
+    buf[r] = '\0';
+
+    bigger_than_delim = buf;
+  }
+
+  string skip_after_delim;
+
   do {
+    if (skip_after_delim > cur_marker) {
+      cur_marker = skip_after_delim;
+      ldout(cct, 20) << "setting cur_marker=" << cur_marker << dendl;
+    }
     std::map<string, RGWObjEnt> ent_map;
     int r = cls_bucket_list(bucket, cur_marker, cur_prefix, max - count, ent_map,
                             &truncated, &cur_marker);
@@ -2142,6 +2164,12 @@ int RGWRados::list_objects(rgw_bucket& bucket, int max, string& prefix, string&
 
         if (delim_pos >= 0) {
           common_prefixes[obj.substr(0, delim_pos + 1)] = true;
+
+          skip_after_delim = obj.substr(0, delim_pos);
+          skip_after_delim.append(bigger_than_delim);
+
+          ldout(cct, 20) << "skip_after_delim=" << skip_after_delim << dendl;
+
           continue;
         }
       }