rgw: improve delmited listing of bucket

If found a prefix, calculate a string greater than that so that next
request we can skip to that. This is still not the most efficient way to
do it. It'll be better to push it down to the objclass, but that'll
require a much bigger change.

Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
(cherry picked from commit e6cf618c257f26f97f60a4c1df1d23a14496cab0)

diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index 3a15c73918aff676358090a7c1fdaacf324a4e24..33fc4c436cada57bd1456c3348daa5c8c44dcfbc 100644 (file)
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -3,6 +3,7 @@
 #include <sys/types.h>
 
 #include "common/ceph_json.h"
+#include "common/utf8.h"
 
 #include "common/errno.h"
 #include "common/Formatter.h"
@@ -1734,7 +1735,28 @@ int RGWRados::list_objects(rgw_bucket& bucket, int max, string& prefix, string&
   prefix_obj.set_obj(prefix);
   string cur_prefix = prefix_obj.object;
 
+  string bigger_than_delim;
+
+  if (!delim.empty()) {
+    unsigned long val = decode_utf8((unsigned char *)delim.c_str(), delim.size());
+    char buf[delim.size() + 16];
+    int r = encode_utf8(val + 1, (unsigned char *)buf);
+    if (r < 0) {
+      ldout(cct,0) << "ERROR: encode_utf8() failed" << dendl;
+      return -EINVAL;
+    }
+    buf[r] = '\0';
+
+    bigger_than_delim = buf;
+  }
+
+  string skip_after_delim;
+
   do {
+    if (skip_after_delim > cur_marker) {
+      cur_marker = skip_after_delim;
+      ldout(cct, 20) << "setting cur_marker=" << cur_marker << dendl;
+    }
     std::map<string, RGWObjEnt> ent_map;
     int r = cls_bucket_list(bucket, cur_marker, cur_prefix, max - count, ent_map,
                             &truncated, &cur_marker);
@@ -1770,6 +1792,12 @@ int RGWRados::list_objects(rgw_bucket& bucket, int max, string& prefix, string&
 
         if (delim_pos >= 0) {
           common_prefixes[obj.substr(0, delim_pos + 1)] = true;
+
+          skip_after_delim = obj.substr(0, delim_pos);
+          skip_after_delim.append(bigger_than_delim);
+
+          ldout(cct, 20) << "skip_after_delim=" << skip_after_delim << dendl;
+
           continue;
         }
       }