rgw: improve delmited listing of bucket

If found a prefix, calculate a string greater than that so that next
request we can skip to that. This is still not the most efficient way to
do it. It'll be better to push it down to the objclass, but that'll
require a much bigger change.

Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>

diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index a73b9c2418918e9a848a69ad9f7bf19a5163e7a8..6b2cc8ce18618eddddc2279a3540c2c5d7d52adf 100644 (file)
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -6,6 +6,7 @@
 #include <sys/types.h>
 
 #include "common/ceph_json.h"
+#include "common/utf8.h"
 
 #include "common/errno.h"
 #include "common/Formatter.h"
@@ -2271,7 +2272,28 @@ int RGWRados::list_objects(rgw_bucket& bucket, int max, string& prefix, string&
   prefix_obj.set_obj(prefix);
   string cur_prefix = prefix_obj.object;
 
+  string bigger_than_delim;
+
+  if (!delim.empty()) {
+    unsigned long val = decode_utf8((unsigned char *)delim.c_str(), delim.size());
+    char buf[delim.size() + 16];
+    int r = encode_utf8(val + 1, (unsigned char *)buf);
+    if (r < 0) {
+      ldout(cct,0) << "ERROR: encode_utf8() failed" << dendl;
+      return -EINVAL;
+    }
+    buf[r] = '\0';
+
+    bigger_than_delim = buf;
+  }
+
+  string skip_after_delim;
+
   do {
+    if (skip_after_delim > cur_marker) {
+      cur_marker = skip_after_delim;
+      ldout(cct, 20) << "setting cur_marker=" << cur_marker << dendl;
+    }
     std::map<string, RGWObjEnt> ent_map;
     int r = cls_bucket_list(bucket, cur_marker, cur_prefix, max - count, ent_map,
                             &truncated, &cur_marker);
@@ -2307,6 +2329,12 @@ int RGWRados::list_objects(rgw_bucket& bucket, int max, string& prefix, string&
 
         if (delim_pos >= 0) {
           common_prefixes[obj.substr(0, delim_pos + 1)] = true;
+
+          skip_after_delim = obj.substr(0, delim_pos);
+          skip_after_delim.append(bigger_than_delim);
+
+          ldout(cct, 20) << "skip_after_delim=" << skip_after_delim << dendl;
+
           continue;
         }
       }