rgw: account common prefixes for MaxKeys in bucket listing

To be more in line with the S3 api. Beforehand we didn't account the
common prefixes towards the MaxKeys (a single common prefix counts as a
single key). Also need to adjust the marker now if it is pointing at a
common prefix.

Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
(cherry picked from commit 82d2d612e700f94a0bb2d9fb7555abf327be379b)

diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index 6492f25a4ffd188eabbb2027e1b9fcbc572cb816..3c0fef32021d0c40f12f33ade6005e880310ca39 100644 (file)
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -1720,7 +1720,7 @@ int RGWRados::list_objects(rgw_bucket& bucket, int max, string& prefix, string&
                            bool *is_truncated, RGWAccessListFilter *filter)
 {
   int count = 0;
-  bool truncated;
+  bool truncated = true;
 
   if (bucket_is_system(bucket)) {
     return -EINVAL;
@@ -1753,7 +1753,16 @@ int RGWRados::list_objects(rgw_bucket& bucket, int max, string& prefix, string&
 
   string skip_after_delim;
 
-  do {
+  /* if marker points at a common prefix, fast forward it into its upperbound string */
+  if (!delim.empty()) {
+    int delim_pos = cur_marker.find(delim, prefix.size());
+    if (delim_pos >= 0) {
+      cur_marker = cur_marker.substr(0, delim_pos);
+      cur_marker.append(bigger_than_delim);
+    }
+  }
+
+  while (truncated && count < max) {
     if (skip_after_delim > cur_marker) {
       cur_marker = skip_after_delim;
       ldout(cct, 20) << "setting cur_marker=" << cur_marker << dendl;
@@ -1796,12 +1805,21 @@ int RGWRados::list_objects(rgw_bucket& bucket, int max, string& prefix, string&
         int delim_pos = obj.find(delim, prefix.size());
 
         if (delim_pos >= 0) {
-          common_prefixes[obj.substr(0, delim_pos + 1)] = true;
+          string prefix_key = obj.substr(0, delim_pos + 1);
+
+          if (common_prefixes.find(prefix_key) == common_prefixes.end()) {
+            if (next_marker) {
+              *next_marker = prefix_key;
+            }
+            common_prefixes[prefix_key] = true;
 
-          skip_after_delim = obj.substr(0, delim_pos);
-          skip_after_delim.append(bigger_than_delim);
+            skip_after_delim = obj.substr(0, delim_pos);
+            skip_after_delim.append(bigger_than_delim);
 
-          ldout(cct, 20) << "skip_after_delim=" << skip_after_delim << dendl;
+            ldout(cct, 20) << "skip_after_delim=" << skip_after_delim << dendl;
+
+            count++;
+          }
 
           continue;
         }
@@ -1813,7 +1831,7 @@ int RGWRados::list_objects(rgw_bucket& bucket, int max, string& prefix, string&
       result.push_back(ent);
       count++;
     }
-  } while (truncated && count < max);
+  }
 
 done:
   if (is_truncated)