rgw: fix bucket may redundantly list keys after BI_PREFIX_CHAR

Fixes: http://tracker.ceph.com/issues/39984
Signed-off-by: Tianshan Qu <tianshan@xsky.com>

diff --git a/src/cls/rgw/cls_rgw.cc b/src/cls/rgw/cls_rgw.cc
index ed68d2bc5ca342e445c524646dda5bfffb949246..6c19d9fb13a6a95933c4b97981c6f08a8358c64d 100644 (file)
--- a/src/cls/rgw/cls_rgw.cc
+++ b/src/cls/rgw/cls_rgw.cc
@@ -173,9 +173,13 @@ static int get_obj_vals(cls_method_context_t hctx, const string& start, const st
   auto upper = std::lower_bound(lower, pkeys->end(), new_start, comp);
   pkeys->erase(lower, upper);
 
-  if (num_entries == (int)pkeys->size())
+  if (num_entries == (int)pkeys->size() || !(*pmore))
     return 0;
 
+  if (pkeys->rbegin()->first > new_start) {
+    new_start = pkeys->rbegin()->first;
+  }
+
   map<string, bufferlist> new_keys;
 
   /* now get some more keys */

diff --git a/src/test/cls_rgw/test_cls_rgw.cc b/src/test/cls_rgw/test_cls_rgw.cc
index 15e531809dbd4cd3292d6e986716329c4e9864ce..e5cee78960a655059e1b44ea7c71274cf224b25c 100644 (file)
--- a/src/test/cls_rgw/test_cls_rgw.cc
+++ b/src/test/cls_rgw/test_cls_rgw.cc
@@ -87,17 +87,17 @@ void test_stats(librados::IoCtx& ioctx, string& oid, RGWObjCategory category, ui
 }
 
 void index_prepare(OpMgr& mgr, librados::IoCtx& ioctx, string& oid, RGWModifyOp index_op, string& tag,
-                  string& obj, string& loc, uint16_t bi_flags = 0)
+                  string& obj, string& loc, uint16_t bi_flags = 0, bool log_op = true)
 {
   ObjectWriteOperation *op = mgr.write_op();
   cls_rgw_obj_key key(obj, string());
   rgw_zone_set zones_trace;
-  cls_rgw_bucket_prepare_op(*op, index_op, tag, key, loc, true, bi_flags, zones_trace);
+  cls_rgw_bucket_prepare_op(*op, index_op, tag, key, loc, log_op, bi_flags, zones_trace);
   ASSERT_EQ(0, ioctx.operate(oid, op));
 }
 
 void index_complete(OpMgr& mgr, librados::IoCtx& ioctx, string& oid, RGWModifyOp index_op, string& tag,
-                   int epoch, string& obj, rgw_bucket_dir_entry_meta& meta, uint16_t bi_flags = 0)
+                   int epoch, string& obj, rgw_bucket_dir_entry_meta& meta, uint16_t bi_flags = 0, bool log_op = true)
 {
   ObjectWriteOperation *op = mgr.write_op();
   cls_rgw_obj_key key(obj, string());
@@ -105,7 +105,7 @@ void index_complete(OpMgr& mgr, librados::IoCtx& ioctx, string& oid, RGWModifyOp
   ver.pool = ioctx.get_id();
   ver.epoch = epoch;
   meta.accounted_size = meta.size;
-  cls_rgw_bucket_complete_op(*op, index_op, tag, ver, key, meta, nullptr, true, bi_flags, nullptr);
+  cls_rgw_bucket_complete_op(*op, index_op, tag, ver, key, meta, nullptr, log_op, bi_flags, nullptr);
   ASSERT_EQ(0, ioctx.operate(oid, op));
 }
 
@@ -398,7 +398,7 @@ TEST(cls_rgw, index_list)
 
   uint64_t epoch = 1;
   uint64_t obj_size = 1024;
-  const int num_objs = 5;
+  const int num_objs = 4;
   const string keys[num_objs] = {
     /* single byte utf8 character */
     { static_cast<char>(0x41) },
@@ -408,8 +408,6 @@ TEST(cls_rgw, index_list)
     { static_cast<char>(0xDF), static_cast<char>(0x8F), static_cast<char>(0x8F) },
     /* quadruble byte utf8 character */
     { static_cast<char>(0xF7), static_cast<char>(0x8F), static_cast<char>(0x8F), static_cast<char>(0x8F) },
-    /* BI_PREFIX_CHAR private namespace, for test only */
-    { static_cast<char>(0x80), static_cast<char>(0x41) }
   };
 
   for (int i = 0; i < num_objs; i++) {
@@ -417,14 +415,24 @@ TEST(cls_rgw, index_list)
     string tag = str_int("tag", i);
     string loc = str_int("loc", i);
 
-    index_prepare(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, obj, loc);
+    index_prepare(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, obj, loc, 0 /* bi_falgs */, false /* log_op */);
 
     op = mgr.write_op();
     rgw_bucket_dir_entry_meta meta;
     meta.category = RGWObjCategory::None;
     meta.size = obj_size;
-    index_complete(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, epoch, obj, meta);
+    index_complete(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, epoch, obj, meta, 0 /* bi_falgs */, false /* log_op */);
+  }
+
+  map<string, bufferlist> entries;
+  /* insert 998 omap key starts with BI_PREFIX_CHAR,
+   * so bucket list first time will get one key before 0x80 and one key after */
+  for (int i = 0; i < 998; ++i) {
+    char buf[10];
+    snprintf(buf, sizeof(buf), "%c%s%d", 0x80, "1000_", i);
+    entries.emplace(string{buf}, bufferlist{});
   }
+  ioctx.omap_set(bucket_oid, entries);
 
   test_stats(ioctx, bucket_oid, RGWObjCategory::None, num_objs, obj_size * num_objs);