From: Tianshan Qu <tianshan@xsky.com>
Date: Tue, 21 May 2019 06:15:51 +0000 (+0800)
Subject: rgw: fix bucket may redundantly list keys after BI_PREFIX_CHAR
X-Git-Tag: v15.1.0~2574^2
X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=dc3ffe99b910d04b68d70a1af2f38d877e05cd25;p=ceph-ci.git

rgw: fix bucket may redundantly list keys after BI_PREFIX_CHAR

Fixes: http://tracker.ceph.com/issues/39984

Signed-off-by: Tianshan Qu <tianshan@xsky.com>
---

diff --git a/src/cls/rgw/cls_rgw.cc b/src/cls/rgw/cls_rgw.cc
index ed68d2bc5ca..6c19d9fb13a 100644
--- a/src/cls/rgw/cls_rgw.cc
+++ b/src/cls/rgw/cls_rgw.cc
@@ -173,9 +173,13 @@ static int get_obj_vals(cls_method_context_t hctx, const string& start, const st
   auto upper = std::lower_bound(lower, pkeys->end(), new_start, comp);
   pkeys->erase(lower, upper);
 
-  if (num_entries == (int)pkeys->size())
+  if (num_entries == (int)pkeys->size() || !(*pmore))
     return 0;
 
+  if (pkeys->rbegin()->first > new_start) {
+    new_start = pkeys->rbegin()->first;
+  }
+
   map<string, bufferlist> new_keys;
 
   /* now get some more keys */
diff --git a/src/test/cls_rgw/test_cls_rgw.cc b/src/test/cls_rgw/test_cls_rgw.cc
index 15e531809db..e5cee78960a 100644
--- a/src/test/cls_rgw/test_cls_rgw.cc
+++ b/src/test/cls_rgw/test_cls_rgw.cc
@@ -87,17 +87,17 @@ void test_stats(librados::IoCtx& ioctx, string& oid, RGWObjCategory category, ui
 }
 
 void index_prepare(OpMgr& mgr, librados::IoCtx& ioctx, string& oid, RGWModifyOp index_op, string& tag,
-		   string& obj, string& loc, uint16_t bi_flags = 0)
+		   string& obj, string& loc, uint16_t bi_flags = 0, bool log_op = true)
 {
   ObjectWriteOperation *op = mgr.write_op();
   cls_rgw_obj_key key(obj, string());
   rgw_zone_set zones_trace;
-  cls_rgw_bucket_prepare_op(*op, index_op, tag, key, loc, true, bi_flags, zones_trace);
+  cls_rgw_bucket_prepare_op(*op, index_op, tag, key, loc, log_op, bi_flags, zones_trace);
   ASSERT_EQ(0, ioctx.operate(oid, op));
 }
 
 void index_complete(OpMgr& mgr, librados::IoCtx& ioctx, string& oid, RGWModifyOp index_op, string& tag,
-		    int epoch, string& obj, rgw_bucket_dir_entry_meta& meta, uint16_t bi_flags = 0)
+		    int epoch, string& obj, rgw_bucket_dir_entry_meta& meta, uint16_t bi_flags = 0, bool log_op = true)
 {
   ObjectWriteOperation *op = mgr.write_op();
   cls_rgw_obj_key key(obj, string());
@@ -105,7 +105,7 @@ void index_complete(OpMgr& mgr, librados::IoCtx& ioctx, string& oid, RGWModifyOp
   ver.pool = ioctx.get_id();
   ver.epoch = epoch;
   meta.accounted_size = meta.size;
-  cls_rgw_bucket_complete_op(*op, index_op, tag, ver, key, meta, nullptr, true, bi_flags, nullptr);
+  cls_rgw_bucket_complete_op(*op, index_op, tag, ver, key, meta, nullptr, log_op, bi_flags, nullptr);
   ASSERT_EQ(0, ioctx.operate(oid, op));
 }
 
@@ -398,7 +398,7 @@ TEST(cls_rgw, index_list)
 
   uint64_t epoch = 1;
   uint64_t obj_size = 1024;
-  const int num_objs = 5;
+  const int num_objs = 4;
   const string keys[num_objs] = {
     /* single byte utf8 character */
     { static_cast<char>(0x41) },
@@ -408,8 +408,6 @@ TEST(cls_rgw, index_list)
     { static_cast<char>(0xDF), static_cast<char>(0x8F), static_cast<char>(0x8F) },
     /* quadruble byte utf8 character */
     { static_cast<char>(0xF7), static_cast<char>(0x8F), static_cast<char>(0x8F), static_cast<char>(0x8F) },
-    /* BI_PREFIX_CHAR private namespace, for test only */
-    { static_cast<char>(0x80), static_cast<char>(0x41) }
   };
 
   for (int i = 0; i < num_objs; i++) {
@@ -417,14 +415,24 @@ TEST(cls_rgw, index_list)
     string tag = str_int("tag", i);
     string loc = str_int("loc", i);
 
-    index_prepare(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, obj, loc);
+    index_prepare(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, obj, loc, 0 /* bi_falgs */, false /* log_op */);
 
     op = mgr.write_op();
     rgw_bucket_dir_entry_meta meta;
     meta.category = RGWObjCategory::None;
     meta.size = obj_size;
-    index_complete(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, epoch, obj, meta);
+    index_complete(mgr, ioctx, bucket_oid, CLS_RGW_OP_ADD, tag, epoch, obj, meta, 0 /* bi_falgs */, false /* log_op */);
+  }
+
+  map<string, bufferlist> entries;
+  /* insert 998 omap key starts with BI_PREFIX_CHAR,
+   * so bucket list first time will get one key before 0x80 and one key after */
+  for (int i = 0; i < 998; ++i) {
+    char buf[10];
+    snprintf(buf, sizeof(buf), "%c%s%d", 0x80, "1000_", i);
+    entries.emplace(string{buf}, bufferlist{});
   }
+  ioctx.omap_set(bucket_oid, entries);
 
   test_stats(ioctx, bucket_oid, RGWObjCategory::None, num_objs, obj_size * num_objs);