rgw/dbstore: Handle prefix/delim in Bucket::List op

Given a prefix, fetch only those objects matching the prefix.
In addition, skip the entries with "delim" and instead include
those entries in common_prefixes

Signed-off-by: Soumya Koduri <skoduri@redhat.com>

diff --git a/src/rgw/store/dbstore/common/dbstore.cc b/src/rgw/store/dbstore/common/dbstore.cc
index b42f9b3b081d900950565ad81d133e9a86504a80..dfb2d6490c4fba4be75f5cba009b53c3e8e9691b 100644 (file)
--- a/src/rgw/store/dbstore/common/dbstore.cc
+++ b/src/rgw/store/dbstore/common/dbstore.cc
@@ -723,12 +723,30 @@ out:
   return ret;
 }
 
+/**
+ * Get ordered listing of the objects in a bucket.
+ *
+ * max_p: maximum number of results to return
+ * bucket: bucket to list contents of
+ * prefix: only return results that match this prefix
+ * delim: do not include results that match this string.
+ *     Any skipped results will have the matching portion of their name
+ *     inserted in common_prefixes with a "true" mark.
+ * marker: if filled in, begin the listing with this object.
+ * end_marker: if filled in, end the listing with this object.
+ * result: the objects are put in here.
+ * common_prefixes: if delim is filled in, any matching prefixes are
+ * placed here.
+ * is_truncated: if number of objects in the bucket is bigger than
+ * max, then truncated.
+ */
 int DB::Bucket::List::list_objects(const DoutPrefixProvider *dpp, int64_t max,
                           vector<rgw_bucket_dir_entry> *result,
                           map<string, bool> *common_prefixes, bool *is_truncated)
 {
   int ret = 0;
   DB *store = target->get_store();
+  int64_t count = 0;
 
   DBOpParams db_params = {};
   store->InitializeParams(dpp, &db_params);
@@ -737,6 +755,7 @@ int DB::Bucket::List::list_objects(const DoutPrefixProvider *dpp, int64_t max,
   /* XXX: Handle whole marker? key -> name, instance, ns? */
   db_params.op.obj.min_marker = params.marker.name;
   db_params.op.obj.max_marker = params.end_marker.name;
+  db_params.op.obj.prefix = params.prefix + "%";
   db_params.op.list_max_count = max + 1; /* +1 for next_marker */
 
   ret = store->ProcessOp(dpp, "ListBucketObjects", &db_params);
@@ -746,19 +765,39 @@ int DB::Bucket::List::list_objects(const DoutPrefixProvider *dpp, int64_t max,
     goto out;
   }
 
-  if (db_params.op.obj.list_entries.size() >= (uint64_t)max) {
-    *is_truncated = true;
-    next_marker.name = db_params.op.obj.list_entries.back().key.name;
-    next_marker.instance = db_params.op.obj.list_entries.back().key.instance;
-    db_params.op.obj.list_entries.pop_back();
-  }
 
   for (auto& entry : db_params.op.obj.list_entries) {
+    if (count >= max) {
+      *is_truncated = true;
+      next_marker.name = entry.key.name;
+      next_marker.instance = entry.key.instance;
+      break;
+    }
+    if (!params.delim.empty()) {
+    const std::string& objname = entry.key.name;
+       const int delim_pos = objname.find(params.delim, params.prefix.size());
+         if (delim_pos >= 0) {
+           /* extract key -with trailing delimiter- for CommonPrefix */
+           const std::string& prefix_key =
+             objname.substr(0, delim_pos + params.delim.length());
+
+           if (common_prefixes &&
+               common_prefixes->find(prefix_key) == common_prefixes->end()) {
+          next_marker = prefix_key;
+          (*common_prefixes)[prefix_key] = true;
+          count++;
+        }
+        continue;
+      }
+    }
+
     if (!params.end_marker.name.empty() &&
         params.end_marker.name.compare(entry.key.name) <= 0) {
+      // should not include end_marker
       *is_truncated = false;
       break;
     }
+    count++;
     result->push_back(std::move(entry));
   }
 out:
@@ -1151,7 +1190,7 @@ int DB::raw_obj::write(const DoutPrefixProvider *dpp, int64_t ofs, int64_t write
 }
 
 int DB::Object::follow_olh(const DoutPrefixProvider *dpp,
-                           const RGWBucketInfo& bucket_info, RGWObjState *state,
+                           const RGWBucketInfo& bucket_info, RGWObjState* state,
                            const rgw_obj& olh_obj, rgw_obj *target)
 {
   auto iter = state->attrset.find(RGW_ATTR_OLH_INFO);
@@ -1202,7 +1241,7 @@ int DB::Object::get_olh_target_state(const DoutPrefixProvider *dpp,
 
 int DB::Object::get_obj_state(const DoutPrefixProvider *dpp,
                               const RGWBucketInfo& bucket_info, const rgw_obj& obj,
-                              bool follow_olh, RGWObjState **state)
+                              bool follow_olh, RGWObjState** state)
 {
   int ret = 0;
 
@@ -1243,14 +1282,14 @@ out:
 
 }
 
-int DB::Object::get_state(const DoutPrefixProvider *dpp, RGWObjState **pstate, bool follow_olh)
+int DB::Object::get_state(const DoutPrefixProvider *dpp, RGWObjState** pstate, bool follow_olh)
 {
   return get_obj_state(dpp, bucket_info, obj, follow_olh, pstate);
 }
 
 int DB::Object::Read::get_attr(const DoutPrefixProvider *dpp, const char *name, bufferlist& dest)
 {
-  RGWObjState *state;
+  RGWObjState* state;
   int r = source->get_state(dpp, &state, true);
   if (r < 0)
     return r;
@@ -1271,7 +1310,7 @@ int DB::Object::Read::prepare(const DoutPrefixProvider *dpp)
 
   map<string, bufferlist>::iterator iter;
 
-  RGWObjState *astate;
+  RGWObjState* astate;
 
   /* XXX Read obj_id too */
   int r = source->get_state(dpp, &astate, true);
@@ -1359,7 +1398,7 @@ int DB::Object::Read::read(int64_t ofs, int64_t end, bufferlist& bl, const DoutP
   bufferlist read_bl;
   uint64_t max_chunk_size = store->get_max_chunk_size();
 
-  RGWObjState *astate;
+  RGWObjState* astate;
   int r = source->get_state(dpp, &astate, true);
   if (r < 0)
     return r;
@@ -1425,7 +1464,7 @@ int DB::Object::Read::read(int64_t ofs, int64_t end, bufferlist& bl, const DoutP
 static int _get_obj_iterate_cb(const DoutPrefixProvider *dpp,
     const DB::raw_obj& read_obj, off_t obj_ofs,
     off_t len, bool is_head_obj,
-    RGWObjState *astate, void *arg)
+    RGWObjState* astate, void *arg)
 {
   struct db_get_obj_data* d = static_cast<struct db_get_obj_data*>(arg);
   return d->store->get_obj_iterate_cb(dpp, read_obj, obj_ofs, len,
@@ -1435,7 +1474,7 @@ static int _get_obj_iterate_cb(const DoutPrefixProvider *dpp,
 int DB::get_obj_iterate_cb(const DoutPrefixProvider *dpp,
     const raw_obj& read_obj, off_t obj_ofs,
     off_t len, bool is_head_obj,
-    RGWObjState *astate, void *arg)
+    RGWObjState* astate, void *arg)
 {
   struct db_get_obj_data* d = static_cast<struct db_get_obj_data*>(arg);
   bufferlist bl;
@@ -1495,7 +1534,7 @@ int DB::Object::iterate_obj(const DoutPrefixProvider *dpp,
 {
   DB *store = get_store();
   uint64_t len;
-  RGWObjState *astate;
+  RGWObjState* astate;
 
   int r = get_state(dpp, &astate, true);
   if (r < 0) {
@@ -1613,7 +1652,7 @@ int DB::Object::Write::_do_write_meta(const DoutPrefixProvider *dpp,
 {
   DB *store = target->get_store();
 
-  RGWObjState *state = &obj_state;
+  RGWObjState* state = &obj_state;
   map<string, bufferlist> *attrset;
   DBOpParams params = {};
   int ret = 0;
@@ -1749,7 +1788,7 @@ int DB::Object::Write::write_meta(const DoutPrefixProvider *dpp, uint64_t size,
 int DB::Object::Delete::delete_obj(const DoutPrefixProvider *dpp) {
   int ret = 0;
   DB *store = target->get_store();
-  RGWObjState *astate;
+  RGWObjState* astate;
 
   int r = target->get_state(dpp, &astate, true);
   if (r < 0)

diff --git a/src/rgw/store/dbstore/common/dbstore.h b/src/rgw/store/dbstore/common/dbstore.h
index da7c3a970adbfaf816f1ae139079485bde83fc34..0779323a5daab53127569313536644a4b22785ab 100644 (file)
--- a/src/rgw/store/dbstore/common/dbstore.h
+++ b/src/rgw/store/dbstore/common/dbstore.h
@@ -86,6 +86,7 @@ struct DBOpObjectInfo {
   bufferlist head_data;
   std::string min_marker;
   std::string max_marker;
+  std::string prefix;
   std::list<rgw_bucket_dir_entry> list_entries;
   /* XXX: Maybe use std::vector instead of std::list */
 };
@@ -279,6 +280,7 @@ struct DBOpObjectPrepareInfo {
   static constexpr const char* head_data = ":head_data";
   static constexpr const char* min_marker = ":min_marker";
   static constexpr const char* max_marker = ":max_marker";
+  static constexpr const char* prefix = ":prefix";
   /* Below used to update mp_parts obj name
    * from meta object to src object on completion */
   static constexpr const char* new_obj_name = ":new_obj_name";
@@ -1083,7 +1085,7 @@ class ListBucketObjectsOp: virtual public DBOp {
       ObjID, TailInstance, HeadPlacementRuleName, HeadPlacementRuleStorageClass, \
       TailPlacementRuleName, TailPlacementStorageClass, \
       ManifestPartObjs, ManifestPartRules, Omap, IsMultipart, MPPartsList, HeadData from '{}' \
-      where BucketName = {} and ObjName > {} ORDER BY ObjName ASC LIMIT {}";
+      where BucketName = {} and ObjName >= {} and ObjName LIKE {} ORDER BY ObjName ASC LIMIT {}";
   public:
     virtual ~ListBucketObjectsOp() {}
 
@@ -1093,6 +1095,7 @@ class ListBucketObjectsOp: virtual public DBOp {
           params.object_table,
           params.op.bucket.bucket_name,
           params.op.obj.min_marker,
+          params.op.obj.prefix,
           params.op.list_max_count);
     }
 };

diff --git a/src/rgw/store/dbstore/sqlite/sqliteDB.cc b/src/rgw/store/dbstore/sqlite/sqliteDB.cc
index 8266614ad253dc6bdd0becc8ec0e623438db02f0..347958abf16cd1996122f18ca19bed0ec1f6a244 100644 (file)
--- a/src/rgw/store/dbstore/sqlite/sqliteDB.cc
+++ b/src/rgw/store/dbstore/sqlite/sqliteDB.cc
@@ -2325,6 +2325,9 @@ int SQLListBucketObjects::Bind(const DoutPrefixProvider *dpp, struct DBOpParams
   SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.min_marker, sdb);
   SQL_BIND_TEXT(dpp, stmt, index, params->op.obj.min_marker.c_str(), sdb);
 
+  SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.prefix, sdb);
+  SQL_BIND_TEXT(dpp, stmt, index, params->op.obj.prefix.c_str(), sdb);
+
   SQL_BIND_INDEX(dpp, stmt, index, p_params.op.list_max_count, sdb);
   SQL_BIND_INT(dpp, stmt, index, params->op.list_max_count, sdb);
 

diff --git a/src/rgw/store/dbstore/tests/dbstore_tests.cc b/src/rgw/store/dbstore/tests/dbstore_tests.cc
index b21be929f11e3213c290ae1d3add92c0b60f61d9..10f7274984810e9992bfe5d026d34487ed7df6e3 100644 (file)
--- a/src/rgw/store/dbstore/tests/dbstore_tests.cc
+++ b/src/rgw/store/dbstore/tests/dbstore_tests.cc
@@ -687,8 +687,7 @@ TEST_F(DBStoreTest, GetObject) {
 TEST_F(DBStoreTest, GetObjectState) {
   struct DBOpParams params = GlobalParams;
   int ret = -1;
-  RGWObjState state;
-  RGWObjState *s = &state;
+  RGWObjState* s;
 
   params.op.obj.state.obj.key.name = "object2";
   params.op.obj.state.obj.key.instance = "inst2";
@@ -698,14 +697,14 @@ TEST_F(DBStoreTest, GetObjectState) {
   ret = op_target.get_obj_state(dpp, params.op.bucket.info, params.op.obj.state.obj,
       false, &s);
   ASSERT_EQ(ret, 0);
-  ASSERT_EQ(state.size, 12);
-  ASSERT_EQ(state.is_olh, false);
+  ASSERT_EQ(s->size, 12);
+  ASSERT_EQ(s->is_olh, false);
 
   /* Recheck with get_state API */
   ret = op_target.get_state(dpp, &s, false);
   ASSERT_EQ(ret, 0);
-  ASSERT_EQ(state.size, 12);
-  ASSERT_EQ(state.is_olh, false);
+  ASSERT_EQ(s->size, 12);
+  ASSERT_EQ(s->is_olh, false);
 }
 
 TEST_F(DBStoreTest, ObjAttrs) {
@@ -880,8 +879,7 @@ TEST_F(DBStoreTest, ListBucketObjects) {
 TEST_F(DBStoreTest, DeleteObj) {
   struct DBOpParams params = GlobalParams;
   int ret = -1;
-  RGWObjState state;
-  RGWObjState *s = &state;
+  RGWObjState *s;
 
   /* delete object2 */
   params.op.obj.state.obj.key.name = "object2";