From e51de26939f29bd507ab3dbe726d345c757ea6f6 Mon Sep 17 00:00:00 2001 From: Soumya Koduri Date: Fri, 15 Apr 2022 01:21:17 +0530 Subject: [PATCH] rgw/dbstore: Handle prefix/delim in Bucket::List op Given a prefix, fetch only those objects matching the prefix. In addition, skip the entries with "delim" and instead include those entries in common_prefixes Signed-off-by: Soumya Koduri --- src/rgw/store/dbstore/common/dbstore.cc | 73 +++++++++++++++----- src/rgw/store/dbstore/common/dbstore.h | 5 +- src/rgw/store/dbstore/sqlite/sqliteDB.cc | 3 + src/rgw/store/dbstore/tests/dbstore_tests.cc | 14 ++-- 4 files changed, 69 insertions(+), 26 deletions(-) diff --git a/src/rgw/store/dbstore/common/dbstore.cc b/src/rgw/store/dbstore/common/dbstore.cc index b42f9b3b081d9..dfb2d6490c4fb 100644 --- a/src/rgw/store/dbstore/common/dbstore.cc +++ b/src/rgw/store/dbstore/common/dbstore.cc @@ -723,12 +723,30 @@ out: return ret; } +/** + * Get ordered listing of the objects in a bucket. + * + * max_p: maximum number of results to return + * bucket: bucket to list contents of + * prefix: only return results that match this prefix + * delim: do not include results that match this string. + * Any skipped results will have the matching portion of their name + * inserted in common_prefixes with a "true" mark. + * marker: if filled in, begin the listing with this object. + * end_marker: if filled in, end the listing with this object. + * result: the objects are put in here. + * common_prefixes: if delim is filled in, any matching prefixes are + * placed here. + * is_truncated: if number of objects in the bucket is bigger than + * max, then truncated. + */ int DB::Bucket::List::list_objects(const DoutPrefixProvider *dpp, int64_t max, vector *result, map *common_prefixes, bool *is_truncated) { int ret = 0; DB *store = target->get_store(); + int64_t count = 0; DBOpParams db_params = {}; store->InitializeParams(dpp, &db_params); @@ -737,6 +755,7 @@ int DB::Bucket::List::list_objects(const DoutPrefixProvider *dpp, int64_t max, /* XXX: Handle whole marker? key -> name, instance, ns? */ db_params.op.obj.min_marker = params.marker.name; db_params.op.obj.max_marker = params.end_marker.name; + db_params.op.obj.prefix = params.prefix + "%"; db_params.op.list_max_count = max + 1; /* +1 for next_marker */ ret = store->ProcessOp(dpp, "ListBucketObjects", &db_params); @@ -746,19 +765,39 @@ int DB::Bucket::List::list_objects(const DoutPrefixProvider *dpp, int64_t max, goto out; } - if (db_params.op.obj.list_entries.size() >= (uint64_t)max) { - *is_truncated = true; - next_marker.name = db_params.op.obj.list_entries.back().key.name; - next_marker.instance = db_params.op.obj.list_entries.back().key.instance; - db_params.op.obj.list_entries.pop_back(); - } for (auto& entry : db_params.op.obj.list_entries) { + if (count >= max) { + *is_truncated = true; + next_marker.name = entry.key.name; + next_marker.instance = entry.key.instance; + break; + } + if (!params.delim.empty()) { + const std::string& objname = entry.key.name; + const int delim_pos = objname.find(params.delim, params.prefix.size()); + if (delim_pos >= 0) { + /* extract key -with trailing delimiter- for CommonPrefix */ + const std::string& prefix_key = + objname.substr(0, delim_pos + params.delim.length()); + + if (common_prefixes && + common_prefixes->find(prefix_key) == common_prefixes->end()) { + next_marker = prefix_key; + (*common_prefixes)[prefix_key] = true; + count++; + } + continue; + } + } + if (!params.end_marker.name.empty() && params.end_marker.name.compare(entry.key.name) <= 0) { + // should not include end_marker *is_truncated = false; break; } + count++; result->push_back(std::move(entry)); } out: @@ -1151,7 +1190,7 @@ int DB::raw_obj::write(const DoutPrefixProvider *dpp, int64_t ofs, int64_t write } int DB::Object::follow_olh(const DoutPrefixProvider *dpp, - const RGWBucketInfo& bucket_info, RGWObjState *state, + const RGWBucketInfo& bucket_info, RGWObjState* state, const rgw_obj& olh_obj, rgw_obj *target) { auto iter = state->attrset.find(RGW_ATTR_OLH_INFO); @@ -1202,7 +1241,7 @@ int DB::Object::get_olh_target_state(const DoutPrefixProvider *dpp, int DB::Object::get_obj_state(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw_obj& obj, - bool follow_olh, RGWObjState **state) + bool follow_olh, RGWObjState** state) { int ret = 0; @@ -1243,14 +1282,14 @@ out: } -int DB::Object::get_state(const DoutPrefixProvider *dpp, RGWObjState **pstate, bool follow_olh) +int DB::Object::get_state(const DoutPrefixProvider *dpp, RGWObjState** pstate, bool follow_olh) { return get_obj_state(dpp, bucket_info, obj, follow_olh, pstate); } int DB::Object::Read::get_attr(const DoutPrefixProvider *dpp, const char *name, bufferlist& dest) { - RGWObjState *state; + RGWObjState* state; int r = source->get_state(dpp, &state, true); if (r < 0) return r; @@ -1271,7 +1310,7 @@ int DB::Object::Read::prepare(const DoutPrefixProvider *dpp) map::iterator iter; - RGWObjState *astate; + RGWObjState* astate; /* XXX Read obj_id too */ int r = source->get_state(dpp, &astate, true); @@ -1359,7 +1398,7 @@ int DB::Object::Read::read(int64_t ofs, int64_t end, bufferlist& bl, const DoutP bufferlist read_bl; uint64_t max_chunk_size = store->get_max_chunk_size(); - RGWObjState *astate; + RGWObjState* astate; int r = source->get_state(dpp, &astate, true); if (r < 0) return r; @@ -1425,7 +1464,7 @@ int DB::Object::Read::read(int64_t ofs, int64_t end, bufferlist& bl, const DoutP static int _get_obj_iterate_cb(const DoutPrefixProvider *dpp, const DB::raw_obj& read_obj, off_t obj_ofs, off_t len, bool is_head_obj, - RGWObjState *astate, void *arg) + RGWObjState* astate, void *arg) { struct db_get_obj_data* d = static_cast(arg); return d->store->get_obj_iterate_cb(dpp, read_obj, obj_ofs, len, @@ -1435,7 +1474,7 @@ static int _get_obj_iterate_cb(const DoutPrefixProvider *dpp, int DB::get_obj_iterate_cb(const DoutPrefixProvider *dpp, const raw_obj& read_obj, off_t obj_ofs, off_t len, bool is_head_obj, - RGWObjState *astate, void *arg) + RGWObjState* astate, void *arg) { struct db_get_obj_data* d = static_cast(arg); bufferlist bl; @@ -1495,7 +1534,7 @@ int DB::Object::iterate_obj(const DoutPrefixProvider *dpp, { DB *store = get_store(); uint64_t len; - RGWObjState *astate; + RGWObjState* astate; int r = get_state(dpp, &astate, true); if (r < 0) { @@ -1613,7 +1652,7 @@ int DB::Object::Write::_do_write_meta(const DoutPrefixProvider *dpp, { DB *store = target->get_store(); - RGWObjState *state = &obj_state; + RGWObjState* state = &obj_state; map *attrset; DBOpParams params = {}; int ret = 0; @@ -1749,7 +1788,7 @@ int DB::Object::Write::write_meta(const DoutPrefixProvider *dpp, uint64_t size, int DB::Object::Delete::delete_obj(const DoutPrefixProvider *dpp) { int ret = 0; DB *store = target->get_store(); - RGWObjState *astate; + RGWObjState* astate; int r = target->get_state(dpp, &astate, true); if (r < 0) diff --git a/src/rgw/store/dbstore/common/dbstore.h b/src/rgw/store/dbstore/common/dbstore.h index da7c3a970adbf..0779323a5daab 100644 --- a/src/rgw/store/dbstore/common/dbstore.h +++ b/src/rgw/store/dbstore/common/dbstore.h @@ -86,6 +86,7 @@ struct DBOpObjectInfo { bufferlist head_data; std::string min_marker; std::string max_marker; + std::string prefix; std::list list_entries; /* XXX: Maybe use std::vector instead of std::list */ }; @@ -279,6 +280,7 @@ struct DBOpObjectPrepareInfo { static constexpr const char* head_data = ":head_data"; static constexpr const char* min_marker = ":min_marker"; static constexpr const char* max_marker = ":max_marker"; + static constexpr const char* prefix = ":prefix"; /* Below used to update mp_parts obj name * from meta object to src object on completion */ static constexpr const char* new_obj_name = ":new_obj_name"; @@ -1083,7 +1085,7 @@ class ListBucketObjectsOp: virtual public DBOp { ObjID, TailInstance, HeadPlacementRuleName, HeadPlacementRuleStorageClass, \ TailPlacementRuleName, TailPlacementStorageClass, \ ManifestPartObjs, ManifestPartRules, Omap, IsMultipart, MPPartsList, HeadData from '{}' \ - where BucketName = {} and ObjName > {} ORDER BY ObjName ASC LIMIT {}"; + where BucketName = {} and ObjName >= {} and ObjName LIKE {} ORDER BY ObjName ASC LIMIT {}"; public: virtual ~ListBucketObjectsOp() {} @@ -1093,6 +1095,7 @@ class ListBucketObjectsOp: virtual public DBOp { params.object_table, params.op.bucket.bucket_name, params.op.obj.min_marker, + params.op.obj.prefix, params.op.list_max_count); } }; diff --git a/src/rgw/store/dbstore/sqlite/sqliteDB.cc b/src/rgw/store/dbstore/sqlite/sqliteDB.cc index 8266614ad253d..347958abf16cd 100644 --- a/src/rgw/store/dbstore/sqlite/sqliteDB.cc +++ b/src/rgw/store/dbstore/sqlite/sqliteDB.cc @@ -2325,6 +2325,9 @@ int SQLListBucketObjects::Bind(const DoutPrefixProvider *dpp, struct DBOpParams SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.min_marker, sdb); SQL_BIND_TEXT(dpp, stmt, index, params->op.obj.min_marker.c_str(), sdb); + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.prefix, sdb); + SQL_BIND_TEXT(dpp, stmt, index, params->op.obj.prefix.c_str(), sdb); + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.list_max_count, sdb); SQL_BIND_INT(dpp, stmt, index, params->op.list_max_count, sdb); diff --git a/src/rgw/store/dbstore/tests/dbstore_tests.cc b/src/rgw/store/dbstore/tests/dbstore_tests.cc index b21be929f11e3..10f7274984810 100644 --- a/src/rgw/store/dbstore/tests/dbstore_tests.cc +++ b/src/rgw/store/dbstore/tests/dbstore_tests.cc @@ -687,8 +687,7 @@ TEST_F(DBStoreTest, GetObject) { TEST_F(DBStoreTest, GetObjectState) { struct DBOpParams params = GlobalParams; int ret = -1; - RGWObjState state; - RGWObjState *s = &state; + RGWObjState* s; params.op.obj.state.obj.key.name = "object2"; params.op.obj.state.obj.key.instance = "inst2"; @@ -698,14 +697,14 @@ TEST_F(DBStoreTest, GetObjectState) { ret = op_target.get_obj_state(dpp, params.op.bucket.info, params.op.obj.state.obj, false, &s); ASSERT_EQ(ret, 0); - ASSERT_EQ(state.size, 12); - ASSERT_EQ(state.is_olh, false); + ASSERT_EQ(s->size, 12); + ASSERT_EQ(s->is_olh, false); /* Recheck with get_state API */ ret = op_target.get_state(dpp, &s, false); ASSERT_EQ(ret, 0); - ASSERT_EQ(state.size, 12); - ASSERT_EQ(state.is_olh, false); + ASSERT_EQ(s->size, 12); + ASSERT_EQ(s->is_olh, false); } TEST_F(DBStoreTest, ObjAttrs) { @@ -880,8 +879,7 @@ TEST_F(DBStoreTest, ListBucketObjects) { TEST_F(DBStoreTest, DeleteObj) { struct DBOpParams params = GlobalParams; int ret = -1; - RGWObjState state; - RGWObjState *s = &state; + RGWObjState *s; /* delete object2 */ params.op.obj.state.obj.key.name = "object2"; -- 2.39.5