From 65ca2f53b80416ba3070293b294048c16efcda9a Mon Sep 17 00:00:00 2001 From: Soumya Koduri Date: Fri, 4 Feb 2022 00:25:22 +0530 Subject: [PATCH] rgw/dbstore: Handle read vs delete races Now that tail objects are associated with objectID, they are not deleted as part of this DeleteObj operation. Such tail objects (with no head object in *.object.table are cleaned up later by GC thread. To avoid races between writes/reads & GC delete, mtime is maintained for each tail object. This mtime is updated when tail object is written and also when its corresponding head object is deleted (like here in this case). Also done some cleanup. Signed-off-by: Soumya Koduri --- src/rgw/store/dbstore/common/dbstore.cc | 60 ++--- src/rgw/store/dbstore/common/dbstore.h | 86 +++--- src/rgw/store/dbstore/sqlite/sqliteDB.cc | 262 +++++++++---------- src/rgw/store/dbstore/sqlite/sqliteDB.h | 9 +- src/rgw/store/dbstore/tests/dbstore_tests.cc | 14 +- 5 files changed, 209 insertions(+), 222 deletions(-) diff --git a/src/rgw/store/dbstore/common/dbstore.cc b/src/rgw/store/dbstore/common/dbstore.cc index e74f278d5a615..6ea53bb925b80 100644 --- a/src/rgw/store/dbstore/common/dbstore.cc +++ b/src/rgw/store/dbstore/common/dbstore.cc @@ -229,6 +229,7 @@ int DB::InitializeParams(const DoutPrefixProvider *dpp, string Op, DBOpParams *p //reset params here params->user_table = user_table; params->bucket_table = bucket_table; + params->quota_table = quota_table; params->lc_entry_table = lc_entry_table; params->lc_head_table = lc_head_table; @@ -772,8 +773,6 @@ int DB::raw_obj::InitializeParamsfromRawObj(const DoutPrefixProvider *dpp, if (!params) return -1; - params->object_table = obj_table; - params->objectdata_table = obj_data_table; params->op.bucket.info.bucket.name = bucket_name; params->op.obj.state.obj.key.name = obj_name; params->op.obj.state.obj.key.instance = obj_instance; @@ -800,8 +799,6 @@ int DB::Object::InitializeParamsfromObject(const DoutPrefixProvider *dpp, if (!params) return -1; - params->object_table = store->getObjectTable(bucket); - params->objectdata_table = store->getObjectDataTable(bucket); params->op.bucket.info.bucket.name = bucket; params->op.obj.state.obj = obj; params->op.obj.obj_id = obj_id; @@ -1736,7 +1733,6 @@ int DB::Object::Write::_do_write_meta(const DoutPrefixProvider *dpp, *meta.mtime = meta.set_mtime; } - /* XXX: handle multipart */ params.op.query_str = "meta"; params.op.obj.obj_id = target->obj_id; ret = store->ProcessOp(dpp, "PutObject", ¶ms); @@ -1768,27 +1764,6 @@ int DB::Object::Write::write_meta(const DoutPrefixProvider *dpp, uint64_t size, return r; } -int DB::Object::Write::update_mp_parts(const DoutPrefixProvider *dpp, rgw_obj_key new_obj_key) -{ - int ret = 0; - DBOpParams params = {}; - DB *store = target->get_store(); - - store->InitializeParams(dpp, "UpdateObjectData", ¶ms); - target->InitializeParamsfromObject(dpp, ¶ms); - - params.op.obj.new_obj_key = new_obj_key; - - ret = store->ProcessOp(dpp, "UpdateObjectData", ¶ms); - - if (ret) { - ldpp_dout(dpp, 0)<<"In UpdateObjectData failed err:(" <get_store(); @@ -1811,15 +1786,31 @@ int DB::Object::Delete::delete_obj(const DoutPrefixProvider *dpp) { store->InitializeParams(dpp, "DeleteObject", &del_params); target->InitializeParamsfromObject(dpp, &del_params); - /* As it is cascade delete, it will delete the objectdata table entries also */ ret = store->ProcessOp(dpp, "DeleteObject", &del_params); if (ret) { ldpp_dout(dpp, 0) << "In DeleteObject failed err:(" <shadow_obj; // objectID is copied here in get_state() + update_params.op.obj.state.mtime = real_clock::now(); + ret = store->ProcessOp(dpp, "UpdateObjectData", &update_params); + + if (ret) { + ldpp_dout(dpp, 0) << "Updating tail objects mtime failed err:(" <list_buckets(dpp, "all", user, marker, string(), max, false, &buckets, &is_truncated); @@ -2059,12 +2051,12 @@ void *DB::GC::entry() { user_marker = user.id; /* XXX: If using locks, unlock here and reacquire in the next iteration */ - std::this_thread::sleep_for(std::chrono::milliseconds(10)); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); } } while(is_truncated); bucket_marker.clear(); - std::this_thread::sleep_for(std::chrono::milliseconds(gc_interval)); + std::this_thread::sleep_for(std::chrono::milliseconds(gc_interval*10)); } while(1); diff --git a/src/rgw/store/dbstore/common/dbstore.h b/src/rgw/store/dbstore/common/dbstore.h index 2a0e0bc273637..8f9cbb314d288 100644 --- a/src/rgw/store/dbstore/common/dbstore.h +++ b/src/rgw/store/dbstore/common/dbstore.h @@ -87,9 +87,7 @@ struct DBOpObjectInfo { std::string min_marker; std::string max_marker; std::list list_entries; - /* Below used to update mp_parts obj name - * from meta object to src object on completion */ - rgw_obj_key new_obj_key; + /* XXX: Maybe use std::vector instead of std::list */ }; struct DBOpObjectDataInfo { @@ -142,8 +140,8 @@ struct DBOpParams { /* Ops*/ DBOpInfo op; - /* Below are subject to change */ std::string objectdata_table; + std::string object_view; std::string quota_table; std::string lc_head_table; std::string lc_entry_table; @@ -331,8 +329,8 @@ struct DBOpPrepareParams { DBOpPrepareInfo op; - /* below subject to change */ std::string objectdata_table; + std::string object_view; std::string quota_table; std::string lc_head_table; std::string lc_entry_table; @@ -550,7 +548,7 @@ class DBOp { ObjAttrs BLOB, \ HeadSize INTEGER, \ MaxHeadSize INTEGER, \ - ObjID TEXT, \ + ObjID TEXT NOT NULL, \ TailInstance TEXT, \ HeadPlacementRuleName TEXT, \ HeadPlacementRuleStorageClass TEXT, \ @@ -581,7 +579,7 @@ class DBOp { ObjInstance TEXT, \ ObjNS TEXT, \ BucketName TEXT NOT NULL , \ - ObjID String, \ + ObjID TEXT NOT NULL , \ MultipartPartStr TEXT, \ PartNum INTEGER NOT NULL, \ Offset INTEGER, \ @@ -592,6 +590,22 @@ class DBOp { FOREIGN KEY (BucketName) \ REFERENCES '{}' (BucketName) ON DELETE CASCADE ON UPDATE CASCADE \n);"; + const std::string CreateObjectViewQ = + /* This query creats temporary view with entries from ObjectData table which have + * corresponding head object (i.e, with same ObjName, ObjInstance, ObjNS, ObjID) + * in the Object table. + * + * GC thread can use this view to delete stale entries from the ObjectData table which + * do not exist in this view. + * + * XXX: This view is throwing ForeignKey mismatch error, mostly may be because all the keys + * of objectdata table are not referenced here. So this view is not used atm. + */ + "CREATE TEMP VIEW IF NOT EXISTS '{}' AS \ + SELECT s.ObjName, s.ObjInstance, s.ObjID from '{}' as s INNER JOIN '{}' USING \ + (ObjName, BucketName, ObjInstance, ObjID);"; + + const std::string CreateQuotaTableQ = "CREATE TABLE IF NOT EXISTS '{}' ( \ QuotaID INTEGER PRIMARY KEY AUTOINCREMENT UNIQUE , \ @@ -643,6 +657,11 @@ class DBOp { return fmt::format(CreateObjectDataTableQ.c_str(), params->objectdata_table.c_str(), params->bucket_table.c_str()); + if (!type.compare("ObjectView")) + return fmt::format(CreateObjectTableQ.c_str(), + params->object_view.c_str(), + params->objectdata_table.c_str(), + params->object_table.c_str()); if (!type.compare("Quota")) return fmt::format(CreateQuotaTableQ.c_str(), params->quota_table.c_str()); @@ -1198,9 +1217,8 @@ class UpdateObjectDataOp: virtual public DBOp { private: const std::string Query = "UPDATE '{}' \ - SET ObjName = {}, ObjInstance = {}, ObjNS = {} \ - WHERE ObjName = {} and ObjInstance = {} and ObjNS = {} and \ - BucketName = {}"; + SET Mtime = {} WHERE ObjName = {} and ObjInstance = {} and \ + BucketName = {} and ObjID = {}"; public: virtual ~UpdateObjectDataOp() {} @@ -1208,11 +1226,10 @@ class UpdateObjectDataOp: virtual public DBOp { std::string Schema(DBOpPrepareParams ¶ms) { return fmt::format(Query.c_str(), params.objectdata_table.c_str(), - params.op.obj.new_obj_name, params.op.obj.new_obj_instance, - params.op.obj.new_obj_ns, + params.op.obj.mtime, params.op.obj.obj_name, params.op.obj.obj_instance, - params.op.obj.obj_ns, - params.op.bucket.bucket_name.c_str()); + params.op.bucket.bucket_name.c_str(), + params.op.obj.obj_id); } }; class GetObjectDataOp: virtual public DBOp { @@ -1256,7 +1273,7 @@ class DeleteObjectDataOp: virtual public DBOp { class DeleteStaleObjectDataOp: virtual public DBOp { private: const std::string Query = - "DELETE from '{}' WHERE (ObjName, ObjInstance, ObjID) NOT IN (SELECT s.ObjName, s.ObjInstance, s.ObjID from '{}' as s INNER JOIN '{}' USING (ObjName, BucketName, ObjInstance, ObjID)) and Mtime > {}"; + "DELETE from '{}' WHERE (ObjName, ObjInstance, ObjID) NOT IN (SELECT s.ObjName, s.ObjInstance, s.ObjID from '{}' as s INNER JOIN '{}' USING (ObjName, BucketName, ObjInstance, ObjID)) and Mtime < {}"; public: virtual ~DeleteStaleObjectDataOp() {} @@ -1432,22 +1449,22 @@ class DB { public: DB(std::string db_name, CephContext *_cct) : db_name(db_name), - user_table(db_name+".user.table"), - bucket_table(db_name+".bucket.table"), - quota_table(db_name+".quota.table"), - lc_head_table(db_name+".lc_head.table"), - lc_entry_table(db_name+".lc_entry.table"), + user_table(db_name+"_user_table"), + bucket_table(db_name+"_bucket_table"), + quota_table(db_name+"_quota_table"), + lc_head_table(db_name+"_lc_head_table"), + lc_entry_table(db_name+"_lc_entry_table"), cct(_cct), dp(_cct, ceph_subsys_rgw, "rgw DBStore backend: ") {} /* DB() {}*/ DB(CephContext *_cct) : db_name("default_db"), - user_table(db_name+".user.table"), - bucket_table(db_name+".bucket.table"), - quota_table(db_name+".quota.table"), - lc_head_table(db_name+".lc_head.table"), - lc_entry_table(db_name+".lc_entry.table"), + user_table(db_name+"_user_table"), + bucket_table(db_name+"_bucket_table"), + quota_table(db_name+"_quota_table"), + lc_head_table(db_name+"_lc_head_table"), + lc_entry_table(db_name+"_lc_entry_table"), cct(_cct), dp(_cct, ceph_subsys_rgw, "rgw DBStore backend: ") {} @@ -1461,9 +1478,11 @@ class DB { const std::string getLCHeadTable() { return lc_head_table; } const std::string getLCEntryTable() { return lc_entry_table; } const std::string getObjectTable(std::string bucket) { - return db_name+"."+bucket+".object.table"; } + return db_name+"_"+bucket+"_object_table"; } const std::string getObjectDataTable(std::string bucket) { - return db_name+"."+bucket+".objectdata.table"; } + return db_name+"_"+bucket+"_objectdata_table"; } + const std::string getObjectView(std::string bucket) { + return db_name+"_"+bucket+"_object_view"; } std::map getObjectMap(); @@ -1499,7 +1518,9 @@ class DB { virtual int createTables(const DoutPrefixProvider *dpp) { return 0; } virtual int InitializeDBOps(const DoutPrefixProvider *dpp) { return 0; } virtual int FreeDBOps(const DoutPrefixProvider *dpp) { return 0; } - virtual int InitPrepareParams(const DoutPrefixProvider *dpp, DBOpPrepareParams ¶ms) = 0; + virtual int InitPrepareParams(const DoutPrefixProvider *dpp, + DBOpPrepareParams &p_params, + DBOpParams* params) = 0; virtual int createLCTables(const DoutPrefixProvider *dpp) = 0; virtual int ListAllBuckets(const DoutPrefixProvider *dpp, DBOpParams *params) = 0; @@ -1643,8 +1664,8 @@ class DB { * gc_obj_min_wait: Min. time to wait before deleting any data post its creation. * */ - uint32_t gc_interval = 24*60*60*10; //msec ; default: 24*60*60*10 - uint32_t gc_obj_min_wait = 600; //600sec default + uint32_t gc_interval = 24*60*60; //sec ; default: 24*60*60 + uint32_t gc_obj_min_wait = 60*60; //60*60sec default std::string bucket_marker; std::string user_marker; @@ -1823,11 +1844,6 @@ class DB { bool assume_noent, bool modify_tail); int write_meta(const DoutPrefixProvider *dpp, uint64_t size, uint64_t accounted_size, std::map& attrs); - /* Below are used to update mp data rows object name - * from meta to src object name on multipart upload - * completion - */ - int update_mp_parts(const DoutPrefixProvider *dpp, rgw_obj_key new_obj_key); }; struct Delete { diff --git a/src/rgw/store/dbstore/sqlite/sqliteDB.cc b/src/rgw/store/dbstore/sqlite/sqliteDB.cc index ee6eb9667f362..e8955f5705157 100644 --- a/src/rgw/store/dbstore/sqlite/sqliteDB.cc +++ b/src/rgw/store/dbstore/sqlite/sqliteDB.cc @@ -56,6 +56,9 @@ using namespace std; rc = -1; \ goto out; \ } \ + ldpp_dout(dpp, 20)<<"Bind parameter text for index(" \ + <user_table.empty()) { + params->user_table = getUserTable(); + } + if (params->user_table.empty()) { + params->user_table = getUserTable(); + } + if (params->bucket_table.empty()) { + params->bucket_table = getBucketTable(); + } + if (params->quota_table.empty()) { + params->quota_table = getQuotaTable(); + } + if (params->lc_entry_table.empty()) { + params->lc_entry_table = getLCEntryTable(); + } + if (params->lc_head_table.empty()) { + params->lc_head_table = getLCHeadTable(); + } + + p_params.user_table = params->user_table; + p_params.bucket_table = params->bucket_table; + p_params.quota_table = params->quota_table; + p_params.lc_entry_table = params->lc_entry_table; + p_params.lc_head_table = params->lc_head_table; + + p_params.op.query_str = params->op.query_str; + + bucket = params->op.bucket.info.bucket.name; + + if (!bucket.empty()) { + if (params->object_table.empty()) { + params->object_table = getObjectTable(bucket); + } + if (params->objectdata_table.empty()) { + params->objectdata_table = getObjectDataTable(bucket); + } + if (params->object_view.empty()) { + params->object_view = getObjectView(bucket); + } + p_params.object_table = params->object_table; + p_params.objectdata_table = params->objectdata_table; + p_params.object_view = params->object_view; + } + + return 0; +} + static int list_callback(void *None, int argc, char **argv, char **aname) { int i; - for(i=0; iuser_table; + InitPrepareParams(dpp, p_params, params); SQL_PREPARE(dpp, p_params, sdb, stmt, ret, "PrepareInsertUser"); out: @@ -1153,7 +1231,7 @@ int SQLRemoveUser::Prepare(const DoutPrefixProvider *dpp, struct DBOpParams *par goto out; } - p_params.user_table = params->user_table; + InitPrepareParams(dpp, p_params, params); SQL_PREPARE(dpp, p_params, sdb, stmt, ret, "PrepareRemoveUser"); out: @@ -1192,8 +1270,7 @@ int SQLGetUser::Prepare(const DoutPrefixProvider *dpp, struct DBOpParams *params goto out; } - p_params.user_table = params->user_table; - p_params.op.query_str = params->op.query_str; + InitPrepareParams(dpp, p_params, params); if (params->op.query_str == "email") { SQL_PREPARE(dpp, p_params, sdb, email_stmt, ret, "PrepareGetUser"); @@ -1274,7 +1351,7 @@ int SQLInsertBucket::Prepare(const DoutPrefixProvider *dpp, struct DBOpParams *p goto out; } - p_params.bucket_table = params->bucket_table; + InitPrepareParams(dpp, p_params, params); SQL_PREPARE(dpp, p_params, sdb, stmt, ret, "PrepareInsertBucket"); @@ -1382,12 +1459,20 @@ int SQLInsertBucket::Execute(const DoutPrefixProvider *dpp, struct DBOpParams *p int ret = -1; class SQLObjectOp *ObPtr = NULL; string bucket_name = params->op.bucket.info.bucket.name; + struct DBOpPrepareParams p_params = PrepareParams; ObPtr = new SQLObjectOp(sdb, ctx()); objectmapInsert(dpp, bucket_name, ObPtr); SQL_EXECUTE(dpp, params, stmt, NULL); + + /* Once Bucket is inserted created corresponding object(&data) tables + */ + InitPrepareParams(dpp, p_params, params); + + (void)createObjectTable(dpp, params); + (void)createObjectDataTable(dpp, params); out: return ret; } @@ -1402,8 +1487,7 @@ int SQLUpdateBucket::Prepare(const DoutPrefixProvider *dpp, struct DBOpParams *p goto out; } - p_params.op.query_str = params->op.query_str; - p_params.bucket_table = params->bucket_table; + InitPrepareParams(dpp, p_params, params); if (params->op.query_str == "attrs") { SQL_PREPARE(dpp, p_params, sdb, attrs_stmt, ret, "PrepareUpdateBucket"); @@ -1557,7 +1641,7 @@ int SQLRemoveBucket::Prepare(const DoutPrefixProvider *dpp, struct DBOpParams *p goto out; } - p_params.bucket_table = params->bucket_table; + InitPrepareParams(dpp, p_params, params); SQL_PREPARE(dpp, p_params, sdb, stmt, ret, "PrepareRemoveBucket"); @@ -1600,8 +1684,7 @@ int SQLGetBucket::Prepare(const DoutPrefixProvider *dpp, struct DBOpParams *para goto out; } - p_params.bucket_table = params->bucket_table; - p_params.user_table = params->user_table; + InitPrepareParams(dpp, p_params, params); SQL_PREPARE(dpp, p_params, sdb, stmt, ret, "PrepareGetBucket"); @@ -1649,9 +1732,8 @@ int SQLListUserBuckets::Prepare(const DoutPrefixProvider *dpp, struct DBOpParams goto out; } - p_params.bucket_table = params->bucket_table; + InitPrepareParams(dpp, p_params, params); - p_params.op.query_str = params->op.query_str; if (params->op.query_str == "all") { SQL_PREPARE(dpp, p_params, sdb, all_stmt, ret, "PrepareListUserBuckets"); }else { @@ -1707,20 +1789,13 @@ int SQLPutObject::Prepare(const DoutPrefixProvider *dpp, struct DBOpParams *para { int ret = -1; struct DBOpPrepareParams p_params = PrepareParams; - struct DBOpParams copy = *params; - string bucket_name = params->op.bucket.info.bucket.name; if (!*sdb) { ldpp_dout(dpp, 0)<<"In SQLPutObject - no db" << dendl; goto out; } - if (p_params.object_table.empty()) { - p_params.object_table = getObjectTable(bucket_name); - } - params->object_table = p_params.object_table; - (void)createObjectTable(dpp, params); - + InitPrepareParams(dpp, p_params, params); SQL_PREPARE(dpp, p_params, sdb, stmt, ret, "PreparePutObject"); out: @@ -1895,20 +1970,13 @@ int SQLDeleteObject::Prepare(const DoutPrefixProvider *dpp, struct DBOpParams *p { int ret = -1; struct DBOpPrepareParams p_params = PrepareParams; - struct DBOpParams copy = *params; - string bucket_name = params->op.bucket.info.bucket.name; if (!*sdb) { ldpp_dout(dpp, 0)<<"In SQLDeleteObject - no db" << dendl; goto out; } - if (p_params.object_table.empty()) { - p_params.object_table = getObjectTable(bucket_name); - } - params->object_table = p_params.object_table; - (void)createObjectTable(dpp, params); - + InitPrepareParams(dpp, p_params, params); SQL_PREPARE(dpp, p_params, sdb, stmt, ret, "PrepareDeleteObject"); out: @@ -1946,20 +2014,13 @@ int SQLGetObject::Prepare(const DoutPrefixProvider *dpp, struct DBOpParams *para { int ret = -1; struct DBOpPrepareParams p_params = PrepareParams; - struct DBOpParams copy = *params; - string bucket_name = params->op.bucket.info.bucket.name; if (!*sdb) { ldpp_dout(dpp, 0)<<"In SQLGetObject - no db" << dendl; goto out; } - if (p_params.object_table.empty()) { - p_params.object_table = getObjectTable(bucket_name); - } - params->object_table = p_params.object_table; - (void)createObjectTable(dpp, params); - + InitPrepareParams(dpp, p_params, params); SQL_PREPARE(dpp, p_params, sdb, stmt, ret, "PrepareGetObject"); out: @@ -2006,12 +2067,7 @@ int SQLUpdateObject::Prepare(const DoutPrefixProvider *dpp, struct DBOpParams *p goto out; } - if (p_params.object_table.empty()) { - bucket_name = params->op.bucket.info.bucket.name; - p_params.object_table = getObjectTable(bucket_name); - } - - p_params.op.query_str = params->op.query_str; + InitPrepareParams(dpp, p_params, params); if (params->op.query_str == "omap") { SQL_PREPARE(dpp, p_params, sdb, omap_stmt, ret, "PrepareUpdateObject"); @@ -2243,26 +2299,13 @@ int SQLListBucketObjects::Prepare(const DoutPrefixProvider *dpp, struct DBOpPara { int ret = -1; struct DBOpPrepareParams p_params = PrepareParams; - struct DBOpParams copy = *params; - string bucket_name; if (!*sdb) { ldpp_dout(dpp, 0)<<"In SQLListBucketObjects - no db" << dendl; goto out; } - if (p_params.object_table.empty()) { - bucket_name = params->op.bucket.info.bucket.name; - p_params.object_table = getObjectTable(bucket_name); - } - - /* XXX: instead of creating..maybe keep object count in bucket info - * and return if there is no object table created. - */ - params->object_table = p_params.object_table; - (void)createObjectTable(dpp, params); - - p_params.op.query_str = params->op.query_str; + InitPrepareParams(dpp, p_params, params); SQL_PREPARE(dpp, p_params, sdb, stmt, ret, "PrepareListBucketObjects"); @@ -2302,23 +2345,13 @@ int SQLPutObjectData::Prepare(const DoutPrefixProvider *dpp, struct DBOpParams * { int ret = -1; struct DBOpPrepareParams p_params = PrepareParams; - struct DBOpParams copy = *params; - string bucket_name = params->op.bucket.info.bucket.name; if (!*sdb) { ldpp_dout(dpp, 0)<<"In SQLPutObjectData - no db" << dendl; goto out; } - if (p_params.object_table.empty()) { - p_params.object_table = getObjectTable(bucket_name); - } - if (p_params.objectdata_table.empty()) { - p_params.objectdata_table = getObjectDataTable(bucket_name); - } - params->object_table = p_params.object_table; - params->objectdata_table = p_params.objectdata_table; - (void)createObjectDataTable(dpp, params); + InitPrepareParams(dpp, p_params, params); SQL_PREPARE(dpp, p_params, sdb, stmt, ret, "PreparePutObjectData"); @@ -2390,25 +2423,13 @@ int SQLUpdateObjectData::Prepare(const DoutPrefixProvider *dpp, struct DBOpParam { int ret = -1; struct DBOpPrepareParams p_params = PrepareParams; - struct DBOpParams copy = *params; - string bucket_name = params->op.bucket.info.bucket.name; if (!*sdb) { ldpp_dout(dpp, 0)<<"In SQLUpdateObjectData - no db" << dendl; goto out; } - if (p_params.object_table.empty()) { - p_params.object_table = getObjectTable(bucket_name); - } - if (p_params.objectdata_table.empty()) { - p_params.objectdata_table = getObjectDataTable(bucket_name); - } - params->bucket_table = p_params.bucket_table; - params->object_table = p_params.object_table; - params->objectdata_table = p_params.objectdata_table; - (void)createObjectDataTable(dpp, params); - + InitPrepareParams(dpp, p_params, params); SQL_PREPARE(dpp, p_params, sdb, stmt, ret, "PrepareUpdateObjectData"); out: @@ -2422,32 +2443,19 @@ int SQLUpdateObjectData::Bind(const DoutPrefixProvider *dpp, struct DBOpParams * struct DBOpPrepareParams p_params = PrepareParams; SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.obj_name.c_str(), sdb); - SQL_BIND_TEXT(dpp, stmt, index, params->op.obj.state.obj.key.name.c_str(), sdb); SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.obj_instance.c_str(), sdb); - SQL_BIND_TEXT(dpp, stmt, index, params->op.obj.state.obj.key.instance.c_str(), sdb); - SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.obj_ns.c_str(), sdb); - - SQL_BIND_TEXT(dpp, stmt, index, params->op.obj.state.obj.key.ns.c_str(), sdb); - SQL_BIND_INDEX(dpp, stmt, index, p_params.op.bucket.bucket_name.c_str(), sdb); - SQL_BIND_TEXT(dpp, stmt, index, params->op.bucket.info.bucket.name.c_str(), sdb); - SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.new_obj_name.c_str(), sdb); - - SQL_BIND_TEXT(dpp, stmt, index, params->op.obj.new_obj_key.name.c_str(), sdb); - - SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.new_obj_instance.c_str(), sdb); - - SQL_BIND_TEXT(dpp, stmt, index, params->op.obj.new_obj_key.instance.c_str(), sdb); - - SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.new_obj_ns.c_str(), sdb); + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.obj_id.c_str(), sdb); + SQL_BIND_TEXT(dpp, stmt, index, params->op.obj.obj_id.c_str(), sdb); - SQL_BIND_TEXT(dpp, stmt, index, params->op.obj.new_obj_key.ns.c_str(), sdb); + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.mtime.c_str(), sdb); + SQL_ENCODE_BLOB_PARAM(dpp, stmt, index, params->op.obj.state.mtime, sdb); out: return rc; @@ -2466,24 +2474,13 @@ int SQLGetObjectData::Prepare(const DoutPrefixProvider *dpp, struct DBOpParams * { int ret = -1; struct DBOpPrepareParams p_params = PrepareParams; - struct DBOpParams copy = *params; - string bucket_name = params->op.bucket.info.bucket.name; if (!*sdb) { ldpp_dout(dpp, 0)<<"In SQLGetObjectData - no db" << dendl; goto out; } - if (p_params.object_table.empty()) { - p_params.object_table = getObjectTable(bucket_name); - } - if (p_params.objectdata_table.empty()) { - p_params.objectdata_table = getObjectDataTable(bucket_name); - } - params->object_table = p_params.object_table; - params->objectdata_table = p_params.objectdata_table; - (void)createObjectDataTable(dpp, params); - + InitPrepareParams(dpp, p_params, params); SQL_PREPARE(dpp, p_params, sdb, stmt, ret, "PrepareGetObjectData"); out: @@ -2525,24 +2522,13 @@ int SQLDeleteObjectData::Prepare(const DoutPrefixProvider *dpp, struct DBOpParam { int ret = -1; struct DBOpPrepareParams p_params = PrepareParams; - struct DBOpParams copy = *params; - string bucket_name = params->op.bucket.info.bucket.name; if (!*sdb) { ldpp_dout(dpp, 0)<<"In SQLDeleteObjectData - no db" << dendl; goto out; } - if (p_params.object_table.empty()) { - p_params.object_table = getObjectTable(bucket_name); - } - if (p_params.objectdata_table.empty()) { - p_params.objectdata_table = getObjectDataTable(bucket_name); - } - params->object_table = p_params.object_table; - params->objectdata_table = p_params.objectdata_table; - (void)createObjectDataTable(dpp, params); - + InitPrepareParams(dpp, p_params, params); SQL_PREPARE(dpp, p_params, sdb, stmt, ret, "PrepareDeleteObjectData"); out: @@ -2584,24 +2570,13 @@ int SQLDeleteStaleObjectData::Prepare(const DoutPrefixProvider *dpp, struct DBOp { int ret = -1; struct DBOpPrepareParams p_params = PrepareParams; - struct DBOpParams copy = *params; - string bucket_name = params->op.bucket.info.bucket.name; if (!*sdb) { ldpp_dout(dpp, 0)<<"In SQLDeleteStaleObjectData - no db" << dendl; goto out; } - if (p_params.object_table.empty()) { - p_params.object_table = getObjectTable(bucket_name); - } - if (p_params.objectdata_table.empty()) { - p_params.objectdata_table = getObjectDataTable(bucket_name); - } - params->object_table = p_params.object_table; - params->objectdata_table = p_params.objectdata_table; - (void)createObjectDataTable(dpp, params); - + InitPrepareParams(dpp, p_params, params); SQL_PREPARE(dpp, p_params, sdb, stmt, ret, "PrepareDeleteStaleObjectData"); out: @@ -2640,7 +2615,7 @@ int SQLInsertLCEntry::Prepare(const DoutPrefixProvider *dpp, struct DBOpParams * goto out; } - p_params.lc_entry_table = params->lc_entry_table; + InitPrepareParams(dpp, p_params, params); SQL_PREPARE(dpp, p_params, sdb, stmt, ret, "PrepareInsertLCEntry"); @@ -2689,7 +2664,7 @@ int SQLRemoveLCEntry::Prepare(const DoutPrefixProvider *dpp, struct DBOpParams * goto out; } - p_params.lc_entry_table = params->lc_entry_table; + InitPrepareParams(dpp, p_params, params); SQL_PREPARE(dpp, p_params, sdb, stmt, ret, "PrepareRemoveLCEntry"); @@ -2733,8 +2708,7 @@ int SQLGetLCEntry::Prepare(const DoutPrefixProvider *dpp, struct DBOpParams *par goto out; } - p_params.lc_entry_table = params->lc_entry_table; - p_params.op.query_str = params->op.query_str; + InitPrepareParams(dpp, p_params, params); if (params->op.query_str == "get_next_entry") { pstmt = &next_stmt; @@ -2795,7 +2769,7 @@ int SQLListLCEntries::Prepare(const DoutPrefixProvider *dpp, struct DBOpParams * goto out; } - p_params.lc_entry_table = params->lc_entry_table; + InitPrepareParams(dpp, p_params, params); SQL_PREPARE(dpp, p_params, sdb, stmt, ret, "PrepareListLCEntries"); @@ -2841,7 +2815,7 @@ int SQLInsertLCHead::Prepare(const DoutPrefixProvider *dpp, struct DBOpParams *p goto out; } - p_params.lc_head_table = params->lc_head_table; + InitPrepareParams(dpp, p_params, params); SQL_PREPARE(dpp, p_params, sdb, stmt, ret, "PrepareInsertLCHead"); @@ -2887,7 +2861,7 @@ int SQLRemoveLCHead::Prepare(const DoutPrefixProvider *dpp, struct DBOpParams *p goto out; } - p_params.lc_head_table = params->lc_head_table; + InitPrepareParams(dpp, p_params, params); SQL_PREPARE(dpp, p_params, sdb, stmt, ret, "PrepareRemoveLCHead"); @@ -2927,7 +2901,7 @@ int SQLGetLCHead::Prepare(const DoutPrefixProvider *dpp, struct DBOpParams *para goto out; } - p_params.lc_head_table = params->lc_head_table; + InitPrepareParams(dpp, p_params, params); SQL_PREPARE(dpp, p_params, sdb, stmt, ret, "PrepareGetLCHead"); diff --git a/src/rgw/store/dbstore/sqlite/sqliteDB.h b/src/rgw/store/dbstore/sqlite/sqliteDB.h index b83296c5a8605..91575fd0a028e 100644 --- a/src/rgw/store/dbstore/sqlite/sqliteDB.h +++ b/src/rgw/store/dbstore/sqlite/sqliteDB.h @@ -25,10 +25,8 @@ class SQLiteDB : public DB, virtual public DBOp { SQLiteDB(sqlite3 *dbi, std::string db_name, CephContext *_cct) : DB(db_name, _cct), cct(_cct) { db = (void*)dbi; - InitPrepareParams(get_def_dpp(), PrepareParams); } SQLiteDB(std::string db_name, CephContext *_cct) : DB(db_name, _cct), cct(_cct) { - InitPrepareParams(get_def_dpp(), PrepareParams); } ~SQLiteDB() {} @@ -38,7 +36,8 @@ class SQLiteDB : public DB, virtual public DBOp { int InitializeDBOps(const DoutPrefixProvider *dpp) override; int FreeDBOps(const DoutPrefixProvider *dpp) override; - int InitPrepareParams(const DoutPrefixProvider *dpp, DBOpPrepareParams ¶ms) override { return 0; } + int InitPrepareParams(const DoutPrefixProvider *dpp, DBOpPrepareParams &p_params, + DBOpParams* params) override; int exec(const DoutPrefixProvider *dpp, const char *schema, int (*callback)(void*,int,char**,char**)); @@ -52,7 +51,11 @@ class SQLiteDB : public DB, virtual public DBOp { int createUserTable(const DoutPrefixProvider *dpp, DBOpParams *params); int createObjectTable(const DoutPrefixProvider *dpp, DBOpParams *params); int createObjectDataTable(const DoutPrefixProvider *dpp, DBOpParams *params); + int createObjectView(const DoutPrefixProvider *dpp, DBOpParams *params); int createQuotaTable(const DoutPrefixProvider *dpp, DBOpParams *params); + void populate_object_params(const DoutPrefixProvider *dpp, + struct DBOpPrepareParams& p_params, + struct DBOpParams* params, bool data); int createLCTables(const DoutPrefixProvider *dpp) override; diff --git a/src/rgw/store/dbstore/tests/dbstore_tests.cc b/src/rgw/store/dbstore/tests/dbstore_tests.cc index 481ad5be4c28e..7148302a1fc82 100644 --- a/src/rgw/store/dbstore/tests/dbstore_tests.cc +++ b/src/rgw/store/dbstore/tests/dbstore_tests.cc @@ -98,6 +98,7 @@ namespace { GlobalParams.op.obj.state.obj.bucket = GlobalParams.op.bucket.info.bucket; GlobalParams.op.obj.state.obj.key.name = object1; GlobalParams.op.obj.state.obj.key.instance = "inst1"; + GlobalParams.op.obj.obj_id = "obj_id1"; GlobalParams.op.obj_data.part_num = 0; /* As of now InitializeParams doesnt do anything @@ -1006,6 +1007,7 @@ TEST_F(DBStoreTest, PutObjectData) { encode("HELLO WORLD", b1); params.op.obj_data.data = b1; params.op.obj_data.size = 12; + params.op.obj.state.mtime = real_clock::now(); ret = db->ProcessOp(dpp, "PutObjectData", ¶ms); ASSERT_EQ(ret, 0); } @@ -1014,8 +1016,7 @@ TEST_F(DBStoreTest, UpdateObjectData) { struct DBOpParams params = GlobalParams; int ret = -1; - params.op.obj.new_obj_key.name = "object3"; - params.op.obj.new_obj_key.instance = "inst3"; + params.op.obj.state.mtime = bucket_mtime; ret = db->ProcessOp(dpp, "UpdateObjectData", ¶ms); ASSERT_EQ(ret, 0); } @@ -1024,15 +1025,16 @@ TEST_F(DBStoreTest, GetObjectData) { struct DBOpParams params = GlobalParams; int ret = -1; - params.op.obj.state.obj.key.instance = "inst3"; - params.op.obj.state.obj.key.name = "object3"; + params.op.obj.state.obj.key.instance = "inst1"; + params.op.obj.state.obj.key.name = "object1"; ret = db->ProcessOp(dpp, "GetObjectData", ¶ms); ASSERT_EQ(ret, 0); ASSERT_EQ(params.op.obj_data.part_num, 1); ASSERT_EQ(params.op.obj_data.offset, 10); ASSERT_EQ(params.op.obj_data.multipart_part_str, "2"); - ASSERT_EQ(params.op.obj.state.obj.key.instance, "inst3"); - ASSERT_EQ(params.op.obj.state.obj.key.name, "object3"); + ASSERT_EQ(params.op.obj.state.obj.key.instance, "inst1"); + ASSERT_EQ(params.op.obj.state.obj.key.name, "object1"); + ASSERT_EQ(params.op.obj.state.mtime, bucket_mtime); string data; decode(data, params.op.obj_data.data); ASSERT_EQ(data, "HELLO WORLD"); -- 2.39.5