From e05daafdb3fecc8887b4ab7d76edf6461c487df9 Mon Sep 17 00:00:00 2001 From: Soumya Koduri Date: Wed, 1 Sep 2021 17:40:59 +0530 Subject: [PATCH] rgw/dbstore: Object APIs Support for simple Put, Get, Delete, List Ops of Regular Objects on dbstore. Signed-off-by: Soumya Koduri --- src/rgw/rgw_obj_manifest.h | 44 + src/rgw/rgw_rados.h | 45 - src/rgw/rgw_sal.cc | 24 + src/rgw/rgw_sal_dbstore.cc | 503 ++++++++- src/rgw/rgw_sal_dbstore.h | 187 ++++ src/rgw/rgw_sal_rados.cc | 22 - src/rgw/store/dbstore/common/dbstore.cc | 1031 +++++++++++++++++- src/rgw/store/dbstore/common/dbstore.h | 834 +++++++++++++- src/rgw/store/dbstore/sqlite/sqliteDB.cc | 824 ++++++++++++-- src/rgw/store/dbstore/sqlite/sqliteDB.h | 104 +- src/rgw/store/dbstore/tests/dbstore_tests.cc | 390 ++++++- 11 files changed, 3705 insertions(+), 303 deletions(-) diff --git a/src/rgw/rgw_obj_manifest.h b/src/rgw/rgw_obj_manifest.h index a5df7a4b255..98b43e51a92 100644 --- a/src/rgw/rgw_obj_manifest.h +++ b/src/rgw/rgw_obj_manifest.h @@ -539,3 +539,47 @@ public: }; }; WRITE_CLASS_ENCODER(RGWObjManifest) + +struct RGWObjState { + rgw_obj obj; + bool is_atomic{false}; + bool has_attrs{false}; + bool exists{false}; + uint64_t size{0}; //< size of raw object + uint64_t accounted_size{0}; //< size before compression, encryption + ceph::real_time mtime; + uint64_t epoch{0}; + bufferlist obj_tag; + bufferlist tail_tag; + std::string write_tag; + bool fake_tag{false}; + std::optional manifest; + std::string shadow_obj; + bool has_data{false}; + bufferlist data; + bool prefetch_data{false}; + bool keep_tail{false}; + bool is_olh{false}; + bufferlist olh_tag; + uint64_t pg_ver{false}; + uint32_t zone_short_id{0}; + + /* important! don't forget to update copy constructor */ + + RGWObjVersionTracker objv_tracker; + + std::map attrset; + + RGWObjState(); + RGWObjState(const RGWObjState& rhs); + ~RGWObjState(); + + bool get_attr(std::string name, bufferlist& dest) { + std::map::iterator iter = attrset.find(name); + if (iter != attrset.end()) { + dest = iter->second; + return true; + } + return false; + } +}; diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h index 3d6d667662e..26b603b128b 100644 --- a/src/rgw/rgw_rados.h +++ b/src/rgw/rgw_rados.h @@ -155,51 +155,6 @@ struct RGWCloneRangeInfo { uint64_t len; }; -struct RGWObjState { - rgw_obj obj; - bool is_atomic{false}; - bool has_attrs{false}; - bool exists{false}; - uint64_t size{0}; //< size of raw object - uint64_t accounted_size{0}; //< size before compression, encryption - ceph::real_time mtime; - uint64_t epoch{0}; - bufferlist obj_tag; - bufferlist tail_tag; - std::string write_tag; - bool fake_tag{false}; - std::optional manifest; - - std::string shadow_obj; - bool has_data{false}; - bufferlist data; - bool prefetch_data{false}; - bool keep_tail{false}; - bool is_olh{false}; - bufferlist olh_tag; - uint64_t pg_ver{false}; - uint32_t zone_short_id{0}; - - /* important! don't forget to update copy constructor */ - - RGWObjVersionTracker objv_tracker; - - std::map attrset; - - RGWObjState(); - RGWObjState(const RGWObjState& rhs); - ~RGWObjState(); - - bool get_attr(std::string name, bufferlist& dest) { - auto iter = attrset.find(name); - if (iter != attrset.end()) { - dest = iter->second; - return true; - } - return false; - } -}; - class RGWFetchObjFilter { public: virtual ~RGWFetchObjFilter() {} diff --git a/src/rgw/rgw_sal.cc b/src/rgw/rgw_sal.cc index 5917280bca9..4a761ce3c64 100644 --- a/src/rgw/rgw_sal.cc +++ b/src/rgw/rgw_sal.cc @@ -130,3 +130,27 @@ void StoreManager::close_storage(rgw::sal::Store* store) delete store; } + +namespace rgw::sal { +int Object::range_to_ofs(uint64_t obj_size, int64_t &ofs, int64_t &end) +{ + if (ofs < 0) { + ofs += obj_size; + if (ofs < 0) + ofs = 0; + end = obj_size - 1; + } else if (end < 0) { + end = obj_size - 1; + } + + if (obj_size > 0) { + if (ofs >= (off_t)obj_size) { + return -ERANGE; + } + if (end >= (off_t)obj_size) { + end = obj_size - 1; + } + } + return 0; +} +} diff --git a/src/rgw/rgw_sal_dbstore.cc b/src/rgw/rgw_sal_dbstore.cc index a28610f949f..67cbdad666d 100644 --- a/src/rgw/rgw_sal_dbstore.cc +++ b/src/rgw/rgw_sal_dbstore.cc @@ -46,7 +46,7 @@ namespace rgw::sal { buckets.set_truncated(is_truncated); for (const auto& ent : ulist.get_buckets()) { - buckets.add(std::unique_ptr(new DBBucket(this->store, ent.second, this))); + buckets.add(std::make_unique(this->store, ent.second, this)); } return 0; @@ -330,13 +330,37 @@ namespace rgw::sal { std::unique_ptr DBBucket::get_object(const rgw_obj_key& k) { - return nullptr; + return std::make_unique(this->store, k, this); } int DBBucket::list(const DoutPrefixProvider *dpp, ListParams& params, int max, ListResults& results, optional_yield y) { - /* XXX: Objects */ - return 0; + int ret = 0; + + results.objs.clear(); + + DB::Bucket target(store->getDB(), get_info()); + DB::Bucket::List list_op(&target); + + list_op.params.prefix = params.prefix; + list_op.params.delim = params.delim; + list_op.params.marker = params.marker; + list_op.params.ns = params.ns; + list_op.params.end_marker = params.end_marker; + list_op.params.ns = params.ns; + list_op.params.enforce_ns = params.enforce_ns; + list_op.params.filter = params.filter; + list_op.params.list_versions = params.list_versions; + list_op.params.allow_unordered = params.allow_unordered; + + results.objs.clear(); + ret = list_op.list_objects(dpp, max, &results.objs, &results.common_prefixes, &results.is_truncated); + if (ret >= 0) { + results.next_marker = list_op.get_next_marker(); + params.marker = results.next_marker; + } + + return ret; } int DBBucket::list_multiparts(const DoutPrefixProvider *dpp, @@ -416,9 +440,465 @@ namespace rgw::sal { std::unique_ptr DBStore::get_lua_script_manager() { - return std::unique_ptr(new DBLuaScriptManager(this)); + return std::make_unique(this); + } + + int DBObject::get_obj_state(const DoutPrefixProvider* dpp, RGWObjectCtx* rctx, RGWObjState **state, optional_yield y, bool follow_olh) + { + if (!*state) { + *state = new RGWObjState(); + } + DB::Object op_target(store->getDB(), get_bucket()->get_info(), get_obj()); + return op_target.get_obj_state(dpp, get_bucket()->get_info(), get_obj(), follow_olh, state); + } + + int DBObject::read_attrs(const DoutPrefixProvider* dpp, DB::Object::Read &read_op, optional_yield y, rgw_obj* target_obj) + { + read_op.params.attrs = &attrs; + read_op.params.target_obj = target_obj; + read_op.params.obj_size = &obj_size; + read_op.params.lastmod = &mtime; + + return read_op.prepare(dpp); + } + + int DBObject::set_obj_attrs(const DoutPrefixProvider* dpp, RGWObjectCtx* rctx, Attrs* setattrs, Attrs* delattrs, optional_yield y, rgw_obj* target_obj) + { + Attrs empty; + DB::Object op_target(store->getDB(), + get_bucket()->get_info(), target_obj ? *target_obj : get_obj()); + return op_target.set_attrs(dpp, setattrs ? *setattrs : empty, delattrs); + } + + int DBObject::get_obj_attrs(RGWObjectCtx* rctx, optional_yield y, const DoutPrefixProvider* dpp, rgw_obj* target_obj) + { + DB::Object op_target(store->getDB(), get_bucket()->get_info(), get_obj()); + DB::Object::Read read_op(&op_target); + + return read_attrs(dpp, read_op, y, target_obj); + } + + int DBObject::modify_obj_attrs(RGWObjectCtx* rctx, const char* attr_name, bufferlist& attr_val, optional_yield y, const DoutPrefixProvider* dpp) + { + rgw_obj target = get_obj(); + int r = get_obj_attrs(rctx, y, dpp, &target); + if (r < 0) { + return r; + } + set_atomic(rctx); + attrs[attr_name] = attr_val; + return set_obj_attrs(dpp, rctx, &attrs, nullptr, y, &target); + } + + int DBObject::delete_obj_attrs(const DoutPrefixProvider* dpp, RGWObjectCtx* rctx, const char* attr_name, optional_yield y) + { + rgw_obj target = get_obj(); + Attrs rmattr; + bufferlist bl; + + set_atomic(rctx); + rmattr[attr_name] = bl; + return set_obj_attrs(dpp, rctx, nullptr, &rmattr, y, &target); + } + + int DBObject::copy_obj_data(RGWObjectCtx& rctx, Bucket* dest_bucket, + Object* dest_obj, + uint16_t olh_epoch, + std::string* petag, + const DoutPrefixProvider* dpp, + optional_yield y) + { + return 0; + } + + /* RGWObjectCtx will be moved out of sal */ + /* XXX: Placeholder. Should not be needed later after Dan's patch */ + void DBObject::set_atomic(RGWObjectCtx* rctx) const + { + return; + } + + /* RGWObjectCtx will be moved out of sal */ + /* XXX: Placeholder. Should not be needed later after Dan's patch */ + void DBObject::set_prefetch_data(RGWObjectCtx* rctx) + { + return; + } + + bool DBObject::is_expired() { + return false; + } + + void DBObject::gen_rand_obj_instance_name() + { + store->getDB()->gen_rand_obj_instance_name(&key); + } + + + int DBObject::omap_get_vals(const DoutPrefixProvider *dpp, const std::string& marker, uint64_t count, + std::map *m, + bool* pmore, optional_yield y) + { + DB::Object op_target(store->getDB(), + get_bucket()->get_info(), get_obj()); + return op_target.obj_omap_get_vals(dpp, marker, count, m, pmore); + } + + int DBObject::omap_get_all(const DoutPrefixProvider *dpp, std::map *m, + optional_yield y) + { + DB::Object op_target(store->getDB(), + get_bucket()->get_info(), get_obj()); + return op_target.obj_omap_get_all(dpp, m); + } + + int DBObject::omap_get_vals_by_keys(const DoutPrefixProvider *dpp, const std::string& oid, + const std::set& keys, + Attrs* vals) + { + DB::Object op_target(store->getDB(), + get_bucket()->get_info(), get_obj()); + return op_target.obj_omap_get_vals_by_keys(dpp, oid, keys, vals); + } + + int DBObject::omap_set_val_by_key(const DoutPrefixProvider *dpp, const std::string& key, bufferlist& val, + bool must_exist, optional_yield y) + { + DB::Object op_target(store->getDB(), + get_bucket()->get_info(), get_obj()); + return op_target.obj_omap_set_val_by_key(dpp, key, val, must_exist); + } + + MPSerializer* DBObject::get_serializer(const DoutPrefixProvider *dpp, const std::string& lock_name) + { + return nullptr; + } + + int DBObject::transition(RGWObjectCtx& rctx, + Bucket* bucket, + const rgw_placement_rule& placement_rule, + const real_time& mtime, + uint64_t olh_epoch, + const DoutPrefixProvider* dpp, + optional_yield y) + { + return 0; + } + + bool DBObject::placement_rules_match(rgw_placement_rule& r1, rgw_placement_rule& r2) + { + /* XXX: support single default zone and zonegroup for now */ + return true; + } + + int DBObject::get_obj_layout(const DoutPrefixProvider *dpp, optional_yield y, Formatter* f, RGWObjectCtx* obj_ctx) + { + return 0; + } + + std::unique_ptr DBObject::get_read_op(RGWObjectCtx* ctx) + { + return std::make_unique(this, ctx); + } + + DBObject::DBReadOp::DBReadOp(DBObject *_source, RGWObjectCtx *_rctx) : + source(_source), + rctx(_rctx), + op_target(_source->store->getDB(), + _source->get_bucket()->get_info(), + _source->get_obj()), + parent_op(&op_target) + { } + + int DBObject::DBReadOp::prepare(optional_yield y, const DoutPrefixProvider* dpp) + { + uint64_t obj_size; + + parent_op.conds.mod_ptr = params.mod_ptr; + parent_op.conds.unmod_ptr = params.unmod_ptr; + parent_op.conds.high_precision_time = params.high_precision_time; + parent_op.conds.mod_zone_id = params.mod_zone_id; + parent_op.conds.mod_pg_ver = params.mod_pg_ver; + parent_op.conds.if_match = params.if_match; + parent_op.conds.if_nomatch = params.if_nomatch; + parent_op.params.lastmod = params.lastmod; + parent_op.params.target_obj = params.target_obj; + parent_op.params.obj_size = &obj_size; + parent_op.params.attrs = &source->get_attrs(); + + int ret = parent_op.prepare(dpp); + if (ret < 0) + return ret; + + source->set_key(parent_op.state.obj.key); + source->set_obj_size(obj_size); + + return ret; + } + + int DBObject::DBReadOp::read(int64_t ofs, int64_t end, bufferlist& bl, optional_yield y, const DoutPrefixProvider* dpp) + { + return parent_op.read(ofs, end, bl, dpp); + } + + int DBObject::DBReadOp::get_attr(const DoutPrefixProvider* dpp, const char* name, bufferlist& dest, optional_yield y) + { + return parent_op.get_attr(dpp, name, dest); + } + + std::unique_ptr DBObject::get_delete_op(RGWObjectCtx* ctx) + { + return std::make_unique(this, ctx); + } + + DBObject::DBDeleteOp::DBDeleteOp(DBObject *_source, RGWObjectCtx *_rctx) : + source(_source), + rctx(_rctx), + op_target(_source->store->getDB(), + _source->get_bucket()->get_info(), + _source->get_obj()), + parent_op(&op_target) + { } + + int DBObject::DBDeleteOp::delete_obj(const DoutPrefixProvider* dpp, optional_yield y) + { + parent_op.params.bucket_owner = params.bucket_owner.get_id(); + parent_op.params.versioning_status = params.versioning_status; + parent_op.params.obj_owner = params.obj_owner; + parent_op.params.olh_epoch = params.olh_epoch; + parent_op.params.marker_version_id = params.marker_version_id; + parent_op.params.bilog_flags = params.bilog_flags; + parent_op.params.remove_objs = params.remove_objs; + parent_op.params.expiration_time = params.expiration_time; + parent_op.params.unmod_since = params.unmod_since; + parent_op.params.mtime = params.mtime; + parent_op.params.high_precision_time = params.high_precision_time; + parent_op.params.zones_trace = params.zones_trace; + parent_op.params.abortmp = params.abortmp; + parent_op.params.parts_accounted_size = params.parts_accounted_size; + + int ret = parent_op.delete_obj(dpp); + if (ret < 0) + return ret; + + result.delete_marker = parent_op.result.delete_marker; + result.version_id = parent_op.result.version_id; + + return ret; + } + + int DBObject::delete_object(const DoutPrefixProvider* dpp, RGWObjectCtx* obj_ctx, optional_yield y, bool prevent_versioning) + { + DB::Object del_target(store->getDB(), bucket->get_info(), *obj_ctx, get_obj()); + DB::Object::Delete del_op(&del_target); + + del_op.params.bucket_owner = bucket->get_info().owner; + del_op.params.versioning_status = bucket->get_info().versioning_status(); + + return del_op.delete_obj(dpp); + } + + int DBObject::delete_obj_aio(const DoutPrefixProvider* dpp, RGWObjState* astate, + Completions* aio, bool keep_index_consistent, + optional_yield y) + { + /* XXX: Make it async */ + return 0; + } + + int DBObject::copy_object(RGWObjectCtx& obj_ctx, + User* user, + req_info* info, + const rgw_zone_id& source_zone, + rgw::sal::Object* dest_object, + rgw::sal::Bucket* dest_bucket, + rgw::sal::Bucket* src_bucket, + const rgw_placement_rule& dest_placement, + ceph::real_time* src_mtime, + ceph::real_time* mtime, + const ceph::real_time* mod_ptr, + const ceph::real_time* unmod_ptr, + bool high_precision_time, + const char* if_match, + const char* if_nomatch, + AttrsMod attrs_mod, + bool copy_if_newer, + Attrs& attrs, + RGWObjCategory category, + uint64_t olh_epoch, + boost::optional delete_at, + std::string* version_id, + std::string* tag, + std::string* etag, + void (*progress_cb)(off_t, void *), + void* progress_data, + const DoutPrefixProvider* dpp, + optional_yield y) + { + return 0; } + int DBObject::DBReadOp::iterate(const DoutPrefixProvider* dpp, int64_t ofs, int64_t end, RGWGetDataCB* cb, optional_yield y) + { + return parent_op.iterate(dpp, ofs, end, cb); + } + + int DBObject::swift_versioning_restore(RGWObjectCtx* obj_ctx, + bool& restored, + const DoutPrefixProvider* dpp) + { + return 0; + } + + int DBObject::swift_versioning_copy(RGWObjectCtx* obj_ctx, + const DoutPrefixProvider* dpp, + optional_yield y) + { + return 0; + } + + DBAtomicWriter::DBAtomicWriter(const DoutPrefixProvider *dpp, + optional_yield y, + std::unique_ptr _head_obj, + DBStore* _store, + const rgw_user& _owner, RGWObjectCtx& obj_ctx, + const rgw_placement_rule *_ptail_placement_rule, + uint64_t _olh_epoch, + const std::string& _unique_tag) : + Writer(dpp, y), + store(_store), + owner(_owner), + ptail_placement_rule(_ptail_placement_rule), + olh_epoch(_olh_epoch), + unique_tag(_unique_tag), + obj(_store, _head_obj->get_key(), _head_obj->get_bucket()), + op_target(_store->getDB(), obj.get_bucket()->get_info(), obj.get_obj()), + parent_op(&op_target) {} + + int DBAtomicWriter::prepare(optional_yield y) + { + return parent_op.prepare(NULL); /* send dpp */ + } + + int DBAtomicWriter::process(bufferlist&& data, uint64_t offset) + { + total_data_size += data.length(); + + /* XXX: Optimize all bufferlist copies in this function */ + + /* copy head_data into meta. */ + uint64_t head_size = store->getDB()->get_max_head_size(); + unsigned head_len = 0; + uint64_t max_chunk_size = store->getDB()->get_max_chunk_size(); + int excess_size = 0; + + /* Accumulate tail_data till max_chunk_size or flush op */ + bufferlist tail_data; + + if (data.length() != 0) { + if (offset < head_size) { + /* XXX: handle case (if exists) where offset > 0 & < head_size */ + head_len = std::min((uint64_t)data.length(), + head_size - offset); + bufferlist tmp; + data.begin(0).copy(head_len, tmp); + head_data.append(tmp); + + parent_op.meta.data = &head_data; + if (head_len == data.length()) { + return 0; + } + + /* Move offset by copy_len */ + offset = head_len; + } + + /* handle tail parts. + * First accumulate and write data into dbstore in its chunk_size + * parts + */ + if (!tail_part_size) { /* new tail part */ + tail_part_offset = offset; + } + data.begin(head_len).copy(data.length() - head_len, tail_data); + tail_part_size += tail_data.length(); + tail_part_data.append(tail_data); + + if (tail_part_size < max_chunk_size) { + return 0; + } else { + int write_ofs = 0; + while (tail_part_size >= max_chunk_size) { + excess_size = tail_part_size - max_chunk_size; + bufferlist tmp; + tail_part_data.begin(write_ofs).copy(max_chunk_size, tmp); + /* write tail objects data */ + int ret = parent_op.write_data(dpp, tmp, tail_part_offset); + + if (ret < 0) { + return ret; + } + + tail_part_size -= max_chunk_size; + write_ofs += max_chunk_size; + tail_part_offset += max_chunk_size; + } + /* reset tail parts or update if excess data */ + if (excess_size > 0) { /* wrote max_chunk_size data */ + tail_part_size = excess_size; + bufferlist tmp; + tail_part_data.begin(write_ofs).copy(excess_size, tmp); + tail_part_data = tmp; + } else { + tail_part_size = 0; + tail_part_data.clear(); + tail_part_offset = 0; + } + } + } else { + if (tail_part_size == 0) { + return 0; /* nothing more to write */ + } + + /* flush watever tail data is present */ + int ret = parent_op.write_data(dpp, tail_part_data, tail_part_offset); + if (ret < 0) { + return ret; + } + tail_part_size = 0; + tail_part_data.clear(); + tail_part_offset = 0; + } + + return 0; + } + + int DBAtomicWriter::complete(size_t accounted_size, const std::string& etag, + ceph::real_time *mtime, ceph::real_time set_mtime, + std::map& attrs, + ceph::real_time delete_at, + const char *if_match, const char *if_nomatch, + const std::string *user_data, + rgw_zone_set *zones_trace, bool *canceled, + optional_yield y) + { + parent_op.meta.mtime = mtime; + parent_op.meta.delete_at = delete_at; + parent_op.meta.if_match = if_match; + parent_op.meta.if_nomatch = if_nomatch; + parent_op.meta.user_data = user_data; + parent_op.meta.zones_trace = zones_trace; + + /* XXX: handle accounted size */ + accounted_size = total_data_size; + int ret = parent_op.write_meta(dpp, total_data_size, accounted_size, attrs); + if (canceled) { + *canceled = parent_op.meta.canceled; + } + + return ret; + + } std::unique_ptr DBStore::get_role(std::string name, std::string tenant, @@ -481,12 +961,14 @@ namespace rgw::sal { const rgw_placement_rule *ptail_placement_rule, uint64_t olh_epoch, const std::string& unique_tag) { - return nullptr; + return std::make_unique(dpp, y, + std::move(_head_obj), this, owner, obj_ctx, + ptail_placement_rule, olh_epoch, unique_tag); } std::unique_ptr DBStore::get_user(const rgw_user &u) { - return std::unique_ptr(new DBUser(this, u)); + return std::make_unique(this, u); } int DBStore::get_user_by_access_key(const DoutPrefixProvider *dpp, const std::string& key, optional_yield y, std::unique_ptr* user) @@ -545,7 +1027,7 @@ namespace rgw::sal { std::unique_ptr DBStore::get_object(const rgw_obj_key& k) { - return NULL; + return std::make_unique(this, k); } @@ -633,7 +1115,7 @@ namespace rgw::sal { return -EEXIST; }*/ } else { - bucket = std::unique_ptr(new DBBucket(this, b, u)); + bucket = std::make_unique(this, b, u); *existed = false; bucket->set_attrs(attrs); // XXX: For now single default zone and STANDARD storage class @@ -732,7 +1214,7 @@ namespace rgw::sal { struct req_state* s, rgw::notify::EventType event_type, const std::string* object_name) { - return 0; + return std::make_unique(obj, event_type); } int DBStore::log_usage(const DoutPrefixProvider *dpp, map& usage_info) @@ -875,6 +1357,7 @@ extern "C" { store->setDBStoreManager(dbsm); store->setDB(db); + db->set_store((rgw::sal::Store*)store); } return store; diff --git a/src/rgw/rgw_sal_dbstore.h b/src/rgw/rgw_sal_dbstore.h index 5d417b5defa..2f3689fa663 100644 --- a/src/rgw/rgw_sal_dbstore.h +++ b/src/rgw/rgw_sal_dbstore.h @@ -26,6 +26,20 @@ namespace rgw { namespace sal { class DBStore; +class DBNotification : public Notification { +protected: + Object* obj; + rgw::notify::EventType event_type; + + public: + DBNotification(Object* _obj, rgw::notify::EventType _type) : Notification(_obj, _type), obj(_obj), event_type(_type) {} + ~DBNotification() = default; + + virtual int publish_reserve(const DoutPrefixProvider *dpp, RGWObjTags* obj_tags = nullptr) override { return 0;} + virtual int publish_commit(const DoutPrefixProvider* dpp, uint64_t size, + const ceph::real_time& mtime, const std::string& etag, const std::string& version) override { return 0; } +}; + class DBUser : public User { private: DBStore *store; @@ -245,6 +259,179 @@ namespace rgw { namespace sal { } }; + class DBObject : public Object { + private: + DBStore* store; + RGWAccessControlPolicy acls; + /* XXX: to be removed. Till Dan's patch comes, a placeholder + * for RGWObjState + */ + RGWObjState* state; + + public: + struct DBReadOp : public ReadOp { + private: + DBObject* source; + RGWObjectCtx* rctx; + DB::Object op_target; + DB::Object::Read parent_op; + + public: + DBReadOp(DBObject *_source, RGWObjectCtx *_rctx); + + virtual int prepare(optional_yield y, const DoutPrefixProvider* dpp) override; + virtual int read(int64_t ofs, int64_t end, bufferlist& bl, optional_yield y, const DoutPrefixProvider* dpp) override; + virtual int iterate(const DoutPrefixProvider* dpp, int64_t ofs, int64_t end, RGWGetDataCB* cb, optional_yield y) override; + virtual int get_attr(const DoutPrefixProvider* dpp, const char* name, bufferlist& dest, optional_yield y) override; + }; + + struct DBDeleteOp : public DeleteOp { + private: + DBObject* source; + RGWObjectCtx* rctx; + DB::Object op_target; + DB::Object::Delete parent_op; + + public: + DBDeleteOp(DBObject* _source, RGWObjectCtx* _rctx); + + virtual int delete_obj(const DoutPrefixProvider* dpp, optional_yield y) override; + }; + + DBObject() = default; + + DBObject(DBStore *_st, const rgw_obj_key& _k) + : Object(_k), + store(_st), + acls() { + } + DBObject(DBStore *_st, const rgw_obj_key& _k, Bucket* _b) + : Object(_k, _b), + store(_st), + acls() { + } + DBObject(DBObject& _o) = default; + + virtual int delete_object(const DoutPrefixProvider* dpp, + RGWObjectCtx* obj_ctx, + optional_yield y, + bool prevent_versioning = false) override; + virtual int delete_obj_aio(const DoutPrefixProvider* dpp, RGWObjState* astate, Completions* aio, + bool keep_index_consistent, optional_yield y) override; + virtual int copy_object(RGWObjectCtx& obj_ctx, User* user, + req_info* info, const rgw_zone_id& source_zone, + rgw::sal::Object* dest_object, rgw::sal::Bucket* dest_bucket, + rgw::sal::Bucket* src_bucket, + const rgw_placement_rule& dest_placement, + ceph::real_time* src_mtime, ceph::real_time* mtime, + const ceph::real_time* mod_ptr, const ceph::real_time* unmod_ptr, + bool high_precision_time, + const char* if_match, const char* if_nomatch, + AttrsMod attrs_mod, bool copy_if_newer, Attrs& attrs, + RGWObjCategory category, uint64_t olh_epoch, + boost::optional delete_at, + std::string* version_id, std::string* tag, std::string* etag, + void (*progress_cb)(off_t, void *), void* progress_data, + const DoutPrefixProvider* dpp, optional_yield y) override; + virtual RGWAccessControlPolicy& get_acl(void) override { return acls; } + virtual int set_acl(const RGWAccessControlPolicy& acl) override { acls = acl; return 0; } + virtual void set_atomic(RGWObjectCtx* rctx) const override; + virtual void set_prefetch_data(RGWObjectCtx* rctx) override; + + virtual int get_obj_state(const DoutPrefixProvider* dpp, RGWObjectCtx* rctx, RGWObjState **state, optional_yield y, bool follow_olh = true) override; + virtual int set_obj_attrs(const DoutPrefixProvider* dpp, RGWObjectCtx* rctx, Attrs* setattrs, Attrs* delattrs, optional_yield y, rgw_obj* target_obj = NULL) override; + virtual int get_obj_attrs(RGWObjectCtx* rctx, optional_yield y, const DoutPrefixProvider* dpp, rgw_obj* target_obj = NULL) override; + virtual int modify_obj_attrs(RGWObjectCtx* rctx, const char* attr_name, bufferlist& attr_val, optional_yield y, const DoutPrefixProvider* dpp) override; + virtual int delete_obj_attrs(const DoutPrefixProvider* dpp, RGWObjectCtx* rctx, const char* attr_name, optional_yield y) override; + virtual int copy_obj_data(RGWObjectCtx& rctx, Bucket* dest_bucket, Object* dest_obj, uint16_t olh_epoch, std::string* petag, const DoutPrefixProvider* dpp, optional_yield y) override; + virtual bool is_expired() override; + virtual void gen_rand_obj_instance_name() override; + virtual std::unique_ptr clone() override { + return std::unique_ptr(new DBObject(*this)); + } + virtual MPSerializer* get_serializer(const DoutPrefixProvider *dpp, const std::string& lock_name) override; + virtual int transition(RGWObjectCtx& rctx, + Bucket* bucket, + const rgw_placement_rule& placement_rule, + const real_time& mtime, + uint64_t olh_epoch, + const DoutPrefixProvider* dpp, + optional_yield y) override; + virtual bool placement_rules_match(rgw_placement_rule& r1, rgw_placement_rule& r2) override; + virtual int get_obj_layout(const DoutPrefixProvider *dpp, optional_yield y, Formatter* f, RGWObjectCtx* obj_ctx) override; + + /* Swift versioning */ + virtual int swift_versioning_restore(RGWObjectCtx* obj_ctx, + bool& restored, + const DoutPrefixProvider* dpp) override; + virtual int swift_versioning_copy(RGWObjectCtx* obj_ctx, + const DoutPrefixProvider* dpp, + optional_yield y) override; + + /* OPs */ + virtual std::unique_ptr get_read_op(RGWObjectCtx *) override; + virtual std::unique_ptr get_delete_op(RGWObjectCtx*) override; + + /* OMAP */ + virtual int omap_get_vals(const DoutPrefixProvider *dpp, const std::string& marker, uint64_t count, + std::map *m, + bool* pmore, optional_yield y) override; + virtual int omap_get_all(const DoutPrefixProvider *dpp, std::map *m, + optional_yield y) override; + virtual int omap_get_vals_by_keys(const DoutPrefixProvider *dpp, const std::string& oid, + const std::set& keys, + Attrs* vals) override; + virtual int omap_set_val_by_key(const DoutPrefixProvider *dpp, const std::string& key, bufferlist& val, + bool must_exist, optional_yield y) override; + private: + int read_attrs(const DoutPrefixProvider* dpp, DB::Object::Read &read_op, optional_yield y, rgw_obj* target_obj = nullptr); + }; + + class DBAtomicWriter : public Writer { + protected: + rgw::sal::DBStore* store; + const rgw_user& owner; + const rgw_placement_rule *ptail_placement_rule; + uint64_t olh_epoch; + const std::string& unique_tag; + DBObject obj; + DB::Object op_target; + DB::Object::Write parent_op; + uint64_t total_data_size = 0; /* for total data being uploaded */ + bufferlist head_data; + bufferlist tail_part_data; + uint64_t tail_part_offset; + uint64_t tail_part_size = 0; /* corresponds to each tail part being + written to dbstore */ + + public: + DBAtomicWriter(const DoutPrefixProvider *dpp, + optional_yield y, + std::unique_ptr _head_obj, + DBStore* _store, + const rgw_user& _owner, RGWObjectCtx& obj_ctx, + const rgw_placement_rule *_ptail_placement_rule, + uint64_t _olh_epoch, + const std::string& _unique_tag); + ~DBAtomicWriter() = default; + + // prepare to start processing object data + virtual int prepare(optional_yield y) override; + + // Process a bufferlist + virtual int process(bufferlist&& data, uint64_t offset) override; + + // complete the operation and make its result visible to clients + virtual int complete(size_t accounted_size, const std::string& etag, + ceph::real_time *mtime, ceph::real_time set_mtime, + std::map& attrs, + ceph::real_time delete_at, + const char *if_match, const char *if_nomatch, + const std::string *user_data, + rgw_zone_set *zones_trace, bool *canceled, + optional_yield y) override; + }; + class DBStore : public Store { private: /* DBStoreManager is used in case multiple diff --git a/src/rgw/rgw_sal_rados.cc b/src/rgw/rgw_sal_rados.cc index ae11e36a6d6..5ce3b2f0009 100644 --- a/src/rgw/rgw_sal_rados.cc +++ b/src/rgw/rgw_sal_rados.cc @@ -1484,28 +1484,6 @@ int RadosStore::get_obj_head_ioctx(const DoutPrefixProvider *dpp, const RGWBucke return rados->get_obj_head_ioctx(dpp, bucket_info, obj, ioctx); } -int Object::range_to_ofs(uint64_t obj_size, int64_t &ofs, int64_t &end) -{ - if (ofs < 0) { - ofs += obj_size; - if (ofs < 0) - ofs = 0; - end = obj_size - 1; - } else if (end < 0) { - end = obj_size - 1; - } - - if (obj_size > 0) { - if (ofs >= (off_t)obj_size) { - return -ERANGE; - } - if (end >= (off_t)obj_size) { - end = obj_size - 1; - } - } - return 0; -} - RadosObject::~RadosObject() {} int RadosObject::get_obj_state(const DoutPrefixProvider* dpp, RGWObjectCtx* rctx, RGWObjState **state, optional_yield y, bool follow_olh) diff --git a/src/rgw/store/dbstore/common/dbstore.cc b/src/rgw/store/dbstore/common/dbstore.cc index ae2fcbdf9c1..7cf8c44413b 100644 --- a/src/rgw/store/dbstore/common/dbstore.cc +++ b/src/rgw/store/dbstore/common/dbstore.cc @@ -159,12 +159,16 @@ DBOp *DB::getDBOp(const DoutPrefixProvider *dpp, string Op, struct DBOpParams *p Ob = iter->second; - if (!Op.compare("InsertObject")) - return Ob->InsertObject; - if (!Op.compare("RemoveObject")) - return Ob->RemoveObject; - if (!Op.compare("ListObject")) - return Ob->ListObject; + if (!Op.compare("PutObject")) + return Ob->PutObject; + if (!Op.compare("DeleteObject")) + return Ob->DeleteObject; + if (!Op.compare("GetObject")) + return Ob->GetObject; + if (!Op.compare("UpdateObject")) + return Ob->UpdateObject; + if (!Op.compare("ListBucketObjects")) + return Ob->ListBucketObjects; if (!Op.compare("PutObjectData")) return Ob->PutObjectData; if (!Op.compare("GetObjectData")) @@ -193,7 +197,7 @@ int DB::objectmapInsert(const DoutPrefixProvider *dpp, string bucket, void *ptr) } Ob = (class ObjectOp*) ptr; - Ob->InitializeObjectOps(dpp); + Ob->InitializeObjectOps(getDBname(), dpp); DB::objectmap.insert(pair(bucket, Ob)); @@ -717,5 +721,1018 @@ out: return ret; } +int DB::Bucket::List::list_objects(const DoutPrefixProvider *dpp, int64_t max, + vector *result, + map *common_prefixes, bool *is_truncated) +{ + int ret = 0; + DB *store = target->get_store(); + + DBOpParams db_params = {}; + store->InitializeParams(dpp, "ListBucketObjects", &db_params); + + db_params.op.bucket.info = target->get_bucket_info(); + /* XXX: Handle whole marker? key -> name, instance, ns? */ + db_params.op.obj.min_marker = params.marker.name; + db_params.op.obj.max_marker = params.end_marker.name; + db_params.op.list_max_count = max + 1; /* +1 for next_marker */ + + ret = store->ProcessOp(dpp, "ListBucketObjects", &db_params); + + if (ret) { + ldpp_dout(dpp, 0)<<"In ListBucketObjects failed err:(" <= (uint64_t)max) { + *is_truncated = true; + next_marker.name = db_params.op.obj.list_entries.back().key.name; + next_marker.instance = db_params.op.obj.list_entries.back().key.instance; + db_params.op.obj.list_entries.pop_back(); + } + + for (auto& entry : db_params.op.obj.list_entries) { + if (!params.end_marker.name.empty() && + params.end_marker.name.compare(entry.key.name) <= 0) { + *is_truncated = false; + break; + } + result->push_back(std::move(entry)); + } +out: + return ret; +} + +int DB::raw_obj::InitializeParamsfromRawObj(const DoutPrefixProvider *dpp, + DBOpParams* params) { + int ret = 0; + + if (!params) + return -1; + + params->object_table = obj_table; + params->objectdata_table = obj_data_table; + params->op.bucket.info.bucket.name = bucket_name; + params->op.obj.state.obj.key.name = obj_name; + params->op.obj.state.obj.key.instance = obj_instance; + params->op.obj.state.obj.key.ns = obj_ns; + + if (multipart_partnum != 0) { + params->op.obj.is_multipart = true; + } else { + params->op.obj.is_multipart = false; + } + + params->op.obj_data.multipart_part_num = multipart_partnum; + params->op.obj_data.part_num = part_num; + + return ret; +} + +int DB::Object::InitializeParamsfromObject(const DoutPrefixProvider *dpp, + DBOpParams* params) { + int ret = 0; + string bucket = bucket_info.bucket.name; + + if (!params) + return -1; + + params->object_table = store->getObjectTable(bucket); + params->objectdata_table = store->getObjectDataTable(bucket); + params->op.bucket.info.bucket.name = bucket; + params->op.obj.state.obj = obj; + + return ret; +} + +int DB::Object::obj_omap_set_val_by_key(const DoutPrefixProvider *dpp, + const std::string& key, bufferlist& val, + bool must_exist) { + int ret = 0; + + DBOpParams params = {}; + + store->InitializeParams(dpp, "GetObject", ¶ms); + InitializeParamsfromObject(dpp, ¶ms); + + ret = store->ProcessOp(dpp, "GetObject", ¶ms); + + if (ret) { + ldpp_dout(dpp, 0) <<"In GetObject failed err:(" <ProcessOp(dpp, "UpdateObject", ¶ms); + + if (ret) { + ldpp_dout(dpp, 0)<<"In UpdateObject failed err:(" <& keys, + std::map* vals) +{ + int ret = 0; + DBOpParams params = {}; + std::map omap; + + if (!vals) + return -1; + + store->InitializeParams(dpp, "GetObject", ¶ms); + InitializeParamsfromObject(dpp, ¶ms); + + ret = store->ProcessOp(dpp, "GetObject", ¶ms); + + if (ret) { + ldpp_dout(dpp, 0) <<"In GetObject failed err:(" <set_instance(buf); +} + +int DB::Object::obj_omap_get_all(const DoutPrefixProvider *dpp, + std::map *m) +{ + int ret = 0; + DBOpParams params = {}; + std::map omap; + + if (!m) + return -1; + + store->InitializeParams(dpp, "GetObject", ¶ms); + InitializeParamsfromObject(dpp, ¶ms); + + ret = store->ProcessOp(dpp, "GetObject", ¶ms); + + if (ret) { + ldpp_dout(dpp, 0)<<"In GetObject failed err:(" < *m, bool* pmore) +{ + int ret = 0; + DBOpParams params = {}; + std::map omap; + map::iterator iter; + uint64_t count = 0; + + if (!m) + return -1; + + store->InitializeParams(dpp, "GetObject", ¶ms); + InitializeParamsfromObject(dpp, ¶ms); + + ret = store->ProcessOp(dpp, "GetObject", ¶ms); + + if (ret) { + ldpp_dout(dpp, 0)<<"In GetObject failed err:(" <first < marker) + continue; + + if ((++count) > max_count) { + *pmore = true; + break; + } + + (*m)[iter->first] = iter->second; + } + +out: + return ret; +} + +int DB::Object::set_attrs(const DoutPrefixProvider *dpp, + map& setattrs, + map* rmattrs) +{ + int ret = 0; + + DBOpParams params = {}; + rgw::sal::Attrs *attrs; + map::iterator iter; + + store->InitializeParams(dpp, "GetObject", ¶ms); + InitializeParamsfromObject(dpp, ¶ms); + + ret = store->ProcessOp(dpp, "GetObject", ¶ms); + + if (ret) { + ldpp_dout(dpp, 0) <<"In GetObject failed err:(" <begin(); iter != rmattrs->end(); ++iter) { + (*attrs).erase(iter->first); + } + } + for (iter = setattrs.begin(); iter != setattrs.end(); ++iter) { + (*attrs)[iter->first] = iter->second; + } + + params.op.query_str = "attrs"; + params.op.obj.state.mtime = real_clock::now(); + + ret = store->ProcessOp(dpp, "UpdateObject", ¶ms); + + if (ret) { + ldpp_dout(dpp, 0)<<"In UpdateObject failed err:(" <InitializeParams(dpp, "GetObjectData", ¶ms); + InitializeParamsfromRawObj(dpp, ¶ms); + + ret = db->ProcessOp(dpp, "GetObjectData", ¶ms); + + if (ret) { + ldpp_dout(dpp, 0)<<"In GetObjectData failed err:(" <InitializeParams(dpp, "PutObjectData", ¶ms); + InitializeParamsfromRawObj(dpp, ¶ms); + + /* XXX: Check for chunk_size ?? */ + params.op.obj_data.offset = ofs; + unsigned write_len = std::min((uint64_t)bl.length() - write_ofs, len); + bl.begin(write_ofs).copy(write_len, params.op.obj_data.data); + params.op.obj_data.size = params.op.obj_data.data.length(); + + ret = db->ProcessOp(dpp, "PutObjectData", ¶ms); + + if (ret) { + ldpp_dout(dpp, 0)<<"In PutObjectData failed err:(" <attrset.find(RGW_ATTR_OLH_INFO); + if (iter == state->attrset.end()) { + return -EINVAL; + } + + DBOLHInfo olh; + string s; + const bufferlist& bl = iter->second; + try { + auto biter = bl.cbegin(); + decode(olh, biter); + } catch (buffer::error& err) { + return -EIO; + } + + if (olh.removed) { + return -ENOENT; + } + + *target = olh.target; + + return 0; +} + +int DB::Object::get_olh_target_state(const DoutPrefixProvider *dpp, + const RGWBucketInfo& bucket_info, const rgw_obj& obj, + RGWObjState* olh_state, RGWObjState** target) +{ + int ret = 0; + rgw_obj target_obj; + + if (!olh_state->is_olh) { + return EINVAL; + } + + ret = follow_olh(dpp, bucket_info, olh_state, obj, &target_obj); /* might return -EAGAIN */ + if (ret < 0) { + ldpp_dout(dpp, 0)<<"In get_olh_target_state follow_olh() failed err:(" <InitializeParams(dpp, "GetObject", ¶ms); + InitializeParamsfromObject(dpp, ¶ms); + + ret = store->ProcessOp(dpp, "GetObject", ¶ms); + + if (ret) { + ldpp_dout(dpp, 0)<<"In GetObject failed err:(" <manifest); + + return 0; +} + +int DB::Object::Read::get_attr(const DoutPrefixProvider *dpp, const char *name, bufferlist& dest) +{ + RGWObjState base_state; + RGWObjState *state = &base_state; + int r = source->get_state(dpp, &state, true); + if (r < 0) + return r; + if (!state->exists) + return -ENOENT; + if (!state->get_attr(name, dest)) + return -ENODATA; + + return 0; +} + +int DB::Object::Read::prepare(const DoutPrefixProvider *dpp) +{ + DB *store = source->get_store(); + CephContext *cct = store->ctx(); + + bufferlist etag; + + map::iterator iter; + + RGWObjState base_state; + RGWObjState *astate = &base_state; + int r = source->get_state(dpp, &astate, true); + if (r < 0) + return r; + + if (!astate->exists) { + return -ENOENT; + } + + state.obj = astate->obj; + + if (params.target_obj) { + *params.target_obj = state.obj; + } + if (params.attrs) { + *params.attrs = astate->attrset; + if (cct->_conf->subsys.should_gather()) { + for (iter = params.attrs->begin(); iter != params.attrs->end(); ++iter) { + ldpp_dout(dpp, 20) << "Read xattr rgw_rados: " << iter->first << dendl; + } + } + } + + if (conds.if_match || conds.if_nomatch) { + r = get_attr(dpp, RGW_ATTR_ETAG, etag); + if (r < 0) + return r; + + if (conds.if_match) { + string if_match_str = rgw_string_unquote(conds.if_match); + ldpp_dout(dpp, 10) << "ETag: " << string(etag.c_str(), etag.length()) << " " << " If-Match: " << if_match_str << dendl; + if (if_match_str.compare(0, etag.length(), etag.c_str(), etag.length()) != 0) { + return -ERR_PRECONDITION_FAILED; + } + } + + if (conds.if_nomatch) { + string if_nomatch_str = rgw_string_unquote(conds.if_nomatch); + ldpp_dout(dpp, 10) << "ETag: " << string(etag.c_str(), etag.length()) << " " << " If-NoMatch: " << if_nomatch_str << dendl; + if (if_nomatch_str.compare(0, etag.length(), etag.c_str(), etag.length()) == 0) { + return -ERR_NOT_MODIFIED; + } + } + } + + if (params.obj_size) + *params.obj_size = astate->size; + if (params.lastmod) + *params.lastmod = astate->mtime; + + return 0; +} + +int DB::Object::Read::range_to_ofs(uint64_t obj_size, int64_t &ofs, int64_t &end) +{ + if (ofs < 0) { + ofs += obj_size; + if (ofs < 0) + ofs = 0; + end = obj_size - 1; + } else if (end < 0) { + end = obj_size - 1; + } + + if (obj_size > 0) { + if (ofs >= (off_t)obj_size) { + return -ERANGE; + } + if (end >= (off_t)obj_size) { + end = obj_size - 1; + } + } + return 0; +} + +int DB::Object::Read::read(int64_t ofs, int64_t end, bufferlist& bl, const DoutPrefixProvider *dpp) +{ + DB *store = source->get_store(); + + uint64_t read_ofs = ofs; + uint64_t len, read_len; + + bufferlist read_bl; + uint64_t max_chunk_size = store->get_max_chunk_size(); + + RGWObjState base_state; + RGWObjState *astate = &base_state; + int r = source->get_state(dpp, &astate, true); + if (r < 0) + return r; + + if (!astate->exists) { + return -ENOENT; + } + + if (astate->size == 0) { + end = 0; + } else if (end >= (int64_t)astate->size) { + end = astate->size - 1; + } + + if (end < 0) + len = 0; + else + len = end - ofs + 1; + + + if (len > max_chunk_size) { + len = max_chunk_size; + } + + int head_data_size = astate->data.length(); + bool reading_from_head = (ofs < head_data_size); + + if (reading_from_head) { + if (astate) { // && astate->prefetch_data)? + if (!ofs && astate->data.length() >= len) { + bl = astate->data; + return bl.length(); + } + + if (ofs < astate->data.length()) { + unsigned copy_len = std::min((uint64_t)head_data_size - ofs, len); + astate->data.begin(ofs).copy(copy_len, bl); + return bl.length(); + } + } + } + + /* tail object */ + int part_num = (ofs / max_chunk_size); + /* XXX: Handle multipart_num */ + raw_obj read_obj(store, source->get_bucket_info().bucket.name, astate->obj.key.name, + astate->obj.key.instance, astate->obj.key.ns, 0, part_num); + + read_len = len; + + ldpp_dout(dpp, 20) << "dbstore->read obj-ofs=" << ofs << " read_ofs=" << read_ofs << " read_len=" << read_len << dendl; + + // read from non head object + r = read_obj.read(dpp, read_ofs, read_len, bl); + + if (r < 0) { + return r; + } + + return bl.length(); +} + +static int _get_obj_iterate_cb(const DoutPrefixProvider *dpp, + const DB::raw_obj& read_obj, off_t obj_ofs, + off_t len, bool is_head_obj, + RGWObjState *astate, void *arg) +{ + struct db_get_obj_data* d = static_cast(arg); + return d->store->get_obj_iterate_cb(dpp, read_obj, obj_ofs, len, + is_head_obj, astate, arg); +} + +int DB::get_obj_iterate_cb(const DoutPrefixProvider *dpp, + const raw_obj& read_obj, off_t obj_ofs, + off_t len, bool is_head_obj, + RGWObjState *astate, void *arg) +{ + struct db_get_obj_data* d = static_cast(arg); + bufferlist bl; + int r = 0; + + if (is_head_obj) { + bl = astate->data; + } else { + // read from non head object + raw_obj robj = read_obj; + /* read entire data. So pass offset as '0' & len as '-1' */ + r = robj.read(dpp, 0, -1, bl); + + if (r < 0) { + return r; + } + } + + unsigned read_ofs = 0, read_len = 0; + while (read_ofs < bl.length()) { + unsigned chunk_len = std::min((uint64_t)bl.length() - read_ofs, (uint64_t)len); + r = d->client_cb->handle_data(bl, read_ofs, chunk_len); + if (r < 0) + return r; + read_ofs += chunk_len; + read_len += chunk_len; + ldpp_dout(dpp, 20) << "dbstore->get_obj_iterate_cb obj-ofs=" << obj_ofs << " len=" << len << " chunk_len = " << chunk_len << " read_len = " << read_len << dendl; + } + + + d->offset += read_len; + + return read_len; +} + +int DB::Object::Read::iterate(const DoutPrefixProvider *dpp, int64_t ofs, int64_t end, RGWGetDataCB *cb) +{ + DB *store = source->get_store(); + const uint64_t chunk_size = store->get_max_chunk_size(); + + db_get_obj_data data(store, cb, ofs); + + int r = source->iterate_obj(dpp, source->get_bucket_info(), state.obj, + ofs, end, chunk_size, _get_obj_iterate_cb, &data); + if (r < 0) { + ldpp_dout(dpp, 0) << "iterate_obj() failed with " << r << dendl; + return r; + } + + return 0; +} + +int DB::Object::iterate_obj(const DoutPrefixProvider *dpp, + const RGWBucketInfo& bucket_info, const rgw_obj& obj, + off_t ofs, off_t end, uint64_t max_chunk_size, + iterate_obj_cb cb, void *arg) +{ + DB *store = get_store(); + uint64_t len; + RGWObjState base_state; + RGWObjState *astate = &base_state; + + int r = get_state(dpp, &astate, true); + if (r < 0) { + return r; + } + + if (!astate->exists) { + return -ENOENT; + } + + if (end < 0) + len = 0; + else + len = end - ofs + 1; + + /* XXX: Will it really help to store all parts info in astate like manifest in Rados? */ + int part_num = 0; + int head_data_size = astate->data.length(); + + while (ofs <= end && (uint64_t)ofs < astate->size) { + part_num = (ofs / max_chunk_size); + uint64_t read_len = std::min(len, max_chunk_size); + + /* XXX: Handle multipart_num */ + raw_obj read_obj(store, get_bucket_info().bucket.name, astate->obj.key.name, + astate->obj.key.instance, astate->obj.key.ns, 0, part_num); + bool reading_from_head = (ofs < head_data_size); + + r = cb(dpp, read_obj, ofs, read_len, reading_from_head, astate, arg); + if (r <= 0) { + return r; + } + /* r refers to chunk_len (no. of bytes) handled in cb */ + len -= r; + ofs += r; + } + + return 0; +} + +int DB::Object::Write::prepare(const DoutPrefixProvider* dpp) +{ + DB *store = target->get_store(); + + DBOpParams params = {}; + int ret = -1; + + /* XXX: handle assume_noent */ + store->InitializeParams(dpp, "GetObject", ¶ms); + target->InitializeParamsfromObject(dpp, ¶ms); + + ret = store->ProcessOp(dpp, "GetObject", ¶ms); + + if (ret) { + ldpp_dout(dpp, 0)<<"In GetObject failed err:(" <bucket_info.bucket.name << ", Object:"<< target->obj.key.name << ") exists" << dendl; + + } else { /* create object entry in the object table */ + params.op.obj.storage_class = "STANDARD"; /* XXX: handle storage class */ + ret = store->ProcessOp(dpp, "PutObject", ¶ms); + + if (ret) { + ldpp_dout(dpp, 0)<<"In PutObject failed err:(" <get_store(); + /* tail objects */ + /* XXX: Split into parts each of max_chunk_size. But later make tail + * object chunk size limit to sqlite blob limit */ + int part_num = 0; + uint64_t max_chunk_size, max_head_size; + + max_head_size = store->get_max_head_size(); + max_chunk_size = store->get_max_chunk_size(); + + /* tail_obj ofs should be greater than max_head_size */ + if (ofs < max_head_size) { + return -1; + } + + uint64_t end = data.length(); + uint64_t write_ofs = 0; + /* as we are writing max_chunk_size at a time in sal_dbstore DBAtomicWriter::process(), + * maybe this while loop is not needed + */ + while (write_ofs < end) { + part_num = (ofs / max_chunk_size); + uint64_t len = std::min(end, max_chunk_size); + + /* XXX: Handle multipart_num */ + raw_obj write_obj(store, target->get_bucket_info().bucket.name, obj_state.obj.key.name, + obj_state.obj.key.instance, obj_state.obj.key.ns, 0, part_num); + + + ldpp_dout(dpp, 20) << "dbstore->write obj-ofs=" << ofs << " write_len=" << len << dendl; + + // write into non head object + int r = write_obj.write(dpp, ofs, write_ofs, len, data); + if (r < 0) { + return r; + } + /* r refers to chunk_len (no. of bytes) handled in raw_obj::write */ + len -= r; + ofs += r; + write_ofs += r; + } + + return 0; +} + +/* Write metadata & head object data */ +int DB::Object::Write::_do_write_meta(const DoutPrefixProvider *dpp, + uint64_t size, uint64_t accounted_size, + map& attrs, + bool assume_noent, bool modify_tail) +{ + DB *store = target->get_store(); + + RGWObjState *state = &obj_state; + map *attrset; + DBOpParams params = {}; + int ret = 0; + string etag; + string content_type; + bufferlist acl_bl; + string storage_class; + + map::iterator iter; + + store->InitializeParams(dpp, "PutObject", ¶ms); + target->InitializeParamsfromObject(dpp, ¶ms); + + obj_state = params.op.obj.state; + + if (real_clock::is_zero(meta.set_mtime)) { + meta.set_mtime = real_clock::now(); + } + + attrset = &state->attrset; + if (target->bucket_info.obj_lock_enabled() && target->bucket_info.obj_lock.has_rule()) { + // && meta.flags == PUT_OBJ_CREATE) { + auto iter = attrs.find(RGW_ATTR_OBJECT_RETENTION); + if (iter == attrs.end()) { + real_time lock_until_date = target->bucket_info.obj_lock.get_lock_until_date(meta.set_mtime); + string mode = target->bucket_info.obj_lock.get_mode(); + RGWObjectRetention obj_retention(mode, lock_until_date); + bufferlist bl; + obj_retention.encode(bl); + (*attrset)[RGW_ATTR_OBJECT_RETENTION] = bl; + } + } + + if (state->is_olh) { + (*attrset)[RGW_ATTR_OLH_ID_TAG] = state->olh_tag; + } + + state->mtime = meta.set_mtime; + + if (meta.data) { + /* if we want to overwrite the data, we also want to overwrite the + xattrs, so just remove the object */ + params.op.obj.head_data = *meta.data; + } + + if (meta.rmattrs) { + for (iter = meta.rmattrs->begin(); iter != meta.rmattrs->end(); ++iter) { + const string& name = iter->first; + (*attrset).erase(name.c_str()); + } + } + + if (meta.manifest) { + storage_class = meta.manifest->get_tail_placement().placement_rule.storage_class; + + /* remove existing manifest attr */ + iter = attrs.find(RGW_ATTR_MANIFEST); + if (iter != attrs.end()) + attrs.erase(iter); + + bufferlist bl; + encode(*meta.manifest, bl); + (*attrset)[RGW_ATTR_MANIFEST] = bl; + } + + for (iter = attrs.begin(); iter != attrs.end(); ++iter) { + const string& name = iter->first; + bufferlist& bl = iter->second; + + if (!bl.length()) + continue; + + (*attrset)[name.c_str()] = bl; + + if (name.compare(RGW_ATTR_ETAG) == 0) { + etag = rgw_bl_str(bl); + params.op.obj.etag = etag; + } else if (name.compare(RGW_ATTR_CONTENT_TYPE) == 0) { + content_type = rgw_bl_str(bl); + } else if (name.compare(RGW_ATTR_ACL) == 0) { + acl_bl = bl; + } + } + + if (!storage_class.empty()) { + bufferlist bl; + bl.append(storage_class); + (*attrset)[RGW_ATTR_STORAGE_CLASS] = bl; + } + + params.op.obj.state = *state ; + params.op.obj.state.exists = true; + params.op.obj.state.size = size; + params.op.obj.state.accounted_size = accounted_size; + params.op.obj.owner = target->get_bucket_info().owner.id; + + /* XXX: handle versioning */ + if (meta.mtime) { + *meta.mtime = meta.set_mtime; + } + + /* XXX: handle multipart */ + params.op.query_str = "meta"; + ret = store->ProcessOp(dpp, "UpdateObject", ¶ms); + + if (ret) { + ldpp_dout(dpp, 0)<<"In UpdateObject failed err:(" <InitializeParamsfromObject(dpp, &del_params); + + /* As it is cascade delete, it will delete the objectdata table entries also */ + ret = store->ProcessOp(dpp, "DeleteObject", &del_params); + if (ret) { + ldpp_dout(dpp, 0) << "In DeleteObject failed err:(" < list_entries; }; +struct DBOpObjectInfo { + RGWAccessControlPolicy acls; + RGWObjState state; + + /* Below are taken from rgw_bucket_dir_entry */ + RGWObjCategory category; + std::string etag; + std::string owner; + std::string owner_display_name; + std::string content_type; + std::string storage_class; + bool appendable; + uint64_t index_ver; + std::string tag; + uint16_t flags; + uint64_t versioned_epoch; + + /* from state.manifest (RGWObjManifest) */ + map objs; + uint64_t head_size{0}; + rgw_placement_rule head_placement_rule; + uint64_t max_head_size{0}; + string prefix; + rgw_bucket_placement tail_placement; /* might be different than the original bucket, + as object might have been copied across pools */ + map rules; + string tail_instance; /* tail object's instance */ + + + /* Obj's omap store */ + std::map omap; + + /* Extra fields */ + bool is_multipart; + bufferlist head_data; + string min_marker; + string max_marker; + list list_entries; +}; + +struct DBOpObjectDataInfo { + RGWObjState state; + uint64_t part_num; + uint64_t multipart_part_num; + uint64_t offset; + uint64_t size; + bufferlist data{}; +}; + struct DBOpInfo { string name; // Op name /* Support only single access_key for now. So store @@ -58,6 +108,8 @@ struct DBOpInfo { DBOpUserInfo user; string query_str; DBOpBucketInfo bucket; + DBOpObjectInfo obj; + DBOpObjectDataInfo obj_data; uint64_t list_max_count; }; @@ -75,10 +127,7 @@ struct DBOpParams { /* Below are subject to change */ string objectdata_table; string quota_table; - string object; - size_t offset; - string data; - size_t datalen; + string obj; }; /* Used for prepared schemas. @@ -162,30 +211,87 @@ struct DBOpBucketPrepareInfo { string max_marker = ":max_marker"; }; +struct DBOpObjectPrepareInfo { + string obj_name = ":obj_name"; + string obj_instance = ":obj_instance"; + string obj_ns = ":obj_ns"; + string acls = ":acls"; + string index_ver = ":index_ver"; + string tag = ":tag"; + string flags = ":flags"; + string versioned_epoch = ":versioned_epoch"; + string obj_category = ":obj_category"; + string etag = ":etag"; + string owner = ":owner"; + string owner_display_name = ":owner_display_name"; + string storage_class = ":storage_class"; + string appendable = ":appendable"; + string content_type = ":content_type"; + string index_hash_source = ":index_hash_source"; + string obj_size = ":obj_size"; + string accounted_size = ":accounted_size"; + string mtime = ":mtime"; + string epoch = ":epoch"; + string obj_tag = ":obj_tag"; + string tail_tag = ":tail_tag"; + string write_tag = ":write_tag"; + string fake_tag = ":fake_tag"; + string shadow_obj = ":shadow_obj"; + string has_data = ":has_data"; + string is_olh = ":is_ols"; + string olh_tag = ":olh_tag"; + string pg_ver = ":pg_ver"; + string zone_short_id = ":zone_short_id"; + string obj_version = ":obj_version"; + string obj_version_tag = ":obj_version_tag"; + string obj_attrs = ":obj_attrs"; + string head_size = ":head_size"; + string max_head_size = ":max_head_size"; + string prefix = ":prefix"; + string tail_instance = ":tail_instance"; + string head_placement_rule_name = ":head_placement_rule_name"; + string head_placement_storage_class = ":head_placement_storage_class"; + string tail_placement_rule_name = ":tail_placement_rule_name"; + string tail_placement_storage_class = ":tail_placement_storage_class"; + string manifest_part_objs = ":manifest_part_objs"; + string manifest_part_rules = ":manifest_part_rules"; + string omap = ":omap"; + string is_multipart = ":is_multipart"; + string head_data = ":head_data"; + string min_marker = ":min_marker"; + string max_marker = ":max_marker"; +}; + +struct DBOpObjectDataPrepareInfo { + string part_num = ":part_num"; + string offset = ":offset"; + string data = ":data"; + string size = ":size"; + string multipart_part_num = ":multipart_part_num"; +}; + struct DBOpPrepareInfo { DBOpUserPrepareInfo user; string query_str = ":query_str"; DBOpBucketPrepareInfo bucket; + DBOpObjectPrepareInfo obj; + DBOpObjectDataPrepareInfo obj_data; string list_max_count = ":list_max_count"; }; struct DBOpPrepareParams { /* Tables */ - string user_table = ":user_table"; - string bucket_table = ":bucket_table"; - string object_table = ":object_table"; + string user_table; + string bucket_table; + string object_table; /* Ops */ DBOpPrepareInfo op; /* below subject to change */ - string objectdata_table = ":objectdata_table"; - string quota_table = ":quota_table"; - string object = ":object"; - string offset = ":offset"; - string data = ":data"; - string datalen = ":datalen"; + string objectdata_table; + string quota_table; }; struct DBOps { @@ -205,21 +311,23 @@ class ObjectOp { virtual ~ObjectOp() {} - class InsertObjectOp *InsertObject; - class RemoveObjectOp *RemoveObject; - class ListObjectOp *ListObject; + class PutObjectOp *PutObject; + class DeleteObjectOp *DeleteObject; + class GetObjectOp *GetObject; + class UpdateObjectOp *UpdateObject; + class ListBucketObjectsOp *ListBucketObjects; class PutObjectDataOp *PutObjectData; class GetObjectDataOp *GetObjectData; class DeleteObjectDataOp *DeleteObjectData; - virtual int InitializeObjectOps(const DoutPrefixProvider *dpp) { return 0; } + virtual int InitializeObjectOps(string db_name, const DoutPrefixProvider *dpp) { return 0; } virtual int FreeObjectOps(const DoutPrefixProvider *dpp) { return 0; } }; class DBOp { private: const string CreateUserTableQ = - /* Corresponds to RGWUser + /* Corresponds to rgw::sal::User * * For now only UserID is made Primary key. * If multiple tenants are stored in single .db handle, should @@ -265,7 +373,7 @@ class DBOp { PRIMARY KEY (UserID) \n);"; const string CreateBucketTableQ = - /* Corresponds to RGWBucket + /* Corresponds to rgw::sal::Bucket * * For now only BucketName is made Primary key. * If multiple tenants are stored in single .db handle, should @@ -318,23 +426,112 @@ class DBOp { PRIMARY KEY (BucketName) \ FOREIGN KEY (OwnerID) \ REFERENCES '{}' (UserID) ON DELETE CASCADE ON UPDATE CASCADE \n);"; + const string CreateObjectTableQ = + /* Corresponds to rgw::sal::Object + * + * For now only BucketName, ObjName is made Primary key. + * If multiple tenants are stored in single .db handle, should + * include Tenant too in the Primary Key. Also should + * reference (BucketID, Tenant) as Foreign key. + * + * referring to + * - rgw_bucket_dir_entry - following are added for now + * flags, + * versioned_epoch + * tag + * index_ver + * meta.category + * meta.etag + * meta.storageclass + * meta.appendable + * meta.content_type + * meta.owner + * meta.owner_display_name + * + * - RGWObjState. Below are omitted from that struct + * as they seem in-memory variables + * * is_atomic, has_atts, exists, prefetch_data, keep_tail, + * - RGWObjManifest + * + * Extra field added "IsMultipart" to flag multipart uploads, + * HeadData to store first chunk data. + */ "CREATE TABLE IF NOT EXISTS '{}' ( \ + ObjName TEXT NOT NULL , \ + ObjInstance TEXT, \ + ObjNS TEXT, \ BucketName TEXT NOT NULL , \ - ObjectName TEXT NOT NULL , \ - PRIMARY KEY (BucketName, ObjectName), \ + ACLs BLOB, \ + IndexVer INTEGER, \ + Tag TEXT, \ + Flags INTEGER, \ + VersionedEpoch INTEGER, \ + ObjCategory INTEGER, \ + Etag TEXT, \ + Owner TEXT, \ + OwnerDisplayName TEXT, \ + StorageClass TEXT, \ + Appendable BOOL, \ + ContentType TEXT, \ + IndexHashSource TEXT, \ + ObjSize INTEGER, \ + AccountedSize INTEGER, \ + Mtime BLOB, \ + Epoch INTEGER, \ + ObjTag BLOB, \ + TailTag BLOB, \ + WriteTag TEXT, \ + FakeTag BOOL, \ + ShadowObj TEXT, \ + HasData BOOL, \ + IsOLH BOOL, \ + OLHTag BLOB, \ + PGVer INTEGER, \ + ZoneShortID INTEGER, \ + ObjVersion INTEGER, \ + ObjVersionTag TEXT, \ + ObjAttrs BLOB, \ + HeadSize INTEGER, \ + MaxHeadSize INTEGER, \ + Prefix String, \ + TailInstance String, \ + HeadPlacementRuleName String, \ + HeadPlacementRuleStorageClass String, \ + TailPlacementRuleName String, \ + TailPlacementStorageClass String, \ + ManifestPartObjs BLOB, \ + ManifestPartRules BLOB, \ + Omap BLOB, \ + IsMultipart BOOL, \ + HeadData BLOB, \ + PRIMARY KEY (ObjName, ObjInstance, BucketName), \ FOREIGN KEY (BucketName) \ REFERENCES '{}' (BucketName) ON DELETE CASCADE ON UPDATE CASCADE \n);"; + const string CreateObjectDataTableQ = + /* Extra field 'MultipartPartNum' added which signifies multipart upload + * part number. For regular object, it is '0' + * + * - part: a collection of stripes that make a contiguous part of an + object. A regular object will only have one part (although might have + many stripes), a multipart object might have many parts. Each part + has a fixed stripe size (ObjChunkSize), although the last stripe of a + part might be smaller than that. + */ "CREATE TABLE IF NOT EXISTS '{}' ( \ + ObjName TEXT NOT NULL , \ + ObjInstance TEXT, \ + ObjNS TEXT, \ BucketName TEXT NOT NULL , \ - ObjectName TEXT NOT NULL , \ - Offset INTEGER NOT NULL, \ + PartNum INTEGER NOT NULL, \ + Offset INTEGER, \ Data BLOB, \ - Size INTEGER NOT NULL, \ - PRIMARY KEY (BucketName, ObjectName, Offset), \ - FOREIGN KEY (BucketName, ObjectName) \ - REFERENCES '{}' (BucketName, ObjectName) ON DELETE CASCADE ON UPDATE CASCADE \n);"; + Size INTEGER, \ + MultipartPartNum INTEGER, \ + PRIMARY KEY (ObjName, BucketName, ObjInstance, MultipartPartNum, PartNum), \ + FOREIGN KEY (BucketName, ObjName, ObjInstance) \ + REFERENCES '{}' (BucketName, ObjName, ObjInstance) ON DELETE CASCADE ON UPDATE CASCADE \n);"; const string CreateQuotaTableQ = "CREATE TABLE IF NOT EXISTS '{}' ( \ @@ -658,55 +855,210 @@ class ListUserBucketsOp: public DBOp { } }; -class InsertObjectOp: public DBOp { +class PutObjectOp: public DBOp { private: const string Query = - "INSERT OR REPLACE INTO '{}' (BucketName, ObjectName) VALUES ({}, {})"; + "INSERT OR REPLACE INTO '{}' \ + (ObjName, ObjInstance, ObjNS, BucketName, ACLs, IndexVer, Tag, \ + Flags, VersionedEpoch, ObjCategory, Etag, Owner, OwnerDisplayName, \ + StorageClass, Appendable, ContentType, IndexHashSource, ObjSize, \ + AccountedSize, Mtime, Epoch, ObjTag, TailTag, WriteTag, FakeTag, \ + ShadowObj, HasData, IsOLH, OLHTag, PGVer, ZoneShortID, \ + ObjVersion, ObjVersionTag, ObjAttrs, HeadSize, MaxHeadSize, \ + Prefix, TailInstance, HeadPlacementRuleName, HeadPlacementRuleStorageClass, \ + TailPlacementRuleName, TailPlacementStorageClass, \ + ManifestPartObjs, ManifestPartRules, Omap, IsMultipart, HeadData ) \ + VALUES ({}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, \ + {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, \ + {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {})"; public: - virtual ~InsertObjectOp() {} + virtual ~PutObjectOp() {} string Schema(DBOpPrepareParams ¶ms) { return fmt::format(Query.c_str(), - params.object_table.c_str(), params.op.bucket.bucket_name.c_str(), - params.object.c_str()); + params.object_table.c_str(), params.op.obj.obj_name, + params.op.obj.obj_instance, params.op.obj.obj_ns, + params.op.bucket.bucket_name, params.op.obj.acls, params.op.obj.index_ver, + params.op.obj.tag, params.op.obj.flags, params.op.obj.versioned_epoch, + params.op.obj.obj_category, params.op.obj.etag, params.op.obj.owner, + params.op.obj.owner_display_name, params.op.obj.storage_class, + params.op.obj.appendable, params.op.obj.content_type, + params.op.obj.index_hash_source, params.op.obj.obj_size, + params.op.obj.accounted_size, params.op.obj.mtime, + params.op.obj.epoch, params.op.obj.obj_tag, params.op.obj.tail_tag, + params.op.obj.write_tag, params.op.obj.fake_tag, params.op.obj.shadow_obj, + params.op.obj.has_data, params.op.obj.is_olh, params.op.obj.olh_tag, + params.op.obj.pg_ver, params.op.obj.zone_short_id, + params.op.obj.obj_version, params.op.obj.obj_version_tag, + params.op.obj.obj_attrs, params.op.obj.head_size, + params.op.obj.max_head_size, params.op.obj.prefix, + params.op.obj.tail_instance, + params.op.obj.head_placement_rule_name, + params.op.obj.head_placement_storage_class, + params.op.obj.tail_placement_rule_name, + params.op.obj.tail_placement_storage_class, + params.op.obj.manifest_part_objs, + params.op.obj.manifest_part_rules, params.op.obj.omap, + params.op.obj.is_multipart, params.op.obj.head_data); } }; -class RemoveObjectOp: public DBOp { +class DeleteObjectOp: public DBOp { private: const string Query = - "DELETE from '{}' where BucketName = {} and ObjectName = {}"; + "DELETE from '{}' where BucketName = {} and ObjName = {} and ObjInstance = {}"; public: - virtual ~RemoveObjectOp() {} + virtual ~DeleteObjectOp() {} string Schema(DBOpPrepareParams ¶ms) { return fmt::format(Query.c_str(), params.object_table.c_str(), - params.op.bucket.bucket_name.c_str(), params.object.c_str()); + params.op.bucket.bucket_name.c_str(), + params.op.obj.obj_name.c_str(), + params.op.obj.obj_instance.c_str()); } }; -class ListObjectOp: public DBOp { +class GetObjectOp: public DBOp { private: const string Query = - "SELECT * from '{}' where BucketName = {} and ObjectName = {}"; - // XXX: Include queries for specific bucket and user too + "SELECT \ + ObjName, ObjInstance, ObjNS, BucketName, ACLs, IndexVer, Tag, \ + Flags, VersionedEpoch, ObjCategory, Etag, Owner, OwnerDisplayName, \ + StorageClass, Appendable, ContentType, IndexHashSource, ObjSize, \ + AccountedSize, Mtime, Epoch, ObjTag, TailTag, WriteTag, FakeTag, \ + ShadowObj, HasData, IsOLH, OLHTag, PGVer, ZoneShortID, \ + ObjVersion, ObjVersionTag, ObjAttrs, HeadSize, MaxHeadSize, \ + Prefix, TailInstance, HeadPlacementRuleName, HeadPlacementRuleStorageClass, \ + TailPlacementRuleName, TailPlacementStorageClass, \ + ManifestPartObjs, ManifestPartRules, Omap, IsMultipart, HeadData from '{}' \ + where BucketName = {} and ObjName = {} and ObjInstance = {}"; public: - virtual ~ListObjectOp() {} + virtual ~GetObjectOp() {} string Schema(DBOpPrepareParams ¶ms) { - return fmt::format(Query.c_str(), params.object_table.c_str(), - params.op.bucket.bucket_name.c_str(), params.object.c_str()); + return fmt::format(Query.c_str(), + params.object_table.c_str(), + params.op.bucket.bucket_name.c_str(), + params.op.obj.obj_name.c_str(), + params.op.obj.obj_instance.c_str()); + } +}; + +class ListBucketObjectsOp: public DBOp { + private: + // once we have stats also stored, may have to update this query to join + // these two tables. + const string Query = + "SELECT \ + ObjName, ObjInstance, ObjNS, BucketName, ACLs, IndexVer, Tag, \ + Flags, VersionedEpoch, ObjCategory, Etag, Owner, OwnerDisplayName, \ + StorageClass, Appendable, ContentType, IndexHashSource, ObjSize, \ + AccountedSize, Mtime, Epoch, ObjTag, TailTag, WriteTag, FakeTag, \ + ShadowObj, HasData, IsOLH, OLHTag, PGVer, ZoneShortID, \ + ObjVersion, ObjVersionTag, ObjAttrs, HeadSize, MaxHeadSize, \ + Prefix, TailInstance, HeadPlacementRuleName, HeadPlacementRuleStorageClass, \ + TailPlacementRuleName, TailPlacementStorageClass, \ + ManifestPartObjs, ManifestPartRules, Omap, IsMultipart, HeadData from '{}' \ + where BucketName = {} and ObjName > {} ORDER BY ObjName ASC LIMIT {}"; + public: + virtual ~ListBucketObjectsOp() {} + + string Schema(DBOpPrepareParams ¶ms) { + /* XXX: Include prefix, delim */ + return fmt::format(Query.c_str(), + params.object_table.c_str(), + params.op.bucket.bucket_name.c_str(), + params.op.obj.min_marker.c_str(), + params.op.list_max_count.c_str()); + } +}; + +class UpdateObjectOp: public DBOp { + private: + // Updates Omap + const string OmapQuery = + "UPDATE '{}' SET Omap = {}, Mtime = {} \ + where BucketName = {} and ObjName = {} and ObjInstance = {}"; + const string AttrsQuery = + "UPDATE '{}' SET ObjAttrs = {}, Mtime = {} \ + where BucketName = {} and ObjName = {} and ObjInstance = {}"; + const string MetaQuery = + "UPDATE '{}' SET \ + ObjNS = {}, ACLs = {}, IndexVer = {}, Tag = {}, Flags = {}, VersionedEpoch = {}, \ + ObjCategory = {}, Etag = {}, Owner = {}, OwnerDisplayName = {}, \ + StorageClass = {}, Appendable = {}, ContentType = {}, \ + IndexHashSource = {}, ObjSize = {}, AccountedSize = {}, Mtime = {}, \ + Epoch = {}, ObjTag = {}, TailTag = {}, WriteTag = {}, FakeTag = {}, \ + ShadowObj = {}, HasData = {}, IsOLH = {}, OLHTag = {}, PGVer = {}, \ + ZoneShortID = {}, ObjVersion = {}, ObjVersionTag = {}, ObjAttrs = {}, \ + HeadSize = {}, MaxHeadSize = {}, Prefix = {}, TailInstance = {}, \ + HeadPlacementRuleName = {}, HeadPlacementRuleStorageClass = {}, \ + TailPlacementRuleName = {}, TailPlacementStorageClass = {}, \ + ManifestPartObjs = {}, ManifestPartRules = {}, Omap = {}, \ + IsMultipart = {}, HeadData = {} \ + WHERE ObjName = {} and ObjInstance = {} and BucketName = {}"; + + public: + virtual ~UpdateObjectOp() {} + + string Schema(DBOpPrepareParams ¶ms) { + if (params.op.query_str == "omap") { + return fmt::format(OmapQuery.c_str(), + params.object_table.c_str(), params.op.obj.omap.c_str(), + params.op.obj.mtime.c_str(), + params.op.bucket.bucket_name.c_str(), + params.op.obj.obj_name.c_str(), + params.op.obj.obj_instance.c_str()); + } + if (params.op.query_str == "attrs") { + return fmt::format(AttrsQuery.c_str(), + params.object_table.c_str(), params.op.obj.obj_attrs.c_str(), + params.op.obj.mtime.c_str(), + params.op.bucket.bucket_name.c_str(), + params.op.obj.obj_name.c_str(), + params.op.obj.obj_instance.c_str()); + } + if (params.op.query_str == "meta") { + return fmt::format(MetaQuery.c_str(), + params.object_table.c_str(), + params.op.obj.obj_ns, params.op.obj.acls, params.op.obj.index_ver, + params.op.obj.tag, params.op.obj.flags, params.op.obj.versioned_epoch, + params.op.obj.obj_category, params.op.obj.etag, params.op.obj.owner, + params.op.obj.owner_display_name, params.op.obj.storage_class, + params.op.obj.appendable, params.op.obj.content_type, + params.op.obj.index_hash_source, params.op.obj.obj_size, + params.op.obj.accounted_size, params.op.obj.mtime, + params.op.obj.epoch, params.op.obj.obj_tag, params.op.obj.tail_tag, + params.op.obj.write_tag, params.op.obj.fake_tag, params.op.obj.shadow_obj, + params.op.obj.has_data, params.op.obj.is_olh, params.op.obj.olh_tag, + params.op.obj.pg_ver, params.op.obj.zone_short_id, + params.op.obj.obj_version, params.op.obj.obj_version_tag, + params.op.obj.obj_attrs, params.op.obj.head_size, + params.op.obj.max_head_size, params.op.obj.prefix, + params.op.obj.tail_instance, + params.op.obj.head_placement_rule_name, + params.op.obj.head_placement_storage_class, + params.op.obj.tail_placement_rule_name, + params.op.obj.tail_placement_storage_class, + params.op.obj.manifest_part_objs, + params.op.obj.manifest_part_rules, params.op.obj.omap, + params.op.obj.is_multipart, params.op.obj.head_data, + params.op.obj.obj_name, params.op.obj.obj_instance, + params.op.bucket.bucket_name); + } + return ""; } }; class PutObjectDataOp: public DBOp { private: const string Query = - "INSERT OR REPLACE INTO '{}' (BucketName, ObjectName, Offset, Data, Size) \ - VALUES ({}, {}, {}, {}, {})"; + "INSERT OR REPLACE INTO '{}' \ + (ObjName, ObjInstance, ObjNS, BucketName, PartNum, Offset, Data, Size, MultipartPartNum) \ + VALUES ({}, {}, {}, {}, {}, {}, {}, {}, {})"; public: virtual ~PutObjectDataOp() {} @@ -714,45 +1066,76 @@ class PutObjectDataOp: public DBOp { string Schema(DBOpPrepareParams ¶ms) { return fmt::format(Query.c_str(), params.objectdata_table.c_str(), - params.op.bucket.bucket_name.c_str(), params.object.c_str(), - params.offset.c_str(), params.data.c_str(), - params.datalen.c_str()); + params.op.obj.obj_name, params.op.obj.obj_instance, + params.op.obj.obj_ns, + params.op.bucket.bucket_name.c_str(), + params.op.obj_data.part_num, + params.op.obj_data.offset.c_str(), params.op.obj_data.data.c_str(), + params.op.obj_data.size, params.op.obj_data.multipart_part_num); } }; class GetObjectDataOp: public DBOp { private: const string Query = - "SELECT * from '{}' where BucketName = {} and ObjectName = {}"; + "SELECT \ + ObjName, ObjInstance, ObjNS, BucketName, PartNum, Offset, Data, Size, \ + MultipartPartNum from '{}' where BucketName = {} and ObjName = {} and ObjInstance = {}"; public: virtual ~GetObjectDataOp() {} string Schema(DBOpPrepareParams ¶ms) { return fmt::format(Query.c_str(), - params.objectdata_table.c_str(), params.op.bucket.bucket_name.c_str(), - params.object.c_str()); + params.objectdata_table.c_str(), + params.op.bucket.bucket_name.c_str(), + params.op.obj.obj_name.c_str(), + params.op.obj.obj_instance.c_str()); } }; class DeleteObjectDataOp: public DBOp { private: const string Query = - "DELETE from '{}' where BucketName = {} and ObjectName = {}"; + "DELETE from '{}' where BucketName = {} and ObjName = {} and ObjInstance = {}"; public: virtual ~DeleteObjectDataOp() {} string Schema(DBOpPrepareParams ¶ms) { return fmt::format(Query.c_str(), - params.objectdata_table.c_str(), params.op.bucket.bucket_name.c_str(), - params.object.c_str()); + params.objectdata_table.c_str(), + params.op.bucket.bucket_name.c_str(), + params.op.obj.obj_name.c_str(), + params.op.obj.obj_instance.c_str()); } }; +/* taken from rgw_rados.h::RGWOLHInfo */ +struct DBOLHInfo { + rgw_obj target; + bool removed; + DBOLHInfo() : removed(false) {} + void encode(bufferlist& bl) const { + ENCODE_START(1, 1, bl); + encode(target, bl); + encode(removed, bl); + ENCODE_FINISH(bl); + } + + void decode(bufferlist::const_iterator& bl) { + DECODE_START(1, bl); + decode(target, bl); + decode(removed, bl); + DECODE_FINISH(bl); + } +}; +WRITE_CLASS_ENCODER(DBOLHInfo) + class DB { private: const string db_name; + rgw::sal::Store* store; const string user_table; const string bucket_table; const string quota_table; @@ -769,6 +1152,9 @@ class DB { CephContext *cct; const DoutPrefix dp; uint64_t max_bucket_id = 0; + // XXX: default ObjStripeSize or ObjChunk size - 4M, make them configurable? + uint64_t ObjHeadSize = 1024; /* 1K - default head data size */ + uint64_t ObjChunkSize = (get_blob_limit() - 1000); /* 1000 to accommodate other fields */ public: DB(string db_name, CephContext *_cct) : db_name(db_name), @@ -781,23 +1167,32 @@ class DB { /* DB() {}*/ DB(CephContext *_cct) : db_name("default_db"), - user_table("user.table"), - bucket_table("bucket.table"), - quota_table("quota.table"), + user_table(db_name+".user.table"), + bucket_table(db_name+".bucket.table"), + quota_table(db_name+".quota.table"), cct(_cct), dp(_cct, dout_subsys, "rgw DBStore backend: ") {} virtual ~DB() {} - const string getDBname() { return db_name + ".db"; } + const string getDBname() { return db_name; } + const string getDBfile() { return db_name + ".db"; } const string getUserTable() { return user_table; } const string getBucketTable() { return bucket_table; } const string getQuotaTable() { return quota_table; } + const string getObjectTable(string bucket) { + return db_name+"."+bucket+".object.table"; } + const string getObjectDataTable(string bucket) { + return db_name+"."+bucket+".objectdata.table"; } map getObjectMap(); struct DBOps dbops; // DB operations, make it private? + void set_store(rgw::sal::Store* _store) { + store = _store; + } + void set_context(CephContext *_cct) { cct = _cct; } @@ -818,6 +1213,7 @@ class DB { int objectmapInsert(const DoutPrefixProvider *dpp, string bucket, void *ptr); int objectmapDelete(const DoutPrefixProvider *dpp, string bucket); + virtual uint64_t get_blob_limit() { return 0; }; virtual void *openDB(const DoutPrefixProvider *dpp) { return NULL; } virtual int closeDB(const DoutPrefixProvider *dpp) { return 0; } virtual int createTables(const DoutPrefixProvider *dpp) { return 0; } @@ -872,6 +1268,330 @@ class DB { RGWBucketInfo& info, bool exclusive, const rgw_user* powner_id, map* pattrs, ceph::real_time* pmtime, RGWObjVersionTracker* pobjv); + + int get_max_head_size() { return ObjHeadSize; } + int get_max_chunk_size() { return ObjChunkSize; } + void gen_rand_obj_instance_name(rgw_obj_key *target_key); + + // db raw obj string is of format - + // "____" + const string raw_obj_oid = "{0}_{1}_{2}_{3}_{4}"; + + inline string to_oid(const string& bucket, const string& obj_name, const string& obj_instance, + uint64_t mp_num, uint64_t partnum) { + string s = fmt::format(raw_obj_oid.c_str(), bucket, obj_name, obj_instance, mp_num, partnum); + return s; + } + inline int from_oid(const string& oid, string& bucket, string& obj_name, + string& obj_instance, + uint64_t& mp_num, uint64_t& partnum) { + vector result; + boost::split(result, oid, boost::is_any_of("_")); + bucket = result[0]; + obj_name = result[1]; + obj_instance = result[2]; + mp_num = stoi(result[3]); + partnum = stoi(result[4]); + + return 0; + } + + struct raw_obj { + DB* db; + + string bucket_name; + string obj_name; + string obj_instance; + string obj_ns; + uint64_t multipart_partnum; + uint64_t part_num; + + string obj_table; + string obj_data_table; + + raw_obj(DB* _db) { + db = _db; + } + + raw_obj(DB* _db, string& _bname, string& _obj_name, string& _obj_instance, + string& _obj_ns, int _mp_partnum, int _part_num) { + db = _db; + bucket_name = _bname; + obj_name = _obj_name; + obj_instance = _obj_instance; + obj_ns = _obj_ns; + multipart_partnum = _mp_partnum; + part_num = _part_num; + + obj_table = bucket_name+".object.table"; + obj_data_table = bucket_name+".objectdata.table"; + } + + raw_obj(DB* _db, string& oid) { + int r; + + db = _db; + r = db->from_oid(oid, bucket_name, obj_name, obj_instance, multipart_partnum, + part_num); + if (r < 0) { + multipart_partnum = 0; + part_num = 0; + } + + obj_table = db->getObjectTable(bucket_name); + obj_data_table = db->getObjectDataTable(bucket_name); + } + + int InitializeParamsfromRawObj (const DoutPrefixProvider *dpp, DBOpParams* params); + + int read(const DoutPrefixProvider *dpp, int64_t ofs, uint64_t end, bufferlist& bl); + int write(const DoutPrefixProvider *dpp, int64_t ofs, int64_t write_ofs, uint64_t len, bufferlist& bl); + }; + + class Bucket { + friend class DB; + DB* store; + + RGWBucketInfo bucket_info; + + public: + Bucket(DB *_store, const RGWBucketInfo& _binfo) : store(_store), bucket_info(_binfo) {} + DB *get_store() { return store; } + rgw_bucket& get_bucket() { return bucket_info.bucket; } + RGWBucketInfo& get_bucket_info() { return bucket_info; } + + class List { + protected: + // absolute maximum number of objects that + // list_objects_(un)ordered can return + static constexpr int64_t bucket_list_objects_absolute_max = 25000; + + DB::Bucket *target; + rgw_obj_key next_marker; + + public: + + struct Params { + string prefix; + string delim; + rgw_obj_key marker; + rgw_obj_key end_marker; + string ns; + bool enforce_ns; + RGWAccessListFilter *filter; + bool list_versions; + bool allow_unordered; + + Params() : + enforce_ns(true), + filter(NULL), + list_versions(false), + allow_unordered(false) + {} + } params; + + explicit List(DB::Bucket *_target) : target(_target) {} + + /* XXX: Handle ordered and unordered separately. + * For now returning only ordered entries */ + int list_objects(const DoutPrefixProvider *dpp, int64_t max, + vector *result, + map *common_prefixes, bool *is_truncated); + rgw_obj_key& get_next_marker() { + return next_marker; + } + }; + }; + + class Object { + friend class DB; + DB* store; + + RGWBucketInfo bucket_info; + rgw_obj obj; + + RGWObjState *state; + + bool versioning_disabled; + + bool bs_initialized; + + public: + Object(DB *_store, const RGWBucketInfo& _bucket_info, RGWObjectCtx& _ctx, const rgw_obj& _obj) : store(_store), bucket_info(_bucket_info), + obj(_obj), + state(NULL), versioning_disabled(false), + bs_initialized(false) {} + + Object(DB *_store, const RGWBucketInfo& _bucket_info, const rgw_obj& _obj) : store(_store), bucket_info(_bucket_info), obj(_obj) {} + + struct Read { + DB::Object *source; + + struct GetObjState { + rgw_obj obj; + } state; + + struct ConditionParams { + const ceph::real_time *mod_ptr; + const ceph::real_time *unmod_ptr; + bool high_precision_time; + uint32_t mod_zone_id; + uint64_t mod_pg_ver; + const char *if_match; + const char *if_nomatch; + + ConditionParams() : + mod_ptr(NULL), unmod_ptr(NULL), high_precision_time(false), mod_zone_id(0), mod_pg_ver(0), + if_match(NULL), if_nomatch(NULL) {} + } conds; + + struct Params { + ceph::real_time *lastmod; + uint64_t *obj_size; + map *attrs; + rgw_obj *target_obj; + + Params() : lastmod(nullptr), obj_size(nullptr), attrs(nullptr), + target_obj(nullptr) {} + } params; + + explicit Read(DB::Object *_source) : source(_source) {} + + int prepare(const DoutPrefixProvider *dpp); + static int range_to_ofs(uint64_t obj_size, int64_t &ofs, int64_t &end); + int read(int64_t ofs, int64_t end, bufferlist& bl, const DoutPrefixProvider *dpp); + int iterate(const DoutPrefixProvider *dpp, int64_t ofs, int64_t end, RGWGetDataCB *cb); + int get_attr(const DoutPrefixProvider *dpp, const char *name, bufferlist& dest); + }; + + struct Write { + DB::Object *target; + RGWObjState obj_state; + + struct MetaParams { + ceph::real_time *mtime; + map* rmattrs; + const bufferlist *data; + RGWObjManifest *manifest; + const string *ptag; + list *remove_objs; + ceph::real_time set_mtime; + rgw_user owner; + RGWObjCategory category; + int flags; + const char *if_match; + const char *if_nomatch; + std::optional olh_epoch; + ceph::real_time delete_at; + bool canceled; + const string *user_data; + rgw_zone_set *zones_trace; + bool modify_tail; + bool completeMultipart; + bool appendable; + + MetaParams() : mtime(NULL), rmattrs(NULL), data(NULL), manifest(NULL), ptag(NULL), + remove_objs(NULL), category(RGWObjCategory::Main), flags(0), + if_match(NULL), if_nomatch(NULL), canceled(false), user_data(nullptr), zones_trace(nullptr), + modify_tail(false), completeMultipart(false), appendable(false) {} + } meta; + + explicit Write(DB::Object *_target) : target(_target) {} + + int prepare(const DoutPrefixProvider* dpp); + int write_data(const DoutPrefixProvider* dpp, + bufferlist& data, uint64_t ofs); + int _do_write_meta(const DoutPrefixProvider *dpp, + uint64_t size, uint64_t accounted_size, + map& attrs, + bool assume_noent, bool modify_tail); + int write_meta(const DoutPrefixProvider *dpp, uint64_t size, + uint64_t accounted_size, map& attrs); + }; + + struct Delete { + DB::Object *target; + + struct DeleteParams { + rgw_user bucket_owner; + int versioning_status; + ACLOwner obj_owner; /* needed for creation of deletion marker */ + uint64_t olh_epoch; + string marker_version_id; + uint32_t bilog_flags; + list *remove_objs; + ceph::real_time expiration_time; + ceph::real_time unmod_since; + ceph::real_time mtime; /* for setting delete marker mtime */ + bool high_precision_time; + rgw_zone_set *zones_trace; + bool abortmp; + uint64_t parts_accounted_size; + + DeleteParams() : versioning_status(0), olh_epoch(0), bilog_flags(0), remove_objs(NULL), high_precision_time(false), zones_trace(nullptr), abortmp(false), parts_accounted_size(0) {} + } params; + + struct DeleteResult { + bool delete_marker; + string version_id; + + DeleteResult() : delete_marker(false) {} + } result; + + explicit Delete(DB::Object *_target) : target(_target) {} + + int delete_obj(const DoutPrefixProvider *dpp); + }; + + /* XXX: the parameters may be subject to change. All we need is bucket name + * & obj name,instance - keys */ + int get_obj_state(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, + const rgw_obj& obj, + bool follow_olh, RGWObjState **state); + int get_olh_target_state(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw_obj& obj, + RGWObjState* olh_state, RGWObjState** target); + int follow_olh(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, RGWObjState *state, + const rgw_obj& olh_obj, rgw_obj *target); + + int get_state(const DoutPrefixProvider *dpp, RGWObjState **pstate, bool follow_olh); + int get_manifest(const DoutPrefixProvider *dpp, RGWObjManifest **pmanifest); + + DB *get_store() { return store; } + rgw_obj& get_obj() { return obj; } + RGWBucketInfo& get_bucket_info() { return bucket_info; } + + int InitializeParamsfromObject(const DoutPrefixProvider *dpp, DBOpParams* params); + int set_attrs(const DoutPrefixProvider *dpp, map& setattrs, + map* rmattrs); + int obj_omap_set_val_by_key(const DoutPrefixProvider *dpp, const std::string& key, bufferlist& val, bool must_exist); + int obj_omap_get_vals_by_keys(const DoutPrefixProvider *dpp, const std::string& oid, + const std::set& keys, + std::map* vals); + int obj_omap_get_all(const DoutPrefixProvider *dpp, std::map *m); + int obj_omap_get_vals(const DoutPrefixProvider *dpp, const std::string& marker, uint64_t count, + std::map *m, bool* pmore); + using iterate_obj_cb = int (*)(const DoutPrefixProvider*, const raw_obj&, off_t, off_t, + bool, RGWObjState*, void*); + + int iterate_obj(const DoutPrefixProvider *dpp, + const RGWBucketInfo& bucket_info, const rgw_obj& obj, + off_t ofs, off_t end, uint64_t max_chunk_size, + iterate_obj_cb cb, void *arg); + }; + int get_obj_iterate_cb(const DoutPrefixProvider *dpp, + const raw_obj& read_obj, off_t obj_ofs, + off_t len, bool is_head_obj, + RGWObjState *astate, void *arg); +}; + +struct db_get_obj_data { + DB* store; + RGWGetDataCB* client_cb = nullptr; + uint64_t offset; // next offset to write to client + + db_get_obj_data(DB* db, RGWGetDataCB* cb, uint64_t offset) : + store(db), client_cb(cb), offset(offset) {} + ~db_get_obj_data() {} }; } } // namespace rgw::store diff --git a/src/rgw/store/dbstore/sqlite/sqliteDB.cc b/src/rgw/store/dbstore/sqlite/sqliteDB.cc index 0a2d323112b..b634a846ae5 100644 --- a/src/rgw/store/dbstore/sqlite/sqliteDB.cc +++ b/src/rgw/store/dbstore/sqlite/sqliteDB.cc @@ -144,6 +144,7 @@ static int list_callback(void *None, int argc, char **argv, char **aname) int i; for(i=0; idb, cct); - dbops.RemoveUser = new SQLRemoveUser(&this->db, cct); - dbops.GetUser = new SQLGetUser(&this->db, cct); - dbops.InsertBucket = new SQLInsertBucket(&this->db, cct); - dbops.UpdateBucket = new SQLUpdateBucket(&this->db, cct); - dbops.RemoveBucket = new SQLRemoveBucket(&this->db, cct); - dbops.GetBucket = new SQLGetBucket(&this->db, cct); - dbops.ListUserBuckets = new SQLListUserBuckets(&this->db, cct); + dbops.InsertUser = new SQLInsertUser(&this->db, this->getDBname(), cct); + dbops.RemoveUser = new SQLRemoveUser(&this->db, this->getDBname(), cct); + dbops.GetUser = new SQLGetUser(&this->db, this->getDBname(), cct); + dbops.InsertBucket = new SQLInsertBucket(&this->db, this->getDBname(), cct); + dbops.UpdateBucket = new SQLUpdateBucket(&this->db, this->getDBname(), cct); + dbops.RemoveBucket = new SQLRemoveBucket(&this->db, this->getDBname(), cct); + dbops.GetBucket = new SQLGetBucket(&this->db, this->getDBname(), cct); + dbops.ListUserBuckets = new SQLListUserBuckets(&this->db, this->getDBname(), cct); return 0; } @@ -375,7 +508,7 @@ void *SQLiteDB::openDB(const DoutPrefixProvider *dpp) string dbname; int rc = 0; - dbname = getDBname(); + dbname = getDBfile(); if (dbname.empty()) { ldpp_dout(dpp, 0)<<"dbname is NULL" << dendl; goto out; @@ -697,8 +830,7 @@ int SQLiteDB::ListAllObjects(const DoutPrefixProvider *dpp, DBOpParams *params) for (iter = objectmap.begin(); iter != objectmap.end(); ++iter) { bucket = iter->first; - params->object_table = bucket + - ".object.table"; + params->object_table = getObjectTable(bucket); schema = ListTableSchema(params->object_table); ret = exec(dpp, schema.c_str(), &list_callback); @@ -711,23 +843,26 @@ int SQLiteDB::ListAllObjects(const DoutPrefixProvider *dpp, DBOpParams *params) return ret; } -int SQLObjectOp::InitializeObjectOps(const DoutPrefixProvider *dpp) +int SQLObjectOp::InitializeObjectOps(string db_name, const DoutPrefixProvider *dpp) { - InsertObject = new SQLInsertObject(sdb, cct); - RemoveObject = new SQLRemoveObject(sdb, cct); - ListObject = new SQLListObject(sdb, cct); - PutObjectData = new SQLPutObjectData(sdb, cct); - GetObjectData = new SQLGetObjectData(sdb, cct); - DeleteObjectData = new SQLDeleteObjectData(sdb, cct); + PutObject = new SQLPutObject(sdb, db_name, cct); + DeleteObject = new SQLDeleteObject(sdb, db_name, cct); + GetObject = new SQLGetObject(sdb, db_name, cct); + UpdateObject = new SQLUpdateObject(sdb, db_name, cct); + ListBucketObjects = new SQLListBucketObjects(sdb, db_name, cct); + PutObjectData = new SQLPutObjectData(sdb, db_name, cct); + GetObjectData = new SQLGetObjectData(sdb, db_name, cct); + DeleteObjectData = new SQLDeleteObjectData(sdb, db_name, cct); return 0; } int SQLObjectOp::FreeObjectOps(const DoutPrefixProvider *dpp) { - delete InsertObject; - delete RemoveObject; - delete ListObject; + delete PutObject; + delete DeleteObject; + delete GetObject; + delete UpdateObject; delete PutObjectData; delete GetObjectData; delete DeleteObjectData; @@ -1108,10 +1243,6 @@ int SQLInsertBucket::Execute(const DoutPrefixProvider *dpp, struct DBOpParams *p objectmapInsert(dpp, bucket_name, ObPtr); - params->object_table = bucket_name + ".object.table"; - - (void)createObjectTable(dpp, params); - SQL_EXECUTE(dpp, params, stmt, NULL); out: return ret; @@ -1351,8 +1482,14 @@ out: int SQLGetBucket::Execute(const DoutPrefixProvider *dpp, struct DBOpParams *params) { int ret = -1; + class SQLObjectOp *ObPtr = NULL; params->op.name = "GetBucket"; + + ObPtr = new SQLObjectOp(sdb, ctx()); + + /* For the case when the server restarts, need to reinsert objectmap*/ + objectmapInsert(dpp, params->op.bucket.info.bucket.name, ObPtr); SQL_EXECUTE(dpp, params, stmt, list_bucket); out: return ret; @@ -1404,46 +1541,234 @@ out: return ret; } -int SQLInsertObject::Prepare(const DoutPrefixProvider *dpp, struct DBOpParams *params) +int SQLPutObject::Prepare(const DoutPrefixProvider *dpp, struct DBOpParams *params) { int ret = -1; struct DBOpPrepareParams p_params = PrepareParams; struct DBOpParams copy = *params; - string bucket_name; + string bucket_name = params->op.bucket.info.bucket.name; if (!*sdb) { - ldpp_dout(dpp, 0)<<"In SQLInsertObject - no db" << dendl; + ldpp_dout(dpp, 0)<<"In SQLPutObject - no db" << dendl; goto out; } - bucket_name = params->op.bucket.info.bucket.name; - p_params.object_table = bucket_name + ".object.table"; + if (p_params.object_table.empty()) { + p_params.object_table = getObjectTable(bucket_name); + } + params->object_table = p_params.object_table; + (void)createObjectTable(dpp, params); - SQL_PREPARE(dpp, p_params, sdb, stmt, ret, "PrepareInsertObject"); + SQL_PREPARE(dpp, p_params, sdb, stmt, ret, "PreparePutObject"); out: return ret; } -int SQLInsertObject::Bind(const DoutPrefixProvider *dpp, struct DBOpParams *params) +int SQLPutObject::Bind(const DoutPrefixProvider *dpp, struct DBOpParams *params) { int index = -1; int rc = 0; struct DBOpPrepareParams p_params = PrepareParams; - SQL_BIND_INDEX(dpp, stmt, index, p_params.object.c_str(), sdb); - - SQL_BIND_TEXT(dpp, stmt, index, params->object.c_str(), sdb); + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.obj_name.c_str(), sdb); + SQL_BIND_TEXT(dpp, stmt, index, params->op.obj.state.obj.key.name.c_str(), sdb); SQL_BIND_INDEX(dpp, stmt, index, p_params.op.bucket.bucket_name.c_str(), sdb); + SQL_BIND_TEXT(dpp, stmt, index, params->op.bucket.info.bucket.name.c_str(), sdb); + + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.obj_instance.c_str(), sdb); + SQL_BIND_TEXT(dpp, stmt, index, params->op.obj.state.obj.key.instance.c_str(), sdb); + + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.obj_ns.c_str(), sdb); + SQL_BIND_TEXT(dpp, stmt, index, params->op.obj.state.obj.key.ns.c_str(), sdb); + + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.acls.c_str(), sdb); + SQL_ENCODE_BLOB_PARAM(dpp, stmt, index, params->op.obj.acls, sdb); + + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.index_ver.c_str(), sdb); + SQL_BIND_INT(dpp, stmt, index, params->op.obj.index_ver, sdb); + + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.tag.c_str(), sdb); + SQL_BIND_TEXT(dpp, stmt, index, params->op.obj.tag.c_str(), sdb); + + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.flags.c_str(), sdb); + SQL_BIND_INT(dpp, stmt, index, params->op.obj.flags, sdb); + + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.versioned_epoch.c_str(), sdb); + SQL_BIND_INT(dpp, stmt, index, params->op.obj.versioned_epoch, sdb); + + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.obj_category.c_str(), sdb); + SQL_BIND_INT(dpp, stmt, index, (uint8_t)(params->op.obj.category), sdb); + + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.etag.c_str(), sdb); + SQL_BIND_TEXT(dpp, stmt, index, params->op.obj.etag.c_str(), sdb); + + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.owner.c_str(), sdb); + SQL_BIND_TEXT(dpp, stmt, index, params->op.obj.owner.c_str(), sdb); + + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.owner_display_name.c_str(), sdb); + SQL_BIND_TEXT(dpp, stmt, index, params->op.obj.owner_display_name.c_str(), sdb); + + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.storage_class.c_str(), sdb); + SQL_BIND_TEXT(dpp, stmt, index, params->op.obj.storage_class.c_str(), sdb); + + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.appendable.c_str(), sdb); + SQL_BIND_INT(dpp, stmt, index, params->op.obj.appendable, sdb); + + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.content_type.c_str(), sdb); + SQL_BIND_TEXT(dpp, stmt, index, params->op.obj.content_type.c_str(), sdb); + + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.index_hash_source.c_str(), sdb); + SQL_BIND_TEXT(dpp, stmt, index, params->op.obj.state.obj.index_hash_source.c_str(), sdb); + + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.obj_size.c_str(), sdb); + SQL_BIND_INT(dpp, stmt, index, params->op.obj.state.size, sdb); + + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.accounted_size.c_str(), sdb); + SQL_BIND_INT(dpp, stmt, index, params->op.obj.state.accounted_size, sdb); + + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.mtime.c_str(), sdb); + SQL_ENCODE_BLOB_PARAM(dpp, stmt, index, params->op.obj.state.mtime, sdb); + + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.epoch.c_str(), sdb); + SQL_BIND_INT(dpp, stmt, index, params->op.obj.state.epoch, sdb); + + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.obj_tag.c_str(), sdb); + SQL_ENCODE_BLOB_PARAM(dpp, stmt, index, params->op.obj.state.obj_tag, sdb); + + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.tail_tag.c_str(), sdb); + SQL_ENCODE_BLOB_PARAM(dpp, stmt, index, params->op.obj.state.tail_tag, sdb); + + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.write_tag.c_str(), sdb); + SQL_BIND_TEXT(dpp, stmt, index, params->op.obj.state.write_tag.c_str(), sdb); + + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.fake_tag.c_str(), sdb); + SQL_BIND_INT(dpp, stmt, index, params->op.obj.state.fake_tag, sdb); + + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.shadow_obj.c_str(), sdb); + SQL_BIND_TEXT(dpp, stmt, index, params->op.obj.state.shadow_obj.c_str(), sdb); + + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.has_data.c_str(), sdb); + SQL_BIND_INT(dpp, stmt, index, params->op.obj.state.has_data, sdb); + + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.is_olh.c_str(), sdb); + SQL_BIND_INT(dpp, stmt, index, params->op.obj.state.is_olh, sdb); + + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.olh_tag.c_str(), sdb); + SQL_ENCODE_BLOB_PARAM(dpp, stmt, index, params->op.obj.state.olh_tag, sdb); + + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.pg_ver.c_str(), sdb); + SQL_BIND_INT(dpp, stmt, index, params->op.obj.state.pg_ver, sdb); + + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.zone_short_id.c_str(), sdb); + SQL_BIND_INT(dpp, stmt, index, params->op.obj.state.zone_short_id, sdb); + + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.obj_version.c_str(), sdb); + SQL_BIND_INT(dpp, stmt, index, params->op.obj.state.objv_tracker.read_version.ver, sdb); + + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.obj_version_tag.c_str(), sdb); + SQL_BIND_TEXT(dpp, stmt, index, params->op.obj.state.objv_tracker.read_version.tag.c_str(), sdb); + + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.obj_attrs.c_str(), sdb); + SQL_ENCODE_BLOB_PARAM(dpp, stmt, index, params->op.obj.state.attrset, sdb); + + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.head_size.c_str(), sdb); + SQL_BIND_INT(dpp, stmt, index, params->op.obj.head_size, sdb); + + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.max_head_size.c_str(), sdb); + SQL_BIND_INT(dpp, stmt, index, params->op.obj.max_head_size, sdb); + + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.prefix.c_str(), sdb); + SQL_BIND_TEXT(dpp, stmt, index, params->op.obj.prefix.c_str(), sdb); + + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.tail_instance.c_str(), sdb); + SQL_BIND_TEXT(dpp, stmt, index, params->op.obj.tail_instance.c_str(), sdb); + + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.head_placement_rule_name.c_str(), sdb); + SQL_BIND_TEXT(dpp, stmt, index, params->op.obj.head_placement_rule.name.c_str(), sdb); + + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.head_placement_storage_class.c_str(), sdb); + SQL_BIND_TEXT(dpp, stmt, index, params->op.obj.head_placement_rule.storage_class.c_str(), sdb); + + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.tail_placement_rule_name.c_str(), sdb); + SQL_BIND_TEXT(dpp, stmt, index, params->op.obj.tail_placement.placement_rule.name.c_str(), sdb); + + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.tail_placement_storage_class.c_str(), sdb); + SQL_BIND_TEXT(dpp, stmt, index, params->op.obj.tail_placement.placement_rule.storage_class.c_str(), sdb); + + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.manifest_part_objs.c_str(), sdb); + SQL_ENCODE_BLOB_PARAM(dpp, stmt, index, params->op.obj.objs, sdb); + + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.manifest_part_rules.c_str(), sdb); + SQL_ENCODE_BLOB_PARAM(dpp, stmt, index, params->op.obj.rules, sdb); + + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.omap.c_str(), sdb); + SQL_ENCODE_BLOB_PARAM(dpp, stmt, index, params->op.obj.omap, sdb); + + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.is_multipart.c_str(), sdb); + SQL_BIND_INT(dpp, stmt, index, params->op.obj.is_multipart, sdb); + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.head_data.c_str(), sdb); + SQL_ENCODE_BLOB_PARAM(dpp, stmt, index, params->op.obj.head_data, sdb); + + +out: + return rc; +} + +int SQLPutObject::Execute(const DoutPrefixProvider *dpp, struct DBOpParams *params) +{ + int ret = -1; + + SQL_EXECUTE(dpp, params, stmt, NULL); +out: + return ret; +} + +int SQLDeleteObject::Prepare(const DoutPrefixProvider *dpp, struct DBOpParams *params) +{ + int ret = -1; + struct DBOpPrepareParams p_params = PrepareParams; + struct DBOpParams copy = *params; + string bucket_name = params->op.bucket.info.bucket.name; + + if (!*sdb) { + ldpp_dout(dpp, 0)<<"In SQLDeleteObject - no db" << dendl; + goto out; + } + + if (p_params.object_table.empty()) { + p_params.object_table = getObjectTable(bucket_name); + } + params->object_table = p_params.object_table; + (void)createObjectTable(dpp, params); + + SQL_PREPARE(dpp, p_params, sdb, stmt, ret, "PrepareDeleteObject"); + +out: + return ret; +} + +int SQLDeleteObject::Bind(const DoutPrefixProvider *dpp, struct DBOpParams *params) +{ + int index = -1; + int rc = 0; + struct DBOpPrepareParams p_params = PrepareParams; + + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.bucket.bucket_name.c_str(), sdb); SQL_BIND_TEXT(dpp, stmt, index, params->op.bucket.info.bucket.name.c_str(), sdb); + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.obj_name.c_str(), sdb); + SQL_BIND_TEXT(dpp, stmt, index, params->op.obj.state.obj.key.name.c_str(), sdb); + + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.obj_instance.c_str(), sdb); + SQL_BIND_TEXT(dpp, stmt, index, params->op.obj.state.obj.key.instance.c_str(), sdb); out: return rc; } -int SQLInsertObject::Execute(const DoutPrefixProvider *dpp, struct DBOpParams *params) +int SQLDeleteObject::Execute(const DoutPrefixProvider *dpp, struct DBOpParams *params) { int ret = -1; @@ -1452,7 +1777,59 @@ out: return ret; } -int SQLRemoveObject::Prepare(const DoutPrefixProvider *dpp, struct DBOpParams *params) +int SQLGetObject::Prepare(const DoutPrefixProvider *dpp, struct DBOpParams *params) +{ + int ret = -1; + struct DBOpPrepareParams p_params = PrepareParams; + struct DBOpParams copy = *params; + string bucket_name = params->op.bucket.info.bucket.name; + + if (!*sdb) { + ldpp_dout(dpp, 0)<<"In SQLGetObject - no db" << dendl; + goto out; + } + + if (p_params.object_table.empty()) { + p_params.object_table = getObjectTable(bucket_name); + } + params->object_table = p_params.object_table; + (void)createObjectTable(dpp, params); + + SQL_PREPARE(dpp, p_params, sdb, stmt, ret, "PrepareGetObject"); + +out: + return ret; +} + +int SQLGetObject::Bind(const DoutPrefixProvider *dpp, struct DBOpParams *params) +{ + int index = -1; + int rc = 0; + struct DBOpPrepareParams p_params = PrepareParams; + + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.bucket.bucket_name.c_str(), sdb); + SQL_BIND_TEXT(dpp, stmt, index, params->op.bucket.info.bucket.name.c_str(), sdb); + + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.obj_name.c_str(), sdb); + SQL_BIND_TEXT(dpp, stmt, index, params->op.obj.state.obj.key.name.c_str(), sdb); + + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.obj_instance.c_str(), sdb); + SQL_BIND_TEXT(dpp, stmt, index, params->op.obj.state.obj.key.instance.c_str(), sdb); + +out: + return rc; +} + +int SQLGetObject::Execute(const DoutPrefixProvider *dpp, struct DBOpParams *params) +{ + int ret = -1; + + SQL_EXECUTE(dpp, params, stmt, list_object); +out: + return ret; +} + +int SQLUpdateObject::Prepare(const DoutPrefixProvider *dpp, struct DBOpParams *params) { int ret = -1; struct DBOpPrepareParams p_params = PrepareParams; @@ -1460,50 +1837,231 @@ int SQLRemoveObject::Prepare(const DoutPrefixProvider *dpp, struct DBOpParams *p string bucket_name; if (!*sdb) { - ldpp_dout(dpp, 0)<<"In SQLRemoveObject - no db" << dendl; + ldpp_dout(dpp, 0)<<"In SQLUpdateObject - no db" << dendl; goto out; } - bucket_name = params->op.bucket.info.bucket.name; - p_params.object_table = bucket_name + ".object.table"; - copy.object_table = bucket_name + ".object.table"; + if (p_params.object_table.empty()) { + bucket_name = params->op.bucket.info.bucket.name; + p_params.object_table = getObjectTable(bucket_name); + } - (void)createObjectTable(dpp, ©); + p_params.op.query_str = params->op.query_str; - SQL_PREPARE(dpp, p_params, sdb, stmt, ret, "PrepareRemoveObject"); + if (params->op.query_str == "omap") { + SQL_PREPARE(dpp, p_params, sdb, omap_stmt, ret, "PrepareUpdateObject"); + } else if (params->op.query_str == "attrs") { + SQL_PREPARE(dpp, p_params, sdb, attrs_stmt, ret, "PrepareUpdateObject"); + } else if (params->op.query_str == "meta") { + SQL_PREPARE(dpp, p_params, sdb, meta_stmt, ret, "PrepareUpdateObject"); + } else { + ldpp_dout(dpp, 0)<<"In SQLUpdateObject invalid query_str:" << + params->op.query_str << dendl; + goto out; + } out: return ret; } -int SQLRemoveObject::Bind(const DoutPrefixProvider *dpp, struct DBOpParams *params) +int SQLUpdateObject::Bind(const DoutPrefixProvider *dpp, struct DBOpParams *params) { int index = -1; int rc = 0; struct DBOpPrepareParams p_params = PrepareParams; + sqlite3_stmt** stmt = NULL; // Prepared statement + + /* All below fields for attrs */ + if (params->op.query_str == "omap") { + stmt = &omap_stmt; + } else if (params->op.query_str == "attrs") { + stmt = &attrs_stmt; + } else if (params->op.query_str == "meta") { + stmt = &meta_stmt; + } else { + ldpp_dout(dpp, 0)<<"In SQLUpdateObject invalid query_str:" << + params->op.query_str << dendl; + goto out; + } + + SQL_BIND_INDEX(dpp, *stmt, index, p_params.op.bucket.bucket_name.c_str(), sdb); + SQL_BIND_TEXT(dpp, *stmt, index, params->op.bucket.info.bucket.name.c_str(), sdb); - SQL_BIND_INDEX(dpp, stmt, index, p_params.object.c_str(), sdb); + SQL_BIND_INDEX(dpp, *stmt, index, p_params.op.obj.obj_name.c_str(), sdb); + SQL_BIND_TEXT(dpp, *stmt, index, params->op.obj.state.obj.key.name.c_str(), sdb); - SQL_BIND_TEXT(dpp, stmt, index, params->object.c_str(), sdb); + SQL_BIND_INDEX(dpp, *stmt, index, p_params.op.obj.obj_instance.c_str(), sdb); + SQL_BIND_TEXT(dpp, *stmt, index, params->op.obj.state.obj.key.instance.c_str(), sdb); - SQL_BIND_INDEX(dpp, stmt, index, p_params.op.bucket.bucket_name.c_str(), sdb); + SQL_BIND_INDEX(dpp, *stmt, index, p_params.op.obj.mtime.c_str(), sdb); + SQL_ENCODE_BLOB_PARAM(dpp, *stmt, index, params->op.obj.state.mtime, sdb); - SQL_BIND_TEXT(dpp, stmt, index, params->op.bucket.info.bucket.name.c_str(), sdb); + if (params->op.query_str == "omap") { + SQL_BIND_INDEX(dpp, *stmt, index, p_params.op.obj.omap.c_str(), sdb); + SQL_ENCODE_BLOB_PARAM(dpp, *stmt, index, params->op.obj.omap, sdb); + } + if (params->op.query_str == "attrs") { + SQL_BIND_INDEX(dpp, *stmt, index, p_params.op.obj.obj_attrs.c_str(), sdb); + SQL_ENCODE_BLOB_PARAM(dpp, *stmt, index, params->op.obj.state.attrset, sdb); + } + if (params->op.query_str == "meta") { + SQL_BIND_INDEX(dpp, *stmt, index, p_params.op.obj.obj_ns.c_str(), sdb); + SQL_BIND_TEXT(dpp, *stmt, index, params->op.obj.state.obj.key.ns.c_str(), sdb); + + SQL_BIND_INDEX(dpp, *stmt, index, p_params.op.obj.acls.c_str(), sdb); + SQL_ENCODE_BLOB_PARAM(dpp, *stmt, index, params->op.obj.acls, sdb); + + SQL_BIND_INDEX(dpp, *stmt, index, p_params.op.obj.index_ver.c_str(), sdb); + SQL_BIND_INT(dpp, *stmt, index, params->op.obj.index_ver, sdb); + + SQL_BIND_INDEX(dpp, *stmt, index, p_params.op.obj.tag.c_str(), sdb); + SQL_BIND_TEXT(dpp, *stmt, index, params->op.obj.tag.c_str(), sdb); + + SQL_BIND_INDEX(dpp, *stmt, index, p_params.op.obj.flags.c_str(), sdb); + SQL_BIND_INT(dpp, *stmt, index, params->op.obj.flags, sdb); + + SQL_BIND_INDEX(dpp, *stmt, index, p_params.op.obj.versioned_epoch.c_str(), sdb); + SQL_BIND_INT(dpp, *stmt, index, params->op.obj.versioned_epoch, sdb); + + SQL_BIND_INDEX(dpp, *stmt, index, p_params.op.obj.obj_category.c_str(), sdb); + SQL_BIND_INT(dpp, *stmt, index, (uint8_t)(params->op.obj.category), sdb); + + SQL_BIND_INDEX(dpp, *stmt, index, p_params.op.obj.etag.c_str(), sdb); + SQL_BIND_TEXT(dpp, *stmt, index, params->op.obj.etag.c_str(), sdb); + + SQL_BIND_INDEX(dpp, *stmt, index, p_params.op.obj.owner.c_str(), sdb); + SQL_BIND_TEXT(dpp, *stmt, index, params->op.obj.owner.c_str(), sdb); + + SQL_BIND_INDEX(dpp, *stmt, index, p_params.op.obj.owner_display_name.c_str(), sdb); + SQL_BIND_TEXT(dpp, *stmt, index, params->op.obj.owner_display_name.c_str(), sdb); + + SQL_BIND_INDEX(dpp, *stmt, index, p_params.op.obj.storage_class.c_str(), sdb); + SQL_BIND_TEXT(dpp, *stmt, index, params->op.obj.storage_class.c_str(), sdb); + + SQL_BIND_INDEX(dpp, *stmt, index, p_params.op.obj.appendable.c_str(), sdb); + SQL_BIND_INT(dpp, *stmt, index, params->op.obj.appendable, sdb); + + SQL_BIND_INDEX(dpp, *stmt, index, p_params.op.obj.content_type.c_str(), sdb); + SQL_BIND_TEXT(dpp, *stmt, index, params->op.obj.content_type.c_str(), sdb); + + SQL_BIND_INDEX(dpp, *stmt, index, p_params.op.obj.index_hash_source.c_str(), sdb); + SQL_BIND_TEXT(dpp, *stmt, index, params->op.obj.state.obj.index_hash_source.c_str(), sdb); + + SQL_BIND_INDEX(dpp, *stmt, index, p_params.op.obj.obj_size.c_str(), sdb); + SQL_BIND_INT(dpp, *stmt, index, params->op.obj.state.size, sdb); + + SQL_BIND_INDEX(dpp, *stmt, index, p_params.op.obj.accounted_size.c_str(), sdb); + SQL_BIND_INT(dpp, *stmt, index, params->op.obj.state.accounted_size, sdb); + + SQL_BIND_INDEX(dpp, *stmt, index, p_params.op.obj.epoch.c_str(), sdb); + SQL_BIND_INT(dpp, *stmt, index, params->op.obj.state.epoch, sdb); + + SQL_BIND_INDEX(dpp, *stmt, index, p_params.op.obj.obj_tag.c_str(), sdb); + SQL_ENCODE_BLOB_PARAM(dpp, *stmt, index, params->op.obj.state.obj_tag, sdb); + + SQL_BIND_INDEX(dpp, *stmt, index, p_params.op.obj.tail_tag.c_str(), sdb); + SQL_ENCODE_BLOB_PARAM(dpp, *stmt, index, params->op.obj.state.tail_tag, sdb); + + SQL_BIND_INDEX(dpp, *stmt, index, p_params.op.obj.write_tag.c_str(), sdb); + SQL_BIND_TEXT(dpp, *stmt, index, params->op.obj.state.write_tag.c_str(), sdb); + + SQL_BIND_INDEX(dpp, *stmt, index, p_params.op.obj.fake_tag.c_str(), sdb); + SQL_BIND_INT(dpp, *stmt, index, params->op.obj.state.fake_tag, sdb); + + SQL_BIND_INDEX(dpp, *stmt, index, p_params.op.obj.shadow_obj.c_str(), sdb); + SQL_BIND_TEXT(dpp, *stmt, index, params->op.obj.state.shadow_obj.c_str(), sdb); + + SQL_BIND_INDEX(dpp, *stmt, index, p_params.op.obj.has_data.c_str(), sdb); + SQL_BIND_INT(dpp, *stmt, index, params->op.obj.state.has_data, sdb); + + SQL_BIND_INDEX(dpp, *stmt, index, p_params.op.obj.is_olh.c_str(), sdb); + SQL_BIND_INT(dpp, *stmt, index, params->op.obj.state.is_olh, sdb); + + SQL_BIND_INDEX(dpp, *stmt, index, p_params.op.obj.olh_tag.c_str(), sdb); + SQL_ENCODE_BLOB_PARAM(dpp, *stmt, index, params->op.obj.state.olh_tag, sdb); + + SQL_BIND_INDEX(dpp, *stmt, index, p_params.op.obj.pg_ver.c_str(), sdb); + SQL_BIND_INT(dpp, *stmt, index, params->op.obj.state.pg_ver, sdb); + + SQL_BIND_INDEX(dpp, *stmt, index, p_params.op.obj.zone_short_id.c_str(), sdb); + SQL_BIND_INT(dpp, *stmt, index, params->op.obj.state.zone_short_id, sdb); + + SQL_BIND_INDEX(dpp, *stmt, index, p_params.op.obj.obj_version.c_str(), sdb); + SQL_BIND_INT(dpp, *stmt, index, params->op.obj.state.objv_tracker.read_version.ver, sdb); + + SQL_BIND_INDEX(dpp, *stmt, index, p_params.op.obj.obj_version_tag.c_str(), sdb); + SQL_BIND_TEXT(dpp, *stmt, index, params->op.obj.state.objv_tracker.read_version.tag.c_str(), sdb); + + SQL_BIND_INDEX(dpp, *stmt, index, p_params.op.obj.obj_attrs.c_str(), sdb); + SQL_ENCODE_BLOB_PARAM(dpp, *stmt, index, params->op.obj.state.attrset, sdb); + + SQL_BIND_INDEX(dpp, *stmt, index, p_params.op.obj.head_size.c_str(), sdb); + SQL_BIND_INT(dpp, *stmt, index, params->op.obj.head_size, sdb); + + SQL_BIND_INDEX(dpp, *stmt, index, p_params.op.obj.max_head_size.c_str(), sdb); + SQL_BIND_INT(dpp, *stmt, index, params->op.obj.max_head_size, sdb); + + SQL_BIND_INDEX(dpp, *stmt, index, p_params.op.obj.prefix.c_str(), sdb); + SQL_BIND_TEXT(dpp, *stmt, index, params->op.obj.prefix.c_str(), sdb); + + SQL_BIND_INDEX(dpp, *stmt, index, p_params.op.obj.tail_instance.c_str(), sdb); + SQL_BIND_TEXT(dpp, *stmt, index, params->op.obj.tail_instance.c_str(), sdb); + + SQL_BIND_INDEX(dpp, *stmt, index, p_params.op.obj.head_placement_rule_name.c_str(), sdb); + SQL_BIND_TEXT(dpp, *stmt, index, params->op.obj.head_placement_rule.name.c_str(), sdb); + + SQL_BIND_INDEX(dpp, *stmt, index, p_params.op.obj.head_placement_storage_class.c_str(), sdb); + SQL_BIND_TEXT(dpp, *stmt, index, params->op.obj.head_placement_rule.storage_class.c_str(), sdb); + + SQL_BIND_INDEX(dpp, *stmt, index, p_params.op.obj.tail_placement_rule_name.c_str(), sdb); + SQL_BIND_TEXT(dpp, *stmt, index, params->op.obj.tail_placement.placement_rule.name.c_str(), sdb); + + SQL_BIND_INDEX(dpp, *stmt, index, p_params.op.obj.tail_placement_storage_class.c_str(), sdb); + SQL_BIND_TEXT(dpp, *stmt, index, params->op.obj.tail_placement.placement_rule.storage_class.c_str(), sdb); + + SQL_BIND_INDEX(dpp, *stmt, index, p_params.op.obj.manifest_part_objs.c_str(), sdb); + SQL_ENCODE_BLOB_PARAM(dpp, *stmt, index, params->op.obj.objs, sdb); + + SQL_BIND_INDEX(dpp, *stmt, index, p_params.op.obj.manifest_part_rules.c_str(), sdb); + SQL_ENCODE_BLOB_PARAM(dpp, *stmt, index, params->op.obj.rules, sdb); + + SQL_BIND_INDEX(dpp, *stmt, index, p_params.op.obj.omap.c_str(), sdb); + SQL_ENCODE_BLOB_PARAM(dpp, *stmt, index, params->op.obj.omap, sdb); + + SQL_BIND_INDEX(dpp, *stmt, index, p_params.op.obj.is_multipart.c_str(), sdb); + SQL_BIND_INT(dpp, *stmt, index, params->op.obj.is_multipart, sdb); + + SQL_BIND_INDEX(dpp, *stmt, index, p_params.op.obj.head_data.c_str(), sdb); + SQL_ENCODE_BLOB_PARAM(dpp, *stmt, index, params->op.obj.head_data, sdb); + } out: return rc; } -int SQLRemoveObject::Execute(const DoutPrefixProvider *dpp, struct DBOpParams *params) +int SQLUpdateObject::Execute(const DoutPrefixProvider *dpp, struct DBOpParams *params) { int ret = -1; + sqlite3_stmt** stmt = NULL; // Prepared statement - SQL_EXECUTE(dpp, params, stmt, NULL); + if (params->op.query_str == "omap") { + stmt = &omap_stmt; + } else if (params->op.query_str == "attrs") { + stmt = &attrs_stmt; + } else if (params->op.query_str == "meta") { + stmt = &meta_stmt; + } else { + ldpp_dout(dpp, 0)<<"In SQLUpdateObject invalid query_str:" << + params->op.query_str << dendl; + goto out; + } + + SQL_EXECUTE(dpp, params, *stmt, NULL); out: return ret; } -int SQLListObject::Prepare(const DoutPrefixProvider *dpp, struct DBOpParams *params) +int SQLListBucketObjects::Prepare(const DoutPrefixProvider *dpp, struct DBOpParams *params) { int ret = -1; struct DBOpPrepareParams p_params = PrepareParams; @@ -1511,42 +2069,49 @@ int SQLListObject::Prepare(const DoutPrefixProvider *dpp, struct DBOpParams *par string bucket_name; if (!*sdb) { - ldpp_dout(dpp, 0)<<"In SQLListObject - no db" << dendl; + ldpp_dout(dpp, 0)<<"In SQLListBucketObjects - no db" << dendl; goto out; } - bucket_name = params->op.bucket.info.bucket.name; - p_params.object_table = bucket_name + ".object.table"; - copy.object_table = bucket_name + ".object.table"; + if (p_params.object_table.empty()) { + bucket_name = params->op.bucket.info.bucket.name; + p_params.object_table = getObjectTable(bucket_name); + } - (void)createObjectTable(dpp, ©); + /* XXX: instead of creating..maybe keep object count in bucket info + * and return if there is no object table created. + */ + params->object_table = p_params.object_table; + (void)createObjectTable(dpp, params); + p_params.op.query_str = params->op.query_str; - SQL_PREPARE(dpp, p_params, sdb, stmt, ret, "PrepareListObject"); + SQL_PREPARE(dpp, p_params, sdb, stmt, ret, "PrepareListBucketObjects"); out: return ret; } -int SQLListObject::Bind(const DoutPrefixProvider *dpp, struct DBOpParams *params) +int SQLListBucketObjects::Bind(const DoutPrefixProvider *dpp, struct DBOpParams *params) { int index = -1; int rc = 0; struct DBOpPrepareParams p_params = PrepareParams; - SQL_BIND_INDEX(dpp, stmt, index, p_params.object.c_str(), sdb); - - SQL_BIND_TEXT(dpp, stmt, index, params->object.c_str(), sdb); - SQL_BIND_INDEX(dpp, stmt, index, p_params.op.bucket.bucket_name.c_str(), sdb); - SQL_BIND_TEXT(dpp, stmt, index, params->op.bucket.info.bucket.name.c_str(), sdb); + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.min_marker.c_str(), sdb); + SQL_BIND_TEXT(dpp, stmt, index, params->op.obj.min_marker.c_str(), sdb); + + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.list_max_count.c_str(), sdb); + SQL_BIND_INT(dpp, stmt, index, params->op.list_max_count, sdb); + out: return rc; } -int SQLListObject::Execute(const DoutPrefixProvider *dpp, struct DBOpParams *params) +int SQLListBucketObjects::Execute(const DoutPrefixProvider *dpp, struct DBOpParams *params) { int ret = -1; @@ -1560,20 +2125,23 @@ int SQLPutObjectData::Prepare(const DoutPrefixProvider *dpp, struct DBOpParams * int ret = -1; struct DBOpPrepareParams p_params = PrepareParams; struct DBOpParams copy = *params; - string bucket_name; + string bucket_name = params->op.bucket.info.bucket.name; if (!*sdb) { ldpp_dout(dpp, 0)<<"In SQLPutObjectData - no db" << dendl; goto out; } - bucket_name = params->op.bucket.info.bucket.name; - p_params.object_table = bucket_name + ".object.table"; - p_params.objectdata_table = bucket_name + ".objectdata.table"; - copy.object_table = bucket_name + ".object.table"; - copy.objectdata_table = bucket_name + ".objectdata.table"; - - (void)createObjectDataTable(dpp, ©); + if (p_params.object_table.empty()) { + p_params.object_table = getObjectTable(bucket_name); + } + if (p_params.objectdata_table.empty()) { + p_params.objectdata_table = getObjectDataTable(bucket_name); + } + params->bucket_table = p_params.bucket_table; + params->object_table = p_params.object_table; + params->objectdata_table = p_params.objectdata_table; + (void)createObjectDataTable(dpp, params); SQL_PREPARE(dpp, p_params, sdb, stmt, ret, "PreparePutObjectData"); @@ -1587,25 +2155,40 @@ int SQLPutObjectData::Bind(const DoutPrefixProvider *dpp, struct DBOpParams *par int rc = 0; struct DBOpPrepareParams p_params = PrepareParams; - SQL_BIND_INDEX(dpp, stmt, index, p_params.object.c_str(), sdb); + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.obj_name.c_str(), sdb); + + SQL_BIND_TEXT(dpp, stmt, index, params->op.obj.state.obj.key.name.c_str(), sdb); + + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.obj_instance.c_str(), sdb); + + SQL_BIND_TEXT(dpp, stmt, index, params->op.obj.state.obj.key.instance.c_str(), sdb); - SQL_BIND_TEXT(dpp, stmt, index, params->object.c_str(), sdb); + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.obj_ns.c_str(), sdb); + + SQL_BIND_TEXT(dpp, stmt, index, params->op.obj.state.obj.key.ns.c_str(), sdb); SQL_BIND_INDEX(dpp, stmt, index, p_params.op.bucket.bucket_name.c_str(), sdb); SQL_BIND_TEXT(dpp, stmt, index, params->op.bucket.info.bucket.name.c_str(), sdb); - SQL_BIND_INDEX(dpp, stmt, index, p_params.offset.c_str(), sdb); + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj_data.part_num.c_str(), sdb); + + SQL_BIND_INT(dpp, stmt, index, params->op.obj_data.part_num, sdb); + + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj_data.offset.c_str(), sdb); + + SQL_BIND_INT(dpp, stmt, index, params->op.obj_data.offset, sdb); - SQL_BIND_INT(dpp, stmt, 3, params->offset, sdb); + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj_data.data.c_str(), sdb); + SQL_ENCODE_BLOB_PARAM(dpp, stmt, index, params->op.obj_data.data, sdb); - SQL_BIND_INDEX(dpp, stmt, index, p_params.data.c_str(), sdb); + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj_data.size.c_str(), sdb); - SQL_BIND_BLOB(dpp, stmt, index, params->data.c_str(), params->data.length(), sdb); + SQL_BIND_INT(dpp, stmt, index, params->op.obj_data.size, sdb); - SQL_BIND_INDEX(dpp, stmt, index, p_params.datalen.c_str(), sdb); + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj_data.multipart_part_num.c_str(), sdb); - SQL_BIND_INT(dpp, stmt, index, params->data.length(), sdb); + SQL_BIND_INT(dpp, stmt, index, params->op.obj_data.multipart_part_num, sdb); out: return rc; @@ -1625,20 +2208,22 @@ int SQLGetObjectData::Prepare(const DoutPrefixProvider *dpp, struct DBOpParams * int ret = -1; struct DBOpPrepareParams p_params = PrepareParams; struct DBOpParams copy = *params; - string bucket_name; + string bucket_name = params->op.bucket.info.bucket.name; if (!*sdb) { ldpp_dout(dpp, 0)<<"In SQLGetObjectData - no db" << dendl; goto out; } - bucket_name = params->op.bucket.info.bucket.name; - p_params.object_table = bucket_name + ".object.table"; - p_params.objectdata_table = bucket_name + ".objectdata.table"; - copy.object_table = bucket_name + ".object.table"; - copy.objectdata_table = bucket_name + ".objectdata.table"; - - (void)createObjectDataTable(dpp, ©); + if (p_params.object_table.empty()) { + p_params.object_table = getObjectTable(bucket_name); + } + if (p_params.objectdata_table.empty()) { + p_params.objectdata_table = getObjectDataTable(bucket_name); + } + params->object_table = p_params.object_table; + params->objectdata_table = p_params.objectdata_table; + (void)createObjectDataTable(dpp, params); SQL_PREPARE(dpp, p_params, sdb, stmt, ret, "PrepareGetObjectData"); @@ -1652,13 +2237,15 @@ int SQLGetObjectData::Bind(const DoutPrefixProvider *dpp, struct DBOpParams *par int rc = 0; struct DBOpPrepareParams p_params = PrepareParams; - SQL_BIND_INDEX(dpp, stmt, index, p_params.object.c_str(), sdb); + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.bucket.bucket_name.c_str(), sdb); + SQL_BIND_TEXT(dpp, stmt, index, params->op.bucket.info.bucket.name.c_str(), sdb); - SQL_BIND_TEXT(dpp, stmt, index, params->object.c_str(), sdb); + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.obj_name.c_str(), sdb); + SQL_BIND_TEXT(dpp, stmt, index, params->op.obj.state.obj.key.name.c_str(), sdb); - SQL_BIND_INDEX(dpp, stmt, index, p_params.op.bucket.bucket_name.c_str(), sdb); + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.obj_instance.c_str(), sdb); + SQL_BIND_TEXT(dpp, stmt, index, params->op.obj.state.obj.key.instance.c_str(), sdb); - SQL_BIND_TEXT(dpp, stmt, index, params->op.bucket.info.bucket.name.c_str(), sdb); out: return rc; } @@ -1677,20 +2264,22 @@ int SQLDeleteObjectData::Prepare(const DoutPrefixProvider *dpp, struct DBOpParam int ret = -1; struct DBOpPrepareParams p_params = PrepareParams; struct DBOpParams copy = *params; - string bucket_name; + string bucket_name = params->op.bucket.info.bucket.name; if (!*sdb) { ldpp_dout(dpp, 0)<<"In SQLDeleteObjectData - no db" << dendl; goto out; } - bucket_name = params->op.bucket.info.bucket.name; - p_params.object_table = bucket_name + ".object.table"; - p_params.objectdata_table = bucket_name + ".objectdata.table"; - copy.object_table = bucket_name + ".object.table"; - copy.objectdata_table = bucket_name + ".objectdata.table"; - - (void)createObjectDataTable(dpp, ©); + if (p_params.object_table.empty()) { + p_params.object_table = getObjectTable(bucket_name); + } + if (p_params.objectdata_table.empty()) { + p_params.objectdata_table = getObjectDataTable(bucket_name); + } + params->object_table = p_params.object_table; + params->objectdata_table = p_params.objectdata_table; + (void)createObjectDataTable(dpp, params); SQL_PREPARE(dpp, p_params, sdb, stmt, ret, "PrepareDeleteObjectData"); @@ -1704,13 +2293,14 @@ int SQLDeleteObjectData::Bind(const DoutPrefixProvider *dpp, struct DBOpParams * int rc = 0; struct DBOpPrepareParams p_params = PrepareParams; - SQL_BIND_INDEX(dpp, stmt, index, p_params.object.c_str(), sdb); - - SQL_BIND_TEXT(dpp, stmt, index, params->object.c_str(), sdb); - SQL_BIND_INDEX(dpp, stmt, index, p_params.op.bucket.bucket_name.c_str(), sdb); - SQL_BIND_TEXT(dpp, stmt, index, params->op.bucket.info.bucket.name.c_str(), sdb); + + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.obj_name.c_str(), sdb); + SQL_BIND_TEXT(dpp, stmt, index, params->op.obj.state.obj.key.name.c_str(), sdb); + + SQL_BIND_INDEX(dpp, stmt, index, p_params.op.obj.obj_instance.c_str(), sdb); + SQL_BIND_TEXT(dpp, stmt, index, params->op.obj.state.obj.key.instance.c_str(), sdb); out: return rc; } diff --git a/src/rgw/store/dbstore/sqlite/sqliteDB.h b/src/rgw/store/dbstore/sqlite/sqliteDB.h index bbab2c3cac4..ce6f2980512 100644 --- a/src/rgw/store/dbstore/sqlite/sqliteDB.h +++ b/src/rgw/store/dbstore/sqlite/sqliteDB.h @@ -24,15 +24,16 @@ class SQLiteDB : public DB, public DBOp{ sqlite3_stmt *stmt = NULL; DBOpPrepareParams PrepareParams; - SQLiteDB(string db_name, CephContext *_cct) : DB(db_name, _cct), cct(_cct) { + SQLiteDB(sqlite3 *dbi, string db_name, CephContext *_cct) : DB(db_name, _cct), cct(_cct) { + db = (void*)dbi; InitPrepareParams(get_def_dpp(), PrepareParams); } - SQLiteDB(sqlite3 *dbi, CephContext *_cct) : DB(_cct), cct(_cct) { - db = (void*)dbi; + SQLiteDB(string db_name, CephContext *_cct) : DB(db_name, _cct), cct(_cct) { InitPrepareParams(get_def_dpp(), PrepareParams); } ~SQLiteDB() {} + uint64_t get_blob_limit() override { return SQLITE_LIMIT_LENGTH; } void *openDB(const DoutPrefixProvider *dpp) override; int closeDB(const DoutPrefixProvider *dpp) override; int InitializeDBOps(const DoutPrefixProvider *dpp) override; @@ -73,7 +74,7 @@ class SQLObjectOp : public ObjectOp { SQLObjectOp(sqlite3 **sdbi, CephContext *_cct) : sdb(sdbi), cct(_cct) {}; ~SQLObjectOp() {} - int InitializeObjectOps(const DoutPrefixProvider *dpp); + int InitializeObjectOps(string db_name, const DoutPrefixProvider *dpp); int FreeObjectOps(const DoutPrefixProvider *dpp); }; @@ -83,7 +84,7 @@ class SQLInsertUser : public SQLiteDB, public InsertUserOp { sqlite3_stmt *stmt = NULL; // Prepared statement public: - SQLInsertUser(void **db, CephContext *cct) : SQLiteDB((sqlite3 *)(*db), cct), sdb((sqlite3 **)db) {} + SQLInsertUser(void **db, string db_name, CephContext *cct) : SQLiteDB((sqlite3 *)(*db), db_name, cct), sdb((sqlite3 **)db) {} ~SQLInsertUser() { if (stmt) sqlite3_finalize(stmt); @@ -99,7 +100,7 @@ class SQLRemoveUser : public SQLiteDB, public RemoveUserOp { sqlite3_stmt *stmt = NULL; // Prepared statement public: - SQLRemoveUser(void **db, CephContext *cct) : SQLiteDB((sqlite3 *)(*db), cct), sdb((sqlite3 **)db) {} + SQLRemoveUser(void **db, string db_name, CephContext *cct) : SQLiteDB((sqlite3 *)(*db), db_name, cct), sdb((sqlite3 **)db) {} ~SQLRemoveUser() { if (stmt) sqlite3_finalize(stmt); @@ -118,7 +119,7 @@ class SQLGetUser : public SQLiteDB, public GetUserOp { sqlite3_stmt *userid_stmt = NULL; // Prepared statement to query by user_id public: - SQLGetUser(void **db, CephContext *cct) : SQLiteDB((sqlite3 *)(*db), cct), sdb((sqlite3 **)db) {} + SQLGetUser(void **db, string db_name, CephContext *cct) : SQLiteDB((sqlite3 *)(*db), db_name, cct), sdb((sqlite3 **)db) {} ~SQLGetUser() { if (stmt) sqlite3_finalize(stmt); @@ -140,7 +141,7 @@ class SQLInsertBucket : public SQLiteDB, public InsertBucketOp { sqlite3_stmt *stmt = NULL; // Prepared statement public: - SQLInsertBucket(void **db, CephContext *cct) : SQLiteDB((sqlite3 *)(*db), cct), sdb((sqlite3 **)db) {} + SQLInsertBucket(void **db, string db_name, CephContext *cct) : SQLiteDB((sqlite3 *)(*db), db_name, cct), sdb((sqlite3 **)db) {} ~SQLInsertBucket() { if (stmt) sqlite3_finalize(stmt); @@ -158,7 +159,7 @@ class SQLUpdateBucket : public SQLiteDB, public UpdateBucketOp { sqlite3_stmt *owner_stmt = NULL; // Prepared statement public: - SQLUpdateBucket(void **db, CephContext *cct) : SQLiteDB((sqlite3 *)(*db), cct), sdb((sqlite3 **)db) {} + SQLUpdateBucket(void **db, string db_name, CephContext *cct) : SQLiteDB((sqlite3 *)(*db), db_name, cct), sdb((sqlite3 **)db) {} ~SQLUpdateBucket() { if (info_stmt) sqlite3_finalize(info_stmt); @@ -178,7 +179,7 @@ class SQLRemoveBucket : public SQLiteDB, public RemoveBucketOp { sqlite3_stmt *stmt = NULL; // Prepared statement public: - SQLRemoveBucket(void **db, CephContext *cct) : SQLiteDB((sqlite3 *)(*db), cct), sdb((sqlite3 **)db) {} + SQLRemoveBucket(void **db, string db_name, CephContext *cct) : SQLiteDB((sqlite3 *)(*db), db_name, cct), sdb((sqlite3 **)db) {} ~SQLRemoveBucket() { if (stmt) sqlite3_finalize(stmt); @@ -194,7 +195,7 @@ class SQLGetBucket : public SQLiteDB, public GetBucketOp { sqlite3_stmt *stmt = NULL; // Prepared statement public: - SQLGetBucket(void **db, CephContext *cct) : SQLiteDB((sqlite3 *)(*db), cct), sdb((sqlite3 **)db) {} + SQLGetBucket(void **db, string db_name, CephContext *cct) : SQLiteDB((sqlite3 *)(*db), db_name, cct), sdb((sqlite3 **)db) {} ~SQLGetBucket() { if (stmt) sqlite3_finalize(stmt); @@ -210,7 +211,7 @@ class SQLListUserBuckets : public SQLiteDB, public ListUserBucketsOp { sqlite3_stmt *stmt = NULL; // Prepared statement public: - SQLListUserBuckets(void **db, CephContext *cct) : SQLiteDB((sqlite3 *)(*db), cct), sdb((sqlite3 **)db) {} + SQLListUserBuckets(void **db, string db_name, CephContext *cct) : SQLiteDB((sqlite3 *)(*db), db_name, cct), sdb((sqlite3 **)db) {} ~SQLListUserBuckets() { if (stmt) sqlite3_finalize(stmt); @@ -220,16 +221,16 @@ class SQLListUserBuckets : public SQLiteDB, public ListUserBucketsOp { int Bind(const DoutPrefixProvider *dpp, DBOpParams *params); }; -class SQLInsertObject : public SQLiteDB, public InsertObjectOp { +class SQLPutObject : public SQLiteDB, public PutObjectOp { private: sqlite3 **sdb = NULL; sqlite3_stmt *stmt = NULL; // Prepared statement public: - SQLInsertObject(void **db, CephContext *cct) : SQLiteDB((sqlite3 *)(*db), cct), sdb((sqlite3 **)db) {} - SQLInsertObject(sqlite3 **sdbi, CephContext *cct) : SQLiteDB(*sdbi, cct), sdb(sdbi) {} + SQLPutObject(void **db, string db_name, CephContext *cct) : SQLiteDB((sqlite3 *)(*db), db_name, cct), sdb((sqlite3 **)db) {} + SQLPutObject(sqlite3 **sdbi, string db_name, CephContext *cct) : SQLiteDB(*sdbi, db_name, cct), sdb(sdbi) {} - ~SQLInsertObject() { + ~SQLPutObject() { if (stmt) sqlite3_finalize(stmt); } @@ -238,16 +239,16 @@ class SQLInsertObject : public SQLiteDB, public InsertObjectOp { int Bind(const DoutPrefixProvider *dpp, DBOpParams *params); }; -class SQLRemoveObject : public SQLiteDB, public RemoveObjectOp { +class SQLDeleteObject : public SQLiteDB, public DeleteObjectOp { private: sqlite3 **sdb = NULL; sqlite3_stmt *stmt = NULL; // Prepared statement public: - SQLRemoveObject(void **db, CephContext *cct) : SQLiteDB((sqlite3 *)(*db), cct), sdb((sqlite3 **)db) {} - SQLRemoveObject(sqlite3 **sdbi, CephContext *cct) : SQLiteDB(*sdbi, cct), sdb(sdbi) {} + SQLDeleteObject(void **db, string db_name, CephContext *cct) : SQLiteDB((sqlite3 *)(*db), db_name, cct), sdb((sqlite3 **)db) {} + SQLDeleteObject(sqlite3 **sdbi, string db_name, CephContext *cct) : SQLiteDB(*sdbi, db_name, cct), sdb(sdbi) {} - ~SQLRemoveObject() { + ~SQLDeleteObject() { if (stmt) sqlite3_finalize(stmt); } @@ -256,16 +257,59 @@ class SQLRemoveObject : public SQLiteDB, public RemoveObjectOp { int Bind(const DoutPrefixProvider *dpp, DBOpParams *params); }; -class SQLListObject : public SQLiteDB, public ListObjectOp { +class SQLGetObject : public SQLiteDB, public GetObjectOp { + private: + sqlite3 **sdb = NULL; + sqlite3_stmt *stmt = NULL; // Prepared statement + + public: + SQLGetObject(void **db, string db_name, CephContext *cct) : SQLiteDB((sqlite3 *)(*db), db_name, cct), sdb((sqlite3 **)db) {} + SQLGetObject(sqlite3 **sdbi, string db_name, CephContext *cct) : SQLiteDB(*sdbi, db_name, cct), sdb(sdbi) {} + + ~SQLGetObject() { + if (stmt) + sqlite3_finalize(stmt); + } + int Prepare(const DoutPrefixProvider *dpp, DBOpParams *params); + int Execute(const DoutPrefixProvider *dpp, DBOpParams *params); + int Bind(const DoutPrefixProvider *dpp, DBOpParams *params); +}; + +class SQLUpdateObject : public SQLiteDB, public UpdateObjectOp { + private: + sqlite3 **sdb = NULL; + sqlite3_stmt *omap_stmt = NULL; // Prepared statement + sqlite3_stmt *attrs_stmt = NULL; // Prepared statement + sqlite3_stmt *meta_stmt = NULL; // Prepared statement + + public: + SQLUpdateObject(void **db, string db_name, CephContext *cct) : SQLiteDB((sqlite3 *)(*db), db_name, cct), sdb((sqlite3 **)db) {} + SQLUpdateObject(sqlite3 **sdbi, string db_name, CephContext *cct) : SQLiteDB(*sdbi, db_name, cct), sdb(sdbi) {} + + ~SQLUpdateObject() { + if (omap_stmt) + sqlite3_finalize(omap_stmt); + if (attrs_stmt) + sqlite3_finalize(attrs_stmt); + if (meta_stmt) + sqlite3_finalize(meta_stmt); + } + + int Prepare(const DoutPrefixProvider *dpp, DBOpParams *params); + int Execute(const DoutPrefixProvider *dpp, DBOpParams *params); + int Bind(const DoutPrefixProvider *dpp, DBOpParams *params); +}; + +class SQLListBucketObjects : public SQLiteDB, public ListBucketObjectsOp { private: sqlite3 **sdb = NULL; sqlite3_stmt *stmt = NULL; // Prepared statement public: - SQLListObject(void **db, CephContext *cct) : SQLiteDB((sqlite3 *)(*db), cct), sdb((sqlite3 **)db) {} - SQLListObject(sqlite3 **sdbi, CephContext *cct) : SQLiteDB(*sdbi, cct), sdb(sdbi) {} + SQLListBucketObjects(void **db, string db_name, CephContext *cct) : SQLiteDB((sqlite3 *)(*db), db_name, cct), sdb((sqlite3 **)db) {} + SQLListBucketObjects(sqlite3 **sdbi, string db_name, CephContext *cct) : SQLiteDB(*sdbi, db_name, cct), sdb(sdbi) {} - ~SQLListObject() { + ~SQLListBucketObjects() { if (stmt) sqlite3_finalize(stmt); } @@ -280,8 +324,8 @@ class SQLPutObjectData : public SQLiteDB, public PutObjectDataOp { sqlite3_stmt *stmt = NULL; // Prepared statement public: - SQLPutObjectData(void **db, CephContext *cct) : SQLiteDB((sqlite3 *)(*db), cct), sdb((sqlite3 **)db) {} - SQLPutObjectData(sqlite3 **sdbi, CephContext *cct) : SQLiteDB(*sdbi, cct), sdb(sdbi) {} + SQLPutObjectData(void **db, string db_name, CephContext *cct) : SQLiteDB((sqlite3 *)(*db), db_name, cct), sdb((sqlite3 **)db) {} + SQLPutObjectData(sqlite3 **sdbi, string db_name, CephContext *cct) : SQLiteDB(*sdbi, db_name, cct), sdb(sdbi) {} ~SQLPutObjectData() { if (stmt) @@ -298,8 +342,8 @@ class SQLGetObjectData : public SQLiteDB, public GetObjectDataOp { sqlite3_stmt *stmt = NULL; // Prepared statement public: - SQLGetObjectData(void **db, CephContext *cct) : SQLiteDB((sqlite3 *)(*db), cct), sdb((sqlite3 **)db) {} - SQLGetObjectData(sqlite3 **sdbi, CephContext *cct) : SQLiteDB(*sdbi, cct), sdb(sdbi) {} + SQLGetObjectData(void **db, string db_name, CephContext *cct) : SQLiteDB((sqlite3 *)(*db), db_name, cct), sdb((sqlite3 **)db) {} + SQLGetObjectData(sqlite3 **sdbi, string db_name, CephContext *cct) : SQLiteDB(*sdbi, db_name, cct), sdb(sdbi) {} ~SQLGetObjectData() { if (stmt) @@ -316,8 +360,8 @@ class SQLDeleteObjectData : public SQLiteDB, public DeleteObjectDataOp { sqlite3_stmt *stmt = NULL; // Prepared statement public: - SQLDeleteObjectData(void **db, CephContext *cct) : SQLiteDB((sqlite3 *)(*db), cct), sdb((sqlite3 **)db) {} - SQLDeleteObjectData(sqlite3 **sdbi, CephContext *cct) : SQLiteDB(*sdbi, cct), sdb(sdbi) {} + SQLDeleteObjectData(void **db, string db_name, CephContext *cct) : SQLiteDB((sqlite3 *)(*db), db_name, cct), sdb((sqlite3 **)db) {} + SQLDeleteObjectData(sqlite3 **sdbi, string db_name, CephContext *cct) : SQLiteDB(*sdbi, db_name, cct), sdb(sdbi) {} ~SQLDeleteObjectData() { if (stmt) diff --git a/src/rgw/store/dbstore/tests/dbstore_tests.cc b/src/rgw/store/dbstore/tests/dbstore_tests.cc index d049935aedf..170d5d48d12 100644 --- a/src/rgw/store/dbstore/tests/dbstore_tests.cc +++ b/src/rgw/store/dbstore/tests/dbstore_tests.cc @@ -58,6 +58,19 @@ namespace gtest { ceph::real_time bucket_mtime = real_clock::now(); string marker1; +class DBGetDataCB : public RGWGetDataCB { + public: + bufferlist data_bl; + off_t data_ofs, data_len; + + int handle_data(bufferlist& bl, off_t bl_ofs, off_t bl_len) { + data_bl = bl; + data_ofs = bl_ofs; + data_len = bl_len; + return 0; + } +}; + namespace { class DBStoreTest : public ::testing::Test { @@ -82,10 +95,11 @@ namespace { GlobalParams.op.user.uinfo.display_name = user1; GlobalParams.op.user.uinfo.user_id.id = user_id1; GlobalParams.op.bucket.info.bucket.name = bucket1; - GlobalParams.object = object1; - GlobalParams.offset = 0; - GlobalParams.data = data; - GlobalParams.datalen = data.length(); + GlobalParams.op.obj.state.obj.bucket = GlobalParams.op.bucket.info.bucket; + GlobalParams.op.obj.state.obj.key.name = object1; + GlobalParams.op.obj.state.obj.key.instance = "inst1"; + GlobalParams.op.obj_data.part_num = 0; + /* As of now InitializeParams doesnt do anything * special based on fop. Hence its okay to do @@ -581,34 +595,374 @@ TEST_F(DBStoreTest, ListAllBuckets) { ASSERT_EQ(ret, 0); } -TEST_F(DBStoreTest, InsertObject) { +TEST_F(DBStoreTest, PutObject) { struct DBOpParams params = GlobalParams; int ret = -1; - ret = db->ProcessOp(dpp, "InsertObject", ¶ms); + params.op.obj.category = RGWObjCategory::Main; + params.op.obj.storage_class = "STANDARD"; + bufferlist b1; + encode("HELLO WORLD", b1); + cout<<"XXXXXXXXX Insert b1.length " << b1.length() << "\n"; + params.op.obj.head_data = b1; + params.op.obj.state.size = 12; + params.op.obj.state.is_olh = false; + ret = db->ProcessOp(dpp, "PutObject", ¶ms); + ASSERT_EQ(ret, 0); + + /* Insert another object */ + params.op.obj.state.obj.key.name = "object2"; + params.op.obj.state.obj.key.instance = "inst2"; + ret = db->ProcessOp(dpp, "PutObject", ¶ms); ASSERT_EQ(ret, 0); } -TEST_F(DBStoreTest, ListObject) { +TEST_F(DBStoreTest, ListAllObjects) { struct DBOpParams params = GlobalParams; int ret = -1; - ret = db->ProcessOp(dpp, "ListObject", ¶ms); + ret = db->ListAllObjects(dpp, ¶ms); + ASSERT_GE(ret, 0); +} + +TEST_F(DBStoreTest, GetObject) { + struct DBOpParams params = GlobalParams; + int ret = -1; + + ret = db->ProcessOp(dpp, "GetObject", ¶ms); ASSERT_EQ(ret, 0); + ASSERT_EQ(params.op.obj.category, RGWObjCategory::Main); + ASSERT_EQ(params.op.obj.storage_class, "STANDARD"); + string data; + decode(data, params.op.obj.head_data); + ASSERT_EQ(data, "HELLO WORLD"); + ASSERT_EQ(params.op.obj.state.size, 12); } -TEST_F(DBStoreTest, ListAllObjects) { +TEST_F(DBStoreTest, GetObjectState) { struct DBOpParams params = GlobalParams; int ret = -1; + RGWObjState state; + RGWObjState *s = &state; - ret = db->ListAllObjects(dpp, ¶ms); + params.op.obj.state.obj.key.name = "object2"; + params.op.obj.state.obj.key.instance = "inst2"; + DB::Object op_target(db, params.op.bucket.info, + params.op.obj.state.obj); + + ret = op_target.get_obj_state(dpp, params.op.bucket.info, params.op.obj.state.obj, + false, &s); ASSERT_EQ(ret, 0); + ASSERT_EQ(state.size, 12); + ASSERT_EQ(state.is_olh, false); + + /* Recheck with get_state API */ + ret = op_target.get_state(dpp, &s, false); + ASSERT_EQ(ret, 0); + ASSERT_EQ(state.size, 12); + ASSERT_EQ(state.is_olh, false); +} + +TEST_F(DBStoreTest, ObjAttrs) { + struct DBOpParams params = GlobalParams; + int ret = -1; + map setattrs; + map rmattrs; + map readattrs; + + bufferlist b1, b2, b3; + encode("ACL", b1); + setattrs[RGW_ATTR_ACL] = b1; + encode("LC", b2); + setattrs[RGW_ATTR_LC] = b2; + encode("ETAG", b3); + setattrs[RGW_ATTR_ETAG] = b3; + + DB::Object op_target(db, params.op.bucket.info, + params.op.obj.state.obj); + + /* Set some attrs */ + ret = op_target.set_attrs(dpp, setattrs, nullptr); + ASSERT_EQ(ret, 0); + + /* read those attrs */ + DB::Object::Read read_op(&op_target); + read_op.params.attrs = &readattrs; + ret = read_op.prepare(dpp); + ASSERT_EQ(ret, 0); + + string val; + decode(val, readattrs[RGW_ATTR_ACL]); + ASSERT_EQ(val, "ACL"); + decode(val, readattrs[RGW_ATTR_LC]); + ASSERT_EQ(val, "LC"); + decode(val, readattrs[RGW_ATTR_ETAG]); + ASSERT_EQ(val, "ETAG"); + + /* Remove some attrs */ + rmattrs[RGW_ATTR_ACL] = b1; + map empty; + ret = op_target.set_attrs(dpp, empty, &rmattrs); + ASSERT_EQ(ret, 0); + + /* read those attrs */ + ret = read_op.prepare(dpp); + ASSERT_EQ(ret, 0); + + ASSERT_EQ(readattrs.count(RGW_ATTR_ACL), 0); + decode(val, readattrs[RGW_ATTR_LC]); + ASSERT_EQ(val, "LC"); + decode(val, readattrs[RGW_ATTR_ETAG]); + ASSERT_EQ(val, "ETAG"); +} + +TEST_F(DBStoreTest, WriteObject) { + struct DBOpParams params = GlobalParams; + int ret = -1; + map setattrs; + params.op.obj.state.obj.key.name = "object3"; + params.op.obj.state.obj.key.instance = "inst3"; + DB::Object op_target(db, params.op.bucket.info, + params.op.obj.state.obj); + DB::Object::Write write_op(&op_target); + ret = write_op.prepare(dpp); + ASSERT_EQ(ret, 0); + + write_op.meta.mtime = &bucket_mtime; + write_op.meta.category = RGWObjCategory::Main; + write_op.meta.owner = params.op.user.uinfo.user_id; + + bufferlist b1; + encode("HELLO WORLD - Object3", b1); + cout<<"XXXXXXXXX Insert b1.length " << b1.length() << "\n"; + write_op.meta.data = &b1; + + bufferlist b2; + encode("ACL", b2); + setattrs[RGW_ATTR_ACL] = b2; + + ret = write_op.write_meta(0, 22, 25, setattrs); + ASSERT_EQ(ret, 0); +} + +TEST_F(DBStoreTest, ReadObject) { + struct DBOpParams params = GlobalParams; + int ret = -1; + map readattrs; + params.op.obj.state.obj.key.name = "object3"; + params.op.obj.state.obj.key.instance = "inst3"; + uint64_t obj_size; + DB::Object op_target(db, params.op.bucket.info, + params.op.obj.state.obj); + DB::Object::Read read_op(&op_target); + read_op.params.attrs = &readattrs; + read_op.params.obj_size = &obj_size; + ret = read_op.prepare(dpp); + ASSERT_EQ(ret, 0); + + bufferlist bl; + ret = read_op.read(0, 25, bl, dpp); + cout<<"XXXXXXXXX Insert bl.length " << bl.length() << "\n"; + ASSERT_EQ(ret, 25); + + string data; + decode(data, bl); + ASSERT_EQ(data, "HELLO WORLD - Object3"); + ASSERT_EQ(obj_size, 22); +} + +TEST_F(DBStoreTest, IterateObject) { + struct DBOpParams params = GlobalParams; + int ret = -1; + map readattrs; + uint64_t obj_size; + DBGetDataCB cb; + + DB::Object op_target(db, params.op.bucket.info, + params.op.obj.state.obj); + DB::Object::Read read_op(&op_target); + read_op.params.attrs = &readattrs; + read_op.params.obj_size = &obj_size; + ret = read_op.prepare(dpp); + ASSERT_EQ(ret, 0); + + bufferlist bl; + ret = read_op.iterate(dpp, 0, 15, &cb); + ASSERT_EQ(ret, 0); + string data; + decode(data, cb.data_bl); + cout << "XXXXXXXXXX iterate data is " << data << ", bl_ofs = " << cb.data_ofs << ", bl_len = " << cb.data_len << "\n"; + ASSERT_EQ(data, "HELLO WORLD"); + ASSERT_EQ(cb.data_ofs, 0); + ASSERT_EQ(cb.data_len, 15); +} + +TEST_F(DBStoreTest, ListBucketObjects) { + struct DBOpParams params = GlobalParams; + int ret = -1; + + int max = 2; + bool is_truncated = false; + rgw_obj_key marker1; + DB::Bucket target(db, params.op.bucket.info); + DB::Bucket::List list_op(&target); + + vector dir_list; + + marker1.name = ""; + do { + is_truncated = false; + list_op.params.marker = marker1; + ret = list_op.list_objects(dpp, max, &dir_list, nullptr, &is_truncated); + ASSERT_EQ(ret, 0); + + cout << "marker1 :" << marker1.name << "\n"; + + cout << "is_truncated :" << is_truncated << "\n"; + + for (const auto& ent: dir_list) { + cls_rgw_obj_key key = ent.key; + cout << "###################### \n"; + cout << "key.name : " << key.name << "\n"; + cout << "key.instance : " << key.instance << "\n"; + + marker1 = list_op.get_next_marker(); + } + dir_list.clear(); + } while(is_truncated); +} + +TEST_F(DBStoreTest, DeleteObj) { + struct DBOpParams params = GlobalParams; + int ret = -1; + RGWObjState state; + RGWObjState *s = &state; + + /* delete object2 */ + params.op.obj.state.obj.key.name = "object2"; + params.op.obj.state.obj.key.instance = "inst2"; + DB::Object op_target(db, params.op.bucket.info, + params.op.obj.state.obj); + + DB::Object::Delete delete_op(&op_target); + ret = delete_op.delete_obj(dpp); + ASSERT_EQ(ret, 0); + + /* Should return ENOENT */ + ret = op_target.get_state(dpp, &s, false); + ASSERT_EQ(ret, -2); +} + +TEST_F(DBStoreTest, ObjectOmapSetVal) { + struct DBOpParams params = GlobalParams; + int ret = -1; + + DB::Object op_target(db, params.op.bucket.info, + params.op.obj.state.obj); + + string val = "part1_val"; + bufferlist bl; + encode(val, bl); + ret = op_target.obj_omap_set_val_by_key(dpp, "part1", bl, false); + ASSERT_EQ(ret, 0); + + val = "part2_val"; + bl.clear(); + encode(val, bl); + ret = op_target.obj_omap_set_val_by_key(dpp, "part2", bl, false); + ASSERT_EQ(ret, 0); + + val = "part3_val"; + bl.clear(); + encode(val, bl); + ret = op_target.obj_omap_set_val_by_key(dpp, "part3", bl, false); + ASSERT_EQ(ret, 0); + + val = "part4_val"; + bl.clear(); + encode(val, bl); + ret = op_target.obj_omap_set_val_by_key(dpp, "part4", bl, false); + ASSERT_EQ(ret, 0); +} + +TEST_F(DBStoreTest, ObjectOmapGetValsByKeys) { + struct DBOpParams params = GlobalParams; + int ret = -1; + std::set keys; + std::map vals; + + DB::Object op_target(db, params.op.bucket.info, + params.op.obj.state.obj); + + keys.insert("part2"); + keys.insert("part4"); + + ret = op_target.obj_omap_get_vals_by_keys(dpp, "", keys, &vals); + ASSERT_EQ(ret, 0); + ASSERT_EQ(vals.size(), 2); + + string val; + decode(val, vals["part2"]); + ASSERT_EQ(val, "part2_val"); + decode(val, vals["part4"]); + ASSERT_EQ(val, "part4_val"); +} + +TEST_F(DBStoreTest, ObjectOmapGetAll) { + struct DBOpParams params = GlobalParams; + int ret = -1; + std::map vals; + + DB::Object op_target(db, params.op.bucket.info, + params.op.obj.state.obj); + + ret = op_target.obj_omap_get_all(dpp, &vals); + ASSERT_EQ(ret, 0); + ASSERT_EQ(vals.size(), 4); + + string val; + decode(val, vals["part1"]); + ASSERT_EQ(val, "part1_val"); + decode(val, vals["part2"]); + ASSERT_EQ(val, "part2_val"); + decode(val, vals["part3"]); + ASSERT_EQ(val, "part3_val"); + decode(val, vals["part4"]); + ASSERT_EQ(val, "part4_val"); +} + +TEST_F(DBStoreTest, ObjectOmapGetVals) { + struct DBOpParams params = GlobalParams; + int ret = -1; + std::set keys; + std::map vals; + bool pmore; + + DB::Object op_target(db, params.op.bucket.info, + params.op.obj.state.obj); + + ret = op_target.obj_omap_get_vals(dpp, "part3", 10, &vals, &pmore); + ASSERT_EQ(ret, 0); + ASSERT_EQ(vals.size(), 2); + + string val; + decode(val, vals["part3"]); + ASSERT_EQ(val, "part3_val"); + decode(val, vals["part4"]); + ASSERT_EQ(val, "part4_val"); } TEST_F(DBStoreTest, PutObjectData) { struct DBOpParams params = GlobalParams; int ret = -1; + params.op.obj_data.part_num = 1; + params.op.obj_data.offset = 10; + params.op.obj_data.multipart_part_num = 2; + bufferlist b1; + encode("HELLO WORLD", b1); + params.op.obj_data.data = b1; + params.op.obj_data.size = 12; ret = db->ProcessOp(dpp, "PutObjectData", ¶ms); ASSERT_EQ(ret, 0); } @@ -619,6 +973,12 @@ TEST_F(DBStoreTest, GetObjectData) { ret = db->ProcessOp(dpp, "GetObjectData", ¶ms); ASSERT_EQ(ret, 0); + ASSERT_EQ(params.op.obj_data.part_num, 1); + ASSERT_EQ(params.op.obj_data.offset, 10); + ASSERT_EQ(params.op.obj_data.multipart_part_num, 2); + string data; + decode(data, params.op.obj_data.data); + ASSERT_EQ(data, "HELLO WORLD"); } TEST_F(DBStoreTest, DeleteObjectData) { @@ -629,11 +989,11 @@ TEST_F(DBStoreTest, DeleteObjectData) { ASSERT_EQ(ret, 0); } -TEST_F(DBStoreTest, RemoveObject) { +TEST_F(DBStoreTest, DeleteObject) { struct DBOpParams params = GlobalParams; int ret = -1; - ret = db->ProcessOp(dpp, "RemoveObject", ¶ms); + ret = db->ProcessOp(dpp, "DeleteObject", ¶ms); ASSERT_EQ(ret, 0); } @@ -678,9 +1038,9 @@ int main(int argc, char **argv) // format: ./dbstore-tests logfile loglevel if (argc == 3) { - c_logfile = argv[1]; - c_loglevel = (atoi)(argv[2]); - cout << "logfile:" << c_logfile << ", loglevel set to " << c_loglevel << "\n"; + c_logfile = argv[1]; + c_loglevel = (atoi)(argv[2]); + cout << "logfile:" << c_logfile << ", loglevel set to " << c_loglevel << "\n"; } ::testing::InitGoogleTest(&argc, argv); -- 2.39.5