From: Jane Zhu Date: Mon, 17 Jun 2024 07:41:51 +0000 (-0400) Subject: rgw/multipart: use cls_version to avoid racing between part upload and multipart... X-Git-Tag: testing/wip-vshankar-testing-20241122.180955-squid-debug~15^2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=2488ec88626ca0c3b95ef759717bec03122b5c5e;p=ceph-ci.git rgw/multipart: use cls_version to avoid racing between part upload and multipart complete Signed-off-by: Jane Zhu (cherry picked from commit 451b70dedb9975a605458b2dae83de61a107c936) --- diff --git a/src/rgw/driver/daos/rgw_sal_daos.cc b/src/rgw/driver/daos/rgw_sal_daos.cc index bafd4dcaf91..0b169a949da 100644 --- a/src/rgw/driver/daos/rgw_sal_daos.cc +++ b/src/rgw/driver/daos/rgw_sal_daos.cc @@ -1198,7 +1198,8 @@ int DaosObject::DaosDeleteOp::delete_obj(const DoutPrefixProvider* dpp, } int DaosObject::delete_object(const DoutPrefixProvider* dpp, optional_yield y, - uint32_t flags) { + uint32_t flags, std::list* remove_objs, + RGWObjVersionTracker* objv) { ldpp_dout(dpp, 20) << "DEBUG: delete_object" << dendl; DaosObject::DaosDeleteOp del_op(this); del_op.params.bucket_owner = bucket->get_info().owner; @@ -1678,7 +1679,8 @@ int DaosMultipartUpload::complete( map& part_etags, list& remove_objs, uint64_t& accounted_size, bool& compressed, RGWCompressionInfo& cs_info, off_t& off, std::string& tag, ACLOwner& owner, uint64_t olh_epoch, - rgw::sal::Object* target_obj) { + rgw::sal::Object* target_obj, + prefix_map_t& processed_prefixes) { ldpp_dout(dpp, 20) << "DEBUG: complete" << dendl; char final_etag[CEPH_CRYPTO_MD5_DIGESTSIZE]; char final_etag_str[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 16]; @@ -1923,6 +1925,15 @@ int DaosMultipartUpload::complete( return ret; } +int DaosMultipartUpload::cleanup_orphaned_parts(const DoutPrefixProvider *dpp, + CephContext *cct, optional_yield y, + const rgw_obj& obj, + std::list& remove_objs, + prefix_map_t& processed_prefixes) +{ + return -ENOTSUP; +} + int DaosMultipartUpload::get_info(const DoutPrefixProvider* dpp, optional_yield y, rgw_placement_rule** rule, rgw::sal::Attrs* attrs) { diff --git a/src/rgw/driver/daos/rgw_sal_daos.h b/src/rgw/driver/daos/rgw_sal_daos.h index cf1583fc174..91122e3e063 100644 --- a/src/rgw/driver/daos/rgw_sal_daos.h +++ b/src/rgw/driver/daos/rgw_sal_daos.h @@ -596,7 +596,8 @@ class DaosObject : public StoreObject { virtual ~DaosObject(); virtual int delete_object(const DoutPrefixProvider* dpp, optional_yield y, - uint32_t flags) override; + uint32_t flags, std::list* remove_objs, + RGWObjVersionTracker* objv) override; virtual int copy_object( const ACLOwner& owner, const rgw_user& remote_user, req_info* info, const rgw_zone_id& source_zone, @@ -855,7 +856,13 @@ class DaosMultipartUpload : public StoreMultipartUpload { uint64_t& accounted_size, bool& compressed, RGWCompressionInfo& cs_info, off_t& off, std::string& tag, ACLOwner& owner, uint64_t olh_epoch, - rgw::sal::Object* target_obj) override; + rgw::sal::Object* target_obj, + prefix_map_t& processed_prefixes) override; + virtual int cleanup_orphaned_parts(const DoutPrefixProvider *dpp, + CephContext *cct, optional_yield y, + const rgw_obj& obj, + std::list& remove_objs, + prefix_map_t& processed_prefixes) override; virtual int get_info(const DoutPrefixProvider* dpp, optional_yield y, rgw_placement_rule** rule, rgw::sal::Attrs* attrs = nullptr) override; diff --git a/src/rgw/driver/motr/rgw_sal_motr.cc b/src/rgw/driver/motr/rgw_sal_motr.cc index 0099c9c107f..6a078bdaa25 100644 --- a/src/rgw/driver/motr/rgw_sal_motr.cc +++ b/src/rgw/driver/motr/rgw_sal_motr.cc @@ -585,7 +585,7 @@ int MotrBucket::remove(const DoutPrefixProvider *dpp, bool delete_children, opti std::unique_ptr object = get_object(key); - ret = object->delete_object(dpp, null_yield, rgw::sal::FLAG_LOG_OP); + ret = object->delete_object(dpp, null_yield, rgw::sal::FLAG_LOG_OP, nullptr, nullptr); if (ret < 0 && ret != -ENOENT) { ldpp_dout(dpp, 0) << "ERROR: remove_bucket rgw_remove_object failed rc=" << ret << dendl; return ret; @@ -1502,7 +1502,11 @@ int MotrObject::MotrDeleteOp::delete_obj(const DoutPrefixProvider* dpp, optional return 0; } -int MotrObject::delete_object(const DoutPrefixProvider* dpp, optional_yield y, uint32_t flags) +int MotrObject::delete_object(const DoutPrefixProvider* dpp, + optional_yield y, + uint32_t flags, + std::list* remove_objs, + RGWObjVersionTracker* objv) { MotrObject::MotrDeleteOp del_op(this); del_op.params.bucket_owner = bucket->get_info().owner; @@ -2668,7 +2672,8 @@ int MotrMultipartUpload::complete(const DoutPrefixProvider *dpp, RGWCompressionInfo& cs_info, off_t& off, std::string& tag, ACLOwner& owner, uint64_t olh_epoch, - rgw::sal::Object* target_obj) + rgw::sal::Object* target_obj, + prefix_map_t& processed_prefixes) { char final_etag[CEPH_CRYPTO_MD5_DIGESTSIZE]; char final_etag_str[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 16]; @@ -2878,6 +2883,15 @@ int MotrMultipartUpload::complete(const DoutPrefixProvider *dpp, M0_IC_DEL, meta_obj->get_key().get_oid(), bl); } +int MotrMultipartUpload::cleanup_orphaned_parts(const DoutPrefixProvider *dpp, + CephContext *cct, optional_yield y, + const rgw_obj& obj, + std::list& remove_objs, + prefix_map_t& processed_prefixes) +{ + return -ENOTSUP; +} + int MotrMultipartUpload::get_info(const DoutPrefixProvider *dpp, optional_yield y, rgw_placement_rule** rule, rgw::sal::Attrs* attrs) { if (!rule && !attrs) { diff --git a/src/rgw/driver/motr/rgw_sal_motr.h b/src/rgw/driver/motr/rgw_sal_motr.h index e278728c7e7..5e1e4ae271a 100644 --- a/src/rgw/driver/motr/rgw_sal_motr.h +++ b/src/rgw/driver/motr/rgw_sal_motr.h @@ -657,7 +657,9 @@ class MotrObject : public StoreObject { virtual int delete_object(const DoutPrefixProvider* dpp, optional_yield y, - uint32_t flags) override; + uint32_t flags, + td::list* remove_objs, + GWObjVersionTracker* objv) override; virtual int copy_object(const ACLOwner& owner, const rgw_user& remote_user, req_info* info, const rgw_zone_id& source_zone, @@ -928,7 +930,13 @@ public: RGWCompressionInfo& cs_info, off_t& off, std::string& tag, ACLOwner& owner, uint64_t olh_epoch, - rgw::sal::Object* target_obj) override; + rgw::sal::Object* target_obj, + prefix_map_t& processed_prefixes) override; + virtual int cleanup_orphaned_parts(const DoutPrefixProvider *dpp, + CephContext *cct, optional_yield y, + const rgw_obj& obj, + std::list& remove_objs, + prefix_map_t& processed_prefixes) override; virtual int get_info(const DoutPrefixProvider *dpp, optional_yield y, rgw_placement_rule** rule, rgw::sal::Attrs* attrs = nullptr) override; virtual std::unique_ptr get_writer(const DoutPrefixProvider *dpp, optional_yield y, diff --git a/src/rgw/driver/posix/rgw_sal_posix.cc b/src/rgw/driver/posix/rgw_sal_posix.cc index 9b1b34fa9e4..534b0ec7db2 100644 --- a/src/rgw/driver/posix/rgw_sal_posix.cc +++ b/src/rgw/driver/posix/rgw_sal_posix.cc @@ -1376,7 +1376,7 @@ int POSIXBucket::copy(const DoutPrefixProvider *dpp, optional_yield y, std::unique_ptr dsb; // Delete the target, in case it's not a multipart - int ret = dest->delete_object(dpp, y, rgw::sal::FLAG_LOG_OP); + int ret = dest->delete_object(dpp, y, rgw::sal::FLAG_LOG_OP, nullptr, nullptr); if (ret < 0) { ldpp_dout(dpp, 0) << "ERROR: could not remove dest object " << dest->get_name() << dendl; @@ -1428,7 +1428,9 @@ int POSIXBucket::copy(const DoutPrefixProvider *dpp, optional_yield y, int POSIXObject::delete_object(const DoutPrefixProvider* dpp, optional_yield y, - uint32_t flags) + uint32_t flags, + std::list* remove_objs, + RGWObjVersionTracker* objv) { POSIXBucket *b = static_cast(get_bucket()); if (!b) { @@ -1922,7 +1924,7 @@ int POSIXObject::link_temp_file(const DoutPrefixProvider *dpp, optional_yield y, } // Delete the target, in case it's a multipart - ret = delete_object(dpp, y, flags); + ret = delete_object(dpp, y, flags, nullptr, nullptr); if (ret < 0) { ldpp_dout(dpp, 0) << "ERROR: could not remove dest object " << get_name() << dendl; @@ -2375,7 +2377,7 @@ int POSIXObject::POSIXReadOp::get_attr(const DoutPrefixProvider* dpp, const char int POSIXObject::POSIXDeleteOp::delete_obj(const DoutPrefixProvider* dpp, optional_yield y, uint32_t flags) { - return source->delete_object(dpp, y, flags); + return source->delete_object(dpp, y, flags, nullptr, nullptr); } int POSIXObject::copy(const DoutPrefixProvider *dpp, optional_yield y, @@ -2391,7 +2393,7 @@ int POSIXObject::copy(const DoutPrefixProvider *dpp, optional_yield y, } // Delete the target, in case it's a multipart - ret = dobj->delete_object(dpp, y, rgw::sal::FLAG_LOG_OP); + ret = dobj->delete_object(dpp, y, rgw::sal::FLAG_LOG_OP, nullptr, nullptr); if (ret < 0) { ldpp_dout(dpp, 0) << "ERROR: could not remove dest object " << dobj->get_name() << dendl; @@ -2601,7 +2603,8 @@ int POSIXMultipartUpload::complete(const DoutPrefixProvider *dpp, RGWCompressionInfo& cs_info, off_t& ofs, std::string& tag, ACLOwner& owner, uint64_t olh_epoch, - rgw::sal::Object* target_obj) + rgw::sal::Object* target_obj, + prefix_map_t& processed_prefixes) { char final_etag[CEPH_CRYPTO_MD5_DIGESTSIZE]; char final_etag_str[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 16]; @@ -2734,6 +2737,15 @@ int POSIXMultipartUpload::complete(const DoutPrefixProvider *dpp, return shadow->rename(dpp, y, target_obj); } +int POSIXMultipartUpload::cleanup_orphaned_parts(const DoutPrefixProvider *dpp, + CephContext *cct, optional_yield y, + const rgw_obj& obj, + std::list& remove_objs, + prefix_map_t& processed_prefixes) +{ + return -ENOTSUP; +} + int POSIXMultipartUpload::get_info(const DoutPrefixProvider *dpp, optional_yield y, rgw_placement_rule** rule, rgw::sal::Attrs* attrs) { diff --git a/src/rgw/driver/posix/rgw_sal_posix.h b/src/rgw/driver/posix/rgw_sal_posix.h index 587bf783e90..fc99060993d 100644 --- a/src/rgw/driver/posix/rgw_sal_posix.h +++ b/src/rgw/driver/posix/rgw_sal_posix.h @@ -320,7 +320,9 @@ public: virtual int delete_object(const DoutPrefixProvider* dpp, optional_yield y, - uint32_t flags) override; + uint32_t flags, + std::list* remove_objs, + RGWObjVersionTracker* objv) override; virtual int copy_object(const ACLOwner& owner, const rgw_user& remote_user, req_info* info, const rgw_zone_id& source_zone, @@ -556,7 +558,13 @@ public: RGWCompressionInfo& cs_info, off_t& ofs, std::string& tag, ACLOwner& owner, uint64_t olh_epoch, - rgw::sal::Object* target_obj) override; + rgw::sal::Object* target_obj, + prefix_map_t& processed_prefixes) override; + virtual int cleanup_orphaned_parts(const DoutPrefixProvider *dpp, + CephContext *cct, optional_yield y, + const rgw_obj& obj, + std::list& remove_objs, + prefix_map_t& processed_prefixes) override; virtual int get_info(const DoutPrefixProvider *dpp, optional_yield y, rgw_placement_rule** rule, rgw::sal::Attrs* attrs) override; diff --git a/src/rgw/driver/rados/rgw_bucket.cc b/src/rgw/driver/rados/rgw_bucket.cc index c51e61a2755..a844cc45dbe 100644 --- a/src/rgw/driver/rados/rgw_bucket.cc +++ b/src/rgw/driver/rados/rgw_bucket.cc @@ -152,7 +152,7 @@ int rgw_remove_object(const DoutPrefixProvider *dpp, rgw::sal::Driver* driver, r std::unique_ptr object = bucket->get_object(key); - return object->delete_object(dpp, y, rgw::sal::FLAG_LOG_OP); + return object->delete_object(dpp, y, rgw::sal::FLAG_LOG_OP, nullptr, nullptr); } static void set_err_msg(std::string *sink, std::string msg) diff --git a/src/rgw/driver/rados/rgw_object_expirer_core.cc b/src/rgw/driver/rados/rgw_object_expirer_core.cc index c285443d0b0..a5d788ea469 100644 --- a/src/rgw/driver/rados/rgw_object_expirer_core.cc +++ b/src/rgw/driver/rados/rgw_object_expirer_core.cc @@ -225,7 +225,7 @@ int RGWObjectExpirer::garbage_single_object(const DoutPrefixProvider *dpp, objex std::unique_ptr obj = bucket->get_object(key); obj->set_atomic(); - ret = obj->delete_object(dpp, null_yield, rgw::sal::FLAG_LOG_OP); + ret = obj->delete_object(dpp, null_yield, rgw::sal::FLAG_LOG_OP, nullptr, nullptr); return ret; } diff --git a/src/rgw/driver/rados/rgw_putobj_processor.cc b/src/rgw/driver/rados/rgw_putobj_processor.cc index d41678cdb06..5716ad3113e 100644 --- a/src/rgw/driver/rados/rgw_putobj_processor.cc +++ b/src/rgw/driver/rados/rgw_putobj_processor.cc @@ -22,6 +22,8 @@ #include "services/svc_zone.h" #include "rgw_sal_rados.h" +#include "cls/version/cls_version_client.h" + #define dout_subsys ceph_subsys_rgw using namespace std; @@ -568,7 +570,9 @@ int MultipartObjectProcessor::complete(size_t accounted_size, } librados::ObjectWriteOperation op; + op.assert_exists(); cls_rgw_mp_upload_part_info_update(op, p, info); + cls_version_inc(op); r = rgw_rados_operate(rctx.dpp, meta_obj_ref.ioctx, meta_obj_ref.obj.oid, &op, rctx.y); ldpp_dout(rctx.dpp, 20) << "Update meta: " << meta_obj_ref.obj.oid << " part " << p << " prefix " << info.manifest.get_prefix() << " return " << r << dendl; @@ -583,8 +587,10 @@ int MultipartObjectProcessor::complete(size_t accounted_size, op = librados::ObjectWriteOperation{}; op.assert_exists(); // detect races with abort op.omap_set(m); + cls_version_inc(op); r = rgw_rados_operate(rctx.dpp, meta_obj_ref.ioctx, meta_obj_ref.obj.oid, &op, rctx.y); } + if (r < 0) { return r == -ENOENT ? -ERR_NO_SUCH_UPLOAD : r; } diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc index 4ad5ce96e10..2c83347e76c 100644 --- a/src/rgw/driver/rados/rgw_rados.cc +++ b/src/rgw/driver/rados/rgw_rados.cc @@ -5842,6 +5842,10 @@ int RGWRados::Object::Delete::delete_obj(optional_yield y, const DoutPrefixProvi store->remove_rgw_head_obj(op); + if (params.check_objv != nullptr) { + cls_version_check(op, *params.check_objv, VER_COND_EQ); + } + auto& ioctx = ref.ioctx; r = rgw_rados_operate(dpp, ioctx, ref.obj.oid, &op, y); @@ -6034,7 +6038,7 @@ int RGWRados::get_obj_state_impl(const DoutPrefixProvider *dpp, RGWObjectCtx *oc int r = -ENOENT; if (!assume_noent) { - r = RGWRados::raw_obj_stat(dpp, raw_obj, &s->size, &s->mtime, &s->epoch, &s->attrset, (s->prefetch_data ? &s->data : NULL), NULL, y); + r = RGWRados::raw_obj_stat(dpp, raw_obj, &s->size, &s->mtime, &s->epoch, &s->attrset, (s->prefetch_data ? &s->data : NULL), &s->objv_tracker, y); } if (r == -ENOENT) { @@ -6757,6 +6761,10 @@ int RGWRados::Object::Read::prepare(optional_yield y, const DoutPrefixProvider * return -ENOENT; } + if (params.objv_tracker) { + *params.objv_tracker = astate->objv_tracker; + } + RGWBucketInfo& bucket_info = source->get_bucket_info(); if (params.part_num) { @@ -8674,6 +8682,7 @@ int RGWRados::raw_obj_stat(const DoutPrefixProvider *dpp, if (first_chunk) { op.read(0, cct->_conf->rgw_max_chunk_size, first_chunk, NULL); } + bufferlist outbl; r = rgw_rados_operate(dpp, ref.ioctx, ref.obj.oid, &op, &outbl, y); diff --git a/src/rgw/driver/rados/rgw_rados.h b/src/rgw/driver/rados/rgw_rados.h index 3450c6cc37b..0e229d659cb 100644 --- a/src/rgw/driver/rados/rgw_rados.h +++ b/src/rgw/driver/rados/rgw_rados.h @@ -771,6 +771,7 @@ public: uint64_t *epoch; int* part_num = nullptr; std::optional parts_count; + RGWObjVersionTracker *objv_tracker = nullptr; Params() : lastmod(nullptr), obj_size(nullptr), attrs(nullptr), target_obj(nullptr), epoch(nullptr) @@ -853,8 +854,9 @@ public: rgw_zone_set *zones_trace; bool abortmp; uint64_t parts_accounted_size; + obj_version *check_objv; - DeleteParams() : versioning_status(0), olh_epoch(0), bilog_flags(0), remove_objs(NULL), high_precision_time(false), zones_trace(nullptr), abortmp(false), parts_accounted_size(0) {} + DeleteParams() : versioning_status(0), olh_epoch(0), bilog_flags(0), remove_objs(NULL), high_precision_time(false), zones_trace(nullptr), abortmp(false), parts_accounted_size(0), check_objv(nullptr) {} } params; struct DeleteResult { diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc index 3ef5c5b03e0..3e9f9070296 100644 --- a/src/rgw/driver/rados/rgw_sal_rados.cc +++ b/src/rgw/driver/rados/rgw_sal_rados.cc @@ -2280,6 +2280,7 @@ int RadosObject::read_attrs(const DoutPrefixProvider* dpp, RGWRados::Object::Rea read_op.params.target_obj = target_obj; read_op.params.obj_size = &state.size; read_op.params.lastmod = &state.mtime; + read_op.params.objv_tracker = &state.objv_tracker; return read_op.prepare(y, dpp); } @@ -2808,6 +2809,9 @@ int RadosObject::RadosDeleteOp::delete_obj(const DoutPrefixProvider* dpp, option parent_op.params.zones_trace = params.zones_trace; parent_op.params.abortmp = params.abortmp; parent_op.params.parts_accounted_size = params.parts_accounted_size; + if (params.objv_tracker) { + parent_op.params.check_objv = params.objv_tracker->version_for_check(); + } int ret = parent_op.delete_obj(y, dpp, flags & FLAG_LOG_OP); if (ret < 0) @@ -2821,7 +2825,9 @@ int RadosObject::RadosDeleteOp::delete_obj(const DoutPrefixProvider* dpp, option int RadosObject::delete_object(const DoutPrefixProvider* dpp, optional_yield y, - uint32_t flags) + uint32_t flags, + std::list* remove_objs, + RGWObjVersionTracker* objv) { RGWRados::Object del_target(store->getRados(), bucket->get_info(), *rados_ctx, get_obj()); RGWRados::Object::Delete del_op(&del_target); @@ -2829,6 +2835,10 @@ int RadosObject::delete_object(const DoutPrefixProvider* dpp, del_op.params.bucket_owner = bucket->get_info().owner; del_op.params.versioning_status = (flags & FLAG_PREVENT_VERSIONING) ? 0 : bucket->get_info().versioning_status(); + del_op.params.remove_objs = remove_objs; + if (objv) { + del_op.params.check_objv = objv->version_for_check(); + } return del_op.delete_obj(y, dpp, flags & FLAG_LOG_OP); } @@ -2924,13 +2934,84 @@ int RadosObject::swift_versioning_copy(const ACLOwner& owner, const rgw_user& re y); } +int RadosMultipartUpload::cleanup_orphaned_parts(const DoutPrefixProvider *dpp, + CephContext *cct, optional_yield y, + const rgw_obj& obj, + list& remove_objs, + prefix_map_t& processed_prefixes) +{ + bool truncated; + int ret; + int max_parts = 1000; + int marker = 0; + cls_rgw_obj_chain chain; + + do { + ret = list_parts(dpp, cct, max_parts, marker, &marker, &truncated, y); + + if (ret < 0) { + ldpp_dout(dpp, 20) << __func__ << ": RadosMultipartUpload::list_parts returned " << ret << dendl; + return (ret == -ENOENT) ? -ERR_NO_SUCH_UPLOAD : ret; + } + + for (auto part_it = parts.begin(); part_it != parts.end(); ++part_it) { + RadosMultipartPart* part = dynamic_cast(part_it->second.get()); + + auto& part_prefixes = processed_prefixes[part->info.num]; + + if (!part->info.manifest.empty()) { + auto manifest_prefix = part->info.manifest.get_prefix(); + if (not manifest_prefix.empty() && part_prefixes.find(manifest_prefix) == part_prefixes.end()) { + store->getRados()->update_gc_chain(dpp, obj, part->info.manifest, &chain); + + RGWObjManifest::obj_iterator oiter = part->info.manifest.obj_begin(dpp); + if (oiter != part->info.manifest.obj_end(dpp)) { + rgw_raw_obj raw_head = oiter.get_location().get_raw_obj(store->getRados()); + + rgw_obj head_obj; + RGWSI_Tier_RADOS::raw_obj_to_obj(bucket->get_key(), raw_head, &head_obj); + + rgw_obj_index_key remove_key; + head_obj.key.get_index_key(&remove_key); + remove_objs.push_back(remove_key); + } + } + } + cleanup_part_history(dpp, y, part, remove_objs, part_prefixes); + } + } while (truncated); + + if (store->getRados()->get_gc() == nullptr) { + //Delete objects inline if gc hasn't been initialised (in case when bypass gc is specified) + store->getRados()->delete_objs_inline(dpp, chain, mp_obj.get_upload_id()); + } else { + /* use upload id as tag and do it synchronously */ + auto [ret, leftover_chain] = store->getRados()->send_chain_to_gc(chain, mp_obj.get_upload_id(), y); + if (ret < 0 && leftover_chain) { + ldpp_dout(dpp, 5) << __func__ << ": gc->send_chain() returned " << ret << dendl; + if (ret == -ENOENT) { + return -ERR_NO_SUCH_UPLOAD; + } + //Delete objects inline if send chain to gc fails + store->getRados()->delete_objs_inline(dpp, *leftover_chain, mp_obj.get_upload_id()); + } + } + return 0; +} + int RadosMultipartUpload::cleanup_part_history(const DoutPrefixProvider* dpp, optional_yield y, RadosMultipartPart *part, - list& remove_objs) + list& remove_objs, + boost::container::flat_set& processed_prefixes) { cls_rgw_obj_chain chain; for (auto& ppfx : part->get_past_prefixes()) { + auto [it, inserted] = processed_prefixes.emplace(ppfx); + if (!inserted) { + continue; // duplicate + } + rgw_obj past_obj; past_obj.init_ns(bucket->get_key(), ppfx + "." + std::to_string(part->info.num), mp_ns); rgw_obj_index_key past_key; @@ -2978,77 +3059,105 @@ int RadosMultipartUpload::abort(const DoutPrefixProvider *dpp, CephContext *cct, int ret; uint64_t parts_accounted_size = 0; - do { - ret = list_parts(dpp, cct, 1000, marker, &marker, &truncated, y); + prefix_map_t processed_prefixes; + + static constexpr auto MAX_DELETE_RETRIES = 15u; + for (auto i = 0u; i < MAX_DELETE_RETRIES; i++) { + ret = meta_obj->get_obj_attrs(y, dpp); if (ret < 0) { - ldpp_dout(dpp, 20) << __func__ << ": RadosMultipartUpload::list_parts returned " << - ret << dendl; + ldpp_dout(dpp, 0) << __func__ << ": ERROR: failed to get obj attrs, obj=" << meta_obj + << " ret=" << ret << dendl; return (ret == -ENOENT) ? -ERR_NO_SUCH_UPLOAD : ret; } - for (auto part_it = parts.begin(); - part_it != parts.end(); - ++part_it) { - RadosMultipartPart* obj_part = dynamic_cast(part_it->second.get()); - if (obj_part->info.manifest.empty()) { - std::unique_ptr obj = bucket->get_object( - rgw_obj_key(obj_part->oid, std::string(), RGW_OBJ_NS_MULTIPART)); - obj->set_hash_source(mp_obj.get_key()); - ret = obj->delete_object(dpp, y, 0); - if (ret < 0 && ret != -ENOENT) - return ret; - } else { - auto target = meta_obj->get_obj(); - store->getRados()->update_gc_chain(dpp, target, obj_part->info.manifest, &chain); - RGWObjManifest::obj_iterator oiter = obj_part->info.manifest.obj_begin(dpp); - if (oiter != obj_part->info.manifest.obj_end(dpp)) { - std::unique_ptr head = bucket->get_object(rgw_obj_key()); - rgw_raw_obj raw_head = oiter.get_location().get_raw_obj(store->getRados()); - dynamic_cast(head.get())->raw_obj_to_obj(raw_head); - - rgw_obj_index_key key; - head->get_key().get_index_key(&key); - remove_objs.push_back(key); - - cleanup_part_history(dpp, null_yield, obj_part, remove_objs); + RGWObjVersionTracker objv_tracker = meta_obj->get_version_tracker(); + + do { + ret = list_parts(dpp, cct, 1000, marker, &marker, &truncated, y); + if (ret < 0) { + ldpp_dout(dpp, 20) << __func__ << ": RadosMultipartUpload::list_parts returned " << ret << dendl; + return (ret == -ENOENT) ? -ERR_NO_SUCH_UPLOAD : ret; + } + + for (auto part_it = parts.begin(); part_it != parts.end(); ++part_it) { + RadosMultipartPart* obj_part = dynamic_cast(part_it->second.get()); + + if (obj_part->info.manifest.empty()) { + std::unique_ptr obj = bucket->get_object( + rgw_obj_key(obj_part->oid, std::string(), RGW_OBJ_NS_MULTIPART)); + obj->set_hash_source(mp_obj.get_key()); + ret = obj->delete_object(dpp, y, 0, nullptr, nullptr); + if (ret < 0 && ret != -ENOENT) + return ret; + } else { + auto manifest_prefix = obj_part->info.manifest.get_prefix(); + auto [it, inserted] = processed_prefixes.emplace(obj_part->info.num, boost::container::flat_set{}); + if (not manifest_prefix.empty()) { + if (it->second.find(manifest_prefix) != it->second.end()) { + continue; + } + it->second.emplace(manifest_prefix); + } + + auto target = meta_obj->get_obj(); + store->getRados()->update_gc_chain(dpp, target, obj_part->info.manifest, &chain); + RGWObjManifest::obj_iterator oiter = obj_part->info.manifest.obj_begin(dpp); + if (oiter != obj_part->info.manifest.obj_end(dpp)) { + std::unique_ptr head = bucket->get_object(rgw_obj_key()); + rgw_raw_obj raw_head = oiter.get_location().get_raw_obj(store->getRados()); + dynamic_cast(head.get())->raw_obj_to_obj(raw_head); + + rgw_obj_index_key key; + head->get_key().get_index_key(&key); + remove_objs.push_back(key); + + cleanup_part_history(dpp, null_yield, obj_part, remove_objs, it->second); + } } + parts_accounted_size += obj_part->info.accounted_size; } - parts_accounted_size += obj_part->info.accounted_size; - } - } while (truncated); + } while (truncated); - if (store->getRados()->get_gc() == nullptr) { - //Delete objects inline if gc hasn't been initialised (in case when bypass gc is specified) - store->getRados()->delete_objs_inline(dpp, chain, mp_obj.get_upload_id()); - } else { - /* use upload id as tag and do it synchronously */ - auto [ret, leftover_chain] = store->getRados()->send_chain_to_gc(chain, mp_obj.get_upload_id(), y); - if (ret < 0 && leftover_chain) { - ldpp_dout(dpp, 5) << __func__ << ": gc->send_chain() returned " << ret << dendl; - if (ret == -ENOENT) { - return -ERR_NO_SUCH_UPLOAD; + if (store->getRados()->get_gc() == nullptr) { + //Delete objects inline if gc hasn't been initialised (in case when bypass gc is specified) + store->getRados()->delete_objs_inline(dpp, chain, mp_obj.get_upload_id()); + } else { + /* use upload id as tag and do it synchronously */ + auto [ret, leftover_chain] = store->getRados()->send_chain_to_gc(chain, mp_obj.get_upload_id(), y); + if (ret < 0 && leftover_chain) { + ldpp_dout(dpp, 5) << __func__ << ": gc->send_chain() returned " << ret << dendl; + if (ret == -ENOENT) { + return -ERR_NO_SUCH_UPLOAD; + } + //Delete objects inline if send chain to gc fails + store->getRados()->delete_objs_inline(dpp, *leftover_chain, mp_obj.get_upload_id()); } - //Delete objects inline if send chain to gc fails - store->getRados()->delete_objs_inline(dpp, *leftover_chain, mp_obj.get_upload_id()); } - } - std::unique_ptr del_op = meta_obj->get_delete_op(); - del_op->params.bucket_owner = bucket->get_info().owner; - del_op->params.versioning_status = 0; - if (!remove_objs.empty()) { - del_op->params.remove_objs = &remove_objs; - } + std::unique_ptr del_op = meta_obj->get_delete_op(); + del_op->params.bucket_owner = bucket->get_info().owner; + del_op->params.versioning_status = 0; + if (!remove_objs.empty()) { + del_op->params.remove_objs = &remove_objs; + } - del_op->params.abortmp = true; - del_op->params.parts_accounted_size = parts_accounted_size; + del_op->params.abortmp = true; + del_op->params.parts_accounted_size = parts_accounted_size; + del_op->params.objv_tracker = &objv_tracker; - // and also remove the metadata obj - ret = del_op->delete_obj(dpp, y, 0); - if (ret < 0) { - ldpp_dout(dpp, 20) << __func__ << ": del_op.delete_obj returned " << - ret << dendl; + // and also remove the metadata obj + ret = del_op->delete_obj(dpp, y, 0); + if (ret != -ECANCELED) { + if (ret < 0) { + ldpp_dout(dpp, 20) << __func__ << ": del_op.delete_obj returned " << ret << dendl; + } + break; + } + ldpp_dout(dpp, 20) << "deleting meta_obj is cancelled due to mismatch cls_version: " << objv_tracker << dendl; + chain.objs.clear(); + marker = 0; } + return (ret == -ENOENT) ? -ERR_NO_SUCH_UPLOAD : ret; } @@ -3228,7 +3337,8 @@ int RadosMultipartUpload::complete(const DoutPrefixProvider *dpp, RGWCompressionInfo& cs_info, off_t& ofs, std::string& tag, ACLOwner& owner, uint64_t olh_epoch, - rgw::sal::Object* target_obj) + rgw::sal::Object* target_obj, + prefix_map_t& processed_prefixes) { char final_etag[CEPH_CRYPTO_MD5_DIGESTSIZE]; char final_etag_str[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 16]; @@ -3300,6 +3410,8 @@ int RadosMultipartUpload::complete(const DoutPrefixProvider *dpp, rgw_obj src_obj; src_obj.init_ns(bucket->get_key(), oid, mp_ns); + auto [it, inserted] = processed_prefixes.emplace(part->info.num, boost::container::flat_set{}); + if (obj_part.manifest.empty()) { ldpp_dout(dpp, 0) << "ERROR: empty manifest for object part: obj=" << src_obj << dendl; @@ -3311,6 +3423,7 @@ int RadosMultipartUpload::complete(const DoutPrefixProvider *dpp, if (not manifest_prefix.empty()) { // It has an explicit prefix. Override the default one. src_obj.init_ns(bucket->get_key(), manifest_prefix + "." + std::to_string(part->info.num), mp_ns); + it->second.emplace(manifest_prefix); } } @@ -3356,7 +3469,7 @@ int RadosMultipartUpload::complete(const DoutPrefixProvider *dpp, remove_objs.push_back(remove_key); - cleanup_part_history(dpp, y, part, remove_objs); + cleanup_part_history(dpp, y, part, remove_objs, it->second); ofs += obj_part.size; accounted_size += obj_part.accounted_size; diff --git a/src/rgw/driver/rados/rgw_sal_rados.h b/src/rgw/driver/rados/rgw_sal_rados.h index 9f5d021987a..0a4e03a102c 100644 --- a/src/rgw/driver/rados/rgw_sal_rados.h +++ b/src/rgw/driver/rados/rgw_sal_rados.h @@ -559,7 +559,9 @@ class RadosObject : public StoreObject { rados_ctx->invalidate(get_obj()); } virtual int delete_object(const DoutPrefixProvider* dpp, - optional_yield y, uint32_t flags) override; + optional_yield y, uint32_t flags, + std::list* remove_objs, + RGWObjVersionTracker* objv) override; virtual int copy_object(const ACLOwner& owner, const rgw_user& remote_user, req_info* info, const rgw_zone_id& source_zone, @@ -816,7 +818,13 @@ public: RGWCompressionInfo& cs_info, off_t& ofs, std::string& tag, ACLOwner& owner, uint64_t olh_epoch, - rgw::sal::Object* target_obj) override; + rgw::sal::Object* target_obj, + prefix_map_t& processed_prefixes) override; + virtual int cleanup_orphaned_parts(const DoutPrefixProvider *dpp, + CephContext *cct, optional_yield y, + const rgw_obj& obj, + std::list& remove_objs, + prefix_map_t& processed_prefixes) override; virtual int get_info(const DoutPrefixProvider *dpp, optional_yield y, rgw_placement_rule** rule, rgw::sal::Attrs* attrs = nullptr) override; virtual std::unique_ptr get_writer(const DoutPrefixProvider *dpp, optional_yield y, @@ -829,7 +837,8 @@ protected: int cleanup_part_history(const DoutPrefixProvider* dpp, optional_yield y, RadosMultipartPart* part, - std::list& remove_objs); + std::list& remove_objs, + boost::container::flat_set& processed_prefixes); }; class MPRadosSerializer : public StoreMPSerializer { diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index 24796ea1aa4..d3c616a78a0 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -6397,12 +6397,61 @@ void RGWCompleteMultipart::execute(optional_yield y) return; } - op_ret = upload->complete(this, y, s->cct, parts->parts, remove_objs, accounted_size, compressed, cs_info, ofs, s->req_id, s->owner, olh_epoch, s->object.get()); + RGWObjVersionTracker& objv_tracker = meta_obj->get_version_tracker(); + + using prefix_map_t = rgw::sal::MultipartUpload::prefix_map_t; + prefix_map_t processed_prefixes; + + op_ret = + upload->complete(this, y, s->cct, parts->parts, remove_objs, accounted_size, + compressed, cs_info, ofs, s->req_id, s->owner, olh_epoch, + s->object.get(), processed_prefixes); if (op_ret < 0) { ldpp_dout(this, 0) << "ERROR: upload complete failed ret=" << op_ret << dendl; return; } + remove_objs.clear(); + + // use cls_version_check() when deleting the meta object to detect part uploads that raced + // with upload->complete(). any parts that finish after that won't be part of the final + // upload, so they need to be gc'd and removed from the bucket index before retrying + // deletion of the multipart meta object + static constexpr auto MAX_DELETE_RETRIES = 15u; + for (auto i = 0u; i < MAX_DELETE_RETRIES; i++) { + // remove the upload meta object ; the meta object is not versioned + // when the bucket is, as that would add an unneeded delete marker + int ret = meta_obj->delete_object(this, y, rgw::sal::FLAG_PREVENT_VERSIONING, &remove_objs, &objv_tracker); + if (ret != -ECANCELED || i == MAX_DELETE_RETRIES - 1) { + if (ret >= 0) { + /* serializer's exclusive lock is released */ + serializer->clear_locked(); + } else { + ldpp_dout(this, 1) << "ERROR: failed to remove object " << meta_obj << ", ret: " << ret << dendl; + } + break; + } + + ldpp_dout(this, 20) << "deleting meta_obj is cancelled due to mismatch cls_version: " << objv_tracker << dendl; + objv_tracker.clear(); + + ret = meta_obj->get_obj_attrs(s->yield, this); + if (ret < 0) { + ldpp_dout(this, 1) << "ERROR: failed to get obj attrs, obj=" << meta_obj + << " ret=" << ret << dendl; + + if (ret != -ENOENT) { + ldpp_dout(this, 0) << "ERROR: failed to remove object " << meta_obj << dendl; + } + break; + } + + ret = upload->cleanup_orphaned_parts(this, s->cct, y, meta_obj->get_obj(), remove_objs, processed_prefixes); + if (ret < 0) { + ldpp_dout(this, 0) << "ERROR: failed to clenup orphaned parts. ret=" << ret << dendl; + } + } + const ceph::real_time upload_time = upload->get_mtime(); etag = s->object->get_attrs()[RGW_ATTR_ETAG].to_str(); @@ -6412,17 +6461,6 @@ void RGWCompleteMultipart::execute(optional_yield y) ldpp_dout(this, 1) << "ERROR: publishing notification failed, with error: " << ret << dendl; // too late to rollback operation, hence op_ret is not set here } - - // remove the upload meta object ; the meta object is not versioned - // when the bucket is, as that would add an unneeded delete marker - ret = meta_obj->delete_object(this, y, rgw::sal::FLAG_PREVENT_VERSIONING); - if (ret >= 0) { - /* serializer's exclusive lock is released */ - serializer->clear_locked(); - } else { - ldpp_dout(this, 4) << "WARNING: failed to remove object " << meta_obj << ", ret: " << ret << dendl; - } - } // RGWCompleteMultipart::execute bool RGWCompleteMultipart::check_previously_completed(const RGWMultiCompleteUpload* parts) diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h index cd29e9c7428..6aa055bcaa4 100644 --- a/src/rgw/rgw_sal.h +++ b/src/rgw/rgw_sal.h @@ -1136,6 +1136,7 @@ class Object { rgw_zone_set* zones_trace{nullptr}; bool abortmp{false}; uint64_t parts_accounted_size{0}; + RGWObjVersionTracker* objv_tracker = nullptr; } params; struct Result { @@ -1155,7 +1156,9 @@ class Object { /** Shortcut synchronous delete call for common deletes */ virtual int delete_object(const DoutPrefixProvider* dpp, optional_yield y, - uint32_t flags) = 0; + uint32_t flags, + std::list* remove_objs, + RGWObjVersionTracker* objv) = 0; /** Copy an this object to another object. */ virtual int copy_object(const ACLOwner& owner, const rgw_user& remote_user, req_info* info, const rgw_zone_id& source_zone, @@ -1299,6 +1302,9 @@ class Object { virtual int get_torrent_info(const DoutPrefixProvider* dpp, optional_yield y, bufferlist& bl) = 0; + /** Get the version tracker for this object */ + virtual RGWObjVersionTracker& get_version_tracker() = 0; + /** Get the OMAP values matching the given set of keys */ virtual int omap_get_vals_by_keys(const DoutPrefixProvider *dpp, const std::string& oid, const std::set& keys, @@ -1380,6 +1386,8 @@ public: */ class MultipartUpload { public: + using prefix_map_t = boost::container::flat_map>; + //object lock std::optional obj_retention = std::nullopt; std::optional obj_legal_hold = std::nullopt; @@ -1425,7 +1433,14 @@ public: RGWCompressionInfo& cs_info, off_t& ofs, std::string& tag, ACLOwner& owner, uint64_t olh_epoch, - rgw::sal::Object* target_obj) = 0; + rgw::sal::Object* target_obj, + prefix_map_t& processed_prefixes) = 0; + /** Cleanup orphaned parts caused by racing condition involving part upload retry */ + virtual int cleanup_orphaned_parts(const DoutPrefixProvider *dpp, + CephContext *cct, optional_yield y, + const rgw_obj& obj, + std::list& remove_objs, + prefix_map_t& processed_prefixes) = 0; /** Get placement and/or attribute info for this upload */ virtual int get_info(const DoutPrefixProvider *dpp, optional_yield y, rgw_placement_rule** rule, rgw::sal::Attrs* attrs = nullptr) = 0; diff --git a/src/rgw/rgw_sal_dbstore.cc b/src/rgw/rgw_sal_dbstore.cc index 874935f1619..d6ee772bc2e 100644 --- a/src/rgw/rgw_sal_dbstore.cc +++ b/src/rgw/rgw_sal_dbstore.cc @@ -716,7 +716,11 @@ namespace rgw::sal { return ret; } - int DBObject::delete_object(const DoutPrefixProvider* dpp, optional_yield y, uint32_t flags) + int DBObject::delete_object(const DoutPrefixProvider* dpp, + optional_yield y, + uint32_t flags, + std::list* remove_objs, + RGWObjVersionTracker* objv) { DB::Object del_target(store->getDB(), bucket->get_info(), get_obj()); DB::Object::Delete del_op(&del_target); @@ -908,7 +912,8 @@ namespace rgw::sal { RGWCompressionInfo& cs_info, off_t& ofs, std::string& tag, ACLOwner& owner, uint64_t olh_epoch, - rgw::sal::Object* target_obj) + rgw::sal::Object* target_obj, + prefix_map_t& processed_prefixes) { char final_etag[CEPH_CRYPTO_MD5_DIGESTSIZE]; char final_etag_str[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 16]; @@ -1018,6 +1023,15 @@ namespace rgw::sal { return ret; } + int DBMultipartUpload::cleanup_orphaned_parts(const DoutPrefixProvider *dpp, + CephContext *cct, optional_yield y, + const rgw_obj& obj, + std::list& remove_objs, + prefix_map_t& processed_prefixes) + { + return -ENOTSUP; + } + int DBMultipartUpload::get_info(const DoutPrefixProvider *dpp, optional_yield y, rgw_placement_rule** rule, rgw::sal::Attrs* attrs) { if (!rule && !attrs) { diff --git a/src/rgw/rgw_sal_dbstore.h b/src/rgw/rgw_sal_dbstore.h index eaa5fda5060..417cc7111c6 100644 --- a/src/rgw/rgw_sal_dbstore.h +++ b/src/rgw/rgw_sal_dbstore.h @@ -449,7 +449,13 @@ protected: RGWCompressionInfo& cs_info, off_t& ofs, std::string& tag, ACLOwner& owner, uint64_t olh_epoch, - rgw::sal::Object* target_obj) override; + rgw::sal::Object* target_obj, + prefix_map_t& processed_prefixes) override; + virtual int cleanup_orphaned_parts(const DoutPrefixProvider *dpp, + CephContext *cct, optional_yield y, + const rgw_obj& obj, + std::list& remove_objs, + prefix_map_t& processed_prefixes) override; virtual int get_info(const DoutPrefixProvider *dpp, optional_yield y, rgw_placement_rule** rule, rgw::sal::Attrs* attrs = nullptr) override; virtual std::unique_ptr get_writer(const DoutPrefixProvider *dpp, optional_yield y, @@ -521,7 +527,9 @@ protected: virtual int delete_object(const DoutPrefixProvider* dpp, optional_yield y, - uint32_t flags) override; + uint32_t flags, + std::list* remove_objs, + RGWObjVersionTracker* objv) override; virtual int copy_object(const ACLOwner& owner, const rgw_user& remote_user, req_info* info, const rgw_zone_id& source_zone, diff --git a/src/rgw/rgw_sal_filter.cc b/src/rgw/rgw_sal_filter.cc index d6d51b20461..a04eb733af2 100644 --- a/src/rgw/rgw_sal_filter.cc +++ b/src/rgw/rgw_sal_filter.cc @@ -994,9 +994,11 @@ int FilterBucket::abort_multiparts(const DoutPrefixProvider* dpp, CephContext* c int FilterObject::delete_object(const DoutPrefixProvider* dpp, optional_yield y, - uint32_t flags) + uint32_t flags, + std::list* remove_objs, + RGWObjVersionTracker* objv) { - return next->delete_object(dpp, y, flags); + return next->delete_object(dpp, y, flags, remove_objs, objv); } int FilterObject::copy_object(const ACLOwner& owner, @@ -1283,11 +1285,21 @@ int FilterMultipartUpload::complete(const DoutPrefixProvider *dpp, RGWCompressionInfo& cs_info, off_t& ofs, std::string& tag, ACLOwner& owner, uint64_t olh_epoch, - rgw::sal::Object* target_obj) + rgw::sal::Object* target_obj, + prefix_map_t& processed_prefixes) { return next->complete(dpp, y, cct, part_etags, remove_objs, accounted_size, compressed, cs_info, ofs, tag, owner, olh_epoch, - nextObject(target_obj)); + nextObject(target_obj), processed_prefixes); +} + +int FilterMultipartUpload::cleanup_orphaned_parts(const DoutPrefixProvider *dpp, + CephContext *cct, optional_yield y, + const rgw_obj& obj, + std::list& remove_objs, + prefix_map_t& processed_prefixes) +{ + return next->cleanup_orphaned_parts(dpp, cct, y, obj, remove_objs, processed_prefixes); } int FilterMultipartUpload::get_info(const DoutPrefixProvider *dpp, diff --git a/src/rgw/rgw_sal_filter.h b/src/rgw/rgw_sal_filter.h index eaa8af2cb43..344aaf9d5b5 100644 --- a/src/rgw/rgw_sal_filter.h +++ b/src/rgw/rgw_sal_filter.h @@ -727,7 +727,9 @@ public: virtual int delete_object(const DoutPrefixProvider* dpp, optional_yield y, - uint32_t flags) override; + uint32_t flags, + std::list* remove_objs, + RGWObjVersionTracker* objv) override; virtual int copy_object(const ACLOwner& owner, const rgw_user& remote_user, req_info* info, const rgw_zone_id& source_zone, @@ -833,6 +835,8 @@ public: virtual int get_torrent_info(const DoutPrefixProvider* dpp, optional_yield y, bufferlist& bl) override; + virtual RGWObjVersionTracker& get_version_tracker() override { return next->get_version_tracker(); } + virtual int omap_get_vals_by_keys(const DoutPrefixProvider *dpp, const std::string& oid, const std::set& keys, @@ -907,7 +911,13 @@ public: RGWCompressionInfo& cs_info, off_t& ofs, std::string& tag, ACLOwner& owner, uint64_t olh_epoch, - rgw::sal::Object* target_obj) override; + rgw::sal::Object* target_obj, + prefix_map_t& processed_prefixes) override; + virtual int cleanup_orphaned_parts(const DoutPrefixProvider *dpp, + CephContext *cct, optional_yield y, + const rgw_obj& obj, + std::list& remove_objs, + prefix_map_t& processed_prefixes) override; virtual int get_info(const DoutPrefixProvider *dpp, optional_yield y, rgw_placement_rule** rule, diff --git a/src/rgw/rgw_sal_store.h b/src/rgw/rgw_sal_store.h index 8bfff802526..1e3d7630984 100644 --- a/src/rgw/rgw_sal_store.h +++ b/src/rgw/rgw_sal_store.h @@ -273,6 +273,8 @@ class StoreObject : public Object { return -ENOENT; } + virtual RGWObjVersionTracker& get_version_tracker() override { return state.objv_tracker; } + virtual void print(std::ostream& out) const override { if (bucket) out << bucket << ":";