From c83fd2e1b1b77edbbeb248c45f05b5bebe54f287 Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Sun, 3 Dec 2023 17:40:23 -0500 Subject: [PATCH] rgw/metadata: use raw RGWMetadataHandler for bucket instances Signed-off-by: Casey Bodley --- src/rgw/driver/rados/rgw_bucket.cc | 381 +++++++++++++----------- src/rgw/driver/rados/rgw_bucket.h | 35 +-- src/rgw/driver/rados/rgw_data_sync.cc | 9 +- src/rgw/driver/rados/rgw_service.cc | 17 +- src/rgw/driver/rados/rgw_sync_module.cc | 9 +- src/rgw/driver/rados/rgw_sync_module.h | 8 +- src/rgw/services/svc_bi.h | 4 +- src/rgw/services/svc_bi_rados.cc | 5 +- src/rgw/services/svc_bi_rados.h | 4 +- src/rgw/services/svc_bucket.h | 7 +- src/rgw/services/svc_bucket_sobj.cc | 115 +++---- src/rgw/services/svc_bucket_sobj.h | 15 +- src/rgw/services/svc_bucket_sync.h | 2 - src/rgw/services/svc_bucket_types.h | 38 --- 14 files changed, 305 insertions(+), 344 deletions(-) delete mode 100644 src/rgw/services/svc_bucket_types.h diff --git a/src/rgw/driver/rados/rgw_bucket.cc b/src/rgw/driver/rados/rgw_bucket.cc index 02c401ef9a5..3152aedd85f 100644 --- a/src/rgw/driver/rados/rgw_bucket.cc +++ b/src/rgw/driver/rados/rgw_bucket.cc @@ -2280,8 +2280,8 @@ int RGWBucketMetadataHandler::put(std::string& entry, RGWMetadataObject* obj, } int update_bucket_topic_mappings(const DoutPrefixProvider* dpp, - RGWBucketCompleteInfo* orig_bci, - RGWBucketCompleteInfo* current_bci, + const RGWBucketCompleteInfo* orig_bci, + const RGWBucketCompleteInfo* current_bci, rgw::sal::Driver* driver) { const auto decode_attrs = [](const rgw::sal::Attrs& attrs, rgw_pubsub_bucket_topics& bucket_topics) -> int { @@ -2658,28 +2658,28 @@ class RGWArchiveBucketMetadataHandler : public RGWBucketMetadataHandler { } }; -class RGWBucketInstanceMetadataHandler : public RGWBucketInstanceMetadataHandlerBase { -public: - struct Svc { - RGWSI_Zone *zone{nullptr}; - RGWSI_Bucket *bucket{nullptr}; - RGWSI_BucketIndex *bi{nullptr}; - } svc; - +class RGWBucketInstanceMetadataHandler : public RGWMetadataHandler { rgw::sal::Driver* driver; - - RGWBucketInstanceMetadataHandler(rgw::sal::Driver* driver) - : driver(driver) {} - - void init(RGWSI_Zone *zone_svc, - RGWSI_Bucket *bucket_svc, - RGWSI_BucketIndex *bi_svc) override { - base_init(bucket_svc->ctx(), - bucket_svc->get_bi_be_handler().get()); - svc.zone = zone_svc; - svc.bucket = bucket_svc; - svc.bi = bi_svc; - } + RGWSI_Zone* svc_zone{nullptr}; + RGWSI_Bucket* svc_bucket{nullptr}; + RGWSI_BucketIndex* svc_bi{nullptr}; + + int put_prepare(const DoutPrefixProvider* dpp, optional_yield y, + const std::string& entry, RGWBucketCompleteInfo& bci, + const std::optional& old_bci, + const RGWObjVersionTracker& objv_tracker, + bool from_remote_zone); + int put_post(const DoutPrefixProvider* dpp, optional_yield y, + const RGWBucketCompleteInfo& bci, + const std::optional& old_bci, + RGWObjVersionTracker& objv_tracker); + public: + RGWBucketInstanceMetadataHandler(rgw::sal::Driver* driver, + RGWSI_Zone* svc_zone, + RGWSI_Bucket* svc_bucket, + RGWSI_BucketIndex* svc_bi) + : driver(driver), svc_zone(svc_zone), + svc_bucket(svc_bucket), svc_bi(svc_bi) {} string get_type() override { return "bucket.instance"; } @@ -2695,86 +2695,78 @@ public: return new RGWBucketInstanceMetadataObject(bci, objv, mtime); } - int do_get(RGWSI_MetaBackend_Handler::Op *op, string& entry, RGWMetadataObject **obj, optional_yield y, const DoutPrefixProvider *dpp) override { - RGWBucketCompleteInfo bci; - real_time mtime; + int get(std::string& entry, RGWMetadataObject** obj, optional_yield y, + const DoutPrefixProvider *dpp) override; + int put(std::string& entry, RGWMetadataObject* obj, + RGWObjVersionTracker& objv_tracker, + optional_yield y, const DoutPrefixProvider* dpp, + RGWMDLogSyncType type, bool from_remote_zone) override; + int remove(std::string& entry, RGWObjVersionTracker& objv_tracker, + optional_yield y, const DoutPrefixProvider *dpp) override; - int ret = svc.bucket->read_bucket_instance_info(entry, &bci.info, &mtime, &bci.attrs, y, dpp); - if (ret < 0) - return ret; + int mutate(const std::string& entry, const ceph::real_time& mtime, + RGWObjVersionTracker* objv_tracker, optional_yield y, + const DoutPrefixProvider* dpp, RGWMDLogStatus op_type, + std::function f) override; - RGWBucketInstanceMetadataObject *mdo = new RGWBucketInstanceMetadataObject(bci, bci.info.objv_tracker.read_version, mtime); + int list_keys_init(const DoutPrefixProvider* dpp, const std::string& marker, + void** phandle) override; + int list_keys_next(const DoutPrefixProvider* dpp, void* handle, int max, + std::list& keys, bool* truncated) override; + void list_keys_complete(void *handle) override; + std::string get_marker(void *handle) override; +}; - *obj = mdo; +int RGWBucketInstanceMetadataHandler::get(std::string& entry, RGWMetadataObject **obj, + optional_yield y, const DoutPrefixProvider *dpp) +{ + RGWBucketCompleteInfo bci; + real_time mtime; - return 0; + int ret = svc_bucket->read_bucket_instance_info(entry, &bci.info, &mtime, &bci.attrs, y, dpp); + if (ret < 0) { + return ret; } - int do_put(RGWSI_MetaBackend_Handler::Op *op, string& entry, - RGWMetadataObject *_obj, RGWObjVersionTracker& objv_tracker, - optional_yield y, const DoutPrefixProvider *dpp, - RGWMDLogSyncType sync_type, bool from_remote_zone) override; - - int do_remove(RGWSI_MetaBackend_Handler::Op *op, string& entry, RGWObjVersionTracker& objv_tracker, - optional_yield y, const DoutPrefixProvider *dpp) override { - RGWBucketCompleteInfo bci; - - int ret = svc.bucket->read_bucket_instance_info(entry, &bci.info, nullptr, - &bci.attrs, y, dpp); - if (ret < 0 && ret != -ENOENT) - return ret; + *obj = new RGWBucketInstanceMetadataObject(bci, bci.info.objv_tracker.read_version, mtime); + return 0; +} - ret = svc.bucket->remove_bucket_instance_info( - entry, bci.info, &bci.info.objv_tracker, y, dpp); - if (ret < 0) - return ret; - ret = update_bucket_topic_mappings(dpp, &bci, /*current_bci=*/nullptr, - driver); - // update_bucket_topic_mapping error is swallowed. - return 0; +int RGWBucketInstanceMetadataHandler::put(std::string& entry, RGWMetadataObject* obj, + RGWObjVersionTracker& objv_tracker, + optional_yield y, const DoutPrefixProvider *dpp, + RGWMDLogSyncType sync_type, bool from_remote_zone) +{ + // read existing bucket instance + std::optional old = RGWBucketCompleteInfo{}; + int ret = svc_bucket->read_bucket_instance_info(entry, &old->info, nullptr, + &old->attrs, y, dpp); + if (ret == -ENOENT) { + old = std::nullopt; + } else if (ret < 0) { + return ret; } -}; -class RGWMetadataHandlerPut_BucketInstance : public RGWMetadataHandlerPut_SObj -{ - CephContext *cct; - optional_yield y; - RGWBucketInstanceMetadataHandler *bihandler; - RGWBucketInstanceMetadataObject *obj; -public: - RGWMetadataHandlerPut_BucketInstance(CephContext *_cct, - RGWBucketInstanceMetadataHandler *_handler, - RGWSI_MetaBackend_Handler::Op *_op, string& entry, - RGWMetadataObject *_obj, RGWObjVersionTracker& objv_tracker, - optional_yield y, - RGWMDLogSyncType type, bool from_remote_zone) : RGWMetadataHandlerPut_SObj(_handler, _op, entry, _obj, objv_tracker, y, type, from_remote_zone), - cct(_cct), y(y), bihandler(_handler) { - obj = static_cast(_obj); + auto newobj = static_cast(obj); + RGWBucketCompleteInfo& bci = newobj->get_bci(); - auto& bci = obj->get_bci(); - obj->set_pattrs(&bci.attrs); + // initializate/update the bucket instance info + ret = put_prepare(dpp, y, entry, bci, old, objv_tracker, from_remote_zone); + if (ret < 0) { + return ret; } - void encode_obj(bufferlist *bl) override { - obj->get_bucket_info().encode(*bl); + // write updated instance + RGWBucketInfo* old_info = (old ? &old->info : nullptr); + auto mtime = obj->get_mtime(); + ret = svc_bucket->store_bucket_instance_info(entry, bci.info, old_info, false, + mtime, &bci.attrs, y, dpp); + if (ret < 0) { + return ret; } - int put_check(const DoutPrefixProvider *dpp) override; - int put_checked(const DoutPrefixProvider *dpp) override; - int put_post(const DoutPrefixProvider *dpp) override; -}; - -int RGWBucketInstanceMetadataHandler::do_put(RGWSI_MetaBackend_Handler::Op *op, - string& entry, - RGWMetadataObject *obj, - RGWObjVersionTracker& objv_tracker, - optional_yield y, - const DoutPrefixProvider *dpp, - RGWMDLogSyncType type, bool from_remote_zone) -{ - RGWMetadataHandlerPut_BucketInstance put_op(svc.bucket->ctx(), this, op, entry, obj, - objv_tracker, y, type, from_remote_zone); - return do_put_operate(&put_op, dpp); + // update related state on success + return put_post(dpp, y, bci, old, objv_tracker); } void init_default_bucket_layout(CephContext *cct, rgw::BucketLayout& layout, @@ -2799,32 +2791,28 @@ void init_default_bucket_layout(CephContext *cct, rgw::BucketLayout& layout, } } -int RGWMetadataHandlerPut_BucketInstance::put_check(const DoutPrefixProvider *dpp) +int RGWBucketInstanceMetadataHandler::put_prepare( + const DoutPrefixProvider* dpp, optional_yield y, + const std::string& entry, RGWBucketCompleteInfo& bci, + const std::optional& old_bci, + const RGWObjVersionTracker& objv_tracker, + bool from_remote_zone) { - int ret; - - RGWBucketCompleteInfo& bci = obj->get_bci(); - - RGWBucketInstanceMetadataObject *orig_obj = static_cast(old_obj); - - RGWBucketCompleteInfo *old_bci = (orig_obj ? &orig_obj->get_bci() : nullptr); - - const bool exists = (!!orig_obj); - if (from_remote_zone) { - // don't sync bucket layout changes - if (!exists) { + // bucket layout information is local. don't overwrite existing layout with + // information from a remote zone + if (old_bci) { + bci.info.layout = old_bci->info.layout; + } else { // replace peer's layout with default-constructed, then apply our defaults bci.info.layout = rgw::BucketLayout{}; - init_default_bucket_layout(cct, bci.info.layout, - bihandler->svc.zone->get_zone(), + init_default_bucket_layout(dpp->get_cct(), bci.info.layout, + svc_zone->get_zone(), std::nullopt); - } else { - bci.info.layout = old_bci->info.layout; } } - if (!exists || old_bci->info.bucket.bucket_id != bci.info.bucket.bucket_id) { + if (!old_bci || old_bci->info.bucket.bucket_id != bci.info.bucket.bucket_id) { /* a new bucket, we need to select a new bucket placement for it */ string tenant_name; string bucket_name; @@ -2836,8 +2824,8 @@ int RGWMetadataHandlerPut_BucketInstance::put_check(const DoutPrefixProvider *dp bci.info.bucket.bucket_id = bucket_instance; bci.info.bucket.tenant = tenant_name; // if the sync module never writes data, don't require the zone to specify all placement targets - if (bihandler->svc.zone->sync_module_supports_writes()) { - ret = bihandler->svc.zone->select_bucket_location_by_rule(dpp, bci.info.placement_rule, &rule_info, y); + if (svc_zone->sync_module_supports_writes()) { + int ret = svc_zone->select_bucket_location_by_rule(dpp, bci.info.placement_rule, &rule_info, y); if (ret < 0) { ldpp_dout(dpp, 0) << "ERROR: select_bucket_placement() returned " << ret << dendl; return ret; @@ -2851,7 +2839,7 @@ int RGWMetadataHandlerPut_BucketInstance::put_check(const DoutPrefixProvider *dp } //always keep bucket versioning enabled on archive zone - if (bihandler->driver->get_zone()->get_tier_type() == "archive") { + if (driver->get_zone()->get_tier_type() == "archive") { bci.info.flags = (bci.info.flags & ~BUCKET_VERSIONS_SUSPENDED) | BUCKET_VERSIONED; } @@ -2862,71 +2850,52 @@ int RGWMetadataHandlerPut_BucketInstance::put_check(const DoutPrefixProvider *dp return 0; } -int RGWMetadataHandlerPut_BucketInstance::put_checked(const DoutPrefixProvider *dpp) +int RGWBucketInstanceMetadataHandler::put_post( + const DoutPrefixProvider* dpp, optional_yield y, + const RGWBucketCompleteInfo& bci, + const std::optional& old_bci, + RGWObjVersionTracker& objv_tracker) { - RGWBucketInstanceMetadataObject *orig_obj = static_cast(old_obj); - - RGWBucketInfo *orig_info = (orig_obj ? &orig_obj->get_bucket_info() : nullptr); - - auto& info = obj->get_bucket_info(); - auto mtime = obj->get_mtime(); - auto pattrs = obj->get_pattrs(); - - return bihandler->svc.bucket->store_bucket_instance_info(entry, - info, - orig_info, - false, - mtime, - pattrs, - y, - dpp); -} - -int RGWMetadataHandlerPut_BucketInstance::put_post(const DoutPrefixProvider *dpp) -{ - RGWBucketCompleteInfo& bci = obj->get_bci(); - - objv_tracker = bci.info.objv_tracker; - - int ret = bihandler->svc.bi->init_index(dpp, bci.info, bci.info.layout.current_index); + int ret = svc_bi->init_index(dpp, bci.info, bci.info.layout.current_index); if (ret < 0) { return ret; } - /* update lifecyle policy */ + /* update lc list on changes to lifecyle policy */ { - auto bucket = bihandler->driver->get_bucket(bci.info); - - auto lc = bihandler->driver->get_rgwlc(); + auto bucket = driver->get_bucket(bci.info); + auto lc = driver->get_rgwlc(); auto lc_it = bci.attrs.find(RGW_ATTR_LC); if (lc_it != bci.attrs.end()) { ldpp_dout(dpp, 20) << "set lc config for " << bci.info.bucket.name << dendl; ret = lc->set_bucket_config(dpp, y, bucket.get(), bci.attrs, nullptr); if (ret < 0) { - ldpp_dout(dpp, 0) << __func__ << " failed to set lc config for " - << bci.info.bucket.name - << dendl; - return ret; + ldpp_dout(dpp, 0) << __func__ << " failed to set lc config for " + << bci.info.bucket.name + << dendl; + return ret; } - } else { - ldpp_dout(dpp, 20) << "remove lc config for " << bci.info.bucket.name << dendl; - ret = lc->remove_bucket_config(dpp, y, bucket.get(), bci.attrs, false /* cannot merge attrs */); - if (ret < 0) { - ldpp_dout(dpp, 0) << __func__ << " failed to remove lc config for " - << bci.info.bucket.name - << dendl; - return ret; + } else if (old_bci) { + lc_it = old_bci->attrs.find(RGW_ATTR_LC); + if (lc_it != old_bci->attrs.end()) { + ldpp_dout(dpp, 20) << "remove lc config for " << old_bci->info.bucket.name << dendl; + ret = lc->remove_bucket_config(dpp, y, bucket.get(), old_bci->attrs, false /* cannot merge attrs */); + if (ret < 0) { + ldpp_dout(dpp, 0) << __func__ << " failed to remove lc config for " + << old_bci->info.bucket.name + << dendl; + return ret; + } } } } /* update lc */ /* update bucket topic mapping */ { - auto* orig_obj = static_cast(old_obj); - auto* orig_bci = (orig_obj ? &orig_obj->get_bci() : nullptr); - ret = update_bucket_topic_mappings(dpp, orig_bci, &bci, bihandler->driver); + auto* orig_bci = (old_bci ? &*old_bci : nullptr); + ret = update_bucket_topic_mappings(dpp, orig_bci, &bci, driver); if (ret < 0) { ldpp_dout(dpp, 0) << __func__ << " failed to apply bucket topic mapping for " @@ -2937,17 +2906,85 @@ int RGWMetadataHandlerPut_BucketInstance::put_post(const DoutPrefixProvider *dpp << " successfully applied bucket topic mapping for " << bci.info.bucket.name << dendl; } + + objv_tracker = bci.info.objv_tracker; + return STATUS_APPLIED; } +int RGWBucketInstanceMetadataHandler::remove(std::string& entry, RGWObjVersionTracker& objv_tracker, + optional_yield y, const DoutPrefixProvider *dpp) +{ + RGWBucketCompleteInfo bci; + int ret = svc_bucket->read_bucket_instance_info(entry, &bci.info, nullptr, + &bci.attrs, y, dpp); + if (ret == -ENOENT) { + return 0; + } + if (ret < 0) { + return ret; + } + + ret = svc_bucket->remove_bucket_instance_info( + entry, bci.info, &bci.info.objv_tracker, y, dpp); + if (ret < 0) + return ret; + std::ignore = update_bucket_topic_mappings(dpp, &bci, /*current_bci=*/nullptr, + driver); + return 0; +} + +int RGWBucketInstanceMetadataHandler::mutate(const std::string& entry, const ceph::real_time& mtime, + RGWObjVersionTracker* objv_tracker, optional_yield y, + const DoutPrefixProvider* dpp, RGWMDLogStatus op_type, + std::function f) +{ + return -ENOTSUP; // unused +} + +int RGWBucketInstanceMetadataHandler::list_keys_init(const DoutPrefixProvider* dpp, + const std::string& marker, + void** phandle) +{ + std::unique_ptr lister; + int ret = svc_bucket->create_instance_lister(dpp, marker, lister); + if (ret < 0) { + return ret; + } + *phandle = lister.release(); // release ownership + return 0; +} + +int RGWBucketInstanceMetadataHandler::list_keys_next(const DoutPrefixProvider* dpp, + void* handle, int max, + std::list& keys, + bool* truncated) +{ + auto lister = static_cast(handle); + return lister->get_next(dpp, max, keys, truncated); +} + +void RGWBucketInstanceMetadataHandler::list_keys_complete(void *handle) +{ + delete static_cast(handle); +} + +std::string RGWBucketInstanceMetadataHandler::get_marker(void *handle) +{ + auto lister = static_cast(handle); + return lister->get_marker(); +} + + class RGWArchiveBucketInstanceMetadataHandler : public RGWBucketInstanceMetadataHandler { -public: - RGWArchiveBucketInstanceMetadataHandler(rgw::sal::Driver* driver) - : RGWBucketInstanceMetadataHandler(driver) {} + public: + using RGWBucketInstanceMetadataHandler::RGWBucketInstanceMetadataHandler; - // N.B. replication of lifecycle policy relies on logic in RGWBucketInstanceMetadataHandler::do_put(...), override with caution + // N.B. replication of lifecycle policy relies on logic in RGWBucketInstanceMetadataHandler::put(...), override with caution - int do_remove(RGWSI_MetaBackend_Handler::Op *op, string& entry, RGWObjVersionTracker& objv_tracker, optional_yield y, const DoutPrefixProvider *dpp) override { + int remove(std::string& entry, RGWObjVersionTracker& objv_tracker, + optional_yield y, const DoutPrefixProvider *dpp) override + { ldpp_dout(dpp, 0) << "SKIP: bucket instance removal is not allowed on archive zone: bucket.instance:" << entry << dendl; return 0; } @@ -2968,16 +3005,11 @@ RGWBucketCtl::RGWBucketCtl(RGWSI_Zone *zone_svc, } void RGWBucketCtl::init(RGWUserCtl *user_ctl, - RGWBucketInstanceMetadataHandler *_bmi_handler, RGWDataChangesLog *datalog, const DoutPrefixProvider *dpp) { ctl.user = user_ctl; - bmi_handler = _bmi_handler; - - bi_be_handler = bmi_handler->get_be_handler(); - datalog->set_bucket_filter( [this](const rgw_bucket& bucket, optional_yield y, const DoutPrefixProvider *dpp) { return bucket_exports_data(bucket, y, dpp); @@ -3456,9 +3488,14 @@ auto create_bucket_metadata_handler(librados::Rados& rados, rados, svc_bucket, ctl_bucket); } -RGWBucketInstanceMetadataHandlerBase* RGWBucketInstanceMetaHandlerAllocator::alloc(rgw::sal::Driver* driver) +auto create_bucket_instance_metadata_handler(rgw::sal::Driver* driver, + RGWSI_Zone* svc_zone, + RGWSI_Bucket* svc_bucket, + RGWSI_BucketIndex* svc_bi) + -> std::unique_ptr { - return new RGWBucketInstanceMetadataHandler(driver); + return std::make_unique(driver, svc_zone, + svc_bucket, svc_bi); } auto create_archive_bucket_metadata_handler(librados::Rados& rados, @@ -3470,12 +3507,16 @@ auto create_archive_bucket_metadata_handler(librados::Rados& rados, rados, svc_bucket, ctl_bucket); } -RGWBucketInstanceMetadataHandlerBase* RGWArchiveBucketInstanceMetaHandlerAllocator::alloc(rgw::sal::Driver* driver) +auto create_archive_bucket_instance_metadata_handler(rgw::sal::Driver* driver, + RGWSI_Zone* svc_zone, + RGWSI_Bucket* svc_bucket, + RGWSI_BucketIndex* svc_bi) + -> std::unique_ptr { - return new RGWArchiveBucketInstanceMetadataHandler(driver); + return std::make_unique(driver, svc_zone, + svc_bucket, svc_bi); } - void RGWBucketEntryPoint::generate_test_instances(list& o) { RGWBucketEntryPoint *bp = new RGWBucketEntryPoint(); diff --git a/src/rgw/driver/rados/rgw_bucket.h b/src/rgw/driver/rados/rgw_bucket.h index ffc46dfc4b2..a93a02a274b 100644 --- a/src/rgw/driver/rados/rgw_bucket.h +++ b/src/rgw/driver/rados/rgw_bucket.h @@ -26,7 +26,6 @@ #include "rgw_formats.h" -#include "services/svc_bucket_types.h" #include "services/svc_bucket_sync.h" // define as static when RGWBucket implementation completes @@ -171,24 +170,18 @@ public: }; WRITE_CLASS_ENCODER(RGWUserBuckets) -class RGWBucketInstanceMetadataHandlerBase : public RGWMetadataHandler_GenericMetaBE { -public: - virtual ~RGWBucketInstanceMetadataHandlerBase() {} - virtual void init(RGWSI_Zone *zone_svc, - RGWSI_Bucket *bucket_svc, - RGWSI_BucketIndex *bi_svc) = 0; -}; - // bucket entrypoint metadata handler factory auto create_bucket_metadata_handler(librados::Rados& rados, RGWSI_Bucket* svc_bucket, RGWBucketCtl* ctl_bucket) -> std::unique_ptr; -class RGWBucketInstanceMetaHandlerAllocator { -public: - static RGWBucketInstanceMetadataHandlerBase *alloc(rgw::sal::Driver* driver); -}; +// bucket instance metadata handler factory +auto create_bucket_instance_metadata_handler(rgw::sal::Driver* driver, + RGWSI_Zone* svc_zone, + RGWSI_Bucket* svc_bucket, + RGWSI_BucketIndex* svc_bi) + -> std::unique_ptr; // archive bucket entrypoint metadata handler factory auto create_archive_bucket_metadata_handler(librados::Rados& rados, @@ -196,10 +189,13 @@ auto create_archive_bucket_metadata_handler(librados::Rados& rados, RGWBucketCtl* ctl_bucket) -> std::unique_ptr; -class RGWArchiveBucketInstanceMetaHandlerAllocator { -public: - static RGWBucketInstanceMetadataHandlerBase *alloc(rgw::sal::Driver* driver); -}; +// archive bucket instance metadata handler factory +auto create_archive_bucket_instance_metadata_handler(rgw::sal::Driver* driver, + RGWSI_Zone* svc_zone, + RGWSI_Bucket* svc_bucket, + RGWSI_BucketIndex* svc_bi) + -> std::unique_ptr; + extern int rgw_remove_object(const DoutPrefixProvider *dpp, rgw::sal::Driver* driver, rgw::sal::Bucket* bucket, rgw_obj_key& key, optional_yield y); @@ -435,10 +431,6 @@ class RGWBucketCtl { RGWUserCtl *user{nullptr}; } ctl; - RGWBucketInstanceMetadataHandler *bmi_handler; - - RGWSI_BucketInstance_BE_Handler bi_be_handler; /* bucket instance backend handler */ - public: RGWBucketCtl(RGWSI_Zone *zone_svc, RGWSI_Bucket *bucket_svc, @@ -447,7 +439,6 @@ public: RGWSI_User* user_svc); void init(RGWUserCtl *user_ctl, - RGWBucketInstanceMetadataHandler *_bmi_handler, RGWDataChangesLog *datalog, const DoutPrefixProvider *dpp); diff --git a/src/rgw/driver/rados/rgw_data_sync.cc b/src/rgw/driver/rados/rgw_data_sync.cc index 248b45845d4..b6cc23bd74a 100644 --- a/src/rgw/driver/rados/rgw_data_sync.cc +++ b/src/rgw/driver/rados/rgw_data_sync.cc @@ -3126,8 +3126,13 @@ public: -> std::unique_ptr override { return create_archive_bucket_metadata_handler(rados, svc_bucket, ctl_bucket); } - RGWBucketInstanceMetadataHandlerBase *alloc_bucket_instance_meta_handler(rgw::sal::Driver* driver) override { - return RGWArchiveBucketInstanceMetaHandlerAllocator::alloc(driver); + auto alloc_bucket_instance_meta_handler(rgw::sal::Driver* driver, + RGWSI_Zone* svc_zone, + RGWSI_Bucket* svc_bucket, + RGWSI_BucketIndex* svc_bi) + -> std::unique_ptr override { + return create_archive_bucket_instance_metadata_handler( + driver, svc_zone, svc_bucket, svc_bi); } }; diff --git a/src/rgw/driver/rados/rgw_service.cc b/src/rgw/driver/rados/rgw_service.cc index ba1ebc0626f..b6239ac3ea4 100644 --- a/src/rgw/driver/rados/rgw_service.cc +++ b/src/rgw/driver/rados/rgw_service.cc @@ -93,7 +93,7 @@ int RGWServices_Def::init(CephContext *cct, bilog_rados.get(), datalog_rados.get()); bilog_rados->init(bi_rados.get()); bucket_sobj->init(zone.get(), sysobj.get(), sysobj_cache.get(), - bi_rados.get(), meta.get(), mdlog.get(), meta_be_sobj.get(), + bi_rados.get(), meta.get(), mdlog.get(), sync_modules.get(), bucket_sync_sobj.get()); bucket_sync_sobj->init(zone.get(), sysobj.get(), @@ -375,10 +375,12 @@ int RGWCtlDef::init(RGWServices& svc, rgw::sal::Driver* driver, auto sync_module = svc.sync_modules->get_sync_module(); if (sync_module) { meta.bucket = sync_module->alloc_bucket_meta_handler(rados, svc.bucket, bucket.get()); - meta.bucket_instance.reset(sync_module->alloc_bucket_instance_meta_handler(driver)); + meta.bucket_instance = sync_module->alloc_bucket_instance_meta_handler( + driver, svc.zone, svc.bucket, svc.bi); } else { meta.bucket = create_bucket_metadata_handler(rados, svc.bucket, bucket.get()); - meta.bucket_instance.reset(RGWBucketInstanceMetaHandlerAllocator::alloc(driver)); + meta.bucket_instance = create_bucket_instance_metadata_handler( + driver, svc.zone, svc.bucket, svc.bi); } meta.otp = rgwrados::otp::create_metadata_handler( @@ -391,10 +393,6 @@ int RGWCtlDef::init(RGWServices& svc, rgw::sal::Driver* driver, user.reset(new RGWUserCtl(svc.zone, svc.user, (RGWUserMetadataHandler *)meta.user.get())); - RGWBucketInstanceMetadataHandlerBase *bi_meta_handler = static_cast(meta.bucket_instance.get()); - - bi_meta_handler->init(svc.zone, svc.bucket, svc.bi); - meta.topic_cache = std::make_unique>(); meta.topic_cache->init(svc.cache); @@ -403,10 +401,7 @@ int RGWCtlDef::init(RGWServices& svc, rgw::sal::Driver* driver, svc.zone->get_zone_params(), *meta.topic_cache); user->init(bucket.get()); - bucket->init(user.get(), - (RGWBucketInstanceMetadataHandler *)bi_meta_handler, - svc.datalog_rados, - dpp); + bucket->init(user.get(), svc.datalog_rados, dpp); return 0; } diff --git a/src/rgw/driver/rados/rgw_sync_module.cc b/src/rgw/driver/rados/rgw_sync_module.cc index f99565ff55c..ca25dc0a44e 100644 --- a/src/rgw/driver/rados/rgw_sync_module.cc +++ b/src/rgw/driver/rados/rgw_sync_module.cc @@ -24,9 +24,14 @@ auto RGWSyncModuleInstance::alloc_bucket_meta_handler(librados::Rados& rados, return create_bucket_metadata_handler(rados, svc_bucket, ctl_bucket); } -RGWBucketInstanceMetadataHandlerBase* RGWSyncModuleInstance::alloc_bucket_instance_meta_handler(rgw::sal::Driver* driver) +auto RGWSyncModuleInstance::alloc_bucket_instance_meta_handler(rgw::sal::Driver* driver, + RGWSI_Zone* svc_zone, + RGWSI_Bucket* svc_bucket, + RGWSI_BucketIndex* svc_bi) + -> std::unique_ptr { - return RGWBucketInstanceMetaHandlerAllocator::alloc(driver); + return create_bucket_instance_metadata_handler(driver, svc_zone, + svc_bucket, svc_bi); } RGWStatRemoteObjCBCR::RGWStatRemoteObjCBCR(RGWDataSyncCtx *_sc, diff --git a/src/rgw/driver/rados/rgw_sync_module.h b/src/rgw/driver/rados/rgw_sync_module.h index 8f7b1d1f0c4..cd9b2a1ba16 100644 --- a/src/rgw/driver/rados/rgw_sync_module.h +++ b/src/rgw/driver/rados/rgw_sync_module.h @@ -45,6 +45,8 @@ class RGWRESTMgr; class RGWMetadataHandler; class RGWBucketInstanceMetadataHandlerBase; class RGWSI_Bucket; +class RGWSI_BucketIndex; +class RGWSI_Zone; class RGWBucketCtl; class RGWSyncModuleInstance { @@ -62,7 +64,11 @@ public: RGWSI_Bucket* svc_bucket, RGWBucketCtl* ctl_bucket) -> std::unique_ptr; - virtual RGWBucketInstanceMetadataHandlerBase *alloc_bucket_instance_meta_handler(rgw::sal::Driver* driver); + virtual auto alloc_bucket_instance_meta_handler(rgw::sal::Driver* driver, + RGWSI_Zone* svc_zone, + RGWSI_Bucket* svc_bucket, + RGWSI_BucketIndex* svc_bi) + -> std::unique_ptr; // indication whether the sync module start with full sync (default behavior) // incremental sync would follow anyway diff --git a/src/rgw/services/svc_bi.h b/src/rgw/services/svc_bi.h index 901c28d59fd..3ab61e7421a 100644 --- a/src/rgw/services/svc_bi.h +++ b/src/rgw/services/svc_bi.h @@ -30,11 +30,11 @@ public: virtual ~RGWSI_BucketIndex() {} virtual int init_index(const DoutPrefixProvider *dpp, - RGWBucketInfo& bucket_info, + const RGWBucketInfo& bucket_info, const rgw::bucket_index_layout_generation& idx_layout, bool judge_support_logrecord = false) = 0; virtual int clean_index(const DoutPrefixProvider *dpp, - RGWBucketInfo& bucket_info, + const RGWBucketInfo& bucket_info, const rgw::bucket_index_layout_generation& idx_layout) = 0; virtual int read_stats(const DoutPrefixProvider *dpp, diff --git a/src/rgw/services/svc_bi_rados.cc b/src/rgw/services/svc_bi_rados.cc index 15cd5cd58ed..61d4edc0c98 100644 --- a/src/rgw/services/svc_bi_rados.cc +++ b/src/rgw/services/svc_bi_rados.cc @@ -352,7 +352,7 @@ int RGWSI_BucketIndex_RADOS::cls_bucket_head(const DoutPrefixProvider *dpp, } int RGWSI_BucketIndex_RADOS::init_index(const DoutPrefixProvider *dpp, - RGWBucketInfo& bucket_info, + const RGWBucketInfo& bucket_info, const rgw::bucket_index_layout_generation& idx_layout, bool judge_support_logrecord) { @@ -380,7 +380,8 @@ int RGWSI_BucketIndex_RADOS::init_index(const DoutPrefixProvider *dpp, } } -int RGWSI_BucketIndex_RADOS::clean_index(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info, const rgw::bucket_index_layout_generation& idx_layout) +int RGWSI_BucketIndex_RADOS::clean_index(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, + const rgw::bucket_index_layout_generation& idx_layout) { librados::IoCtx index_pool; diff --git a/src/rgw/services/svc_bi_rados.h b/src/rgw/services/svc_bi_rados.h index 7acf5c08807..b8990133fe0 100644 --- a/src/rgw/services/svc_bi_rados.h +++ b/src/rgw/services/svc_bi_rados.h @@ -122,11 +122,11 @@ public: } int init_index(const DoutPrefixProvider *dpp, - RGWBucketInfo& bucket_info, + const RGWBucketInfo& bucket_info, const rgw::bucket_index_layout_generation& idx_layout, bool judge_support_logrecord = false) override; int clean_index(const DoutPrefixProvider *dpp, - RGWBucketInfo& bucket_info, + const RGWBucketInfo& bucket_info, const rgw::bucket_index_layout_generation& idx_layout) override; /* RADOS specific */ diff --git a/src/rgw/services/svc_bucket.h b/src/rgw/services/svc_bucket.h index 540f57db885..6bb00e39a48 100644 --- a/src/rgw/services/svc_bucket.h +++ b/src/rgw/services/svc_bucket.h @@ -20,8 +20,6 @@ #include #include "rgw_service.h" -#include "svc_bucket_types.h" - class RGWMetadataLister; class RGWSI_Bucket : public RGWServiceInstance @@ -33,11 +31,12 @@ public: static std::string get_entrypoint_meta_key(const rgw_bucket& bucket); static std::string get_bi_meta_key(const rgw_bucket& bucket); - virtual RGWSI_BucketInstance_BE_Handler& get_bi_be_handler() = 0; - virtual int create_entrypoint_lister(const DoutPrefixProvider* dpp, const std::string& marker, std::unique_ptr& lister) = 0; + virtual int create_instance_lister(const DoutPrefixProvider* dpp, + const std::string& marker, + std::unique_ptr& lister) = 0; virtual int read_bucket_entrypoint_info(const std::string& key, RGWBucketEntryPoint *entry_point, diff --git a/src/rgw/services/svc_bucket_sobj.cc b/src/rgw/services/svc_bucket_sobj.cc index b4f0e280421..beb767717c7 100644 --- a/src/rgw/services/svc_bucket_sobj.cc +++ b/src/rgw/services/svc_bucket_sobj.cc @@ -22,7 +22,7 @@ using namespace std; -constexpr std::string_view instance_oid_prefix = ".bucket.meta."; +static const std::string instance_oid_prefix = ".bucket.meta."; // convert bucket instance oids back to the tenant/ format for metadata keys. // it's safe to parse 'tenant:' only for oids, because they won't contain the @@ -63,55 +63,6 @@ static std::string instance_oid_to_meta_key(const std::string& oid) return key; } -class RGWSI_BucketInstance_SObj_Module : public RGWSI_MBSObj_Handler_Module { - RGWSI_Bucket_SObj::Svc& svc; - - const string prefix; -public: - RGWSI_BucketInstance_SObj_Module(RGWSI_Bucket_SObj::Svc& _svc) : RGWSI_MBSObj_Handler_Module("bucket.instance"), - svc(_svc), prefix(instance_oid_prefix) {} - - void get_pool_and_oid(const string& key, rgw_pool *pool, string *oid) override { - if (pool) { - *pool = svc.zone->get_zone_params().domain_root; - } - if (oid) { - *oid = key_to_oid(key); - } - } - - const string& get_oid_prefix() override { - return prefix; - } - - bool is_valid_oid(const string& oid) override { - return (oid.compare(0, prefix.size(), prefix) == 0); - } - - string key_to_oid(const string& key) override { - return instance_meta_key_to_oid(key); - } - - string oid_to_key(const string& oid) override { - return instance_oid_to_meta_key(oid); - } - - /* - * hash entry for mdlog placement. Use the same hash key we'd have for the bucket entry - * point, so that the log entries end up at the same log shard, so that we process them - * in order - */ - string get_hash_key(const string& key) override { - string k = "bucket:"; - int pos = key.find(':'); - if (pos < 0) - k.append(key); - else - k.append(key.substr(0, pos)); - - return k; - } -}; RGWSI_Bucket_SObj::RGWSI_Bucket_SObj(CephContext *cct): RGWSI_Bucket(cct) { } @@ -122,7 +73,6 @@ RGWSI_Bucket_SObj::~RGWSI_Bucket_SObj() { void RGWSI_Bucket_SObj::init(RGWSI_Zone *_zone_svc, RGWSI_SysObj *_sysobj_svc, RGWSI_SysObj_Cache *_cache_svc, RGWSI_BucketIndex *_bi, RGWSI_Meta *_meta_svc, RGWSI_MDLog* mdlog_svc, - RGWSI_MetaBackend *_meta_be_svc, RGWSI_SyncModules *_sync_modules_svc, RGWSI_Bucket_Sync *_bucket_sync_svc) { @@ -133,7 +83,6 @@ void RGWSI_Bucket_SObj::init(RGWSI_Zone *_zone_svc, RGWSI_SysObj *_sysobj_svc, svc.bi = _bi; svc.meta = _meta_svc; svc.mdlog = mdlog_svc; - svc.meta_be = _meta_be_svc; svc.sync_modules = _sync_modules_svc; svc.bucket_sync = _bucket_sync_svc; } @@ -142,25 +91,6 @@ int RGWSI_Bucket_SObj::do_start(optional_yield, const DoutPrefixProvider *dpp) { binfo_cache.reset(new RGWChainedCacheImpl); binfo_cache->init(svc.cache); - - /* create a backend handler for bucket instance */ - - RGWSI_MetaBackend_Handler *bi_handler; - - int r = svc.meta->create_be_handler(RGWSI_MetaBackend::Type::MDBE_SOBJ, &bi_handler); - if (r < 0) { - ldpp_dout(dpp, 0) << "ERROR: failed to create be handler: r=" << r << dendl; - return r; - } - - bi_be_handler = bi_handler; - - RGWSI_MetaBackend_Handler_SObj *bi_bh = static_cast(bi_handler); - - auto bi_module = new RGWSI_BucketInstance_SObj_Module(svc); - bi_be_module.reset(bi_module); - bi_bh->set_module(bi_module); - return 0; } @@ -199,6 +129,36 @@ int RGWSI_Bucket_SObj::create_entrypoint_lister( return 0; } + +class BucketInstanceLister : public RGWMetadataLister { + public: + using RGWMetadataLister::RGWMetadataLister; + + void filter_transform(std::vector& oids, + std::list& keys) override + { + // transform instance oids to metadata keys + std::transform(oids.begin(), oids.end(), + std::back_inserter(keys), + instance_oid_to_meta_key); + } +}; + +int RGWSI_Bucket_SObj::create_instance_lister( + const DoutPrefixProvider* dpp, + const std::string& marker, + std::unique_ptr& lister) +{ + const rgw_pool& pool = svc.zone->get_zone_params().domain_root; + auto p = std::make_unique(svc.sysobj->get_pool(pool)); + int r = p->init(dpp, marker, instance_oid_prefix); + if (r < 0) { + return r; + } + lister = std::move(p); + return 0; +} + int RGWSI_Bucket_SObj::read_bucket_entrypoint_info(const string& key, RGWBucketEntryPoint *entry_point, RGWObjVersionTracker *objv_tracker, @@ -518,9 +478,13 @@ int RGWSI_Bucket_SObj::store_bucket_instance_info(const string& key, int ret = rgw_put_system_obj(dpp, svc.sysobj, pool, oid, bl, exclusive, &info.objv_tracker, mtime, y, pattrs); if (ret >= 0) { - int r = svc.bucket_sync->handle_bi_update(dpp, info, - orig_info.value_or(nullptr), - y); + int r = svc.mdlog->complete_entry(dpp, y, "bucket.instance", + key, &info.objv_tracker); + if (r < 0) { + return r; + } + + r = svc.bucket_sync->handle_bi_update(dpp, info, orig_info.value_or(nullptr), y); if (r < 0) { return r; } @@ -565,7 +529,8 @@ int RGWSI_Bucket_SObj::remove_bucket_instance_info(const string& key, */ } - return 0; + return svc.mdlog->complete_entry(dpp, y, "bucket.instance", + key, objv_tracker); } int RGWSI_Bucket_SObj::read_bucket_stats(const RGWBucketInfo& bucket_info, diff --git a/src/rgw/services/svc_bucket_sobj.h b/src/rgw/services/svc_bucket_sobj.h index 3bf3a3a59ff..6028222ed25 100644 --- a/src/rgw/services/svc_bucket_sobj.h +++ b/src/rgw/services/svc_bucket_sobj.h @@ -19,8 +19,6 @@ #include "rgw_service.h" -#include "svc_meta_be.h" -#include "svc_bucket_types.h" #include "svc_bucket.h" #include "svc_bucket_sync.h" @@ -47,9 +45,6 @@ class RGWSI_Bucket_SObj : public RGWSI_Bucket using RGWChainedCacheImpl_bucket_info_cache_entry = RGWChainedCacheImpl; std::unique_ptr binfo_cache; - RGWSI_BucketInstance_BE_Handler bi_be_handler; - std::unique_ptr bi_be_module; - int do_start(optional_yield, const DoutPrefixProvider *dpp) override; int do_read_bucket_instance_info(const std::string& key, @@ -75,7 +70,6 @@ public: RGWSI_SysObj_Cache *cache{nullptr}; RGWSI_Meta *meta{nullptr}; RGWSI_MDLog *mdlog{nullptr}; - RGWSI_MetaBackend *meta_be{nullptr}; RGWSI_SyncModules *sync_modules{nullptr}; RGWSI_Bucket_Sync *bucket_sync{nullptr}; } svc; @@ -83,17 +77,12 @@ public: RGWSI_Bucket_SObj(CephContext *cct); ~RGWSI_Bucket_SObj(); - RGWSI_BucketInstance_BE_Handler& get_bi_be_handler() override { - return bi_be_handler; - } - void init(RGWSI_Zone *_zone_svc, RGWSI_SysObj *_sysobj_svc, RGWSI_SysObj_Cache *_cache_svc, RGWSI_BucketIndex *_bi, RGWSI_Meta *_meta_svc, RGWSI_MDLog *mdlog_svc, - RGWSI_MetaBackend *_meta_be_svc, RGWSI_SyncModules *_sync_modules_svc, RGWSI_Bucket_Sync *_bucket_sync_svc); @@ -101,6 +90,10 @@ public: const std::string& marker, std::unique_ptr& lister) override; + int create_instance_lister(const DoutPrefixProvider* dpp, + const std::string& marker, + std::unique_ptr& lister) override; + int read_bucket_entrypoint_info(const std::string& key, RGWBucketEntryPoint *entry_point, RGWObjVersionTracker *objv_tracker, diff --git a/src/rgw/services/svc_bucket_sync.h b/src/rgw/services/svc_bucket_sync.h index 29b2c493612..b72740b386d 100644 --- a/src/rgw/services/svc_bucket_sync.h +++ b/src/rgw/services/svc_bucket_sync.h @@ -19,8 +19,6 @@ #include "driver/rados/rgw_service.h" // FIXME: subclass dependency -#include "svc_bucket_types.h" - class RGWBucketSyncPolicyHandler; using RGWBucketSyncPolicyHandlerRef = std::shared_ptr; diff --git a/src/rgw/services/svc_bucket_types.h b/src/rgw/services/svc_bucket_types.h deleted file mode 100644 index 30e5309d56d..00000000000 --- a/src/rgw/services/svc_bucket_types.h +++ /dev/null @@ -1,38 +0,0 @@ - -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab ft=cpp - -/* - * Ceph - scalable distributed file system - * - * Copyright (C) 2019 Red Hat, Inc. - * - * This is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License version 2.1, as published by the Free Software - * Foundation. See file COPYING. - * - */ - - -#pragma once - -#include "common/ptr_wrapper.h" - -#include "svc_meta_be.h" -#include "svc_meta_be_types.h" - -class RGWSI_MetaBackend_Handler; - -using RGWSI_Bucket_BE_Handler = ptr_wrapper; -using RGWSI_BucketInstance_BE_Handler = ptr_wrapper; - - -using RGWSI_Bucket_EP_Ctx = ptr_wrapper; -using RGWSI_Bucket_BI_Ctx = ptr_wrapper; - -struct RGWSI_Bucket_X_Ctx { - RGWSI_Bucket_EP_Ctx ep; - RGWSI_Bucket_BI_Ctx bi; -}; - -- 2.39.5