From: Shilpa Jagannath Date: Wed, 7 Aug 2024 19:33:57 +0000 (-0400) Subject: rgw/multisite: move the remote bucket listing logic into a separate X-Git-Tag: v20.3.0~161^2~11 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=4b343ac6d899f8cb8d3790e5a34c7190f51a23f3;p=ceph.git rgw/multisite: move the remote bucket listing logic into a separate function. Signed-off-by: Shilpa Jagannath --- diff --git a/PendingReleaseNotes b/PendingReleaseNotes index e0d2e12e5a189..64688e8630d97 100644 --- a/PendingReleaseNotes +++ b/PendingReleaseNotes @@ -136,6 +136,10 @@ `ceph fs subvolume earmark rm` have been added to set, get and remove earmark from a given subvolume. * RADOS: Add ``messenger dump`` command to retrieve runtime information on connections, sockets, and kernel TCP stats from the messenger. +* RGW: deleted buckets are automatically cleaned up as part of trimming process. + DeleteBucket will start returning 409 BucketNotEmpty errors until empty + on all zones when sync policy is enabled. + `radosgw-admin bucket list-deleted` lists such buckets. * RADOS: A performance botteneck in the balancer mgr module has been fixed. Related Tracker: https://tracker.ceph.com/issues/68657 diff --git a/src/rgw/driver/rados/rgw_cr_rados.cc b/src/rgw/driver/rados/rgw_cr_rados.cc index 1ac07306b09c0..fc18cfd162ab9 100644 --- a/src/rgw/driver/rados/rgw_cr_rados.cc +++ b/src/rgw/driver/rados/rgw_cr_rados.cc @@ -1214,7 +1214,7 @@ RGWStatRemoteBucketCR::RGWStatRemoteBucketCR(const DoutPrefixProvider *dpp, std::vector& peer_result) : RGWCoroutine(store->ctx()), dpp(dpp), store(store), source_zone(source_zone), bucket(bucket), http(http), - zids(zids), peer_result(peer_result) {} + zids(std::move(zids)), peer_result(peer_result) {} int RGWStatRemoteBucketCR::operate(const DoutPrefixProvider *dpp) { reenter(this) { diff --git a/src/rgw/driver/rados/rgw_cr_rados.h b/src/rgw/driver/rados/rgw_cr_rados.h index 6a5ef9851f3a4..c01132fa3555e 100644 --- a/src/rgw/driver/rados/rgw_cr_rados.h +++ b/src/rgw/driver/rados/rgw_cr_rados.h @@ -16,8 +16,6 @@ #include "services/svc_sys_obj.h" #include "services/svc_bucket.h" -#include -using namespace std; struct rgw_http_param_pair; class RGWRESTConn; @@ -1725,12 +1723,12 @@ struct bucket_list_entry { rgw_obj_key key; bool is_latest; real_time mtime; - string etag; + std::string etag; uint64_t size; - string storage_class; + std::string storage_class; rgw_bucket_entry_owner owner; uint64_t versioned_epoch; - string rgw_tag; + std::string rgw_tag; bucket_list_entry() : delete_marker(false), is_latest(false), size(0), versioned_epoch(0) {} @@ -1739,7 +1737,7 @@ struct bucket_list_entry { JSONDecoder::decode_json("Key", key.name, obj); JSONDecoder::decode_json("VersionId", key.instance, obj); JSONDecoder::decode_json("IsLatest", is_latest, obj); - string mtime_str; + std::string mtime_str; JSONDecoder::decode_json("RgwxMtime", mtime_str, obj); struct tm t; @@ -1773,12 +1771,12 @@ struct bucket_list_entry { }; struct bucket_unordered_list_result { - string name; - string prefix; + std::string name; + std::string prefix; int max_keys; bool is_truncated; bool allow_unordered; - list entries; + std::list entries; bucket_unordered_list_result() : max_keys(0), is_truncated(false) {} @@ -1800,7 +1798,7 @@ class RGWStatRemoteBucketCR: public RGWCoroutine { const rgw_bucket& bucket; RGWHTTPManager* http; std::vector zids; - vector& peer_result; + std::vector& peer_result; public: RGWStatRemoteBucketCR(const DoutPrefixProvider *dpp, diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc index 3fb9a4eb09e50..1941a1b4e3174 100644 --- a/src/rgw/driver/rados/rgw_rados.cc +++ b/src/rgw/driver/rados/rgw_rados.cc @@ -5590,7 +5590,79 @@ int RGWRados::store_delete_bucket_info_flag(RGWBucketInfo& bucket_info, std::map return r; } - + +int get_zone_ids(const DoutPrefixProvider *dpp, + rgw::sal::RadosStore* const driver, + const rgw_zone_id source_zone, + const rgw_bucket& bucket, + std::vector& zids, + optional_yield y) +{ + RGWBucketSyncPolicyHandlerRef source_handler; + int ret = driver->get_sync_policy_handler(dpp, source_zone, bucket, &source_handler, y); + if (ret < 0) { + ldpp_dout(dpp, 10) << "failed to get bucket sync policy handler (r=" << ret << ")" << dendl; + return ret; + } + + auto all_dests = source_handler->get_all_dests(); + rgw_zone_id last_zid; + for (auto& diter : all_dests) { + const auto& zid = diter.first; + if (zid == last_zid) { + continue; + } + last_zid = zid; + zids.push_back(zid); + } + return 0; +} + +int list_remote_buckets(const DoutPrefixProvider *dpp, + rgw::sal::RadosStore* const driver, + const rgw_zone_id source_zone, + const rgw_bucket& bucket, + optional_yield y) +{ + + std::vector zids; + int ret = get_zone_ids(dpp, driver, source_zone, bucket, zids, y); + if (ret < 0) { + ldpp_dout(dpp, 10) << "failed to get remote zones (r=" << ret << ")" << dendl; + return ret; + } + + std::vector peer_status; + peer_status.resize(zids.size()); + + RGWCoroutinesManager crs(driver->ctx(), driver->getRados()->get_cr_registry()); + RGWHTTPManager http(driver->ctx(), crs.get_completion_mgr()); + ret = http.start(); + if (ret < 0) { + ldpp_dout(dpp, 0) << "failed in http_manager.start() ret=" << ret << dendl; + return ret; + } + + ret = crs.run(dpp, new RGWStatRemoteBucketCR(dpp, driver, source_zone, bucket, &http, zids, peer_status)); + if (ret < 0) { + ldpp_dout(dpp, 0) << "failed to fetch remote bucket stats " << cpp_strerror(ret) << dendl; + return ret; + } + + for (const auto& list_result: peer_status) { + auto entries_iter = list_result.entries.begin(); + for (; entries_iter != list_result.entries.end(); ++entries_iter) { + std::string ns; + rgw_obj_key obj; + + if (rgw_obj_key::oid_to_key_in_ns(entries_iter->key.name, &obj, ns)) { + return -ENOTEMPTY; + } + } + } + return 0; +} + /** * Delete a bucket. * bucket: the name of the bucket to delete @@ -5614,58 +5686,20 @@ int RGWRados::delete_bucket(RGWBucketInfo& bucket_info, std::mapis_syncing_bucket_meta(bucket)) { auto bs_policy = bucket_info.sync_policy; if (bs_policy) { - ldpp_dout(dpp, 10) << "bucket policy exists" << dendl; + ldpp_dout(dpp, 10) << "bucket policy exists. listing remote zones" << dendl; const rgw_zone_id source_zone = svc.zone->get_zone_params().get_id(); - RGWBucketSyncPolicyHandlerRef source_handler; - int ret = driver->get_sync_policy_handler(dpp, source_zone, bucket, &source_handler, y); - if (ret < 0) { - ldpp_dout(dpp, 10) << "could not get bucket sync policy handler (r=" << ret << ")" << dendl; - } - - auto all_dests = source_handler->get_all_dests(); - std::vector zids; - rgw_zone_id last_zid; - for (auto& diter : all_dests) { - const auto& zid = diter.first; - if (zid == last_zid) { - continue; - } - last_zid = zid; - zids.push_back(zid); - } - - std::vector peer_status; - peer_status.resize(zids.size()); - - RGWCoroutinesManager crs(driver->ctx(), driver->getRados()->get_cr_registry()); - RGWHTTPManager http(driver->ctx(), crs.get_completion_mgr()); - ret = http.start(); - if (ret < 0) { - ldpp_dout(dpp, 0) << "failed in http_manager.start() ret=" << ret << dendl; - } - ret = crs.run(dpp, new RGWStatRemoteBucketCR(dpp, driver, source_zone, bucket, &http, zids, peer_status)); - if (ret < 0) { - ldpp_dout(dpp, 0) << "failed to fetch remote bucket stats " << cpp_strerror(ret) << dendl; - } - - for (const auto& list_result: peer_status) { - auto entries_iter = list_result.entries.begin(); - for (; entries_iter != list_result.entries.end(); ++entries_iter) { - std::string ns; - rgw_obj_key obj; - - if (rgw_obj_key::oid_to_key_in_ns(entries_iter->key.name, &obj, ns)) { - ldpp_dout(dpp, 0) << "cannot delete bucket. objects exist in the bucket in another zone " << dendl; - return -ENOTEMPTY; - } - } + r = list_remote_buckets(dpp, driver, source_zone, bucket, y); + if (r == -ENOTEMPTY) { + ldpp_dout(dpp, 0) << "ERROR: cannot delete bucket. objects exist in the bucket on another zone " << dendl; + return r; + } else if (r < 0) { + ldpp_dout(dpp, 10) << "failed to list remote buckets" << dendl; + // don't return. } } }