From 804dc9519e93f805bdbed24f47045496b1a8ca63 Mon Sep 17 00:00:00 2001 From: "J. Eric Ivancich" Date: Mon, 17 Jan 2022 16:14:16 -0500 Subject: [PATCH] rgw: add indexless bucket logic to "bucket radoslist" The "bucket radoslist" sub-command of radosgw-admin is supposed to list all rados objects tied to one or all directories and thereby provide a way to determine orphaned rados objects. But indexless buckets don't provide an index to employ for this purpose. So warnings or errors should be provided depending on the circumstances. Signed-off-by: J. Eric Ivancich --- src/rgw/rgw_admin.cc | 4 +++- src/rgw/rgw_orphan.cc | 47 +++++++++++++++++++++++++++++++++---------- src/rgw/rgw_orphan.h | 7 +++++-- 3 files changed, 44 insertions(+), 14 deletions(-) diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc index e7bb912b840f0..9ba9979f628d9 100644 --- a/src/rgw/rgw_admin.cc +++ b/src/rgw/rgw_admin.cc @@ -6860,7 +6860,9 @@ int main(int argc, const char **argv) } if (bucket_name.empty()) { - ret = lister.run(dpp()); + // yes_i_really_mean_it means continue with listing even if + // there are indexless buckets + ret = lister.run(dpp(), yes_i_really_mean_it); } else { ret = lister.run(dpp(), bucket_name); } diff --git a/src/rgw/rgw_orphan.cc b/src/rgw/rgw_orphan.cc index a43d2a6754b5d..e36231f2eb36b 100644 --- a/src/rgw/rgw_orphan.cc +++ b/src/rgw/rgw_orphan.cc @@ -1319,7 +1319,8 @@ int RGWRadosList::process_bucket( } -int RGWRadosList::run(const DoutPrefixProvider *dpp) +int RGWRadosList::run(const DoutPrefixProvider *dpp, + const bool yes_i_really_mean_it) { int ret; void* handle = nullptr; @@ -1332,17 +1333,36 @@ int RGWRadosList::run(const DoutPrefixProvider *dpp) return ret; } - const int max_keys = 1000; + constexpr int max_keys = 1000; bool truncated = true; + bool warned_indexless = false; do { std::list buckets; ret = store->meta_list_keys_next(dpp, handle, max_keys, buckets, &truncated); for (std::string& bucket_id : buckets) { - ret = run(dpp, bucket_id); + ret = run(dpp, bucket_id, true); if (ret == -ENOENT) { continue; + } else if (ret == -EINVAL) { + if (! warned_indexless) { + if (yes_i_really_mean_it) { + std::cerr << + "WARNING: because there is at least one indexless bucket (" << + bucket_id << + ") the results of radoslist are *incomplete*; continuing due to --yes-i-really-mean-it" << + std::endl; + warned_indexless = true; + } else { + std::cerr << "ERROR: because there is at least one indexless bucket (" << + bucket_id << + ") the results of radoslist are *incomplete*; use --yes-i-really-mean-it to bypass error" << + std::endl; + return ret; + } + } + continue; } else if (ret < 0) { return ret; } @@ -1350,13 +1370,13 @@ int RGWRadosList::run(const DoutPrefixProvider *dpp) } while (truncated); return 0; -} // RGWRadosList::run() +} // RGWRadosList::run(DoutPrefixProvider, bool) -int RGWRadosList::run(const DoutPrefixProvider *dpp, const std::string& start_bucket_name) +int RGWRadosList::run(const DoutPrefixProvider *dpp, + const std::string& start_bucket_name, + const bool silent_indexless) { - RGWObjectCtx obj_ctx(store); - std::unique_ptr bucket; int ret; add_bucket_entire(start_bucket_name); @@ -1380,6 +1400,12 @@ int RGWRadosList::run(const DoutPrefixProvider *dpp, const std::string& start_bu std::cerr << "ERROR: could not get info for bucket " << bucket_name << " -- " << cpp_strerror(-ret) << std::endl; return ret; + } else if (bucket->get_info().is_indexless()) { + if (! silent_indexless) { + std::cerr << "ERROR: unable to run radoslist on indexless bucket " << + bucket_name << std::endl; + } + return -EINVAL; } const std::string bucket_id = bucket->get_key().get_key(); @@ -1428,12 +1454,13 @@ int RGWRadosList::run(const DoutPrefixProvider *dpp, const std::string& start_bu } // while (! bucket_process_map.empty()) if (include_rgw_obj_name) { - goto done; + return 0; } // now handle incomplete multipart uploads by going back to the // initial bucket + std::unique_ptr bucket; ret = store->get_bucket(dpp, nullptr, tenant_name, start_bucket_name, &bucket, null_yield); if (ret == -ENOENT) { // bucket deletion race? @@ -1451,10 +1478,8 @@ int RGWRadosList::run(const DoutPrefixProvider *dpp, const std::string& start_bu return ret; } -done: - return 0; -} // RGWRadosList::run(string) +} // RGWRadosList::run(DoutPrefixProvider, string, bool) int RGWRadosList::do_incomplete_multipart(const DoutPrefixProvider *dpp, diff --git a/src/rgw/rgw_orphan.h b/src/rgw/rgw_orphan.h index a5081374e88fd..4ecbd0e97ccd5 100644 --- a/src/rgw/rgw_orphan.h +++ b/src/rgw/rgw_orphan.h @@ -291,8 +291,11 @@ public: int build_linked_oids_index(); - int run(const DoutPrefixProvider *dpp, const std::string& bucket_id); - int run(const DoutPrefixProvider *dpp); + int run(const DoutPrefixProvider *dpp, + const std::string& bucket_id, + const bool silent_indexless = false); + int run(const DoutPrefixProvider *dpp, + const bool yes_i_really_mean_it = false); // if there's a non-empty field separator, that means we'll display // bucket and object names -- 2.39.5