From 81d6260c1a42400bfa8c677c419e2c3f76d7a920 Mon Sep 17 00:00:00 2001 From: "J. Eric Ivancich" Date: Thu, 17 Dec 2020 18:32:20 -0500 Subject: [PATCH] rgw: add radosgw-admin bucket radoslist bucket/obj mapping A new command-line option "--rgw-obj-fs" is added to radosgw-admin. When used with the "bucket radoslist" subcommand, will output lines with a rados object, bucket name, and object name, separated by the field separator specified. Without this command-line option, only the rados object is output, which is the previous behavior. Signed-off-by: J. Eric Ivancich --- src/rgw/rgw_admin.cc | 13 +++++++++++++ src/rgw/rgw_orphan.cc | 28 ++++++++++++++++++++++++---- src/rgw/rgw_orphan.h | 20 +++++++++++++++----- src/test/cli/radosgw-admin/help.t | 6 ++++++ 4 files changed, 58 insertions(+), 9 deletions(-) diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc index 5c833b6b21a..475e67cf325 100644 --- a/src/rgw/rgw_admin.cc +++ b/src/rgw/rgw_admin.cc @@ -438,6 +438,11 @@ void usage() cout << " --context context in which the script runs. one of: preRequest, postRequest\n"; cout << " --package name of the lua package that should be added/removed to/from the allowlist\n"; cout << " --allow-compilation package is allowed to compile C code as part of its installation\n"; + cout << "\nradoslist options:\n"; + cout << " --rgw-obj-fs the field separator that will separate the rados\n"; + cout << " object name from the rgw object name;\n"; + cout << " additionally rados objects for incomplete\n"; + cout << " multipart uploads will not be output\n"; cout << "\n"; generic_client_usage(); } @@ -3227,6 +3232,8 @@ int main(int argc, const char **argv) SimpleCmd cmd(all_cmds, cmd_aliases); + std::optional rgw_obj_fs; // radoslist field separator + for (std::vector::iterator i = args.begin(); i != args.end(); ) { if (ceph_argparse_double_dash(args, i)) { break; @@ -3650,6 +3657,8 @@ int main(int argc, const char **argv) script_package = val; } else if (ceph_argparse_binary_flag(args, i, &allow_compilation, NULL, "--allow-compilation", (char*)NULL)) { // do nothing + } else if (ceph_argparse_witharg(args, i, &val, "--rgw-obj-fs", (char*)NULL)) { + rgw_obj_fs = val; } else if (strncmp(*i, "-", 1) == 0) { cerr << "ERROR: invalid flag " << *i << std::endl; return EINVAL; @@ -6101,6 +6110,10 @@ int main(int argc, const char **argv) if (opt_cmd == OPT::BUCKET_RADOS_LIST) { RGWRadosList lister(store, max_concurrent_ios, orphan_stale_secs, tenant); + if (rgw_obj_fs) { + lister.set_field_separator(*rgw_obj_fs); + } + if (bucket_name.empty()) { ret = lister.run(dpp()); } else { diff --git a/src/rgw/rgw_orphan.cc b/src/rgw/rgw_orphan.cc index 50574e4ac88..62408fcc114 100644 --- a/src/rgw/rgw_orphan.cc +++ b/src/rgw/rgw_orphan.cc @@ -925,6 +925,8 @@ int RGWOrphanSearch::finish() int RGWRadosList::handle_stat_result(RGWRados::Object::Stat::Result& result, + std::string& bucket_name, + rgw_obj_key& obj_key, std::set& obj_oids) { obj_oids.clear(); @@ -949,6 +951,9 @@ int RGWRadosList::handle_stat_result(RGWRados::Object::Stat::Result& result, return 0; } + bucket_name = bucket.name; + obj_key = result.obj.key; + if (!result.manifest) { /* a very very old object, or part of a multipart upload during upload */ obj_oids.insert(oid); @@ -1049,7 +1054,9 @@ int RGWRadosList::pop_and_handle_stat_op( RGWObjectCtx& obj_ctx, std::deque& ops) { - std::set obj_oids; + std::string bucket_name; + rgw_obj_key obj_key; + std::set obj_oids; RGWRados::Object::Stat& front_op = ops.front(); int ret = front_op.wait(); @@ -1061,7 +1068,7 @@ int RGWRadosList::pop_and_handle_stat_op( goto done; } - ret = handle_stat_result(front_op.result, obj_oids); + ret = handle_stat_result(front_op.result, bucket_name, obj_key, obj_oids); if (ret < 0) { lderr(store->ctx()) << "ERROR: handle_stat_result() returned error: " << cpp_strerror(-ret) << dendl; @@ -1069,7 +1076,14 @@ int RGWRadosList::pop_and_handle_stat_op( // output results for (const auto& o : obj_oids) { - std::cout << o << std::endl; + if (include_rgw_obj_name) { + std::cout << o << + field_separator << bucket_name << + field_separator << obj_key << + std::endl; + } else { + std::cout << o << std::endl; + } } done: @@ -1343,6 +1357,7 @@ int RGWRadosList::run(const DoutPrefixProvider *dpp, const std::string& start_bu { RGWSysObjectCtx sys_obj_ctx = store->svc()->sysobj->init_obj_ctx(); RGWObjectCtx obj_ctx(store); + RGWBucketInfo bucket_info; int ret; add_bucket_entire(start_bucket_name); @@ -1419,10 +1434,13 @@ int RGWRadosList::run(const DoutPrefixProvider *dpp, const std::string& start_bu } } // while (! bucket_process_map.empty()) + if (include_rgw_obj_name) { + goto done; + } + // now handle incomplete multipart uploads by going back to the // initial bucket - RGWBucketInfo bucket_info; ret = store->getRados()->get_bucket_info(store->svc(), tenant_name, start_bucket_name, @@ -1446,6 +1464,8 @@ int RGWRadosList::run(const DoutPrefixProvider *dpp, const std::string& start_bu return ret; } +done: + return 0; } // RGWRadosList::run(string) diff --git a/src/rgw/rgw_orphan.h b/src/rgw/rgw_orphan.h index fc4fc0ea16f..10e9f523909 100644 --- a/src/rgw/rgw_orphan.h +++ b/src/rgw/rgw_orphan.h @@ -13,8 +13,7 @@ * */ -#ifndef CEPH_RGW_ORPHAN_H -#define CEPH_RGW_ORPHAN_H +#pragma once #include "common/config.h" #include "common/Formatter.h" @@ -258,7 +257,12 @@ class RGWRadosList { uint64_t stale_secs; std::string tenant_name; + bool include_rgw_obj_name; + std::string field_separator; + int handle_stat_result(RGWRados::Object::Stat::Result& result, + std::string& bucket_name, + rgw_obj_key& obj_key, std::set& obj_oids); int pop_and_handle_stat_op(RGWObjectCtx& obj_ctx, std::deque& ops); @@ -272,7 +276,8 @@ public: store(_store), max_concurrent_ios(_max_ios), stale_secs(_stale_secs), - tenant_name(_tenant_name) + tenant_name(_tenant_name), + include_rgw_obj_name(false) {} int process_bucket(const DoutPrefixProvider *dpp, @@ -288,6 +293,11 @@ public: int run(const DoutPrefixProvider *dpp, const std::string& bucket_id); int run(const DoutPrefixProvider *dpp); -}; // class RGWRadosList -#endif + // if there's a non-empty field separator, that means we'll display + // bucket and object names + void set_field_separator(const std::string& fs) { + field_separator = fs; + include_rgw_obj_name = !field_separator.empty(); + } +}; // class RGWRadosList diff --git a/src/test/cli/radosgw-admin/help.t b/src/test/cli/radosgw-admin/help.t index 86e6110d0bb..490499f24a0 100644 --- a/src/test/cli/radosgw-admin/help.t +++ b/src/test/cli/radosgw-admin/help.t @@ -340,6 +340,12 @@ --package name of the lua package that should be added/removed to/from the allowlist --allow-compilation package is allowed to compile C code as part of its installation + radoslist options: + --rgw-obj-fs the field separator that will separate the rados + object name from the rgw object name; + additionally rados objects for incomplete + multipart uploads will not be output + --conf/-c FILE read configuration from the given configuration file --id ID set ID portion of my name --name/-n TYPE.ID set name -- 2.39.5