From: J. Eric Ivancich Date: Thu, 17 Dec 2020 23:32:20 +0000 (-0500) Subject: rgw: add radosgw-admin bucket radoslist bucket/obj mapping X-Git-Tag: v15.2.13~2^2~18^2~2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=7bd0ecd53d220a2ed2766283c746b20c77c6481c;p=ceph.git rgw: add radosgw-admin bucket radoslist bucket/obj mapping A new command-line option "--rgw-obj-fs" is added to radosgw-admin. When used with the "bucket radoslist" subcommand, will output lines with a rados object, bucket name, and object name, separated by the field separator specified. Without this command-line option, only the rados object is output, which is the previous behavior. Signed-off-by: J. Eric Ivancich (cherry picked from commit 81d6260c1a42400bfa8c677c419e2c3f76d7a920) --- diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc index a6eef9a54205..20850c153cbd 100644 --- a/src/rgw/rgw_admin.cc +++ b/src/rgw/rgw_admin.cc @@ -415,6 +415,11 @@ void usage() cout << " --totp-seconds the time resolution that is being used for TOTP generation\n"; cout << " --totp-window the number of TOTP tokens that are checked before and after the current token when validating token\n"; cout << " --totp-pin the valid value of a TOTP token at a certain time\n"; + cout << "\nradoslist options:\n"; + cout << " --rgw-obj-fs the field separator that will separate the rados\n"; + cout << " object name from the rgw object name;\n"; + cout << " additionally rados objects for incomplete\n"; + cout << " multipart uploads will not be output\n"; cout << "\n"; generic_client_usage(); } @@ -3249,6 +3254,8 @@ int main(int argc, const char **argv) SimpleCmd cmd(all_cmds, cmd_aliases); + std::optional rgw_obj_fs; // radoslist field separator + for (std::vector::iterator i = args.begin(); i != args.end(); ) { if (ceph_argparse_double_dash(args, i)) { break; @@ -3668,6 +3675,8 @@ int main(int argc, const char **argv) opt_dest_owner = val; } else if (ceph_argparse_binary_flag(args, i, &detail, NULL, "--detail", (char*)NULL)) { // do nothing + } else if (ceph_argparse_witharg(args, i, &val, "--rgw-obj-fs", (char*)NULL)) { + rgw_obj_fs = val; } else if (strncmp(*i, "-", 1) == 0) { cerr << "ERROR: invalid flag " << *i << std::endl; return EINVAL; @@ -6099,6 +6108,10 @@ int main(int argc, const char **argv) if (opt_cmd == OPT::BUCKET_RADOS_LIST) { RGWRadosList lister(store, max_concurrent_ios, orphan_stale_secs, tenant); + if (rgw_obj_fs) { + lister.set_field_separator(*rgw_obj_fs); + } + if (bucket_name.empty()) { ret = lister.run(); } else { diff --git a/src/rgw/rgw_orphan.cc b/src/rgw/rgw_orphan.cc index f9bffb7ca38c..d6ad28ff0e94 100644 --- a/src/rgw/rgw_orphan.cc +++ b/src/rgw/rgw_orphan.cc @@ -925,6 +925,8 @@ int RGWOrphanSearch::finish() int RGWRadosList::handle_stat_result(RGWRados::Object::Stat::Result& result, + std::string& bucket_name, + rgw_obj_key& obj_key, std::set& obj_oids) { obj_oids.clear(); @@ -949,6 +951,9 @@ int RGWRadosList::handle_stat_result(RGWRados::Object::Stat::Result& result, return 0; } + bucket_name = bucket.name; + obj_key = result.obj.key; + if (!result.manifest) { /* a very very old object, or part of a multipart upload during upload */ obj_oids.insert(oid); @@ -1049,7 +1054,9 @@ int RGWRadosList::pop_and_handle_stat_op( RGWObjectCtx& obj_ctx, std::deque& ops) { - std::set obj_oids; + std::string bucket_name; + rgw_obj_key obj_key; + std::set obj_oids; RGWRados::Object::Stat& front_op = ops.front(); int ret = front_op.wait(); @@ -1061,7 +1068,7 @@ int RGWRadosList::pop_and_handle_stat_op( goto done; } - ret = handle_stat_result(front_op.result, obj_oids); + ret = handle_stat_result(front_op.result, bucket_name, obj_key, obj_oids); if (ret < 0) { lderr(store->ctx()) << "ERROR: handle_stat_result() returned error: " << cpp_strerror(-ret) << dendl; @@ -1069,7 +1076,14 @@ int RGWRadosList::pop_and_handle_stat_op( // output results for (const auto& o : obj_oids) { - std::cout << o << std::endl; + if (include_rgw_obj_name) { + std::cout << o << + field_separator << bucket_name << + field_separator << obj_key << + std::endl; + } else { + std::cout << o << std::endl; + } } done: @@ -1341,6 +1355,7 @@ int RGWRadosList::run(const std::string& start_bucket_name) { RGWSysObjectCtx sys_obj_ctx = store->svc()->sysobj->init_obj_ctx(); RGWObjectCtx obj_ctx(store); + RGWBucketInfo bucket_info; int ret; add_bucket_entire(start_bucket_name); @@ -1416,10 +1431,13 @@ int RGWRadosList::run(const std::string& start_bucket_name) } } // while (! bucket_process_map.empty()) + if (include_rgw_obj_name) { + goto done; + } + // now handle incomplete multipart uploads by going back to the // initial bucket - RGWBucketInfo bucket_info; ret = store->getRados()->get_bucket_info(store->svc(), tenant_name, start_bucket_name, @@ -1442,6 +1460,8 @@ int RGWRadosList::run(const std::string& start_bucket_name) return ret; } +done: + return 0; } // RGWRadosList::run(string) diff --git a/src/rgw/rgw_orphan.h b/src/rgw/rgw_orphan.h index ae1e61b780e7..2339ae044812 100644 --- a/src/rgw/rgw_orphan.h +++ b/src/rgw/rgw_orphan.h @@ -13,8 +13,7 @@ * */ -#ifndef CEPH_RGW_ORPHAN_H -#define CEPH_RGW_ORPHAN_H +#pragma once #include "common/config.h" #include "common/Formatter.h" @@ -258,7 +257,12 @@ class RGWRadosList { uint64_t stale_secs; std::string tenant_name; + bool include_rgw_obj_name; + std::string field_separator; + int handle_stat_result(RGWRados::Object::Stat::Result& result, + std::string& bucket_name, + rgw_obj_key& obj_key, std::set& obj_oids); int pop_and_handle_stat_op(RGWObjectCtx& obj_ctx, std::deque& ops); @@ -272,7 +276,8 @@ public: store(_store), max_concurrent_ios(_max_ios), stale_secs(_stale_secs), - tenant_name(_tenant_name) + tenant_name(_tenant_name), + include_rgw_obj_name(false) {} int process_bucket(const std::string& bucket_instance_id, @@ -286,6 +291,11 @@ public: int run(const std::string& bucket_id); int run(); -}; // class RGWRadosList -#endif + // if there's a non-empty field separator, that means we'll display + // bucket and object names + void set_field_separator(const std::string& fs) { + field_separator = fs; + include_rgw_obj_name = !field_separator.empty(); + } +}; // class RGWRadosList diff --git a/src/test/cli/radosgw-admin/help.t b/src/test/cli/radosgw-admin/help.t index 27c01c4ddaa7..2dd1ea384cfb 100644 --- a/src/test/cli/radosgw-admin/help.t +++ b/src/test/cli/radosgw-admin/help.t @@ -316,6 +316,12 @@ --totp-window the number of TOTP tokens that are checked before and after the current token when validating token --totp-pin the valid value of a TOTP token at a certain time + radoslist options: + --rgw-obj-fs the field separator that will separate the rados + object name from the rgw object name; + additionally rados objects for incomplete + multipart uploads will not be output + --conf/-c FILE read configuration from the given configuration file --id ID set ID portion of my name --name/-n TYPE.ID set name