rgw: add radosgw-admin bucket radoslist bucket/obj mapping

A new command-line option "--rgw-obj-fs" is added to
radosgw-admin. When used with the "bucket radoslist" subcommand, will
output lines with a rados object, bucket name, and object name,
separated by the field separator specified. Without this command-line
option, only the rados object is output, which is the previous
behavior.

Signed-off-by: J. Eric Ivancich <ivancich@redhat.com>

diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index 5c833b6b21a706a56619f91e5bbc24e517970675..475e67cf32568bf7103d46bc4c5f9e00da95dde0 100644 (file)
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -438,6 +438,11 @@ void usage()
   cout << "   --context                 context in which the script runs. one of: preRequest, postRequest\n";
   cout << "   --package                 name of the lua package that should be added/removed to/from the allowlist\n";
   cout << "   --allow-compilation       package is allowed to compile C code as part of its installation\n";
+  cout << "\nradoslist options:\n";
+  cout << "   --rgw-obj-fs              the field separator that will separate the rados\n";
+  cout << "                             object name from the rgw object name;\n";
+  cout << "                             additionally rados objects for incomplete\n";
+  cout << "                             multipart uploads will not be output\n";
   cout << "\n";
   generic_client_usage();
 }
@@ -3227,6 +3232,8 @@ int main(int argc, const char **argv)
 
   SimpleCmd cmd(all_cmds, cmd_aliases);
 
+  std::optional<std::string> rgw_obj_fs; // radoslist field separator
+
   for (std::vector<const char*>::iterator i = args.begin(); i != args.end(); ) {
     if (ceph_argparse_double_dash(args, i)) {
       break;
@@ -3650,6 +3657,8 @@ int main(int argc, const char **argv)
       script_package = val;
     } else if (ceph_argparse_binary_flag(args, i, &allow_compilation, NULL, "--allow-compilation", (char*)NULL)) {
       // do nothing
+    } else if (ceph_argparse_witharg(args, i, &val, "--rgw-obj-fs", (char*)NULL)) {
+      rgw_obj_fs = val;
     } else if (strncmp(*i, "-", 1) == 0) {
       cerr << "ERROR: invalid flag " << *i << std::endl;
       return EINVAL;
@@ -6101,6 +6110,10 @@ int main(int argc, const char **argv)
   if (opt_cmd == OPT::BUCKET_RADOS_LIST) {
     RGWRadosList lister(store,
                        max_concurrent_ios, orphan_stale_secs, tenant);
+    if (rgw_obj_fs) {
+      lister.set_field_separator(*rgw_obj_fs);
+    }
+
     if (bucket_name.empty()) {
       ret = lister.run(dpp());
     } else {

diff --git a/src/rgw/rgw_orphan.cc b/src/rgw/rgw_orphan.cc
index 50574e4ac88972839f84ac56daf69506d448e437..62408fcc1141e4697e44dac0a5f89161f4447f82 100644 (file)
--- a/src/rgw/rgw_orphan.cc
+++ b/src/rgw/rgw_orphan.cc
@@ -925,6 +925,8 @@ int RGWOrphanSearch::finish()
 
 
 int RGWRadosList::handle_stat_result(RGWRados::Object::Stat::Result& result,
+                                    std::string& bucket_name,
+                                    rgw_obj_key& obj_key,
                                      std::set<string>& obj_oids)
 {
   obj_oids.clear();
@@ -949,6 +951,9 @@ int RGWRadosList::handle_stat_result(RGWRados::Object::Stat::Result& result,
     return 0;
   }
 
+  bucket_name = bucket.name;
+  obj_key = result.obj.key;
+
   if (!result.manifest) {
     /* a very very old object, or part of a multipart upload during upload */
     obj_oids.insert(oid);
@@ -1049,7 +1054,9 @@ int RGWRadosList::pop_and_handle_stat_op(
   RGWObjectCtx& obj_ctx,
   std::deque<RGWRados::Object::Stat>& ops)
 {
-  std::set<string> obj_oids;
+  std::string bucket_name;
+  rgw_obj_key obj_key;
+  std::set<std::string> obj_oids;
   RGWRados::Object::Stat& front_op = ops.front();
 
   int ret = front_op.wait();
@@ -1061,7 +1068,7 @@ int RGWRadosList::pop_and_handle_stat_op(
     goto done;
   }
 
-  ret = handle_stat_result(front_op.result, obj_oids);
+  ret = handle_stat_result(front_op.result, bucket_name, obj_key, obj_oids);
   if (ret < 0) {
     lderr(store->ctx()) << "ERROR: handle_stat_result() returned error: " <<
       cpp_strerror(-ret) << dendl;
@@ -1069,7 +1076,14 @@ int RGWRadosList::pop_and_handle_stat_op(
 
   // output results
   for (const auto& o : obj_oids) {
-    std::cout << o << std::endl;
+    if (include_rgw_obj_name) {
+      std::cout << o <<
+       field_separator << bucket_name <<
+       field_separator << obj_key <<
+       std::endl;
+    } else {
+      std::cout << o << std::endl;
+    }
   }
 
 done:
@@ -1343,6 +1357,7 @@ int RGWRadosList::run(const DoutPrefixProvider *dpp, const std::string& start_bu
 {
   RGWSysObjectCtx sys_obj_ctx = store->svc()->sysobj->init_obj_ctx();
   RGWObjectCtx obj_ctx(store);
+  RGWBucketInfo bucket_info;
   int ret;
 
   add_bucket_entire(start_bucket_name);
@@ -1419,10 +1434,13 @@ int RGWRadosList::run(const DoutPrefixProvider *dpp, const std::string& start_bu
     }
   } // while (! bucket_process_map.empty())
 
+  if (include_rgw_obj_name) {
+    goto done;
+  }
+
   // now handle incomplete multipart uploads by going back to the
   // initial bucket
 
-  RGWBucketInfo bucket_info;
   ret = store->getRados()->get_bucket_info(store->svc(),
                                           tenant_name,
                                           start_bucket_name,
@@ -1446,6 +1464,8 @@ int RGWRadosList::run(const DoutPrefixProvider *dpp, const std::string& start_bu
     return ret;
   }
 
+done:
+
   return 0;
 } // RGWRadosList::run(string)
 

diff --git a/src/rgw/rgw_orphan.h b/src/rgw/rgw_orphan.h
index fc4fc0ea16f5a881511e172a5cdaa16dd4c917e0..10e9f52390995c5dad95f538fe90637de39b53cc 100644 (file)
--- a/src/rgw/rgw_orphan.h
+++ b/src/rgw/rgw_orphan.h
@@ -13,8 +13,7 @@
  * 
  */
 
-#ifndef CEPH_RGW_ORPHAN_H
-#define CEPH_RGW_ORPHAN_H
+#pragma once
 
 #include "common/config.h"
 #include "common/Formatter.h"
@@ -258,7 +257,12 @@ class RGWRadosList {
   uint64_t stale_secs;
   std::string tenant_name;
 
+  bool include_rgw_obj_name;
+  std::string field_separator;
+
   int handle_stat_result(RGWRados::Object::Stat::Result& result,
+                        std::string& bucket_name,
+                        rgw_obj_key& obj_key,
                         std::set<string>& obj_oids);
   int pop_and_handle_stat_op(RGWObjectCtx& obj_ctx,
                             std::deque<RGWRados::Object::Stat>& ops);
@@ -272,7 +276,8 @@ public:
     store(_store),
     max_concurrent_ios(_max_ios),
     stale_secs(_stale_secs),
-    tenant_name(_tenant_name)
+    tenant_name(_tenant_name),
+    include_rgw_obj_name(false)
   {}
 
   int process_bucket(const DoutPrefixProvider *dpp, 
@@ -288,6 +293,11 @@ public:
 
   int run(const DoutPrefixProvider *dpp, const std::string& bucket_id);
   int run(const DoutPrefixProvider *dpp);
-}; // class RGWRadosList
 
-#endif
+  // if there's a non-empty field separator, that means we'll display
+  // bucket and object names
+  void set_field_separator(const std::string& fs) {
+    field_separator = fs;
+    include_rgw_obj_name = !field_separator.empty();
+  }
+}; // class RGWRadosList

diff --git a/src/test/cli/radosgw-admin/help.t b/src/test/cli/radosgw-admin/help.t
index 86e6110d0bb720fb3d9dbc9b733893474c21b857..490499f24a0296ffa641a00c1a38e49037bcb5cf 100644 (file)
--- a/src/test/cli/radosgw-admin/help.t
+++ b/src/test/cli/radosgw-admin/help.t
@@ -340,6 +340,12 @@
      --package                 name of the lua package that should be added/removed to/from the allowlist
      --allow-compilation       package is allowed to compile C code as part of its installation
   
+  radoslist options:
+     --rgw-obj-fs              the field separator that will separate the rados
+                               object name from the rgw object name;
+                               additionally rados objects for incomplete
+                               multipart uploads will not be output
+  
     --conf/-c FILE    read configuration from the given configuration file
     --id ID           set ID portion of my name
     --name/-n TYPE.ID set name