common, tools, test: Add "rados purge" feature to remove all objects from a pool

This required creating an Object type which is a pair of strings an
object id and object namespace.  Functionally, nothing has changed
with regards to the bench and cleanup command semantics.  Those
commands still allow operation in the default or a specified namespace.

Fixes: #12262
Signed-off-by: David Zafman <dzafman@redhat.com>

diff --git a/src/common/obj_bencher.cc b/src/common/obj_bencher.cc
index f729a10eda7e812d8987a42686bbd49a63116469..d53681367965e58cdc2384fbdcef15a093b57156 100644 (file)
--- a/src/common/obj_bencher.cc
+++ b/src/common/obj_bencher.cc
@@ -1075,8 +1075,8 @@ int ObjBencher::clean_up(int num_objects, int prevPid, int concurrentios) {
  * @returns true if there are any objects in the store which match
  * the prefix, false if there are no more
  */
-bool ObjBencher::more_objects_matching_prefix(const std::string& prefix, std::list<std::string>* objects) {
-  std::list<std::string> unfiltered_objects;
+bool ObjBencher::more_objects_matching_prefix(const std::string& prefix, std::list<Object>* objects) {
+  std::list<Object> unfiltered_objects;
 
   objects->clear();
 
@@ -1085,12 +1085,10 @@ bool ObjBencher::more_objects_matching_prefix(const std::string& prefix, std::li
     if (!objects_remain)
       return false;
 
-    std::list<std::string>::const_iterator i = unfiltered_objects.begin();
+    std::list<Object>::const_iterator i = unfiltered_objects.begin();
     for ( ; i != unfiltered_objects.end(); ++i) {
-      const std::string& next = *i;
-
-      if (next.substr(0, prefix.length()) == prefix) {
-        objects->push_back(next);
+      if (i->first.substr(0, prefix.length()) == prefix) {
+        objects->push_back(*i);
       }
     }
   }
@@ -1104,12 +1102,12 @@ int ObjBencher::clean_up_slow(const std::string& prefix, int concurrentios) {
   if (concurrentios <= 0) 
     return -EINVAL;
 
-  std::vector<string> name(concurrentios);
-  std::string newName;
+  std::vector<Object> name(concurrentios);
+  Object newName;
   int r = 0;
   utime_t runtime;
   int slot = 0;
-  std::list<std::string> objects;
+  std::list<Object> objects;
   bool objects_remain = true;
 
   lock.Lock();
@@ -1144,7 +1142,8 @@ int ObjBencher::clean_up_slow(const std::string& prefix, int concurrentios) {
   //start initial removes
   for (int i = 0; i < concurrentios; ++i) {
     create_completion(i, _aio_cb, (void *)&lc);
-    r = aio_remove(name[i], i);
+    set_namespace(name[i].second);
+    r = aio_remove(name[i].first, i);
     if (r < 0) { //naughty, doesn't clean up heap
       cerr << "r = " << r << std::endl;
       goto ERR;
@@ -1206,7 +1205,8 @@ int ObjBencher::clean_up_slow(const std::string& prefix, int concurrentios) {
 
     //start new remove and check data if requested
     create_completion(slot, _aio_cb, (void *)&lc);
-    r = aio_remove(newName, slot);
+    set_namespace(newName.second);
+    r = aio_remove(newName.first, slot);
     if (r < 0) {
       goto ERR;
     }

diff --git a/src/common/obj_bencher.h b/src/common/obj_bencher.h
index 9ba63ed74a265243440572f0776f88cd7a1f79d7..832461bb99d233df8b9baa287f1b455eb1dfccac 100644 (file)
--- a/src/common/obj_bencher.h
+++ b/src/common/obj_bencher.h
@@ -53,6 +53,9 @@ const int OP_WRITE     = 1;
 const int OP_SEQ_READ  = 2;
 const int OP_RAND_READ = 3;
 
+// Object is composed of <oid,namespace>
+typedef std::pair<std::string, std::string> Object;
+
 class ObjBencher {
   bool show_time;
 public:
@@ -71,8 +74,7 @@ protected:
   int rand_read_bench(int secondsToRun, int num_objects, int concurrentios, int writePid, bool no_verify=false);
 
   int clean_up(int num_objects, int prevPid, int concurrentios);
-  int clean_up_slow(const std::string& prefix, int concurrentios);
-  bool more_objects_matching_prefix(const std::string& prefix, std::list<std::string>* name);
+  bool more_objects_matching_prefix(const std::string& prefix, std::list<Object>* name);
 
   virtual int completions_init(int concurrentios) = 0;
   virtual void completions_done() = 0;
@@ -91,7 +93,8 @@ protected:
   virtual int sync_write(const std::string& oid, bufferlist& bl, size_t len) = 0;
   virtual int sync_remove(const std::string& oid) = 0;
 
-  virtual bool get_objects(std::list<std::string>* objects, int num) = 0;
+  virtual bool get_objects(std::list< std::pair<std::string, std::string> >* objects, int num) = 0;
+  virtual void set_namespace(const std::string&) {}
 
   ostream& out(ostream& os);
   ostream& out(ostream& os, utime_t& t);
@@ -106,6 +109,7 @@ public:
   void set_show_time(bool dt) {
     show_time = dt;
   }
+  int clean_up_slow(const std::string& prefix, int concurrentios);
 };
 
 

diff --git a/src/test/test_rados_tool.sh b/src/test/test_rados_tool.sh
index fc79dd0fe8fd1926ac8b21b26ac0a37022bab648..08756ed98e7ceca0614523d56dc16cf7539c01d4 100755 (executable)
--- a/src/test/test_rados_tool.sh
+++ b/src/test/test_rados_tool.sh
@@ -98,6 +98,13 @@ run_expect_succ "$RADOS_TOOL" -p "$POOL" --create import "$TDIR/expa"
 # inaccessible import src should fail
 run_expect_fail "$RADOS_TOOL" -p "$POOL" import "$TDIR/dir_nonexistent"
 
+# export an empty pool to test purge
+run_expect_succ "$RADOS_TOOL" purge "$POOL" --yes-i-really-really-mean-it
+run_expect_succ "$RADOS_TOOL" -p "$POOL" export "$TDIR/empty"
+cmp -s "$TDIR/expb" "$TDIR/empty" \
+    || die "failed to export the same stuff we imported!"
+rm -f "$TDIR/empty"
+
 # import some stuff with extended attributes on it
 run_expect_succ "$RADOS_TOOL" -p "$POOL" import "$TDIR/expc"
 VAL=`"$RADOS_TOOL" -p "$POOL" getxattr foo "rados.toothbrush"`

diff --git a/src/tools/rados/rados.cc b/src/tools/rados/rados.cc
index 165f7343cb0d43683dfe6d4f794511bdf2ffbe4e..40850cc9a575e5955bb3e6d489bc0bcfafb98dab 100644 (file)
--- a/src/tools/rados/rados.cc
+++ b/src/tools/rados/rados.cc
@@ -68,6 +68,8 @@ void usage(ostream& out)
 "   cppool <pool-name> <dest-pool>   copy content of a pool\n"
 "   rmpool <pool-name> [<pool-name> --yes-i-really-really-mean-it]\n"
 "                                    remove pool <pool-name>'\n"
+"   purge <pool-name> --yes-i-really-really-mean-it\n"
+"                                    remove all objects from pool <pool-name> without removing it\n"
 "   df                               show per-pool and total usage\n"
 "   ls                               list objects in pool\n\n"
 "   chown 123                        change the pool owner to auid 123\n"
@@ -882,7 +884,7 @@ protected:
     return completions[slot]->get_return_value();
   }
 
-  bool get_objects(std::list<std::string>* objects, int num) {
+  bool get_objects(std::list<Object>* objects, int num) {
     int count = 0;
 
     if (!iterator_valid) {
@@ -899,13 +901,18 @@ protected:
 
     objects->clear();
     for ( ; oi != ei && count < num; ++oi) {
-      objects->push_back(oi->get_oid());
+      Object obj(oi->get_oid(), oi->get_nspace());
+      objects->push_back(obj);
       ++count;
     }
 
     return true;
   }
 
+  void set_namespace( const std::string& ns) {
+    io_ctx.set_namespace(ns);
+  }
+
 public:
   RadosBencher(CephContext *cct_, librados::Rados& _r, librados::IoCtx& _i)
     : ObjBencher(cct_), completions(NULL), rados(_r), io_ctx(_i), iterator_valid(false) {}
@@ -2273,6 +2280,32 @@ static int rados_tool_common(const std::map < std::string, std::string > &opts,
       cerr << "pool " << nargs[1] << " could not be removed" << std::endl;
     }
   }
+  else if (strcmp(nargs[0], "purge") == 0) {
+    if (nargs.size() < 2)
+      usage_exit();
+    if (nargs.size() < 3 ||
+       strcmp(nargs[2], "--yes-i-really-really-mean-it") != 0) {
+      cerr << "WARNING:\n"
+          << "  This will PERMANENTLY DESTROY all objects from a pool with no way back.\n"
+          << "  To confirm, follow pool with --yes-i-really-really-mean-it" << std::endl;
+      ret = -1;
+      goto out;
+    }
+    ret = rados.ioctx_create(nargs[1], io_ctx);
+    if (ret < 0) {
+      cerr << "error pool " << nargs[1] << ": "
+          << cpp_strerror(ret) << std::endl;
+      goto out;
+    }
+    io_ctx.set_namespace(all_nspaces);
+    RadosBencher bencher(g_ceph_context, rados, io_ctx);
+    ret = bencher.clean_up_slow("", concurrent_ios);
+    if (ret >= 0) {
+      cout << "successfully purged pool " << nargs[1] << std::endl;
+    } else { //error
+      cerr << "pool " << nargs[1] << " could not be purged" << std::endl;
+    }
+  }
   else if (strcmp(nargs[0], "lssnap") == 0) {
     if (!pool_name || nargs.size() != 1)
       usage_exit();

diff --git a/src/tools/rest_bench.cc b/src/tools/rest_bench.cc
index 159f91cbd5672e8e0539b7164a457163a2b99160..010beee87efb690eb9239c41e0e9ea2c834518e1 100644 (file)
--- a/src/tools/rest_bench.cc
+++ b/src/tools/rest_bench.cc
@@ -90,7 +90,7 @@ struct req_context : public RefCountedObject {
   uint64_t off;
   uint64_t len;
   const char *list_start;
-  std::list<std::string>* list_objects;
+  std::list<Object>* list_objects;
   int list_count;
   string oid;
   Mutex lock;
@@ -201,7 +201,8 @@ static S3Status list_bucket_callback(int is_truncated, const char *next_marker,
   ctx->list_start = next_marker;
 
   for (int i = 0; i < count; ++i) {
-    ctx->list_objects->push_back(objects[i].key);
+    // RGW doesn't support namespaces yet
+    ctx->list_objects->push_back(Object(objects[i].key, ""));
   }
 
   return S3StatusOK;
@@ -532,7 +533,7 @@ protected:
     return ret;
   }
 
-  bool get_objects(std::list<std::string>* objects, int num) {
+  bool get_objects(std::list<Object>* objects, int num) {
     if (bucket_list_done) {
       bucket_list_done = false;
       return false;