From: David Zafman Date: Tue, 14 Jul 2015 02:07:07 +0000 (-0700) Subject: common, tools, test: Add "rados purge" feature to remove all objects from a pool X-Git-Tag: v9.1.0~523^2~1 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=90fdbbfbe623407057a5c1e0cf13ae318a0ed626;p=ceph.git common, tools, test: Add "rados purge" feature to remove all objects from a pool This required creating an Object type which is a pair of strings an object id and object namespace. Functionally, nothing has changed with regards to the bench and cleanup command semantics. Those commands still allow operation in the default or a specified namespace. Fixes: #12262 Signed-off-by: David Zafman --- diff --git a/src/common/obj_bencher.cc b/src/common/obj_bencher.cc index f729a10eda7e..d53681367965 100644 --- a/src/common/obj_bencher.cc +++ b/src/common/obj_bencher.cc @@ -1075,8 +1075,8 @@ int ObjBencher::clean_up(int num_objects, int prevPid, int concurrentios) { * @returns true if there are any objects in the store which match * the prefix, false if there are no more */ -bool ObjBencher::more_objects_matching_prefix(const std::string& prefix, std::list* objects) { - std::list unfiltered_objects; +bool ObjBencher::more_objects_matching_prefix(const std::string& prefix, std::list* objects) { + std::list unfiltered_objects; objects->clear(); @@ -1085,12 +1085,10 @@ bool ObjBencher::more_objects_matching_prefix(const std::string& prefix, std::li if (!objects_remain) return false; - std::list::const_iterator i = unfiltered_objects.begin(); + std::list::const_iterator i = unfiltered_objects.begin(); for ( ; i != unfiltered_objects.end(); ++i) { - const std::string& next = *i; - - if (next.substr(0, prefix.length()) == prefix) { - objects->push_back(next); + if (i->first.substr(0, prefix.length()) == prefix) { + objects->push_back(*i); } } } @@ -1104,12 +1102,12 @@ int ObjBencher::clean_up_slow(const std::string& prefix, int concurrentios) { if (concurrentios <= 0) return -EINVAL; - std::vector name(concurrentios); - std::string newName; + std::vector name(concurrentios); + Object newName; int r = 0; utime_t runtime; int slot = 0; - std::list objects; + std::list objects; bool objects_remain = true; lock.Lock(); @@ -1144,7 +1142,8 @@ int ObjBencher::clean_up_slow(const std::string& prefix, int concurrentios) { //start initial removes for (int i = 0; i < concurrentios; ++i) { create_completion(i, _aio_cb, (void *)&lc); - r = aio_remove(name[i], i); + set_namespace(name[i].second); + r = aio_remove(name[i].first, i); if (r < 0) { //naughty, doesn't clean up heap cerr << "r = " << r << std::endl; goto ERR; @@ -1206,7 +1205,8 @@ int ObjBencher::clean_up_slow(const std::string& prefix, int concurrentios) { //start new remove and check data if requested create_completion(slot, _aio_cb, (void *)&lc); - r = aio_remove(newName, slot); + set_namespace(newName.second); + r = aio_remove(newName.first, slot); if (r < 0) { goto ERR; } diff --git a/src/common/obj_bencher.h b/src/common/obj_bencher.h index 9ba63ed74a26..832461bb99d2 100644 --- a/src/common/obj_bencher.h +++ b/src/common/obj_bencher.h @@ -53,6 +53,9 @@ const int OP_WRITE = 1; const int OP_SEQ_READ = 2; const int OP_RAND_READ = 3; +// Object is composed of +typedef std::pair Object; + class ObjBencher { bool show_time; public: @@ -71,8 +74,7 @@ protected: int rand_read_bench(int secondsToRun, int num_objects, int concurrentios, int writePid, bool no_verify=false); int clean_up(int num_objects, int prevPid, int concurrentios); - int clean_up_slow(const std::string& prefix, int concurrentios); - bool more_objects_matching_prefix(const std::string& prefix, std::list* name); + bool more_objects_matching_prefix(const std::string& prefix, std::list* name); virtual int completions_init(int concurrentios) = 0; virtual void completions_done() = 0; @@ -91,7 +93,8 @@ protected: virtual int sync_write(const std::string& oid, bufferlist& bl, size_t len) = 0; virtual int sync_remove(const std::string& oid) = 0; - virtual bool get_objects(std::list* objects, int num) = 0; + virtual bool get_objects(std::list< std::pair >* objects, int num) = 0; + virtual void set_namespace(const std::string&) {} ostream& out(ostream& os); ostream& out(ostream& os, utime_t& t); @@ -106,6 +109,7 @@ public: void set_show_time(bool dt) { show_time = dt; } + int clean_up_slow(const std::string& prefix, int concurrentios); }; diff --git a/src/test/test_rados_tool.sh b/src/test/test_rados_tool.sh index fc79dd0fe8fd..08756ed98e7c 100755 --- a/src/test/test_rados_tool.sh +++ b/src/test/test_rados_tool.sh @@ -98,6 +98,13 @@ run_expect_succ "$RADOS_TOOL" -p "$POOL" --create import "$TDIR/expa" # inaccessible import src should fail run_expect_fail "$RADOS_TOOL" -p "$POOL" import "$TDIR/dir_nonexistent" +# export an empty pool to test purge +run_expect_succ "$RADOS_TOOL" purge "$POOL" --yes-i-really-really-mean-it +run_expect_succ "$RADOS_TOOL" -p "$POOL" export "$TDIR/empty" +cmp -s "$TDIR/expb" "$TDIR/empty" \ + || die "failed to export the same stuff we imported!" +rm -f "$TDIR/empty" + # import some stuff with extended attributes on it run_expect_succ "$RADOS_TOOL" -p "$POOL" import "$TDIR/expc" VAL=`"$RADOS_TOOL" -p "$POOL" getxattr foo "rados.toothbrush"` diff --git a/src/tools/rados/rados.cc b/src/tools/rados/rados.cc index 165f7343cb0d..40850cc9a575 100644 --- a/src/tools/rados/rados.cc +++ b/src/tools/rados/rados.cc @@ -68,6 +68,8 @@ void usage(ostream& out) " cppool copy content of a pool\n" " rmpool [ --yes-i-really-really-mean-it]\n" " remove pool '\n" +" purge --yes-i-really-really-mean-it\n" +" remove all objects from pool without removing it\n" " df show per-pool and total usage\n" " ls list objects in pool\n\n" " chown 123 change the pool owner to auid 123\n" @@ -882,7 +884,7 @@ protected: return completions[slot]->get_return_value(); } - bool get_objects(std::list* objects, int num) { + bool get_objects(std::list* objects, int num) { int count = 0; if (!iterator_valid) { @@ -899,13 +901,18 @@ protected: objects->clear(); for ( ; oi != ei && count < num; ++oi) { - objects->push_back(oi->get_oid()); + Object obj(oi->get_oid(), oi->get_nspace()); + objects->push_back(obj); ++count; } return true; } + void set_namespace( const std::string& ns) { + io_ctx.set_namespace(ns); + } + public: RadosBencher(CephContext *cct_, librados::Rados& _r, librados::IoCtx& _i) : ObjBencher(cct_), completions(NULL), rados(_r), io_ctx(_i), iterator_valid(false) {} @@ -2273,6 +2280,32 @@ static int rados_tool_common(const std::map < std::string, std::string > &opts, cerr << "pool " << nargs[1] << " could not be removed" << std::endl; } } + else if (strcmp(nargs[0], "purge") == 0) { + if (nargs.size() < 2) + usage_exit(); + if (nargs.size() < 3 || + strcmp(nargs[2], "--yes-i-really-really-mean-it") != 0) { + cerr << "WARNING:\n" + << " This will PERMANENTLY DESTROY all objects from a pool with no way back.\n" + << " To confirm, follow pool with --yes-i-really-really-mean-it" << std::endl; + ret = -1; + goto out; + } + ret = rados.ioctx_create(nargs[1], io_ctx); + if (ret < 0) { + cerr << "error pool " << nargs[1] << ": " + << cpp_strerror(ret) << std::endl; + goto out; + } + io_ctx.set_namespace(all_nspaces); + RadosBencher bencher(g_ceph_context, rados, io_ctx); + ret = bencher.clean_up_slow("", concurrent_ios); + if (ret >= 0) { + cout << "successfully purged pool " << nargs[1] << std::endl; + } else { //error + cerr << "pool " << nargs[1] << " could not be purged" << std::endl; + } + } else if (strcmp(nargs[0], "lssnap") == 0) { if (!pool_name || nargs.size() != 1) usage_exit(); diff --git a/src/tools/rest_bench.cc b/src/tools/rest_bench.cc index 159f91cbd567..010beee87efb 100644 --- a/src/tools/rest_bench.cc +++ b/src/tools/rest_bench.cc @@ -90,7 +90,7 @@ struct req_context : public RefCountedObject { uint64_t off; uint64_t len; const char *list_start; - std::list* list_objects; + std::list* list_objects; int list_count; string oid; Mutex lock; @@ -201,7 +201,8 @@ static S3Status list_bucket_callback(int is_truncated, const char *next_marker, ctx->list_start = next_marker; for (int i = 0; i < count; ++i) { - ctx->list_objects->push_back(objects[i].key); + // RGW doesn't support namespaces yet + ctx->list_objects->push_back(Object(objects[i].key, "")); } return S3StatusOK; @@ -532,7 +533,7 @@ protected: return ret; } - bool get_objects(std::list* objects, int num) { + bool get_objects(std::list* objects, int num) { if (bucket_list_done) { bucket_list_done = false; return false;