cerr << " replicalog get get replica metadata log entry\n";
cerr << " replicalog update update replica metadata log entry\n";
cerr << " replicalog delete delete replica metadata log entry\n";
- cout << " orphans find init and run search for leaked rados objects\n";
- cout << " orphans finish clean up search for leaked rados objects\n";
+ cerr << " orphans find init and run search for leaked rados objects\n";
+ cerr << " orphans finish clean up search for leaked rados objects\n";
cerr << "options:\n";
cerr << " --uid=<id> user id\n";
cerr << " --subuser=<name> subuser name\n";
cerr << " --categories=<list> comma separated list of categories, used in usage show\n";
cerr << " --caps=<caps> list of caps (e.g., \"usage=read, write; user=read\"\n";
cerr << " --yes-i-really-mean-it required for certain operations\n";
- cerr << " --reset-regions reset regionmap when regionmap update";
+ cerr << " --reset-regions reset regionmap when regionmap update\n";
+ cerr << " --bypass-gc when specified with bucket deletion, triggers\n";
+ cerr << " object deletions by not involving GC\n";
+ cerr << " --inconsistent-index when specified with bucket deletion and bypass-gc set to true,\n";
+ cerr << " ignores bucket index consistency\n";
cerr << "\n";
cerr << "<date> := \"YYYY-MM-DD[ hh:mm:ss]\"\n";
cerr << "\nQuota options:\n";
cerr << " --max-objects specify max objects (negative value to disable)\n";
cerr << " --max-size specify max size (in bytes, negative value to disable)\n";
cerr << " --quota-scope scope of quota (bucket, user)\n";
- cout << "\nOrphans search options:\n";
- cout << " --pool data pool to scan for leaked rados objects in\n";
- cout << " --num-shards num of shards to use for keeping the temporary scan info\n";
+ cerr << "\nOrphans search options:\n";
+ cerr << " --pool data pool to scan for leaked rados objects in\n";
+ cerr << " --num-shards num of shards to use for keeping the temporary scan info\n";
cerr << "\n";
generic_client_usage();
}
int max_concurrent_ios = 32;
uint64_t orphan_stale_secs = (24 * 3600);
+ int bypass_gc = false;
+ int inconsistent_index = false;
+
std::string val;
std::ostringstream errs;
string err;
// do nothing
} else if (ceph_argparse_binary_flag(args, i, &reset_regions, NULL, "--reset-regions", (char*)NULL)) {
// do nothing
+ } else if (ceph_argparse_binary_flag(args, i, &bypass_gc, NULL, "--bypass-gc", (char*)NULL)) {
+ // do nothing
+ } else if (ceph_argparse_binary_flag(args, i, &inconsistent_index, NULL, "--inconsistent-index", (char*)NULL)) {
+ // do nothing
} else if (ceph_argparse_witharg(args, i, &val, "--caps", (char*)NULL)) {
caps = val;
} else if (ceph_argparse_witharg(args, i, &val, "-i", "--infile", (char*)NULL)) {
bucket_op.set_check_objects(check_objects);
bucket_op.set_delete_children(delete_child_objects);
bucket_op.set_fix_index(fix);
+ bucket_op.set_max_aio(max_concurrent_ios);
// required to gather errors from operations
std::string err_msg;
}
if (opt_cmd == OPT_BUCKET_RM) {
- RGWBucketAdminOp::remove_bucket(store, bucket_op);
+ if (inconsistent_index == false) {
+ RGWBucketAdminOp::remove_bucket(store, bucket_op, bypass_gc, true);
+ } else {
+ RGWBucketAdminOp::remove_bucket(store, bucket_op, bypass_gc, false);
+ }
}
if (opt_cmd == OPT_GC_LIST) {
#include "rgw_user.h"
#include "rgw_string.h"
+#include "include/rados/librados.hpp"
// until everything is moved from rgw_common
#include "rgw_common.h"
map<RGWObjCategory, RGWStorageStats> stats;
std::vector<RGWObjEnt> objs;
map<string, bool> common_prefixes;
- rgw_obj obj;
RGWBucketInfo info;
- bufferlist bl;
RGWObjectCtx obj_ctx(store);
string bucket_ver, master_ver;
if (ret < 0)
return ret;
- obj.bucket = bucket;
-
ret = store->get_bucket_info(obj_ctx, bucket.name, info, NULL);
if (ret < 0)
return ret;
while (!objs.empty()) {
std::vector<RGWObjEnt>::iterator it = objs.begin();
- for (it = objs.begin(); it != objs.end(); ++it) {
+ for (; it != objs.end(); ++it) {
ret = rgw_remove_object(store, info, bucket, (*it).key);
if (ret < 0)
return ret;
return ret;
}
+static int aio_wait(librados::AioCompletion *handle)
+{
+ librados::AioCompletion *c = (librados::AioCompletion *)handle;
+ c->wait_for_complete();
+ int ret = c->get_return_value();
+ c->release();
+ return ret;
+}
+
+static int drain_handles(list<librados::AioCompletion *>& pending)
+{
+ int ret = 0;
+ while (!pending.empty()) {
+ librados::AioCompletion *handle = pending.front();
+ pending.pop_front();
+ int r = aio_wait(handle);
+ if (r < 0) {
+ ret = r;
+ }
+ }
+ return ret;
+}
+
+int rgw_remove_bucket_bypass_gc(RGWRados *store, rgw_bucket& bucket,
+ int concurrent_max, bool keep_index_consistent)
+{
+ int ret;
+ map<RGWObjCategory, RGWStorageStats> stats;
+ std::vector<RGWObjEnt> objs;
+ map<string, bool> common_prefixes;
+ RGWBucketInfo info;
+ RGWObjectCtx obj_ctx(store);
+
+ string bucket_ver, master_ver;
+
+ ret = store->get_bucket_stats(bucket, &bucket_ver, &master_ver, stats, NULL);
+ if (ret < 0)
+ return ret;
+
+ ret = store->get_bucket_info(obj_ctx, bucket.name, info, NULL);
+ if (ret < 0)
+ return ret;
+
+
+ RGWRados::Bucket target(store, info.bucket);
+ RGWRados::Bucket::List list_op(&target);
+
+ list_op.params.list_versions = true;
+
+ std::list<librados::AioCompletion*> handles;
+
+ int max = 1000;
+ int max_aio = concurrent_max;
+ ret = list_op.list_objects(max, &objs, &common_prefixes, NULL);
+ if (ret < 0)
+ return ret;
+
+ while (!objs.empty()) {
+ std::vector<RGWObjEnt>::iterator it = objs.begin();
+ for (; it != objs.end(); ++it) {
+ RGWObjState *astate = NULL;
+ rgw_obj obj(bucket, (*it).key.name);
+ obj.set_instance((*it).key.instance);
+
+ ret = store->get_obj_state(&obj_ctx, obj, &astate, NULL);
+ if (ret == -ENOENT) {
+ dout(1) << "WARNING: cannot find obj state for obj " << obj.get_object() << dendl;
+ continue;
+ }
+ if (ret < 0) {
+ lderr(store->ctx()) << "ERROR: get obj state returned with error " << ret << dendl;
+ return ret;
+ }
+
+ if (astate->has_manifest) {
+ rgw_obj head_obj;
+ RGWObjManifest& manifest = astate->manifest;
+ RGWObjManifest::obj_iterator miter = manifest.obj_begin();
+
+ if (miter.get_location().ns.empty()) {
+ head_obj = miter.get_location();
+ }
+
+ for (; miter != manifest.obj_end() && max_aio--; ++miter) {
+ if (!max_aio) {
+ ret = drain_handles(handles);
+ if (ret < 0) {
+ lderr(store->ctx()) << "ERROR: could not drain handles as aio completion returned with " << ret << dendl;
+ return ret;
+ }
+ max_aio = concurrent_max;
+ }
+
+ rgw_obj last_obj = miter.get_location();
+ if (last_obj == head_obj) {
+ // have the head obj deleted at the end
+ continue;
+ }
+
+ ret = store->delete_obj_aio(last_obj, bucket, info, astate, handles, keep_index_consistent);
+ if (ret < 0) {
+ lderr(store->ctx()) << "ERROR: delete obj aio failed with " << ret << dendl;
+ return ret;
+ }
+ } // for all shadow objs
+
+ ret = store->delete_obj_aio(head_obj, bucket, info, astate, handles, keep_index_consistent);
+ if (ret < 0) {
+ lderr(store->ctx()) << "ERROR: delete obj aio failed with " << ret << dendl;
+ return ret;
+ }
+ }
+
+ if (!max_aio) {
+ ret = drain_handles(handles);
+ if (ret < 0) {
+ lderr(store->ctx()) << "ERROR: could not drain handles as aio completion returned with " << ret << dendl;
+ return ret;
+ }
+ max_aio = concurrent_max;
+ }
+ } // for all RGW objects
+ objs.clear();
+
+ ret = list_op.list_objects(max, &objs, &common_prefixes, NULL);
+ if (ret < 0)
+ return ret;
+ }
+
+ ret = drain_handles(handles);
+ if (ret < 0) {
+ lderr(store->ctx()) << "ERROR: could not drain handles as aio completion returned with " << ret << dendl;
+ return ret;
+ }
+
+ ret = rgw_bucket_sync_user_stats(store, bucket.name);
+ if (ret < 0) {
+ dout(1) << "WARNING: failed sync user stats before bucket delete. ret=" << ret << dendl;
+ }
+
+ RGWObjVersionTracker objv_tracker;
+
+ ret = rgw_bucket_delete_bucket_obj(store, bucket.name, objv_tracker);
+ if (ret < 0) {
+ lderr(store->ctx()) << "ERROR: could not remove bucket " << bucket.name << "with ret as " << ret << dendl;
+ return ret;
+ }
+
+ if (!store->is_syncing_bucket_meta(bucket)) {
+ RGWObjVersionTracker objv_tracker;
+ string entry;
+ store->get_bucket_instance_entry(bucket, entry);
+ ret = rgw_bucket_instance_remove_entry(store, entry, &objv_tracker);
+ if (ret < 0) {
+ lderr(store->ctx()) << "ERROR: could not remove bucket instance entry" << bucket.name << "with ret as " << ret << dendl;
+ return ret;
+ }
+ }
+
+ ret = rgw_unlink_bucket(store, info.owner, bucket.name, false);
+ if (ret < 0) {
+ lderr(store->ctx()) << "ERROR: unable to remove user bucket information" << dendl;
+ }
+
+ return ret;
+}
+
int rgw_bucket_delete_bucket_obj(RGWRados *store, string& bucket_name, RGWObjVersionTracker& objv_tracker)
{
return store->meta_mgr->remove_entry(bucket_meta_handler, bucket_name, &objv_tracker);
return r;
}
-int RGWBucket::remove(RGWBucketAdminOpState& op_state, std::string *err_msg)
+int RGWBucket::remove(RGWBucketAdminOpState& op_state, bool bypass_gc,
+ bool keep_index_consistent, std::string *err_msg)
{
bool delete_children = op_state.will_delete_children();
rgw_bucket bucket = op_state.get_bucket();
+ int ret;
+
+ if (bypass_gc) {
+ if (delete_children) {
+ ret = rgw_remove_bucket_bypass_gc(store, bucket, op_state.get_max_aio(), keep_index_consistent);
+ } else {
+ set_err_msg(err_msg, "purge objects should be set for gc to be bypassed");
+ return -EINVAL;
+ }
+ } else {
+ ret = rgw_remove_bucket(store, bucket_info.owner, bucket, delete_children);
+ }
- int ret = rgw_remove_bucket(store, bucket_info.owner, bucket, delete_children);
if (ret < 0) {
set_err_msg(err_msg, "unable to remove bucket" + cpp_strerror(-ret));
return ret;
return 0;
}
-int RGWBucketAdminOp::remove_bucket(RGWRados *store, RGWBucketAdminOpState& op_state)
+int RGWBucketAdminOp::remove_bucket(RGWRados *store, RGWBucketAdminOpState& op_state,
+ bool bypass_gc, bool keep_index_consistent)
{
RGWBucket bucket;
if (ret < 0)
return ret;
- return bucket.remove(op_state);
+ return bucket.remove(op_state, bypass_gc, keep_index_consistent);
}
int RGWBucketAdminOp::remove_object(RGWRados *store, RGWBucketAdminOpState& op_state)
extern int rgw_remove_object(RGWRados *store, RGWBucketInfo& bucket_info, rgw_bucket& bucket, rgw_obj_key& key);
extern int rgw_remove_bucket(RGWRados *store, const string& bucket_owner, rgw_bucket& bucket, bool delete_children);
+extern int rgw_remove_bucket_bypass_gc(RGWRados *store, rgw_bucket& bucket, int concurrent_max);
extern int rgw_bucket_set_attrs(RGWRados *store, RGWBucketInfo& bucket_info,
map<string, bufferlist>& attrs,
bool fix_index;
bool delete_child_objects;
bool bucket_stored;
+ int max_aio;
rgw_bucket bucket;
void set_fix_index(bool value) { fix_index = value; }
void set_delete_children(bool value) { delete_child_objects = value; }
+ void set_max_aio(int value) { max_aio = value; }
+
void set_user_id(std::string& user_id) {
if (!user_id.empty())
uid = user_id;
bool is_user_op() { return !uid.empty(); }
bool is_system_op() { return uid.empty(); }
bool has_bucket_stored() { return bucket_stored; }
+ int get_max_aio() { return max_aio; }
RGWBucketAdminOpState() : list_buckets(false), stat_buckets(false), check_objects(false),
fix_index(false), delete_child_objects(false),
map<RGWObjCategory, RGWStorageStats>& calculated_stats,
std::string *err_msg = NULL);
- int remove(RGWBucketAdminOpState& op_state, std::string *err_msg = NULL);
+ int remove(RGWBucketAdminOpState& op_state, bool bypass_gc = false, bool keep_index_consistent = true, std::string *err_msg = NULL);
int link(RGWBucketAdminOpState& op_state, std::string *err_msg = NULL);
int unlink(RGWBucketAdminOpState& op_state, std::string *err_msg = NULL);
static int check_index(RGWRados *store, RGWBucketAdminOpState& op_state,
RGWFormatterFlusher& flusher);
- static int remove_bucket(RGWRados *store, RGWBucketAdminOpState& op_state);
+ static int remove_bucket(RGWRados *store, RGWBucketAdminOpState& op_state, bool bypass_gc = false, bool keep_index_consistent = true);
static int remove_object(RGWRados *store, RGWBucketAdminOpState& op_state);
static int info(RGWRados *store, RGWBucketAdminOpState& op_state, RGWFormatterFlusher& flusher);
};
}
}
+int RGWRados::delete_obj_aio(rgw_obj& obj, rgw_bucket& bucket,
+ RGWBucketInfo& bucket_info, RGWObjState *astate,
+ list<librados::AioCompletion *>& handles, bool keep_index_consistent)
+{
+ rgw_rados_ref ref;
+ int ret = get_obj_ref(obj, &ref, &bucket);
+ if (ret < 0) {
+ lderr(cct) << "ERROR: failed to get obj ref with ret=" << ret << dendl;
+ return ret;
+ }
+
+ if (keep_index_consistent) {
+ RGWRados::Bucket bop(this, bucket_info.bucket);
+ RGWRados::Bucket::UpdateIndex index_op(&bop, obj, astate);
+
+ ret = index_op.prepare(CLS_RGW_OP_DEL);
+ if (ret < 0) {
+ lderr(cct) << "ERROR: failed to prepare index op with ret=" << ret << dendl;
+ return ret;
+ }
+ }
+
+ ObjectWriteOperation op;
+ list<string> prefixes;
+ cls_rgw_remove_obj(op, prefixes);
+
+ AioCompletion *c = librados::Rados::aio_create_completion(NULL, NULL, NULL);
+ ret = ref.ioctx.aio_operate(ref.oid, c, &op);
+ if (ret < 0) {
+ lderr(cct) << "ERROR: AioOperate failed with ret=" << ret << dendl;
+ return ret;
+ }
+
+ handles.push_back(c);
+
+ if (keep_index_consistent) {
+ ret = delete_obj_index(obj);
+ if (ret < 0) {
+ lderr(cct) << "ERROR: failed to delete obj index with ret=" << ret << dendl;
+ return ret;
+ }
+ }
+ return ret;
+}
+
librados::Rados* get_rados_handle();
+ int delete_obj_aio(rgw_obj& obj, rgw_bucket& bucket, RGWBucketInfo& info, RGWObjState *astate,
+ list<librados::AioCompletion *>& handles, bool keep_index_consistent);
private:
/**
* This is a helper method, it generates a list of bucket index objects with the given
--caps=<caps> list of caps (e.g., "usage=read, write; user=read"
--yes-i-really-mean-it required for certain operations
--reset-regions reset regionmap when regionmap update
+ --bypass-gc when specified with bucket deletion, triggers
+ object deletions by not involving GC
+ --inconsistent-index when specified with bucket deletion and bypass-gc set to true,
+ ignores bucket index consistency
+
<date> := "YYYY-MM-DD[ hh:mm:ss]"
Quota options:
projects / ceph.git / commitdiff