From: Yehuda Sadeh Date: Wed, 28 Nov 2012 21:50:08 +0000 (-0800) Subject: rgw: bucket check --check-objects X-Git-Tag: v0.56~139^2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=5d6da25c78465d64481675d2272b6e1fbdff8656;p=ceph.git rgw: bucket check --check-objects A new option that goes through the indexed objects, verifies their actual state and updates the index accordingly. Signed-off-by: Yehuda Sadeh --- diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc index 665ac5516e86..6a7d7915a692 100644 --- a/src/rgw/rgw_admin.cc +++ b/src/rgw/rgw_admin.cc @@ -90,6 +90,8 @@ void _usage() cerr << " --end-date=\n"; cerr << " --bucket-id=\n"; cerr << " --fix besides checking bucket index, will also fix it\n"; + cerr << " --check-objects bucket check: rebuilds bucket index according to\n"; + cerr << " actual objects state\n"; cerr << " --format= specify output format for certain operations: xml,\n"; cerr << " json\n"; cerr << " --purge-data when specified, user removal will also purge all the\n"; @@ -648,6 +650,13 @@ static int remove_bucket(RGWRados *store, rgw_bucket& bucket, bool delete_childr return ret; } +static bool bucket_object_check_filter(const string& name) +{ + string ns; + string obj = name; + return rgw_obj::translate_raw_obj_to_obj_in_ns(obj, ns); +} + int main(int argc, char **argv) { vector args; @@ -691,6 +700,7 @@ int main(int argc, char **argv) int max_buckets = -1; map categories; string caps; + int check_objects = false; std::string val; std::ostringstream errs; @@ -786,6 +796,8 @@ int main(int argc, char **argv) // do nothing } else if (ceph_argparse_binary_flag(args, i, &fix, NULL, "--fix", (char*)NULL)) { // do nothing + } else if (ceph_argparse_binary_flag(args, i, &check_objects, NULL, "--check-objects", (char*)NULL)) { + // do nothing } else if (ceph_argparse_witharg(args, i, &val, "--caps", (char*)NULL)) { caps = val; } else { @@ -1608,6 +1620,45 @@ next: map existing_stats; map calculated_stats; + if (check_objects) { + if (!fix) { + cerr << "--check-objects flag requires --fix" << std::endl; + return 1; + } +#define BUCKET_TAG_TIMEOUT 30 + cout << "Checking objects, decreasing bucket 2-phase commit timeout.\n" + "** Note that timeout will reset only when operation completes successfully **" << std::endl; + + store->cls_obj_set_bucket_tag_timeout(bucket, BUCKET_TAG_TIMEOUT); + + string prefix; + string marker; + bool is_truncated = true; + + while (is_truncated) { + map result; + string ns; + int r = store->cls_bucket_list(bucket, marker, prefix, 1000, + result, &is_truncated, &marker, + bucket_object_check_filter); + + if (r < 0 && r != -ENOENT) { + cerr << "ERROR: failed operation r=" << r << std::endl; + } + + if (r == -ENOENT) + break; + + map::iterator iter; + for (iter = result.begin(); iter != result.end(); ++iter) { + cout << iter->first << std::endl; + } + + } + + store->cls_obj_set_bucket_tag_timeout(bucket, 0); + + } int r = store->bucket_check_index(bucket, &existing_stats, &calculated_stats); if (r < 0) { cerr << "failed to check index err=" << cpp_strerror(-r) << std::endl; @@ -1631,6 +1682,7 @@ next: return r; } } + } if (opt_cmd == OPT_BUCKET_RM) { diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index f877883c7eba..70220b4176d0 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -3076,9 +3076,27 @@ int RGWRados::cls_obj_complete_cancel(rgw_bucket& bucket, string& tag, string& n return cls_obj_complete_op(bucket, CLS_RGW_OP_ADD, tag, 0, ent, RGW_OBJ_CATEGORY_NONE, NULL); } +int RGWRados::cls_obj_set_bucket_tag_timeout(rgw_bucket& bucket, uint64_t timeout) +{ + librados::IoCtx io_ctx; + string oid; + + int r = open_bucket(bucket, io_ctx, oid); + if (r < 0) + return r; + + ObjectWriteOperation o; + cls_rgw_bucket_set_tag_timeout(o, timeout); + + r = io_ctx.operate(oid, &o); + + return r; +} + int RGWRados::cls_bucket_list(rgw_bucket& bucket, string start, string prefix, uint32_t num, map& m, - bool *is_truncated, string *last_entry) + bool *is_truncated, string *last_entry, + bool (*force_check_filter)(const string& name)) { ldout(cct, 10) << "cls_bucket_list " << bucket << " start " << start << " num " << num << dendl; @@ -3114,7 +3132,9 @@ int RGWRados::cls_bucket_list(rgw_bucket& bucket, string start, string prefix, continue; } - if (!dirent.exists || !dirent.pending_map.empty()) { + bool force_check = force_check_filter && force_check_filter(dirent.name); + + if (!dirent.exists || !dirent.pending_map.empty() || force_check) { /* there are uncommitted ops. We need to check the current state, * and if the tags are old we need to do cleanup as well. */ librados::IoCtx sub_ctx; diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h index 50d8572f0819..e2ab4e244994 100644 --- a/src/rgw/rgw_rados.h +++ b/src/rgw/rgw_rados.h @@ -683,9 +683,10 @@ public: int cls_obj_complete_add(rgw_bucket& bucket, string& tag, uint64_t epoch, RGWObjEnt& ent, RGWObjCategory category, list *remove_objs); int cls_obj_complete_del(rgw_bucket& bucket, string& tag, uint64_t epoch, string& name); int cls_obj_complete_cancel(rgw_bucket& bucket, string& tag, string& name); + int cls_obj_set_bucket_tag_timeout(rgw_bucket& bucket, uint64_t timeout); int cls_bucket_list(rgw_bucket& bucket, string start, string prefix, uint32_t num, map& m, bool *is_truncated, - string *last_entry = NULL); + string *last_entry, bool (*force_check_filter)(const string& name) = NULL); int cls_bucket_head(rgw_bucket& bucket, struct rgw_bucket_dir_header& header); int prepare_update_index(RGWObjState *state, rgw_bucket& bucket, rgw_obj& oid, string& tag); diff --git a/src/test/cli/radosgw-admin/help.t b/src/test/cli/radosgw-admin/help.t index 4dd8634c28cb..a9269ecb6914 100644 --- a/src/test/cli/radosgw-admin/help.t +++ b/src/test/cli/radosgw-admin/help.t @@ -56,6 +56,8 @@ --end-date= --bucket-id= --fix besides checking bucket index, will also fix it + --check-objects bucket check: rebuilds bucket index according to + actual objects state --format= specify output format for certain operations: xml, json --purge-data when specified, user removal will also purge all the