From: Yehuda Sadeh Date: Fri, 26 Jun 2015 23:56:28 +0000 (-0700) Subject: rgw: conversion tool to fix broken multipart objects X-Git-Tag: v0.94.3~3^2 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=e19f928bd770a37f2f631c4cd796e2e30a494234;p=ceph.git rgw: conversion tool to fix broken multipart objects Fixes: #12079 Broken multipart objects: multipart objects that created on 0.94.2 and that start with underscore have a bad locator on their tail objects. This extends the tool that was needed for older issue we've had with hammer multipart objects (that start with underscore). The same usage applies: $ radosgw-admin bucket check --check-head-obj-locator \ --bucket= [--fix] Signed-off-by: Yehuda Sadeh (cherry picked from commit f02ca6107172cecd80a490df9f0d66204e62326c) --- diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc index 7cf3126a2687d..4549a65a4f574 100644 --- a/src/rgw/rgw_admin.cc +++ b/src/rgw/rgw_admin.cc @@ -911,6 +911,7 @@ int check_min_obj_stripe_size(RGWRados *store, RGWBucketInfo& bucket_info, rgw_o int check_obj_locator_underscore(RGWBucketInfo& bucket_info, rgw_obj& obj, rgw_obj_key& key, bool fix, bool remove_bad, Formatter *f) { f->open_object_section("object"); f->open_object_section("key"); + f->dump_string("type", "head"); f->dump_string("name", key.name); f->dump_string("instance", key.instance); f->close_section(); @@ -953,6 +954,36 @@ done: return 0; } +int check_obj_tail_locator_underscore(RGWBucketInfo& bucket_info, rgw_obj& obj, rgw_obj_key& key, bool fix, Formatter *f) { + f->open_object_section("object"); + f->open_object_section("key"); + f->dump_string("type", "tail"); + f->dump_string("name", key.name); + f->dump_string("instance", key.instance); + f->close_section(); + + string oid; + string locator; + + bool needs_fixing; + string status; + + int ret = store->fix_tail_obj_locator(obj.bucket, key, fix, &needs_fixing); + if (ret < 0) { + cerr << "ERROR: fix_tail_object_locator_underscore() returned ret=" << ret << std::endl; + status = "failed"; + } else { + status = (needs_fixing && !fix ? "needs_fixing" : "ok"); + } + + f->dump_bool("needs_fixing", needs_fixing); + f->dump_string("status", status); + + f->close_section(); + + return 0; +} + int do_check_object_locator(const string& bucket_name, bool fix, bool remove_bad, Formatter *f) { if (remove_bad && !fix) { @@ -1012,6 +1043,10 @@ int do_check_object_locator(const string& bucket_name, bool fix, bool remove_bad if (key.name[0] == '_') { ret = check_obj_locator_underscore(bucket_info, obj, key, fix, remove_bad, f); /* ignore return code, move to the next one */ + + if (ret >= 0) { + ret = check_obj_tail_locator_underscore(bucket_info, obj, key, fix, f); + } } } f->flush(cout); diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index 0584bcf243966..0beba4cd9b5ff 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -3045,6 +3045,149 @@ int RGWRados::fix_head_obj_locator(rgw_bucket& bucket, bool copy_obj, bool remov return 0; } +int RGWRados::move_rados_obj(librados::IoCtx& src_ioctx, + const string& src_oid, const string& src_locator, + librados::IoCtx& dst_ioctx, + const string& dst_oid, const string& dst_locator) +{ + +#define COPY_BUF_SIZE (4 * 1024 * 1024) + bool done = false; + uint64_t chunk_size = COPY_BUF_SIZE; + uint64_t ofs = 0; + int ret = 0; + time_t mtime = 0; + uint64_t size; + + if (src_oid == dst_oid && src_locator == dst_locator) { + return 0; + } + + src_ioctx.locator_set_key(src_locator); + dst_ioctx.locator_set_key(dst_locator); + + do { + bufferlist data; + ObjectReadOperation rop; + ObjectWriteOperation wop; + + if (ofs == 0) { + rop.stat(&size, &mtime, NULL); + } + rop.read(ofs, chunk_size, &data, NULL); + ret = src_ioctx.operate(src_oid, &rop, NULL); + if (ret < 0) { + goto done_err; + } + + if (data.length() == 0) { + break; + } + + if (ofs == 0) { + wop.create(true); /* make it exclusive */ + wop.mtime(&mtime); + } + wop.write(ofs, data); + ret = dst_ioctx.operate(dst_oid, &wop); + ofs += data.length(); + done = data.length() != chunk_size; + } while (!done); + + if (ofs != size) { + lderr(cct) << "ERROR: " << __func__ << ": copying " << src_oid << " -> " << dst_oid + << ": expected " << size << " bytes to copy, ended up with " << ofs << dendl; + ret = -EIO; + goto done_err; + } + + src_ioctx.remove(src_oid); + + return 0; + +done_err: + lderr(cct) << "ERROR: failed to copy " << src_oid << " -> " << dst_oid << dendl; + return ret; +} + +/* + * fixes an issue where head objects were supposed to have a locator created, but ended + * up without one + */ +int RGWRados::fix_tail_obj_locator(rgw_bucket& bucket, rgw_obj_key& key, bool fix, bool *need_fix) +{ + string locator; + + rgw_obj obj(bucket, key); + + if (need_fix) { + *need_fix = false; + } + + rgw_rados_ref ref; + int r = get_obj_ref(obj, &ref, &bucket); + if (r < 0) { + return r; + } + + RGWObjState *astate = NULL; + RGWObjectCtx rctx(this); + r = get_obj_state(&rctx, obj, &astate, NULL); + if (r < 0) + return r; + + if (astate->has_manifest) { + RGWObjManifest::obj_iterator miter; + RGWObjManifest& manifest = astate->manifest; + for (miter = manifest.obj_begin(); miter != manifest.obj_end(); ++miter) { + rgw_obj loc = miter.get_location(); + string oid; + string locator; + + if (loc.ns.empty()) { + /* continue, we're only interested in tail objects */ + continue; + } + + get_obj_bucket_and_oid_loc(loc, bucket, oid, locator); + ref.ioctx.locator_set_key(locator); + + ldout(cct, 20) << __func__ << ": key=" << key << " oid=" << oid << " locator=" << locator << dendl; + + r = ref.ioctx.stat(oid, NULL, NULL); + if (r != -ENOENT) { + continue; + } + + string bad_loc; + prepend_bucket_marker(bucket, loc.get_orig_obj(), bad_loc); + + /* create a new ioctx with the bad locator */ + librados::IoCtx src_ioctx; + src_ioctx.dup(ref.ioctx); + src_ioctx.locator_set_key(bad_loc); + + r = src_ioctx.stat(oid, NULL, NULL); + if (r != 0) { + /* cannot find a broken part */ + continue; + } + ldout(cct, 20) << __func__ << ": found bad object part: " << loc << dendl; + if (need_fix) { + *need_fix = true; + } + if (fix) { + r = move_rados_obj(src_ioctx, oid, bad_loc, ref.ioctx, oid, locator); + if (r < 0) { + lderr(cct) << "ERROR: copy_rados_obj() on oid=" << oid << " returned r=" << r << dendl; + } + } + } + } + + return 0; +} + int RGWRados::BucketShard::init(rgw_bucket& _bucket, rgw_obj& obj) { bucket = _bucket; diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h index 51b6a728ff9db..18213cf3a5499 100644 --- a/src/rgw/rgw_rados.h +++ b/src/rgw/rgw_rados.h @@ -2031,7 +2031,12 @@ public: map *calculated_stats); int bucket_rebuild_index(rgw_bucket& bucket); int remove_objs_from_index(rgw_bucket& bucket, list& oid_list); + int move_rados_obj(librados::IoCtx& src_ioctx, + const string& src_oid, const string& src_locator, + librados::IoCtx& dst_ioctx, + const string& dst_oid, const string& dst_locator); int fix_head_obj_locator(rgw_bucket& bucket, bool copy_obj, bool remove_bad, rgw_obj_key& key); + int fix_tail_obj_locator(rgw_bucket& bucket, rgw_obj_key& key, bool fix, bool *need_fix); int cls_user_get_header(const string& user_id, cls_user_header *header); int cls_user_get_header_async(const string& user_id, RGWGetUserHeader_CB *ctx);