From e396064d9afa6fcba7b6bb1e3a500ef9e718e0d2 Mon Sep 17 00:00:00 2001 From: "J. Eric Ivancich" Date: Wed, 18 Sep 2019 18:25:50 -0400 Subject: [PATCH] rgw: add `rgw-orphan-list` tool & `radosgw-admin bucket radoslist ...` Adds a `radosgw-admin` subcommand and walks the associated bucket indices and manifests to generate the list of rados objects that represent the rgw objects in the bucket(s). Also adds a tool named `rgw-orphan-list`, which uses the radoslist subcommand, that produces a list in a local file of what appear to be rgw orphans. Signed-off-by: J. Eric Ivancich --- ceph.spec.in | 2 + debian/radosgw.install | 2 + src/rgw/CMakeLists.txt | 3 + src/rgw/rgw-orphan-list | 90 +++++ src/rgw/rgw_admin.cc | 35 +- src/rgw/rgw_op.h | 7 + src/rgw/rgw_orphan.cc | 648 ++++++++++++++++++++++++++++++ src/rgw/rgw_orphan.h | 77 ++++ src/rgw/rgw_rados.cc | 8 +- src/rgw/services/svc_tier_rados.h | 6 + src/test/cli/radosgw-admin/help.t | 1 + src/tools/CMakeLists.txt | 4 + src/tools/ceph-diff-sorted.cc | 173 ++++++++ 13 files changed, 1052 insertions(+), 4 deletions(-) create mode 100755 src/rgw/rgw-orphan-list create mode 100644 src/tools/ceph-diff-sorted.cc diff --git a/ceph.spec.in b/ceph.spec.in index 408f0b1c1d7ec..6ee83cc399578 100644 --- a/ceph.spec.in +++ b/ceph.spec.in @@ -1931,10 +1931,12 @@ fi %{_mandir}/man8/rbd-nbd.8* %files radosgw +%{_bindir}/ceph-diff-sorted %{_bindir}/radosgw %{_bindir}/radosgw-token %{_bindir}/radosgw-es %{_bindir}/radosgw-object-expirer +%{_bindir}/rgw-orphan-list %{_libdir}/libradosgw.so* %{_mandir}/man8/radosgw.8* %dir %{_localstatedir}/lib/ceph/radosgw diff --git a/debian/radosgw.install b/debian/radosgw.install index 303791694c25a..b2930879adc7d 100644 --- a/debian/radosgw.install +++ b/debian/radosgw.install @@ -1,7 +1,9 @@ lib/systemd/system/ceph-radosgw* +usr/bin/ceph-diff-sorted usr/bin/radosgw usr/bin/radosgw-es usr/bin/radosgw-object-expirer usr/bin/radosgw-token +usr/bin/rgw-orphan-list usr/lib/libradosgw.so* usr/share/man/man8/radosgw.8 diff --git a/src/rgw/CMakeLists.txt b/src/rgw/CMakeLists.txt index 4af33be7d3d89..02b82a0a65db4 100644 --- a/src/rgw/CMakeLists.txt +++ b/src/rgw/CMakeLists.txt @@ -413,3 +413,6 @@ if(WITH_TESTS) ceph_rgw_multiparser DESTINATION bin) endif(WITH_TESTS) + +install(PROGRAMS rgw-orphan-list + DESTINATION bin) diff --git a/src/rgw/rgw-orphan-list b/src/rgw/rgw-orphan-list new file mode 100755 index 0000000000000..5af5282e03761 --- /dev/null +++ b/src/rgw/rgw-orphan-list @@ -0,0 +1,90 @@ +#!/usr/bin/env bash + +# version 1 + +# IMPORTANT: affects order produced by 'sort' and 'ceph-diff-sorted' +# relies on this ordering +export LANG=C + +out_dir="." +temp_file=/tmp/temp.$$ +timestamp=$(date -u +%Y%m%d%k%M) +lspools_err="${out_dir}/lspools-${timestamp}.error" +rados_out="${out_dir}/rados-${timestamp}.intermediate" +rados_err="${out_dir}/rados-${timestamp}.error" +rgwadmin_out="${out_dir}/radosgw-admin-${timestamp}.intermediate" +rgwadmin_err="${out_dir}/radosgw-admin-${timestamp}.error" +delta_out="${out_dir}/orphan-list-${timestamp}.out" + +error_out() { + echo "An error was encountered while running '$1'. Aborting." + if [ $# -gt 1 ] ;then + echo "Review file '$2' for details." + fi + echo "***" + echo "*** WARNING: The results are incomplete. Do not use! ***" + echo "***" + exit 1 +} + +prompt_pool() { + echo "Available pools:" + rados lspools >"$temp_file" 2>"$lspools_err" + if [ "$?" -ne 0 ] ;then + error_out "rados lspools" "$lspools_err" + fi + sed 's/^/ /' $temp_file # list pools and indent + printf "Which pool do you want to search for orphans? " + local mypool + read mypool + echo $mypool +} + +if [ $# -eq 0 ] ;then + pool="$(prompt_pool)" +elif [ $# -eq 1 ] ;then + pool="$1" +else + error_out "Usage: $0 [pool]" +fi + +echo "Pool is \"$pool\"." + +echo "Note: output files produced will be tagged with the current timestamp -- ${timestamp}." + +echo "running 'rados ls' at $(date)" +rados ls --pool="$pool" >"$rados_out" 2>"$rados_err" +if [ "$?" -ne 0 ] ;then + error_out "rados ls" "$rados_err" +fi +sort -u "$rados_out" >"$temp_file" +mv -f "$temp_file" "$rados_out" + +echo "running 'radosgw-admin bucket radoslist' at $(date)" +radosgw-admin bucket radoslist >"$rgwadmin_out" 2>"$rgwadmin_err" +if [ "$?" -ne 0 ] ;then + error_out "radosgw-admin radoslist" "$rgwadmin_err" +fi +sort -u "$rgwadmin_out" >"$temp_file" +mv -f "$temp_file" "$rgwadmin_out" + +echo "computing delta at $(date)" +ceph-diff-sorted "$rados_out" "$rgwadmin_out" | grep "^<" | sed 's/^< *//' >"$delta_out" +# use PIPESTATUS to get at exit status of first process in above pipe; +# 0 means same, 1 means different, >1 means error +if [ "${PIPESTATUS[0]}" -gt 1 ] ;then + error_out "ceph-diff-sorted" +fi + +found="$(wc -l < $delta_out)" +possible="$(wc -l < $rados_out)" +percentage=$(expr 100 \* $found / $possible) + +echo "$found potential orphans found out of a possible $possible (${percentage}%)." +echo "The results can be found in ${delta_out}." +echo " Intermediate files: ${rados_out} and ${rgwadmin_out}" +echo "***" +echo "*** WARNING: This is EXPERIMENTAL code and the results should be used" +echo "*** only with CAUTION!" +echo "***" +echo "Done at $(date)." diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc index a1a011e0153e6..7c01bc2f172b6 100644 --- a/src/rgw/rgw_admin.cc +++ b/src/rgw/rgw_admin.cc @@ -136,6 +136,7 @@ void usage() cout << " bucket rewrite rewrite all objects in the specified bucket\n"; cout << " bucket sync disable disable bucket sync\n"; cout << " bucket sync enable enable bucket sync\n"; + cout << " bucket radoslist list rados objects backing bucket's objects\n"; cout << " bi get retrieve bucket index object entries\n"; cout << " bi put store bucket index object entries\n"; cout << " bi list list raw bucket index entries\n"; @@ -586,6 +587,7 @@ enum class OPT { BUCKET_REWRITE, BUCKET_RESHARD, BUCKET_CHOWN, + BUCKET_RADOS_LIST, POLICY, POOL_ADD, POOL_RM, @@ -785,6 +787,8 @@ static SimpleCmd::Commands all_cmds = { { "bucket rewrite", OPT::BUCKET_REWRITE }, { "bucket reshard", OPT::BUCKET_RESHARD }, { "bucket chown", OPT::BUCKET_CHOWN }, + { "bucket radoslist", OPT::BUCKET_RADOS_LIST }, + { "bucket rados list", OPT::BUCKET_RADOS_LIST }, { "policy", OPT::POLICY }, { "pool add", OPT::POOL_ADD }, { "pool rm", OPT::POOL_RM }, @@ -1076,8 +1080,12 @@ public: } }; -static int init_bucket(const string& tenant_name, const string& bucket_name, const string& bucket_id, - RGWBucketInfo& bucket_info, rgw_bucket& bucket, map *pattrs = nullptr) +static int init_bucket(const string& tenant_name, + const string& bucket_name, + const string& bucket_id, + RGWBucketInfo& bucket_info, + rgw_bucket& bucket, + map *pattrs = nullptr) { if (!bucket_name.empty()) { auto obj_ctx = store->svc()->sysobj->init_obj_ctx(); @@ -6076,6 +6084,29 @@ int main(int argc, const char **argv) } /* have bucket_name */ } /* OPT::BUCKETS_LIST */ + if (opt_cmd == OPT::BUCKET_RADOS_LIST) { + RGWRadosList lister(store, + max_concurrent_ios, orphan_stale_secs, tenant); + if (bucket_name.empty()) { + ret = lister.run(); + } else { + ret = lister.run(bucket_name); + } + + if (ret < 0) { + std::cerr << + "ERROR: bucket radoslist failed to finish before " << + "encountering error: " << cpp_strerror(-ret) << std::endl; + std::cerr << "************************************" + "************************************" << std::endl; + std::cerr << "WARNING: THE RESULTS ARE NOT RELIABLE AND SHOULD NOT " << + "BE USED IN DELETING ORPHANS" << std::endl; + std::cerr << "************************************" + "************************************" << std::endl; + return -ret; + } + } + if (opt_cmd == OPT::BUCKET_STATS) { if (bucket_name.empty() && !bucket_id.empty()) { rgw_bucket bucket; diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h index 906b26d236996..dc554a37b714b 100644 --- a/src/rgw/rgw_op.h +++ b/src/rgw/rgw_op.h @@ -1829,6 +1829,13 @@ public: struct RGWMultipartUploadEntry { rgw_bucket_dir_entry obj; RGWMPObj mp; + + friend std::ostream& operator<<(std::ostream& out, + const RGWMultipartUploadEntry& e) { + constexpr char quote = '"'; + return out << "RGWMultipartUploadEntry{ obj.key=" << + quote << e.obj.key << quote << " mp=" << e.mp << " }"; + } }; class RGWListBucketMultiparts : public RGWOp { diff --git a/src/rgw/rgw_orphan.cc b/src/rgw/rgw_orphan.cc index 7a1174039cbdf..ce4f5f69ac01a 100644 --- a/src/rgw/rgw_orphan.cc +++ b/src/rgw/rgw_orphan.cc @@ -9,6 +9,8 @@ #include "common/errno.h" #include "rgw_rados.h" +#include "rgw_op.h" +#include "rgw_multi.h" #include "rgw_orphan.h" #include "rgw_zone.h" #include "rgw_bucket.h" @@ -57,6 +59,30 @@ static string obj_fingerprint(const string& oid, const char *force_ns = NULL) return s.substr(0, i + 1); } + +static string obj_force_ns(const string& oid, const char *force_ns) +{ + ssize_t pos = oid.find('_'); + if (pos < 0) { + cerr << "ERROR: object does not have a bucket marker: " << oid << std::endl; + } + + string obj_marker = oid.substr(0, pos); + + rgw_obj_key obj; + rgw_obj_key::parse_raw_oid(oid.substr(pos + 1), &obj); + + if (obj.ns.empty()) { + return oid; + } + + obj.set_ns(force_ns); + + string s = obj_marker + obj.get_oid(); + + return s; +} + int RGWOrphanStore::read_job(const string& job_name, RGWOrphanSearchState & state) { set keys; @@ -920,3 +946,625 @@ int RGWOrphanSearch::finish() return r; } + + +int RGWRadosList::handle_stat_result(RGWRados::Object::Stat::Result& result, + std::set& obj_oids) +{ + obj_oids.clear(); + + rgw_bucket& bucket = result.obj.bucket; + + ldout(store->ctx(), 20) << "RGWRadosList::" << __func__ << + " bucket=" << bucket << + ", has_manifest=" << result.manifest.has_value() << + dendl; + + // iterator to store result of dlo/slo attribute find + decltype(result.attrs)::iterator attr_it = result.attrs.end(); + const std::string oid = bucket.marker + "_" + result.obj.get_oid(); + ldout(store->ctx(), 20) << "radoslist processing object=\"" << + oid << "\"" << dendl; + if (visited_oids.find(oid) != visited_oids.end()) { + // apparently we hit a loop; don't continue with this oid + ldout(store->ctx(), 15) << + "radoslist stopped loop at already visited object=\"" << + oid << "\"" << dendl; + return 0; + } + + if (!result.manifest) { + /* a very very old object, or part of a multipart upload during upload */ + obj_oids.insert(oid); + + /* + * multipart parts don't have manifest on them, it's in the meta + * object; we'll process them in + * RGWRadosList::do_incomplete_multipart + */ + } else if ((attr_it = result.attrs.find(RGW_ATTR_USER_MANIFEST)) != + result.attrs.end()) { + // *** handle DLO object *** + + obj_oids.insert(oid); + visited_oids.insert(oid); // prevent dlo loops + ldout(store->ctx(), 15) << "radoslist added to visited list DLO=\"" << + oid << "\"" << dendl; + + char* prefix_path_c = attr_it->second.c_str(); + const std::string& prefix_path = prefix_path_c; + + const size_t sep_pos = prefix_path.find('/'); + if (string::npos == sep_pos) { + return -EINVAL; + } + + const std::string bucket_name = prefix_path.substr(0, sep_pos); + const std::string prefix = prefix_path.substr(sep_pos + 1); + + add_bucket_prefix(bucket_name, prefix); + ldout(store->ctx(), 25) << "radoslist DLO oid=\"" << oid << + "\" added bucket=\"" << bucket_name << "\" prefix=\"" << + prefix << "\" to process list" << dendl; + } else if ((attr_it = result.attrs.find(RGW_ATTR_SLO_MANIFEST)) != + result.attrs.end()) { + // *** handle SLO object *** + + obj_oids.insert(oid); + visited_oids.insert(oid); // prevent slo loops + ldout(store->ctx(), 15) << "radoslist added to visited list SLO=\"" << + oid << "\"" << dendl; + + RGWSLOInfo slo_info; + bufferlist::const_iterator bliter = attr_it->second.begin(); + try { + ::decode(slo_info, bliter); + } catch (buffer::error& err) { + ldout(store->ctx(), 0) << + "ERROR: failed to decode slo manifest for " << oid << dendl; + return -EIO; + } + + for (const auto& iter : slo_info.entries) { + const string& path_str = iter.path; + + const size_t sep_pos = path_str.find('/', 1 /* skip initial slash */); + if (string::npos == sep_pos) { + return -EINVAL; + } + + std::string bucket_name; + std::string obj_name; + + bucket_name = url_decode(path_str.substr(1, sep_pos - 1)); + obj_name = url_decode(path_str.substr(sep_pos + 1)); + + const rgw_obj_key obj_key(obj_name); + add_bucket_filter(bucket_name, obj_key); + ldout(store->ctx(), 25) << "radoslist SLO oid=\"" << oid << + "\" added bucket=\"" << bucket_name << "\" obj_key=\"" << + obj_key << "\" to process list" << dendl; + } + } else { + RGWObjManifest& manifest = *result.manifest; + + // in multipart, the head object contains no data and just has the + // manifest AND empty objects have no manifest, but they're + // realized as empty rados objects + if (0 == manifest.get_max_head_size() || + manifest.obj_begin() == manifest.obj_end()) { + obj_oids.insert(oid); + // first_insert = true; + } + + RGWObjManifest::obj_iterator miter; + for (miter = manifest.obj_begin(); miter != manifest.obj_end(); ++miter) { + const rgw_raw_obj& loc = + miter.get_location().get_raw_obj(store->getRados()); + string s = loc.oid; + obj_oids.insert(s); + } + } + + return 0; +} // RGWRadosList::handle_stat_result + +int RGWRadosList::pop_and_handle_stat_op( + RGWObjectCtx& obj_ctx, + std::deque& ops) +{ + std::set obj_oids; + RGWRados::Object::Stat& front_op = ops.front(); + + int ret = front_op.wait(); + if (ret < 0) { + if (ret != -ENOENT) { + lderr(store->ctx()) << "ERROR: stat_async() returned error: " << + cpp_strerror(-ret) << dendl; + } + goto done; + } + + ret = handle_stat_result(front_op.result, obj_oids); + if (ret < 0) { + lderr(store->ctx()) << "ERROR: handle_stat_result() returned error: " << + cpp_strerror(-ret) << dendl; + } + + // output results + for (const auto& o : obj_oids) { + std::cout << o << std::endl; + } + +done: + + // invalidate object context for this object to avoid memory leak + // (see pr https://github.com/ceph/ceph/pull/30174) + obj_ctx.invalidate(front_op.result.obj); + + ops.pop_front(); + return ret; +} + + +#if 0 // code that may be the basis for expansion +int RGWRadosList::build_buckets_instance_index() +{ + void *handle; + int max = 1000; + string section = "bucket.instance"; + int ret = store->meta_mgr->list_keys_init(section, &handle); + if (ret < 0) { + lderr(store->ctx()) << "ERROR: can't get key: " << cpp_strerror(-ret) << dendl; + return ret; + } + + map > instances; + + bool truncated; + + RGWObjectCtx obj_ctx(store); + + int count = 0; + uint64_t total = 0; + + do { + list keys; + ret = store->meta_mgr->list_keys_next(handle, max, keys, &truncated); + if (ret < 0) { + lderr(store->ctx()) << "ERROR: lists_keys_next(): " << cpp_strerror(-ret) << dendl; + return ret; + } + + for (list::iterator iter = keys.begin(); iter != keys.end(); ++iter) { + ++total; + ldout(store->ctx(), 10) << "bucket_instance=" << *iter << " total=" << total << dendl; + int shard = orphan_shard(*iter); + instances[shard].push_back(*iter); + + if (++count >= COUNT_BEFORE_FLUSH) { + ret = log_oids(buckets_instance_index, instances); + if (ret < 0) { + lderr(store->ctx()) << __func__ << ": ERROR: log_oids() returned ret=" << ret << dendl; + return ret; + } + count = 0; + instances.clear(); + } + } + } while (truncated); + + ret = log_oids(buckets_instance_index, instances); + if (ret < 0) { + lderr(store->ctx()) << __func__ << ": ERROR: log_oids() returned ret=" << ret << dendl; + return ret; + } + store->meta_mgr->list_keys_complete(handle); + + return 0; +} +#endif + + +int RGWRadosList::process_bucket( + const std::string& bucket_instance_id, + const std::string& prefix, + const std::set& entries_filter) +{ + ldout(store->ctx(), 10) << "RGWRadosList::" << __func__ << + " bucket_instance_id=" << bucket_instance_id << + ", prefix=" << prefix << + ", entries_filter.size=" << entries_filter.size() << dendl; + + RGWBucketInfo bucket_info; + RGWSysObjectCtx sys_obj_ctx = store->svc()->sysobj->init_obj_ctx(); + int ret = store->getRados()->get_bucket_instance_info(sys_obj_ctx, + bucket_instance_id, + bucket_info, + nullptr, + nullptr, + null_yield); + if (ret < 0) { + if (ret == -ENOENT) { + // probably raced with bucket removal + return 0; + } + lderr(store->ctx()) << __func__ << + ": ERROR: RGWRados::get_bucket_instance_info() returned ret=" << + ret << dendl; + return ret; + } + + RGWRados::Bucket target(store->getRados(), bucket_info); + RGWRados::Bucket::List list_op(&target); + + std::string marker; + list_op.params.marker = rgw_obj_key(marker); + list_op.params.list_versions = true; + list_op.params.enforce_ns = false; + list_op.params.allow_unordered = false; + list_op.params.prefix = prefix; + + bool truncated; + + std::deque stat_ops; + std::string prev_versioned_key_name = ""; + + RGWObjectCtx obj_ctx(store); + + do { + std::vector result; + + constexpr int64_t LIST_OBJS_MAX_ENTRIES = 100; + ret = list_op.list_objects(LIST_OBJS_MAX_ENTRIES, &result, + NULL, &truncated, null_yield); + if (ret == -ENOENT) { + // race with bucket delete? + ret = 0; + break; + } else if (ret < 0) { + std::cerr << "ERROR: store->list_objects(): " << cpp_strerror(-ret) << + std::endl; + return ret; + } + + for (std::vector::iterator iter = result.begin(); + iter != result.end(); + ++iter) { + rgw_bucket_dir_entry& entry = *iter; + + if (entry.key.instance.empty()) { + ldout(store->ctx(), 20) << "obj entry: " << entry.key.name << dendl; + } else { + ldout(store->ctx(), 20) << "obj entry: " << entry.key.name << + " [" << entry.key.instance << "]" << dendl; + } + + ldout(store->ctx(), 20) << __func__ << ": entry.key.name=" << + entry.key.name << " entry.key.instance=" << entry.key.instance << + dendl; + + // ignore entries that are not in the filter if there is a filter + if (!entries_filter.empty() && + entries_filter.find(entry.key) == entries_filter.cend()) { + continue; + } + + // we need to do this in two cases below, so use a lambda + auto do_stat_key = + [&](const rgw_obj_key& key) -> int { + int ret; + + rgw_obj obj(bucket_info.bucket, key); + + RGWRados::Object op_target(store->getRados(), bucket_info, + obj_ctx, obj); + + stat_ops.push_back(RGWRados::Object::Stat(&op_target)); + RGWRados::Object::Stat& op = stat_ops.back(); + + ret = op.stat_async(); + if (ret < 0) { + lderr(store->ctx()) << "ERROR: stat_async() returned error: " << + cpp_strerror(-ret) << dendl; + return ret; + } + + if (stat_ops.size() >= max_concurrent_ios) { + ret = pop_and_handle_stat_op(obj_ctx, stat_ops); + if (ret < 0) { + if (ret != -ENOENT) { + lderr(store->ctx()) << + "ERROR: pop_and_handle_stat_op() returned error: " << + cpp_strerror(-ret) << dendl; + } + + // clear error, so we'll continue processing directory + ret = 0; + } + } + + return ret; + }; // do_stat_key lambda + + // for versioned objects, make sure the head object is handled + // as well by ignoring the instance identifier + if (!entry.key.instance.empty() && + entry.key.name != prev_versioned_key_name) { + // don't do the same key twice; even though out bucket index + // listing allows unordered, since all versions of an object + // use the same bucket index key, they'll all end up together + // and sorted + prev_versioned_key_name = entry.key.name; + + rgw_obj_key uninstanced(entry.key.name); + + ret = do_stat_key(uninstanced); + if (ret < 0) { + return ret; + } + } + + ret = do_stat_key(entry.key); + if (ret < 0) { + return ret; + } + } // for iter loop + } while (truncated); + + while (!stat_ops.empty()) { + ret = pop_and_handle_stat_op(obj_ctx, stat_ops); + if (ret < 0) { + if (ret != -ENOENT) { + lderr(store->ctx()) << "ERROR: stat_async() returned error: " << + cpp_strerror(-ret) << dendl; + } + } + } + + return 0; +} + + +int RGWRadosList::run() +{ + int ret; + void* handle = nullptr; + + ret = store->ctl()->meta.mgr->list_keys_init("bucket", &handle); + if (ret < 0) { + lderr(store->ctx()) << "RGWRadosList::" << __func__ << + " ERROR: list_keys_init returned " << + cpp_strerror(-ret) << dendl; + return ret; + } + + const int max_keys = 1000; + bool truncated = true; + + do { + std::list buckets; + ret = store->ctl()->meta.mgr->list_keys_next(handle, max_keys, + buckets, &truncated); + + for (std::string& bucket_id : buckets) { + ret = run(bucket_id); + if (ret == -ENOENT) { + continue; + } else if (ret < 0) { + return ret; + } + } + } while (truncated); + + return 0; +} // RGWRadosList::run() + + +int RGWRadosList::run(const std::string& start_bucket_name) +{ + RGWSysObjectCtx sys_obj_ctx = store->svc()->sysobj->init_obj_ctx(); + RGWObjectCtx obj_ctx(store); + int ret; + + add_bucket_entire(start_bucket_name); + + while (! bucket_process_map.empty()) { + // pop item from map and capture its key data + auto front = bucket_process_map.begin(); + std::string bucket_name = front->first; + process_t process; + std::swap(process, front->second); + bucket_process_map.erase(front); + + RGWBucketInfo bucket_info; + ret = store->getRados()->get_bucket_info(store->svc(), + tenant_name, + bucket_name, + bucket_info, + nullptr, + null_yield); + if (ret == -ENOENT) { + std::cerr << "WARNING: bucket " << bucket_name << + " does not exist; could it have been deleted very recently?" << + std::endl; + continue; + } else if (ret < 0) { + std::cerr << "ERROR: could not get info for bucket " << bucket_name << + " -- " << cpp_strerror(-ret) << std::endl; + return ret; + } + + const std::string bucket_id = bucket_info.bucket.get_key(); + + static const std::set empty_filter; + static const std::string empty_prefix; + + auto do_process_bucket = + [&bucket_id, this] + (const std::string& prefix, + const std::set& entries_filter) -> int { + int ret = process_bucket(bucket_id, prefix, entries_filter); + if (ret == -ENOENT) { + // bucket deletion race? + return 0; + } if (ret < 0) { + lderr(store->ctx()) << "RGWRadosList::" << __func__ << + ": ERROR: process_bucket(); bucket_id=" << + bucket_id << " returned ret=" << ret << dendl; + } + + return ret; + }; + + // either process the whole bucket *or* process the filters and/or + // the prefixes + if (process.entire_container) { + ret = do_process_bucket(empty_prefix, empty_filter); + if (ret < 0) { + return ret; + } + } else { + if (! process.filter_keys.empty()) { + ret = do_process_bucket(empty_prefix, process.filter_keys); + if (ret < 0) { + return ret; + } + } + for (const auto& p : process.prefixes) { + ret = do_process_bucket(p, empty_filter); + if (ret < 0) { + return ret; + } + } + } + } // while (! bucket_process_map.empty()) + + // now handle incomplete multipart uploads by going back to the + // initial bucket + + RGWBucketInfo bucket_info; + ret = store->getRados()->get_bucket_info(store->svc(), + tenant_name, + start_bucket_name, + bucket_info, + nullptr, + null_yield); + if (ret == -ENOENT) { + // bucket deletion race? + return 0; + } else if (ret < 0) { + lderr(store->ctx()) << "RGWRadosList::" << __func__ << + ": ERROR: get_bucket_info returned ret=" << ret << dendl; + return ret; + } + + ret = do_incomplete_multipart(store, bucket_info); + if (ret < 0) { + lderr(store->ctx()) << "RGWRadosList::" << __func__ << + ": ERROR: do_incomplete_multipart returned ret=" << ret << dendl; + return ret; + } + + return 0; +} // RGWRadosList::run(string) + + +int RGWRadosList::do_incomplete_multipart( + rgw::sal::RGWRadosStore* store, + RGWBucketInfo& bucket_info) +{ + constexpr int max_uploads = 1000; + constexpr int max_parts = 1000; + static const std::string mp_ns = RGW_OBJ_NS_MULTIPART; + static MultipartMetaFilter mp_filter; + + int ret; + + RGWRados::Bucket target(store->getRados(), bucket_info); + RGWRados::Bucket::List list_op(&target); + + bool is_listing_truncated; + std::string empty_string; + RGWMultipartUploadEntry next_uploads_marker; + + do { + RGWMPObj uploads_marker = next_uploads_marker.mp; + const std::string& marker_meta = uploads_marker.get_meta(); + list_op.params.marker = marker_meta; + list_op.params.ns = mp_ns; + list_op.params.filter = &mp_filter; + + // use empty strings for list_op.params.{prefix,delim} + + std::vector objs; + std::map common_prefixes; + ret = list_op.list_objects(max_uploads, &objs, &common_prefixes, + &is_listing_truncated, null_yield); + if (ret == -ENOENT) { + // could bucket have been removed while this is running? + ldout(store->ctx(), 20) << "RGWRadosList::" << __func__ << + ": WARNING: call to list_objects of multipart namespace got ENOENT; " + "assuming bucket removal race" << dendl; + break; + } else if (ret < 0) { + lderr(store->ctx()) << "RGWRadosList::" << __func__ << + ": ERROR: list_objects op returned ret=" << ret << dendl; + return ret; + } + + if (!objs.empty()) { + std::vector uploads; + RGWMultipartUploadEntry entry; + for (const rgw_bucket_dir_entry& obj : objs) { + const rgw_obj_key& key = obj.key; + if (!entry.mp.from_meta(key.name)) { + // we only want the meta objects, so skip all the components + continue; + } + entry.obj = obj; + uploads.push_back(entry); + ldout(store->ctx(), 20) << "RGWRadosList::" << __func__ << + " processing incomplete multipart entry " << + entry << dendl; + } + next_uploads_marker = entry; + + // now process the uploads vector + + int parts_marker = 0; + bool is_parts_truncated = false; + do { + map parts; + + for (const auto& upload : uploads) { + const RGWMPObj& mp = upload.mp; + ret = list_multipart_parts(store, bucket_info, store->ctx(), + mp.get_upload_id(), mp.get_meta(), + max_parts, + parts_marker, parts, NULL, &is_parts_truncated); + if (ret == -ENOENT) { + continue; + } else if (ret < 0) { + lderr(store->ctx()) << "RGWRadosList::" << __func__ << + ": ERROR: list_multipart_parts returned ret=" << ret << dendl; + return ret; + } + + for (auto& p : parts) { + RGWObjManifest& manifest = p.second.manifest; + for (auto obj_it = manifest.obj_begin(); + obj_it != manifest.obj_end(); + ++obj_it) { + const rgw_raw_obj& loc = + obj_it.get_location().get_raw_obj(store->getRados()); + std::cout << loc.oid << std::endl; + } + } + } + } while (is_parts_truncated); + } // if objs not empty + } while(is_listing_truncated); + + return 0; +} // RGWRadosList::do_incomplete_multipart diff --git a/src/rgw/rgw_orphan.h b/src/rgw/rgw_orphan.h index d27e71d6a4a75..ae1e61b780e7d 100644 --- a/src/rgw/rgw_orphan.h +++ b/src/rgw/rgw_orphan.h @@ -210,5 +210,82 @@ public: }; +class RGWRadosList { + + /* + * process_t describes how to process a irectory, we will either + * process the whole thing (entire_container == true) or a portion + * of it (entire_container == false). When we only process a + * portion, we will list the specific keys and/or specific lexical + * prefixes. + */ + struct process_t { + bool entire_container; + std::set filter_keys; + std::set prefixes; + + process_t() : + entire_container(false) + {} + }; + + std::map bucket_process_map; + std::set visited_oids; + + void add_bucket_entire(const std::string& bucket_name) { + auto p = bucket_process_map.emplace(std::make_pair(bucket_name, + process_t())); + p.first->second.entire_container = true; + } + + void add_bucket_prefix(const std::string& bucket_name, + const std::string& prefix) { + auto p = bucket_process_map.emplace(std::make_pair(bucket_name, + process_t())); + p.first->second.prefixes.insert(prefix); + } + + void add_bucket_filter(const std::string& bucket_name, + const rgw_obj_key& obj_key) { + auto p = bucket_process_map.emplace(std::make_pair(bucket_name, + process_t())); + p.first->second.filter_keys.insert(obj_key); + } + + rgw::sal::RGWRadosStore* store; + + uint16_t max_concurrent_ios; + uint64_t stale_secs; + std::string tenant_name; + + int handle_stat_result(RGWRados::Object::Stat::Result& result, + std::set& obj_oids); + int pop_and_handle_stat_op(RGWObjectCtx& obj_ctx, + std::deque& ops); + +public: + + RGWRadosList(rgw::sal::RGWRadosStore* _store, + int _max_ios, + uint64_t _stale_secs, + const std::string& _tenant_name) : + store(_store), + max_concurrent_ios(_max_ios), + stale_secs(_stale_secs), + tenant_name(_tenant_name) + {} + + int process_bucket(const std::string& bucket_instance_id, + const std::string& prefix, + const std::set& entries_filter); + + int do_incomplete_multipart(rgw::sal::RGWRadosStore* store, + RGWBucketInfo& bucket_info); + + int build_linked_oids_index(); + + int run(const std::string& bucket_id); + int run(); +}; // class RGWRadosList #endif diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index a2847c7721fde..42532ec482c1e 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -7552,8 +7552,12 @@ int RGWRados::get_bucket_stats_async(RGWBucketInfo& bucket_info, int shard_id, R return r; } -int RGWRados::get_bucket_instance_info(RGWSysObjectCtx& obj_ctx, const string& meta_key, RGWBucketInfo& info, - real_time *pmtime, map *pattrs, optional_yield y) +int RGWRados::get_bucket_instance_info(RGWSysObjectCtx& obj_ctx, + const string& meta_key, + RGWBucketInfo& info, + real_time *pmtime, + map *pattrs, + optional_yield y) { rgw_bucket bucket; rgw_bucket_parse_bucket_key(cct, meta_key, &bucket, nullptr); diff --git a/src/rgw/services/svc_tier_rados.h b/src/rgw/services/svc_tier_rados.h index a86b0a45111de..f2424990e550a 100644 --- a/src/rgw/services/svc_tier_rados.h +++ b/src/rgw/services/svc_tier_rados.h @@ -16,6 +16,8 @@ #pragma once +#include + #include "rgw/rgw_service.h" #include "svc_rados.h" @@ -84,6 +86,10 @@ public: meta = ""; upload_id = ""; } + friend std::ostream& operator<<(std::ostream& out, const RGWMPObj& obj) { + return out << "RGWMPObj:{ prefix=" << std::quoted(obj.prefix) << + ", meta=" << std::quoted(obj.meta) << " }"; + } }; // class RGWMPObj /** diff --git a/src/test/cli/radosgw-admin/help.t b/src/test/cli/radosgw-admin/help.t index 4f2e1bfc85e0b..4f82c00a35d47 100644 --- a/src/test/cli/radosgw-admin/help.t +++ b/src/test/cli/radosgw-admin/help.t @@ -31,6 +31,7 @@ bucket rewrite rewrite all objects in the specified bucket bucket sync disable disable bucket sync bucket sync enable enable bucket sync + bucket radoslist list rados objects backing bucket's objects bi get retrieve bucket index object entries bi put store bucket index object entries bi list list raw bucket index entries diff --git a/src/tools/CMakeLists.txt b/src/tools/CMakeLists.txt index 730635c8dd11b..9a429abc7fb5c 100644 --- a/src/tools/CMakeLists.txt +++ b/src/tools/CMakeLists.txt @@ -88,6 +88,10 @@ add_executable(osdmaptool ${osdomaptool_srcs}) target_link_libraries(osdmaptool global) install(TARGETS osdmaptool DESTINATION bin) +set(ceph-diff-sorted_srcs ceph-diff-sorted.cc) +add_executable(ceph-diff-sorted ${ceph-diff-sorted_srcs}) +install(TARGETS ceph-diff-sorted DESTINATION bin) + if(WITH_TESTS) set(ceph_psim_srcs psim.cc) add_executable(ceph_psim ${ceph_psim_srcs}) diff --git a/src/tools/ceph-diff-sorted.cc b/src/tools/ceph-diff-sorted.cc new file mode 100644 index 0000000000000..f8e4c28e64789 --- /dev/null +++ b/src/tools/ceph-diff-sorted.cc @@ -0,0 +1,173 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +/* + * diffsorted -- a utility to compute a line-by-line diff on two + * sorted input files + * + * Copyright © 2019 Red Hat + * + * Author: J. Eric Ivancich + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. + */ + + +/* + * SUMMARY + * + * The `diffsorted` utility does a line-by-line diff on two sorted text + * files and indicating lines that are in one file but not the other + * using diff-style notation (although line numbers are not indicated). + * + * USAGE + * + * rgw-diff-sorted file1.txt file2.txt + * + * NOTES + * + * Each files should have its lines in sorted order and should have no + * empty lines. + * + * A potential input file can be sorted using the `sort` utility provided + * that LANG=C to insure byte lexical order. For example: + * + * LANG=C sort unsorted.txt >sorted.txt + * + * or: + * + * export LANG=C + * sort unsorted.txt >sorted.txt + * + * EXIT STATUS + * + * 0 : files same + * 1 : files different + * 2 : usage problem (e.g., wrong number of command-line arguments) + * 3 : problem opening input file + * 4 : bad file content (e.g., unsorted order or empty lines) + */ + + +#include +#include + + +struct FileOfLines { + const char* filename; + std::ifstream input; + std::string this_line, prev_line; + bool next_eof; + bool is_eof; + + FileOfLines(const char* _filename) : + filename(_filename), + input(filename), + next_eof(false), + is_eof(false) + { } + + void dump(const std::string& prefix) { + do { + std::cout << prefix << this_line << std::endl; + advance(); + } while (!eof()); + } + + bool eof() const { + return is_eof; + } + + bool good() const { + return input.good(); + } + + void advance() { + if (next_eof) { + is_eof = true; + return; + } + + prev_line = this_line; + std::getline(input, this_line); + if (this_line.empty()) { + if (!input.eof()) { + std::cerr << "Error: " << filename << " has an empty line." << + std::endl; + exit(4); + } + is_eof = true; + return; + } else if (input.eof()) { + next_eof = true; + } + + if (this_line < prev_line) { + std::cerr << "Error: " << filename << " is not in sorted order; \"" << + this_line << "\" follows \"" << prev_line << "\"." << std::endl; + exit(4); + } + } + + const std::string line() const { + return this_line; + } +}; + +int main(int argc, const char* argv[]) { + if (argc != 3) { + std::cerr << "Usage: " << argv[0] << " " << std::endl; + exit(2); + } + + FileOfLines input1(argv[1]); + if (!input1.good()) { + std::cerr << "Error opening " << argv[1] << + "." << std::endl; + exit(3); + } + + FileOfLines input2(argv[2]); + if (!input2.good()) { + std::cerr << "Error opening " << argv[2] << + "." << std::endl; + exit(3); + } + + bool files_same = true; + + input1.advance(); + input2.advance(); + + while (!input1.eof() && !input2.eof()) { + if (input1.line() == input2.line()) { + input1.advance(); + input2.advance(); + } else if (input1.line() < input2.line()) { + files_same = false; + std::cout << "< " << input1.line() << std::endl; + input1.advance(); + } else { + files_same = false; + std::cout << "> " << input2.line() << std::endl; + input2.advance(); + } + } + + if (!input1.eof()) { + files_same = false; + input1.dump("< "); + } else if (!input2.eof()) { + files_same = false; + input2.dump("> "); + } + + if (files_same) { + exit(0); + } else { + exit(1); + } +} -- 2.39.5