From 36738bf388f99d984ed1b366e818051275d7af6a Mon Sep 17 00:00:00 2001 From: "J. Eric Ivancich" Date: Wed, 18 Sep 2019 18:25:50 -0400 Subject: [PATCH] rgw: add `rgw-orphan-list` tool & `radosgw-admin bucket radoslist ...` Adds a radosgw-admin subcommand and possibly a bucket name (if not, all buckets are assumed) and walks the bucket(s) listing(s) and the manifest (if it exists) for each object in the listing to generate the rados objects that represent the rgw objects in the bucket. Also adds a tool named rgw-orphan-list that will produce a list in a local file of what appear to be rgw orphans. NOTE: This is not a cherry-pick from master because the feature was originally written for an older version of Ceph and is being gradually forward-ported to newer and newer versions. Signed-off-by: J. Eric Ivancich --- ceph.spec.in | 2 + debian/radosgw.install | 2 + src/rgw/CMakeLists.txt | 3 + src/rgw/rgw-orphan-list | 90 +++++ src/rgw/rgw_admin.cc | 36 +- src/rgw/rgw_op.h | 7 + src/rgw/rgw_orphan.cc | 633 ++++++++++++++++++++++++++++++ src/rgw/rgw_orphan.h | 76 ++++ src/rgw/rgw_rados.h | 19 +- src/test/cli/radosgw-admin/help.t | 1 + src/tools/CMakeLists.txt | 4 + src/tools/ceph-diff-sorted.cc | 173 ++++++++ 12 files changed, 1037 insertions(+), 9 deletions(-) create mode 100755 src/rgw/rgw-orphan-list create mode 100644 src/tools/ceph-diff-sorted.cc diff --git a/ceph.spec.in b/ceph.spec.in index 03a73dad010b2..dff86047d32f9 100644 --- a/ceph.spec.in +++ b/ceph.spec.in @@ -1847,10 +1847,12 @@ fi %{_mandir}/man8/rbd-nbd.8* %files radosgw +%{_bindir}/ceph-diff-sorted %{_bindir}/radosgw %{_bindir}/radosgw-token %{_bindir}/radosgw-es %{_bindir}/radosgw-object-expirer +%{_bindir}/rgw-orphan-list %{_mandir}/man8/radosgw.8* %dir %{_localstatedir}/lib/ceph/radosgw %{_unitdir}/ceph-radosgw@.service diff --git a/debian/radosgw.install b/debian/radosgw.install index 329ea0e480fab..c76870ea3baa9 100644 --- a/debian/radosgw.install +++ b/debian/radosgw.install @@ -1,6 +1,8 @@ lib/systemd/system/ceph-radosgw* +usr/bin/ceph-diff-sorted usr/bin/radosgw usr/bin/radosgw-es usr/bin/radosgw-object-expirer usr/bin/radosgw-token +usr/bin/rgw-orphan-list usr/share/man/man8/radosgw.8 diff --git a/src/rgw/CMakeLists.txt b/src/rgw/CMakeLists.txt index 63212ca5cc656..9bfde0e457c20 100644 --- a/src/rgw/CMakeLists.txt +++ b/src/rgw/CMakeLists.txt @@ -397,3 +397,6 @@ if(WITH_TESTS) ceph_rgw_multiparser DESTINATION bin) endif(WITH_TESTS) + +install(PROGRAMS rgw-orphan-list + DESTINATION bin) diff --git a/src/rgw/rgw-orphan-list b/src/rgw/rgw-orphan-list new file mode 100755 index 0000000000000..5af5282e03761 --- /dev/null +++ b/src/rgw/rgw-orphan-list @@ -0,0 +1,90 @@ +#!/usr/bin/env bash + +# version 1 + +# IMPORTANT: affects order produced by 'sort' and 'ceph-diff-sorted' +# relies on this ordering +export LANG=C + +out_dir="." +temp_file=/tmp/temp.$$ +timestamp=$(date -u +%Y%m%d%k%M) +lspools_err="${out_dir}/lspools-${timestamp}.error" +rados_out="${out_dir}/rados-${timestamp}.intermediate" +rados_err="${out_dir}/rados-${timestamp}.error" +rgwadmin_out="${out_dir}/radosgw-admin-${timestamp}.intermediate" +rgwadmin_err="${out_dir}/radosgw-admin-${timestamp}.error" +delta_out="${out_dir}/orphan-list-${timestamp}.out" + +error_out() { + echo "An error was encountered while running '$1'. Aborting." + if [ $# -gt 1 ] ;then + echo "Review file '$2' for details." + fi + echo "***" + echo "*** WARNING: The results are incomplete. Do not use! ***" + echo "***" + exit 1 +} + +prompt_pool() { + echo "Available pools:" + rados lspools >"$temp_file" 2>"$lspools_err" + if [ "$?" -ne 0 ] ;then + error_out "rados lspools" "$lspools_err" + fi + sed 's/^/ /' $temp_file # list pools and indent + printf "Which pool do you want to search for orphans? " + local mypool + read mypool + echo $mypool +} + +if [ $# -eq 0 ] ;then + pool="$(prompt_pool)" +elif [ $# -eq 1 ] ;then + pool="$1" +else + error_out "Usage: $0 [pool]" +fi + +echo "Pool is \"$pool\"." + +echo "Note: output files produced will be tagged with the current timestamp -- ${timestamp}." + +echo "running 'rados ls' at $(date)" +rados ls --pool="$pool" >"$rados_out" 2>"$rados_err" +if [ "$?" -ne 0 ] ;then + error_out "rados ls" "$rados_err" +fi +sort -u "$rados_out" >"$temp_file" +mv -f "$temp_file" "$rados_out" + +echo "running 'radosgw-admin bucket radoslist' at $(date)" +radosgw-admin bucket radoslist >"$rgwadmin_out" 2>"$rgwadmin_err" +if [ "$?" -ne 0 ] ;then + error_out "radosgw-admin radoslist" "$rgwadmin_err" +fi +sort -u "$rgwadmin_out" >"$temp_file" +mv -f "$temp_file" "$rgwadmin_out" + +echo "computing delta at $(date)" +ceph-diff-sorted "$rados_out" "$rgwadmin_out" | grep "^<" | sed 's/^< *//' >"$delta_out" +# use PIPESTATUS to get at exit status of first process in above pipe; +# 0 means same, 1 means different, >1 means error +if [ "${PIPESTATUS[0]}" -gt 1 ] ;then + error_out "ceph-diff-sorted" +fi + +found="$(wc -l < $delta_out)" +possible="$(wc -l < $rados_out)" +percentage=$(expr 100 \* $found / $possible) + +echo "$found potential orphans found out of a possible $possible (${percentage}%)." +echo "The results can be found in ${delta_out}." +echo " Intermediate files: ${rados_out} and ${rgwadmin_out}" +echo "***" +echo "*** WARNING: This is EXPERIMENTAL code and the results should be used" +echo "*** only with CAUTION!" +echo "***" +echo "Done at $(date)." diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc index e25ab8a92619d..ad7808352e787 100644 --- a/src/rgw/rgw_admin.cc +++ b/src/rgw/rgw_admin.cc @@ -108,6 +108,7 @@ void usage() cout << " bucket rewrite rewrite all objects in the specified bucket\n"; cout << " bucket sync disable disable bucket sync\n"; cout << " bucket sync enable enable bucket sync\n"; + cout << " bucket radoslist list rados objects backing bucket's objects\n"; cout << " bi get retrieve bucket index object entries\n"; cout << " bi put store bucket index object entries\n"; cout << " bi list list raw bucket index entries\n"; @@ -413,6 +414,8 @@ enum { OPT_BUCKET_RM, OPT_BUCKET_REWRITE, OPT_BUCKET_RESHARD, + OPT_BUCKET_CHOWN, + OPT_BUCKET_RADOS_LIST, OPT_POLICY, OPT_POOL_ADD, OPT_POOL_RM, @@ -686,6 +689,8 @@ static int get_cmd(const char *cmd, const char *prev_cmd, const char *prev_prev_ return OPT_BUCKET_RESHARD; if (strcmp(cmd, "check") == 0) return OPT_BUCKET_CHECK; + if (strcmp(cmd, "radoslist") == 0) + return OPT_BUCKET_RADOS_LIST; if (strcmp(cmd, "sync") == 0) { *need_more = true; return 0; @@ -1187,8 +1192,12 @@ public: } }; -static int init_bucket(const string& tenant_name, const string& bucket_name, const string& bucket_id, - RGWBucketInfo& bucket_info, rgw_bucket& bucket, map *pattrs = nullptr) +static int init_bucket(const string& tenant_name, + const string& bucket_name, + const string& bucket_id, + RGWBucketInfo& bucket_info, + rgw_bucket& bucket, + map *pattrs = nullptr) { if (!bucket_name.empty()) { auto obj_ctx = store->svc.sysobj->init_obj_ctx(); @@ -5593,6 +5602,29 @@ int main(int argc, const char **argv) } /* have bucket_name */ } /* OPT_BUCKETS_LIST */ + if (opt_cmd == OPT_BUCKET_RADOS_LIST) { + RGWRadosList lister(store, + max_concurrent_ios, orphan_stale_secs, tenant); + if (bucket_name.empty()) { + ret = lister.run(); + } else { + ret = lister.run(bucket_name); + } + + if (ret < 0) { + std::cerr << + "ERROR: bucket radoslist failed to finish before " << + "encountering error: " << cpp_strerror(-ret) << std::endl; + std::cerr << "************************************" + "************************************" << std::endl; + std::cerr << "WARNING: THE RESULTS ARE NOT RELIABLE AND SHOULD NOT " << + "BE USED IN DELETING ORPHANS" << std::endl; + std::cerr << "************************************" + "************************************" << std::endl; + return -ret; + } + } + if (opt_cmd == OPT_BUCKET_STATS) { if (bucket_name.empty() && !bucket_id.empty()) { rgw_bucket bucket; diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h index 56454460a51c2..e76b12581f96c 100644 --- a/src/rgw/rgw_op.h +++ b/src/rgw/rgw_op.h @@ -1749,6 +1749,13 @@ public: struct RGWMultipartUploadEntry { rgw_bucket_dir_entry obj; RGWMPObj mp; + + friend std::ostream& operator<<(std::ostream& out, + const RGWMultipartUploadEntry& e) { + constexpr char quote = '"'; + return out << "RGWMultipartUploadEntry{ obj.key=" << + quote << e.obj.key << quote << " mp=" << e.mp << " }"; + } }; class RGWListBucketMultiparts : public RGWOp { diff --git a/src/rgw/rgw_orphan.cc b/src/rgw/rgw_orphan.cc index fc6fd84abd779..054483bcdc885 100644 --- a/src/rgw/rgw_orphan.cc +++ b/src/rgw/rgw_orphan.cc @@ -9,6 +9,8 @@ #include "common/errno.h" #include "rgw_rados.h" +#include "rgw_op.h" +#include "rgw_multi.h" #include "rgw_orphan.h" #include "rgw_zone.h" #include "rgw_bucket.h" @@ -57,6 +59,30 @@ static string obj_fingerprint(const string& oid, const char *force_ns = NULL) return s.substr(0, i + 1); } + +static string obj_force_ns(const string& oid, const char *force_ns) +{ + ssize_t pos = oid.find('_'); + if (pos < 0) { + cerr << "ERROR: object does not have a bucket marker: " << oid << std::endl; + } + + string obj_marker = oid.substr(0, pos); + + rgw_obj_key obj; + rgw_obj_key::parse_raw_oid(oid.substr(pos + 1), &obj); + + if (obj.ns.empty()) { + return oid; + } + + obj.set_ns(force_ns); + + string s = obj_marker + obj.get_oid(); + + return s; +} + int RGWOrphanStore::read_job(const string& job_name, RGWOrphanSearchState & state) { set keys; @@ -920,3 +946,610 @@ int RGWOrphanSearch::finish() return r; } + + +int RGWRadosList::handle_stat_result(RGWRados::Object::Stat::Result& result, + std::set& obj_oids) +{ + obj_oids.clear(); + + rgw_bucket& bucket = result.obj.bucket; + + ldout(store->ctx(), 20) << "RGWRadosList::" << __func__ << + " bucket=" << bucket << ", has_manifest=" << result.has_manifest << + dendl; + + // iterator to store result of dlo/slo attribute find + decltype(result.attrs)::iterator attr_it = result.attrs.end(); + const std::string oid = bucket.marker + "_" + result.obj.get_oid(); + ldout(store->ctx(), 20) << "radoslist processing object=\"" << + oid << "\"" << dendl; + if (visited_oids.find(oid) != visited_oids.end()) { + // apparently we hit a loop; don't continue with this oid + ldout(store->ctx(), 15) << + "radoslist stopped loop at already visited object=\"" << + oid << "\"" << dendl; + return 0; + } + + if (!result.has_manifest) { + /* a very very old object, or part of a multipart upload during upload */ + obj_oids.insert(oid); + + /* + * multipart parts don't have manifest on them, it's in the meta + * object; we'll process them in + * RGWRadosList::do_incomplete_multipart + */ + } else if ((attr_it = result.attrs.find(RGW_ATTR_USER_MANIFEST)) != + result.attrs.end()) { + // *** handle DLO object *** + + obj_oids.insert(oid); + visited_oids.insert(oid); // prevent dlo loops + ldout(store->ctx(), 15) << "radoslist added to visited list DLO=\"" << + oid << "\"" << dendl; + + char* prefix_path_c = attr_it->second.c_str(); + const std::string& prefix_path = prefix_path_c; + + const size_t sep_pos = prefix_path.find('/'); + if (string::npos == sep_pos) { + return -EINVAL; + } + + const std::string bucket_name = prefix_path.substr(0, sep_pos); + const std::string prefix = prefix_path.substr(sep_pos + 1); + + add_bucket_prefix(bucket_name, prefix); + ldout(store->ctx(), 25) << "radoslist DLO oid=\"" << oid << + "\" added bucket=\"" << bucket_name << "\" prefix=\"" << + prefix << "\" to process list" << dendl; + } else if ((attr_it = result.attrs.find(RGW_ATTR_SLO_MANIFEST)) != + result.attrs.end()) { + // *** handle SLO object *** + + obj_oids.insert(oid); + visited_oids.insert(oid); // prevent slo loops + ldout(store->ctx(), 15) << "radoslist added to visited list SLO=\"" << + oid << "\"" << dendl; + + RGWSLOInfo slo_info; + bufferlist::const_iterator bliter = attr_it->second.begin(); + try { + ::decode(slo_info, bliter); + } catch (buffer::error& err) { + ldout(store->ctx(), 0) << + "ERROR: failed to decode slo manifest for " << oid << dendl; + return -EIO; + } + + for (const auto& iter : slo_info.entries) { + const string& path_str = iter.path; + + const size_t sep_pos = path_str.find('/', 1 /* skip initial slash */); + if (string::npos == sep_pos) { + return -EINVAL; + } + + std::string bucket_name; + std::string obj_name; + + bucket_name = url_decode(path_str.substr(1, sep_pos - 1)); + obj_name = url_decode(path_str.substr(sep_pos + 1)); + + const rgw_obj_key obj_key(obj_name); + add_bucket_filter(bucket_name, obj_key); + ldout(store->ctx(), 25) << "radoslist SLO oid=\"" << oid << + "\" added bucket=\"" << bucket_name << "\" obj_key=\"" << + obj_key << "\" to process list" << dendl; + } + } else { + RGWObjManifest& manifest = result.manifest; + + // in multipart, the head object contains no data and just has the + // manifest AND empty objects have no manifest, but they're + // realized as empty rados objects + if (0 == manifest.get_max_head_size() || + manifest.obj_begin() == manifest.obj_end()) { + obj_oids.insert(oid); + // first_insert = true; + } + + RGWObjManifest::obj_iterator miter; + for (miter = manifest.obj_begin(); miter != manifest.obj_end(); ++miter) { + const rgw_raw_obj& loc = miter.get_location().get_raw_obj(store); + string s = loc.oid; + obj_oids.insert(s); + } + } + + return 0; +} // RGWRadosList::handle_stat_result + +int RGWRadosList::pop_and_handle_stat_op( + RGWObjectCtx& obj_ctx, + std::deque& ops) +{ + std::set obj_oids; + RGWRados::Object::Stat& front_op = ops.front(); + + int ret = front_op.wait(); + if (ret < 0) { + if (ret != -ENOENT) { + lderr(store->ctx()) << "ERROR: stat_async() returned error: " << + cpp_strerror(-ret) << dendl; + } + goto done; + } + + ret = handle_stat_result(front_op.result, obj_oids); + if (ret < 0) { + lderr(store->ctx()) << "ERROR: handle_stat_result() returned error: " << + cpp_strerror(-ret) << dendl; + } + + // output results + for (const auto& o : obj_oids) { + std::cout << o << std::endl; + } + +done: + + // invalidate object context for this object to avoid memory leak + // (see pr https://github.com/ceph/ceph/pull/30174) + obj_ctx.invalidate(front_op.result.obj); + + ops.pop_front(); + return ret; +} + + +#if 0 // code that may be the basis for expansion +int RGWRadosList::build_buckets_instance_index() +{ + void *handle; + int max = 1000; + string section = "bucket.instance"; + int ret = store->meta_mgr->list_keys_init(section, &handle); + if (ret < 0) { + lderr(store->ctx()) << "ERROR: can't get key: " << cpp_strerror(-ret) << dendl; + return ret; + } + + map > instances; + + bool truncated; + + RGWObjectCtx obj_ctx(store); + + int count = 0; + uint64_t total = 0; + + do { + list keys; + ret = store->meta_mgr->list_keys_next(handle, max, keys, &truncated); + if (ret < 0) { + lderr(store->ctx()) << "ERROR: lists_keys_next(): " << cpp_strerror(-ret) << dendl; + return ret; + } + + for (list::iterator iter = keys.begin(); iter != keys.end(); ++iter) { + ++total; + ldout(store->ctx(), 10) << "bucket_instance=" << *iter << " total=" << total << dendl; + int shard = orphan_shard(*iter); + instances[shard].push_back(*iter); + + if (++count >= COUNT_BEFORE_FLUSH) { + ret = log_oids(buckets_instance_index, instances); + if (ret < 0) { + lderr(store->ctx()) << __func__ << ": ERROR: log_oids() returned ret=" << ret << dendl; + return ret; + } + count = 0; + instances.clear(); + } + } + } while (truncated); + + ret = log_oids(buckets_instance_index, instances); + if (ret < 0) { + lderr(store->ctx()) << __func__ << ": ERROR: log_oids() returned ret=" << ret << dendl; + return ret; + } + store->meta_mgr->list_keys_complete(handle); + + return 0; +} +#endif + + +int RGWRadosList::process_bucket( + const std::string& bucket_instance_id, + const std::string& prefix, + const std::set& entries_filter) +{ + ldout(store->ctx(), 10) << "RGWRadosList::" << __func__ << + " bucket_instance_id=" << bucket_instance_id << + ", prefix=" << prefix << + ", entries_filter.size=" << entries_filter.size() << dendl; + + RGWBucketInfo bucket_info; + RGWSysObjectCtx sys_obj_ctx = store->svc.sysobj->init_obj_ctx(); + int ret = store->get_bucket_instance_info(sys_obj_ctx, bucket_instance_id, + bucket_info, nullptr, nullptr); + if (ret < 0) { + if (ret == -ENOENT) { + // probably raced with bucket removal + return 0; + } + lderr(store->ctx()) << __func__ << + ": ERROR: RGWRados::get_bucket_instance_info() returned ret=" << + ret << dendl; + return ret; + } + + RGWRados::Bucket target(store, bucket_info); + RGWRados::Bucket::List list_op(&target); + + std::string marker; + list_op.params.marker = rgw_obj_key(marker); + list_op.params.list_versions = true; + list_op.params.enforce_ns = false; + list_op.params.allow_unordered = false; + list_op.params.prefix = prefix; + + bool truncated; + + std::deque stat_ops; + std::string prev_versioned_key_name = ""; + + RGWObjectCtx obj_ctx(store); + + do { + std::vector result; + + constexpr int64_t LIST_OBJS_MAX_ENTRIES = 100; + ret = list_op.list_objects(LIST_OBJS_MAX_ENTRIES, &result, + NULL, &truncated); + if (ret == -ENOENT) { + // race with bucket delete? + ret = 0; + break; + } else if (ret < 0) { + std::cerr << "ERROR: store->list_objects(): " << cpp_strerror(-ret) << + std::endl; + return ret; + } + + for (std::vector::iterator iter = result.begin(); + iter != result.end(); + ++iter) { + rgw_bucket_dir_entry& entry = *iter; + + if (entry.key.instance.empty()) { + ldout(store->ctx(), 20) << "obj entry: " << entry.key.name << dendl; + } else { + ldout(store->ctx(), 20) << "obj entry: " << entry.key.name << + " [" << entry.key.instance << "]" << dendl; + } + + ldout(store->ctx(), 20) << __func__ << ": entry.key.name=" << + entry.key.name << " entry.key.instance=" << entry.key.instance << + dendl; + + // ignore entries that are not in the filter if there is a filter + if (!entries_filter.empty() && + entries_filter.find(entry.key) == entries_filter.cend()) { + continue; + } + + // we need to do this in two cases below, so use a lambda + auto do_stat_key = + [&](const rgw_obj_key& key) -> int { + int ret; + + rgw_obj obj(bucket_info.bucket, key); + + RGWRados::Object op_target(store, bucket_info, obj_ctx, obj); + + stat_ops.push_back(RGWRados::Object::Stat(&op_target)); + RGWRados::Object::Stat& op = stat_ops.back(); + + ret = op.stat_async(); + if (ret < 0) { + lderr(store->ctx()) << "ERROR: stat_async() returned error: " << + cpp_strerror(-ret) << dendl; + return ret; + } + + if (stat_ops.size() >= max_concurrent_ios) { + ret = pop_and_handle_stat_op(obj_ctx, stat_ops); + if (ret < 0) { + if (ret != -ENOENT) { + lderr(store->ctx()) << + "ERROR: pop_and_handle_stat_op() returned error: " << + cpp_strerror(-ret) << dendl; + } + + // clear error, so we'll continue processing directory + ret = 0; + } + } + + return ret; + }; // do_stat_key lambda + + // for versioned objects, make sure the head object is handled + // as well by ignoring the instance identifier + if (!entry.key.instance.empty() && + entry.key.name != prev_versioned_key_name) { + // don't do the same key twice; even though out bucket index + // listing allows unordered, since all versions of an object + // use the same bucket index key, they'll all end up together + // and sorted + prev_versioned_key_name = entry.key.name; + + rgw_obj_key uninstanced(entry.key.name); + + ret = do_stat_key(uninstanced); + if (ret < 0) { + return ret; + } + } + + ret = do_stat_key(entry.key); + if (ret < 0) { + return ret; + } + } // for iter loop + } while (truncated); + + while (!stat_ops.empty()) { + ret = pop_and_handle_stat_op(obj_ctx, stat_ops); + if (ret < 0) { + if (ret != -ENOENT) { + lderr(store->ctx()) << "ERROR: stat_async() returned error: " << + cpp_strerror(-ret) << dendl; + } + } + } + + return 0; +} + + +int RGWRadosList::run() +{ + int ret; + void* handle = nullptr; + + ret = store->meta_mgr->list_keys_init("bucket", &handle); + if (ret < 0) { + lderr(store->ctx()) << "RGWRadosList::" << __func__ << + " ERROR: list_keys_init returned " << + cpp_strerror(-ret) << dendl; + return ret; + } + + const int max_keys = 1000; + bool truncated = true; + + do { + std::list buckets; + ret = store->meta_mgr->list_keys_next(handle, max_keys, buckets, &truncated); + + for (std::string& bucket_id : buckets) { + ret = run(bucket_id); + if (ret == -ENOENT) { + continue; + } else if (ret < 0) { + return ret; + } + } + } while (truncated); + + return 0; +} // RGWRadosList::run() + + +int RGWRadosList::run(const std::string& start_bucket_name) +{ + RGWSysObjectCtx sys_obj_ctx = store->svc.sysobj->init_obj_ctx(); + RGWObjectCtx obj_ctx(store); + int ret; + + add_bucket_entire(start_bucket_name); + + while (! bucket_process_map.empty()) { + // pop item from map and capture its key data + auto front = bucket_process_map.begin(); + std::string bucket_name = front->first; + process_t process; + std::swap(process, front->second); + bucket_process_map.erase(front); + + RGWBucketInfo bucket_info; + ret = store->get_bucket_info(sys_obj_ctx, + tenant_name, bucket_name, bucket_info, + nullptr, nullptr); + if (ret == -ENOENT) { + std::cerr << "WARNING: bucket " << bucket_name << + " does not exist; could it have been deleted very recently?" << + std::endl; + continue; + } else if (ret < 0) { + std::cerr << "ERROR: could not get info for bucket " << bucket_name << + " -- " << cpp_strerror(-ret) << std::endl; + return ret; + } + + const std::string bucket_id = bucket_info.bucket.get_key(); + + static const std::set empty_filter; + static const std::string empty_prefix; + + auto do_process_bucket = + [&bucket_id, this] + (const std::string& prefix, + const std::set& entries_filter) -> int { + int ret = process_bucket(bucket_id, prefix, entries_filter); + if (ret == -ENOENT) { + // bucket deletion race? + return 0; + } if (ret < 0) { + lderr(store->ctx()) << "RGWRadosList::" << __func__ << + ": ERROR: process_bucket(); bucket_id=" << + bucket_id << " returned ret=" << ret << dendl; + } + + return ret; + }; + + // either process the whole bucket *or* process the filters and/or + // the prefixes + if (process.entire_container) { + ret = do_process_bucket(empty_prefix, empty_filter); + if (ret < 0) { + return ret; + } + } else { + if (! process.filter_keys.empty()) { + ret = do_process_bucket(empty_prefix, process.filter_keys); + if (ret < 0) { + return ret; + } + } + for (const auto& p : process.prefixes) { + ret = do_process_bucket(p, empty_filter); + if (ret < 0) { + return ret; + } + } + } + } // while (! bucket_process_map.empty()) + + // now handle incomplete multipart uploads by going back to the + // initial bucket + + RGWBucketInfo bucket_info; + ret = store->get_bucket_info(sys_obj_ctx, + tenant_name, start_bucket_name, bucket_info, + nullptr, nullptr); + if (ret == -ENOENT) { + // bucket deletion race? + return 0; + } else if (ret < 0) { + lderr(store->ctx()) << "RGWRadosList::" << __func__ << + ": ERROR: get_bucket_info returned ret=" << ret << dendl; + return ret; + } + + ret = do_incomplete_multipart(store, bucket_info); + if (ret < 0) { + lderr(store->ctx()) << "RGWRadosList::" << __func__ << + ": ERROR: do_incomplete_multipart returned ret=" << ret << dendl; + return ret; + } + + return 0; +} // RGWRadosList::run(string) + + +int RGWRadosList::do_incomplete_multipart( + RGWRados* store, + RGWBucketInfo& bucket_info) +{ + constexpr int max_uploads = 1000; + constexpr int max_parts = 1000; + static const std::string mp_ns = RGW_OBJ_NS_MULTIPART; + static MultipartMetaFilter mp_filter; + + int ret; + + RGWRados::Bucket target(store, bucket_info); + RGWRados::Bucket::List list_op(&target); + + bool is_listing_truncated; + std::string empty_string; + RGWMultipartUploadEntry next_uploads_marker; + + do { + RGWMPObj uploads_marker = next_uploads_marker.mp; + const std::string& marker_meta = uploads_marker.get_meta(); + list_op.params.marker = marker_meta; + list_op.params.ns = mp_ns; + list_op.params.filter = &mp_filter; + + // use empty strings for list_op.params.{prefix,delim} + + std::vector objs; + std::map common_prefixes; + ret = list_op.list_objects(max_uploads, &objs, &common_prefixes, + &is_listing_truncated); + if (ret == -ENOENT) { + // could bucket have been removed while this is running? + ldout(store->ctx(), 20) << "RGWRadosList::" << __func__ << + ": WARNING: call to list_objects of multipart namespace got ENOENT; " + "assuming bucket removal race" << dendl; + break; + } else if (ret < 0) { + lderr(store->ctx()) << "RGWRadosList::" << __func__ << + ": ERROR: list_objects op returned ret=" << ret << dendl; + return ret; + } + + if (!objs.empty()) { + std::vector uploads; + RGWMultipartUploadEntry entry; + for (const rgw_bucket_dir_entry& obj : objs) { + const rgw_obj_key& key = obj.key; + if (!entry.mp.from_meta(key.name)) { + // we only want the meta objects, so skip all the components + continue; + } + entry.obj = obj; + uploads.push_back(entry); + ldout(store->ctx(), 20) << "RGWRadosList::" << __func__ << + " processing incomplete multipart entry " << + entry << dendl; + } + next_uploads_marker = entry; + + // now process the uploads vector + + int parts_marker = 0; + bool is_parts_truncated = false; + do { + map parts; + + for (const auto& upload : uploads) { + const RGWMPObj& mp = upload.mp; + ret = list_multipart_parts(store, bucket_info, store->ctx(), + mp.get_upload_id(), mp.get_meta(), + max_parts, + parts_marker, parts, NULL, &is_parts_truncated); + if (ret == -ENOENT) { + continue; + } else if (ret < 0) { + lderr(store->ctx()) << "RGWRadosList::" << __func__ << + ": ERROR: list_multipart_parts returned ret=" << ret << dendl; + return ret; + } + + for (auto& p : parts) { + RGWObjManifest& manifest = p.second.manifest; + for (auto obj_it = manifest.obj_begin(); + obj_it != manifest.obj_end(); + ++obj_it) { + const rgw_raw_obj& loc = obj_it.get_location().get_raw_obj(store); + std::cout << loc.oid << std::endl; + } + } + } + } while (is_parts_truncated); + } // if objs not empty + } while(is_listing_truncated); + + return 0; +} // RGWRadosList::do_incomplete_multipart diff --git a/src/rgw/rgw_orphan.h b/src/rgw/rgw_orphan.h index 7d808eeda65d3..fe737b4f52f13 100644 --- a/src/rgw/rgw_orphan.h +++ b/src/rgw/rgw_orphan.h @@ -210,5 +210,81 @@ public: }; +class RGWRadosList { + + /* + * process_t describes how to process a irectory, we will either + * process the whole thing (entire_container == true) or a portion + * of it (entire_container == false). When we only process a + * portion, we will list the specific keys and/or specific lexical + * prefixes. + */ + struct process_t { + bool entire_container; + std::set filter_keys; + std::set prefixes; + + process_t() : + entire_container(false) + {} + }; + + std::map bucket_process_map; + std::set visited_oids; + + void add_bucket_entire(const std::string& bucket_name) { + auto p = bucket_process_map.emplace(std::make_pair(bucket_name, + process_t())); + p.first->second.entire_container = true; + } + + void add_bucket_prefix(const std::string& bucket_name, + const std::string& prefix) { + auto p = bucket_process_map.emplace(std::make_pair(bucket_name, + process_t())); + p.first->second.prefixes.insert(prefix); + } + + void add_bucket_filter(const std::string& bucket_name, + const rgw_obj_key& obj_key) { + auto p = bucket_process_map.emplace(std::make_pair(bucket_name, + process_t())); + p.first->second.filter_keys.insert(obj_key); + } + + RGWRados *store; + + uint16_t max_concurrent_ios; + uint64_t stale_secs; + std::string tenant_name; + + int handle_stat_result(RGWRados::Object::Stat::Result& result, + std::set& obj_oids); + int pop_and_handle_stat_op(RGWObjectCtx& obj_ctx, + std::deque& ops); + +public: + + RGWRadosList(RGWRados *_store, + int _max_ios, + uint64_t _stale_secs, + const std::string& _tenant_name) : + store(_store), + max_concurrent_ios(_max_ios), + stale_secs(_stale_secs), + tenant_name(_tenant_name) + {} + + int process_bucket(const std::string& bucket_instance_id, + const std::string& prefix, + const std::set& entries_filter); + + int do_incomplete_multipart(RGWRados* store, RGWBucketInfo& bucket_info); + + int build_linked_oids_index(); + + int run(const std::string& bucket_id); + int run(); +}; // class RGWRadosList #endif diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h index 0c7bf5718fba6..2250c672180a5 100644 --- a/src/rgw/rgw_rados.h +++ b/src/rgw/rgw_rados.h @@ -5,6 +5,7 @@ #define CEPH_RGWRADOS_H #include +#include #include "include/rados/librados.hpp" #include "include/Context.h" @@ -726,18 +727,18 @@ public: void seek(uint64_t ofs); void operator++(); - bool operator==(const obj_iterator& rhs) { + bool operator==(const obj_iterator& rhs) const { return (ofs == rhs.ofs); } - bool operator!=(const obj_iterator& rhs) { + bool operator!=(const obj_iterator& rhs) const { return (ofs != rhs.ofs); } - const rgw_obj_select& get_location() { + const rgw_obj_select& get_location() const { return location; } /* start of current stripe */ - uint64_t get_stripe_ofs() { + uint64_t get_stripe_ofs() const { if (manifest->explicit_objs) { return explicit_iter->first; } @@ -755,7 +756,7 @@ public: } /* current stripe size */ - uint64_t get_stripe_size() { + uint64_t get_stripe_size() const { if (manifest->explicit_objs) { return explicit_iter->second.size; } @@ -763,7 +764,7 @@ public: } /* offset where data starts within current stripe */ - uint64_t location_ofs() { + uint64_t location_ofs() const { if (manifest->explicit_objs) { return explicit_iter->second.loc_ofs; } @@ -2539,7 +2540,7 @@ public: const string& get_key() const { return oid; } - bool from_meta(string& meta) { + bool from_meta(const string& meta) { int end_pos = meta.rfind('.'); // search for ".meta" if (end_pos < 0) return false; @@ -2557,6 +2558,10 @@ public: meta = ""; upload_id = ""; } + friend std::ostream& operator<<(std::ostream& out, const RGWMPObj& obj) { + return out << "RGWMPObj:{ prefix=" << std::quoted(obj.prefix) << + ", meta=" << std::quoted(obj.meta) << " }"; + } }; // class RGWMPObj diff --git a/src/test/cli/radosgw-admin/help.t b/src/test/cli/radosgw-admin/help.t index da3425c3d884d..026e32ce05690 100644 --- a/src/test/cli/radosgw-admin/help.t +++ b/src/test/cli/radosgw-admin/help.t @@ -29,6 +29,7 @@ bucket rewrite rewrite all objects in the specified bucket bucket sync disable disable bucket sync bucket sync enable enable bucket sync + bucket radoslist list rados objects backing bucket's objects bi get retrieve bucket index object entries bi put store bucket index object entries bi list list raw bucket index entries diff --git a/src/tools/CMakeLists.txt b/src/tools/CMakeLists.txt index 678ee66cc2d61..fc8539ff7ab07 100644 --- a/src/tools/CMakeLists.txt +++ b/src/tools/CMakeLists.txt @@ -88,6 +88,10 @@ add_executable(osdmaptool ${osdomaptool_srcs}) target_link_libraries(osdmaptool global) install(TARGETS osdmaptool DESTINATION bin) +set(ceph-diff-sorted_srcs ceph-diff-sorted.cc) +add_executable(ceph-diff-sorted ${ceph-diff-sorted_srcs}) +install(TARGETS ceph-diff-sorted DESTINATION bin) + if(WITH_TESTS) set(ceph_psim_srcs psim.cc) add_executable(ceph_psim ${ceph_psim_srcs}) diff --git a/src/tools/ceph-diff-sorted.cc b/src/tools/ceph-diff-sorted.cc new file mode 100644 index 0000000000000..f8e4c28e64789 --- /dev/null +++ b/src/tools/ceph-diff-sorted.cc @@ -0,0 +1,173 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +/* + * diffsorted -- a utility to compute a line-by-line diff on two + * sorted input files + * + * Copyright © 2019 Red Hat + * + * Author: J. Eric Ivancich + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. + */ + + +/* + * SUMMARY + * + * The `diffsorted` utility does a line-by-line diff on two sorted text + * files and indicating lines that are in one file but not the other + * using diff-style notation (although line numbers are not indicated). + * + * USAGE + * + * rgw-diff-sorted file1.txt file2.txt + * + * NOTES + * + * Each files should have its lines in sorted order and should have no + * empty lines. + * + * A potential input file can be sorted using the `sort` utility provided + * that LANG=C to insure byte lexical order. For example: + * + * LANG=C sort unsorted.txt >sorted.txt + * + * or: + * + * export LANG=C + * sort unsorted.txt >sorted.txt + * + * EXIT STATUS + * + * 0 : files same + * 1 : files different + * 2 : usage problem (e.g., wrong number of command-line arguments) + * 3 : problem opening input file + * 4 : bad file content (e.g., unsorted order or empty lines) + */ + + +#include +#include + + +struct FileOfLines { + const char* filename; + std::ifstream input; + std::string this_line, prev_line; + bool next_eof; + bool is_eof; + + FileOfLines(const char* _filename) : + filename(_filename), + input(filename), + next_eof(false), + is_eof(false) + { } + + void dump(const std::string& prefix) { + do { + std::cout << prefix << this_line << std::endl; + advance(); + } while (!eof()); + } + + bool eof() const { + return is_eof; + } + + bool good() const { + return input.good(); + } + + void advance() { + if (next_eof) { + is_eof = true; + return; + } + + prev_line = this_line; + std::getline(input, this_line); + if (this_line.empty()) { + if (!input.eof()) { + std::cerr << "Error: " << filename << " has an empty line." << + std::endl; + exit(4); + } + is_eof = true; + return; + } else if (input.eof()) { + next_eof = true; + } + + if (this_line < prev_line) { + std::cerr << "Error: " << filename << " is not in sorted order; \"" << + this_line << "\" follows \"" << prev_line << "\"." << std::endl; + exit(4); + } + } + + const std::string line() const { + return this_line; + } +}; + +int main(int argc, const char* argv[]) { + if (argc != 3) { + std::cerr << "Usage: " << argv[0] << " " << std::endl; + exit(2); + } + + FileOfLines input1(argv[1]); + if (!input1.good()) { + std::cerr << "Error opening " << argv[1] << + "." << std::endl; + exit(3); + } + + FileOfLines input2(argv[2]); + if (!input2.good()) { + std::cerr << "Error opening " << argv[2] << + "." << std::endl; + exit(3); + } + + bool files_same = true; + + input1.advance(); + input2.advance(); + + while (!input1.eof() && !input2.eof()) { + if (input1.line() == input2.line()) { + input1.advance(); + input2.advance(); + } else if (input1.line() < input2.line()) { + files_same = false; + std::cout << "< " << input1.line() << std::endl; + input1.advance(); + } else { + files_same = false; + std::cout << "> " << input2.line() << std::endl; + input2.advance(); + } + } + + if (!input1.eof()) { + files_same = false; + input1.dump("< "); + } else if (!input2.eof()) { + files_same = false; + input2.dump("> "); + } + + if (files_same) { + exit(0); + } else { + exit(1); + } +} -- 2.39.5