From ed98de9fc45e4d28ba01b389fe2b2f225a4b8606 Mon Sep 17 00:00:00 2001 From: Matt Benjamin Date: Fri, 24 Apr 2020 17:32:01 -0400 Subject: [PATCH] rgwlc: properly accumulate in RGWLC::list_lc_progress() Also track shard/index progress, clearing marker when the shard is advanced. Signed-off-by: Matt Benjamin --- src/cls/rgw/cls_rgw_client.cc | 2 +- src/rgw/rgw_admin.cc | 4 ++-- src/rgw/rgw_lc.cc | 21 ++++++++++++++++----- src/rgw/rgw_lc.h | 4 ++-- src/rgw/rgw_rados.cc | 7 ++++--- src/rgw/rgw_rados.h | 6 +++--- 6 files changed, 28 insertions(+), 16 deletions(-) diff --git a/src/cls/rgw/cls_rgw_client.cc b/src/cls/rgw/cls_rgw_client.cc index faf0b3675b5f2..95d6ffbc16a84 100644 --- a/src/cls/rgw/cls_rgw_client.cc +++ b/src/cls/rgw/cls_rgw_client.cc @@ -947,7 +947,7 @@ int cls_rgw_lc_list(IoCtx& io_ctx, const string& oid, [](const cls_rgw_lc_entry& a, const cls_rgw_lc_entry& b) { return a.bucket < b.bucket; }); entries = std::move(ret.entries); - return r; + return r; } void cls_rgw_reshard_add(librados::ObjectWriteOperation& op, const cls_rgw_reshard_entry& entry) diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc index 045f370464f98..d4c87e5002246 100644 --- a/src/rgw/rgw_admin.cc +++ b/src/rgw/rgw_admin.cc @@ -7185,13 +7185,14 @@ next: formatter->open_array_section("lifecycle_list"); vector bucket_lc_map; string marker; + int index{0}; #define MAX_LC_LIST_ENTRIES 100 if (max_entries < 0) { max_entries = MAX_LC_LIST_ENTRIES; } do { int ret = store->getRados()->list_lc_progress(marker, max_entries, - bucket_lc_map); + bucket_lc_map, index); if (ret < 0) { cerr << "ERROR: failed to list objs: " << cpp_strerror(-ret) << std::endl; @@ -7211,7 +7212,6 @@ next: formatter->dump_string("status", lc_status); formatter->close_section(); // objs formatter->flush(cout); - marker = entry.bucket; } } while (!bucket_lc_map.empty()); diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc index 36868068de0b4..ff8eb8e793767 100644 --- a/src/rgw/rgw_lc.cc +++ b/src/rgw/rgw_lc.cc @@ -1540,14 +1540,16 @@ clean: } while (true); } -int RGWLC::list_lc_progress(const string& marker, uint32_t max_entries, - vector& progress_map) +int RGWLC::list_lc_progress(string& marker, uint32_t max_entries, + vector& progress_map, + int& index) { - int index = 0; - for(; index entries; int ret = cls_rgw_lc_list(store->getRados()->lc_pool_ctx, obj_names[index], marker, - max_entries, progress_map); + max_entries, entries); if (ret < 0) { if (ret == -ENOENT) { ldpp_dout(this, 10) << __func__ << "() ignoring unfound lc object=" @@ -1557,6 +1559,15 @@ int RGWLC::list_lc_progress(const string& marker, uint32_t max_entries, return ret; } } + progress_map.reserve(progress_map.size() + entries.size()); + progress_map.insert(progress_map.end(), entries.begin(), entries.end()); + + /* update index, marker tuple */ + if (progress_map.size() > 0) + marker = progress_map.back().bucket; + + if (progress_map.size() >= max_entries) + break; } return 0; } diff --git a/src/rgw/rgw_lc.h b/src/rgw/rgw_lc.h index 8cfc1d78f03b3..03209b7ea987d 100644 --- a/src/rgw/rgw_lc.h +++ b/src/rgw/rgw_lc.h @@ -505,8 +505,8 @@ public: bool if_already_run_today(time_t start_date); bool expired_session(time_t started); time_t thread_stop_at(); - int list_lc_progress(const string& marker, uint32_t max_entries, - vector&); + int list_lc_progress(string& marker, uint32_t max_entries, + vector&, int& index); int bucket_lc_prepare(int index, LCWorker* worker); int bucket_lc_process(string& shard_id, LCWorker* worker, time_t stop_at, bool once); diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index 8c8a5af989859..0cc647038f2bc 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -8040,10 +8040,11 @@ int RGWRados::process_gc(bool expired_only) return gc->process(expired_only); } -int RGWRados::list_lc_progress(const string& marker, uint32_t max_entries, - vector& progress_map) +int RGWRados::list_lc_progress(string& marker, uint32_t max_entries, + vector& progress_map, + int& index) { - return lc->list_lc_progress(marker, max_entries, progress_map); + return lc->list_lc_progress(marker, max_entries, progress_map, index); } int RGWRados::process_lc() diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h index bcfb8739201d8..bbc3780bf0f6a 100644 --- a/src/rgw/rgw_rados.h +++ b/src/rgw/rgw_rados.h @@ -1444,9 +1444,9 @@ public: int defer_gc(void *ctx, const RGWBucketInfo& bucket_info, const rgw_obj& obj, optional_yield y); int process_lc(); - int list_lc_progress(const string& marker, uint32_t max_entries, - vector& progress_map); - + int list_lc_progress(string& marker, uint32_t max_entries, + vector& progress_map, int& index); + int bucket_check_index(RGWBucketInfo& bucket_info, map *existing_stats, map *calculated_stats); -- 2.39.5