rgwlc: properly accumulate in RGWLC::list_lc_progress()

Also track shard/index progress, clearing marker when the shard
is advanced.

Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
(cherry picked from commit ed98de9fc45e4d28ba01b389fe2b2f225a4b8606)

diff --git a/src/cls/rgw/cls_rgw_client.cc b/src/cls/rgw/cls_rgw_client.cc
index 270f9aa2e2bdb1b93b657dcbd6cd3b3d57381481..182897ab4323d8b045d6b9e19655d80c14019143 100644 (file)
--- a/src/cls/rgw/cls_rgw_client.cc
+++ b/src/cls/rgw/cls_rgw_client.cc
@@ -939,7 +939,7 @@ int cls_rgw_lc_list(IoCtx& io_ctx, const string& oid,
            [](const cls_rgw_lc_entry& a, const cls_rgw_lc_entry& b)
              { return a.bucket < b.bucket; });
   entries = std::move(ret.entries);
- return r;
+  return r;
 }
 
 void cls_rgw_reshard_add(librados::ObjectWriteOperation& op, const cls_rgw_reshard_entry& entry)

diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index 84239157c7cfbdbd34da6a97acbc055e25abd446..6831a78ed23e94aacb787754d5f6fa31ddd31f4c 100644 (file)
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -7226,13 +7226,14 @@ next:
     formatter->open_array_section("lifecycle_list");
     vector<cls_rgw_lc_entry> bucket_lc_map;
     string marker;
+    int index{0};
 #define MAX_LC_LIST_ENTRIES 100
     if (max_entries < 0) {
       max_entries = MAX_LC_LIST_ENTRIES;
     }
     do {
       int ret = store->getRados()->list_lc_progress(marker, max_entries,
-                                                   bucket_lc_map);
+                                                   bucket_lc_map, index);
       if (ret < 0) {
         cerr << "ERROR: failed to list objs: " << cpp_strerror(-ret)
             << std::endl;
@@ -7252,7 +7253,6 @@ next:
         formatter->dump_string("status", lc_status);
         formatter->close_section(); // objs
         formatter->flush(cout);
-        marker = entry.bucket;
       }
     } while (!bucket_lc_map.empty());
 

diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc
index e57b82a6802494751099ecc4e54f83190a230cb7..d2b3202469203cb8a553f932bb75f2c706460f2a 100644 (file)
--- a/src/rgw/rgw_lc.cc
+++ b/src/rgw/rgw_lc.cc
@@ -1542,14 +1542,16 @@ clean:
   } while (true);
 }
 
-int RGWLC::list_lc_progress(const string& marker, uint32_t max_entries,
-                           vector<cls_rgw_lc_entry>& progress_map)
+int RGWLC::list_lc_progress(string& marker, uint32_t max_entries,
+                           vector<cls_rgw_lc_entry>& progress_map,
+                           int& index)
 {
-  int index = 0;
-  for(; index <max_objs; index++) {
+  progress_map.clear();
+  for(; index < max_objs; index++, marker="") {
+    vector<cls_rgw_lc_entry> entries;
     int ret =
       cls_rgw_lc_list(store->getRados()->lc_pool_ctx, obj_names[index], marker,
-                     max_entries, progress_map);
+                     max_entries, entries);
     if (ret < 0) {
       if (ret == -ENOENT) {
         ldpp_dout(this, 10) << __func__ << "() ignoring unfound lc object="
@@ -1559,6 +1561,15 @@ int RGWLC::list_lc_progress(const string& marker, uint32_t max_entries,
         return ret;
       }
     }
+    progress_map.reserve(progress_map.size() + entries.size());
+    progress_map.insert(progress_map.end(), entries.begin(), entries.end());
+
+    /* update index, marker tuple */
+    if (progress_map.size() > 0)
+      marker = progress_map.back().bucket;
+
+    if (progress_map.size() >= max_entries)
+      break;
   }
   return 0;
 }

diff --git a/src/rgw/rgw_lc.h b/src/rgw/rgw_lc.h
index 8cfc1d78f03b383b4ee4364157baa2e169aa4cd0..03209b7ea987d2ae58605a4cd10e767558a2ee5b 100644 (file)
--- a/src/rgw/rgw_lc.h
+++ b/src/rgw/rgw_lc.h
@@ -505,8 +505,8 @@ public:
   bool if_already_run_today(time_t start_date);
   bool expired_session(time_t started);
   time_t thread_stop_at();
-  int list_lc_progress(const string& marker, uint32_t max_entries,
-                      vector<cls_rgw_lc_entry>&);
+  int list_lc_progress(string& marker, uint32_t max_entries,
+                      vector<cls_rgw_lc_entry>&, int& index);
   int bucket_lc_prepare(int index, LCWorker* worker);
   int bucket_lc_process(string& shard_id, LCWorker* worker, time_t stop_at,
                        bool once);

diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index d193e1c8e34f35a2db1eb5001bf342a46187fe89..f6e9f9c304eca9457b1f61b391b737564cc7ac82 100644 (file)
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -8035,10 +8035,11 @@ int RGWRados::process_gc(bool expired_only)
   return gc->process(expired_only);
 }
 
-int RGWRados::list_lc_progress(const string& marker, uint32_t max_entries,
-                              vector<cls_rgw_lc_entry>& progress_map)
+int RGWRados::list_lc_progress(string& marker, uint32_t max_entries,
+                              vector<cls_rgw_lc_entry>& progress_map,
+                              int& index)
 {
-  return lc->list_lc_progress(marker, max_entries, progress_map);
+  return lc->list_lc_progress(marker, max_entries, progress_map, index);
 }
 
 int RGWRados::process_lc()

diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h
index 6ac58fc305792f95b08bdd10d1e4f7a42e04f746..99e7cbbfd39051833b3a4dccdc57810a0bb95b9b 100644 (file)
--- a/src/rgw/rgw_rados.h
+++ b/src/rgw/rgw_rados.h
@@ -1444,9 +1444,9 @@ public:
   int defer_gc(void *ctx, const RGWBucketInfo& bucket_info, const rgw_obj& obj, optional_yield y);
 
   int process_lc();
-  int list_lc_progress(const string& marker, uint32_t max_entries,
-                      vector<cls_rgw_lc_entry>& progress_map);
-  
+  int list_lc_progress(string& marker, uint32_t max_entries,
+                      vector<cls_rgw_lc_entry>& progress_map, int& index);
+
   int bucket_check_index(RGWBucketInfo& bucket_info,
                          map<RGWObjCategory, RGWStorageStats> *existing_stats,
                          map<RGWObjCategory, RGWStorageStats> *calculated_stats);