rgwlc: properly accumulate in RGWLC::list_lc_progress()

Also track shard/index progress, clearing marker when the shard
is advanced.

Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>

diff --git a/src/cls/rgw/cls_rgw_client.cc b/src/cls/rgw/cls_rgw_client.cc
index faf0b3675b5f27241b46fef3ce6c036898471f7d..95d6ffbc16a84ee00590a6054a1ba350e6ff454d 100644 (file)
--- a/src/cls/rgw/cls_rgw_client.cc
+++ b/src/cls/rgw/cls_rgw_client.cc
@@ -947,7 +947,7 @@ int cls_rgw_lc_list(IoCtx& io_ctx, const string& oid,
            [](const cls_rgw_lc_entry& a, const cls_rgw_lc_entry& b)
              { return a.bucket < b.bucket; });
   entries = std::move(ret.entries);
- return r;
+  return r;
 }
 
 void cls_rgw_reshard_add(librados::ObjectWriteOperation& op, const cls_rgw_reshard_entry& entry)

diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index 045f370464f988d08081862bc15b1e23794441d4..d4c87e5002246c9d6374c94b1dcd2a133702eeae 100644 (file)
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -7185,13 +7185,14 @@ next:
     formatter->open_array_section("lifecycle_list");
     vector<cls_rgw_lc_entry> bucket_lc_map;
     string marker;
+    int index{0};
 #define MAX_LC_LIST_ENTRIES 100
     if (max_entries < 0) {
       max_entries = MAX_LC_LIST_ENTRIES;
     }
     do {
       int ret = store->getRados()->list_lc_progress(marker, max_entries,
-                                                   bucket_lc_map);
+                                                   bucket_lc_map, index);
       if (ret < 0) {
         cerr << "ERROR: failed to list objs: " << cpp_strerror(-ret)
             << std::endl;
@@ -7211,7 +7212,6 @@ next:
         formatter->dump_string("status", lc_status);
         formatter->close_section(); // objs
         formatter->flush(cout);
-        marker = entry.bucket;
       }
     } while (!bucket_lc_map.empty());
 

diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc
index 36868068de0b456ded0399099d735442b3ffb31a..ff8eb8e7937672c9b715646e32d33cfc7201001c 100644 (file)
--- a/src/rgw/rgw_lc.cc
+++ b/src/rgw/rgw_lc.cc
@@ -1540,14 +1540,16 @@ clean:
   } while (true);
 }
 
-int RGWLC::list_lc_progress(const string& marker, uint32_t max_entries,
-                           vector<cls_rgw_lc_entry>& progress_map)
+int RGWLC::list_lc_progress(string& marker, uint32_t max_entries,
+                           vector<cls_rgw_lc_entry>& progress_map,
+                           int& index)
 {
-  int index = 0;
-  for(; index <max_objs; index++) {
+  progress_map.clear();
+  for(; index < max_objs; index++, marker="") {
+    vector<cls_rgw_lc_entry> entries;
     int ret =
       cls_rgw_lc_list(store->getRados()->lc_pool_ctx, obj_names[index], marker,
-                     max_entries, progress_map);
+                     max_entries, entries);
     if (ret < 0) {
       if (ret == -ENOENT) {
         ldpp_dout(this, 10) << __func__ << "() ignoring unfound lc object="
@@ -1557,6 +1559,15 @@ int RGWLC::list_lc_progress(const string& marker, uint32_t max_entries,
         return ret;
       }
     }
+    progress_map.reserve(progress_map.size() + entries.size());
+    progress_map.insert(progress_map.end(), entries.begin(), entries.end());
+
+    /* update index, marker tuple */
+    if (progress_map.size() > 0)
+      marker = progress_map.back().bucket;
+
+    if (progress_map.size() >= max_entries)
+      break;
   }
   return 0;
 }

diff --git a/src/rgw/rgw_lc.h b/src/rgw/rgw_lc.h
index 8cfc1d78f03b383b4ee4364157baa2e169aa4cd0..03209b7ea987d2ae58605a4cd10e767558a2ee5b 100644 (file)
--- a/src/rgw/rgw_lc.h
+++ b/src/rgw/rgw_lc.h
@@ -505,8 +505,8 @@ public:
   bool if_already_run_today(time_t start_date);
   bool expired_session(time_t started);
   time_t thread_stop_at();
-  int list_lc_progress(const string& marker, uint32_t max_entries,
-                      vector<cls_rgw_lc_entry>&);
+  int list_lc_progress(string& marker, uint32_t max_entries,
+                      vector<cls_rgw_lc_entry>&, int& index);
   int bucket_lc_prepare(int index, LCWorker* worker);
   int bucket_lc_process(string& shard_id, LCWorker* worker, time_t stop_at,
                        bool once);

diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index 8c8a5af9898596993c8c64b71f4466036708e5ba..0cc647038f2bcf9eb1b000065abb57458c5d5d19 100644 (file)
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -8040,10 +8040,11 @@ int RGWRados::process_gc(bool expired_only)
   return gc->process(expired_only);
 }
 
-int RGWRados::list_lc_progress(const string& marker, uint32_t max_entries,
-                              vector<cls_rgw_lc_entry>& progress_map)
+int RGWRados::list_lc_progress(string& marker, uint32_t max_entries,
+                              vector<cls_rgw_lc_entry>& progress_map,
+                              int& index)
 {
-  return lc->list_lc_progress(marker, max_entries, progress_map);
+  return lc->list_lc_progress(marker, max_entries, progress_map, index);
 }
 
 int RGWRados::process_lc()

diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h
index bcfb8739201d8b7f0a3e2e555fd57b28c290fd7b..bbc3780bf0f6adac7512d93eee6b9a6a3e4923c1 100644 (file)
--- a/src/rgw/rgw_rados.h
+++ b/src/rgw/rgw_rados.h
@@ -1444,9 +1444,9 @@ public:
   int defer_gc(void *ctx, const RGWBucketInfo& bucket_info, const rgw_obj& obj, optional_yield y);
 
   int process_lc();
-  int list_lc_progress(const string& marker, uint32_t max_entries,
-                      vector<cls_rgw_lc_entry>& progress_map);
-  
+  int list_lc_progress(string& marker, uint32_t max_entries,
+                      vector<cls_rgw_lc_entry>& progress_map, int& index);
+
   int bucket_check_index(RGWBucketInfo& bucket_info,
                          map<RGWObjCategory, RGWStorageStats> *existing_stats,
                          map<RGWObjCategory, RGWStorageStats> *calculated_stats);