osd/OSD: batch-list objects to reduce memory consumption

For PG with a huge amount of objects, it wouldn't be an ideal
way to list all of them at a time. Split them into small batches
which we can handle individually efficiently should instead be
the preferred option.

Signed-off-by: xie xingguo <xie.xingguo@zte.com.cn>

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 505cf57d4023328a6cfc29ec48baa57a92be7899..bdb728ad3d617d051f6a19fe2aad6cbb1640a149 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -3742,26 +3742,27 @@ void OSD::recursive_remove_collection(CephContext* cct,
   ObjectStore::Transaction t;
   SnapMapper mapper(cct, &driver, 0, 0, 0, pgid.shard);
 
+  ghobject_t next;
+  int max = cct->_conf->osd_target_transaction_size;
   vector<ghobject_t> objects;
-  store->collection_list(ch, ghobject_t(), ghobject_t::get_max(),
-                        INT_MAX, &objects, 0);
-  generic_dout(10) << __func__ << " " << objects << dendl;
-  // delete them.
-  int removed = 0;
-  for (vector<ghobject_t>::iterator p = objects.begin();
-       p != objects.end();
-       ++p, removed++) {
-    OSDriver::OSTransaction _t(driver.get_transaction(&t));
-    int r = mapper.remove_oid(p->hobj, &_t);
-    if (r != 0 && r != -ENOENT)
-      ceph_abort();
-    t.remove(tmp, *p);
-    if (removed > cct->_conf->osd_target_transaction_size) {
-      int r = store->queue_transaction(ch, std::move(t));
-      assert(r == 0);
-      t = ObjectStore::Transaction();
-      removed = 0;
-    }
+  objects.reserve(max);
+  while (true) {
+    objects.clear();
+    store->collection_list(ch, next, ghobject_t::get_max(),
+      max, &objects, &next);
+    generic_dout(10) << __func__ << " " << objects << dendl;
+    if (objects.empty())
+      break;
+    for (auto& p: objects) {
+      OSDriver::OSTransaction _t(driver.get_transaction(&t));
+      int r = mapper.remove_oid(p.hobj, &_t);
+      if (r != 0 && r != -ENOENT)
+        ceph_abort();
+      t.remove(tmp, p);
+    }
+    int r = store->queue_transaction(ch, std::move(t));
+    assert(r == 0);
+    t = ObjectStore::Transaction();
   }
   t.remove_collection(tmp);
   int r = store->queue_transaction(ch, std::move(t));