From: xie xingguo Date: Wed, 7 Mar 2018 08:36:03 +0000 (+0800) Subject: osd/OSD: batch-list objects to reduce memory consumption X-Git-Tag: v13.0.2~51^2 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=020a031b62db7474ddf65791d9ba260e21174337;p=ceph.git osd/OSD: batch-list objects to reduce memory consumption For PG with a huge amount of objects, it wouldn't be an ideal way to list all of them at a time. Split them into small batches which we can handle individually efficiently should instead be the preferred option. Signed-off-by: xie xingguo --- diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index 505cf57d40233..bdb728ad3d617 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -3742,26 +3742,27 @@ void OSD::recursive_remove_collection(CephContext* cct, ObjectStore::Transaction t; SnapMapper mapper(cct, &driver, 0, 0, 0, pgid.shard); + ghobject_t next; + int max = cct->_conf->osd_target_transaction_size; vector objects; - store->collection_list(ch, ghobject_t(), ghobject_t::get_max(), - INT_MAX, &objects, 0); - generic_dout(10) << __func__ << " " << objects << dendl; - // delete them. - int removed = 0; - for (vector::iterator p = objects.begin(); - p != objects.end(); - ++p, removed++) { - OSDriver::OSTransaction _t(driver.get_transaction(&t)); - int r = mapper.remove_oid(p->hobj, &_t); - if (r != 0 && r != -ENOENT) - ceph_abort(); - t.remove(tmp, *p); - if (removed > cct->_conf->osd_target_transaction_size) { - int r = store->queue_transaction(ch, std::move(t)); - assert(r == 0); - t = ObjectStore::Transaction(); - removed = 0; - } + objects.reserve(max); + while (true) { + objects.clear(); + store->collection_list(ch, next, ghobject_t::get_max(), + max, &objects, &next); + generic_dout(10) << __func__ << " " << objects << dendl; + if (objects.empty()) + break; + for (auto& p: objects) { + OSDriver::OSTransaction _t(driver.get_transaction(&t)); + int r = mapper.remove_oid(p.hobj, &_t); + if (r != 0 && r != -ENOENT) + ceph_abort(); + t.remove(tmp, p); + } + int r = store->queue_transaction(ch, std::move(t)); + assert(r == 0); + t = ObjectStore::Transaction(); } t.remove_collection(tmp); int r = store->queue_transaction(ch, std::move(t));