From f9829cfff6ff22a008c76a43062ae124c339c67f Mon Sep 17 00:00:00 2001 From: Aishwarya Mathuria Date: Tue, 5 Dec 2023 15:28:50 +0000 Subject: [PATCH] osd: Change PG Deletion cost for mClock With the osd_delete_sleep_ssd and osd_delete_sleep_hdd options disabled with mClock, it was noticed that PG deletion was completing much faster with mClock scheduler. In order to give mClock a more accurate cost of the PG Deletion operation, we calculate it by taking into consideration how many objects are being deleted. Signed-off-by: Aishwarya Mathuria (cherry picked from commit 34bed3e36a05b8281b6248fc9a1340bd0f1828a8) --- src/osd/OSD.cc | 13 ++++++++++--- src/osd/OSD.h | 2 +- src/osd/PG.cc | 9 ++++++--- src/osd/PG.h | 4 +++- src/osd/PrimaryLogPG.cc | 4 +++- 5 files changed, 23 insertions(+), 9 deletions(-) diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index 39a0fb30d42..b0a6fc9241b 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -1877,14 +1877,21 @@ void OSDService::queue_scrub_next_chunk(PG *pg, Scrub::scrub_prio_t with_priorit queue_scrub_event_msg(pg, with_priority); } -void OSDService::queue_for_pg_delete(spg_t pgid, epoch_t e) +void OSDService::queue_for_pg_delete(spg_t pgid, epoch_t e, int64_t num_objects) { dout(10) << __func__ << " on " << pgid << " e " << e << dendl; + uint64_t cost_for_queue = [this, num_objects] { + if (op_queue_type_t::mClockScheduler == osd->osd_op_queue_type()) { + return num_objects * cct->_conf->osd_pg_delete_cost; + } else { + return cct->_conf->osd_pg_delete_cost; + } + }(); enqueue_back( OpSchedulerItem( unique_ptr( new PGDelete(pgid, e)), - cct->_conf->osd_pg_delete_cost, + cost_for_queue, cct->_conf->osd_pg_delete_priority, ceph_clock_now(), 0, @@ -10223,7 +10230,7 @@ void OSD::maybe_override_cost_for_qos() // If the scheduler enabled is mclock, override the default PG deletion cost // so that mclock can meet the QoS goals. if (op_queue_type_t::mClockScheduler == osd_op_queue_type()) { - uint64_t pg_delete_cost = 15728640; + uint64_t pg_delete_cost = 1048576; cct->_conf.set_val("osd_pg_delete_cost", std::to_string(pg_delete_cost)); } } diff --git a/src/osd/OSD.h b/src/osd/OSD.h index 8c03e019203..5909ad246f7 100644 --- a/src/osd/OSD.h +++ b/src/osd/OSD.h @@ -563,7 +563,7 @@ public: unsigned int qu_priority, Scrub::act_token_t act_token); - void queue_for_pg_delete(spg_t pgid, epoch_t e); + void queue_for_pg_delete(spg_t pgid, epoch_t e, int64_t num_objects); bool try_finish_pg_delete(PG *pg, unsigned old_pg_num); private: diff --git a/src/osd/PG.cc b/src/osd/PG.cc index cb12dae7c5b..7a6b7b01180 100644 --- a/src/osd/PG.cc +++ b/src/osd/PG.cc @@ -2665,7 +2665,8 @@ void PG::C_DeleteMore::complete(int r) { ceph_assert(r == 0); pg->lock(); if (!pg->pg_has_reset_since(epoch)) { - pg->osd->queue_for_pg_delete(pg->get_pgid(), epoch); + pg->osd->queue_for_pg_delete(pg->get_pgid(), epoch, + num_objects); } pg->unlock(); delete this; @@ -2689,7 +2690,9 @@ std::pair PG::do_delete_work( std::scoped_lock locker{*this}; delete_needs_sleep = false; if (!pg_has_reset_since(e)) { - osd->queue_for_pg_delete(get_pgid(), e); + // We pass 1 for num_objects here as only wpq uses this code path + // and it will be ignored + osd->queue_for_pg_delete(get_pgid(), e, 1); } }); @@ -2762,7 +2765,7 @@ std::pair PG::do_delete_work( bool running = true; if (num) { dout(20) << __func__ << " deleting " << num << " objects" << dendl; - Context *fin = new C_DeleteMore(this, get_osdmap_epoch()); + Context *fin = new C_DeleteMore(this, get_osdmap_epoch(), num); t.register_on_commit(fin); } else { if (cct->_conf->osd_inject_failure_on_pg_removal) { diff --git a/src/osd/PG.h b/src/osd/PG.h index 677f4e93a07..9abf92e9085 100644 --- a/src/osd/PG.h +++ b/src/osd/PG.h @@ -776,7 +776,9 @@ public: struct C_DeleteMore : public Context { PGRef pg; epoch_t epoch; - C_DeleteMore(PG *p, epoch_t e) : pg(p), epoch(e) {} + int64_t num_objects; + C_DeleteMore(PG *p, epoch_t e, int64_t num) : pg(p), epoch(e), + num_objects(num){} void finish(int r) override { ceph_abort(); } diff --git a/src/osd/PrimaryLogPG.cc b/src/osd/PrimaryLogPG.cc index 214b8c533ae..3d1aa380289 100644 --- a/src/osd/PrimaryLogPG.cc +++ b/src/osd/PrimaryLogPG.cc @@ -12851,7 +12851,9 @@ void PrimaryLogPG::on_removal(ObjectStore::Transaction &t) on_shutdown(); - t.register_on_commit(new C_DeleteMore(this, get_osdmap_epoch())); + // starting PG deletion, num_objects can be 1 + // do_delete_work will update num_objects + t.register_on_commit(new C_DeleteMore(this, get_osdmap_epoch(), 1)); } void PrimaryLogPG::clear_async_reads() -- 2.39.5