osd: Change PG Deletion cost for mClock

With the osd_delete_sleep_ssd and osd_delete_sleep_hdd options disabled with mClock, it was noticed that PG deletion was completing much faster with mClock scheduler.
In order to give mClock a more accurate cost of the PG Deletion operation, we calculate it by taking into consideration how many objects are being deleted.

Signed-off-by: Aishwarya Mathuria <amathuri@redhat.com>
(cherry picked from commit 34bed3e36a05b8281b6248fc9a1340bd0f1828a8)

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 39a0fb30d424ae31c53f5e96b3dfa855bdfb5f24..b0a6fc9241bb570c9dd25408715a1a3780542cb5 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -1877,14 +1877,21 @@ void OSDService::queue_scrub_next_chunk(PG *pg, Scrub::scrub_prio_t with_priorit
   queue_scrub_event_msg<PGScrubGetNextChunk>(pg, with_priority);
 }
 
-void OSDService::queue_for_pg_delete(spg_t pgid, epoch_t e)
+void OSDService::queue_for_pg_delete(spg_t pgid, epoch_t e, int64_t num_objects)
 {
   dout(10) << __func__ << " on " << pgid << " e " << e  << dendl;
+  uint64_t cost_for_queue = [this, num_objects] {
+    if (op_queue_type_t::mClockScheduler == osd->osd_op_queue_type()) {
+      return num_objects * cct->_conf->osd_pg_delete_cost;
+    } else {
+      return cct->_conf->osd_pg_delete_cost;
+    }
+  }();
   enqueue_back(
     OpSchedulerItem(
       unique_ptr<OpSchedulerItem::OpQueueable>(
        new PGDelete(pgid, e)),
-      cct->_conf->osd_pg_delete_cost,
+      cost_for_queue,
       cct->_conf->osd_pg_delete_priority,
       ceph_clock_now(),
       0,
@@ -10223,7 +10230,7 @@ void OSD::maybe_override_cost_for_qos()
   // If the scheduler enabled is mclock, override the default PG deletion cost
   // so that mclock can meet the QoS goals.
   if (op_queue_type_t::mClockScheduler == osd_op_queue_type()) {
-    uint64_t pg_delete_cost = 15728640;
+    uint64_t pg_delete_cost = 1048576;
     cct->_conf.set_val("osd_pg_delete_cost", std::to_string(pg_delete_cost));
   }
 }

diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index 8c03e0192033992b48943da8d9c6c308c79ca574..5909ad246f736605fd27a68c0aa8b0d7c6e55b82 100644 (file)
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -563,7 +563,7 @@ public:
                                   unsigned int qu_priority,
                                   Scrub::act_token_t act_token);
 
-  void queue_for_pg_delete(spg_t pgid, epoch_t e);
+  void queue_for_pg_delete(spg_t pgid, epoch_t e, int64_t num_objects);
   bool try_finish_pg_delete(PG *pg, unsigned old_pg_num);
 
 private:

diff --git a/src/osd/PG.cc b/src/osd/PG.cc
index cb12dae7c5b100977d2a4760b99155d81b1b6da9..7a6b7b0118034d597998810eee3b748c64b9cfed 100644 (file)
--- a/src/osd/PG.cc
+++ b/src/osd/PG.cc
@@ -2665,7 +2665,8 @@ void PG::C_DeleteMore::complete(int r) {
   ceph_assert(r == 0);
   pg->lock();
   if (!pg->pg_has_reset_since(epoch)) {
-    pg->osd->queue_for_pg_delete(pg->get_pgid(), epoch);
+    pg->osd->queue_for_pg_delete(pg->get_pgid(), epoch,
+                                num_objects);
   }
   pg->unlock();
   delete this;
@@ -2689,7 +2690,9 @@ std::pair<ghobject_t, bool> PG::do_delete_work(
         std::scoped_lock locker{*this};
         delete_needs_sleep = false;
         if (!pg_has_reset_since(e)) {
-          osd->queue_for_pg_delete(get_pgid(), e);
+         // We pass 1 for num_objects here as only wpq uses this code path
+         // and it will be ignored
+          osd->queue_for_pg_delete(get_pgid(), e, 1);
         }
       });
 
@@ -2762,7 +2765,7 @@ std::pair<ghobject_t, bool> PG::do_delete_work(
   bool running = true;
   if (num) {
     dout(20) << __func__ << " deleting " << num << " objects" << dendl;
-    Context *fin = new C_DeleteMore(this, get_osdmap_epoch());
+    Context *fin = new C_DeleteMore(this, get_osdmap_epoch(), num);
     t.register_on_commit(fin);
   } else {
     if (cct->_conf->osd_inject_failure_on_pg_removal) {

diff --git a/src/osd/PG.h b/src/osd/PG.h
index 677f4e93a07954df34627a6528c52f21fcd788e8..9abf92e90855a52544d672f5da73a038c3b77890 100644 (file)
--- a/src/osd/PG.h
+++ b/src/osd/PG.h
@@ -776,7 +776,9 @@ public:
   struct C_DeleteMore : public Context {
     PGRef pg;
     epoch_t epoch;
-    C_DeleteMore(PG *p, epoch_t e) : pg(p), epoch(e) {}
+    int64_t num_objects;
+    C_DeleteMore(PG *p, epoch_t e, int64_t num) : pg(p), epoch(e),
+                                                 num_objects(num){}
     void finish(int r) override {
       ceph_abort();
     }

diff --git a/src/osd/PrimaryLogPG.cc b/src/osd/PrimaryLogPG.cc
index 214b8c533aefa9525447a9b3e1019585d16b428d..3d1aa3802890fa1a981f142f2951bf9673505607 100644 (file)
--- a/src/osd/PrimaryLogPG.cc
+++ b/src/osd/PrimaryLogPG.cc
@@ -12851,7 +12851,9 @@ void PrimaryLogPG::on_removal(ObjectStore::Transaction &t)
 
   on_shutdown();
 
-  t.register_on_commit(new C_DeleteMore(this, get_osdmap_epoch()));
+  // starting PG deletion, num_objects can be 1
+  // do_delete_work will update num_objects
+  t.register_on_commit(new C_DeleteMore(this, get_osdmap_epoch(), 1));
 }
 
 void PrimaryLogPG::clear_async_reads()