From 2da091229bd3a9c4d81fecacb60b918a614aeb84 Mon Sep 17 00:00:00 2001
From: Sridhar Seshasayee <sseshasa@redhat.com>
Date: Thu, 4 Mar 2021 17:08:58 +0530
Subject: [PATCH] osd: Add config options for cost per io & byte for the mclock
 scheduler

The cost per io and cost per byte options for hdd and ssd are specified
and set to default values determined using experiments on hdds and ssds
using a cost model. The values are used in calc_scaled_cost() to
determine the scaled cost for every OpSchedulerItem that is enqueued
within the mClockScheduler.

Signed-off-by: Sridhar Seshasayee <sseshasa@redhat.com>
---
 src/common/options.cc                | 44 +++++++++++++------
 src/osd/scheduler/mClockScheduler.cc | 65 +++++++++++++++++++---------
 src/osd/scheduler/mClockScheduler.h  |  7 ++-
 3 files changed, 80 insertions(+), 36 deletions(-)

diff --git a/src/common/options.cc b/src/common/options.cc
index 8673abc4cf05..5d2f095eefe3 100644
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -3039,22 +3039,40 @@ std::vector<Option> get_global_options() {
     .set_description("mclock anticipation timeout in seconds")
     .set_long_description("the amount of time that mclock waits until the unused resource is forfeited"),
 
-    Option("osd_mclock_cost_per_io_msec", Option::TYPE_UINT, Option::LEVEL_DEV)
-    .set_default(0)
-    .set_description("Cost per IO in milliseconds to consider per OSD (overrides _ssd and _hdd if non-zero)")
-    .set_long_description("This option specifies the cost factor to consider in msec per OSD. This is considered by the mclock_scheduler to set an additional cost factor in QoS calculations. Only considered for osd_op_queue = mclock_scheduler")
+    Option("osd_mclock_cost_per_io_usec", Option::TYPE_FLOAT, Option::LEVEL_DEV)
+    .set_default(0.0)
+    .set_description("Cost per IO in microseconds to consider per OSD (overrides _ssd and _hdd if non-zero)")
+    .set_long_description("This option specifies the cost factor to consider in usec per OSD. This is considered by the mclock scheduler to set an additional cost factor in QoS calculations. Only considered for osd_op_queue = mclock_scheduler")
     .set_flag(Option::FLAG_RUNTIME),
 
-    Option("osd_mclock_cost_per_io_msec_hdd", Option::TYPE_UINT, Option::LEVEL_DEV)
-    .set_default(0)
-    .set_description("Cost per IO in milliseconds to consider per OSD (for rotational media)")
-    .set_long_description("This option specifies the cost factor to consider in msec per OSD for rotational device type. This is considered by the mclock_scheduler to set an additional cost factor in QoS calculations. Only considered for osd_op_queue = mclock_scheduler")
+    Option("osd_mclock_cost_per_io_usec_hdd", Option::TYPE_FLOAT, Option::LEVEL_DEV)
+    .set_default(25000.0)
+    .set_description("Cost per IO in microseconds to consider per OSD (for rotational media)")
+    .set_long_description("This option specifies the cost factor to consider in usec per OSD for rotational device type. This is considered by the mclock_scheduler to set an additional cost factor in QoS calculations. Only considered for osd_op_queue = mclock_scheduler")
     .set_flag(Option::FLAG_RUNTIME),
 
-    Option("osd_mclock_cost_per_io_msec_ssd", Option::TYPE_UINT, Option::LEVEL_DEV)
-    .set_default(0)
-    .set_description("Cost per IO in milliseconds to consider per OSD (for solid state media)")
-    .set_long_description("This option specifies the cost factor to consider in msec per OSD for solid state device type. This is considered by the mclock_scheduler to set an additional cost factor in QoS calculations. Only considered for osd_op_queue = mclock_scheduler")
+    Option("osd_mclock_cost_per_io_usec_ssd", Option::TYPE_FLOAT, Option::LEVEL_DEV)
+    .set_default(50.0)
+    .set_description("Cost per IO in microseconds to consider per OSD (for solid state media)")
+    .set_long_description("This option specifies the cost factor to consider in usec per OSD for solid state device type. This is considered by the mclock_scheduler to set an additional cost factor in QoS calculations. Only considered for osd_op_queue = mclock_scheduler")
+    .set_flag(Option::FLAG_RUNTIME),
+
+    Option("osd_mclock_cost_per_byte_usec", Option::TYPE_FLOAT, Option::LEVEL_DEV)
+    .set_default(0.0)
+    .set_description("Cost per byte in microseconds to consider per OSD (overrides _ssd and _hdd if non-zero)")
+    .set_long_description("This option specifies the cost per byte to consider in microseconds per OSD. This is considered by the mclock scheduler to set an additional cost factor in QoS calculations. Only considered for osd_op_queue = mclock_scheduler")
+    .set_flag(Option::FLAG_RUNTIME),
+
+    Option("osd_mclock_cost_per_byte_usec_hdd", Option::TYPE_FLOAT, Option::LEVEL_DEV)
+    .set_default(5.2)
+    .set_description("Cost per byte in microseconds to consider per OSD (for rotational media)")
+    .set_long_description("This option specifies the cost per byte to consider in microseconds per OSD for rotational device type. This is considered by the mclock_scheduler to set an additional cost factor in QoS calculations. Only considered for osd_op_queue = mclock_scheduler")
+    .set_flag(Option::FLAG_RUNTIME),
+
+    Option("osd_mclock_cost_per_byte_usec_ssd", Option::TYPE_FLOAT, Option::LEVEL_DEV)
+    .set_default(0.011)
+    .set_description("Cost per byte in microseconds to consider per OSD (for solid state media)")
+    .set_long_description("This option specifies the cost per byte to consider in microseconds per OSD for solid state device type. This is considered by the mclock_scheduler to set an additional cost factor in QoS calculations. Only considered for osd_op_queue = mclock_scheduler")
     .set_flag(Option::FLAG_RUNTIME),
 
     Option("osd_mclock_max_capacity_iops", Option::TYPE_FLOAT, Option::LEVEL_BASIC)
@@ -3064,7 +3082,7 @@ std::vector<Option> get_global_options() {
     .set_flag(Option::FLAG_RUNTIME),
 
     Option("osd_mclock_max_capacity_iops_hdd", Option::TYPE_FLOAT, Option::LEVEL_BASIC)
-    .set_default(10000.0)
+    .set_default(315.0)
     .set_description("Max IOPs capacity (at 4KiB block size) to consider per OSD (for rotational media)")
     .set_long_description("This option specifies the max OSD capacity in iops per OSD. Helps in QoS calculations when enabling a dmclock profile. Only considered for osd_op_queue = mclock_scheduler")
     .set_flag(Option::FLAG_RUNTIME),
diff --git a/src/osd/scheduler/mClockScheduler.cc b/src/osd/scheduler/mClockScheduler.cc
index 979311d8c7b4..c30b50e27ff5 100644
--- a/src/osd/scheduler/mClockScheduler.cc
+++ b/src/osd/scheduler/mClockScheduler.cc
@@ -110,31 +110,62 @@ void mClockScheduler::set_max_osd_capacity()
         cct->_conf.get_val<double>("osd_mclock_max_capacity_iops_ssd");
     }
   }
-  // Set max osd bandwidth across all shards (at 4KiB blocksize)
-  max_osd_bandwidth = max_osd_capacity * 4 * 1024;
   // Set per op-shard iops limit
   max_osd_capacity /= num_shards;
+  dout(1) << __func__ << " #op shards: " << num_shards
+          << " max osd capacity(iops) per shard: " << max_osd_capacity << dendl;
 }
 
 void mClockScheduler::set_osd_mclock_cost_per_io()
 {
-  if (cct->_conf.get_val<uint64_t>("osd_mclock_cost_per_io_msec")) {
-    osd_mclock_cost_per_io_msec =
-      cct->_conf.get_val<uint64_t>("osd_mclock_cost_per_io_msec");
+  std::chrono::seconds sec(1);
+  if (cct->_conf.get_val<double>("osd_mclock_cost_per_io_usec")) {
+    osd_mclock_cost_per_io =
+      cct->_conf.get_val<double>("osd_mclock_cost_per_io_usec");
   } else {
     if (is_rotational) {
-      osd_mclock_cost_per_io_msec =
-        cct->_conf.get_val<uint64_t>("osd_mclock_cost_per_io_msec_hdd");
+      osd_mclock_cost_per_io =
+        cct->_conf.get_val<double>("osd_mclock_cost_per_io_usec_hdd");
+      // For HDDs, convert value to seconds
+      osd_mclock_cost_per_io /= std::chrono::microseconds(sec).count();
     } else {
-      osd_mclock_cost_per_io_msec =
-        cct->_conf.get_val<uint64_t>("osd_mclock_cost_per_io_msec_ssd");
+      // For SSDs, convert value to milliseconds
+      osd_mclock_cost_per_io =
+        cct->_conf.get_val<double>("osd_mclock_cost_per_io_usec_ssd");
+      osd_mclock_cost_per_io /= std::chrono::milliseconds(sec).count();
     }
   }
+  dout(1) << __func__ << " osd_mclock_cost_per_io: "
+          << std::fixed << osd_mclock_cost_per_io << dendl;
+}
+
+void mClockScheduler::set_osd_mclock_cost_per_byte()
+{
+  std::chrono::seconds sec(1);
+  if (cct->_conf.get_val<double>("osd_mclock_cost_per_byte_usec")) {
+    osd_mclock_cost_per_byte =
+      cct->_conf.get_val<double>("osd_mclock_cost_per_byte_usec");
+  } else {
+    if (is_rotational) {
+      osd_mclock_cost_per_byte =
+        cct->_conf.get_val<double>("osd_mclock_cost_per_byte_usec_hdd");
+      // For HDDs, convert value to seconds
+      osd_mclock_cost_per_byte /= std::chrono::microseconds(sec).count();
+    } else {
+      osd_mclock_cost_per_byte =
+        cct->_conf.get_val<double>("osd_mclock_cost_per_byte_usec_ssd");
+      // For SSDs, convert value to milliseconds
+      osd_mclock_cost_per_byte /= std::chrono::milliseconds(sec).count();
+    }
+  }
+  dout(1) << __func__ << " osd_mclock_cost_per_byte: "
+          << std::fixed << osd_mclock_cost_per_byte << dendl;
 }
 
 void mClockScheduler::set_mclock_profile()
 {
   mclock_profile = cct->_conf.get_val<std::string>("osd_mclock_profile");
+  dout(1) << __func__ << " mclock profile: " << mclock_profile << dendl;
 }
 
 std::string mClockScheduler::get_mclock_profile()
@@ -358,19 +389,11 @@ void mClockScheduler::set_global_recovery_options()
   cct->_conf.apply_changes(nullptr);
 }
 
-int mClockScheduler::calc_scaled_cost(int cost)
+int mClockScheduler::calc_scaled_cost(int item_cost)
 {
-  // Calculate scaled cost in msecs based on item cost
-  int scaled_cost = std::floor((cost / max_osd_bandwidth) * 1000);
-
-  // Scale the cost down by an additional cost factor if specified
-  // to account for different device characteristics (hdd, ssd).
-  // This option can be used to further tune the performance further
-  // if necessary (disabled by default).
-  if (osd_mclock_cost_per_io_msec > 0) {
-    scaled_cost *= osd_mclock_cost_per_io_msec / 1000.0;
-  }
-
+  // Calculate total scaled cost in secs
+  int scaled_cost =
+    std::round(osd_mclock_cost_per_io + (osd_mclock_cost_per_byte * item_cost));
   return std::max(scaled_cost, 1);
 }
 
diff --git a/src/osd/scheduler/mClockScheduler.h b/src/osd/scheduler/mClockScheduler.h
index 8727af617983..2e513bb6ce30 100644
--- a/src/osd/scheduler/mClockScheduler.h
+++ b/src/osd/scheduler/mClockScheduler.h
@@ -67,8 +67,8 @@ class mClockScheduler : public OpScheduler, md_config_obs_t {
   const uint32_t num_shards;
   bool is_rotational;
   double max_osd_capacity;
-  double max_osd_bandwidth;
-  uint64_t osd_mclock_cost_per_io_msec;
+  double osd_mclock_cost_per_io;
+  double osd_mclock_cost_per_byte;
   std::string mclock_profile = "high_client_ops";
   struct ClientAllocs {
     uint64_t res;
@@ -145,6 +145,9 @@ public:
   // Set the cost per io for the osd
   void set_osd_mclock_cost_per_io();
 
+  // Set the cost per byte for the osd
+  void set_osd_mclock_cost_per_byte();
+
   // Set the mclock profile type to enable
   void set_mclock_profile();
 
-- 
2.47.3