mClock: Replace immediate queue with high priority queue

With the introduction of a high priority queue in the mClock workflow, we no longer need a separate queue for ops that need to be processed immediately.
We will be using the high_priority queue with a very high priority for all items having op_scheduler_class as immediate.

Signed-off-by: Aishwarya Mathuria <amathuri@redhat.com>

diff --git a/src/osd/scheduler/mClockScheduler.cc b/src/osd/scheduler/mClockScheduler.cc
index 5aa4f71530e354d9648bed817406fd869601f348..729caa4500f69496d45d509a607275bf7a582ae5 100644 (file)
--- a/src/osd/scheduler/mClockScheduler.cc
+++ b/src/osd/scheduler/mClockScheduler.cc
@@ -400,7 +400,6 @@ void mClockScheduler::dump(ceph::Formatter &f) const
 {
   // Display queue sizes
   f.open_object_section("queue_sizes");
-  f.dump_int("immediate", immediate.size());
   f.dump_int("high_priority_queue", high_priority.size());
   f.dump_int("scheduler", scheduler.request_count());
   f.close_section();
@@ -435,7 +434,7 @@ void mClockScheduler::enqueue(OpSchedulerItem&& item)
   
   // TODO: move this check into OpSchedulerItem, handle backwards compat
   if (op_scheduler_class::immediate == id.class_id) {
-    immediate.push_front(std::move(item));
+    enqueue_high(immediate_class_priority, std::move(item));
   } else if (priority >= cutoff) {
     enqueue_high(priority, std::move(item));
   } else {
@@ -454,7 +453,7 @@ void mClockScheduler::enqueue(OpSchedulerItem&& item)
   }
 
  dout(20) << __func__ << " client_count: " << scheduler.client_count()
-          << " queue_sizes: [ imm: " << immediate.size()
+          << " queue_sizes: [ "
          << " high_priority_queue: " << high_priority.size()
           << " sched: " << scheduler.request_count() << " ]"
           << dendl;
@@ -473,7 +472,7 @@ void mClockScheduler::enqueue_front(OpSchedulerItem&& item)
   auto id = get_scheduler_id(item);
 
   if (op_scheduler_class::immediate == id.class_id) {
-    immediate.push_back(std::move(item));
+    enqueue_high(immediate_class_priority, std::move(item), true);
   } else if (priority >= cutoff) {
     enqueue_high(priority, std::move(item), true);
   } else {
@@ -496,11 +495,7 @@ void mClockScheduler::enqueue_high(unsigned priority,
 
 WorkItem mClockScheduler::dequeue()
 {
-  if (!immediate.empty()) {
-    WorkItem work_item{std::move(immediate.back())};
-    immediate.pop_back();
-    return work_item;
-  } else if (!high_priority.empty()) {
+  if (!high_priority.empty()) {
     auto iter = high_priority.begin();
     // invariant: high_priority entries are never empty
     assert(!iter->second.empty());

diff --git a/src/osd/scheduler/mClockScheduler.h b/src/osd/scheduler/mClockScheduler.h
index 65e9c3e38a6f12be7b40ff5efa2e4e12a4942473..088df31ee190c6e230010a4bf3d0bc9ca41aedda 100644 (file)
--- a/src/osd/scheduler/mClockScheduler.h
+++ b/src/osd/scheduler/mClockScheduler.h
@@ -135,7 +135,6 @@ class mClockScheduler : public OpScheduler, md_config_obs_t {
   using SubQueue = std::map<priority_t,
        std::list<OpSchedulerItem>,
        std::greater<priority_t>>;
-  using SubQueueIter = SubQueue::iterator;
   mclock_queue_t scheduler;
   /**
    * high_priority
@@ -144,7 +143,7 @@ class mClockScheduler : public OpScheduler, md_config_obs_t {
    * Invariant: entries are never empty
    */
   SubQueue high_priority;
-  std::list<OpSchedulerItem> immediate;
+  priority_t immediate_class_priority = std::numeric_limits<priority_t>::max();
 
   static scheduler_id_t get_scheduler_id(const OpSchedulerItem &item) {
     return scheduler_id_t{
@@ -212,7 +211,7 @@ public:
   // Enqueue op in the back of the regular queue
   void enqueue(OpSchedulerItem &&item) final;
 
-  // Enqueue the op in the front of the high priority queue or the immediate queue (based on priority)
+  // Enqueue the op in the front of the high priority queue
   void enqueue_front(OpSchedulerItem &&item) final;
 
   // Return an op to be dispatch
@@ -220,7 +219,7 @@ public:
 
   // Returns if the queue is empty
   bool empty() const final {
-    return immediate.empty() && scheduler.empty() && high_priority.empty();
+    return scheduler.empty() && high_priority.empty();
   }
 
   // Formatted output of the queue

diff --git a/src/test/osd/TestMClockScheduler.cc b/src/test/osd/TestMClockScheduler.cc
index 40a830000aaf05f191156be84a10a309aceb87d4..685d20bd842bac53c62a59dfaf1ec9b40a0d577c 100644 (file)
--- a/src/test/osd/TestMClockScheduler.cc
+++ b/src/test/osd/TestMClockScheduler.cc
@@ -218,7 +218,7 @@ TEST_F(mClockSchedulerTest, TestAllQueuesEnqueueDequeue) {
     std::this_thread::sleep_for(std::chrono::microseconds(1));
   }
 
-  // Insert ops into the immediate queue
+  // Insert Immediate ops
   for (unsigned i = 103; i < 105; ++i) {
     q.enqueue(create_item(i, client1, op_scheduler_class::immediate));
     std::this_thread::sleep_for(std::chrono::microseconds(1));
@@ -232,7 +232,7 @@ TEST_F(mClockSchedulerTest, TestAllQueuesEnqueueDequeue) {
 
   ASSERT_FALSE(q.empty());
   auto r = get_item(q.dequeue());
-  // Immediate queue should be dequeued first
+  // Ops classified as Immediate should be dequeued first
   ASSERT_EQ(103u, r.get_map_epoch());
   r = get_item(q.dequeue());
   ASSERT_EQ(104u, r.get_map_epoch());