From 860de9b4b280d63d66e94d36df92c238026887c5 Mon Sep 17 00:00:00 2001
From: David Zafman <dzafman@redhat.com>
Date: Tue, 9 Oct 2018 09:11:29 -0700
Subject: [PATCH] osd: Separate removal and recovery thread pools

The name "disk_tp" was misleading because it was only used for
removal and recovery.

Signed-off-by: David Zafman <dzafman@redhat.com>

This backport is not cherry-picked from master since the remove
threadpool was folded into the main op queue post-luminous, so the
code is almost entirely different.
---
 src/common/legacy_config_opts.h |  3 ++-
 src/common/options.cc           |  6 +++++-
 src/osd/OSD.cc                  | 28 ++++++++++++++++++----------
 src/osd/OSD.h                   |  3 ++-
 4 files changed, 27 insertions(+), 13 deletions(-)

diff --git a/src/common/legacy_config_opts.h b/src/common/legacy_config_opts.h
index a51870ef64289..8232fdc3d788e 100644
--- a/src/common/legacy_config_opts.h
+++ b/src/common/legacy_config_opts.h
@@ -665,7 +665,8 @@ OPTION(osd_peering_wq_threads, OPT_INT)
 OPTION(osd_peering_wq_batch_size, OPT_U64)
 OPTION(osd_op_pq_max_tokens_per_priority, OPT_U64)
 OPTION(osd_op_pq_min_cost, OPT_U64)
-OPTION(osd_disk_threads, OPT_INT)
+OPTION(osd_remove_threads, OPT_INT)
+OPTION(osd_recovery_threads, OPT_INT)
 OPTION(osd_disk_thread_ioprio_class, OPT_STR) // rt realtime be best effort idle
 OPTION(osd_disk_thread_ioprio_priority, OPT_INT) // 0-7
 OPTION(osd_recover_clone_overlap, OPT_BOOL)   // preserve clone_overlap during recovery/migration
diff --git a/src/common/options.cc b/src/common/options.cc
index ff3bb1a1be193..970a12f3fe614 100644
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -1950,7 +1950,11 @@ std::vector<Option> get_global_options() {
     .set_default(65536)
     .set_description(""),
 
-    Option("osd_disk_threads", Option::TYPE_INT, Option::LEVEL_ADVANCED)
+    Option("osd_remove_threads", Option::TYPE_INT, Option::LEVEL_ADVANCED)
+    .set_default(1)
+    .set_description(""),
+
+    Option("osd_recovery_threads", Option::TYPE_INT, Option::LEVEL_ADVANCED)
     .set_default(1)
     .set_description(""),
 
diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 810e8583b73e8..75063ef84ae44 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -216,7 +216,7 @@ OSDService::OSDService(OSD *osd) :
   monc(osd->monc),
   peering_wq(osd->peering_wq),
   recovery_gen_wq("recovery_gen_wq", cct->_conf->osd_recovery_thread_timeout,
-		  &osd->disk_tp),
+		  &osd->recovery_tp),
   class_handler(osd->class_handler),
   pg_epoch_lock("OSDService::pg_epoch_lock"),
   publish_lock("OSDService::publish_lock"),
@@ -1972,7 +1972,8 @@ OSD::OSD(CephContext *cct_, ObjectStore *store_,
 	     "osd_peering_tp_threads"),
   osd_op_tp(cct, "OSD::osd_op_tp", "tp_osd_tp",
 	    get_num_op_threads()),
-  disk_tp(cct, "OSD::disk_tp", "tp_osd_disk", cct->_conf->osd_disk_threads, "osd_disk_threads"),
+  remove_tp(cct, "OSD::remove_tp", "tp_osd_remove", cct->_conf->osd_remove_threads, "osd_remove_threads"),
+  recovery_tp(cct, "OSD::recovery_tp", "tp_osd_recovery", cct->_conf->osd_recovery_threads, "osd_recovery_threads"),
   command_tp(cct, "OSD::command_tp", "tp_osd_cmd",  1),
   session_waiting_lock("OSD::session_waiting_lock"),
   osdmap_subscribe_lock("OSD::osdmap_subscribe_lock"),
@@ -2023,7 +2024,7 @@ OSD::OSD(CephContext *cct_, ObjectStore *store_,
     store,
     cct->_conf->osd_remove_thread_timeout,
     cct->_conf->osd_remove_thread_suicide_timeout,
-    &disk_tp),
+    &remove_tp),
   service(this)
 {
   monc->set_messenger(client_messenger);
@@ -2671,7 +2672,8 @@ int OSD::init()
   service.max_oldest_map = superblock.oldest_map;
 
   osd_op_tp.start();
-  disk_tp.start();
+  remove_tp.start();
+  recovery_tp.start();
   command_tp.start();
 
   set_disk_tp_priority();
@@ -3424,9 +3426,13 @@ int OSD::shutdown()
   command_tp.stop();
   dout(10) << "command tp stopped" << dendl;
 
-  disk_tp.drain();
-  disk_tp.stop();
-  dout(10) << "disk tp paused (new)" << dendl;
+  remove_tp.drain();
+  remove_tp.stop();
+  dout(10) << "remove tp paused (new)" << dendl;
+
+  recovery_tp.drain();
+  recovery_tp.stop();
+  dout(10) << "recovery tp paused (new)" << dendl;
 
   dout(10) << "stopping agent" << dendl;
   service.agent_stop();
@@ -10086,12 +10092,14 @@ void OSD::set_disk_tp_priority()
     return;
   int cls =
     ceph_ioprio_string_to_class(cct->_conf->osd_disk_thread_ioprio_class);
-  if (cls < 0)
+  if (cls < 0) {
     derr << __func__ << cpp_strerror(cls) << ": "
 	 << "osd_disk_thread_ioprio_class is " << cct->_conf->osd_disk_thread_ioprio_class
 	 << " but only the following values are allowed: idle, be or rt" << dendl;
-  else
-    disk_tp.set_ioprio(cls, cct->_conf->osd_disk_thread_ioprio_priority);
+  } else {
+    remove_tp.set_ioprio(cls, cct->_conf->osd_disk_thread_ioprio_priority);
+    recovery_tp.set_ioprio(cls, cct->_conf->osd_disk_thread_ioprio_priority);
+  }
 }
 
 // --------------------------------
diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index 4523fb2807f2e..003f3b7b51d09 100644
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -1392,7 +1392,8 @@ private:
 
   ThreadPool peering_tp;
   ShardedThreadPool osd_op_tp;
-  ThreadPool disk_tp;
+  ThreadPool remove_tp;
+  ThreadPool recovery_tp;
   ThreadPool command_tp;
 
   void set_disk_tp_priority();
-- 
2.39.5