From: Kefu Chai <kchai@redhat.com>
Date: Thu, 2 Jul 2020 11:27:40 +0000 (+0800)
Subject: crimson/os: pin the last cpu core for the alien worker threads
X-Git-Tag: wip-pdonnell-testing-20200918.022351~791^2~1
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=2ba13668a6fc6d9fdc7109d4b53cc6b1dc72ecfa;p=ceph-ci.git

crimson/os: pin the last cpu core for the alien worker threads

before this change, we assume that we have at least current_shared + 10
cores. but that's not always true. so in this change, the last core
is used for performing the alien tasks scheduled by reactor.

Signed-off-by: Kefu Chai <kchai@redhat.com>
---

diff --git a/src/crimson/os/alienstore/alien_store.cc b/src/crimson/os/alienstore/alien_store.cc
index 1048f978ed7..445dd78b346 100644
--- a/src/crimson/os/alienstore/alien_store.cc
+++ b/src/crimson/os/alienstore/alien_store.cc
@@ -66,7 +66,15 @@ AlienStore::AlienStore(const std::string& path, const ConfigValues& values)
   g_ceph_context = cct.get();
   cct->_conf.set_config_values(values);
   store = std::make_unique<BlueStore>(cct.get(), path);
-  tp = std::make_unique<crimson::os::ThreadPool>(1, 128, seastar::this_shard_id() + 10);
+
+  long cpu_id = 0;
+  if (long nr_cpus = sysconf(_SC_NPROCESSORS_ONLN); nr_cpus != -1) {
+    cpu_id = nr_cpus - 1;
+  } else {
+    logger().error("{}: unable to get nproc: {}", __func__, errno);
+    cpu_id = -1;
+  }
+  tp = std::make_unique<crimson::os::ThreadPool>(1, 128, cpu_id);
 }
 
 seastar::future<> AlienStore::start()
diff --git a/src/crimson/os/alienstore/thread_pool.cc b/src/crimson/os/alienstore/thread_pool.cc
index 323bea52024..488599e2748 100644
--- a/src/crimson/os/alienstore/thread_pool.cc
+++ b/src/crimson/os/alienstore/thread_pool.cc
@@ -13,14 +13,16 @@ namespace crimson::os {
 
 ThreadPool::ThreadPool(size_t n_threads,
                        size_t queue_sz,
-                       unsigned cpu_id)
+                       long cpu_id)
   : queue_size{round_up_to(queue_sz, seastar::smp::count)},
     pending{queue_size}
 {
   auto queue_max_wait = std::chrono::seconds(local_conf()->threadpool_empty_queue_max_wait);
   for (size_t i = 0; i < n_threads; i++) {
     threads.emplace_back([this, cpu_id, queue_max_wait] {
-      pin(cpu_id);
+      if (cpu_id >= 0) {
+        pin(cpu_id);
+      }
       crimson::os::AlienStore::configure_thread_memory();
       loop(queue_max_wait);
     });
diff --git a/src/crimson/os/alienstore/thread_pool.h b/src/crimson/os/alienstore/thread_pool.h
index ec3e450a578..f8d773319c7 100644
--- a/src/crimson/os/alienstore/thread_pool.h
+++ b/src/crimson/os/alienstore/thread_pool.h
@@ -101,7 +101,7 @@ public:
    * @note each @c Task has its own crimson::thread::Condition, which possesses
    * an fd, so we should keep the size of queue under a reasonable limit.
    */
-  ThreadPool(size_t n_threads, size_t queue_sz, unsigned cpu);
+  ThreadPool(size_t n_threads, size_t queue_sz, long cpu);
   ~ThreadPool();
   seastar::future<> start();
   seastar::future<> stop();