crimson/os: pin the last cpu core for the alien worker threads

before this change, we assume that we have at least current_shared + 10
cores. but that's not always true. so in this change, the last core
is used for performing the alien tasks scheduled by reactor.

Signed-off-by: Kefu Chai <kchai@redhat.com>

diff --git a/src/crimson/os/alienstore/alien_store.cc b/src/crimson/os/alienstore/alien_store.cc
index 1048f978ed73806bc93a29a9fa75b9cef8fb026a..445dd78b346ae7ff5d5b04f7a97580dd4f324d64 100644 (file)
--- a/src/crimson/os/alienstore/alien_store.cc
+++ b/src/crimson/os/alienstore/alien_store.cc
@@ -66,7 +66,15 @@ AlienStore::AlienStore(const std::string& path, const ConfigValues& values)
   g_ceph_context = cct.get();
   cct->_conf.set_config_values(values);
   store = std::make_unique<BlueStore>(cct.get(), path);
-  tp = std::make_unique<crimson::os::ThreadPool>(1, 128, seastar::this_shard_id() + 10);
+
+  long cpu_id = 0;
+  if (long nr_cpus = sysconf(_SC_NPROCESSORS_ONLN); nr_cpus != -1) {
+    cpu_id = nr_cpus - 1;
+  } else {
+    logger().error("{}: unable to get nproc: {}", __func__, errno);
+    cpu_id = -1;
+  }
+  tp = std::make_unique<crimson::os::ThreadPool>(1, 128, cpu_id);
 }
 
 seastar::future<> AlienStore::start()

diff --git a/src/crimson/os/alienstore/thread_pool.cc b/src/crimson/os/alienstore/thread_pool.cc
index 323bea5202423f7df47c9d5252d7c11a1cd41834..488599e27488002def0891038c5273ecaec8dad0 100644 (file)
--- a/src/crimson/os/alienstore/thread_pool.cc
+++ b/src/crimson/os/alienstore/thread_pool.cc
@@ -13,14 +13,16 @@ namespace crimson::os {
 
 ThreadPool::ThreadPool(size_t n_threads,
                        size_t queue_sz,
-                       unsigned cpu_id)
+                       long cpu_id)
   : queue_size{round_up_to(queue_sz, seastar::smp::count)},
     pending{queue_size}
 {
   auto queue_max_wait = std::chrono::seconds(local_conf()->threadpool_empty_queue_max_wait);
   for (size_t i = 0; i < n_threads; i++) {
     threads.emplace_back([this, cpu_id, queue_max_wait] {
-      pin(cpu_id);
+      if (cpu_id >= 0) {
+        pin(cpu_id);
+      }
       crimson::os::AlienStore::configure_thread_memory();
       loop(queue_max_wait);
     });

diff --git a/src/crimson/os/alienstore/thread_pool.h b/src/crimson/os/alienstore/thread_pool.h
index ec3e450a578891bceff2b53d5b3def1c0959769d..f8d773319c78facb05383f225df691868de009c2 100644 (file)
--- a/src/crimson/os/alienstore/thread_pool.h
+++ b/src/crimson/os/alienstore/thread_pool.h
@@ -101,7 +101,7 @@ public:
    * @note each @c Task has its own crimson::thread::Condition, which possesses
    * an fd, so we should keep the size of queue under a reasonable limit.
    */
-  ThreadPool(size_t n_threads, size_t queue_sz, unsigned cpu);
+  ThreadPool(size_t n_threads, size_t queue_sz, long cpu);
   ~ThreadPool();
   seastar::future<> start();
   seastar::future<> stop();