From: Xuehan Xu <xxhdx1985126@gmail.com>
Date: Mon, 1 Mar 2021 11:23:27 +0000 (+0800)
Subject: crimson/os/alienstore: scatter alienstore's threads to different cores
X-Git-Tag: v17.1.0~2515^2~2
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=8b30a9ed12d5c44d05c106d751b4431cf4cd3916;p=ceph.git

crimson/os/alienstore: scatter alienstore's threads to different cores

This is for the purpose of performance optimization. According to tests,
there are circumstances in which the single cpu core that's holding all
alienstore threads is the perf bottleneck.

Signed-off-by: Xuehan Xu <xxhdx1985126@gmail.com>
---

diff --git a/src/crimson/os/alienstore/alien_store.cc b/src/crimson/os/alienstore/alien_store.cc
index 827d9f9af29..6b789e49742 100644
--- a/src/crimson/os/alienstore/alien_store.cc
+++ b/src/crimson/os/alienstore/alien_store.cc
@@ -63,17 +63,10 @@ AlienStore::AlienStore(const std::string& path, const ConfigValues& values)
   g_ceph_context = cct.get();
   cct->_conf.set_config_values(values);
   store = std::make_unique<BlueStore>(cct.get(), path);
-
-  long cpu_id = 0;
-  if (long nr_cpus = sysconf(_SC_NPROCESSORS_ONLN); nr_cpus != -1) {
-    cpu_id = nr_cpus - 1;
-  } else {
-    logger().error("{}: unable to get nproc: {}", __func__, errno);
-    cpu_id = -1;
-  }
   const auto num_threads =
     cct->_conf.get_val<uint64_t>("crimson_alien_op_num_threads");
-  tp = std::make_unique<crimson::os::ThreadPool>(num_threads, 128, cpu_id);
+  std::vector<uint64_t> cpu_cores = _parse_cpu_cores();
+  tp = std::make_unique<crimson::os::ThreadPool>(num_threads, 128, cpu_cores);
 }
 
 seastar::future<> AlienStore::start()
@@ -567,4 +560,26 @@ int AlienStore::AlienOmapIterator::status() const
   return iter->status();
 }
 
+std::vector<uint64_t> AlienStore::_parse_cpu_cores()
+{
+  std::vector<uint64_t> cpu_cores;
+  auto cpu_string =
+    cct->_conf.get_val<std::string>("crimson_alien_thread_cpu_cores");
+
+  std::string token;
+  std::istringstream token_stream(cpu_string);
+  while (std::getline(token_stream, token, ',')) {
+    std::istringstream cpu_stream(token);
+    std::string cpu;
+    std::getline(cpu_stream, cpu, '-');
+    uint64_t start_cpu = std::stoull(cpu);
+    std::getline(cpu_stream, cpu, '-');
+    uint64_t end_cpu = std::stoull(cpu);
+    for (uint64_t i = start_cpu; i < end_cpu; i++) {
+      cpu_cores.push_back(i);
+    }
+  }
+  return cpu_cores;
+}
+
 }
diff --git a/src/crimson/os/alienstore/alien_store.h b/src/crimson/os/alienstore/alien_store.h
index ae4830981e6..c94da3ee580 100644
--- a/src/crimson/os/alienstore/alien_store.h
+++ b/src/crimson/os/alienstore/alien_store.h
@@ -119,5 +119,6 @@ private:
   std::unique_ptr<CephContext> cct;
   seastar::gate transaction_gate;
   std::unordered_map<coll_t, CollectionRef> coll_map;
+  std::vector<uint64_t> _parse_cpu_cores();
 };
 }
diff --git a/src/crimson/os/alienstore/thread_pool.cc b/src/crimson/os/alienstore/thread_pool.cc
index e127d87d524..c42947bfa2f 100644
--- a/src/crimson/os/alienstore/thread_pool.cc
+++ b/src/crimson/os/alienstore/thread_pool.cc
@@ -1,3 +1,6 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:nil -*-
+// vim: ts=8 sw=2 smarttab expandtab
+
 #include "thread_pool.h"
 
 #include <chrono>
@@ -12,15 +15,15 @@ namespace crimson::os {
 
 ThreadPool::ThreadPool(size_t n_threads,
                        size_t queue_sz,
-                       long cpu_id)
+                       std::vector<uint64_t> cpus)
   : queue_size{round_up_to(queue_sz, seastar::smp::count)},
     pending{queue_size}
 {
   auto queue_max_wait = std::chrono::seconds(local_conf()->threadpool_empty_queue_max_wait);
   for (size_t i = 0; i < n_threads; i++) {
-    threads.emplace_back([this, cpu_id, queue_max_wait] {
-      if (cpu_id >= 0) {
-        pin(cpu_id);
+    threads.emplace_back([this, cpus, queue_max_wait] {
+      if (!cpus.empty()) {
+        pin(cpus);
       }
       loop(queue_max_wait);
     });
@@ -34,11 +37,13 @@ ThreadPool::~ThreadPool()
   }
 }
 
-void ThreadPool::pin(unsigned cpu_id)
+void ThreadPool::pin(const std::vector<uint64_t>& cpus)
 {
   cpu_set_t cs;
   CPU_ZERO(&cs);
-  CPU_SET(cpu_id, &cs);
+  for (auto cpu : cpus) {
+    CPU_SET(cpu, &cs);
+  }
   [[maybe_unused]] auto r = pthread_setaffinity_np(pthread_self(),
                                                    sizeof(cs), &cs);
   ceph_assert(r == 0);
diff --git a/src/crimson/os/alienstore/thread_pool.h b/src/crimson/os/alienstore/thread_pool.h
index 27840da1891..8b66725dd70 100644
--- a/src/crimson/os/alienstore/thread_pool.h
+++ b/src/crimson/os/alienstore/thread_pool.h
@@ -86,7 +86,7 @@ class ThreadPool {
   bool is_stopping() const {
     return stopping.load(std::memory_order_relaxed);
   }
-  static void pin(unsigned cpu_id);
+  static void pin(const std::vector<uint64_t>& cpus);
   seastar::semaphore& local_free_slots() {
     return submit_queue.local().free_slots;
   }
@@ -102,7 +102,7 @@ public:
    * @note each @c Task has its own crimson::thread::Condition, which possesses
    * an fd, so we should keep the size of queue under a reasonable limit.
    */
-  ThreadPool(size_t n_threads, size_t queue_sz, long cpu);
+  ThreadPool(size_t n_threads, size_t queue_sz, std::vector<uint64_t> cpus);
   ~ThreadPool();
   seastar::future<> start();
   seastar::future<> stop();