From: Kefu Chai <kchai@redhat.com>
Date: Wed, 24 Feb 2021 04:34:26 +0000 (+0800)
Subject: async/dpdk: do not use worker id when creating worker
X-Git-Tag: v16.2.15~8^2~7
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=6376b8014cdbd0eaeabb1f9fa18276f76b37e38c;p=ceph.git

async/dpdk: do not use worker id when creating worker

so we can drop the `i` parameter in a later change.

also restructure DPDKStack::spawn_worker() to capture variables by value
instead of by reference, we cannot assume that the variables allocated on
stack are still available when the function is scheduled on another
stack and core.

Signed-off-by: Kefu Chai <kchai@redhat.com>
(cherry picked from commit db8d9f6641ea7280e46cd985f8a3bb17fe10808e)
---

diff --git a/src/msg/async/dpdk/DPDKStack.cc b/src/msg/async/dpdk/DPDKStack.cc
index 9a73dac5db14..f17767394260 100644
--- a/src/msg/async/dpdk/DPDKStack.cc
+++ b/src/msg/async/dpdk/DPDKStack.cc
@@ -246,7 +246,7 @@ void DPDKStack::spawn_worker(unsigned i, std::function<void ()> &&func)
 {
   // create a extra master thread
   //
-  funcs[i] = std::move(func);
+  funcs.push_back(std::move(func));
   int r = 0;
   r = dpdk::eal::init(cct);
   if (r < 0) {
@@ -255,16 +255,17 @@ void DPDKStack::spawn_worker(unsigned i, std::function<void ()> &&func)
   }
   // if dpdk::eal::init already called by NVMEDevice, we will select 1..n
   // cores
-  ceph_assert(rte_lcore_count() >= i + 1);
+  unsigned nr_worker = funcs.size();
+  ceph_assert(rte_lcore_count() >= nr_worker);
   unsigned core_id;
-  int j = i;
   RTE_LCORE_FOREACH_SLAVE(core_id) {
-    if (i-- == 0) {
+    if (--nr_worker == 0) {
       break;
     }
   }
-  dpdk::eal::execute_on_master([&]() {
-    r = rte_eal_remote_launch(dpdk_thread_adaptor, static_cast<void*>(&funcs[j]), core_id);
+  void *adapted_func = static_cast<void*>(funcs.back());
+  dpdk::eal::execute_on_master([adapted_func, core_id, this]() {
+    int r = rte_eal_remote_launch(dpdk_thread_adaptor, adapted_func, core_id);
     if (r < 0) {
       lderr(cct) << __func__ << " remote launch failed, r=" << r << dendl;
       ceph_abort();
diff --git a/src/msg/async/dpdk/DPDKStack.h b/src/msg/async/dpdk/DPDKStack.h
index 926adaffcbb7..8a1f0089978c 100644
--- a/src/msg/async/dpdk/DPDKStack.h
+++ b/src/msg/async/dpdk/DPDKStack.h
@@ -254,9 +254,8 @@ class DPDKStack : public NetworkStack {
   }
 
  public:
-  explicit DPDKStack(CephContext *cct): NetworkStack(cct) {
-    funcs.resize(cct->_conf->ms_async_max_op_threads);
-  }
+  explicit DPDKStack(CephContext *cct): NetworkStack(cct)
+  {}
   virtual bool support_local_listen_table() const override { return true; }
 
   virtual void spawn_worker(unsigned i, std::function<void ()> &&func) override;