async/dpdk: do not use worker id when creating worker

so we can drop the `i` parameter in a later change.

also restructure DPDKStack::spawn_worker() to capture variables by value
instead of by reference, we cannot assume that the variables allocated on
stack are still available when the function is scheduled on another
stack and core.

Signed-off-by: Kefu Chai <kchai@redhat.com>
(cherry picked from commit db8d9f6641ea7280e46cd985f8a3bb17fe10808e)

diff --git a/src/msg/async/dpdk/DPDKStack.cc b/src/msg/async/dpdk/DPDKStack.cc
index 9a73dac5db1420a86017c37038307f756baf95c0..f177673942603bbb6f81b4a6c45502e8a1657787 100644 (file)
--- a/src/msg/async/dpdk/DPDKStack.cc
+++ b/src/msg/async/dpdk/DPDKStack.cc
@@ -246,7 +246,7 @@ void DPDKStack::spawn_worker(unsigned i, std::function<void ()> &&func)
 {
   // create a extra master thread
   //
-  funcs[i] = std::move(func);
+  funcs.push_back(std::move(func));
   int r = 0;
   r = dpdk::eal::init(cct);
   if (r < 0) {
@@ -255,16 +255,17 @@ void DPDKStack::spawn_worker(unsigned i, std::function<void ()> &&func)
   }
   // if dpdk::eal::init already called by NVMEDevice, we will select 1..n
   // cores
-  ceph_assert(rte_lcore_count() >= i + 1);
+  unsigned nr_worker = funcs.size();
+  ceph_assert(rte_lcore_count() >= nr_worker);
   unsigned core_id;
-  int j = i;
   RTE_LCORE_FOREACH_SLAVE(core_id) {
-    if (i-- == 0) {
+    if (--nr_worker == 0) {
       break;
     }
   }
-  dpdk::eal::execute_on_master([&]() {
-    r = rte_eal_remote_launch(dpdk_thread_adaptor, static_cast<void*>(&funcs[j]), core_id);
+  void *adapted_func = static_cast<void*>(funcs.back());
+  dpdk::eal::execute_on_master([adapted_func, core_id, this]() {
+    int r = rte_eal_remote_launch(dpdk_thread_adaptor, adapted_func, core_id);
     if (r < 0) {
       lderr(cct) << __func__ << " remote launch failed, r=" << r << dendl;
       ceph_abort();

diff --git a/src/msg/async/dpdk/DPDKStack.h b/src/msg/async/dpdk/DPDKStack.h
index 926adaffcbb7a87ca1850cd7464fb964602ca9b2..8a1f0089978c72b3d5edf2161859c34a8ad20613 100644 (file)
--- a/src/msg/async/dpdk/DPDKStack.h
+++ b/src/msg/async/dpdk/DPDKStack.h
@@ -254,9 +254,8 @@ class DPDKStack : public NetworkStack {
   }
 
  public:
-  explicit DPDKStack(CephContext *cct): NetworkStack(cct) {
-    funcs.resize(cct->_conf->ms_async_max_op_threads);
-  }
+  explicit DPDKStack(CephContext *cct): NetworkStack(cct)
+  {}
   virtual bool support_local_listen_table() const override { return true; }
 
   virtual void spawn_worker(unsigned i, std::function<void ()> &&func) override;