ms_dpdk_coremask and ms_async_op_threads has some relationship,

dpdk library will create threasds as many as bit 1 number in
ms_dpdk_coremask and the thredas num should be equal ms_async_op_threads+1,
by ceph default confiuration,ms_async_op_threads =3,ms_dpdk_coremask =1,
so dpdk library can't create enough threads for dpdk workers. the sytem
will stuck at NetworkStack::start().so set ms_dpdk_coremask default 0xF.
and add assert to comapre the two configurations.

Signed-off-by: chunmei <chunmei.liu@intel.com>

diff --git a/src/common/options.cc b/src/common/options.cc
index 89005ae358903e3ac2513f9d734dfeecdeb4991c..f3ff00ec8ab59163b9bafa04bddb692259e6458e 100644 (file)
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -821,9 +821,10 @@ std::vector<Option> get_global_options() {
     .set_description(""),
 
     Option("ms_dpdk_coremask", Option::TYPE_STR, Option::LEVEL_ADVANCED)
-    .set_default("1")
+    .set_default("0xF")         //begin with 0x for the string
     .set_description("")
-    .set_safe(),
+    .set_safe()
+    .add_see_also("ms_async_op_threads"),
 
     Option("ms_dpdk_memory_channel", Option::TYPE_STR, Option::LEVEL_ADVANCED)
     .set_default("4")

diff --git a/src/msg/async/dpdk/dpdk_rte.cc b/src/msg/async/dpdk/dpdk_rte.cc
index 6510df103b201f71ab971a6ab454b025ce9037cb..abaa3d2e212c3aecc549b090c07f213dd8fc9a69 100644 (file)
--- a/src/msg/async/dpdk/dpdk_rte.cc
+++ b/src/msg/async/dpdk/dpdk_rte.cc
@@ -53,6 +53,12 @@ namespace dpdk {
     }
 
     bool done = false;
+    const char *hexstring = c->_conf->get_val<std::string>("ms_dpdk_coremaskbit").c_str();
+    int num = (int)strtol(hexstring, NULL, 0);
+    unsigned int coremaskbit = bitcount(num);
+
+    ceph_assert(coremaskbit > c->_conf->ms_async_op_threads);
+
     t = std::thread([&]() {
       // TODO: Inherit these from the app parameters - "opts"
       std::vector<std::vector<char>> args {