:Default: ``5``
-``ms async set affinity``
-
-:Description: Set to true to bind Async Messenger workers to particular CPU cores.
-:Type: Boolean
-:Required: No
-:Default: ``true``
-
-
-``ms async affinity cores``
-
-:Description: When ``ms async set affinity`` is true, this string specifies how Async
- Messenger workers are bound to CPU cores. For example, "0,2" will bind
- workers #1 and #2 to CPU cores #0 and #2, respectively.
- NOTE: when manually setting affinity, make sure to not assign workers to
- processors that are virtual CPUs created as an effect of Hyperthreading
- or similar technology, because they are slower than regular CPU cores.
-:Type: String
-:Required: No
-:Default: ``(empty)``
-
-
``ms async send inline``
:Description: Send messages directly from the thread that generated them instead of
OPTION(ms_dump_corrupt_message_level, OPT_INT) // debug level to hexdump undecodeable messages at
OPTION(ms_async_op_threads, OPT_U64) // number of worker processing threads for async messenger created on init
OPTION(ms_async_max_op_threads, OPT_U64) // max number of worker processing threads for async messenger
-OPTION(ms_async_set_affinity, OPT_BOOL)
-// example: ms_async_affinity_cores = 0,1
-// The number of coreset is expected to equal to ms_async_op_threads, otherwise
-// extra op threads will loop ms_async_affinity_cores again.
-// If ms_async_affinity_cores is empty, all threads will be bind to current running
-// core
-OPTION(ms_async_affinity_cores, OPT_STR)
OPTION(ms_async_rdma_device_name, OPT_STR)
OPTION(ms_async_rdma_enable_hugepage, OPT_BOOL)
OPTION(ms_async_rdma_buffer_size, OPT_INT)
.set_description("Maximum threadpool size of AsyncMessenger")
.add_see_also("ms_async_op_threads"),
- Option("ms_async_set_affinity", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
- .set_default(true)
- .set_description("Set CPU affinity for AsyncMessenger worker threads"),
-
- Option("ms_async_affinity_cores", Option::TYPE_STR, Option::LEVEL_ADVANCED)
- .set_default("")
- .set_description("List of cores to set worker thread affinity")
- .add_see_also("ms_async_set_affinity"),
-
Option("ms_async_rdma_device_name", Option::TYPE_STR, Option::LEVEL_ADVANCED)
.set_default("")
.set_description(""),
PosixNetworkStack::PosixNetworkStack(CephContext *c, const string &t)
: NetworkStack(c, t)
{
- vector<string> corestrs;
- get_str_vec(cct->_conf->ms_async_affinity_cores, corestrs);
- for (auto & corestr : corestrs) {
- string err;
- int coreid = strict_strtol(corestr.c_str(), 10, &err);
- if (err == "")
- coreids.push_back(coreid);
- else
- lderr(cct) << __func__ << " failed to parse " << corestr << " in " << cct->_conf->ms_async_affinity_cores << dendl;
- }
}
};
class PosixNetworkStack : public NetworkStack {
- vector<int> coreids;
vector<std::thread> threads;
public:
explicit PosixNetworkStack(CephContext *c, const string &t);
- int get_cpuid(int id) const {
- if (coreids.empty())
- return -1;
- return coreids[id % coreids.size()];
- }
void spawn_worker(unsigned i, std::function<void ()> &&func) override {
threads.resize(i+1);
threads[i] = std::thread(func);