From: Vu Pham Date: Wed, 23 Mar 2016 18:18:56 +0000 (-0700) Subject: xio: identify heavy messengers and properly allocate resources X-Git-Tag: v11.0.0~884^2~1 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=f729177d6daca8ea5b1b0445871dc8bfdb60f92a;p=ceph.git xio: identify heavy messengers and properly allocate resources Identify heavy messengers (osd client and cluster) and properly allocate resources ie. number of portals/contexts, max number of connection per portal/context; otherwise, configure with moderate defaults for those messengers Signed-off-by: Vu Pham --- diff --git a/src/common/config_opts.h b/src/common/config_opts.h index c2a577fa31d9..0f2b78830947 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -93,6 +93,7 @@ OPTION(xio_mp_max_1k, OPT_INT, 8192) // max 1K chunks OPTION(xio_mp_max_page, OPT_INT, 4096) // max 1K chunks OPTION(xio_mp_max_hint, OPT_INT, 4096) // max size-hint chunks OPTION(xio_portal_threads, OPT_INT, 2) // xio portal threads per messenger +OPTION(xio_max_conns_per_portal, OPT_INT, 32) // max xio_connections per portal/ctx OPTION(xio_transport_type, OPT_STR, "rdma") // xio transport type: {rdma or tcp} OPTION(xio_max_send_inline, OPT_INT, 512) // xio maximum threshold to send inline diff --git a/src/msg/xio/XioMessenger.cc b/src/msg/xio/XioMessenger.cc index e3afde040e14..e6ff7dbf1352 100644 --- a/src/msg/xio/XioMessenger.cc +++ b/src/msg/xio/XioMessenger.cc @@ -366,7 +366,7 @@ XioMessenger::XioMessenger(CephContext *cct, entity_name_t name, XioInit(cct), nsessions(0), shutdown_called(false), - portals(this, cct->_conf->xio_portal_threads), + portals(this, get_nportals(), get_nconns_per_portal()), dispatch_strategy(ds), loop_con(new XioLoopbackConnection(this)), special_handling(0), @@ -391,6 +391,12 @@ XioMessenger::XioMessenger(CephContext *cct, entity_name_t name, local_features = features; loop_con->set_features(features); + ldout(cct,2) << "Create msgr: " << this << " instance: " + << nInstances.read() << " type: " << name.type_str() + << " subtype: " << mname << " nportals: " << get_nportals() + << " nconns_per_portal: " << get_nconns_per_portal() << " features: " + << features << dendl; + } /* ctor */ int XioMessenger::pool_hint(uint32_t dsize) { @@ -403,6 +409,16 @@ int XioMessenger::pool_hint(uint32_t dsize) { XMSG_MEMPOOL_QUANTUM, 0); } +int XioMessenger::get_nconns_per_portal() +{ + return max(cct->_conf->xio_max_conns_per_portal, 32); +} + +int XioMessenger::get_nportals() +{ + return max(cct->_conf->xio_portal_threads, 1); +} + void XioMessenger::learned_addr(const entity_addr_t &peer_addr_for_me) { // be careful here: multiple threads may block here, and readers of @@ -516,9 +532,9 @@ int XioMessenger::session_event(struct xio_session *session, /* XXXX pre-merge of session startup negotiation ONLY! */ xcon->cstate.state_up_ready(XioConnection::CState::OP_FLAG_NONE); - ldout(cct,2) << "new connection session " << session - << " xcon " << xcon << dendl; - ldout(cct,2) << "server: connected from " << s_inst.addr << " to " << peer_addr_for_me << dendl; + ldout(cct,2) << "New connection session " << session + << " xcon " << xcon << " on msgr: " << this << " portal: " << xcon->portal << dendl; + ldout(cct,2) << "Server: connected from " << s_inst.addr << " to " << peer_addr_for_me << dendl; } break; case XIO_SESSION_CONNECTION_ERROR_EVENT: @@ -1011,8 +1027,9 @@ ConnectionRef XioMessenger::get_connection(const entity_inst_t& dest) /* XXXX pre-merge of session startup negotiation ONLY! */ xcon->cstate.state_up_ready(XioConnection::CState::OP_FLAG_NONE); - ldout(cct,2) << "new connection xcon: " << xcon << - " up_ready on session " << xcon->session << dendl; + ldout(cct,2) << "New connection xcon: " << xcon << + " up_ready on session " << xcon->session << + " on msgr: " << this << " portal: " << xcon->portal << dendl; return xcon->get(); /* nref +1 */ } diff --git a/src/msg/xio/XioMessenger.h b/src/msg/xio/XioMessenger.h index 448b8150366e..86f3e6595999 100644 --- a/src/msg/xio/XioMessenger.h +++ b/src/msg/xio/XioMessenger.h @@ -153,6 +153,9 @@ public: */ void learned_addr(const entity_addr_t& peer_addr_for_me); +private: + int get_nconns_per_portal(); + int get_nportals(); protected: virtual void ready() diff --git a/src/msg/xio/XioPortal.cc b/src/msg/xio/XioPortal.cc index c158764d3aad..f6911850dec4 100644 --- a/src/msg/xio/XioPortal.cc +++ b/src/msg/xio/XioPortal.cc @@ -63,7 +63,7 @@ int XioPortals::bind(struct xio_session_ops *ops, const string& base_uri, /* bind the portals */ for (size_t i = 0; i < portals.size(); i++) { if (!portals[i]) - portals[i] = new XioPortal(msgr); + portals[i] = new XioPortal(msgr, max_conns_per_ctx); uint16_t result_port; if (port != 0) { diff --git a/src/msg/xio/XioPortal.h b/src/msg/xio/XioPortal.h index 04e16d22a7e9..42d580d2b3ef 100644 --- a/src/msg/xio/XioPortal.h +++ b/src/msg/xio/XioPortal.h @@ -131,28 +131,28 @@ private: friend class XioMessenger; public: - explicit XioPortal(Messenger *_msgr) : - msgr(_msgr), ctx(NULL), server(NULL), submit_q(), xio_uri(""), - portal_id(NULL), _shutdown(false), drained(false), - magic(0), - special_handling(0) - { - pthread_spin_init(&sp, PTHREAD_PROCESS_PRIVATE); - pthread_mutex_init(&mtx, NULL); - - /* a portal is an xio_context and event loop */ - ctx = xio_context_create(NULL, 0 /* poll timeout */, -1 /* cpu hint */); - - /* associate this XioPortal object with the xio_context handle */ - struct xio_context_attr xca; - xca.user_context = this; - xio_modify_context(ctx, &xca, XIO_CONTEXT_ATTR_USER_CTX); - - if (magic & (MSG_MAGIC_XIO)) { - printf("XioPortal %p created ev_loop %p ctx %p\n", - this, ev_loop, ctx); - } - } + explicit XioPortal(Messenger *_msgr, int max_conns) : + msgr(_msgr), ctx(NULL), server(NULL), submit_q(), xio_uri(""), + portal_id(NULL), _shutdown(false), drained(false), + magic(0), + special_handling(0) + { + pthread_spin_init(&sp, PTHREAD_PROCESS_PRIVATE); + pthread_mutex_init(&mtx, NULL); + + struct xio_context_params ctx_params; + memset(&ctx_params, 0, sizeof(ctx_params)); + ctx_params.user_context = this; + /* + * hint to Accelio the total number of connections that will share + * this context's resources: internal primary task pool... + */ + ctx_params.max_conns_per_ctx = max_conns; + + /* a portal is an xio_context and event loop */ + ctx = xio_context_create(&ctx_params, 0 /* poll timeout */, -1 /* cpu hint */); + assert(ctx && "Whoops, failed to create portal/ctx"); + } int bind(struct xio_session_ops *ops, const string &base_uri, uint16_t port, uint16_t *assigned_port); @@ -355,20 +355,16 @@ private: vector portals; char **p_vec; int n; - int last_use; + int last_unused; + int max_conns_per_ctx; public: - XioPortals(Messenger *msgr, int _n) : p_vec(NULL) + XioPortals(Messenger *msgr, int _n, int nconns) : p_vec(NULL), last_unused(0) { + max_conns_per_ctx = nconns; + n = max(_n, 1); /* portal0 */ - portals.push_back(new XioPortal(msgr)); - last_use = 0; - - /* enforce at least two portals if bind */ - if (_n < 2) - _n = 2; - n = _n; - + portals.push_back(new XioPortal(msgr, nconns)); /* additional portals allocated on bind() */ } @@ -384,11 +380,11 @@ public: return n; } - int get_last_use() + int get_last_unused() { - int pix = last_use; - if (++last_use >= get_portals_len() - 1) - last_use = 0; + int pix = last_unused; + if (++last_unused >= get_portals_len()) + last_unused = 0; return pix; } @@ -405,11 +401,15 @@ public: void *cb_user_context) { const char **portals_vec = get_vec(); - int pix = get_last_use(); - - return xio_accept(session, - (const char **)&(portals_vec[pix]), - 1, NULL, 0); + int pix = get_last_unused(); + + if (pix == 0) { + return xio_accept(session, NULL, 0, NULL, 0); + } else { + return xio_accept(session, + (const char **)&(portals_vec[pix]), + 1, NULL, 0); + } } void start() @@ -417,14 +417,11 @@ public: XioPortal *portal; int p_ix, nportals = portals.size(); - /* portal_0 is the new-session handler, portal_1+ terminate - * active sessions */ - - p_vec = new char*[(nportals-1)]; - for (p_ix = 1; p_ix < nportals; ++p_ix) { + p_vec = new char*[nportals]; + for (p_ix = 0; p_ix < nportals; ++p_ix) { portal = portals[p_ix]; /* shift left */ - p_vec[(p_ix-1)] = (char*) /* portal->xio_uri.c_str() */ + p_vec[p_ix] = (char*) /* portal->xio_uri.c_str() */ portal->portal_id; }