From aa30423d139501019373e82b1f22fe8315893f15 Mon Sep 17 00:00:00 2001 From: Vu Pham Date: Wed, 23 Mar 2016 11:26:31 -0700 Subject: [PATCH] xio: alloc all portals and use for all connections Allocate all xio_portal_threads and assign/spread both passive and active connections to different portals of heavy messengers. Signed-off-by: Vu Pham --- src/msg/xio/XioConnection.cc | 2 +- src/msg/xio/XioMessenger.cc | 10 +++++----- src/msg/xio/XioMessenger.h | 2 +- src/msg/xio/XioPortal.cc | 3 --- src/msg/xio/XioPortal.h | 19 +++++++++++-------- 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/msg/xio/XioConnection.cc b/src/msg/xio/XioConnection.cc index 40069fad0cf..7f1cc0e913e 100644 --- a/src/msg/xio/XioConnection.cc +++ b/src/msg/xio/XioConnection.cc @@ -81,7 +81,7 @@ XioConnection::XioConnection(XioMessenger *m, XioConnection::type _type, const entity_inst_t& _peer) : Connection(m->cct, m), xio_conn_type(_type), - portal(m->default_portal()), + portal(m->get_portal()), connected(false), peer(_peer), session(NULL), diff --git a/src/msg/xio/XioMessenger.cc b/src/msg/xio/XioMessenger.cc index e6ff7dbf135..4631aee707f 100644 --- a/src/msg/xio/XioMessenger.cc +++ b/src/msg/xio/XioMessenger.cc @@ -541,9 +541,9 @@ int XioMessenger::session_event(struct xio_session *session, case XIO_SESSION_CONNECTION_CLOSED_EVENT: /* orderly discon */ case XIO_SESSION_CONNECTION_DISCONNECTED_EVENT: /* unexpected discon */ case XIO_SESSION_CONNECTION_REFUSED_EVENT: - ldout(cct,2) << xio_session_event_types[event_data->event] - << " user_context " << event_data->conn_user_context << dendl; xcon = static_cast(event_data->conn_user_context); + ldout(cct,2) << xio_session_event_types[event_data->event] + << " xcon " << xcon << " session " << session << dendl; if (likely(!!xcon)) { Spinlock::Locker lckr(conns_sp); XioConnection::EntitySet::iterator conn_iter = @@ -565,9 +565,9 @@ int XioMessenger::session_event(struct xio_session *session, } break; case XIO_SESSION_CONNECTION_TEARDOWN_EVENT: - ldout(cct,2) << xio_session_event_types[event_data->event] - << " user_context " << event_data->conn_user_context << dendl; xcon = static_cast(event_data->conn_user_context); + ldout(cct,2) << xio_session_event_types[event_data->event] + << " xcon " << xcon << " session " << session << dendl; xcon->on_teardown_event(); break; case XIO_SESSION_TEARDOWN_EVENT: @@ -1004,7 +1004,7 @@ ConnectionRef XioMessenger::get_connection(const entity_inst_t& dest) * we can always set it explicitly */ struct xio_connection_params xcp = {}; xcp.session = xcon->session; - xcp.ctx = this->portals.get_portal0()->ctx; + xcp.ctx = xcon->portal->ctx; xcp.conn_user_context = xcon; xcon->conn = xio_connect(&xcp); diff --git a/src/msg/xio/XioMessenger.h b/src/msg/xio/XioMessenger.h index 86f3e659599..a2679622257 100644 --- a/src/msg/xio/XioMessenger.h +++ b/src/msg/xio/XioMessenger.h @@ -68,7 +68,7 @@ public: virtual ~XioMessenger(); - XioPortal* default_portal() { return portals.get_portal0(); } + XioPortal* get_portal() { return portals.get_next_portal(); } virtual void set_myaddr(const entity_addr_t& a) { Messenger::set_myaddr(a); diff --git a/src/msg/xio/XioPortal.cc b/src/msg/xio/XioPortal.cc index f6911850dec..e2379fb3312 100644 --- a/src/msg/xio/XioPortal.cc +++ b/src/msg/xio/XioPortal.cc @@ -62,9 +62,6 @@ int XioPortals::bind(struct xio_session_ops *ops, const string& base_uri, /* bind the portals */ for (size_t i = 0; i < portals.size(); i++) { - if (!portals[i]) - portals[i] = new XioPortal(msgr, max_conns_per_ctx); - uint16_t result_port; if (port != 0) { // bind directly to the given port diff --git a/src/msg/xio/XioPortal.h b/src/msg/xio/XioPortal.h index 42d580d2b3e..e43f95a0cdb 100644 --- a/src/msg/xio/XioPortal.h +++ b/src/msg/xio/XioPortal.h @@ -356,16 +356,19 @@ private: char **p_vec; int n; int last_unused; - int max_conns_per_ctx; public: XioPortals(Messenger *msgr, int _n, int nconns) : p_vec(NULL), last_unused(0) { - max_conns_per_ctx = nconns; n = max(_n, 1); - /* portal0 */ - portals.push_back(new XioPortal(msgr, nconns)); - /* additional portals allocated on bind() */ + + portals.resize(n); + for (int i = 0; i < n; i++) { + if (!portals[i]) { + portals[i] = new XioPortal(msgr, nconns); + assert(portals[i] != nullptr); + } + } } vector& get() { return portals; } @@ -388,9 +391,10 @@ public: return pix; } - XioPortal* get_portal0() + XioPortal* get_next_portal() { - return portals[0]; + int pix = get_last_unused(); + return portals[pix]; } int bind(struct xio_session_ops *ops, const string& base_uri, @@ -420,7 +424,6 @@ public: p_vec = new char*[nportals]; for (p_ix = 0; p_ix < nportals; ++p_ix) { portal = portals[p_ix]; - /* shift left */ p_vec[p_ix] = (char*) /* portal->xio_uri.c_str() */ portal->portal_id; } -- 2.39.5