xio: identify heavy messengers and properly allocate resources

Identify heavy messengers (osd client and cluster) and properly
allocate resources ie. number of portals/contexts, max number of
connection per portal/context; otherwise, configure with moderate
defaults for those messengers

Signed-off-by: Vu Pham <vu@mellanox.com>

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index c2a577fa31d9a638b396d052ae77a06ef03742c9..0f2b78830947d371405720ed4293d2539383e707 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -93,6 +93,7 @@ OPTION(xio_mp_max_1k, OPT_INT, 8192) // max 1K chunks
 OPTION(xio_mp_max_page, OPT_INT, 4096) // max 1K chunks
 OPTION(xio_mp_max_hint, OPT_INT, 4096) // max size-hint chunks
 OPTION(xio_portal_threads, OPT_INT, 2) // xio portal threads per messenger
+OPTION(xio_max_conns_per_portal, OPT_INT, 32) // max xio_connections per portal/ctx
 OPTION(xio_transport_type, OPT_STR, "rdma") // xio transport type: {rdma or tcp}
 OPTION(xio_max_send_inline, OPT_INT, 512) // xio maximum threshold to send inline
 

diff --git a/src/msg/xio/XioMessenger.cc b/src/msg/xio/XioMessenger.cc
index e3afde040e14bb20124cdcf0422900aefbc285b9..e6ff7dbf1352337fcdf2d66b9b5b62553ea70d19 100644 (file)
--- a/src/msg/xio/XioMessenger.cc
+++ b/src/msg/xio/XioMessenger.cc
@@ -366,7 +366,7 @@ XioMessenger::XioMessenger(CephContext *cct, entity_name_t name,
     XioInit(cct),
     nsessions(0),
     shutdown_called(false),
-    portals(this, cct->_conf->xio_portal_threads),
+    portals(this, get_nportals(), get_nconns_per_portal()),
     dispatch_strategy(ds),
     loop_con(new XioLoopbackConnection(this)),
     special_handling(0),
@@ -391,6 +391,12 @@ XioMessenger::XioMessenger(CephContext *cct, entity_name_t name,
   local_features = features;
   loop_con->set_features(features);
 
+  ldout(cct,2) << "Create msgr: " << this << " instance: "
+    << nInstances.read() << " type: " << name.type_str()
+    << " subtype: " << mname << " nportals: " << get_nportals()
+    << " nconns_per_portal: " << get_nconns_per_portal() << " features: "
+    << features << dendl;
+
 } /* ctor */
 
 int XioMessenger::pool_hint(uint32_t dsize) {
@@ -403,6 +409,16 @@ int XioMessenger::pool_hint(uint32_t dsize) {
                                   XMSG_MEMPOOL_QUANTUM, 0);
 }
 
+int XioMessenger::get_nconns_per_portal()
+{
+  return max(cct->_conf->xio_max_conns_per_portal, 32);
+}
+
+int XioMessenger::get_nportals()
+{
+  return max(cct->_conf->xio_portal_threads, 1);
+}
+
 void XioMessenger::learned_addr(const entity_addr_t &peer_addr_for_me)
 {
   // be careful here: multiple threads may block here, and readers of
@@ -516,9 +532,9 @@ int XioMessenger::session_event(struct xio_session *session,
     /* XXXX pre-merge of session startup negotiation ONLY! */
     xcon->cstate.state_up_ready(XioConnection::CState::OP_FLAG_NONE);
 
-    ldout(cct,2) << "new connection session " << session
-                << " xcon " << xcon << dendl;
-    ldout(cct,2) << "server: connected from " << s_inst.addr << " to " << peer_addr_for_me << dendl;
+    ldout(cct,2) << "New connection session " << session
+      << " xcon " << xcon << " on msgr: " << this << " portal: " << xcon->portal << dendl;
+    ldout(cct,2) << "Server: connected from " << s_inst.addr << " to " << peer_addr_for_me << dendl;
   }
   break;
   case XIO_SESSION_CONNECTION_ERROR_EVENT:
@@ -1011,8 +1027,9 @@ ConnectionRef XioMessenger::get_connection(const entity_inst_t& dest)
     /* XXXX pre-merge of session startup negotiation ONLY! */
     xcon->cstate.state_up_ready(XioConnection::CState::OP_FLAG_NONE);
 
-    ldout(cct,2) << "new connection xcon: " << xcon <<
-      " up_ready on session " << xcon->session << dendl;
+    ldout(cct,2) << "New connection xcon: " << xcon <<
+      " up_ready on session " << xcon->session <<
+      " on msgr: " << this << " portal: " << xcon->portal << dendl;
 
     return xcon->get(); /* nref +1 */
   }

diff --git a/src/msg/xio/XioMessenger.h b/src/msg/xio/XioMessenger.h
index 448b8150366ebee82a389ab55d5ed50e68b23934..86f3e65959990f10aab12f81740d2009deed3357 100644 (file)
--- a/src/msg/xio/XioMessenger.h
+++ b/src/msg/xio/XioMessenger.h
@@ -153,6 +153,9 @@ public:
    */
   void learned_addr(const entity_addr_t& peer_addr_for_me);
 
+private:
+  int get_nconns_per_portal();
+  int get_nportals();
 
 protected:
   virtual void ready()

diff --git a/src/msg/xio/XioPortal.cc b/src/msg/xio/XioPortal.cc
index c158764d3aad38c26ee2177e797d0dda61330770..f6911850dec42044a51c2135cc10ea988f367a27 100644 (file)
--- a/src/msg/xio/XioPortal.cc
+++ b/src/msg/xio/XioPortal.cc
@@ -63,7 +63,7 @@ int XioPortals::bind(struct xio_session_ops *ops, const string& base_uri,
   /* bind the portals */
   for (size_t i = 0; i < portals.size(); i++) {
     if (!portals[i])
-      portals[i] = new XioPortal(msgr);
+      portals[i] = new XioPortal(msgr, max_conns_per_ctx);
 
     uint16_t result_port;
     if (port != 0) {

diff --git a/src/msg/xio/XioPortal.h b/src/msg/xio/XioPortal.h
index 04e16d22a7e940ba08416d81fb2a4d1e738ab8bb..42d580d2b3ef6f0cc65e704814820bb519d4a0e9 100644 (file)
--- a/src/msg/xio/XioPortal.h
+++ b/src/msg/xio/XioPortal.h
@@ -131,28 +131,28 @@ private:
   friend class XioMessenger;
 
 public:
-  explicit XioPortal(Messenger *_msgr) :
-  msgr(_msgr), ctx(NULL), server(NULL), submit_q(), xio_uri(""),
-  portal_id(NULL), _shutdown(false), drained(false),
-  magic(0),
-  special_handling(0)
-    {
-      pthread_spin_init(&sp, PTHREAD_PROCESS_PRIVATE);
-      pthread_mutex_init(&mtx, NULL);
-
-      /* a portal is an xio_context and event loop */
-      ctx = xio_context_create(NULL, 0 /* poll timeout */, -1 /* cpu hint */);
-
-      /* associate this XioPortal object with the xio_context handle */
-      struct xio_context_attr xca;
-      xca.user_context = this;
-      xio_modify_context(ctx, &xca, XIO_CONTEXT_ATTR_USER_CTX);
-
-      if (magic & (MSG_MAGIC_XIO)) {
-       printf("XioPortal %p created ev_loop %p ctx %p\n",
-              this, ev_loop, ctx);
-      }
-    }
+  explicit XioPortal(Messenger *_msgr, int max_conns) :
+    msgr(_msgr), ctx(NULL), server(NULL), submit_q(), xio_uri(""),
+    portal_id(NULL), _shutdown(false), drained(false),
+    magic(0),
+    special_handling(0)
+  {
+    pthread_spin_init(&sp, PTHREAD_PROCESS_PRIVATE);
+    pthread_mutex_init(&mtx, NULL);
+
+    struct xio_context_params ctx_params;
+    memset(&ctx_params, 0, sizeof(ctx_params));
+    ctx_params.user_context = this;
+    /*
+     * hint to Accelio the total number of connections that will share
+     * this context's resources: internal primary task pool...
+     */
+    ctx_params.max_conns_per_ctx = max_conns;
+
+    /* a portal is an xio_context and event loop */
+    ctx = xio_context_create(&ctx_params, 0 /* poll timeout */, -1 /* cpu hint */);
+    assert(ctx && "Whoops, failed to create portal/ctx");
+  }
 
   int bind(struct xio_session_ops *ops, const string &base_uri,
           uint16_t port, uint16_t *assigned_port);
@@ -355,20 +355,16 @@ private:
   vector<XioPortal*> portals;
   char **p_vec;
   int n;
-  int last_use;
+  int last_unused;
+  int max_conns_per_ctx;
 
 public:
-  XioPortals(Messenger *msgr, int _n) : p_vec(NULL)
+  XioPortals(Messenger *msgr, int _n, int nconns) : p_vec(NULL), last_unused(0)
   {
+    max_conns_per_ctx = nconns;
+    n = max(_n, 1);
     /* portal0 */
-    portals.push_back(new XioPortal(msgr));
-    last_use = 0;
-
-    /* enforce at least two portals if bind */
-    if (_n < 2)
-      _n = 2;
-    n = _n;
-
+    portals.push_back(new XioPortal(msgr, nconns));
     /* additional portals allocated on bind() */
   }
 
@@ -384,11 +380,11 @@ public:
     return n;
   }
 
-  int get_last_use()
+  int get_last_unused()
   {
-    int pix = last_use;
-    if (++last_use >= get_portals_len() - 1)
-      last_use = 0;
+    int pix = last_unused;
+    if (++last_unused >= get_portals_len())
+      last_unused = 0;
     return pix;
   }
 
@@ -405,11 +401,15 @@ public:
             void *cb_user_context)
   {
     const char **portals_vec = get_vec();
-    int pix = get_last_use();
-
-    return xio_accept(session,
-                     (const char **)&(portals_vec[pix]),
-                     1, NULL, 0);
+    int pix = get_last_unused();
+
+    if (pix == 0) {
+      return xio_accept(session, NULL, 0, NULL, 0);
+    } else {
+      return xio_accept(session,
+                       (const char **)&(portals_vec[pix]),
+                       1, NULL, 0);
+    }
   }
 
   void start()
@@ -417,14 +417,11 @@ public:
     XioPortal *portal;
     int p_ix, nportals = portals.size();
 
-    /* portal_0 is the new-session handler, portal_1+ terminate
-     * active sessions */
-
-    p_vec = new char*[(nportals-1)];
-    for (p_ix = 1; p_ix < nportals; ++p_ix) {
+    p_vec = new char*[nportals];
+    for (p_ix = 0; p_ix < nportals; ++p_ix) {
       portal = portals[p_ix];
       /* shift left */
-      p_vec[(p_ix-1)] = (char*) /* portal->xio_uri.c_str() */
+      p_vec[p_ix] = (char*) /* portal->xio_uri.c_str() */
                        portal->portal_id;
     }