msgr,xio: flexible Messenger::create configuration flags

Widen Messenger::create and XioMessenger constructor to support
per-Messenger instance creation parameters.

This introduce a minimalist generic set of flags to describe
the type of Messenger and its associated resources.

We apply the usage of these flags to ceph-osd's "workhorse",
"heartbeat" and "light" Messenger instances, ceph-mon and
other ceph clients Messengers.

Signed-off-by: Vu Pham <vu@mellanox.com>

diff --git a/src/ceph_mds.cc b/src/ceph_mds.cc
index efe22e6b995ffc59fc91cd5877ee6decffd20aff..ec8b97b4d7489fd736b23ba17bfeec2389288e53 100644 (file)
--- a/src/ceph_mds.cc
+++ b/src/ceph_mds.cc
@@ -138,7 +138,7 @@ int main(int argc, const char **argv)
 
   Messenger *msgr = Messenger::create(g_ceph_context, g_conf->ms_type,
                                      entity_name_t::MDS(-1), "mds",
-                                     getpid());
+                                     getpid(), 0, Messenger::HAS_MANY_CONNECTIONS);
   if (!msgr)
     exit(1);
   msgr->set_cluster_protocol(CEPH_MDS_PROTOCOL);

diff --git a/src/ceph_mon.cc b/src/ceph_mon.cc
index 4bb5c4a5793fcd87d22bed7cd1cae3f158c0e1e9..68b6e371e69460fc2707b48cbf2223c775e48b86 100644 (file)
--- a/src/ceph_mon.cc
+++ b/src/ceph_mon.cc
@@ -663,9 +663,8 @@ int main(int argc, const char **argv)
   // bind
   int rank = monmap.get_rank(g_conf->name.get_id());
   Messenger *msgr = Messenger::create(g_ceph_context, g_conf->ms_type,
-                                     entity_name_t::MON(rank),
-                                     "mon",
-                                     0);
+                                     entity_name_t::MON(rank), "mon",
+                                     0, 0, Messenger::HAS_MANY_CONNECTIONS);
   if (!msgr)
     exit(1);
   msgr->set_cluster_protocol(CEPH_MON_PROTOCOL);

diff --git a/src/ceph_osd.cc b/src/ceph_osd.cc
index 7deb5a159c9520b2973948d409415e2049e905aa..33fafe1a3830f734be5042fe5621243d8321c3d9 100644 (file)
--- a/src/ceph_osd.cc
+++ b/src/ceph_osd.cc
@@ -447,19 +447,23 @@ int main(int argc, const char **argv)
 
   Messenger *ms_public = Messenger::create(g_ceph_context, g_conf->ms_type,
                                           entity_name_t::OSD(whoami), "client",
-                                          getpid());
+                                          getpid(), 0,
+                                          Messenger::HAS_HEAVY_TRAFFIC |
+                                          Messenger::HAS_MANY_CONNECTIONS);
   Messenger *ms_cluster = Messenger::create(g_ceph_context, g_conf->ms_type,
                                            entity_name_t::OSD(whoami), "cluster",
-                                           getpid(), CEPH_FEATURES_ALL);
+                                           getpid(), CEPH_FEATURES_ALL,
+                                           Messenger::HAS_HEAVY_TRAFFIC |
+                                           Messenger::HAS_MANY_CONNECTIONS);
   Messenger *ms_hbclient = Messenger::create(g_ceph_context, g_conf->ms_type,
                                             entity_name_t::OSD(whoami), "hbclient",
-                                            getpid());
+                                            getpid(), 0, Messenger::HEARTBEAT);
   Messenger *ms_hb_back_server = Messenger::create(g_ceph_context, g_conf->ms_type,
                                                   entity_name_t::OSD(whoami), "hb_back_server",
-                                                  getpid());
+                                                  getpid(), 0, Messenger::HEARTBEAT);
   Messenger *ms_hb_front_server = Messenger::create(g_ceph_context, g_conf->ms_type,
                                                    entity_name_t::OSD(whoami), "hb_front_server",
-                                                   getpid());
+                                                   getpid(), 0, Messenger::HEARTBEAT);
   Messenger *ms_objecter = Messenger::create(g_ceph_context, g_conf->ms_type,
                                             entity_name_t::OSD(whoami), "ms_objecter",
                                             getpid());

diff --git a/src/msg/Messenger.cc b/src/msg/Messenger.cc
index 98cc86fcfad3439e283d49c0f86f7af8513dfc94..b6534f9a3208eece2cc2800f159fbabf8f3b2204 100644 (file)
--- a/src/msg/Messenger.cc
+++ b/src/msg/Messenger.cc
@@ -21,7 +21,7 @@ Messenger *Messenger::create_client_messenger(CephContext *cct, string lname)
 
 Messenger *Messenger::create(CephContext *cct, const string &type,
                             entity_name_t name, string lname,
-                            uint64_t nonce, uint64_t features)
+                            uint64_t nonce, uint64_t features, uint64_t cflags)
 {
   int r = -1;
   if (type == "random") {
@@ -36,7 +36,7 @@ Messenger *Messenger::create(CephContext *cct, const string &type,
 #ifdef HAVE_XIO
   else if ((type == "xio") &&
           cct->check_experimental_feature_enabled("ms-type-xio"))
-    return new XioMessenger(cct, name, lname, nonce, features);
+    return new XioMessenger(cct, name, lname, nonce, features, cflags);
 #endif
   lderr(cct) << "unrecognized ms_type '" << type << "'" << dendl;
   return nullptr;

diff --git a/src/msg/Messenger.h b/src/msg/Messenger.h
index 8acfd6816ddf2969c729a85a5ba4f847c3085e4e..08121169bfc7b2556e78f548042632f40a8b9da7 100644 (file)
--- a/src/msg/Messenger.h
+++ b/src/msg/Messenger.h
@@ -52,6 +52,14 @@ protected:
   int socket_priority;
 
 public:
+  /**
+   * Various Messenger conditional config/type flags to allow
+   * different "transport" Messengers to tune themselves
+   */
+  static const int HAS_HEAVY_TRAFFIC    = 0x0001;
+  static const int HAS_MANY_CONNECTIONS = 0x0002;
+  static const int HEARTBEAT            = 0x0004;
+
   /**
    *  The CephContext this Messenger uses. Many other components initialize themselves
    *  from this value.
@@ -151,13 +159,15 @@ public:
    * @param lname logical name of the messenger in this process (e.g., "client")
    * @param nonce nonce value to uniquely identify this instance on the current host
    * @param features bits for the local connection
+   * @param cflags general set of flags to configure transport resources
    */
   static Messenger *create(CephContext *cct,
                            const string &type,
                            entity_name_t name,
                           string lname,
                            uint64_t nonce,
-                          uint64_t features = 0);
+                          uint64_t features = 0,
+                          uint64_t cflags = 0);
 
   /**
    * create a new messenger

diff --git a/src/msg/xio/XioMessenger.cc b/src/msg/xio/XioMessenger.cc
index 5233ce5066dec1e0cc2cad27342668ab9838ab1b..e20dd3c6fb1bed03175f8c9b3f19c0b69ff15816 100644 (file)
--- a/src/msg/xio/XioMessenger.cc
+++ b/src/msg/xio/XioMessenger.cc
@@ -351,12 +351,12 @@ static ostream& _prefix(std::ostream *_dout, XioMessenger *msgr) {
 
 XioMessenger::XioMessenger(CephContext *cct, entity_name_t name,
                           string mname, uint64_t _nonce, uint64_t features,
-                          DispatchStrategy *ds)
+                          uint64_t cflags, DispatchStrategy *ds)
   : SimplePolicyMessenger(cct, name, mname, _nonce),
     XioInit(cct),
     nsessions(0),
     shutdown_called(false),
-    portals(this, get_nportals(), get_nconns_per_portal()),
+    portals(this, get_nportals(cflags), get_nconns_per_portal(cflags)),
     dispatch_strategy(ds),
     loop_con(new XioLoopbackConnection(this)),
     special_handling(0),
@@ -383,8 +383,8 @@ XioMessenger::XioMessenger(CephContext *cct, entity_name_t name,
 
   ldout(cct,2) << "Create msgr: " << this << " instance: "
     << nInstances.read() << " type: " << name.type_str()
-    << " subtype: " << mname << " nportals: " << get_nportals()
-    << " nconns_per_portal: " << get_nconns_per_portal() << " features: "
+    << " subtype: " << mname << " nportals: " << get_nportals(cflags)
+    << " nconns_per_portal: " << get_nconns_per_portal(cflags) << " features: "
     << features << dendl;
 
 } /* ctor */
@@ -399,14 +399,27 @@ int XioMessenger::pool_hint(uint32_t dsize) {
                                   XMSG_MEMPOOL_QUANTUM, 0);
 }
 
-int XioMessenger::get_nconns_per_portal()
+int XioMessenger::get_nconns_per_portal(uint64_t cflags)
 {
-  return max(cct->_conf->xio_max_conns_per_portal, 32);
+  const int XIO_DEFAULT_NUM_CONNS_PER_PORTAL = 8;
+  int nconns = XIO_DEFAULT_NUM_CONNS_PER_PORTAL;
+
+  if (cflags & Messenger::HAS_MANY_CONNECTIONS)
+    nconns = max(cct->_conf->xio_max_conns_per_portal, XIO_DEFAULT_NUM_CONNS_PER_PORTAL);
+  else if (cflags & Messenger::HEARTBEAT)
+    nconns = max(cct->_conf->osd_heartbeat_min_peers * 4, XIO_DEFAULT_NUM_CONNS_PER_PORTAL);
+
+  return nconns;
 }
 
-int XioMessenger::get_nportals()
+int XioMessenger::get_nportals(uint64_t cflags)
 {
-  return max(cct->_conf->xio_portal_threads, 1);
+  int nportals = 1;
+
+  if (cflags & Messenger::HAS_HEAVY_TRAFFIC)
+    nportals = max(cct->_conf->xio_portal_threads, 1);
+
+  return nportals;
 }
 
 void XioMessenger::learned_addr(const entity_addr_t &peer_addr_for_me)

diff --git a/src/msg/xio/XioMessenger.h b/src/msg/xio/XioMessenger.h
index 56dae250aeb8871c65dc4a7ddd62a93337879c65..db7edf6c8d960e4843877ee521f187709798eb18 100644 (file)
--- a/src/msg/xio/XioMessenger.h
+++ b/src/msg/xio/XioMessenger.h
@@ -64,6 +64,7 @@ private:
 public:
   XioMessenger(CephContext *cct, entity_name_t name,
               string mname, uint64_t nonce, uint64_t features,
+              uint64_t cflags = 0,
               DispatchStrategy* ds = new QueueStrategy(1));
 
   virtual ~XioMessenger();
@@ -149,8 +150,8 @@ public:
   void learned_addr(const entity_addr_t& peer_addr_for_me);
 
 private:
-  int get_nconns_per_portal();
-  int get_nportals();
+  int get_nconns_per_portal(uint64_t cflags);
+  int get_nportals(uint64_t cflags);
 
 protected:
   virtual void ready()

diff --git a/src/test/messenger/xio_client.cc b/src/test/messenger/xio_client.cc
index 414668f37e882e79cc1a2d2079b8b042e11dd763..cad3a0544678171132debacd115578bc1f492754 100644 (file)
--- a/src/test/messenger/xio_client.cc
+++ b/src/test/messenger/xio_client.cc
@@ -117,7 +117,7 @@ int main(int argc, const char **argv)
        messenger = new XioMessenger(g_ceph_context,
                                     entity_name_t::MON(-1),
                                     "xio_client",
-                                    0 /* nonce */, XIO_ALL_FEATURES,
+                                    0 /* nonce */, XIO_ALL_FEATURES, 0 /* cflags */,
                                     dstrategy);
 
        // enable timing prints

diff --git a/src/test/messenger/xio_server.cc b/src/test/messenger/xio_server.cc
index 039aa7b1b70a8e2a426132e84360980c699879e2..21ea4e0e5210ebd64f568b8d2c0b35c03b79d145 100644 (file)
--- a/src/test/messenger/xio_server.cc
+++ b/src/test/messenger/xio_server.cc
@@ -88,7 +88,7 @@ int main(int argc, const char **argv)
        messenger = new XioMessenger(g_ceph_context,
                                     entity_name_t::MON(-1),
                                     "xio_server",
-                                    0 /* nonce */, XIO_ALL_FEATURES,
+                                    0 /* nonce */, XIO_ALL_FEATURES, 0 /* cflags */,
                                     dstrategy);
 
        static_cast<XioMessenger*>(messenger)->set_magic(