From: Vu Pham Date: Mon, 23 May 2016 20:52:53 +0000 (-0700) Subject: msgr,xio: flexible Messenger::create configuration flags X-Git-Tag: v11.0.0~269^2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=5b218fb7940c5fbb66f0d52f479580c978a1e3ea;p=ceph.git msgr,xio: flexible Messenger::create configuration flags Widen Messenger::create and XioMessenger constructor to support per-Messenger instance creation parameters. This introduce a minimalist generic set of flags to describe the type of Messenger and its associated resources. We apply the usage of these flags to ceph-osd's "workhorse", "heartbeat" and "light" Messenger instances, ceph-mon and other ceph clients Messengers. Signed-off-by: Vu Pham --- diff --git a/src/ceph_mds.cc b/src/ceph_mds.cc index efe22e6b995f..ec8b97b4d748 100644 --- a/src/ceph_mds.cc +++ b/src/ceph_mds.cc @@ -138,7 +138,7 @@ int main(int argc, const char **argv) Messenger *msgr = Messenger::create(g_ceph_context, g_conf->ms_type, entity_name_t::MDS(-1), "mds", - getpid()); + getpid(), 0, Messenger::HAS_MANY_CONNECTIONS); if (!msgr) exit(1); msgr->set_cluster_protocol(CEPH_MDS_PROTOCOL); diff --git a/src/ceph_mon.cc b/src/ceph_mon.cc index 4bb5c4a5793f..68b6e371e694 100644 --- a/src/ceph_mon.cc +++ b/src/ceph_mon.cc @@ -663,9 +663,8 @@ int main(int argc, const char **argv) // bind int rank = monmap.get_rank(g_conf->name.get_id()); Messenger *msgr = Messenger::create(g_ceph_context, g_conf->ms_type, - entity_name_t::MON(rank), - "mon", - 0); + entity_name_t::MON(rank), "mon", + 0, 0, Messenger::HAS_MANY_CONNECTIONS); if (!msgr) exit(1); msgr->set_cluster_protocol(CEPH_MON_PROTOCOL); diff --git a/src/ceph_osd.cc b/src/ceph_osd.cc index 7deb5a159c95..33fafe1a3830 100644 --- a/src/ceph_osd.cc +++ b/src/ceph_osd.cc @@ -447,19 +447,23 @@ int main(int argc, const char **argv) Messenger *ms_public = Messenger::create(g_ceph_context, g_conf->ms_type, entity_name_t::OSD(whoami), "client", - getpid()); + getpid(), 0, + Messenger::HAS_HEAVY_TRAFFIC | + Messenger::HAS_MANY_CONNECTIONS); Messenger *ms_cluster = Messenger::create(g_ceph_context, g_conf->ms_type, entity_name_t::OSD(whoami), "cluster", - getpid(), CEPH_FEATURES_ALL); + getpid(), CEPH_FEATURES_ALL, + Messenger::HAS_HEAVY_TRAFFIC | + Messenger::HAS_MANY_CONNECTIONS); Messenger *ms_hbclient = Messenger::create(g_ceph_context, g_conf->ms_type, entity_name_t::OSD(whoami), "hbclient", - getpid()); + getpid(), 0, Messenger::HEARTBEAT); Messenger *ms_hb_back_server = Messenger::create(g_ceph_context, g_conf->ms_type, entity_name_t::OSD(whoami), "hb_back_server", - getpid()); + getpid(), 0, Messenger::HEARTBEAT); Messenger *ms_hb_front_server = Messenger::create(g_ceph_context, g_conf->ms_type, entity_name_t::OSD(whoami), "hb_front_server", - getpid()); + getpid(), 0, Messenger::HEARTBEAT); Messenger *ms_objecter = Messenger::create(g_ceph_context, g_conf->ms_type, entity_name_t::OSD(whoami), "ms_objecter", getpid()); diff --git a/src/msg/Messenger.cc b/src/msg/Messenger.cc index 98cc86fcfad3..b6534f9a3208 100644 --- a/src/msg/Messenger.cc +++ b/src/msg/Messenger.cc @@ -21,7 +21,7 @@ Messenger *Messenger::create_client_messenger(CephContext *cct, string lname) Messenger *Messenger::create(CephContext *cct, const string &type, entity_name_t name, string lname, - uint64_t nonce, uint64_t features) + uint64_t nonce, uint64_t features, uint64_t cflags) { int r = -1; if (type == "random") { @@ -36,7 +36,7 @@ Messenger *Messenger::create(CephContext *cct, const string &type, #ifdef HAVE_XIO else if ((type == "xio") && cct->check_experimental_feature_enabled("ms-type-xio")) - return new XioMessenger(cct, name, lname, nonce, features); + return new XioMessenger(cct, name, lname, nonce, features, cflags); #endif lderr(cct) << "unrecognized ms_type '" << type << "'" << dendl; return nullptr; diff --git a/src/msg/Messenger.h b/src/msg/Messenger.h index 8acfd6816ddf..08121169bfc7 100644 --- a/src/msg/Messenger.h +++ b/src/msg/Messenger.h @@ -52,6 +52,14 @@ protected: int socket_priority; public: + /** + * Various Messenger conditional config/type flags to allow + * different "transport" Messengers to tune themselves + */ + static const int HAS_HEAVY_TRAFFIC = 0x0001; + static const int HAS_MANY_CONNECTIONS = 0x0002; + static const int HEARTBEAT = 0x0004; + /** * The CephContext this Messenger uses. Many other components initialize themselves * from this value. @@ -151,13 +159,15 @@ public: * @param lname logical name of the messenger in this process (e.g., "client") * @param nonce nonce value to uniquely identify this instance on the current host * @param features bits for the local connection + * @param cflags general set of flags to configure transport resources */ static Messenger *create(CephContext *cct, const string &type, entity_name_t name, string lname, uint64_t nonce, - uint64_t features = 0); + uint64_t features = 0, + uint64_t cflags = 0); /** * create a new messenger diff --git a/src/msg/xio/XioMessenger.cc b/src/msg/xio/XioMessenger.cc index 5233ce5066de..e20dd3c6fb1b 100644 --- a/src/msg/xio/XioMessenger.cc +++ b/src/msg/xio/XioMessenger.cc @@ -351,12 +351,12 @@ static ostream& _prefix(std::ostream *_dout, XioMessenger *msgr) { XioMessenger::XioMessenger(CephContext *cct, entity_name_t name, string mname, uint64_t _nonce, uint64_t features, - DispatchStrategy *ds) + uint64_t cflags, DispatchStrategy *ds) : SimplePolicyMessenger(cct, name, mname, _nonce), XioInit(cct), nsessions(0), shutdown_called(false), - portals(this, get_nportals(), get_nconns_per_portal()), + portals(this, get_nportals(cflags), get_nconns_per_portal(cflags)), dispatch_strategy(ds), loop_con(new XioLoopbackConnection(this)), special_handling(0), @@ -383,8 +383,8 @@ XioMessenger::XioMessenger(CephContext *cct, entity_name_t name, ldout(cct,2) << "Create msgr: " << this << " instance: " << nInstances.read() << " type: " << name.type_str() - << " subtype: " << mname << " nportals: " << get_nportals() - << " nconns_per_portal: " << get_nconns_per_portal() << " features: " + << " subtype: " << mname << " nportals: " << get_nportals(cflags) + << " nconns_per_portal: " << get_nconns_per_portal(cflags) << " features: " << features << dendl; } /* ctor */ @@ -399,14 +399,27 @@ int XioMessenger::pool_hint(uint32_t dsize) { XMSG_MEMPOOL_QUANTUM, 0); } -int XioMessenger::get_nconns_per_portal() +int XioMessenger::get_nconns_per_portal(uint64_t cflags) { - return max(cct->_conf->xio_max_conns_per_portal, 32); + const int XIO_DEFAULT_NUM_CONNS_PER_PORTAL = 8; + int nconns = XIO_DEFAULT_NUM_CONNS_PER_PORTAL; + + if (cflags & Messenger::HAS_MANY_CONNECTIONS) + nconns = max(cct->_conf->xio_max_conns_per_portal, XIO_DEFAULT_NUM_CONNS_PER_PORTAL); + else if (cflags & Messenger::HEARTBEAT) + nconns = max(cct->_conf->osd_heartbeat_min_peers * 4, XIO_DEFAULT_NUM_CONNS_PER_PORTAL); + + return nconns; } -int XioMessenger::get_nportals() +int XioMessenger::get_nportals(uint64_t cflags) { - return max(cct->_conf->xio_portal_threads, 1); + int nportals = 1; + + if (cflags & Messenger::HAS_HEAVY_TRAFFIC) + nportals = max(cct->_conf->xio_portal_threads, 1); + + return nportals; } void XioMessenger::learned_addr(const entity_addr_t &peer_addr_for_me) diff --git a/src/msg/xio/XioMessenger.h b/src/msg/xio/XioMessenger.h index 56dae250aeb8..db7edf6c8d96 100644 --- a/src/msg/xio/XioMessenger.h +++ b/src/msg/xio/XioMessenger.h @@ -64,6 +64,7 @@ private: public: XioMessenger(CephContext *cct, entity_name_t name, string mname, uint64_t nonce, uint64_t features, + uint64_t cflags = 0, DispatchStrategy* ds = new QueueStrategy(1)); virtual ~XioMessenger(); @@ -149,8 +150,8 @@ public: void learned_addr(const entity_addr_t& peer_addr_for_me); private: - int get_nconns_per_portal(); - int get_nportals(); + int get_nconns_per_portal(uint64_t cflags); + int get_nportals(uint64_t cflags); protected: virtual void ready() diff --git a/src/test/messenger/xio_client.cc b/src/test/messenger/xio_client.cc index 414668f37e88..cad3a0544678 100644 --- a/src/test/messenger/xio_client.cc +++ b/src/test/messenger/xio_client.cc @@ -117,7 +117,7 @@ int main(int argc, const char **argv) messenger = new XioMessenger(g_ceph_context, entity_name_t::MON(-1), "xio_client", - 0 /* nonce */, XIO_ALL_FEATURES, + 0 /* nonce */, XIO_ALL_FEATURES, 0 /* cflags */, dstrategy); // enable timing prints diff --git a/src/test/messenger/xio_server.cc b/src/test/messenger/xio_server.cc index 039aa7b1b70a..21ea4e0e5210 100644 --- a/src/test/messenger/xio_server.cc +++ b/src/test/messenger/xio_server.cc @@ -88,7 +88,7 @@ int main(int argc, const char **argv) messenger = new XioMessenger(g_ceph_context, entity_name_t::MON(-1), "xio_server", - 0 /* nonce */, XIO_ALL_FEATURES, + 0 /* nonce */, XIO_ALL_FEATURES, 0 /* cflags */, dstrategy); static_cast(messenger)->set_magic(