From: Haomai Wang Date: Wed, 16 Nov 2016 15:10:40 +0000 (+0800) Subject: osd: separate hb client messenger to public and cluster X-Git-Tag: v12.0.0~112^2~3 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=551cfee828349e5f20daa61d3bf5831a9c99844d;p=ceph.git osd: separate hb client messenger to public and cluster Signed-off-by: Haomai Wang --- diff --git a/src/ceph_osd.cc b/src/ceph_osd.cc index 31c0e637c8dd..334fb7fa8e17 100644 --- a/src/ceph_osd.cc +++ b/src/ceph_osd.cc @@ -450,8 +450,11 @@ int main(int argc, const char **argv) getpid(), Messenger::HAS_HEAVY_TRAFFIC | Messenger::HAS_MANY_CONNECTIONS); - Messenger *ms_hbclient = Messenger::create(g_ceph_context, g_conf->ms_type, - entity_name_t::OSD(whoami), "hbclient", + Messenger *ms_hb_back_client = Messenger::create(g_ceph_context, g_conf->ms_type, + entity_name_t::OSD(whoami), "hb_back_client", + getpid(), Messenger::HEARTBEAT); + Messenger *ms_hb_front_client = Messenger::create(g_ceph_context, g_conf->ms_type, + entity_name_t::OSD(whoami), "hb_front_client", getpid(), Messenger::HEARTBEAT); Messenger *ms_hb_back_server = Messenger::create(g_ceph_context, g_conf->ms_type, entity_name_t::OSD(whoami), "hb_back_server", @@ -462,10 +465,11 @@ int main(int argc, const char **argv) Messenger *ms_objecter = Messenger::create(g_ceph_context, g_conf->ms_type, entity_name_t::OSD(whoami), "ms_objecter", getpid(), 0); - if (!ms_public || !ms_cluster || !ms_hbclient || !ms_hb_back_server || !ms_hb_front_server || !ms_objecter) + if (!ms_public || !ms_cluster || !ms_hb_front_client || !ms_hb_back_client || !ms_hb_back_server || !ms_hb_front_server || !ms_objecter) exit(1); ms_cluster->set_cluster_protocol(CEPH_OSD_PROTOCOL); - ms_hbclient->set_cluster_protocol(CEPH_OSD_PROTOCOL); + ms_hb_front_client->set_cluster_protocol(CEPH_OSD_PROTOCOL); + ms_hb_back_client->set_cluster_protocol(CEPH_OSD_PROTOCOL); ms_hb_back_server->set_cluster_protocol(CEPH_OSD_PROTOCOL); ms_hb_front_server->set_cluster_protocol(CEPH_OSD_PROTOCOL); @@ -523,7 +527,9 @@ int main(int argc, const char **argv) ms_cluster->set_policy(entity_name_t::TYPE_CLIENT, Messenger::Policy::stateless_server(0, 0)); - ms_hbclient->set_policy(entity_name_t::TYPE_OSD, + ms_hb_front_client->set_policy(entity_name_t::TYPE_OSD, + Messenger::Policy::lossy_client(0, 0)); + ms_hb_back_client->set_policy(entity_name_t::TYPE_OSD, Messenger::Policy::lossy_client(0, 0)); ms_hb_back_server->set_policy(entity_name_t::TYPE_OSD, Messenger::Policy::stateless_server(0, 0)); @@ -540,7 +546,8 @@ int main(int argc, const char **argv) exit(1); if (g_conf->osd_heartbeat_use_min_delay_socket) { - ms_hbclient->set_socket_priority(SOCKET_PRIORITY_MIN_DELAY); + ms_hb_front_client->set_socket_priority(SOCKET_PRIORITY_MIN_DELAY); + ms_hb_back_client->set_socket_priority(SOCKET_PRIORITY_MIN_DELAY); ms_hb_back_server->set_socket_priority(SOCKET_PRIORITY_MIN_DELAY); ms_hb_front_server->set_socket_priority(SOCKET_PRIORITY_MIN_DELAY); } @@ -586,7 +593,8 @@ int main(int argc, const char **argv) whoami, ms_cluster, ms_public, - ms_hbclient, + ms_hb_front_client, + ms_hb_back_client, ms_hb_front_server, ms_hb_back_server, ms_objecter, @@ -602,7 +610,8 @@ int main(int argc, const char **argv) } ms_public->start(); - ms_hbclient->start(); + ms_hb_front_client->start(); + ms_hb_back_client->start(); ms_hb_front_server->start(); ms_hb_back_server->start(); ms_cluster->start(); @@ -632,7 +641,8 @@ int main(int argc, const char **argv) kill(getpid(), SIGTERM); ms_public->wait(); - ms_hbclient->wait(); + ms_hb_front_client->wait(); + ms_hb_back_client->wait(); ms_hb_front_server->wait(); ms_hb_back_server->wait(); ms_cluster->wait(); @@ -646,7 +656,8 @@ int main(int argc, const char **argv) // done delete osd; delete ms_public; - delete ms_hbclient; + delete ms_hb_front_client; + delete ms_hb_back_client; delete ms_hb_front_server; delete ms_hb_back_server; delete ms_cluster; diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index 0347b15117d9..0aa9c0801f15 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -836,9 +836,9 @@ pair OSDService::get_con_osd_hb(int peer, epoch_t f release_map(next_map); return ret; } - ret.first = osd->hbclient_messenger->get_connection(next_map->get_hb_back_inst(peer)); + ret.first = osd->hb_back_client_messenger->get_connection(next_map->get_hb_back_inst(peer)); if (next_map->get_hb_front_addr(peer) != entity_addr_t()) - ret.second = osd->hbclient_messenger->get_connection(next_map->get_hb_front_inst(peer)); + ret.second = osd->hb_front_client_messenger->get_connection(next_map->get_hb_front_inst(peer)); release_map(next_map); return ret; } @@ -1636,7 +1636,8 @@ OSD::OSD(CephContext *cct_, ObjectStore *store_, int id, Messenger *internal_messenger, Messenger *external_messenger, - Messenger *hb_clientm, + Messenger *hb_client_front, + Messenger *hb_client_back, Messenger *hb_front_serverm, Messenger *hb_back_serverm, Messenger *osdc_messenger, @@ -1678,7 +1679,8 @@ OSD::OSD(CephContext *cct_, ObjectStore *store_, heartbeat_lock("OSD::heartbeat_lock"), heartbeat_stop(false), heartbeat_need_update(true), - hbclient_messenger(hb_clientm), + hb_front_client_messenger(hb_client_front), + hb_back_client_messenger(hb_client_back), hb_front_server_messenger(hb_front_serverm), hb_back_server_messenger(hb_back_serverm), daily_loadavg(0.0), @@ -2187,7 +2189,8 @@ int OSD::init() client_messenger->add_dispatcher_head(this); cluster_messenger->add_dispatcher_head(this); - hbclient_messenger->add_dispatcher_head(&heartbeat_dispatcher); + hb_front_client_messenger->add_dispatcher_head(&heartbeat_dispatcher); + hb_back_client_messenger->add_dispatcher_head(&heartbeat_dispatcher); hb_front_server_messenger->add_dispatcher_head(&heartbeat_dispatcher); hb_back_server_messenger->add_dispatcher_head(&heartbeat_dispatcher); @@ -2858,7 +2861,8 @@ int OSD::shutdown() class_handler->shutdown(); client_messenger->shutdown(); cluster_messenger->shutdown(); - hbclient_messenger->shutdown(); + hb_front_client_messenger->shutdown(); + hb_back_client_messenger->shutdown(); objecter_messenger->shutdown(); hb_front_server_messenger->shutdown(); hb_back_server_messenger->shutdown(); @@ -7212,7 +7216,8 @@ void OSD::_committed_osd_maps(epoch_t first, epoch_t last, MOSDMap *m) << " rebind hb_front_server_messenger failed" << dendl; } - hbclient_messenger->mark_down_all(); + hb_front_client_messenger->mark_down_all(); + hb_back_client_messenger->mark_down_all(); reset_heartbeat_peers(); } diff --git a/src/osd/OSD.h b/src/osd/OSD.h index 62f55b12e7d3..5b2f8bf9f705 100644 --- a/src/osd/OSD.h +++ b/src/osd/OSD.h @@ -1648,7 +1648,8 @@ private: std::atomic_bool heartbeat_need_update; map heartbeat_peers; ///< map of osd id to HeartbeatInfo utime_t last_mon_heartbeat; - Messenger *hbclient_messenger; + Messenger *hb_front_client_messenger; + Messenger *hb_back_client_messenger; Messenger *hb_front_server_messenger; Messenger *hb_back_server_messenger; utime_t last_heartbeat_resample; ///< last time we chose random peers in waiting-for-healthy state @@ -2440,7 +2441,8 @@ protected: int id, Messenger *internal, Messenger *external, - Messenger *hb_client, + Messenger *hb_front_client, + Messenger *hb_back_client, Messenger *hb_front_server, Messenger *hb_back_server, Messenger *osdc_messenger,