osd: separate hb client messenger to public and cluster

Signed-off-by: Haomai Wang <haomai@xsky.com>

diff --git a/src/ceph_osd.cc b/src/ceph_osd.cc
index 31c0e637c8ddbdeb990f06d981b24e1c60a11de3..334fb7fa8e1770f8415c96bc107ab1d296d84fd7 100644 (file)
--- a/src/ceph_osd.cc
+++ b/src/ceph_osd.cc
@@ -450,8 +450,11 @@ int main(int argc, const char **argv)
                                            getpid(),
                                            Messenger::HAS_HEAVY_TRAFFIC |
                                            Messenger::HAS_MANY_CONNECTIONS);
-  Messenger *ms_hbclient = Messenger::create(g_ceph_context, g_conf->ms_type,
-                                            entity_name_t::OSD(whoami), "hbclient",
+  Messenger *ms_hb_back_client = Messenger::create(g_ceph_context, g_conf->ms_type,
+                                            entity_name_t::OSD(whoami), "hb_back_client",
+                                            getpid(), Messenger::HEARTBEAT);
+  Messenger *ms_hb_front_client = Messenger::create(g_ceph_context, g_conf->ms_type,
+                                            entity_name_t::OSD(whoami), "hb_front_client",
                                             getpid(), Messenger::HEARTBEAT);
   Messenger *ms_hb_back_server = Messenger::create(g_ceph_context, g_conf->ms_type,
                                                   entity_name_t::OSD(whoami), "hb_back_server",
@@ -462,10 +465,11 @@ int main(int argc, const char **argv)
   Messenger *ms_objecter = Messenger::create(g_ceph_context, g_conf->ms_type,
                                             entity_name_t::OSD(whoami), "ms_objecter",
                                             getpid(), 0);
-  if (!ms_public || !ms_cluster || !ms_hbclient || !ms_hb_back_server || !ms_hb_front_server || !ms_objecter)
+  if (!ms_public || !ms_cluster || !ms_hb_front_client || !ms_hb_back_client || !ms_hb_back_server || !ms_hb_front_server || !ms_objecter)
     exit(1);
   ms_cluster->set_cluster_protocol(CEPH_OSD_PROTOCOL);
-  ms_hbclient->set_cluster_protocol(CEPH_OSD_PROTOCOL);
+  ms_hb_front_client->set_cluster_protocol(CEPH_OSD_PROTOCOL);
+  ms_hb_back_client->set_cluster_protocol(CEPH_OSD_PROTOCOL);
   ms_hb_back_server->set_cluster_protocol(CEPH_OSD_PROTOCOL);
   ms_hb_front_server->set_cluster_protocol(CEPH_OSD_PROTOCOL);
 
@@ -523,7 +527,9 @@ int main(int argc, const char **argv)
   ms_cluster->set_policy(entity_name_t::TYPE_CLIENT,
                         Messenger::Policy::stateless_server(0, 0));
 
-  ms_hbclient->set_policy(entity_name_t::TYPE_OSD,
+  ms_hb_front_client->set_policy(entity_name_t::TYPE_OSD,
+                         Messenger::Policy::lossy_client(0, 0));
+  ms_hb_back_client->set_policy(entity_name_t::TYPE_OSD,
                          Messenger::Policy::lossy_client(0, 0));
   ms_hb_back_server->set_policy(entity_name_t::TYPE_OSD,
                                Messenger::Policy::stateless_server(0, 0));
@@ -540,7 +546,8 @@ int main(int argc, const char **argv)
     exit(1);
 
   if (g_conf->osd_heartbeat_use_min_delay_socket) {
-    ms_hbclient->set_socket_priority(SOCKET_PRIORITY_MIN_DELAY);
+    ms_hb_front_client->set_socket_priority(SOCKET_PRIORITY_MIN_DELAY);
+    ms_hb_back_client->set_socket_priority(SOCKET_PRIORITY_MIN_DELAY);
     ms_hb_back_server->set_socket_priority(SOCKET_PRIORITY_MIN_DELAY);
     ms_hb_front_server->set_socket_priority(SOCKET_PRIORITY_MIN_DELAY);
   }
@@ -586,7 +593,8 @@ int main(int argc, const char **argv)
                 whoami,
                 ms_cluster,
                 ms_public,
-                ms_hbclient,
+                ms_hb_front_client,
+                ms_hb_back_client,
                 ms_hb_front_server,
                 ms_hb_back_server,
                 ms_objecter,
@@ -602,7 +610,8 @@ int main(int argc, const char **argv)
   }
 
   ms_public->start();
-  ms_hbclient->start();
+  ms_hb_front_client->start();
+  ms_hb_back_client->start();
   ms_hb_front_server->start();
   ms_hb_back_server->start();
   ms_cluster->start();
@@ -632,7 +641,8 @@ int main(int argc, const char **argv)
     kill(getpid(), SIGTERM);
 
   ms_public->wait();
-  ms_hbclient->wait();
+  ms_hb_front_client->wait();
+  ms_hb_back_client->wait();
   ms_hb_front_server->wait();
   ms_hb_back_server->wait();
   ms_cluster->wait();
@@ -646,7 +656,8 @@ int main(int argc, const char **argv)
   // done
   delete osd;
   delete ms_public;
-  delete ms_hbclient;
+  delete ms_hb_front_client;
+  delete ms_hb_back_client;
   delete ms_hb_front_server;
   delete ms_hb_back_server;
   delete ms_cluster;

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 0347b15117d991c19abbba8362cb812243381946..0aa9c0801f158d783c7a9462a5df5acdc2672e6b 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -836,9 +836,9 @@ pair<ConnectionRef,ConnectionRef> OSDService::get_con_osd_hb(int peer, epoch_t f
     release_map(next_map);
     return ret;
   }
-  ret.first = osd->hbclient_messenger->get_connection(next_map->get_hb_back_inst(peer));
+  ret.first = osd->hb_back_client_messenger->get_connection(next_map->get_hb_back_inst(peer));
   if (next_map->get_hb_front_addr(peer) != entity_addr_t())
-    ret.second = osd->hbclient_messenger->get_connection(next_map->get_hb_front_inst(peer));
+    ret.second = osd->hb_front_client_messenger->get_connection(next_map->get_hb_front_inst(peer));
   release_map(next_map);
   return ret;
 }
@@ -1636,7 +1636,8 @@ OSD::OSD(CephContext *cct_, ObjectStore *store_,
         int id,
         Messenger *internal_messenger,
         Messenger *external_messenger,
-        Messenger *hb_clientm,
+        Messenger *hb_client_front,
+        Messenger *hb_client_back,
         Messenger *hb_front_serverm,
         Messenger *hb_back_serverm,
         Messenger *osdc_messenger,
@@ -1678,7 +1679,8 @@ OSD::OSD(CephContext *cct_, ObjectStore *store_,
   heartbeat_lock("OSD::heartbeat_lock"),
   heartbeat_stop(false),
   heartbeat_need_update(true),
-  hbclient_messenger(hb_clientm),
+  hb_front_client_messenger(hb_client_front),
+  hb_back_client_messenger(hb_client_back),
   hb_front_server_messenger(hb_front_serverm),
   hb_back_server_messenger(hb_back_serverm),
   daily_loadavg(0.0),
@@ -2187,7 +2189,8 @@ int OSD::init()
   client_messenger->add_dispatcher_head(this);
   cluster_messenger->add_dispatcher_head(this);
 
-  hbclient_messenger->add_dispatcher_head(&heartbeat_dispatcher);
+  hb_front_client_messenger->add_dispatcher_head(&heartbeat_dispatcher);
+  hb_back_client_messenger->add_dispatcher_head(&heartbeat_dispatcher);
   hb_front_server_messenger->add_dispatcher_head(&heartbeat_dispatcher);
   hb_back_server_messenger->add_dispatcher_head(&heartbeat_dispatcher);
 
@@ -2858,7 +2861,8 @@ int OSD::shutdown()
   class_handler->shutdown();
   client_messenger->shutdown();
   cluster_messenger->shutdown();
-  hbclient_messenger->shutdown();
+  hb_front_client_messenger->shutdown();
+  hb_back_client_messenger->shutdown();
   objecter_messenger->shutdown();
   hb_front_server_messenger->shutdown();
   hb_back_server_messenger->shutdown();
@@ -7212,7 +7216,8 @@ void OSD::_committed_osd_maps(epoch_t first, epoch_t last, MOSDMap *m)
                   << " rebind hb_front_server_messenger failed" << dendl;
         }
 
-       hbclient_messenger->mark_down_all();
+       hb_front_client_messenger->mark_down_all();
+       hb_back_client_messenger->mark_down_all();
 
        reset_heartbeat_peers();
       }

diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index 62f55b12e7d3c2aaef0698d9e513803ebb68de30..5b2f8bf9f70541eb6cb9330a978c72783525c03f 100644 (file)
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -1648,7 +1648,8 @@ private:
   std::atomic_bool heartbeat_need_update;   
   map<int,HeartbeatInfo> heartbeat_peers;  ///< map of osd id to HeartbeatInfo
   utime_t last_mon_heartbeat;
-  Messenger *hbclient_messenger;
+  Messenger *hb_front_client_messenger;
+  Messenger *hb_back_client_messenger;
   Messenger *hb_front_server_messenger;
   Messenger *hb_back_server_messenger;
   utime_t last_heartbeat_resample;   ///< last time we chose random peers in waiting-for-healthy state
@@ -2440,7 +2441,8 @@ protected:
       int id,
       Messenger *internal,
       Messenger *external,
-      Messenger *hb_client,
+      Messenger *hb_front_client,
+      Messenger *hb_back_client,
       Messenger *hb_front_server,
       Messenger *hb_back_server,
       Messenger *osdc_messenger,