osd: add a separate messenger for the Objecter

We will give the OSD's Objecter its own messenger so that it does not
interfere with the OSD when it marks things up or down.

Signed-off-by: Sage Weil <sage@inktank.com>

diff --git a/src/ceph_osd.cc b/src/ceph_osd.cc
index dc6f435bdcfc304ecb468fa460934abc3ede0f25..4c6702459e7edd074da53517c32052c408555140 100644 (file)
--- a/src/ceph_osd.cc
+++ b/src/ceph_osd.cc
@@ -331,6 +331,9 @@ int main(int argc, const char **argv)
   Messenger *messenger_hb_front_server = Messenger::create(g_ceph_context,
                                                    entity_name_t::OSD(whoami), "hb_front_server",
                                                    getpid());
+  Messenger *messenger_objecter = Messenger::create(g_ceph_context,
+                                                   entity_name_t::OSD(whoami), "hbclient",
+                                                   getpid());
   cluster_messenger->set_cluster_protocol(CEPH_OSD_PROTOCOL);
   messenger_hbclient->set_cluster_protocol(CEPH_OSD_PROTOCOL);
   messenger_hb_back_server->set_cluster_protocol(CEPH_OSD_PROTOCOL);
@@ -386,6 +389,8 @@ int main(int argc, const char **argv)
   messenger_hb_front_server->set_policy(entity_name_t::TYPE_OSD,
                                        Messenger::Policy::stateless_server(0, 0));
 
+  messenger_objecter->set_default_policy(Messenger::Policy::lossy_client(0, CEPH_FEATURE_OSDREPLYMUX));
+
   r = client_messenger->bind(g_conf->public_addr);
   if (r < 0)
     exit(1);
@@ -432,6 +437,7 @@ int main(int argc, const char **argv)
 
   osd = new OSD(whoami, cluster_messenger, client_messenger,
                messenger_hbclient, messenger_hb_front_server, messenger_hb_back_server,
+               messenger_objecter,
                &mc,
                g_conf->osd_data, g_conf->osd_journal);
 
@@ -450,6 +456,7 @@ int main(int argc, const char **argv)
   messenger_hb_front_server->start();
   messenger_hb_back_server->start();
   cluster_messenger->start();
+  messenger_objecter->start();
 
   // start osd
   err = osd->init();
@@ -475,6 +482,7 @@ int main(int argc, const char **argv)
   messenger_hb_front_server->wait();
   messenger_hb_back_server->wait();
   cluster_messenger->wait();
+  messenger_objecter->wait();
 
   unregister_async_signal_handler(SIGHUP, sighup_handler);
   unregister_async_signal_handler(SIGINT, handle_osd_signal);
@@ -488,6 +496,7 @@ int main(int argc, const char **argv)
   delete messenger_hb_front_server;
   delete messenger_hb_back_server;
   delete cluster_messenger;
+  delete messenger_objecter;
   client_byte_throttler.reset();
   client_msg_throttler.reset();
   g_ceph_context->put();

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 66022a3898a3530fd835915707612800952910ec..5bde23335adb79e1d4597c16824200c96746e261 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -882,6 +882,7 @@ OSD::OSD(int id, Messenger *internal_messenger, Messenger *external_messenger,
         Messenger *hb_clientm,
         Messenger *hb_front_serverm,
         Messenger *hb_back_serverm,
+        Messenger *osdc_messenger,
         MonClient *mc,
         const std::string &dev, const std::string &jdev) :
   Dispatcher(external_messenger->cct),
@@ -897,6 +898,7 @@ OSD::OSD(int id, Messenger *internal_messenger, Messenger *external_messenger,
                                                                      cct->_conf->auth_service_required)),
   cluster_messenger(internal_messenger),
   client_messenger(external_messenger),
+  objecter_messenger(osdc_messenger),
   monc(mc),
   logger(NULL),
   recoverystate_perf(NULL),
@@ -1646,6 +1648,7 @@ int OSD::shutdown()
   client_messenger->shutdown();
   cluster_messenger->shutdown();
   hbclient_messenger->shutdown();
+  objecter_messenger->shutdown();
   hb_front_server_messenger->shutdown();
   hb_back_server_messenger->shutdown();
   peering_wq.clear();

diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index eae4bab21ca4c5509a2b5e3a257f783f1920636e..bef71fdb28f2c93f2c3f622420536c94bfb951ad 100644 (file)
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -627,6 +627,7 @@ protected:
 
   Messenger   *cluster_messenger;
   Messenger   *client_messenger;
+  Messenger   *objecter_messenger;
   MonClient   *monc;
   PerfCounters      *logger;
   PerfCounters      *recoverystate_perf;
@@ -1672,6 +1673,7 @@ protected:
    * be cleaned up properly*/
   OSD(int id, Messenger *internal, Messenger *external,
       Messenger *hb_client, Messenger *hb_front_server, Messenger *hb_back_server,
+      Messenger *osdc_messenger,
       MonClient *mc, const std::string &dev, const std::string &jdev);
   ~OSD();