mds: set dispatcher order

This tries to preserve existing order but uses priorities to make it explicit
and robust to future dispatchers being added. Except:

- The beacon and metrics dispatcher have the highest priorities.  This is to
  ensure we process these messages before trying to acquire any expensive locks
  (like mds_lock).

- The monc dispatcher also has a relatively high priority for the same reasons.
  This change affects other daemons which may have ordered a dispatcher ahead
  of the monc but I cannot think of a legitimate reason to nor do I see an
  instance of it.

Fixes: 7fc04be9332704946ba6f0e95cfcd1afc34fc0fe
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
(cherry picked from commit 3291f3976459fe6c05b5f54e200bd91cf3b78d8a)

diff --git a/src/mds/MDSDaemon.cc b/src/mds/MDSDaemon.cc
index c1acb1ae3621ab3787bb44210d6496b6ab3a6aae..483f51db724aafe48f56b506fd9e1b92b8e47f33 100644 (file)
--- a/src/mds/MDSDaemon.cc
+++ b/src/mds/MDSDaemon.cc
@@ -514,8 +514,10 @@ int MDSDaemon::init()
   dout(10) << sizeof(Capability) << "\tCapability" << dendl;
   dout(10) << sizeof(xlist<void*>::item) << "\txlist<>::item" << dendl;
 
-  messenger->add_dispatcher_tail(&beacon);
-  messenger->add_dispatcher_tail(this);
+  // Ensure beacons are processed ahead of most other dispatchers.
+  messenger->add_dispatcher_head(&beacon, Dispatcher::PRIORITY_HIGH);
+  // order last as MDSDaemon::ms_dispatch2 first acquires the mds_lock
+  messenger->add_dispatcher_head(this, Dispatcher::PRIORITY_LOW);
 
   // init monc
   monc->set_messenger(messenger);

diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index aa6a8c162f4f5cbf1cb06a4b17f78022c5a6347f..448a31a77773e31fb89ba8875027e52399568f98 100644 (file)
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -602,7 +602,7 @@ MDSRank::~MDSRank()
 void MDSRankDispatcher::init()
 {
   objecter->init();
-  messenger->add_dispatcher_head(objecter);
+  messenger->add_dispatcher_tail(objecter); // the default priority
 
   objecter->start();
 
@@ -2110,7 +2110,7 @@ void MDSRank::active_start()
 
   dout(10) << __func__ << ": initializing metrics handler" << dendl;
   metrics_handler.init();
-  messenger->add_dispatcher_tail(&metrics_handler);
+  messenger->add_dispatcher_tail(&metrics_handler, Dispatcher::PRIORITY_HIGH);
 
   // metric aggregation is solely done by rank 0
   if (is_rank0()) {

diff --git a/src/mon/MonClient.cc b/src/mon/MonClient.cc
index f0f567c287d23afb79e28840895aa8495ee5cfe9..7584fe2c43997d9478d81652490b10e958a91299 100644 (file)
--- a/src/mon/MonClient.cc
+++ b/src/mon/MonClient.cc
@@ -129,7 +129,7 @@ int MonClient::get_monmap_and_config()
   messenger = Messenger::create_client_messenger(
     cct, "temp_mon_client");
   ceph_assert(messenger);
-  messenger->add_dispatcher_head(this);
+  messenger->add_dispatcher_head(this, Dispatcher::PRIORITY_HIGH);
   messenger->start();
   auto shutdown_msgr = make_scope_guard([this] {
     messenger->shutdown();
@@ -263,7 +263,7 @@ int MonClient::ping_monitor(const string &mon_id, string *result_reply)
                                                result_reply);
 
   Messenger *smsgr = Messenger::create_client_messenger(cct, "temp_ping_client");
-  smsgr->add_dispatcher_head(pinger);
+  smsgr->add_dispatcher_head(pinger, Dispatcher::PRIORITY_HIGH);
   smsgr->set_auth_client(pinger);
   smsgr->start();
 
@@ -510,7 +510,7 @@ int MonClient::init()
   initialized = true;
 
   messenger->set_auth_client(this);
-  messenger->add_dispatcher_head(this);
+  messenger->add_dispatcher_head(this, Dispatcher::PRIORITY_HIGH);
 
   timer.init();
   schedule_tick();