From: Patrick Donnelly <pdonnell@redhat.com>
Date: Tue, 14 May 2024 19:28:56 +0000 (-0400)
Subject: mds: set dispatcher order
X-Git-Tag: v20.0.0~1881^2
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=3291f3976459fe6c05b5f54e200bd91cf3b78d8a;p=ceph.git

mds: set dispatcher order

This tries to preserve existing order but uses priorities to make it explicit
and robust to future dispatchers being added. Except:

- The beacon and metrics dispatcher have the highest priorities.  This is to
  ensure we process these messages before trying to acquire any expensive locks
  (like mds_lock).

- The monc dispatcher also has a relatively high priority for the same reasons.
  This change affects other daemons which may have ordered a dispatcher ahead
  of the monc but I cannot think of a legitimate reason to nor do I see an
  instance of it.

Fixes: 7fc04be9332704946ba6f0e95cfcd1afc34fc0fe
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---

diff --git a/src/mds/MDSDaemon.cc b/src/mds/MDSDaemon.cc
index cf9a572d4c0..0c2252b1cb0 100644
--- a/src/mds/MDSDaemon.cc
+++ b/src/mds/MDSDaemon.cc
@@ -570,8 +570,10 @@ int MDSDaemon::init()
   dout(10) << sizeof(Capability) << "\tCapability" << dendl;
   dout(10) << sizeof(xlist<void*>::item) << "\txlist<>::item" << dendl;
 
-  messenger->add_dispatcher_tail(&beacon);
-  messenger->add_dispatcher_tail(this);
+  // Ensure beacons are processed ahead of most other dispatchers.
+  messenger->add_dispatcher_head(&beacon, Dispatcher::PRIORITY_HIGH);
+  // order last as MDSDaemon::ms_dispatch2 first acquires the mds_lock
+  messenger->add_dispatcher_head(this, Dispatcher::PRIORITY_LOW);
 
   // init monc
   monc->set_messenger(messenger);
diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index 36ed04425be..e3c8554536e 100644
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -610,7 +610,7 @@ MDSRank::~MDSRank()
 void MDSRankDispatcher::init()
 {
   objecter->init();
-  messenger->add_dispatcher_head(objecter);
+  messenger->add_dispatcher_tail(objecter); // the default priority
 
   objecter->start();
 
@@ -2137,7 +2137,7 @@ void MDSRank::active_start()
 
   dout(10) << __func__ << ": initializing metrics handler" << dendl;
   metrics_handler.init();
-  messenger->add_dispatcher_tail(&metrics_handler);
+  messenger->add_dispatcher_tail(&metrics_handler, Dispatcher::PRIORITY_HIGH);
 
   // metric aggregation is solely done by rank 0
   if (is_rank0()) {
diff --git a/src/mon/MonClient.cc b/src/mon/MonClient.cc
index 4c74b107e75..0667c078b63 100644
--- a/src/mon/MonClient.cc
+++ b/src/mon/MonClient.cc
@@ -131,7 +131,7 @@ int MonClient::get_monmap_and_config()
   messenger = Messenger::create_client_messenger(
     cct, "temp_mon_client");
   ceph_assert(messenger);
-  messenger->add_dispatcher_head(this);
+  messenger->add_dispatcher_head(this, Dispatcher::PRIORITY_HIGH);
   messenger->start();
   auto shutdown_msgr = make_scope_guard([this] {
     messenger->shutdown();
@@ -265,7 +265,7 @@ int MonClient::ping_monitor(const string &mon_id, string *result_reply)
 						result_reply);
 
   Messenger *smsgr = Messenger::create_client_messenger(cct, "temp_ping_client");
-  smsgr->add_dispatcher_head(pinger);
+  smsgr->add_dispatcher_head(pinger, Dispatcher::PRIORITY_HIGH);
   smsgr->set_auth_client(pinger);
   smsgr->start();
 
@@ -512,7 +512,7 @@ int MonClient::init()
   initialized = true;
 
   messenger->set_auth_client(this);
-  messenger->add_dispatcher_head(this);
+  messenger->add_dispatcher_head(this, Dispatcher::PRIORITY_HIGH);
 
   timer.init();
   schedule_tick();