From e765f2d533440cfc4189f36fcaba24617a302e84 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Tue, 5 Nov 2019 19:51:17 -0800
Subject: [PATCH] mds: establish session with mgr only after added to FSMap

Note that we now sub to the mgrmap after init because the MgrClient
connection to the mgr is driven by receipt of the MgrMap.

This is important so that the MDS does not have metadata with the mgr
when the mons are ignoring the MDS otherwise due to CompatSet
incompatibilities.

Fixes: https://tracker.ceph.com/issues/41538
Fixes: https://tracker.ceph.com/issues/42635
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 src/mds/MDSDaemon.cc | 15 ++++++++++-----
 src/mgr/MgrClient.cc |  1 +
 src/mgr/MgrClient.h  |  4 ++++
 3 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/src/mds/MDSDaemon.cc b/src/mds/MDSDaemon.cc
index 6e65323ce027..45356177b990 100644
--- a/src/mds/MDSDaemon.cc
+++ b/src/mds/MDSDaemon.cc
@@ -386,9 +386,6 @@ int MDSDaemon::init()
     return -ETIMEDOUT;
   }
 
-  mgrc.init();
-  messenger->add_dispatcher_head(&mgrc);
-
   mds_lock.lock();
   if (beacon.get_want_state() == CEPH_MDS_STATE_DNE) {
     dout(4) << __func__ << ": terminated already, dropping out" << dendl;
@@ -397,7 +394,6 @@ int MDSDaemon::init()
   }
 
   monc->sub_want("mdsmap", 0, 0);
-  monc->sub_want("mgrmap", 0, 0);
   monc->renew_subs();
 
   mds_lock.unlock();
@@ -799,6 +795,14 @@ void MDSDaemon::handle_mds_map(const cref_t<MMDSMap> &m)
     respawn();
   }
 
+  if (old_state == DS::STATE_NULL && new_state != DS::STATE_NULL) {
+    /* The MDS has been added to the FSMap, now we can init the MgrClient */
+    mgrc.init();
+    messenger->add_dispatcher_tail(&mgrc);
+    monc->sub_want("mgrmap", 0, 0);
+    monc->renew_subs(); /* MgrMap receipt drives connection to ceph-mgr */
+  }
+
   // mark down any failed peers
   for (const auto& [gid, info] : oldmap->get_mds_info()) {
     if (mdsmap->get_mds_info().count(gid) == 0) {
@@ -899,7 +903,8 @@ void MDSDaemon::suicide()
   }
   beacon.shutdown();
 
-  mgrc.shutdown();
+  if (mgrc.is_initialized())
+    mgrc.shutdown();
 
   if (mds_rank) {
     mds_rank->shutdown();
diff --git a/src/mgr/MgrClient.cc b/src/mgr/MgrClient.cc
index 976d297e639a..0667d8c72735 100644
--- a/src/mgr/MgrClient.cc
+++ b/src/mgr/MgrClient.cc
@@ -55,6 +55,7 @@ void MgrClient::init()
   ceph_assert(msgr != nullptr);
 
   timer.init();
+  initialized = true;
 }
 
 void MgrClient::shutdown()
diff --git a/src/mgr/MgrClient.h b/src/mgr/MgrClient.h
index 7e5272dbedbe..e9053ccf1f02 100644
--- a/src/mgr/MgrClient.h
+++ b/src/mgr/MgrClient.h
@@ -168,10 +168,14 @@ public:
     std::map<std::string,std::string> &&task_status);
   void update_daemon_health(std::vector<DaemonHealthMetric>&& metrics);
 
+  bool is_initialized() const { return initialized; }
+
 private:
   void _send_stats();
   void _send_pgstats();
   void _send_report();
+
+  bool initialized = false;
 };
 
 #endif
-- 
2.47.3