mgr: fix spurious PG health messages on mgr restart

Previously, the mgr would send MMonMgrReport indicating
a very unhappy PGMap to the mon right after startup.

This is a change to hold off on sending that report until
all the OSDs have reported in, or until some time has passed.

Signed-off-by: John Spray <john.spray@redhat.com>

diff --git a/src/mgr/DaemonServer.cc b/src/mgr/DaemonServer.cc
index 05aa07cbbb436e7129994dc75a23335ad6723046..6454c8da306a52b96bd631527f2b61c2dc09e5b9 100644 (file)
--- a/src/mgr/DaemonServer.cc
+++ b/src/mgr/DaemonServer.cc
@@ -113,6 +113,8 @@ int DaemonServer::init(uint64_t gid, entity_addr_t client_addr)
   msgr->start();
   msgr->add_dispatcher_tail(this);
 
+  started_at = ceph_clock_now();
+
   return 0;
 }
 
@@ -235,6 +237,7 @@ bool DaemonServer::ms_dispatch(Message *m)
   switch (m->get_type()) {
     case MSG_PGSTATS:
       cluster_state.ingest_pgstats(static_cast<MPGStats*>(m));
+      maybe_ready(m->get_source().num());
       m->put();
       return true;
     case MSG_MGR_REPORT:
@@ -249,6 +252,35 @@ bool DaemonServer::ms_dispatch(Message *m)
   };
 }
 
+void DaemonServer::maybe_ready(int32_t osd_id)
+{
+  if (!pgmap_ready && reported_osds.find(osd_id) == reported_osds.end()) {
+    dout(4) << "initial report from osd " << osd_id << dendl;
+    reported_osds.insert(osd_id);
+    std::set<int32_t> up_osds;
+
+    cluster_state.with_osdmap([&](const OSDMap& osdmap) {
+        osdmap.get_up_osds(up_osds);
+    });
+
+    std::set<int32_t> unreported_osds;
+    std::set_difference(up_osds.begin(), up_osds.end(),
+                        reported_osds.begin(), reported_osds.end(),
+                        std::inserter(unreported_osds, unreported_osds.begin()));
+
+    if (unreported_osds.size() == 0) {
+      dout(4) << "all osds have reported, sending PG state to mon" << dendl;
+      pgmap_ready = true;
+      reported_osds.clear();
+      // Avoid waiting for next tick
+      send_report();
+    } else {
+      dout(4) << "still waiting for " << unreported_osds.size() << " osds"
+                 " to report in before PGMap is ready" << dendl;
+    }
+  }
+}
+
 void DaemonServer::shutdown()
 {
   dout(10) << "begin" << dendl;
@@ -977,6 +1009,19 @@ void DaemonServer::_prune_pending_service_map()
 
 void DaemonServer::send_report()
 {
+  if (!pgmap_ready) {
+    if (ceph_clock_now() - started_at > g_conf->mgr_stats_period * 4.0) {
+      pgmap_ready = true;
+      reported_osds.clear();
+      dout(1) << "Giving up on OSDs that haven't reported yet, sending "
+              << "potentially incomplete PG state to mon" << dendl;
+    } else {
+      dout(1) << "Not sending PG status to monitor yet, waiting for OSDs"
+              << dendl;
+      return;
+    }
+  }
+
   auto m = new MMonMgrReport();
   cluster_state.with_pgmap([&](const PGMap& pg_map) {
       cluster_state.update_delta_stats();

diff --git a/src/mgr/DaemonServer.h b/src/mgr/DaemonServer.h
index 06ee68b8adc80cdca81fb3de10f90a2686944c3f..4877cfe85aed8c0eedef5b1832db77ea68bc5177 100644 (file)
--- a/src/mgr/DaemonServer.h
+++ b/src/mgr/DaemonServer.h
@@ -89,6 +89,11 @@ private:
 
   void _prune_pending_service_map();
 
+  utime_t started_at;
+  bool pgmap_ready = false;
+  std::set<int32_t> reported_osds;
+  void maybe_ready(int32_t osd_id);
+
 public:
   int init(uint64_t gid, entity_addr_t client_addr);
   void shutdown();