mon: allow a MON_DOWN grace period after cluster mkfs

During teuthology tests, the initial cluster bootstrap often starts up
the mon sbut doesn't include all mons in the initial quorum, due to
mon startup misalignment and random delays.  Provide a short grace period
where we will not raise a MON_DOWN alert even though the quorum is not
complete.

Fixes: https://tracker.ceph.com/issues/43584
Signed-off-by: Sage Weil <sage@newdream.net>
(cherry picked from commit eee041f2f070b88b01d45c04624872681dd158be)

 Conflicts:
	src/common/options/mon.yaml.in
		In Pacific, options are in `src/common/options.cc`.

diff --git a/src/common/options.cc b/src/common/options.cc
index 89731a5bbd7791693fb4be74ab02ae3b95dd12f8..b0239e24aa4d3ebe94f2f9700e09edbfd3ec1a01 100644 (file)
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -5634,6 +5634,11 @@ std::vector<Option> get_global_options() {
     .set_description("Period in seconds between monitor-to-manager "
                      "health/status updates"),
 
+    Option("mon_down_mkfs_grace", Option::TYPE_SECS, Option::LEVEL_ADVANCED)
+    .set_default(60)
+    .add_service("mon")
+    .set_description("Period in seconds that the cluster may have a mon down after cluster creation"),
+
     Option("mon_mgr_beacon_grace", Option::TYPE_SECS, Option::LEVEL_ADVANCED)
     .set_default(30)
     .add_service("mon")

diff --git a/src/mon/HealthMonitor.cc b/src/mon/HealthMonitor.cc
index ae92e8b77521bf8806f1a4d676baf343335536ad..a45159e7c669c4e4d4620dbe4fe124430769b4a8 100644 (file)
--- a/src/mon/HealthMonitor.cc
+++ b/src/mon/HealthMonitor.cc
@@ -798,7 +798,9 @@ void HealthMonitor::check_for_mon_down(health_check_map_t *checks)
 {
   int max = mon.monmap->size();
   int actual = mon.get_quorum().size();
-  if (actual < max) {
+  const auto now = ceph::real_clock::now();
+  if (actual < max &&
+      now > mon.monmap->created.to_real_time() + g_conf().get_val<std::chrono::seconds>("mon_down_mkfs_grace")) {
     ostringstream ss;
     ss << (max-actual) << "/" << max << " mons down, quorum "
        << mon.get_quorum_names();