mon: make all paxos-related timeouts relative to mon_lease

It is a pain to adjust these settings for slow/loady clusters.
Instead, make everything a ratio of mon_lease (still default 5).

No change in defaults.

Signed-off-by: Sage Weil <sage@redhat.com>

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index fd16972c30914b453d87c0c79338cd8691736b3f..faca3022754092626afd34948bde95867041e076 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -216,12 +216,13 @@ OPTION(mon_osd_prime_pg_temp_max_time, OPT_FLOAT, .5)  // max time to spend prim
 OPTION(mon_osd_pool_ec_fast_read, OPT_BOOL, false) // whether turn on fast read on the pool or not
 OPTION(mon_stat_smooth_intervals, OPT_INT, 2)  // smooth stats over last N PGMap maps
 OPTION(mon_lease, OPT_FLOAT, 5)       // lease interval
-OPTION(mon_lease_renew_interval, OPT_FLOAT, 3) // on leader, to renew the lease
-OPTION(mon_lease_ack_timeout, OPT_FLOAT, 10.0) // on leader, if lease isn't acked by all peons
+OPTION(mon_lease_renew_interval_factor, OPT_FLOAT, .6) // on leader, to renew the lease
+OPTION(mon_lease_ack_timeout_factor, OPT_FLOAT, 2.0) // on leader, if lease isn't acked by all peons
+OPTION(mon_accept_timeout_factor, OPT_FLOAT, 2.0)    // on leader, if paxos update isn't accepted
+
 OPTION(mon_clock_drift_allowed, OPT_FLOAT, .050) // allowed clock drift between monitors
 OPTION(mon_clock_drift_warn_backoff, OPT_FLOAT, 5) // exponential backoff for clock drift warnings
 OPTION(mon_timecheck_interval, OPT_FLOAT, 300.0) // on leader, timecheck (clock drift check) interval (seconds)
-OPTION(mon_accept_timeout, OPT_FLOAT, 10.0)    // on leader, if paxos update isn't accepted
 OPTION(mon_pg_create_interval, OPT_FLOAT, 30.0) // no more than every 30s
 OPTION(mon_pg_stuck_threshold, OPT_INT, 300) // number of seconds after which pgs can be considered inactive, unclean, or stale (see doc/control.rst under dump_stuck for more info)
 OPTION(mon_pg_warn_min_per_osd, OPT_INT, 30)  // min # pgs per (in) osd before we warn the admin

diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc
index 3dc10ef9f7eaf2b6c055f3a1ffe04ca916a92ef4..b49578d3906d4a89b12f87835d42d6b26f128393 100644 (file)
--- a/src/mon/Monitor.cc
+++ b/src/mon/Monitor.cc
@@ -451,8 +451,9 @@ const char** Monitor::get_tracked_conf_keys() const
   static const char* KEYS[] = {
     "crushtool", // helpful for testing
     "mon_lease",
-    "mon_lease_renew_interval",
-    "mon_lease_ack_timeout",
+    "mon_lease_renew_interval_factor",
+    "mon_lease_ack_timeout_factor",
+    "mon_accept_timeout_factor",
     // clog & admin clog
     "clog_to_monitors",
     "clog_to_syslog",
@@ -517,11 +518,10 @@ int Monitor::sanitize_options()
 
   // mon_lease must be greater than mon_lease_renewal; otherwise we
   // may incur in leases expiring before they are renewed.
-  if (g_conf->mon_lease <= g_conf->mon_lease_renew_interval) {
-    clog->error() << "mon_lease (" << g_conf->mon_lease
-                 << ") must be greater "
-                 << "than mon_lease_renew_interval ("
-                 << g_conf->mon_lease_renew_interval << ")";
+  if (g_conf->mon_lease_renew_interval_factor >= 1.0) {
+    clog->error() << "mon_lease_renew_interval_factor ("
+                 << g_conf->mon_lease_renew_interval_factor
+                 << ") must be less than 1.0";
     r = -EINVAL;
   }
 
@@ -530,13 +530,13 @@ int Monitor::sanitize_options()
   // with the same value, for a given small vale, could mean timing out if
   // the monitors happened to be overloaded -- or even under normal load for
   // a small enough value.
-  if (g_conf->mon_lease_ack_timeout <= g_conf->mon_lease) {
-    clog->error() << "mon_lease_ack_timeout ("
-                 << g_conf->mon_lease_ack_timeout
-                 << ") must be greater than mon_lease ("
-                 << g_conf->mon_lease << ")";
+  if (g_conf->mon_lease_ack_timeout_factor <= 1.0) {
+    clog->error() << "mon_lease_ack_timeout_factor ("
+                 << g_conf->mon_lease_ack_timeout_factor
+                 << ") must be greater than 1.0";
     r = -EINVAL;
   }
+
   return r;
 }
 

diff --git a/src/mon/Paxos.cc b/src/mon/Paxos.cc
index 44b8184aec0d88625872720cc018c48125aca9b1..e0c8039257f5777f5a5542ecf95e567687b6b919 100644 (file)
--- a/src/mon/Paxos.cc
+++ b/src/mon/Paxos.cc
@@ -187,7 +187,9 @@ void Paxos::collect(version_t oldpn)
 
   // set timeout event
   collect_timeout_event = new C_CollectTimeout(this);
-  mon->timer.add_event_after(g_conf->mon_accept_timeout, collect_timeout_event);
+  mon->timer.add_event_after(g_conf->mon_accept_timeout_factor *
+                            g_conf->mon_lease,
+                            collect_timeout_event);
 }
 
 
@@ -686,7 +688,9 @@ void Paxos::begin(bufferlist& v)
 
   // set timeout event
   accept_timeout_event = new C_AcceptTimeout(this);
-  mon->timer.add_event_after(g_conf->mon_accept_timeout, accept_timeout_event);
+  mon->timer.add_event_after(g_conf->mon_accept_timeout_factor *
+                            g_conf->mon_lease,
+                            accept_timeout_event);
 }
 
 // peon
@@ -972,7 +976,8 @@ void Paxos::extend_lease()
   //  if old timeout is still in place, leave it.
   if (!lease_ack_timeout_event) {
     lease_ack_timeout_event = new C_LeaseAckTimeout(this);
-    mon->timer.add_event_after(g_conf->mon_lease_ack_timeout, 
+    mon->timer.add_event_after(g_conf->mon_lease_ack_timeout_factor *
+                              g_conf->mon_lease,
                               lease_ack_timeout_event);
   }
 
@@ -980,7 +985,7 @@ void Paxos::extend_lease()
   lease_renew_event = new C_LeaseRenew(this);
   utime_t at = lease_expire;
   at -= g_conf->mon_lease;
-  at += g_conf->mon_lease_renew_interval;
+  at += g_conf->mon_lease_renew_interval_factor * g_conf->mon_lease;
   mon->timer.add_event_at(at, lease_renew_event);
 }
 
@@ -1163,7 +1168,9 @@ void Paxos::reset_lease_timeout()
   if (lease_timeout_event)
     mon->timer.cancel_event(lease_timeout_event);
   lease_timeout_event = new C_LeaseTimeout(this);
-  mon->timer.add_event_after(g_conf->mon_lease_ack_timeout, lease_timeout_event);
+  mon->timer.add_event_after(g_conf->mon_lease_ack_timeout_factor *
+                            g_conf->mon_lease,
+                            lease_timeout_event);
 }
 
 void Paxos::lease_timeout()