From 2a01bbc4032f04e4db64c818cb2bbf3047abcd58 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Tue, 15 Sep 2015 15:41:04 -0400 Subject: [PATCH] mon: make all paxos-related timeouts relative to mon_lease It is a pain to adjust these settings for slow/loady clusters. Instead, make everything a ratio of mon_lease (still default 5). No change in defaults. Signed-off-by: Sage Weil --- src/common/config_opts.h | 7 ++++--- src/mon/Monitor.cc | 24 ++++++++++++------------ src/mon/Paxos.cc | 17 ++++++++++++----- 3 files changed, 28 insertions(+), 20 deletions(-) diff --git a/src/common/config_opts.h b/src/common/config_opts.h index fd16972c30914..faca302275409 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -216,12 +216,13 @@ OPTION(mon_osd_prime_pg_temp_max_time, OPT_FLOAT, .5) // max time to spend prim OPTION(mon_osd_pool_ec_fast_read, OPT_BOOL, false) // whether turn on fast read on the pool or not OPTION(mon_stat_smooth_intervals, OPT_INT, 2) // smooth stats over last N PGMap maps OPTION(mon_lease, OPT_FLOAT, 5) // lease interval -OPTION(mon_lease_renew_interval, OPT_FLOAT, 3) // on leader, to renew the lease -OPTION(mon_lease_ack_timeout, OPT_FLOAT, 10.0) // on leader, if lease isn't acked by all peons +OPTION(mon_lease_renew_interval_factor, OPT_FLOAT, .6) // on leader, to renew the lease +OPTION(mon_lease_ack_timeout_factor, OPT_FLOAT, 2.0) // on leader, if lease isn't acked by all peons +OPTION(mon_accept_timeout_factor, OPT_FLOAT, 2.0) // on leader, if paxos update isn't accepted + OPTION(mon_clock_drift_allowed, OPT_FLOAT, .050) // allowed clock drift between monitors OPTION(mon_clock_drift_warn_backoff, OPT_FLOAT, 5) // exponential backoff for clock drift warnings OPTION(mon_timecheck_interval, OPT_FLOAT, 300.0) // on leader, timecheck (clock drift check) interval (seconds) -OPTION(mon_accept_timeout, OPT_FLOAT, 10.0) // on leader, if paxos update isn't accepted OPTION(mon_pg_create_interval, OPT_FLOAT, 30.0) // no more than every 30s OPTION(mon_pg_stuck_threshold, OPT_INT, 300) // number of seconds after which pgs can be considered inactive, unclean, or stale (see doc/control.rst under dump_stuck for more info) OPTION(mon_pg_warn_min_per_osd, OPT_INT, 30) // min # pgs per (in) osd before we warn the admin diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc index 3dc10ef9f7eaf..b49578d3906d4 100644 --- a/src/mon/Monitor.cc +++ b/src/mon/Monitor.cc @@ -451,8 +451,9 @@ const char** Monitor::get_tracked_conf_keys() const static const char* KEYS[] = { "crushtool", // helpful for testing "mon_lease", - "mon_lease_renew_interval", - "mon_lease_ack_timeout", + "mon_lease_renew_interval_factor", + "mon_lease_ack_timeout_factor", + "mon_accept_timeout_factor", // clog & admin clog "clog_to_monitors", "clog_to_syslog", @@ -517,11 +518,10 @@ int Monitor::sanitize_options() // mon_lease must be greater than mon_lease_renewal; otherwise we // may incur in leases expiring before they are renewed. - if (g_conf->mon_lease <= g_conf->mon_lease_renew_interval) { - clog->error() << "mon_lease (" << g_conf->mon_lease - << ") must be greater " - << "than mon_lease_renew_interval (" - << g_conf->mon_lease_renew_interval << ")"; + if (g_conf->mon_lease_renew_interval_factor >= 1.0) { + clog->error() << "mon_lease_renew_interval_factor (" + << g_conf->mon_lease_renew_interval_factor + << ") must be less than 1.0"; r = -EINVAL; } @@ -530,13 +530,13 @@ int Monitor::sanitize_options() // with the same value, for a given small vale, could mean timing out if // the monitors happened to be overloaded -- or even under normal load for // a small enough value. - if (g_conf->mon_lease_ack_timeout <= g_conf->mon_lease) { - clog->error() << "mon_lease_ack_timeout (" - << g_conf->mon_lease_ack_timeout - << ") must be greater than mon_lease (" - << g_conf->mon_lease << ")"; + if (g_conf->mon_lease_ack_timeout_factor <= 1.0) { + clog->error() << "mon_lease_ack_timeout_factor (" + << g_conf->mon_lease_ack_timeout_factor + << ") must be greater than 1.0"; r = -EINVAL; } + return r; } diff --git a/src/mon/Paxos.cc b/src/mon/Paxos.cc index 44b8184aec0d8..e0c8039257f57 100644 --- a/src/mon/Paxos.cc +++ b/src/mon/Paxos.cc @@ -187,7 +187,9 @@ void Paxos::collect(version_t oldpn) // set timeout event collect_timeout_event = new C_CollectTimeout(this); - mon->timer.add_event_after(g_conf->mon_accept_timeout, collect_timeout_event); + mon->timer.add_event_after(g_conf->mon_accept_timeout_factor * + g_conf->mon_lease, + collect_timeout_event); } @@ -686,7 +688,9 @@ void Paxos::begin(bufferlist& v) // set timeout event accept_timeout_event = new C_AcceptTimeout(this); - mon->timer.add_event_after(g_conf->mon_accept_timeout, accept_timeout_event); + mon->timer.add_event_after(g_conf->mon_accept_timeout_factor * + g_conf->mon_lease, + accept_timeout_event); } // peon @@ -972,7 +976,8 @@ void Paxos::extend_lease() // if old timeout is still in place, leave it. if (!lease_ack_timeout_event) { lease_ack_timeout_event = new C_LeaseAckTimeout(this); - mon->timer.add_event_after(g_conf->mon_lease_ack_timeout, + mon->timer.add_event_after(g_conf->mon_lease_ack_timeout_factor * + g_conf->mon_lease, lease_ack_timeout_event); } @@ -980,7 +985,7 @@ void Paxos::extend_lease() lease_renew_event = new C_LeaseRenew(this); utime_t at = lease_expire; at -= g_conf->mon_lease; - at += g_conf->mon_lease_renew_interval; + at += g_conf->mon_lease_renew_interval_factor * g_conf->mon_lease; mon->timer.add_event_at(at, lease_renew_event); } @@ -1163,7 +1168,9 @@ void Paxos::reset_lease_timeout() if (lease_timeout_event) mon->timer.cancel_event(lease_timeout_event); lease_timeout_event = new C_LeaseTimeout(this); - mon->timer.add_event_after(g_conf->mon_lease_ack_timeout, lease_timeout_event); + mon->timer.add_event_after(g_conf->mon_lease_ack_timeout_factor * + g_conf->mon_lease, + lease_timeout_event); } void Paxos::lease_timeout() -- 2.39.5