From 0e7cfc6bebe1d4b1b3a6c2c85fc169c64a75f1e9 Mon Sep 17 00:00:00 2001 From: Greg Farnum Date: Wed, 11 Aug 2010 09:49:29 -0700 Subject: [PATCH] mon: backoff clock drift warnings --- src/config.cc | 1 + src/config.h | 1 + src/mon/Paxos.cc | 19 +++++++++++++------ src/mon/Paxos.h | 7 ++++++- 4 files changed, 21 insertions(+), 7 deletions(-) diff --git a/src/config.cc b/src/config.cc index e29a968014a5e..11f7abeee6c36 100644 --- a/src/config.cc +++ b/src/config.cc @@ -330,6 +330,7 @@ static struct config_option config_optionsp[] = { OPTION(mon_lease_renew_interval, 0, OPT_FLOAT, 3), // on leader, to renew the lease OPTION(mon_lease_ack_timeout, 0, OPT_FLOAT, 10.0), // on leader, if lease isn't acked by all peons OPTION(mon_lease_wiggle_room, 0, OPT_FLOAT, .010), // allowed clock drift between monitors + OPTION(mon_lease_time_warn_backoff, 0, OPT_FLOAT, 5), // exponential backoff for clock drift warnings OPTION(mon_lease_timeout, 0, OPT_FLOAT, 10.0), // on peon, if lease isn't extended OPTION(mon_accept_timeout, 0, OPT_FLOAT, 10.0), // on leader, if paxos update isn't accepted OPTION(mon_stop_on_last_unmount, 0, OPT_BOOL, false), diff --git a/src/config.h b/src/config.h index 2809d2c7d3550..98fc6f78003a9 100644 --- a/src/config.h +++ b/src/config.h @@ -150,6 +150,7 @@ struct md_config_t { float mon_lease; float mon_lease_renew_interval; float mon_lease_wiggle_room; + float mon_lease_time_warn_backoff; float mon_lease_ack_timeout; float mon_lease_timeout; float mon_accept_timeout; diff --git a/src/mon/Paxos.cc b/src/mon/Paxos.cc index 286f1158ab76d..d4d63bb9c8b53 100644 --- a/src/mon/Paxos.cc +++ b/src/mon/Paxos.cc @@ -572,12 +572,19 @@ void Paxos::handle_lease(MMonPaxos *lease) } if (lease->sent_timestamp > g_clock.now() + g_conf.mon_lease_wiggle_room) { - stringstream ss; - ss << "lease_expire from mon" << lease->get_source().num() - << " was sent from future time " << lease->sent_timestamp - << ", clocks not synchronized" - << std::endl; - mon->get_logclient()->log(LOG_WARN, ss); + utime_t warn_diff = g_clock.now() - last_lease_time_warn; + if ((last_lease_time_warn == utime_t()) || + (warn_diff > + pow(g_conf.mon_lease_time_warn_backoff, lease_times_warned))) { + stringstream ss; + ss << "lease_expire from mon" << lease->get_source().num() + << " was sent from future time " << lease->sent_timestamp + << ", clocks not synchronized" + << std::endl; + mon->get_logclient()->log(LOG_WARN, ss); + last_lease_time_warn = g_clock.now(); + ++lease_times_warned; + } } // extend lease diff --git a/src/mon/Paxos.h b/src/mon/Paxos.h index fa6c0641c8f3c..d459ba31769c5 100644 --- a/src/mon/Paxos.h +++ b/src/mon/Paxos.h @@ -152,6 +152,10 @@ private: }; map observers; + //synchronization warnings + utime_t last_lease_time_warn; + int lease_times_warned; + class C_CollectTimeout : public Context { Paxos *paxos; @@ -233,7 +237,8 @@ public: lease_renew_event(0), lease_ack_timeout_event(0), lease_timeout_event(0), - accept_timeout_event(0) { } + accept_timeout_event(0), + lease_times_warned(0) { } const char *get_machine_name() const { return machine_name; -- 2.39.5