From: Joao Eduardo Luis Date: Thu, 7 Jan 2016 11:05:44 +0000 (+0000) Subject: mon: Monitor: adaptative clock skew detection interval X-Git-Tag: v10.1.0~209^2~1 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=45e16d0d7407cf4e886dde9321bb91c1a087776d;p=ceph.git mon: Monitor: adaptative clock skew detection interval When in the presence of a clock skew, adjust the checking interval according to how many rounds have gone by since the last clean check. If a skew is detected, instead of waiting an additional 300 seconds we will perform the check more frequently, gradually backing off the frequency if the skew is still in place (up to a maximum of 'mon_timecheck_interval', default: 300s). This will help with transient skews. Signed-off-by: Joao Eduardo Luis --- diff --git a/src/common/config_opts.h b/src/common/config_opts.h index cb6d6b65d549..c596c7fb6b4e 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -248,6 +248,7 @@ OPTION(mon_accept_timeout_factor, OPT_FLOAT, 2.0) // on leader, if paxos upda OPTION(mon_clock_drift_allowed, OPT_FLOAT, .050) // allowed clock drift between monitors OPTION(mon_clock_drift_warn_backoff, OPT_FLOAT, 5) // exponential backoff for clock drift warnings OPTION(mon_timecheck_interval, OPT_FLOAT, 300.0) // on leader, timecheck (clock drift check) interval (seconds) +OPTION(mon_timecheck_skew_interval, OPT_FLOAT, 30.0) // on leader, timecheck (clock drift check) interval when in presence of a skew (seconds) OPTION(mon_pg_create_interval, OPT_FLOAT, 30.0) // no more than every 30s OPTION(mon_pg_stuck_threshold, OPT_INT, 300) // number of seconds after which pgs can be considered inactive, unclean, or stale (see doc/control.rst under dump_stuck for more info) OPTION(mon_pg_min_inactive, OPT_U64, 1) // the number of PGs which have to be inactive longer than 'mon_pg_stuck_threshold' before health goes into ERR. 0 means disabled, never go into ERR. diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc index dcb129037b5e..174f35b7635b 100644 --- a/src/mon/Monitor.cc +++ b/src/mon/Monitor.cc @@ -185,6 +185,7 @@ Monitor::Monitor(CephContext* cct_, string nm, MonitorDBStore *s, timecheck_round(0), timecheck_acks(0), + timecheck_rounds_since_clean(0), timecheck_event(NULL), probe_timeout_event(NULL), @@ -3822,8 +3823,7 @@ void Monitor::timecheck_start_round() timecheck(); out: dout(10) << __func__ << " setting up next event" << dendl; - timecheck_event = new C_TimeCheck(this); - timer.add_event_after(g_conf->mon_timecheck_interval, timecheck_event); + timecheck_reset_event(); } void Monitor::timecheck_finish_round(bool success) @@ -3837,6 +3837,7 @@ void Monitor::timecheck_finish_round(bool success) assert(timecheck_waiting.empty()); assert(timecheck_acks == quorum.size()); timecheck_report(); + timecheck_check_skews(); return; } @@ -3870,6 +3871,65 @@ void Monitor::timecheck_cleanup() timecheck_waiting.clear(); timecheck_skews.clear(); timecheck_latencies.clear(); + + timecheck_rounds_since_clean = 0; +} + +void Monitor::timecheck_reset_event() +{ + if (timecheck_event) { + timer.cancel_event(timecheck_event); + timecheck_event = NULL; + } + + double delay = + cct->_conf->mon_timecheck_skew_interval * timecheck_rounds_since_clean; + + if (delay <= 0 || delay > cct->_conf->mon_timecheck_interval) { + delay = cct->_conf->mon_timecheck_interval; + } + + dout(10) << __func__ << " delay " << delay + << " rounds_since_clean " << timecheck_rounds_since_clean + << dendl; + + timecheck_event = new C_TimeCheck(this); + timer.add_event_after(delay, timecheck_event); +} + +void Monitor::timecheck_check_skews() +{ + dout(10) << __func__ << dendl; + assert(is_leader()); + assert((timecheck_round % 2) == 0); + if (monmap->size() == 1) { + assert(0 == "We are alone; we shouldn't have gotten here!"); + return; + } + assert(timecheck_latencies.size() == timecheck_skews.size()); + + bool found_skew = false; + for (map::iterator p = timecheck_skews.begin(); + p != timecheck_skews.end(); ++p) { + + double abs_skew; + if (timecheck_has_skew(p->second, &abs_skew)) { + dout(10) << __func__ + << " " << p->first << " skew " << abs_skew << dendl; + found_skew = true; + } + } + + if (found_skew) { + ++timecheck_rounds_since_clean; + timecheck_reset_event(); + } else if (timecheck_rounds_since_clean > 0) { + dout(1) << __func__ + << " no clock skews found after " << timecheck_rounds_since_clean + << " rounds" << dendl; + timecheck_rounds_since_clean = 0; + } + } void Monitor::timecheck_report() @@ -3892,7 +3952,8 @@ void Monitor::timecheck_report() m->epoch = get_epoch(); m->round = timecheck_round; - for (map::iterator it = timecheck_skews.begin(); it != timecheck_skews.end(); ++it) { + for (map::iterator it = timecheck_skews.begin(); + it != timecheck_skews.end(); ++it) { double skew = it->second; double latency = timecheck_latencies[it->first]; @@ -3951,10 +4012,10 @@ health_status_t Monitor::timecheck_status(ostringstream &ss, const double latency) { health_status_t status = HEALTH_OK; - double abs_skew = (skew_bound > 0 ? skew_bound : -skew_bound); assert(latency >= 0); - if (abs_skew > g_conf->mon_clock_drift_allowed) { + double abs_skew; + if (timecheck_has_skew(skew_bound, &abs_skew)) { status = HEALTH_WARN; ss << "clock skew " << abs_skew << "s" << " > max " << g_conf->mon_clock_drift_allowed << "s"; diff --git a/src/mon/Monitor.h b/src/mon/Monitor.h index b1ea51d4f2e9..add1c3ed6760 100644 --- a/src/mon/Monitor.h +++ b/src/mon/Monitor.h @@ -53,6 +53,7 @@ #include "include/memory.h" #include "include/str_map.h" #include +#include #include "common/TrackedOp.h" #include "mon/MonOpRequest.h" @@ -502,6 +503,15 @@ private: version_t timecheck_round; unsigned int timecheck_acks; utime_t timecheck_round_start; + /* When we hit a skew we will start a new round based off of + * 'mon_timecheck_skew_interval'. Each new round will be backed off + * until we hit 'mon_timecheck_interval' -- which is the typical + * interval when not in the presence of a skew. + * + * This variable tracks the number of rounds with skews since last clean + * so that we can report to the user and properly adjust the backoff. + */ + uint64_t timecheck_rounds_since_clean; /** * Time Check event. */ @@ -521,6 +531,8 @@ private: void timecheck_finish_round(bool success = true); void timecheck_cancel_round(); void timecheck_cleanup(); + void timecheck_reset_event(); + void timecheck_check_skews(); void timecheck_report(); void timecheck(); health_status_t timecheck_status(ostringstream &ss, @@ -529,6 +541,16 @@ private: void handle_timecheck_leader(MonOpRequestRef op); void handle_timecheck_peon(MonOpRequestRef op); void handle_timecheck(MonOpRequestRef op); + + /** + * Returns 'true' if this is considered to be a skew; 'false' otherwise. + */ + bool timecheck_has_skew(const double skew_bound, double *abs) const { + double abs_skew = std::fabs(skew_bound); + if (abs) + *abs = abs_skew; + return (abs_skew > g_conf->mon_clock_drift_allowed); + } /** * @} */