osd: exponential backoff on pg stats ack timeout

If we don't get a timely response to our pg stats update we fail
the mon connection and reconnect to a new mon.  If the mons aren't
responding because they are overloaded (for example, because they
are overwhelmed with stats updates) this just makes the problem
worse.

Mitigate the situation by doing an exponential backoff on the
timeout.  When we do successfully send an update, slowly decay the
timeout back to the initial value.

Signed-off-by: Sage Weil <sage@redhat.com>

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index 4eda30098f5b5bb50aba6e26b76a691d83dd05df..cf8235de5bbfbaa74119be3c53c136523045894a 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -660,6 +660,8 @@ OPTION(osd_mon_report_interval_max, OPT_INT, 120)
 OPTION(osd_mon_report_interval_min, OPT_INT, 5)  // pg stats, failures, up_thru, boot.
 OPTION(osd_pg_stat_report_interval_max, OPT_INT, 500)  // report pg stats for any given pg at least this often
 OPTION(osd_mon_ack_timeout, OPT_INT, 30) // time out a mon if it doesn't ack stats
+OPTION(osd_stats_ack_timeout_factor, OPT_DOUBLE, 2.0) // multiples of mon_ack_timeout
+OPTION(osd_stats_ack_timeout_decay, OPT_DOUBLE, .9)
 OPTION(osd_default_data_pool_replay_window, OPT_INT, 45)
 OPTION(osd_preserve_trimmed_log, OPT_BOOL, false)
 OPTION(osd_auto_mark_unfound_lost, OPT_BOOL, false)

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 0562eed8f3a88f5432d2ec4949de0cf49c394c45..9130c516fbf988fa6acb6997d79355e76afdb200 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -1566,6 +1566,7 @@ OSD::OSD(CephContext *cct_, ObjectStore *store_,
   debug_drop_pg_create_probability(cct->_conf->osd_debug_drop_pg_create_probability),
   debug_drop_pg_create_duration(cct->_conf->osd_debug_drop_pg_create_duration),
   debug_drop_pg_create_left(-1),
+  stats_ack_timeout(cct->_conf->osd_mon_ack_timeout),
   outstanding_pg_stats(false),
   timeout_mon_on_pg_stats(true),
   up_thru_wanted(0), up_thru_pending(0),
@@ -3988,13 +3989,17 @@ void OSD::tick()
     // mon report?
     utime_t now = ceph_clock_now(cct);
     if (outstanding_pg_stats && timeout_mon_on_pg_stats &&
-       (now - cct->_conf->osd_mon_ack_timeout) > last_pg_stats_ack) {
-      dout(1) << "mon hasn't acked PGStats in " << now - last_pg_stats_ack
+       (now - stats_ack_timeout) > last_pg_stats_ack) {
+      dout(1) << __func__ << " mon hasn't acked PGStats in "
+             << now - last_pg_stats_ack
              << " seconds, reconnecting elsewhere" << dendl;
       monc->reopen_session(new C_MonStatsAckTimer(this));
       timeout_mon_on_pg_stats = false;
       last_pg_stats_ack = ceph_clock_now(cct);  // reset clock
       last_pg_stats_sent = utime_t();
+      stats_ack_timeout =
+       MAX(g_conf->osd_mon_ack_timeout,
+           stats_ack_timeout * g_conf->osd_stats_ack_timeout_factor);
     }
     if (now - last_pg_stats_sent > cct->_conf->osd_mon_report_interval_max) {
       osd_stat_updated = true;
@@ -4800,6 +4805,12 @@ void OSD::handle_pg_stats_ack(MPGStatsAck *ack)
 
   last_pg_stats_ack = ceph_clock_now(cct);
 
+  // decay timeout slowly (analogous to TCP)
+  stats_ack_timeout =
+    MAX(g_conf->osd_mon_ack_timeout,
+       stats_ack_timeout * g_conf->osd_stats_ack_timeout_decay);
+  dout(20) << __func__ << "  timeout now " << stats_ack_timeout << dendl;
+
   pg_stat_queue_lock.Lock();
 
   if (ack->get_tid() > pg_stat_tid_flushed) {

diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index 44a492c83aa6b66988bcd4e5753070c41648d169..00ce4b25bd9addfd373140b8ec8942112b8cccf4 100644 (file)
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -1970,6 +1970,7 @@ protected:
    *  elsewhere.
    */
   utime_t last_pg_stats_ack;
+  float stats_ack_timeout;
   bool outstanding_pg_stats; // some stat updates haven't been acked yet
   bool timeout_mon_on_pg_stats;
   void restart_stats_timer() {