From: Sage Weil <sage@redhat.com>
Date: Tue, 15 Sep 2015 20:08:02 +0000 (-0400)
Subject: osd: exponential backoff on pg stats ack timeout
X-Git-Tag: v10.0.1~26^2~62
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=facd36fc14bb855e950fd55a764444f8887e0828;p=ceph.git

osd: exponential backoff on pg stats ack timeout

If we don't get a timely response to our pg stats update we fail
the mon connection and reconnect to a new mon.  If the mons aren't
responding because they are overloaded (for example, because they
are overwhelmed with stats updates) this just makes the problem
worse.

Mitigate the situation by doing an exponential backoff on the
timeout.  When we do successfully send an update, slowly decay the
timeout back to the initial value.

Signed-off-by: Sage Weil <sage@redhat.com>
---

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index 4eda30098f5..cf8235de5bb 100644
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -660,6 +660,8 @@ OPTION(osd_mon_report_interval_max, OPT_INT, 120)
 OPTION(osd_mon_report_interval_min, OPT_INT, 5)  // pg stats, failures, up_thru, boot.
 OPTION(osd_pg_stat_report_interval_max, OPT_INT, 500)  // report pg stats for any given pg at least this often
 OPTION(osd_mon_ack_timeout, OPT_INT, 30) // time out a mon if it doesn't ack stats
+OPTION(osd_stats_ack_timeout_factor, OPT_DOUBLE, 2.0) // multiples of mon_ack_timeout
+OPTION(osd_stats_ack_timeout_decay, OPT_DOUBLE, .9)
 OPTION(osd_default_data_pool_replay_window, OPT_INT, 45)
 OPTION(osd_preserve_trimmed_log, OPT_BOOL, false)
 OPTION(osd_auto_mark_unfound_lost, OPT_BOOL, false)
diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 0562eed8f3a..9130c516fbf 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -1566,6 +1566,7 @@ OSD::OSD(CephContext *cct_, ObjectStore *store_,
   debug_drop_pg_create_probability(cct->_conf->osd_debug_drop_pg_create_probability),
   debug_drop_pg_create_duration(cct->_conf->osd_debug_drop_pg_create_duration),
   debug_drop_pg_create_left(-1),
+  stats_ack_timeout(cct->_conf->osd_mon_ack_timeout),
   outstanding_pg_stats(false),
   timeout_mon_on_pg_stats(true),
   up_thru_wanted(0), up_thru_pending(0),
@@ -3988,13 +3989,17 @@ void OSD::tick()
     // mon report?
     utime_t now = ceph_clock_now(cct);
     if (outstanding_pg_stats && timeout_mon_on_pg_stats &&
-	(now - cct->_conf->osd_mon_ack_timeout) > last_pg_stats_ack) {
-      dout(1) << "mon hasn't acked PGStats in " << now - last_pg_stats_ack
+	(now - stats_ack_timeout) > last_pg_stats_ack) {
+      dout(1) << __func__ << " mon hasn't acked PGStats in "
+	      << now - last_pg_stats_ack
 	      << " seconds, reconnecting elsewhere" << dendl;
       monc->reopen_session(new C_MonStatsAckTimer(this));
       timeout_mon_on_pg_stats = false;
       last_pg_stats_ack = ceph_clock_now(cct);  // reset clock
       last_pg_stats_sent = utime_t();
+      stats_ack_timeout =
+	MAX(g_conf->osd_mon_ack_timeout,
+	    stats_ack_timeout * g_conf->osd_stats_ack_timeout_factor);
     }
     if (now - last_pg_stats_sent > cct->_conf->osd_mon_report_interval_max) {
       osd_stat_updated = true;
@@ -4800,6 +4805,12 @@ void OSD::handle_pg_stats_ack(MPGStatsAck *ack)
 
   last_pg_stats_ack = ceph_clock_now(cct);
 
+  // decay timeout slowly (analogous to TCP)
+  stats_ack_timeout =
+    MAX(g_conf->osd_mon_ack_timeout,
+	stats_ack_timeout * g_conf->osd_stats_ack_timeout_decay);
+  dout(20) << __func__ << "  timeout now " << stats_ack_timeout << dendl;
+
   pg_stat_queue_lock.Lock();
 
   if (ack->get_tid() > pg_stat_tid_flushed) {
diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index 44a492c83aa..00ce4b25bd9 100644
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -1970,6 +1970,7 @@ protected:
    *  elsewhere.
    */
   utime_t last_pg_stats_ack;
+  float stats_ack_timeout;
   bool outstanding_pg_stats; // some stat updates haven't been acked yet
   bool timeout_mon_on_pg_stats;
   void restart_stats_timer() {