osd: set per-interval cap on promotions

This will help mitigate stampedes by capping the number of
promotes within a single tick interval.

Signed-off-by: Sage Weil <sage@redhat.com>

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 0bcdead51b11edbd87e10030371bf9aa894b19a9..9c7c636f31d9717f1055e8a54235de71bbc0a889 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -236,6 +236,8 @@ OSDService::OSDService(OSD *osd) :
   agent_timer_lock("OSD::agent_timer_lock"),
   agent_timer(osd->client_messenger->cct, agent_timer_lock),
   promote_probability_millis(1000),
+  promote_max_objects(0),
+  promote_max_bytes(0),
   objecter(new Objecter(osd->client_messenger->cct, osd->objecter_messenger, osd->monc, NULL, 0, 0)),
   objecter_finisher(osd->client_messenger->cct),
   watch_lock("OSD::watch_lock"),
@@ -661,6 +663,10 @@ void OSDService::promote_throttle_recalibrate()
           << ", prob " << promote_probability_millis << " -> " << prob
           << dendl;
   promote_probability_millis = prob;
+
+  // set hard limits for this interval to mitigate stampedes
+  promote_max_objects = target_obj_sec * OSD::OSD_TICK_INTERVAL * 2;
+  promote_max_bytes = target_bytes_sec * OSD::OSD_TICK_INTERVAL * 2;
 }
 
 // -------------------------------------

diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index 6f29ce9508d20a84af78f7175b58a816e300a1cf..cf6370aeaca60cec1a58a1552790c435d76de7a4 100644 (file)
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -785,13 +785,20 @@ public:
   unsigned promote_probability_millis; ///< probability thousands. one word.
   PromoteCounter promote_counter;
   utime_t last_recalibrate;
+  unsigned long promote_max_objects, promote_max_bytes;
 
   bool promote_throttle() {
     // NOTE: lockless!  we rely on the probability being a single word.
     promote_counter.attempt();
     if ((unsigned)rand() % 1000 > promote_probability_millis)
-      return true;  //  yes throttle (no promote)
-    return false;   //   no throttle (promote)
+      return true;  // yes throttle (no promote)
+    if (promote_max_objects &&
+       promote_counter.objects.read() > promote_max_objects)
+      return true;  // yes throttle
+    if (promote_max_bytes &&
+       promote_counter.bytes.read() > promote_max_bytes)
+      return true;  // yes throttle
+    return false;   //  no throttle (promote)
   }
   void promote_finish(uint64_t bytes) {
     promote_counter.finish(bytes);