From 07507694bc3db4fc54e6e10955fe9beae56ca080 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Mon, 1 Feb 2016 13:30:26 -0500 Subject: [PATCH] osd: set per-interval cap on promotions This will help mitigate stampedes by capping the number of promotes within a single tick interval. Signed-off-by: Sage Weil --- src/osd/OSD.cc | 6 ++++++ src/osd/OSD.h | 11 +++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index 0bcdead51b1..9c7c636f31d 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -236,6 +236,8 @@ OSDService::OSDService(OSD *osd) : agent_timer_lock("OSD::agent_timer_lock"), agent_timer(osd->client_messenger->cct, agent_timer_lock), promote_probability_millis(1000), + promote_max_objects(0), + promote_max_bytes(0), objecter(new Objecter(osd->client_messenger->cct, osd->objecter_messenger, osd->monc, NULL, 0, 0)), objecter_finisher(osd->client_messenger->cct), watch_lock("OSD::watch_lock"), @@ -661,6 +663,10 @@ void OSDService::promote_throttle_recalibrate() << ", prob " << promote_probability_millis << " -> " << prob << dendl; promote_probability_millis = prob; + + // set hard limits for this interval to mitigate stampedes + promote_max_objects = target_obj_sec * OSD::OSD_TICK_INTERVAL * 2; + promote_max_bytes = target_bytes_sec * OSD::OSD_TICK_INTERVAL * 2; } // ------------------------------------- diff --git a/src/osd/OSD.h b/src/osd/OSD.h index 6f29ce9508d..cf6370aeaca 100644 --- a/src/osd/OSD.h +++ b/src/osd/OSD.h @@ -785,13 +785,20 @@ public: unsigned promote_probability_millis; ///< probability thousands. one word. PromoteCounter promote_counter; utime_t last_recalibrate; + unsigned long promote_max_objects, promote_max_bytes; bool promote_throttle() { // NOTE: lockless! we rely on the probability being a single word. promote_counter.attempt(); if ((unsigned)rand() % 1000 > promote_probability_millis) - return true; // yes throttle (no promote) - return false; // no throttle (promote) + return true; // yes throttle (no promote) + if (promote_max_objects && + promote_counter.objects.read() > promote_max_objects) + return true; // yes throttle + if (promote_max_bytes && + promote_counter.bytes.read() > promote_max_bytes) + return true; // yes throttle + return false; // no throttle (promote) } void promote_finish(uint64_t bytes) { promote_counter.finish(bytes); -- 2.47.3