From 86b082bd947799f6d4dbf0f88f1e64d993d5a488 Mon Sep 17 00:00:00 2001 From: Wencong Wan Date: Wed, 20 May 2020 02:23:01 +0800 Subject: [PATCH] librbd: support duration controllable continuous burst Modify the design of token bucket to save more tokens than burst value, so as to support continuous burst. The desired burst duration can be controlled by rbd_qos_XYZ_burst_seconds. Fixes: https://tracker.ceph.com/issues/45313 Signed-off-by: wencong wan --- doc/rbd/rbd-config-ref.rst | 48 +++++++++++++ src/common/Throttle.cc | 68 +++++++++++++------ src/common/Throttle.h | 14 ++-- src/common/options.cc | 30 ++++++++ src/librbd/ImageCtx.cc | 18 +++-- src/librbd/io/ImageDispatcher.cc | 4 +- src/librbd/io/ImageDispatcher.h | 3 +- src/librbd/io/ImageDispatcherInterface.h | 2 +- src/librbd/io/QosImageDispatch.cc | 6 +- src/librbd/io/QosImageDispatch.h | 3 +- .../librbd/io/test_mock_QosImageDispatch.cc | 6 +- src/test/librbd/mock/io/MockImageDispatcher.h | 2 +- 12 files changed, 160 insertions(+), 44 deletions(-) diff --git a/doc/rbd/rbd-config-ref.rst b/doc/rbd/rbd-config-ref.rst index 3d114741a31..03a5b0160b3 100644 --- a/doc/rbd/rbd-config-ref.rst +++ b/doc/rbd/rbd-config-ref.rst @@ -375,6 +375,54 @@ settings. :Default: ``0`` +``rbd qos iops burst seconds`` + +:Description: The desired burst duration in seconds of IO operations. +:Type: Unsigned Integer +:Required: No +:Default: ``1`` + + +``rbd qos bps burst seconds`` + +:Description: The desired burst duration in seconds of IO bytes. +:Type: Unsigned Integer +:Required: No +:Default: ``1`` + + +``rbd qos read iops burst seconds`` + +:Description: The desired burst duration in seconds of read operations. +:Type: Unsigned Integer +:Required: No +:Default: ``1`` + + +``rbd qos write iops burst seconds`` + +:Description: The desired burst duration in seconds of write operations. +:Type: Unsigned Integer +:Required: No +:Default: ``1`` + + +``rbd qos read bps burst seconds`` + +:Description: The desired burst duration in seconds of read bytes. +:Type: Unsigned Integer +:Required: No +:Default: ``1`` + + +``rbd qos write bps burst seconds`` + +:Description: The desired burst duration in seconds of write bytes. +:Type: Unsigned Integer +:Required: No +:Default: ``1`` + + ``rbd qos schedule tick min`` :Description: The minimum schedule tick (in milliseconds) for QoS. diff --git a/src/common/Throttle.cc b/src/common/Throttle.cc index 9815493cea7..f25fa7cbe6e 100644 --- a/src/common/Throttle.cc +++ b/src/common/Throttle.cc @@ -671,52 +671,76 @@ uint64_t TokenBucketThrottle::Bucket::get(uint64_t c) { } uint64_t got = 0; - if (remain >= c) { + if (available >= c) { // There is enough token in bucket, take c. got = c; + available -= c; remain -= c; } else { - // There is not enough, take all remain. - got = remain; - remain = 0; + // There is not enough, take all available. + got = available; + remain -= available; + available = 0; } return got; } -uint64_t TokenBucketThrottle::Bucket::put(uint64_t c) { +uint64_t TokenBucketThrottle::Bucket::put(uint64_t tokens, double burst_ratio) { if (0 == max) { return 0; } - if (c) { - // put c tokens into bucket + if (tokens) { + // put tokens into bucket uint64_t current = remain; - if ((current + c) <= max) { - remain += c; + if ((current + tokens) <= capacity) { + remain += tokens; } else { - remain = max; + remain = capacity; + } + + // available tokens increase at burst speed + uint64_t available_inc = tokens; + if (burst_ratio > 1) { + available_inc = (uint64_t)(tokens * burst_ratio); + } + uint64_t inc_upper_limit = remain > max ? max : remain; + if ((available + available_inc) <= inc_upper_limit ){ + available += available_inc; + }else{ + available = inc_upper_limit; } + } return remain; } -void TokenBucketThrottle::Bucket::set_max(uint64_t m) { - if (remain > m || 0 == m) { - remain = m; +void TokenBucketThrottle::Bucket::set_max(uint64_t max, uint64_t burst_seconds) { + // the capacity of bucket should not be less than max + if (burst_seconds < 1){ + burst_seconds = 1; } - max = m; + uint64_t new_capacity = max*burst_seconds; + if (capacity != new_capacity){ + capacity = new_capacity; + remain = capacity; + } + if (available > max || 0 == max) { + available = max; + } + this->max = max; } TokenBucketThrottle::TokenBucketThrottle( CephContext *cct, const std::string &name, - uint64_t capacity, + uint64_t burst, uint64_t avg, SafeTimer *timer, ceph::mutex *timer_lock) : m_cct(cct), m_name(name), - m_throttle(m_cct, name + "_bucket", capacity), - m_avg(avg), m_timer(timer), m_timer_lock(timer_lock), + m_throttle(m_cct, name + "_bucket", burst), + m_burst(burst), m_avg(avg), m_timer(timer), m_timer_lock(timer_lock), m_lock(ceph::make_mutex(name + "_lock")) {} @@ -738,7 +762,7 @@ TokenBucketThrottle::~TokenBucketThrottle() { } } -int TokenBucketThrottle::set_limit(uint64_t average, uint64_t burst) { +int TokenBucketThrottle::set_limit(uint64_t average, uint64_t burst, uint64_t burst_seconds) { { std::lock_guard lock{m_lock}; @@ -766,7 +790,7 @@ int TokenBucketThrottle::set_limit(uint64_t average, uint64_t burst) { m_current_tick = 0; // for the default configuration of burst. - m_throttle.set_max(0 == burst ? average : burst); + m_throttle.set_max(0 == burst ? average : burst, burst_seconds); } // turn millisecond to second m_schedule_tick = m_tick / 1000.0; @@ -809,7 +833,11 @@ void TokenBucketThrottle::add_tokens() { { std::lock_guard lock(m_lock); // put tokens into bucket. - m_throttle.put(tokens_this_tick()); + double burst_ratio = 1.0; + if (m_throttle.max > m_avg && m_avg > 0){ + burst_ratio = (double)m_throttle.max/m_avg; + } + m_throttle.put(tokens_this_tick(), burst_ratio); if (0 == m_avg || 0 == m_throttle.max) tmp_blockers.swap(m_blockers); // check the m_blockers from head to tail, if blocker can get diff --git a/src/common/Throttle.h b/src/common/Throttle.h index d4bb77025af..57a9407ce9d 100644 --- a/src/common/Throttle.h +++ b/src/common/Throttle.h @@ -336,13 +336,15 @@ class TokenBucketThrottle { uint64_t remain; uint64_t max; + uint64_t capacity; + uint64_t available; Bucket(CephContext *cct, const std::string &name, uint64_t m) - : cct(cct), name(name), remain(m), max(m) {} + : cct(cct), name(name), remain(m), max(m), capacity(m), available(m) {} uint64_t get(uint64_t c); - uint64_t put(uint64_t c); - void set_max(uint64_t m); + uint64_t put(uint64_t tokens, double burst_ratio); + void set_max(uint64_t max, uint64_t burst_seconds); }; struct Blocker { @@ -356,8 +358,8 @@ class TokenBucketThrottle { CephContext *m_cct; const std::string m_name; Bucket m_throttle; - uint64_t m_avg = 0; uint64_t m_burst = 0; + uint64_t m_avg = 0; SafeTimer *m_timer; ceph::mutex *m_timer_lock; Context *m_token_ctx = nullptr; @@ -405,7 +407,7 @@ class TokenBucketThrottle { public: TokenBucketThrottle(CephContext *cct, const std::string &name, - uint64_t capacity, uint64_t avg, + uint64_t burst, uint64_t avg, SafeTimer *timer, ceph::mutex *timer_lock); ~TokenBucketThrottle(); @@ -450,7 +452,7 @@ public: return wait; } - int set_limit(uint64_t average, uint64_t burst); + int set_limit(uint64_t average, uint64_t burst, uint64_t burst_seconds); void set_schedule_tick_min(uint64_t tick); private: diff --git a/src/common/options.cc b/src/common/options.cc index c39095bfde8..b910ebccafe 100644 --- a/src/common/options.cc +++ b/src/common/options.cc @@ -7463,6 +7463,36 @@ static std::vector