rgw: add accurate Retry-After header to rate-limit responses

When RGW rejects an S3 request with -ERR_RATE_LIMITED, return an HTTP
Retry-After header that tells the client how many seconds to wait before
retrying. The value is computed from the current token-bucket deficit and
replenishment rate, so a polite client that honors the header will retry
right around when its tokens recover.

The delay is threaded out of the rate limiter through req_state and onto
the response in abort_early(). The rate-limit check now returns int64_t
seconds (0 = allowed) instead of bool, with a 1-second floor on the
rejected path.

The seconds-to-recover calculation is factored into a small static helper
RateLimiterEntry::compute_delay(limit, needed, interval) so it can be
unit-tested directly. Each dimension (ops, bytes) calls it independently
and the caller takes the max, leaving room for additional limit
dimensions in the future.

The user-facing documentation is updated to describe the new header and
to suggest proxy-layer remapping for operators who need 429 instead of
the existing 503.

Tracker: https://tracker.ceph.com/issues/64083
Tracker: https://tracker.ceph.com/issues/67655
Signed-off-by: Lumir Sliva <61183145+lumir-sliva@users.noreply.github.com>

diff --git a/doc/radosgw/admin.rst b/doc/radosgw/admin.rst
index 2115845a72b5a9fb7e81c505b9ef7b18fab826d5..18df6218b79fb0da4c618ca5a73bf7e41b04122a 100644 (file)
--- a/doc/radosgw/admin.rst
+++ b/doc/radosgw/admin.rst
@@ -678,6 +678,12 @@ then the ``GET`` action will fail. After "user A" has completed this 1 GB
 operation, RGW blocks the user's requests for up to two accumulation intervals. After this
 time has elapsed, "user A" will be able to send ``GET`` requests again.
 
+Rate-limited requests receive an HTTP ``503`` response with the S3 error code
+``SlowDown``, matching the AWS S3 behavior. The response includes a
+``Retry-After`` header with the estimated number of seconds until the rate-limit
+window allows the next request through, computed from the current token-bucket
+state. Load balancers or reverse proxies can be configured to map the ``503``
+with ``Retry-After`` to ``429 Too Many Requests`` if clients require it.
 
 - **Bucket:** The ``--bucket`` option allows you to specify a rate limit for a
   bucket.

diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h
index 8f95f4ef6716971ba1228754ca6569231817c09b..364a2c6d8b43ce3e87dd5e282d3c513397b11b30 100644 (file)
--- a/src/rgw/rgw_common.h
+++ b/src/rgw/rgw_common.h
@@ -1310,6 +1310,7 @@ struct req_state : DoutPrefixProvider {
   std::shared_ptr<RateLimiter> ratelimit_data;
   RGWRateLimitInfo user_ratelimit;
   RGWRateLimitInfo bucket_ratelimit;
+  int64_t ratelimit_retry_after{0};
   std::string ratelimit_bucket_marker;
   std::string ratelimit_user_name;
   bool content_started{false};

diff --git a/src/rgw/rgw_process.cc b/src/rgw/rgw_process.cc
index 507d4e3ff7b1125380e8a03546d9e09f529120d3..d23295b2a904a43959488e0c4e0fef1813f8ca15 100644 (file)
--- a/src/rgw/rgw_process.cc
+++ b/src/rgw/rgw_process.cc
@@ -140,8 +140,8 @@ bool rate_limit(rgw::sal::Driver* driver, req_state* s) {
   if (s->user->get_id().id == RGW_USER_ANON_ID && global_anon.enabled) {
     *user_ratelimit = global_anon;
   }
-  bool limit_bucket = false;
-  bool limit_user = s->ratelimit_data->should_rate_limit(method, s->ratelimit_user_name, s->time, user_ratelimit, s->info.request_params);
+  int64_t limit_bucket = 0;
+  int64_t limit_user = s->ratelimit_data->should_rate_limit(method, s->ratelimit_user_name, s->time, user_ratelimit, s->info.request_params);
 
   if(!rgw::sal::Bucket::empty(s->bucket.get()))
   {
@@ -169,6 +169,10 @@ bool rate_limit(rgw::sal::Driver* driver, req_state* s) {
   }
   s->user_ratelimit = *user_ratelimit;
   s->bucket_ratelimit = *bucket_ratelimit;
+  int64_t delay = limit_user ? limit_user : limit_bucket;
+  if (delay > 0) {
+    s->ratelimit_retry_after = delay;
+  }
   return (limit_user || limit_bucket);
 }
 

diff --git a/src/rgw/rgw_ratelimit.h b/src/rgw/rgw_ratelimit.h
index 6410d197758a86c5a4ee06cf02fd2405a297568e..04c366500089c57cf113b647c4c70a1d37ccb126 100644 (file)
--- a/src/rgw/rgw_ratelimit.h
+++ b/src/rgw/rgw_ratelimit.h
@@ -9,13 +9,29 @@ enum class OpType { Read, Write, List, Delete };
 
 
 class RateLimiterEntry {
-  /* 
+public:
+  /*
     fixed_point_rgw_ratelimit is important to preserve the precision of the token calculation
     for example: a user have a limit of single op per minute, the user will consume its single token and then will send another request, 1s after it.
     in that case, without this method, the user will get 0 tokens although it should get 0.016 tokens.
     using this method it will add 16 tokens to the user, and the user will have 16 tokens, each time rgw will do comparison rgw will divide by fixed_point_rgw_ratelimit, so the user will be blocked anyway until it has enough tokens.
   */
   static constexpr int64_t fixed_point_rgw_ratelimit = 1000;
+
+  // Compute the seconds needed to replenish `needed` tokens when tokens
+  // replenish at `limit` units per `interval` seconds. All three parameters
+  // are in fixed-point-scaled units (multiply user-visible token counts by
+  // fixed_point_rgw_ratelimit before passing). The result is in seconds,
+  // rounded up. Returns 0 when `limit` or `needed` is non-positive, which
+  // callers treat as "no delay required for this dimension".
+  static int64_t compute_delay(int64_t limit, int64_t needed, int64_t interval) {
+    if (limit <= 0 || needed <= 0) {
+      return 0;
+    }
+    return (needed * interval + limit - 1) / limit;
+  }
+
+private:
   // counters are tracked in multiples of fixed_point_rgw_ratelimit
   struct counters {
     int64_t ops = 0;
@@ -53,66 +69,74 @@ class RateLimiterEntry {
   {
     return write.bytes / fixed_point_rgw_ratelimit;
   }
-  bool should_rate_limit_read(int64_t ops_limit, int64_t bw_limit) {
-    //check if tenants did not reach their bw or ops limits and that the limits are not 0 (which is unlimited)
-    if(((read_ops() - 1 < 0) && (ops_limit > 0)) ||
-      (read_bytes() < 0 && bw_limit > 0))
-  {
-    return true;
-  }
+  // Returns 0 if the request is allowed, or the estimated retry delay
+  // in seconds if rate-limited. compute_delay() is the sole decision
+  // maker: delay > 0 means rate-limited, 0 means allowed.
+  int64_t should_rate_limit_read(int64_t ops_limit, int64_t bw_limit) {
+    const int64_t interval = g_ceph_context->_conf->rgw_ratelimit_interval;
+    const int64_t ops_delay = compute_delay(ops_limit * fixed_point_rgw_ratelimit,
+                                            fixed_point_rgw_ratelimit - read.ops,
+                                            interval);
+    const int64_t bw_delay = compute_delay(bw_limit * fixed_point_rgw_ratelimit,
+                                           -read.bytes,
+                                           interval);
+    const int64_t delay = std::max(ops_delay, bw_delay);
+    if (delay > 0) {
+      return delay;
+    }
     // we don't want to reduce ops' tokens if we've rejected it.
-    if (read_ops() > 0)
-    {
-      // make sure read.ops always greater or equal than 0
-      // to avoid large budget caused by the case that rate limiter was enabled and read.ops set to 0
+    if (read_ops() > 0) {
       read.ops -= fixed_point_rgw_ratelimit;
     }
-    return false;
+    return 0;
   }
-  bool should_rate_limit_list(int64_t ops_limit)
+  int64_t should_rate_limit_list(int64_t ops_limit)
   {
-    if ((list_ops() - 1 < 0) && (ops_limit > 0)) {
-      return true;
+    const int64_t interval = g_ceph_context->_conf->rgw_ratelimit_interval;
+    const int64_t delay = compute_delay(ops_limit * fixed_point_rgw_ratelimit,
+                                        fixed_point_rgw_ratelimit - list.ops,
+                                        interval);
+    if (delay > 0) {
+      return delay;
     }
-    if (list_ops() > 0)
-    {
-      // make sure list.ops always greater or equal than 0
-      // to avoid large budget caused by the case that rate limiter was enabled and list.ops set to 0
+    if (list_ops() > 0) {
       list.ops -= fixed_point_rgw_ratelimit;
     }
-    return false;
+    return 0;
   }
-  bool should_rate_limit_delete(int64_t ops_limit)
+  int64_t should_rate_limit_delete(int64_t ops_limit)
   {
-    if ((delete_ops() - 1 < 0) && (ops_limit > 0)) {
-      return true;
+    const int64_t interval = g_ceph_context->_conf->rgw_ratelimit_interval;
+    const int64_t delay = compute_delay(ops_limit * fixed_point_rgw_ratelimit,
+                                        fixed_point_rgw_ratelimit - del.ops,
+                                        interval);
+    if (delay > 0) {
+      return delay;
     }
-    if (delete_ops() > 0)
-    {
-      // make sure del.ops always greater or equal than 0
-      // to avoid large budget caused by the case that rate limiter was enabled and del.ops set to 0
+    if (delete_ops() > 0) {
       del.ops -= fixed_point_rgw_ratelimit;
     }
-    return false;
+    return 0;
   }
-  bool should_rate_limit_write(int64_t ops_limit, int64_t bw_limit) 
+  int64_t should_rate_limit_write(int64_t ops_limit, int64_t bw_limit)
   {
-    //check if tenants did not reach their bw or ops limits and that the limits are not 0 (which is unlimited)
-    if(((write_ops() - 1 < 0) && (ops_limit > 0)) ||
-      (write_bytes() < 0 && bw_limit > 0))
-    {
-      return true;
+    const int64_t interval = g_ceph_context->_conf->rgw_ratelimit_interval;
+    const int64_t ops_delay = compute_delay(ops_limit * fixed_point_rgw_ratelimit,
+                                            fixed_point_rgw_ratelimit - write.ops,
+                                            interval);
+    const int64_t bw_delay = compute_delay(bw_limit * fixed_point_rgw_ratelimit,
+                                           -write.bytes,
+                                           interval);
+    const int64_t delay = std::max(ops_delay, bw_delay);
+    if (delay > 0) {
+      return delay;
     }
-
-    // we don't want to reduce ops' tokens if we've rejected it.
-    if (write_ops() > 0)
-    {
-      // make sure write.ops always greater or equal than 0
-      // to avoid large budget caused by the case that rate limiter was enabled and write.ops set to 0
+    if (write_ops() > 0) {
       write.ops -= fixed_point_rgw_ratelimit;
     }
-    return false;
+    return 0;
   }
+
   /* The purpose of this function is to minimum time before overriding the stored timestamp
      This function is necessary to force the increase tokens add at least 1 token when it updates the last stored timestamp.
      That way the user/bucket will not lose tokens because of rounding
@@ -167,7 +191,8 @@ class RateLimiterEntry {
   }
 
   public:
-    bool should_rate_limit(OpType op_type, const RGWRateLimitInfo* ratelimit_info, ceph::timespan curr_timestamp)
+    // Returns 0 if allowed, or the retry delay in seconds if rate-limited.
+    int64_t should_rate_limit(OpType op_type, const RGWRateLimitInfo* ratelimit_info, ceph::timespan curr_timestamp)
     {
       std::unique_lock lock(ts_lock);
       increase_tokens(curr_timestamp, ratelimit_info);
@@ -181,7 +206,7 @@ class RateLimiterEntry {
         case OpType::Delete:
           return should_rate_limit_delete(ratelimit_info->max_delete_ops);
       }
-      return false;
+      return 0;
     }
     void decrease_bytes(bool is_read, int64_t amount, const RGWRateLimitInfo* info) {
       std::unique_lock lock(ts_lock);
@@ -283,11 +308,12 @@ class RateLimiter : public DoutPrefix {
       ratelimit_entries.max_load_factor(1000);
     };
 
-    bool should_rate_limit(const char *method, const std::string& key, ceph::coarse_real_time curr_timestamp, const RGWRateLimitInfo* ratelimit_info, const std::string& resource) {
+    // Returns 0 if allowed, or the retry delay in seconds if rate-limited.
+    int64_t should_rate_limit(const char *method, const std::string& key, ceph::coarse_real_time curr_timestamp, const RGWRateLimitInfo* ratelimit_info, const std::string& resource) {
       ldpp_dout(this, 21) << "checking should_rate_limit: key=" << std::quoted(key) << " enabled=" << ratelimit_info->enabled << dendl;
       if (key.empty() || key.length() == 1 || !ratelimit_info->enabled)
       {
-        return false;
+        return 0;
       }
       OpType type = op_type(method, resource, ratelimit_info);
       auto& it = find_or_create(key);

diff --git a/src/rgw/rgw_rest.cc b/src/rgw/rgw_rest.cc
index c592b7d2411d403f4812eb3900804a406a592715..a22baa7bca60974fd0f3397086e026e6b6c097e7 100644 (file)
--- a/src/rgw/rgw_rest.cc
+++ b/src/rgw/rgw_rest.cc
@@ -718,6 +718,10 @@ void abort_early(req_state *s, RGWOp* op, int err_no,
     }
 
     dump_errno(s);
+    if (err_no == -ERR_RATE_LIMITED && s->ratelimit_retry_after > 0) {
+      dump_header(s, "Retry-After",
+                  static_cast<long long>(s->ratelimit_retry_after));
+    }
     dump_bucket_from_state(s);
     if (err_no == -ERR_PERMANENT_REDIRECT || err_no == -ERR_WEBSITE_REDIRECT) {
       string dest_uri;

diff --git a/src/test/rgw/test_rgw_ratelimit.cc b/src/test/rgw/test_rgw_ratelimit.cc
index 88dae1be544f0e8689ff968488dd2547bc22bf49..a1c902f034b4220feabc36733dd113a5b7b03879 100644 (file)
--- a/src/test/rgw/test_rgw_ratelimit.cc
+++ b/src/test/rgw/test_rgw_ratelimit.cc
@@ -8,6 +8,36 @@
 using namespace std::chrono_literals;
 
 
+TEST(RGWRateLimitEntry, compute_delay)
+{
+  // Disabled limit returns 0 regardless of the deficit
+  EXPECT_EQ(0, RateLimiterEntry::compute_delay(0, 0, 1));
+  EXPECT_EQ(0, RateLimiterEntry::compute_delay(0, 1, 1));
+  EXPECT_EQ(0, RateLimiterEntry::compute_delay(0, 1000, 1));
+  EXPECT_EQ(0, RateLimiterEntry::compute_delay(0, 1001, 1));
+  EXPECT_EQ(0, RateLimiterEntry::compute_delay(-1, 1000, 1));
+
+  // Zero or negative deficit returns 0 (caller treats as "no delay needed")
+  EXPECT_EQ(0, RateLimiterEntry::compute_delay(1000, 0, 1));
+  EXPECT_EQ(0, RateLimiterEntry::compute_delay(1000, -1, 1));
+  EXPECT_EQ(0, RateLimiterEntry::compute_delay(1000, -1000, 1));
+
+  // Ceil division at interval=1: tokens replenish at `limit` per second
+  EXPECT_EQ(1, RateLimiterEntry::compute_delay(1000, 1, 1));
+  EXPECT_EQ(1, RateLimiterEntry::compute_delay(1000, 1000, 1));
+  EXPECT_EQ(2, RateLimiterEntry::compute_delay(1000, 1001, 1));
+  EXPECT_EQ(2, RateLimiterEntry::compute_delay(1000, 2000, 1));
+  EXPECT_EQ(3, RateLimiterEntry::compute_delay(1000, 2001, 1));
+
+  // Ceil division at interval=100: tokens replenish at `limit` per 100s
+  EXPECT_EQ(0, RateLimiterEntry::compute_delay(10000, 0, 100));
+  EXPECT_EQ(1, RateLimiterEntry::compute_delay(10000, 1, 100));
+  EXPECT_EQ(1, RateLimiterEntry::compute_delay(10000, 100, 100));
+  EXPECT_EQ(2, RateLimiterEntry::compute_delay(10000, 101, 100));
+  EXPECT_EQ(2, RateLimiterEntry::compute_delay(10000, 200, 100));
+  EXPECT_EQ(3, RateLimiterEntry::compute_delay(10000, 201, 100));
+}
+
 TEST(RGWRateLimit, op_limit_not_enabled)
 {
   // info.enabled = false, so no limit
@@ -17,12 +47,14 @@ TEST(RGWRateLimit, op_limit_not_enabled)
   RGWRateLimitInfo info;
   auto time = ceph::coarse_real_clock::now();
   std::string key = "uuser123";
-  bool success = ratelimit.should_rate_limit("PUT", key, time, &info, "");
-  EXPECT_EQ(false, success);
+  int64_t delay = ratelimit.should_rate_limit("PUT", key, time, &info, "");
+  EXPECT_EQ(0, delay);
 }
 TEST(RGWRateLimit, reject_op_over_limit)
 {
-  // check that request is being rejected because there are not enough tokens
+  // check that request is being rejected because there are not enough tokens,
+  // and that the returned delay matches the configured interval (default 60s)
+  // when the user is exactly one token short.
   std::atomic_bool replacing;
   std::condition_variable cv;
   RateLimiter ratelimit(g_ceph_context, replacing, cv);
@@ -31,10 +63,10 @@ TEST(RGWRateLimit, reject_op_over_limit)
   info.max_read_ops = 1;
   auto time = ceph::coarse_real_clock::now();
   std::string key = "uuser123";
-  bool success = ratelimit.should_rate_limit("GET", key, time, &info, "");
+  int64_t delay = ratelimit.should_rate_limit("GET", key, time, &info, "");
   time = ceph::coarse_real_clock::now();
-  success = ratelimit.should_rate_limit("GET", key, time, &info, "");
-  EXPECT_EQ(true, success);
+  delay = ratelimit.should_rate_limit("GET", key, time, &info, "");
+  EXPECT_EQ(60, delay);
 }
 TEST(RGWRateLimit, accept_op_after_giveback)
 {
@@ -47,11 +79,11 @@ TEST(RGWRateLimit, accept_op_after_giveback)
   info.max_read_ops = 1;
   auto time = ceph::coarse_real_clock::now();
   std::string key = "uuser123";
-  bool success = ratelimit.should_rate_limit("GET", key, time, &info, "");
+  int64_t delay = ratelimit.should_rate_limit("GET", key, time, &info, "");
   ratelimit.giveback_tokens("GET", key, "", &info);
   time = ceph::coarse_real_clock::now();
-  success = ratelimit.should_rate_limit("GET", key, time, &info, "");
-  EXPECT_EQ(false, success);
+  delay = ratelimit.should_rate_limit("GET", key, time, &info, "");
+  EXPECT_EQ(0, delay);
 }
 TEST(RGWRateLimit, accept_op_after_refill)
 {
@@ -64,10 +96,10 @@ TEST(RGWRateLimit, accept_op_after_refill)
   info.max_read_ops = 1;
   auto time = ceph::coarse_real_clock::now();
   std::string key = "uuser123";
-  bool success = ratelimit.should_rate_limit("GET", key, time, &info, "");
+  int64_t delay = ratelimit.should_rate_limit("GET", key, time, &info, "");
   time += 61s;
-  success = ratelimit.should_rate_limit("GET", key, time, &info, "");
-  EXPECT_EQ(false, success);
+  delay = ratelimit.should_rate_limit("GET", key, time, &info, "");
+  EXPECT_EQ(0, delay);
 }
 TEST(RGWRateLimit, reject_bw_over_limit)
 {
@@ -80,11 +112,11 @@ TEST(RGWRateLimit, reject_bw_over_limit)
   info.max_read_bytes = 1;
   auto time = ceph::coarse_real_clock::now();
   std::string key = "uuser123";
-  bool success = ratelimit.should_rate_limit("GET", key, time, &info, "");
+  int64_t delay = ratelimit.should_rate_limit("GET", key, time, &info, "");
   ratelimit.decrease_bytes("GET",key, 2, &info);
   time = ceph::coarse_real_clock::now();
-  success = ratelimit.should_rate_limit("GET", key, time, &info, "");
-  EXPECT_EQ(true, success);
+  delay = ratelimit.should_rate_limit("GET", key, time, &info, "");
+  EXPECT_GT(delay, 0);
 }
 TEST(RGWRateLimit, accept_bw)
 {
@@ -97,11 +129,11 @@ TEST(RGWRateLimit, accept_bw)
   info.max_read_bytes = 2;
   auto time = ceph::coarse_real_clock::now();
   std::string key = "uuser123";
-  bool success = ratelimit.should_rate_limit("GET", key, time, &info, "");
+  int64_t delay = ratelimit.should_rate_limit("GET", key, time, &info, "");
   ratelimit.decrease_bytes("GET",key, 1, &info);
   time = ceph::coarse_real_clock::now();
-  success = ratelimit.should_rate_limit("GET", key, time, &info, "");
-  EXPECT_EQ(false, success);
+  delay = ratelimit.should_rate_limit("GET", key, time, &info, "");
+  EXPECT_EQ(0, delay);
 }
 TEST(RGWRateLimit, check_bw_debt_at_max_120secs)
 {
@@ -114,11 +146,11 @@ TEST(RGWRateLimit, check_bw_debt_at_max_120secs)
   info.max_read_bytes = 2;
   auto time = ceph::coarse_real_clock::now();
   std::string key = "uuser123";
-  bool success = ratelimit.should_rate_limit("GET", key, time, &info, "");
+  int64_t delay = ratelimit.should_rate_limit("GET", key, time, &info, "");
   ratelimit.decrease_bytes("GET",key, 100, &info);
   time += 121s;
-  success = ratelimit.should_rate_limit("GET", key, time, &info, "");
-  EXPECT_EQ(false, success);
+  delay = ratelimit.should_rate_limit("GET", key, time, &info, "");
+  EXPECT_EQ(0, delay);
 }
 TEST(RGWRateLimit, check_that_bw_limit_not_affect_ops)
 {
@@ -132,11 +164,11 @@ TEST(RGWRateLimit, check_that_bw_limit_not_affect_ops)
   info.max_read_bytes = 100000000;
   auto time = ceph::coarse_real_clock::now();
   std::string key = "uuser123";
-  bool success = ratelimit.should_rate_limit("GET", key, time, &info, "");
+  int64_t delay = ratelimit.should_rate_limit("GET", key, time, &info, "");
   ratelimit.decrease_bytes("GET",key, 10000, &info);
   time = ceph::coarse_real_clock::now();
-  success = ratelimit.should_rate_limit("GET", key, time, &info, "");
-  EXPECT_EQ(true, success);
+  delay = ratelimit.should_rate_limit("GET", key, time, &info, "");
+  EXPECT_GT(delay, 0);
 }
 TEST(RGWRateLimit, read_limit_does_not_affect_writes)
 {
@@ -150,11 +182,11 @@ TEST(RGWRateLimit, read_limit_does_not_affect_writes)
   info.max_read_bytes = 100000000;
   auto time = ceph::coarse_real_clock::now();
   std::string key = "uuser123";
-  bool success = ratelimit.should_rate_limit("PUT", key, time, &info, "");
+  int64_t delay = ratelimit.should_rate_limit("PUT", key, time, &info, "");
   ratelimit.decrease_bytes("PUT",key, 10000, &info);
   time = ceph::coarse_real_clock::now();
-  success = ratelimit.should_rate_limit("PUT", key, time, &info, "");
-  EXPECT_EQ(false, success);
+  delay = ratelimit.should_rate_limit("PUT", key, time, &info, "");
+  EXPECT_EQ(0, delay);
 }
 TEST(RGWRateLimit, write_limit_does_not_affect_reads)
 {
@@ -168,11 +200,11 @@ TEST(RGWRateLimit, write_limit_does_not_affect_reads)
   info.max_write_bytes = 100000000;
   auto time = ceph::coarse_real_clock::now();
   std::string key = "uuser123";
-  bool success = ratelimit.should_rate_limit("GET", key, time, &info, "");
+  int64_t delay = ratelimit.should_rate_limit("GET", key, time, &info, "");
   ratelimit.decrease_bytes("GET",key, 10000, &info);
   time = ceph::coarse_real_clock::now();
-  success = ratelimit.should_rate_limit("GET", key, time, &info, "");
-  EXPECT_EQ(false, success);
+  delay = ratelimit.should_rate_limit("GET", key, time, &info, "");
+  EXPECT_EQ(0, delay);
 }
 
 TEST(RGWRateLimit, allow_unlimited_access)
@@ -185,8 +217,8 @@ TEST(RGWRateLimit, allow_unlimited_access)
   info.enabled = true;
   auto time = ceph::coarse_real_clock::now();
   std::string key = "uuser123";
-  bool success = ratelimit.should_rate_limit("GET", key, time, &info, "");
-  EXPECT_EQ(false, success);
+  int64_t delay = ratelimit.should_rate_limit("GET", key, time, &info, "");
+  EXPECT_EQ(0, delay);
 }
 
 TEST(RGWRateLimit, unlimited_access_not_left_large_read_ops_budget)
@@ -202,13 +234,13 @@ TEST(RGWRateLimit, unlimited_access_not_left_large_read_ops_budget)
 
   for (int i = 0; i < 10; i++)
   {
-    bool success = ratelimit.should_rate_limit("GET", key, time, &info, "");
-    EXPECT_EQ(false, success);
+    int64_t delay = ratelimit.should_rate_limit("GET", key, time, &info, "");
+    EXPECT_EQ(0, delay);
   }
   time += 61s;
   info.max_read_ops = 1; // make read ops limited
-  bool success = ratelimit.should_rate_limit("GET", key, time, &info, "");
-  EXPECT_EQ(false, success);
+  int64_t delay = ratelimit.should_rate_limit("GET", key, time, &info, "");
+  EXPECT_EQ(0, delay);
 }
 
 TEST(RGWRateLimit, unlimited_access_not_left_large_write_ops_budget)
@@ -224,13 +256,13 @@ TEST(RGWRateLimit, unlimited_access_not_left_large_write_ops_budget)
 
   for (int i = 0; i < 10; i++)
   {
-    bool success = ratelimit.should_rate_limit("PUT", key, time, &info, "");
-    EXPECT_EQ(false, success);
+    int64_t delay = ratelimit.should_rate_limit("PUT", key, time, &info, "");
+    EXPECT_EQ(0, delay);
   }
   time += 61s;
   info.max_write_ops = 1; // make write ops limited
-  bool success = ratelimit.should_rate_limit("PUT", key, time, &info, "");
-  EXPECT_EQ(false, success);
+  int64_t delay = ratelimit.should_rate_limit("PUT", key, time, &info, "");
+  EXPECT_EQ(0, delay);
 }
 
 TEST(RGWRateLimitGC, NO_GC_AHEAD_OF_TIME)
@@ -276,22 +308,23 @@ TEST(RGWRateLimitEntry, op_limit_not_enabled)
   RateLimiterEntry entry;
   RGWRateLimitInfo info;
   auto time = ceph::coarse_real_clock::now().time_since_epoch();
-  bool success = entry.should_rate_limit(OpType::Read, &info, time);
-  EXPECT_EQ(false, success);
+  int64_t delay = entry.should_rate_limit(OpType::Read, &info, time);
+  EXPECT_EQ(0, delay);
 }
 TEST(RGWRateLimitEntry, reject_op_over_limit)
 {
-  // check that request is being rejected because there are not enough tokens
-
+  // check that request is being rejected because there are not enough tokens,
+  // and that the returned delay matches the configured interval (default 60s)
+  // when the user is exactly one token short.
   RGWRateLimitInfo info;
   RateLimiterEntry entry;
   info.enabled = true;
   info.max_read_ops = 1;
   auto time = ceph::coarse_real_clock::now().time_since_epoch();
-  bool success = entry.should_rate_limit(OpType::Read, &info, time);
+  int64_t delay = entry.should_rate_limit(OpType::Read, &info, time);
   time = ceph::coarse_real_clock::now().time_since_epoch();
-  success = entry.should_rate_limit(OpType::Read, &info, time);
-  EXPECT_EQ(true, success);
+  delay = entry.should_rate_limit(OpType::Read, &info, time);
+  EXPECT_EQ(60, delay);
 }
 TEST(RGWRateLimitEntry, accept_op_after_giveback)
 {
@@ -301,11 +334,11 @@ TEST(RGWRateLimitEntry, accept_op_after_giveback)
   info.enabled = true;
   info.max_read_ops = 1;
   auto time = ceph::coarse_real_clock::now().time_since_epoch();
-  bool success = entry.should_rate_limit(OpType::Read,  &info, time);
+  int64_t delay = entry.should_rate_limit(OpType::Read,  &info, time);
   entry.giveback_tokens(OpType::Read);
   time = ceph::coarse_real_clock::now().time_since_epoch();
-  success = entry.should_rate_limit(OpType::Read,  &info, time);
-  EXPECT_EQ(false, success);
+  delay = entry.should_rate_limit(OpType::Read,  &info, time);
+  EXPECT_EQ(0, delay);
 }
 TEST(RGWRateLimitEntry, accept_op_after_refill)
 {
@@ -315,10 +348,10 @@ TEST(RGWRateLimitEntry, accept_op_after_refill)
   info.enabled = true;
   info.max_read_ops = 1;
   auto time = ceph::coarse_real_clock::now().time_since_epoch();
-  bool success = entry.should_rate_limit(OpType::Read,  &info, time);
+  int64_t delay = entry.should_rate_limit(OpType::Read,  &info, time);
   time += 61s;
-  success = entry.should_rate_limit(OpType::Read,  &info, time);
-  EXPECT_EQ(false, success);
+  delay = entry.should_rate_limit(OpType::Read,  &info, time);
+  EXPECT_EQ(0, delay);
 }
 TEST(RGWRateLimitEntry, reject_bw_over_limit)
 {
@@ -328,11 +361,11 @@ TEST(RGWRateLimitEntry, reject_bw_over_limit)
   info.enabled = true;
   info.max_read_bytes = 1;
   auto time = ceph::coarse_real_clock::now().time_since_epoch();
-  bool success = entry.should_rate_limit(OpType::Read,  &info, time);
+  int64_t delay = entry.should_rate_limit(OpType::Read,  &info, time);
   entry.decrease_bytes(true, 2, &info);
   time = ceph::coarse_real_clock::now().time_since_epoch();
-  success = entry.should_rate_limit(OpType::Read,  &info, time);
-  EXPECT_EQ(true, success);
+  delay = entry.should_rate_limit(OpType::Read,  &info, time);
+  EXPECT_GT(delay, 0);
 }
 TEST(RGWRateLimitEntry, accept_bw)
 {
@@ -342,11 +375,11 @@ TEST(RGWRateLimitEntry, accept_bw)
   info.enabled = true;
   info.max_read_bytes = 2;
   auto time = ceph::coarse_real_clock::now().time_since_epoch();
-  bool success = entry.should_rate_limit(OpType::Read, &info, time);
+  int64_t delay = entry.should_rate_limit(OpType::Read, &info, time);
   entry.decrease_bytes(true, 1, &info);
   time = ceph::coarse_real_clock::now().time_since_epoch();
-  success = entry.should_rate_limit(OpType::Read, &info, time);
-  EXPECT_EQ(false, success);
+  delay = entry.should_rate_limit(OpType::Read, &info, time);
+  EXPECT_EQ(0, delay);
 }
 TEST(RGWRateLimitEntry, check_bw_debt_at_max_120secs)
 {
@@ -356,11 +389,11 @@ TEST(RGWRateLimitEntry, check_bw_debt_at_max_120secs)
   info.enabled = true;
   info.max_read_bytes = 2;
   auto time = ceph::coarse_real_clock::now().time_since_epoch();
-  bool success = entry.should_rate_limit(OpType::Read, &info, time);
+  int64_t delay = entry.should_rate_limit(OpType::Read, &info, time);
   entry.decrease_bytes(true, 100, &info);
   time += 121s;
-  success = entry.should_rate_limit(OpType::Read, &info, time);
-  EXPECT_EQ(false, success);
+  delay = entry.should_rate_limit(OpType::Read, &info, time);
+  EXPECT_EQ(0, delay);
 }
 TEST(RGWRateLimitEntry, check_that_bw_limit_not_affect_ops)
 {
@@ -371,11 +404,11 @@ TEST(RGWRateLimitEntry, check_that_bw_limit_not_affect_ops)
   info.max_read_ops = 1;
   info.max_read_bytes = 100000000;
   auto time = ceph::coarse_real_clock::now().time_since_epoch();
-  bool success = entry.should_rate_limit(OpType::Read, &info, time);
+  int64_t delay = entry.should_rate_limit(OpType::Read, &info, time);
   entry.decrease_bytes(true, 10000, &info);
   time = ceph::coarse_real_clock::now().time_since_epoch();
-  success = entry.should_rate_limit(OpType::Read, &info, time);
-  EXPECT_EQ(true, success);
+  delay = entry.should_rate_limit(OpType::Read, &info, time);
+  EXPECT_GT(delay, 0);
 }
 TEST(RGWRateLimitEntry, read_limit_does_not_affect_writes)
 {
@@ -386,11 +419,11 @@ TEST(RGWRateLimitEntry, read_limit_does_not_affect_writes)
   info.max_read_ops = 1;
   info.max_read_bytes = 100000000;
   auto time = ceph::coarse_real_clock::now().time_since_epoch();
-  bool success = entry.should_rate_limit(OpType::Write, &info, time);
+  int64_t delay = entry.should_rate_limit(OpType::Write, &info, time);
   entry.decrease_bytes(false, 10000, &info);
   time = ceph::coarse_real_clock::now().time_since_epoch();
-  success = entry.should_rate_limit(OpType::Write, &info, time);
-  EXPECT_EQ(false, success);
+  delay = entry.should_rate_limit(OpType::Write, &info, time);
+  EXPECT_EQ(0, delay);
 }
 TEST(RGWRateLimitEntry, write_limit_does_not_affect_reads)
 {
@@ -402,11 +435,11 @@ TEST(RGWRateLimitEntry, write_limit_does_not_affect_reads)
   info.max_write_bytes = 100000000;
   auto time = ceph::coarse_real_clock::now().time_since_epoch();
   std::string key = "uuser123";
-  bool success = entry.should_rate_limit(OpType::Read, &info, time);
+  int64_t delay = entry.should_rate_limit(OpType::Read, &info, time);
   entry.decrease_bytes(true, 10000, &info);
   time = ceph::coarse_real_clock::now().time_since_epoch();
-  success = entry.should_rate_limit(OpType::Read, &info, time);
-  EXPECT_EQ(false, success);
+  delay = entry.should_rate_limit(OpType::Read, &info, time);
+  EXPECT_EQ(0, delay);
 }
 
 TEST(RGWRateLimitEntry, allow_unlimited_access)
@@ -416,8 +449,8 @@ TEST(RGWRateLimitEntry, allow_unlimited_access)
   RGWRateLimitInfo info;
   info.enabled = true;
   auto time = ceph::coarse_real_clock::now().time_since_epoch();
-  bool success = entry.should_rate_limit(OpType::Read, &info, time);
-  EXPECT_EQ(false, success);
+  int64_t delay = entry.should_rate_limit(OpType::Read, &info, time);
+  EXPECT_EQ(0, delay);
 }
 
 
@@ -438,10 +471,10 @@ TEST(RGWRateLimit, reject_list_op_over_limit)
   info.max_list_ops = 1;
   auto time = ceph::coarse_real_clock::now();
   std::string key = "uuser_list";
-  bool success = ratelimit.should_rate_limit("GET", key, time, &info, RES_LIST_TYPE_2);
+  int64_t delay = ratelimit.should_rate_limit("GET", key, time, &info, RES_LIST_TYPE_2);
   time = ceph::coarse_real_clock::now();
-  success = ratelimit.should_rate_limit("GET", key, time, &info, RES_LIST_TYPE_2);
-  EXPECT_EQ(true, success);
+  delay = ratelimit.should_rate_limit("GET", key, time, &info, RES_LIST_TYPE_2);
+  EXPECT_GT(delay, 0);
 }
 
 TEST(RGWRateLimit, accept_list_op_after_giveback)
@@ -455,11 +488,11 @@ TEST(RGWRateLimit, accept_list_op_after_giveback)
   info.max_list_ops = 1;
   auto time = ceph::coarse_real_clock::now();
   std::string key = "uuser_list";
-  bool success = ratelimit.should_rate_limit("GET", key, time, &info, RES_LIST_TYPE_2);
+  int64_t delay = ratelimit.should_rate_limit("GET", key, time, &info, RES_LIST_TYPE_2);
   ratelimit.giveback_tokens("GET", key, RES_LIST_TYPE_2, &info);
   time = ceph::coarse_real_clock::now();
-  success = ratelimit.should_rate_limit("GET", key, time, &info, RES_LIST_TYPE_2);
-  EXPECT_EQ(false, success);
+  delay = ratelimit.should_rate_limit("GET", key, time, &info, RES_LIST_TYPE_2);
+  EXPECT_EQ(0, delay);
 }
 
 TEST(RGWRateLimit, accept_list_op_after_refill)
@@ -473,10 +506,10 @@ TEST(RGWRateLimit, accept_list_op_after_refill)
   info.max_list_ops = 1;
   auto time = ceph::coarse_real_clock::now();
   std::string key = "uuser_list";
-  bool success = ratelimit.should_rate_limit("GET", key, time, &info, RES_LIST_TYPE_2);
+  int64_t delay = ratelimit.should_rate_limit("GET", key, time, &info, RES_LIST_TYPE_2);
   time += 61s;
-  success = ratelimit.should_rate_limit("GET", key, time, &info, RES_LIST_TYPE_2);
-  EXPECT_EQ(false, success);
+  delay = ratelimit.should_rate_limit("GET", key, time, &info, RES_LIST_TYPE_2);
+  EXPECT_EQ(0, delay);
 }
 
 TEST(RGWRateLimit, list_limit_does_not_affect_reads)
@@ -491,10 +524,10 @@ TEST(RGWRateLimit, list_limit_does_not_affect_reads)
   info.max_read_ops = 1;
   auto time = ceph::coarse_real_clock::now();
   std::string key = "uuser_list";
-  bool success = ratelimit.should_rate_limit("GET", key, time, &info, RES_LIST_TYPE_2);
+  int64_t delay = ratelimit.should_rate_limit("GET", key, time, &info, RES_LIST_TYPE_2);
   // Should still be able to do a normal GET (read)
-  success = ratelimit.should_rate_limit("GET", key, time, &info, "");
-  EXPECT_EQ(false, success);
+  delay = ratelimit.should_rate_limit("GET", key, time, &info, "");
+  EXPECT_EQ(0, delay);
 }
 
 TEST(RGWRateLimit, read_limit_does_not_affect_lists)
@@ -509,10 +542,10 @@ TEST(RGWRateLimit, read_limit_does_not_affect_lists)
   info.max_read_ops = 1;
   auto time = ceph::coarse_real_clock::now();
   std::string key = "uuser_list";
-  bool success = ratelimit.should_rate_limit("GET", key, time, &info, "");
+  int64_t delay = ratelimit.should_rate_limit("GET", key, time, &info, "");
   // Should still be able to do a LIST op
-  success = ratelimit.should_rate_limit("GET", key, time, &info, RES_LIST_TYPE_2);
-  EXPECT_EQ(false, success);
+  delay = ratelimit.should_rate_limit("GET", key, time, &info, RES_LIST_TYPE_2);
+  EXPECT_EQ(0, delay);
 }
 
 TEST(RGWRateLimit, list_limit_does_not_affect_writes)
@@ -527,10 +560,10 @@ TEST(RGWRateLimit, list_limit_does_not_affect_writes)
   info.max_write_ops = 1;
   auto time = ceph::coarse_real_clock::now();
   std::string key = "uuser_list";
-  bool success = ratelimit.should_rate_limit("GET", key, time, &info, RES_LIST_TYPE_2);
+  int64_t delay = ratelimit.should_rate_limit("GET", key, time, &info, RES_LIST_TYPE_2);
   // Should still be able to do a PUT (write)
-  success = ratelimit.should_rate_limit("PUT", key, time, &info, "");
-  EXPECT_EQ(false, success);
+  delay = ratelimit.should_rate_limit("PUT", key, time, &info, "");
+  EXPECT_EQ(0, delay);
 }
 
 TEST(RGWRateLimit, unlimited_access_not_left_large_list_ops_budget)
@@ -546,13 +579,13 @@ TEST(RGWRateLimit, unlimited_access_not_left_large_list_ops_budget)
 
   for (int i = 0; i < 10; i++)
   {
-    bool success = ratelimit.should_rate_limit("GET", key, time, &info, RES_LIST_TYPE_2);
-    EXPECT_EQ(false, success);
+    int64_t delay = ratelimit.should_rate_limit("GET", key, time, &info, RES_LIST_TYPE_2);
+    EXPECT_EQ(0, delay);
   }
   time += 61s;
   info.max_list_ops = 1; // make list ops limited
-  bool success = ratelimit.should_rate_limit("GET", key, time, &info, RES_LIST_TYPE_2);
-  EXPECT_EQ(false, success);
+  int64_t delay = ratelimit.should_rate_limit("GET", key, time, &info, RES_LIST_TYPE_2);
+  EXPECT_EQ(0, delay);
 }
 
 TEST(RGWRateLimit, write_limit_does_not_affect_lists)
@@ -567,10 +600,10 @@ TEST(RGWRateLimit, write_limit_does_not_affect_lists)
   info.max_write_ops = 1;
   auto time = ceph::coarse_real_clock::now();
   std::string key = "uuser_list";
-  bool success = ratelimit.should_rate_limit("PUT", key, time, &info, "");
+  int64_t delay = ratelimit.should_rate_limit("PUT", key, time, &info, "");
   // Should still be able to do a LIST op
-  success = ratelimit.should_rate_limit("GET", key, time, &info, RES_LIST_TYPE_2);
-  EXPECT_EQ(false, success);
+  delay = ratelimit.should_rate_limit("GET", key, time, &info, RES_LIST_TYPE_2);
+  EXPECT_EQ(0, delay);
 }
 
 TEST(RGWRateLimit, list_limit_does_not_affect_deletes)
@@ -585,10 +618,10 @@ TEST(RGWRateLimit, list_limit_does_not_affect_deletes)
   info.max_delete_ops = 1;
   auto time = ceph::coarse_real_clock::now();
   std::string key = "uuser_list";
-  bool success = ratelimit.should_rate_limit("GET", key, time, &info, RES_LIST_TYPE_2);
+  int64_t delay = ratelimit.should_rate_limit("GET", key, time, &info, RES_LIST_TYPE_2);
   // Should still be able to do a DELETE op
-  success = ratelimit.should_rate_limit("DELETE", key, time, &info, "");
-  EXPECT_EQ(false, success);
+  delay = ratelimit.should_rate_limit("DELETE", key, time, &info, "");
+  EXPECT_EQ(0, delay);
 }
 
 TEST(RGWRateLimit, delete_limit_does_not_affect_lists)
@@ -603,10 +636,10 @@ TEST(RGWRateLimit, delete_limit_does_not_affect_lists)
   info.max_delete_ops = 1;
   auto time = ceph::coarse_real_clock::now();
   std::string key = "uuser_list";
-  bool success = ratelimit.should_rate_limit("DELETE", key, time, &info, "");
+  int64_t delay = ratelimit.should_rate_limit("DELETE", key, time, &info, "");
   // Should still be able to do a LIST op
-  success = ratelimit.should_rate_limit("GET", key, time, &info, RES_LIST_TYPE_2);
-  EXPECT_EQ(false, success);
+  delay = ratelimit.should_rate_limit("GET", key, time, &info, RES_LIST_TYPE_2);
+  EXPECT_EQ(0, delay);
 }
 
 // LIST RES_DELIMITER minimal tests
@@ -621,10 +654,10 @@ TEST(RGWRateLimit, reject_delimiter_op_over_limit)
   info.max_list_ops = 1;
   auto time = ceph::coarse_real_clock::now();
   std::string key = "uuser_list";
-  bool success = ratelimit.should_rate_limit("GET", key, time, &info, RES_DELIMITER);
+  int64_t delay = ratelimit.should_rate_limit("GET", key, time, &info, RES_DELIMITER);
   time = ceph::coarse_real_clock::now();
-  success = ratelimit.should_rate_limit("GET", key, time, &info, RES_DELIMITER);
-  EXPECT_EQ(true, success);
+  delay = ratelimit.should_rate_limit("GET", key, time, &info, RES_DELIMITER);
+  EXPECT_GT(delay, 0);
 }
 
 TEST(RGWRateLimit, accept_delimiter_op_after_giveback)
@@ -638,11 +671,11 @@ TEST(RGWRateLimit, accept_delimiter_op_after_giveback)
   info.max_list_ops = 1;
   auto time = ceph::coarse_real_clock::now();
   std::string key = "uuser_list";
-  bool success = ratelimit.should_rate_limit("GET", key, time, &info, RES_DELIMITER);
+  int64_t delay = ratelimit.should_rate_limit("GET", key, time, &info, RES_DELIMITER);
   ratelimit.giveback_tokens("GET", key, RES_DELIMITER, &info);
   time = ceph::coarse_real_clock::now();
-  success = ratelimit.should_rate_limit("GET", key, time, &info, RES_DELIMITER);
-  EXPECT_EQ(false, success);
+  delay = ratelimit.should_rate_limit("GET", key, time, &info, RES_DELIMITER);
+  EXPECT_EQ(0, delay);
 }
 
 TEST(RGWRateLimit, accept_delimiter_op_after_refill)
@@ -656,10 +689,10 @@ TEST(RGWRateLimit, accept_delimiter_op_after_refill)
   info.max_list_ops = 1;
   auto time = ceph::coarse_real_clock::now();
   std::string key = "uuser_list";
-  bool success = ratelimit.should_rate_limit("GET", key, time, &info, RES_DELIMITER);
+  int64_t delay = ratelimit.should_rate_limit("GET", key, time, &info, RES_DELIMITER);
   time += 61s;
-  success = ratelimit.should_rate_limit("GET", key, time, &info, RES_DELIMITER);
-  EXPECT_EQ(false, success);
+  delay = ratelimit.should_rate_limit("GET", key, time, &info, RES_DELIMITER);
+  EXPECT_EQ(0, delay);
 }
 
 // LIST RES_PREFIX minimal tests
@@ -674,10 +707,10 @@ TEST(RGWRateLimit, reject_prefix_op_over_limit)
   info.max_list_ops = 1;
   auto time = ceph::coarse_real_clock::now();
   std::string key = "uuser_list";
-  bool success = ratelimit.should_rate_limit("GET", key, time, &info, RES_PREFIX);
+  int64_t delay = ratelimit.should_rate_limit("GET", key, time, &info, RES_PREFIX);
   time = ceph::coarse_real_clock::now();
-  success = ratelimit.should_rate_limit("GET", key, time, &info, RES_PREFIX);
-  EXPECT_EQ(true, success);
+  delay = ratelimit.should_rate_limit("GET", key, time, &info, RES_PREFIX);
+  EXPECT_GT(delay, 0);
 }
 
 TEST(RGWRateLimit, accept_prefix_op_after_giveback)
@@ -691,11 +724,11 @@ TEST(RGWRateLimit, accept_prefix_op_after_giveback)
   info.max_list_ops = 1;
   auto time = ceph::coarse_real_clock::now();
   std::string key = "uuser_list";
-  bool success = ratelimit.should_rate_limit("GET", key, time, &info, RES_PREFIX);
+  int64_t delay = ratelimit.should_rate_limit("GET", key, time, &info, RES_PREFIX);
   ratelimit.giveback_tokens("GET", key, RES_PREFIX, &info);
   time = ceph::coarse_real_clock::now();
-  success = ratelimit.should_rate_limit("GET", key, time, &info, RES_PREFIX);
-  EXPECT_EQ(false, success);
+  delay = ratelimit.should_rate_limit("GET", key, time, &info, RES_PREFIX);
+  EXPECT_EQ(0, delay);
 }
 
 TEST(RGWRateLimit, accept_prefix_op_after_refill)
@@ -709,10 +742,10 @@ TEST(RGWRateLimit, accept_prefix_op_after_refill)
   info.max_list_ops = 1;
   auto time = ceph::coarse_real_clock::now();
   std::string key = "uuser_list";
-  bool success = ratelimit.should_rate_limit("GET", key, time, &info, RES_PREFIX);
+  int64_t delay = ratelimit.should_rate_limit("GET", key, time, &info, RES_PREFIX);
   time += 61s;
-  success = ratelimit.should_rate_limit("GET", key, time, &info, RES_PREFIX);
-  EXPECT_EQ(false, success);
+  delay = ratelimit.should_rate_limit("GET", key, time, &info, RES_PREFIX);
+  EXPECT_EQ(0, delay);
 }
 
 
@@ -724,10 +757,10 @@ TEST(RGWRateLimitEntry, reject_list_op_over_limit)
   info.enabled = true;
   info.max_list_ops = 1;
   auto time = ceph::coarse_real_clock::now().time_since_epoch();
-  bool success = entry.should_rate_limit(OpType::List, &info, time);
+  int64_t delay = entry.should_rate_limit(OpType::List, &info, time);
   time = ceph::coarse_real_clock::now().time_since_epoch();
-  success = entry.should_rate_limit(OpType::List, &info, time);
-  EXPECT_EQ(true, success);
+  delay = entry.should_rate_limit(OpType::List, &info, time);
+  EXPECT_GT(delay, 0);
 }
 
 TEST(RGWRateLimitEntry, accept_list_op_after_giveback)
@@ -738,11 +771,11 @@ TEST(RGWRateLimitEntry, accept_list_op_after_giveback)
   info.enabled = true;
   info.max_list_ops = 1;
   auto time = ceph::coarse_real_clock::now().time_since_epoch();
-  bool success = entry.should_rate_limit(OpType::List, &info, time);
+  int64_t delay = entry.should_rate_limit(OpType::List, &info, time);
   entry.giveback_tokens(OpType::List);
   time = ceph::coarse_real_clock::now().time_since_epoch();
-  success = entry.should_rate_limit(OpType::List, &info, time);
-  EXPECT_EQ(false, success);
+  delay = entry.should_rate_limit(OpType::List, &info, time);
+  EXPECT_EQ(0, delay);
 }
 
 TEST(RGWRateLimitEntry, accept_list_op_after_refill)
@@ -753,10 +786,10 @@ TEST(RGWRateLimitEntry, accept_list_op_after_refill)
   info.enabled = true;
   info.max_list_ops = 1;
   auto time = ceph::coarse_real_clock::now().time_since_epoch();
-  bool success = entry.should_rate_limit(OpType::List, &info, time);
+  int64_t delay = entry.should_rate_limit(OpType::List, &info, time);
   time += 61s;
-  success = entry.should_rate_limit(OpType::List, &info, time);
-  EXPECT_EQ(false, success);
+  delay = entry.should_rate_limit(OpType::List, &info, time);
+  EXPECT_EQ(0, delay);
 }
 
 TEST(RGWRateLimitEntry, list_limit_does_not_affect_reads)
@@ -768,10 +801,10 @@ TEST(RGWRateLimitEntry, list_limit_does_not_affect_reads)
   info.max_list_ops = 1;
   info.max_read_ops = 1;
   auto time = ceph::coarse_real_clock::now().time_since_epoch();
-  bool success = entry.should_rate_limit(OpType::List, &info, time);
+  int64_t delay = entry.should_rate_limit(OpType::List, &info, time);
   time = ceph::coarse_real_clock::now().time_since_epoch();
-  success = entry.should_rate_limit(OpType::Read, &info, time);
-  EXPECT_EQ(false, success);
+  delay = entry.should_rate_limit(OpType::Read, &info, time);
+  EXPECT_EQ(0, delay);
 }
 
 TEST(RGWRateLimitEntry, read_limit_does_not_affect_lists)
@@ -783,10 +816,10 @@ TEST(RGWRateLimitEntry, read_limit_does_not_affect_lists)
   info.max_list_ops = 1;
   info.max_read_ops = 1;
   auto time = ceph::coarse_real_clock::now().time_since_epoch();
-  bool success = entry.should_rate_limit(OpType::Read, &info, time);
+  int64_t delay = entry.should_rate_limit(OpType::Read, &info, time);
   time = ceph::coarse_real_clock::now().time_since_epoch();
-  success = entry.should_rate_limit(OpType::List, &info, time);
-  EXPECT_EQ(false, success);
+  delay = entry.should_rate_limit(OpType::List, &info, time);
+  EXPECT_EQ(0, delay);
 }
 
 TEST(RGWRateLimitEntry, list_limit_does_not_affect_writes)
@@ -798,10 +831,10 @@ TEST(RGWRateLimitEntry, list_limit_does_not_affect_writes)
   info.max_list_ops = 1;
   info.max_write_ops = 1;
   auto time = ceph::coarse_real_clock::now().time_since_epoch();
-  bool success = entry.should_rate_limit(OpType::List, &info, time);
+  int64_t delay = entry.should_rate_limit(OpType::List, &info, time);
   time = ceph::coarse_real_clock::now().time_since_epoch();
-  success = entry.should_rate_limit(OpType::Write, &info, time);
-  EXPECT_EQ(false, success);
+  delay = entry.should_rate_limit(OpType::Write, &info, time);
+  EXPECT_EQ(0, delay);
 }
 
 TEST(RGWRateLimitEntry, write_limit_does_not_affect_lists)
@@ -813,10 +846,10 @@ TEST(RGWRateLimitEntry, write_limit_does_not_affect_lists)
   info.max_list_ops = 1;
   info.max_write_ops = 1;
   auto time = ceph::coarse_real_clock::now().time_since_epoch();
-  bool success = entry.should_rate_limit(OpType::Write, &info, time);
+  int64_t delay = entry.should_rate_limit(OpType::Write, &info, time);
   time = ceph::coarse_real_clock::now().time_since_epoch();
-  success = entry.should_rate_limit(OpType::List, &info, time);
-  EXPECT_EQ(false, success);
+  delay = entry.should_rate_limit(OpType::List, &info, time);
+  EXPECT_EQ(0, delay);
 }
 
 TEST(RGWRateLimitEntry, list_limit_does_not_affect_deletes)
@@ -828,10 +861,10 @@ TEST(RGWRateLimitEntry, list_limit_does_not_affect_deletes)
   info.max_list_ops = 1;
   info.max_delete_ops = 1;
   auto time = ceph::coarse_real_clock::now().time_since_epoch();
-  bool success = entry.should_rate_limit(OpType::List, &info, time);
+  int64_t delay = entry.should_rate_limit(OpType::List, &info, time);
   time = ceph::coarse_real_clock::now().time_since_epoch();
-  success = entry.should_rate_limit(OpType::Delete, &info, time);
-  EXPECT_EQ(false, success);
+  delay = entry.should_rate_limit(OpType::Delete, &info, time);
+  EXPECT_EQ(0, delay);
 }
 
 TEST(RGWRateLimitEntry, delete_limit_does_not_affect_lists)
@@ -843,10 +876,10 @@ TEST(RGWRateLimitEntry, delete_limit_does_not_affect_lists)
   info.max_list_ops = 1;
   info.max_delete_ops = 1;
   auto time = ceph::coarse_real_clock::now().time_since_epoch();
-  bool success = entry.should_rate_limit(OpType::Delete, &info, time);
+  int64_t delay = entry.should_rate_limit(OpType::Delete, &info, time);
   time = ceph::coarse_real_clock::now().time_since_epoch();
-  success = entry.should_rate_limit(OpType::List, &info, time);
-  EXPECT_EQ(false, success);
+  delay = entry.should_rate_limit(OpType::List, &info, time);
+  EXPECT_EQ(0, delay);
 }
 
 
@@ -861,10 +894,10 @@ TEST(RGWRateLimit, reject_delete_op_over_limit)
   info.max_delete_ops = 1;
   auto time = ceph::coarse_real_clock::now();
   std::string key = "uuser_delete";
-  bool success = ratelimit.should_rate_limit("DELETE", key, time, &info, "");
+  int64_t delay = ratelimit.should_rate_limit("DELETE", key, time, &info, "");
   time = ceph::coarse_real_clock::now();
-  success = ratelimit.should_rate_limit("DELETE", key, time, &info, "");
-  EXPECT_EQ(true, success);
+  delay = ratelimit.should_rate_limit("DELETE", key, time, &info, "");
+  EXPECT_GT(delay, 0);
 }
 
 TEST(RGWRateLimit, accept_delete_op_after_giveback)
@@ -878,11 +911,11 @@ TEST(RGWRateLimit, accept_delete_op_after_giveback)
   info.max_delete_ops = 1;
   auto time = ceph::coarse_real_clock::now();
   std::string key = "uuser_delete";
-  bool success = ratelimit.should_rate_limit("DELETE", key, time, &info, "");
+  int64_t delay = ratelimit.should_rate_limit("DELETE", key, time, &info, "");
   ratelimit.giveback_tokens("DELETE", key, "", &info);
   time = ceph::coarse_real_clock::now();
-  success = ratelimit.should_rate_limit("DELETE", key, time, &info, "");
-  EXPECT_EQ(false, success);
+  delay = ratelimit.should_rate_limit("DELETE", key, time, &info, "");
+  EXPECT_EQ(0, delay);
 }
 
 TEST(RGWRateLimit, accept_delete_op_after_refill)
@@ -896,10 +929,10 @@ TEST(RGWRateLimit, accept_delete_op_after_refill)
   info.max_delete_ops = 1;
   auto time = ceph::coarse_real_clock::now();
   std::string key = "uuser_delete";
-  bool success = ratelimit.should_rate_limit("DELETE", key, time, &info, "");
+  int64_t delay = ratelimit.should_rate_limit("DELETE", key, time, &info, "");
   time += 61s;
-  success = ratelimit.should_rate_limit("DELETE", key, time, &info, "");
-  EXPECT_EQ(false, success);
+  delay = ratelimit.should_rate_limit("DELETE", key, time, &info, "");
+  EXPECT_EQ(0, delay);
 }
 
 TEST(RGWRateLimit, delete_limit_does_not_affect_reads)
@@ -914,10 +947,10 @@ TEST(RGWRateLimit, delete_limit_does_not_affect_reads)
   info.max_read_ops = 1;
   auto time = ceph::coarse_real_clock::now();
   std::string key = "uuser_delete";
-  bool success = ratelimit.should_rate_limit("DELETE", key, time, &info, "");
+  int64_t delay = ratelimit.should_rate_limit("DELETE", key, time, &info, "");
   // Should still be able to do a normal GET (read)
-  success = ratelimit.should_rate_limit("GET", key, time, &info, "");
-  EXPECT_EQ(false, success);
+  delay = ratelimit.should_rate_limit("GET", key, time, &info, "");
+  EXPECT_EQ(0, delay);
 }
 
 TEST(RGWRateLimit, read_limit_does_not_affect_deletes)
@@ -932,10 +965,10 @@ TEST(RGWRateLimit, read_limit_does_not_affect_deletes)
   info.max_read_ops = 1;
   auto time = ceph::coarse_real_clock::now();
   std::string key = "uuser_delete";
-  bool success = ratelimit.should_rate_limit("GET", key, time, &info, "");
+  int64_t delay = ratelimit.should_rate_limit("GET", key, time, &info, "");
   // Should still be able to do a DELETE op
-  success = ratelimit.should_rate_limit("DELETE", key, time, &info, "");
-  EXPECT_EQ(false, success);
+  delay = ratelimit.should_rate_limit("DELETE", key, time, &info, "");
+  EXPECT_EQ(0, delay);
 }
 
 TEST(RGWRateLimit, write_limit_does_not_affect_deletes)
@@ -950,10 +983,10 @@ TEST(RGWRateLimit, write_limit_does_not_affect_deletes)
   info.max_write_ops = 1;
   auto time = ceph::coarse_real_clock::now();
   std::string key = "uuser_delete";
-  bool success = ratelimit.should_rate_limit("PUT", key, time, &info, "");
+  int64_t delay = ratelimit.should_rate_limit("PUT", key, time, &info, "");
   // Should still be able to do a DELETE op
-  success = ratelimit.should_rate_limit("DELETE", key, time, &info, "");
-  EXPECT_EQ(false, success);
+  delay = ratelimit.should_rate_limit("DELETE", key, time, &info, "");
+  EXPECT_EQ(0, delay);
 }
 
 TEST(RGWRateLimit, delete_limit_does_not_affect_writes)
@@ -968,10 +1001,10 @@ TEST(RGWRateLimit, delete_limit_does_not_affect_writes)
   info.max_write_ops = 1;
   auto time = ceph::coarse_real_clock::now();
   std::string key = "uuser_delete";
-  bool success = ratelimit.should_rate_limit("DELETE", key, time, &info, "");
+  int64_t delay = ratelimit.should_rate_limit("DELETE", key, time, &info, "");
   // Should still be able to do a PUT (write)
-  success = ratelimit.should_rate_limit("PUT", key, time, &info, "");
-  EXPECT_EQ(false, success);
+  delay = ratelimit.should_rate_limit("PUT", key, time, &info, "");
+  EXPECT_EQ(0, delay);
 }
 
 TEST(RGWRateLimit, unlimited_access_not_left_large_delete_ops_budget)
@@ -987,13 +1020,13 @@ TEST(RGWRateLimit, unlimited_access_not_left_large_delete_ops_budget)
 
   for (int i = 0; i < 10; i++)
   {
-    bool success = ratelimit.should_rate_limit("DELETE", key, time, &info, "");
-    EXPECT_EQ(false, success);
+    int64_t delay = ratelimit.should_rate_limit("DELETE", key, time, &info, "");
+    EXPECT_EQ(0, delay);
   }
   time += 61s;
   info.max_delete_ops = 1; // make delete ops limited
-  bool success = ratelimit.should_rate_limit("DELETE", key, time, &info, "");
-  EXPECT_EQ(false, success);
+  int64_t delay = ratelimit.should_rate_limit("DELETE", key, time, &info, "");
+  EXPECT_EQ(0, delay);
 }
 
 TEST(RGWRateLimitEntry, reject_delete_op_over_limit)
@@ -1004,10 +1037,10 @@ TEST(RGWRateLimitEntry, reject_delete_op_over_limit)
   info.enabled = true;
   info.max_delete_ops = 1;
   auto time = ceph::coarse_real_clock::now().time_since_epoch();
-  bool success = entry.should_rate_limit(OpType::Delete, &info, time);
+  int64_t delay = entry.should_rate_limit(OpType::Delete, &info, time);
   time = ceph::coarse_real_clock::now().time_since_epoch();
-  success = entry.should_rate_limit(OpType::Delete, &info, time);
-  EXPECT_EQ(true, success);
+  delay = entry.should_rate_limit(OpType::Delete, &info, time);
+  EXPECT_GT(delay, 0);
 }
 
 TEST(RGWRateLimitEntry, accept_delete_op_after_giveback)
@@ -1018,11 +1051,11 @@ TEST(RGWRateLimitEntry, accept_delete_op_after_giveback)
   info.enabled = true;
   info.max_delete_ops = 1;
   auto time = ceph::coarse_real_clock::now().time_since_epoch();
-  bool success = entry.should_rate_limit(OpType::Delete, &info, time);
+  int64_t delay = entry.should_rate_limit(OpType::Delete, &info, time);
   entry.giveback_tokens(OpType::Delete);
   time = ceph::coarse_real_clock::now().time_since_epoch();
-  success = entry.should_rate_limit(OpType::Delete, &info, time);
-  EXPECT_EQ(false, success);
+  delay = entry.should_rate_limit(OpType::Delete, &info, time);
+  EXPECT_EQ(0, delay);
 }
 
 TEST(RGWRateLimitEntry, accept_delete_op_after_refill)
@@ -1033,10 +1066,10 @@ TEST(RGWRateLimitEntry, accept_delete_op_after_refill)
   info.enabled = true;
   info.max_delete_ops = 1;
   auto time = ceph::coarse_real_clock::now().time_since_epoch();
-  bool success = entry.should_rate_limit(OpType::Delete, &info, time);
+  int64_t delay = entry.should_rate_limit(OpType::Delete, &info, time);
   time += 61s;
-  success = entry.should_rate_limit(OpType::Delete, &info, time);
-  EXPECT_EQ(false, success);
+  delay = entry.should_rate_limit(OpType::Delete, &info, time);
+  EXPECT_EQ(0, delay);
 }
 
 TEST(RGWRateLimitEntry, delete_limit_does_not_affect_reads)
@@ -1048,10 +1081,10 @@ TEST(RGWRateLimitEntry, delete_limit_does_not_affect_reads)
   info.max_delete_ops = 1;
   info.max_read_ops = 1;
   auto time = ceph::coarse_real_clock::now().time_since_epoch();
-  bool success = entry.should_rate_limit(OpType::Delete, &info, time);
+  int64_t delay = entry.should_rate_limit(OpType::Delete, &info, time);
   time = ceph::coarse_real_clock::now().time_since_epoch();
-  success = entry.should_rate_limit(OpType::Read, &info, time);
-  EXPECT_EQ(false, success);
+  delay = entry.should_rate_limit(OpType::Read, &info, time);
+  EXPECT_EQ(0, delay);
 }
 
 TEST(RGWRateLimitEntry, read_limit_does_not_affect_deletes)
@@ -1063,10 +1096,10 @@ TEST(RGWRateLimitEntry, read_limit_does_not_affect_deletes)
   info.max_delete_ops = 1;
   info.max_read_ops = 1;
   auto time = ceph::coarse_real_clock::now().time_since_epoch();
-  bool success = entry.should_rate_limit(OpType::Read, &info, time);
+  int64_t delay = entry.should_rate_limit(OpType::Read, &info, time);
   time = ceph::coarse_real_clock::now().time_since_epoch();
-  success = entry.should_rate_limit(OpType::Delete, &info, time);
-  EXPECT_EQ(false, success);
+  delay = entry.should_rate_limit(OpType::Delete, &info, time);
+  EXPECT_EQ(0, delay);
 }
 
 TEST(RGWRateLimitEntry, write_limit_does_not_affect_deletes)
@@ -1078,10 +1111,10 @@ TEST(RGWRateLimitEntry, write_limit_does_not_affect_deletes)
   info.max_delete_ops = 1;
   info.max_write_ops = 1;
   auto time = ceph::coarse_real_clock::now().time_since_epoch();
-  bool success = entry.should_rate_limit(OpType::Write, &info, time);
+  int64_t delay = entry.should_rate_limit(OpType::Write, &info, time);
   time = ceph::coarse_real_clock::now().time_since_epoch();
-  success = entry.should_rate_limit(OpType::Delete, &info, time);
-  EXPECT_EQ(false, success);
+  delay = entry.should_rate_limit(OpType::Delete, &info, time);
+  EXPECT_EQ(0, delay);
 }
 
 TEST(RGWRateLimitEntry, delete_limit_does_not_affect_writes)
@@ -1093,9 +1126,9 @@ TEST(RGWRateLimitEntry, delete_limit_does_not_affect_writes)
   info.max_delete_ops = 1;
   info.max_write_ops = 1;
   auto time = ceph::coarse_real_clock::now().time_since_epoch();
-  bool success = entry.should_rate_limit(OpType::Delete, &info, time);
+  int64_t delay = entry.should_rate_limit(OpType::Delete, &info, time);
   time = ceph::coarse_real_clock::now().time_since_epoch();
-  success = entry.should_rate_limit(OpType::Write, &info, time);
-  EXPECT_EQ(false, success);
+  delay = entry.should_rate_limit(OpType::Write, &info, time);
+  EXPECT_EQ(0, delay);
 }