librbd: support burst limit configuration.

Signed-off-by: Shiyang Ruan <ruansy.fnst@cn.fujitsu.com>

diff --git a/qa/suites/rbd/thrash/workloads/rbd_fsx_rate_limit.yaml b/qa/suites/rbd/thrash/workloads/rbd_fsx_rate_limit.yaml
index 6e07e591f59a9a4e4c73441db921fd745b91aa00..6d215c62b4b34c853ad8e09776d0681e80de5884 100644 (file)
--- a/qa/suites/rbd/thrash/workloads/rbd_fsx_rate_limit.yaml
+++ b/qa/suites/rbd/thrash/workloads/rbd_fsx_rate_limit.yaml
@@ -7,3 +7,4 @@ overrides:
     conf:
       client:
         rbd qos iops limit: 50
+        rbd qos iops burst: 100

diff --git a/src/common/Throttle.cc b/src/common/Throttle.cc
index 8349347cec8cbd915609ec0fc8279764d79ad549..015ee09c357e0350106c9187ec01a5d5b33bc965 100644 (file)
--- a/src/common/Throttle.cc
+++ b/src/common/Throttle.cc
@@ -721,34 +721,25 @@ TokenBucketThrottle::~TokenBucketThrottle() {
   }
 }
 
-int TokenBucketThrottle::set_burst(uint64_t burst){
-  std::lock_guard<Mutex> lock(m_lock);
-  if (0 < burst && burst < m_avg) {
-    // the burst should never less than the average.
-    return -EINVAL;
-  } else {
-    m_burst = burst;
-  }
-  // for the default configuration of burst.
-  m_throttle.set_max(0 == m_burst ? m_avg : m_burst);
-  return 0;
-}
-
-int TokenBucketThrottle::set_average(uint64_t avg) {
+int TokenBucketThrottle::set_limit(uint64_t average, uint64_t burst) {
   {
     std::lock_guard<Mutex> lock(m_lock);
-    m_avg = avg;
 
-    if (0 < m_burst && m_burst < avg) {
+    if (0 < burst && burst < average) {
       // the burst should never less than the average.
       return -EINVAL;
-    } else if (0 == avg) {
+    }
+
+    m_avg = average;
+    m_burst = burst;
+
+    if (0 == average) {
       // The limit is not set, and no tokens will be put into the bucket.
       // So, we can schedule the timer slowly, or even cancel it.
       m_tick = 1000;
     } else {
       // calculate the tick(ms), don't less than the minimum.
-      m_tick = 1000 / avg;
+      m_tick = 1000 / average;
       if (m_tick < m_tick_min) {
         m_tick = m_tick_min;
       }
@@ -758,9 +749,7 @@ int TokenBucketThrottle::set_average(uint64_t avg) {
       m_current_tick = 0;
 
       // for the default configuration of burst.
-      if (0 == m_burst) {
-        m_throttle.set_max(m_avg);
-      }
+      m_throttle.set_max(0 == burst ? average : burst);
     }
     // turn millisecond to second
     m_schedule_tick = m_tick / 1000.0;

diff --git a/src/common/Throttle.h b/src/common/Throttle.h
index 96d55d13c2eee64621d33d6fe5d9b4c28f9dc9b3..d66051c1b43c012556c6cabceedbe41c77d4ae18 100644 (file)
--- a/src/common/Throttle.h
+++ b/src/common/Throttle.h
@@ -445,8 +445,7 @@ public:
     return wait;
   }
   
-  int set_burst(uint64_t burst);
-  int set_average(uint64_t avg);
+  int set_limit(uint64_t average, uint64_t burst);
 
 private:
   uint64_t tokens_filled(double tick);

diff --git a/src/common/options.cc b/src/common/options.cc
index ffed4884eff31baeebb2d85a8150b548a4df7811..7f75b07b71987b5b4a376222382275dbbf8b48eb 100644 (file)
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -6676,6 +6676,30 @@ static std::vector<Option> get_rbd_options() {
     .set_default(0)
     .set_description("the desired limit of write bytes per second"),
 
+    Option("rbd_qos_iops_burst", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
+    .set_default(0)
+    .set_description("the desired burst limit of IO operations"),
+
+    Option("rbd_qos_bps_burst", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
+    .set_default(0)
+    .set_description("the desired burst limit of IO bytes"),
+
+    Option("rbd_qos_read_iops_burst", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
+    .set_default(0)
+    .set_description("the desired burst limit of read operations"),
+
+    Option("rbd_qos_write_iops_burst", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
+    .set_default(0)
+    .set_description("the desired burst limit of write operations"),
+
+    Option("rbd_qos_read_bps_burst", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
+    .set_default(0)
+    .set_description("the desired burst limit of read bytes"),
+
+    Option("rbd_qos_write_bps_burst", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
+    .set_default(0)
+    .set_description("the desired burst limit of write bytes"),
+
     Option("rbd_discard_on_zeroed_write_same", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
     .set_default(true)
     .set_description("discard data on zeroed write same instead of writing zero"),

diff --git a/src/librbd/ImageCtx.cc b/src/librbd/ImageCtx.cc
index ec9dc2d0d9913cd1f63ab21ff268f4a22b3c8bd5..341f79fab61f85f0d783a4e191caab0066238246 100644 (file)
--- a/src/librbd/ImageCtx.cc
+++ b/src/librbd/ImageCtx.cc
@@ -810,23 +810,29 @@ public:
     }
 
     io_work_queue->apply_qos_limit(
+      RBD_QOS_IOPS_THROTTLE,
       config.get_val<uint64_t>("rbd_qos_iops_limit"),
-      RBD_QOS_IOPS_THROTTLE);
+      config.get_val<uint64_t>("rbd_qos_iops_burst"));
     io_work_queue->apply_qos_limit(
+      RBD_QOS_BPS_THROTTLE,
       config.get_val<uint64_t>("rbd_qos_bps_limit"),
-      RBD_QOS_BPS_THROTTLE);
+      config.get_val<uint64_t>("rbd_qos_bps_burst"));
     io_work_queue->apply_qos_limit(
+      RBD_QOS_READ_IOPS_THROTTLE,
       config.get_val<uint64_t>("rbd_qos_read_iops_limit"),
-      RBD_QOS_READ_IOPS_THROTTLE);
+      config.get_val<uint64_t>("rbd_qos_read_iops_burst"));
     io_work_queue->apply_qos_limit(
+      RBD_QOS_WRITE_IOPS_THROTTLE,
       config.get_val<uint64_t>("rbd_qos_write_iops_limit"),
-      RBD_QOS_WRITE_IOPS_THROTTLE);
+      config.get_val<uint64_t>("rbd_qos_write_iops_burst"));
     io_work_queue->apply_qos_limit(
+      RBD_QOS_READ_BPS_THROTTLE,
       config.get_val<uint64_t>("rbd_qos_read_bps_limit"),
-      RBD_QOS_READ_BPS_THROTTLE);
+      config.get_val<uint64_t>("rbd_qos_read_bps_burst"));
     io_work_queue->apply_qos_limit(
+      RBD_QOS_WRITE_BPS_THROTTLE,
       config.get_val<uint64_t>("rbd_qos_write_bps_limit"),
-      RBD_QOS_WRITE_BPS_THROTTLE);
+      config.get_val<uint64_t>("rbd_qos_write_bps_burst"));
   }
 
   ExclusiveLock<ImageCtx> *ImageCtx::create_exclusive_lock() {

diff --git a/src/librbd/io/ImageRequestWQ.cc b/src/librbd/io/ImageRequestWQ.cc
index deeef4531cb57191320adb2efeacea702917ad26..62f7e779a3a6119eb92e41aa611910db3aead4a4 100644 (file)
--- a/src/librbd/io/ImageRequestWQ.cc
+++ b/src/librbd/io/ImageRequestWQ.cc
@@ -615,7 +615,10 @@ void ImageRequestWQ<I>::set_require_lock(Direction direction, bool enabled) {
 }
 
 template <typename I>
-void ImageRequestWQ<I>::apply_qos_limit(uint64_t limit, const uint64_t flag) {
+void ImageRequestWQ<I>::apply_qos_limit(const uint64_t flag,
+                                        uint64_t limit,
+                                        uint64_t burst) {
+  CephContext *cct = m_image_ctx.cct;
   TokenBucketThrottle *throttle = nullptr;
   for (auto pair : m_throttles) {
     if (flag == pair.first) {
@@ -624,8 +627,16 @@ void ImageRequestWQ<I>::apply_qos_limit(uint64_t limit, const uint64_t flag) {
     }
   }
   ceph_assert(throttle != nullptr);
-  throttle->set_burst(limit);
-  throttle->set_average(limit);
+
+  int r = throttle->set_limit(limit, burst);
+  if (r < 0) {
+    lderr(cct) << "invalid qos parameter: "
+               << "burst(" << burst << ") is less than "
+               << "limit(" << limit << ")" << dendl;
+    // if apply failed, we should at least make sure the limit works.
+    throttle->set_limit(limit, 0);
+  }
+
   if (limit)
     m_qos_enabled_flag |= flag;
   else

diff --git a/src/librbd/io/ImageRequestWQ.h b/src/librbd/io/ImageRequestWQ.h
index ebb0c4bd305ea985beb5800cf8f122bf758b1169..f1a92d6dfad63226ea33595480071057be735e6a 100644 (file)
--- a/src/librbd/io/ImageRequestWQ.h
+++ b/src/librbd/io/ImageRequestWQ.h
@@ -73,8 +73,7 @@ public:
 
   void set_require_lock(Direction direction, bool enabled);
 
-  void apply_qos_limit(uint64_t limit, const uint64_t flag);
-
+  void apply_qos_limit(const uint64_t flag, uint64_t limit, uint64_t burst);
 protected:
   void *_void_dequeue() override;
   void process(ImageDispatchSpec<ImageCtxT> *req) override;

diff --git a/src/test/librbd/io/test_mock_ImageRequestWQ.cc b/src/test/librbd/io/test_mock_ImageRequestWQ.cc
index b15209608642a99100f8ebeb9ef6239071de934d..719fca2a9544c2b2085c4665eb7aa78a529374fc 100644 (file)
--- a/src/test/librbd/io/test_mock_ImageRequestWQ.cc
+++ b/src/test/librbd/io/test_mock_ImageRequestWQ.cc
@@ -391,7 +391,7 @@ TEST_F(TestMockIoImageRequestWQ, QosNoLimit) {
   InSequence seq;
   MockImageRequestWQ mock_image_request_wq(&mock_image_ctx, "io", 60, nullptr);
 
-  mock_image_request_wq.apply_qos_limit(0, RBD_QOS_BPS_THROTTLE);
+  mock_image_request_wq.apply_qos_limit(RBD_QOS_BPS_THROTTLE, 0, 0);
 
   expect_front(mock_image_request_wq, &mock_queued_image_request);
   expect_is_refresh_request(mock_image_ctx, false);
@@ -414,7 +414,7 @@ TEST_F(TestMockIoImageRequestWQ, BPSQos) {
   InSequence seq;
   MockImageRequestWQ mock_image_request_wq(&mock_image_ctx, "io", 60, nullptr);
 
-  mock_image_request_wq.apply_qos_limit(1, RBD_QOS_BPS_THROTTLE);
+  mock_image_request_wq.apply_qos_limit(RBD_QOS_BPS_THROTTLE, 1, 0);
 
   expect_front(mock_image_request_wq, &mock_queued_image_request);
   expect_tokens_requested(mock_queued_image_request, 2);