Merge pull request #17032 from yangdongsheng/rbd_qos

rbd: implement image qos in tokenbucket algorithm

Reviewed-by: Jason Dillaman <dillaman@redhat.com>

diff --cc src/common/options.cc
Simple merge

diff --cc src/librbd/ImageCtx.cc
Simple merge

diff --cc src/librbd/ImageCtx.h
Simple merge

diff --cc src/librbd/io/ImageRequestWQ.cc
index 7f479f45e96cb7a589eb79d1eb87201d7f6a0d26,9970f8fe3725e09af7ea7af312610545321b93e6..e57aa18fbfbb29e1ba744886f855c5571fd482a8
--- 1/src/librbd/io/ImageRequestWQ.cc
--- 2/src/librbd/io/ImageRequestWQ.cc
+++ b/src/librbd/io/ImageRequestWQ.cc
@@@ -541,6 -553,21 +553,25 @@@ void ImageRequestWQ<I>::set_require_loc
    }
  }
  
 -void ImageRequestWQ<I>::handle_iops_throttle_ready(int r, ImageRequest<I> *item) {
+ template <typename I>
+ void ImageRequestWQ<I>::apply_qos_iops_limit(uint64_t limit) {
+   iops_throttle->set_max(limit);
+   iops_throttle->set_average(limit);
+ }
+ 
+ template <typename I>
++void ImageRequestWQ<I>::handle_iops_throttle_ready(int r,
++                                                   ImageRequest<I> *item) {
++  CephContext *cct = m_image_ctx.cct;
++  ldout(cct, 15) << "r=" << r << ", " << "req=" << item << dendl;
++
+   assert(m_io_blockers.load() > 0);
+   --m_io_blockers;
+   item->set_throttled();
+   this->requeue(item);
+   this->signal();
+ }
+ 
  template <typename I>
  void *ImageRequestWQ<I>::_void_dequeue() {
    CephContext *cct = m_image_ctx.cct;
@@@ -551,6 -578,6 +582,18 @@@
      return nullptr;
    }
  
++  if (!peek_item->was_throttled() &&
++      iops_throttle->get<
++        ImageRequestWQ<I>, ImageRequest<I>,
++        &ImageRequestWQ<I>::handle_iops_throttle_ready>(1, this, peek_item)) {
++    ldout(cct, 15) << "throttling IO " << peek_item << dendl;
++
++    // dequeue the throttled item and block future IO
++    ThreadPool::PointerWQ<ImageRequest<I> >::_void_dequeue();
++    ++m_io_blockers;
++    return nullptr;
++  }
++
    bool lock_required;
    bool refresh_required = m_image_ctx.state->is_refresh_required();
    {