From d7a72b5f703c04e826869d5ca59df6e2cadb8e1b Mon Sep 17 00:00:00 2001 From: Jason Dillaman Date: Wed, 29 Apr 2020 13:05:35 -0400 Subject: [PATCH] librbd: QoS image io dispatch layer Re-implement the existing QoS throttling behavior from ImageRequestWQ as a new image IO dispatch layer. The existing QoS hooks in ImageRequestWQ are temporarily hooked to the new layer. Signed-off-by: Jason Dillaman --- src/librbd/CMakeLists.txt | 1 + src/librbd/ImageCtx.cc | 27 +- src/librbd/io/ImageDispatchSpec.cc | 45 --- src/librbd/io/ImageDispatchSpec.h | 15 - src/librbd/io/ImageDispatcher.cc | 15 + src/librbd/io/ImageDispatcher.h | 7 + src/librbd/io/ImageDispatcherInterface.h | 4 +- src/librbd/io/ImageRequestWQ.cc | 117 +------- src/librbd/io/ImageRequestWQ.h | 19 -- src/librbd/io/QosImageDispatch.cc | 275 ++++++++++++++++++ src/librbd/io/QosImageDispatch.h | 107 +++++++ src/librbd/io/Types.h | 40 ++- .../librbd/io/test_mock_ImageRequestWQ.cc | 106 +------ .../librbd/io/test_mock_QosImageDispatch.cc | 89 ++++++ src/test/librbd/mock/io/MockImageDispatcher.h | 3 + .../librbd/mock/io/MockQosImageDispatch.h | 24 ++ 16 files changed, 570 insertions(+), 324 deletions(-) create mode 100644 src/librbd/io/QosImageDispatch.cc create mode 100644 src/librbd/io/QosImageDispatch.h create mode 100644 src/test/librbd/io/test_mock_QosImageDispatch.cc create mode 100644 src/test/librbd/mock/io/MockQosImageDispatch.h diff --git a/src/librbd/CMakeLists.txt b/src/librbd/CMakeLists.txt index edc17aedc3f..ef9e1baf731 100644 --- a/src/librbd/CMakeLists.txt +++ b/src/librbd/CMakeLists.txt @@ -82,6 +82,7 @@ set(librbd_internal_srcs io/ObjectDispatchSpec.cc io/ObjectDispatcher.cc io/ObjectRequest.cc + io/QosImageDispatch.cc io/ReadResult.cc io/SimpleSchedulerObjectDispatch.cc io/Utils.cc diff --git a/src/librbd/ImageCtx.cc b/src/librbd/ImageCtx.cc index d5f5e5516af..4a0b884cd34 100644 --- a/src/librbd/ImageCtx.cc +++ b/src/librbd/ImageCtx.cc @@ -31,6 +31,7 @@ #include "librbd/io/ImageDispatcher.h" #include "librbd/io/ImageRequestWQ.h" #include "librbd/io/ObjectDispatcher.h" +#include "librbd/io/QosImageDispatch.h" #include "librbd/journal/StandardPolicy.h" #include "osdc/Striper.h" @@ -816,31 +817,31 @@ public: } } - io_work_queue->apply_qos_schedule_tick_min( + io_image_dispatcher->apply_qos_schedule_tick_min( config.get_val("rbd_qos_schedule_tick_min")); - io_work_queue->apply_qos_limit( - RBD_QOS_IOPS_THROTTLE, + io_image_dispatcher->apply_qos_limit( + io::IMAGE_DISPATCH_FLAG_QOS_IOPS_THROTTLE, config.get_val("rbd_qos_iops_limit"), config.get_val("rbd_qos_iops_burst")); - io_work_queue->apply_qos_limit( - RBD_QOS_BPS_THROTTLE, + io_image_dispatcher->apply_qos_limit( + io::IMAGE_DISPATCH_FLAG_QOS_BPS_THROTTLE, config.get_val("rbd_qos_bps_limit"), config.get_val("rbd_qos_bps_burst")); - io_work_queue->apply_qos_limit( - RBD_QOS_READ_IOPS_THROTTLE, + io_image_dispatcher->apply_qos_limit( + io::IMAGE_DISPATCH_FLAG_QOS_READ_IOPS_THROTTLE, config.get_val("rbd_qos_read_iops_limit"), config.get_val("rbd_qos_read_iops_burst")); - io_work_queue->apply_qos_limit( - RBD_QOS_WRITE_IOPS_THROTTLE, + io_image_dispatcher->apply_qos_limit( + io::IMAGE_DISPATCH_FLAG_QOS_WRITE_IOPS_THROTTLE, config.get_val("rbd_qos_write_iops_limit"), config.get_val("rbd_qos_write_iops_burst")); - io_work_queue->apply_qos_limit( - RBD_QOS_READ_BPS_THROTTLE, + io_image_dispatcher->apply_qos_limit( + io::IMAGE_DISPATCH_FLAG_QOS_READ_BPS_THROTTLE, config.get_val("rbd_qos_read_bps_limit"), config.get_val("rbd_qos_read_bps_burst")); - io_work_queue->apply_qos_limit( - RBD_QOS_WRITE_BPS_THROTTLE, + io_image_dispatcher->apply_qos_limit( + io::IMAGE_DISPATCH_FLAG_QOS_WRITE_BPS_THROTTLE, config.get_val("rbd_qos_write_bps_limit"), config.get_val("rbd_qos_write_bps_burst")); diff --git a/src/librbd/io/ImageDispatchSpec.cc b/src/librbd/io/ImageDispatchSpec.cc index 4d2f6497b19..5c21e5969f5 100644 --- a/src/librbd/io/ImageDispatchSpec.cc +++ b/src/librbd/io/ImageDispatchSpec.cc @@ -104,45 +104,6 @@ struct ImageDispatchSpec::IsWriteOpVisitor } }; -template -struct ImageDispatchSpec::TokenRequestedVisitor - : public boost::static_visitor { - ImageDispatchSpec* spec; - uint64_t flag; - uint64_t *tokens; - - TokenRequestedVisitor(ImageDispatchSpec* spec, uint64_t _flag, - uint64_t *tokens) - : spec(spec), flag(_flag), tokens(tokens) { - } - - uint64_t operator()(const Read&) const { - if (flag & RBD_QOS_WRITE_MASK) { - *tokens = 0; - return false; - } - - *tokens = (flag & RBD_QOS_BPS_MASK) ? spec->extents_length() : 1; - return true; - } - - uint64_t operator()(const Flush&) const { - *tokens = 0; - return true; - } - - template - uint64_t operator()(const T&) const { - if (flag & RBD_QOS_READ_MASK) { - *tokens = 0; - return false; - } - - *tokens = (flag & RBD_QOS_BPS_MASK) ? spec->extents_length() : 1; - return true; - } -}; - template void ImageDispatchSpec::send() { boost::apply_visitor(SendVisitor{this}, request); @@ -186,12 +147,6 @@ bool ImageDispatchSpec::is_write_op() const { return boost::apply_visitor(IsWriteOpVisitor(), request); } -template -bool ImageDispatchSpec::tokens_requested(uint64_t flag, uint64_t *tokens) { - return boost::apply_visitor(TokenRequestedVisitor{this, flag, tokens}, - request); -} - template void ImageDispatchSpec::start_op() { aio_comp->start_op(); diff --git a/src/librbd/io/ImageDispatchSpec.h b/src/librbd/io/ImageDispatchSpec.h index f26a297e207..3b17414e31a 100644 --- a/src/librbd/io/ImageDispatchSpec.h +++ b/src/librbd/io/ImageDispatchSpec.h @@ -106,7 +106,6 @@ public: int op_flags; ZTracer::Trace parent_trace; uint64_t tid; - std::atomic throttled_flag = 0; static ImageDispatchSpec* create_read_request( ImageCtxT &image_ctx, ImageDispatchLayer image_dispatch_layer, @@ -182,20 +181,6 @@ public: void start_op(); - bool tokens_requested(uint64_t flag, uint64_t *tokens); - - bool was_throttled(uint64_t flag) { - return throttled_flag & flag; - } - - void set_throttled(uint64_t flag) { - throttled_flag |= flag; - } - - bool were_all_throttled() { - return (throttled_flag & RBD_QOS_MASK) == RBD_QOS_MASK; - } - const Extents& get_image_extents() const; AioCompletion* get_aio_completion() const { diff --git a/src/librbd/io/ImageDispatcher.cc b/src/librbd/io/ImageDispatcher.cc index 5ea46822bc4..f4fdca25a8a 100644 --- a/src/librbd/io/ImageDispatcher.cc +++ b/src/librbd/io/ImageDispatcher.cc @@ -11,6 +11,7 @@ #include "librbd/io/ImageDispatch.h" #include "librbd/io/ImageDispatchInterface.h" #include "librbd/io/ImageDispatchSpec.h" +#include "librbd/io/QosImageDispatch.h" #include #define dout_subsys ceph_subsys_rbd @@ -103,6 +104,20 @@ ImageDispatcher::ImageDispatcher(I* image_ctx) // configure the core image dispatch handler on startup auto image_dispatch = new ImageDispatch(image_ctx); this->register_dispatch(image_dispatch); + + m_qos_image_dispatch = new QosImageDispatch(image_ctx); + this->register_dispatch(m_qos_image_dispatch); +} + +template +void ImageDispatcher::apply_qos_schedule_tick_min(uint64_t tick) { + m_qos_image_dispatch->apply_qos_schedule_tick_min(tick); +} + +template +void ImageDispatcher::apply_qos_limit(uint64_t flag, uint64_t limit, + uint64_t burst) { + m_qos_image_dispatch->apply_qos_limit(flag, limit, burst); } template diff --git a/src/librbd/io/ImageDispatcher.h b/src/librbd/io/ImageDispatcher.h index 1cda3bdf5aa..95ea6866a98 100644 --- a/src/librbd/io/ImageDispatcher.h +++ b/src/librbd/io/ImageDispatcher.h @@ -21,11 +21,16 @@ struct ImageCtx; namespace io { +template struct QosImageDispatch; + template class ImageDispatcher : public Dispatcher { public: ImageDispatcher(ImageCtxT* image_ctx); + void apply_qos_schedule_tick_min(uint64_t tick) override; + void apply_qos_limit(uint64_t flag, uint64_t limit, uint64_t burst) override; + void finish(int r, ImageDispatchLayer image_dispatch_layer, uint64_t tid) override; @@ -37,6 +42,8 @@ protected: private: struct SendVisitor; + QosImageDispatch* m_qos_image_dispatch = nullptr; + }; } // namespace io diff --git a/src/librbd/io/ImageDispatcherInterface.h b/src/librbd/io/ImageDispatcherInterface.h index 5ba8ee47d51..60d0c25f91c 100644 --- a/src/librbd/io/ImageDispatcherInterface.h +++ b/src/librbd/io/ImageDispatcherInterface.h @@ -17,10 +17,12 @@ namespace io { struct ImageDispatcherInterface : public DispatcherInterface { public: + virtual void apply_qos_schedule_tick_min(uint64_t tick) = 0; + virtual void apply_qos_limit(uint64_t flag, uint64_t limit, + uint64_t burst) = 0; virtual void finish(int r, ImageDispatchLayer image_dispatch_layer, uint64_t tid) = 0; - }; } // namespace io diff --git a/src/librbd/io/ImageRequestWQ.cc b/src/librbd/io/ImageRequestWQ.cc index f74d20cfb47..13e6cd70919 100644 --- a/src/librbd/io/ImageRequestWQ.cc +++ b/src/librbd/io/ImageRequestWQ.cc @@ -15,6 +15,8 @@ #include "librbd/io/AioCompletion.h" #include "librbd/io/ImageRequest.h" #include "librbd/io/ImageDispatchSpec.h" +#include "librbd/io/ImageDispatcher.h" +#include "librbd/io/QosImageDispatch.h" #include "common/EventTrace.h" #define dout_subsys ceph_subsys_rbd @@ -83,15 +85,6 @@ struct ImageRequestWQ::C_RefreshFinish : public Context { } }; -static std::map throttle_flags = { - { RBD_QOS_IOPS_THROTTLE, "rbd_qos_iops_throttle" }, - { RBD_QOS_BPS_THROTTLE, "rbd_qos_bps_throttle" }, - { RBD_QOS_READ_IOPS_THROTTLE, "rbd_qos_read_iops_throttle" }, - { RBD_QOS_WRITE_IOPS_THROTTLE, "rbd_qos_write_iops_throttle" }, - { RBD_QOS_READ_BPS_THROTTLE, "rbd_qos_read_bps_throttle" }, - { RBD_QOS_WRITE_BPS_THROTTLE, "rbd_qos_write_bps_throttle" } -}; - template ImageRequestWQ::ImageRequestWQ(I *image_ctx, const string &name, time_t ti, ThreadPool *tp) @@ -102,26 +95,9 @@ ImageRequestWQ::ImageRequestWQ(I *image_ctx, const string &name, CephContext *cct = m_image_ctx.cct; ldout(cct, 5) << "ictx=" << image_ctx << dendl; - SafeTimer *timer; - ceph::mutex *timer_lock; - ImageCtx::get_timer_instance(cct, &timer, &timer_lock); - - for (auto flag : throttle_flags) { - m_throttles.push_back(make_pair( - flag.first, - new TokenBucketThrottle(cct, flag.second, 0, 0, timer, timer_lock))); - } - this->register_work_queue(); } -template -ImageRequestWQ::~ImageRequestWQ() { - for (auto t : m_throttles) { - delete t.second; - } -} - template ssize_t ImageRequestWQ::read(uint64_t off, uint64_t len, ReadResult &&read_result, int op_flags) { @@ -756,86 +732,6 @@ void ImageRequestWQ::set_require_lock(Direction direction, bool enabled) { } } -template -void ImageRequestWQ::apply_qos_schedule_tick_min(uint64_t tick){ - for (auto pair : m_throttles) { - pair.second->set_schedule_tick_min(tick); - } -} - -template -void ImageRequestWQ::apply_qos_limit(const uint64_t flag, - uint64_t limit, - uint64_t burst) { - CephContext *cct = m_image_ctx.cct; - TokenBucketThrottle *throttle = nullptr; - for (auto pair : m_throttles) { - if (flag == pair.first) { - throttle = pair.second; - break; - } - } - ceph_assert(throttle != nullptr); - - int r = throttle->set_limit(limit, burst); - if (r < 0) { - lderr(cct) << throttle->get_name() << ": invalid qos parameter: " - << "burst(" << burst << ") is less than " - << "limit(" << limit << ")" << dendl; - // if apply failed, we should at least make sure the limit works. - throttle->set_limit(limit, 0); - } - - if (limit) - m_qos_enabled_flag |= flag; - else - m_qos_enabled_flag &= ~flag; -} - -template -void ImageRequestWQ::handle_throttle_ready( - ImageDispatchSpec *item, uint64_t flag) { - CephContext *cct = m_image_ctx.cct; - ldout(cct, 15) << "req=" << item << dendl; - - ceph_assert(m_io_throttled.load() > 0); - item->set_throttled(flag); - if (item->were_all_throttled()) { - this->requeue_back(item); - --m_io_throttled; - this->signal(); - } -} - -template -bool ImageRequestWQ::needs_throttle(ImageDispatchSpec *item) { - uint64_t tokens = 0; - uint64_t flag = 0; - bool blocked = false; - TokenBucketThrottle* throttle = nullptr; - - for (auto t : m_throttles) { - flag = t.first; - if (item->was_throttled(flag)) - continue; - - if (!(m_qos_enabled_flag & flag)) { - item->set_throttled(flag); - continue; - } - - throttle = t.second; - if (item->tokens_requested(flag, &tokens) && - throttle->get(tokens, this, &ImageRequestWQ::handle_throttle_ready, - item, flag)) { - blocked = true; - } else { - item->set_throttled(flag); - } - } - return blocked; -} - template void *ImageRequestWQ::_void_dequeue() { CephContext *cct = m_image_ctx.cct; @@ -846,15 +742,6 @@ void *ImageRequestWQ::_void_dequeue() { return nullptr; } - if (needs_throttle(peek_item)) { - ldout(cct, 15) << "throttling IO " << peek_item << dendl; - - ++m_io_throttled; - // dequeue the throttled item - ThreadPool::PointerWQ >::_void_dequeue(); - return nullptr; - } - bool lock_required; bool refresh_required = m_image_ctx.state->is_refresh_required(); { diff --git a/src/librbd/io/ImageRequestWQ.h b/src/librbd/io/ImageRequestWQ.h index 40acfa52390..1a115441073 100644 --- a/src/librbd/io/ImageRequestWQ.h +++ b/src/librbd/io/ImageRequestWQ.h @@ -6,7 +6,6 @@ #include "include/Context.h" #include "common/ceph_mutex.h" -#include "common/Throttle.h" #include "common/WorkQueue.h" #include "librbd/io/Types.h" #include "include/interval_set.h" @@ -30,7 +29,6 @@ class ImageRequestWQ public: ImageRequestWQ(ImageCtxT *image_ctx, const string &name, time_t ti, ThreadPool *tp); - ~ImageRequestWQ(); ssize_t read(uint64_t off, uint64_t len, ReadResult &&read_result, int op_flags); @@ -75,18 +73,9 @@ public: void set_require_lock(Direction direction, bool enabled); - void apply_qos_schedule_tick_min(uint64_t tick); - - void apply_qos_limit(const uint64_t flag, uint64_t limit, uint64_t burst); - protected: void *_void_dequeue() override; void process(ImageDispatchSpec *req) override; - bool _empty() override { - return (ThreadPool::PointerWQ>::_empty() && - m_io_throttled.load() == 0); - } - private: typedef std::list Contexts; @@ -107,7 +96,6 @@ private: std::atomic m_in_flight_ios { 0 }; std::atomic m_in_flight_writes { 0 }; std::atomic m_io_blockers { 0 }; - std::atomic m_io_throttled { 0 }; typedef interval_set ImageExtentIntervals; ImageExtentIntervals m_in_flight_extents; @@ -117,9 +105,6 @@ private: std::set m_queued_or_blocked_io_tids; std::map*> m_queued_flushes; - std::list > m_throttles; - uint64_t m_qos_enabled_flag = 0; - bool m_shutdown = false; Context *m_on_shutdown = nullptr; @@ -134,8 +119,6 @@ private: return (m_queued_writes == 0); } - bool needs_throttle(ImageDispatchSpec *item); - void finish_queued_io(bool write_op); void remove_in_flight_write_ios(uint64_t offset, uint64_t length, bool write_op, uint64_t tid); @@ -157,8 +140,6 @@ private: void handle_acquire_lock(int r, ImageDispatchSpec *req); void handle_refreshed(int r, ImageDispatchSpec *req); void handle_blocked_writes(int r); - - void handle_throttle_ready(ImageDispatchSpec *item, uint64_t flag); }; } // namespace io diff --git a/src/librbd/io/QosImageDispatch.cc b/src/librbd/io/QosImageDispatch.cc new file mode 100644 index 00000000000..3738931fd10 --- /dev/null +++ b/src/librbd/io/QosImageDispatch.cc @@ -0,0 +1,275 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/io/QosImageDispatch.h" +#include "common/dout.h" +#include "common/WorkQueue.h" +#include "librbd/ImageCtx.h" +#include + +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "librbd::io::QosImageDispatch: " << this << " " \ + << __func__ << ": " + +namespace librbd { +namespace io { + +namespace { + +uint64_t get_extent_length(const Extents& extents) { + uint64_t length = 0; + for (auto& extent : extents) { + length += extent.second; + } + return length; +} + +uint64_t calculate_tokens(bool read_op, uint64_t extent_length, uint64_t flag) { + if (read_op && ((flag & IMAGE_DISPATCH_FLAG_QOS_WRITE_MASK) != 0)) { + return 0; + } else if (!read_op && ((flag & IMAGE_DISPATCH_FLAG_QOS_READ_MASK) != 0)) { + return 0; + } + + return (((flag & IMAGE_DISPATCH_FLAG_QOS_BPS_MASK) != 0) ? extent_length : 1); +} + +static std::map throttle_flags = { + {IMAGE_DISPATCH_FLAG_QOS_IOPS_THROTTLE, "rbd_qos_iops_throttle" }, + {IMAGE_DISPATCH_FLAG_QOS_BPS_THROTTLE, "rbd_qos_bps_throttle" }, + {IMAGE_DISPATCH_FLAG_QOS_READ_IOPS_THROTTLE, "rbd_qos_read_iops_throttle" }, + {IMAGE_DISPATCH_FLAG_QOS_WRITE_IOPS_THROTTLE, "rbd_qos_write_iops_throttle" }, + {IMAGE_DISPATCH_FLAG_QOS_READ_BPS_THROTTLE, "rbd_qos_read_bps_throttle" }, + {IMAGE_DISPATCH_FLAG_QOS_WRITE_BPS_THROTTLE, "rbd_qos_write_bps_throttle" } +}; + +} // anonymous namespace + +template +QosImageDispatch::QosImageDispatch(I* image_ctx) + : m_image_ctx(image_ctx) { + auto cct = m_image_ctx->cct; + ldout(cct, 5) << "ictx=" << image_ctx << dendl; + + SafeTimer *timer; + ceph::mutex *timer_lock; + ImageCtx::get_timer_instance(cct, &timer, &timer_lock); + for (auto flag : throttle_flags) { + m_throttles.push_back(make_pair( + flag.first, + new TokenBucketThrottle(cct, flag.second, 0, 0, timer, timer_lock))); + } +} + +template +QosImageDispatch::~QosImageDispatch() { + for (auto t : m_throttles) { + delete t.second; + } +} + +template +void QosImageDispatch::shut_down(Context* on_finish) { + on_finish->complete(0); +} + +template +void QosImageDispatch::apply_qos_schedule_tick_min(uint64_t tick) { + for (auto pair : m_throttles) { + pair.second->set_schedule_tick_min(tick); + } +} + +template +void QosImageDispatch::apply_qos_limit(uint64_t flag, uint64_t limit, + uint64_t burst) { + auto cct = m_image_ctx->cct; + TokenBucketThrottle *throttle = nullptr; + for (auto pair : m_throttles) { + if (flag == pair.first) { + throttle = pair.second; + break; + } + } + ceph_assert(throttle != nullptr); + + int r = throttle->set_limit(limit, burst); + if (r < 0) { + lderr(cct) << throttle->get_name() << ": invalid qos parameter: " + << "burst(" << burst << ") is less than " + << "limit(" << limit << ")" << dendl; + // if apply failed, we should at least make sure the limit works. + throttle->set_limit(limit, 0); + } + + if (limit) { + m_qos_enabled_flag |= flag; + } else { + m_qos_enabled_flag &= ~flag; + } +} + +template +bool QosImageDispatch::read( + AioCompletion* aio_comp, Extents &&image_extents, ReadResult &&read_result, + int op_flags, const ZTracer::Trace &parent_trace, uint64_t tid, + std::atomic* image_dispatch_flags, + DispatchResult* dispatch_result, Context* on_dispatched) { + auto cct = m_image_ctx->cct; + ldout(cct, 20) << "tid=" << tid << ", image_extents=" << image_extents + << dendl; + + if (needs_throttle(true, image_extents, image_dispatch_flags, + dispatch_result, on_dispatched)) { + return true; + } + + return false; +} + +template +bool QosImageDispatch::write( + AioCompletion* aio_comp, Extents &&image_extents, bufferlist &&bl, + int op_flags, const ZTracer::Trace &parent_trace, uint64_t tid, + std::atomic* image_dispatch_flags, + DispatchResult* dispatch_result, Context* on_dispatched) { + auto cct = m_image_ctx->cct; + ldout(cct, 20) << "tid=" << tid << ", image_extents=" << image_extents + << dendl; + + if (needs_throttle(false, image_extents, image_dispatch_flags, + dispatch_result, on_dispatched)) { + return true; + } + + return false; +} + +template +bool QosImageDispatch::discard( + AioCompletion* aio_comp, Extents &&image_extents, + uint32_t discard_granularity_bytes, const ZTracer::Trace &parent_trace, + uint64_t tid, std::atomic* image_dispatch_flags, + DispatchResult* dispatch_result, Context* on_dispatched) { + auto cct = m_image_ctx->cct; + ldout(cct, 20) << "tid=" << tid << ", image_extents=" << image_extents + << dendl; + + if (needs_throttle(false, image_extents, image_dispatch_flags, + dispatch_result, on_dispatched)) { + return true; + } + + return false; +} + +template +bool QosImageDispatch::write_same( + AioCompletion* aio_comp, Extents &&image_extents, bufferlist &&bl, + int op_flags, const ZTracer::Trace &parent_trace, uint64_t tid, + std::atomic* image_dispatch_flags, + DispatchResult* dispatch_result, Context* on_dispatched) { + auto cct = m_image_ctx->cct; + ldout(cct, 20) << "tid=" << tid << ", image_extents=" << image_extents + << dendl; + + if (needs_throttle(false, image_extents, image_dispatch_flags, + dispatch_result, on_dispatched)) { + return true; + } + + return false; +} + +template +bool QosImageDispatch::compare_and_write( + AioCompletion* aio_comp, Extents &&image_extents, bufferlist &&cmp_bl, + bufferlist &&bl, uint64_t *mismatch_offset, int op_flags, + const ZTracer::Trace &parent_trace, uint64_t tid, + std::atomic* image_dispatch_flags, + DispatchResult* dispatch_result, Context* on_dispatched) { + auto cct = m_image_ctx->cct; + ldout(cct, 20) << "tid=" << tid << ", image_extents=" << image_extents + << dendl; + + if (needs_throttle(false, image_extents, image_dispatch_flags, + dispatch_result, on_dispatched)) { + return true; + } + + return false; +} + +template +bool QosImageDispatch::flush( + AioCompletion* aio_comp, FlushSource flush_source, + const ZTracer::Trace &parent_trace, uint64_t tid, + std::atomic* image_dispatch_flags, + DispatchResult* dispatch_result, Context* on_dispatched) { + auto cct = m_image_ctx->cct; + ldout(cct, 20) << "tid=" << tid << dendl; + + return false; +} + +template +bool QosImageDispatch::set_throttle_flag( + std::atomic* image_dispatch_flags, uint32_t flag) { + uint32_t expected = image_dispatch_flags->load(); + uint32_t desired; + do { + desired = expected | flag; + } while (!image_dispatch_flags->compare_exchange_weak(expected, desired)); + + return ((desired & IMAGE_DISPATCH_FLAG_QOS_MASK) == + IMAGE_DISPATCH_FLAG_QOS_MASK); +} + +template +bool QosImageDispatch::needs_throttle( + bool read_op, const Extents& image_extents, + std::atomic* image_dispatch_flags, + DispatchResult* dispatch_result, Context* on_dispatched) { + auto cct = m_image_ctx->cct; + auto extent_length = get_extent_length(image_extents); + bool all_qos_flags_set = false; + + *dispatch_result = DISPATCH_RESULT_CONTINUE; + + auto qos_enabled_flag = m_qos_enabled_flag; + for (auto [flag, throttle] : m_throttles) { + if ((qos_enabled_flag & flag) == 0) { + all_qos_flags_set = set_throttle_flag(image_dispatch_flags, flag); + continue; + } + + auto tokens = calculate_tokens(read_op, extent_length, flag); + if (tokens > 0 && + throttle->get(tokens, this, &QosImageDispatch::handle_throttle_ready, + Tag{image_dispatch_flags, on_dispatched}, flag)) { + ldout(cct, 15) << "on_dispatched=" << on_dispatched << ", " + << "flag=" << flag << dendl; + all_qos_flags_set = false; + } else { + all_qos_flags_set = set_throttle_flag(image_dispatch_flags, flag); + } + } + return !all_qos_flags_set; +} + +template +void QosImageDispatch::handle_throttle_ready(Tag&& tag, uint64_t flag) { + auto cct = m_image_ctx->cct; + ldout(cct, 15) << "on_dispatched=" << tag.on_dispatched << ", " + << "flag=" << flag << dendl; + + if (set_throttle_flag(tag.image_dispatch_flags, flag)) { + tag.on_dispatched->complete(0); + } +} + +} // namespace io +} // namespace librbd + +template class librbd::io::QosImageDispatch; diff --git a/src/librbd/io/QosImageDispatch.h b/src/librbd/io/QosImageDispatch.h new file mode 100644 index 00000000000..07546d9cf29 --- /dev/null +++ b/src/librbd/io/QosImageDispatch.h @@ -0,0 +1,107 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_IO_QOS_IMAGE_DISPATCH_H +#define CEPH_LIBRBD_IO_QOS_IMAGE_DISPATCH_H + +#include "librbd/io/ImageDispatchInterface.h" +#include "include/int_types.h" +#include "include/buffer.h" +#include "common/zipkin_trace.h" +#include "common/Throttle.h" +#include "librbd/io/ReadResult.h" +#include "librbd/io/Types.h" +#include + +struct Context; + +namespace librbd { + +struct ImageCtx; + +namespace io { + +struct AioCompletion; + +template +class QosImageDispatch : public ImageDispatchInterface { +public: + struct Tag { + std::atomic* image_dispatch_flags; + Context* on_dispatched; + + Tag(std::atomic* image_dispatch_flags, Context* on_dispatched) + : image_dispatch_flags(image_dispatch_flags), + on_dispatched(on_dispatched) { + } + }; + + QosImageDispatch(ImageCtxT* image_ctx); + ~QosImageDispatch() override; + + ImageDispatchLayer get_dispatch_layer() const override { + return IMAGE_DISPATCH_LAYER_QOS; + } + + void shut_down(Context* on_finish) override; + + void apply_qos_schedule_tick_min(uint64_t tick); + void apply_qos_limit(uint64_t flag, uint64_t limit, uint64_t burst); + + bool read( + AioCompletion* aio_comp, Extents &&image_extents, + ReadResult &&read_result, int op_flags, + const ZTracer::Trace &parent_trace, uint64_t tid, + std::atomic* image_dispatch_flags, + DispatchResult* dispatch_result, Context* on_dispatched) override; + bool write( + AioCompletion* aio_comp, Extents &&image_extents, bufferlist &&bl, + int op_flags, const ZTracer::Trace &parent_trace, uint64_t tid, + std::atomic* image_dispatch_flags, + DispatchResult* dispatch_result, Context* on_dispatched) override; + bool discard( + AioCompletion* aio_comp, Extents &&image_extents, + uint32_t discard_granularity_bytes, + const ZTracer::Trace &parent_trace, uint64_t tid, + std::atomic* image_dispatch_flags, + DispatchResult* dispatch_result, Context* on_dispatched) override; + bool write_same( + AioCompletion* aio_comp, Extents &&image_extents, bufferlist &&bl, + int op_flags, const ZTracer::Trace &parent_trace, uint64_t tid, + std::atomic* image_dispatch_flags, + DispatchResult* dispatch_result, Context* on_dispatched) override; + bool compare_and_write( + AioCompletion* aio_comp, Extents &&image_extents, bufferlist &&cmp_bl, + bufferlist &&bl, uint64_t *mismatch_offset, int op_flags, + const ZTracer::Trace &parent_trace, uint64_t tid, + std::atomic* image_dispatch_flags, + DispatchResult* dispatch_result, Context* on_dispatched) override; + bool flush( + AioCompletion* aio_comp, FlushSource flush_source, + const ZTracer::Trace &parent_trace, uint64_t tid, + std::atomic* image_dispatch_flags, + DispatchResult* dispatch_result, Context* on_dispatched) override; + + void handle_finished(int r, uint64_t tid) override {} + +private: + ImageCtxT* m_image_ctx; + + std::list > m_throttles; + uint64_t m_qos_enabled_flag = 0; + + bool set_throttle_flag(std::atomic* image_dispatch_flags, + uint32_t flag); + bool needs_throttle(bool read_op, const Extents& image_extents, + std::atomic* image_dispatch_flags, + DispatchResult* dispatch_result, Context* on_dispatched); + void handle_throttle_ready(Tag&& tag, uint64_t flag); + +}; + +} // namespace io +} // namespace librbd + +extern template class librbd::io::QosImageDispatch; + +#endif // CEPH_LIBRBD_IO_QOS_IMAGE_DISPATCH_H diff --git a/src/librbd/io/Types.h b/src/librbd/io/Types.h index fc10337255c..332167a657a 100644 --- a/src/librbd/io/Types.h +++ b/src/librbd/io/Types.h @@ -14,20 +14,6 @@ struct Context; namespace librbd { namespace io { -#define RBD_QOS_IOPS_THROTTLE 1 << 0 -#define RBD_QOS_BPS_THROTTLE 1 << 1 -#define RBD_QOS_READ_IOPS_THROTTLE 1 << 2 -#define RBD_QOS_WRITE_IOPS_THROTTLE 1 << 3 -#define RBD_QOS_READ_BPS_THROTTLE 1 << 4 -#define RBD_QOS_WRITE_BPS_THROTTLE 1 << 5 - -#define RBD_QOS_BPS_MASK (RBD_QOS_BPS_THROTTLE | RBD_QOS_READ_BPS_THROTTLE | RBD_QOS_WRITE_BPS_THROTTLE) -#define RBD_QOS_IOPS_MASK (RBD_QOS_IOPS_THROTTLE | RBD_QOS_READ_IOPS_THROTTLE | RBD_QOS_WRITE_IOPS_THROTTLE) -#define RBD_QOS_READ_MASK (RBD_QOS_READ_BPS_THROTTLE | RBD_QOS_READ_IOPS_THROTTLE) -#define RBD_QOS_WRITE_MASK (RBD_QOS_WRITE_BPS_THROTTLE | RBD_QOS_WRITE_IOPS_THROTTLE) - -#define RBD_QOS_MASK (RBD_QOS_BPS_MASK | RBD_QOS_IOPS_MASK) - typedef enum { AIO_TYPE_NONE = 0, AIO_TYPE_GENERIC, @@ -75,6 +61,32 @@ enum ImageDispatchLayer { IMAGE_DISPATCH_LAYER_LAST }; +enum { + IMAGE_DISPATCH_FLAG_QOS_IOPS_THROTTLE = 1 << 0, + IMAGE_DISPATCH_FLAG_QOS_BPS_THROTTLE = 1 << 1, + IMAGE_DISPATCH_FLAG_QOS_READ_IOPS_THROTTLE = 1 << 2, + IMAGE_DISPATCH_FLAG_QOS_WRITE_IOPS_THROTTLE = 1 << 3, + IMAGE_DISPATCH_FLAG_QOS_READ_BPS_THROTTLE = 1 << 4, + IMAGE_DISPATCH_FLAG_QOS_WRITE_BPS_THROTTLE = 1 << 5, + IMAGE_DISPATCH_FLAG_QOS_BPS_MASK = ( + IMAGE_DISPATCH_FLAG_QOS_BPS_THROTTLE | + IMAGE_DISPATCH_FLAG_QOS_READ_BPS_THROTTLE | + IMAGE_DISPATCH_FLAG_QOS_WRITE_BPS_THROTTLE), + IMAGE_DISPATCH_FLAG_QOS_IOPS_MASK = ( + IMAGE_DISPATCH_FLAG_QOS_IOPS_THROTTLE | + IMAGE_DISPATCH_FLAG_QOS_READ_IOPS_THROTTLE | + IMAGE_DISPATCH_FLAG_QOS_WRITE_IOPS_THROTTLE), + IMAGE_DISPATCH_FLAG_QOS_READ_MASK = ( + IMAGE_DISPATCH_FLAG_QOS_READ_IOPS_THROTTLE | + IMAGE_DISPATCH_FLAG_QOS_READ_BPS_THROTTLE), + IMAGE_DISPATCH_FLAG_QOS_WRITE_MASK = ( + IMAGE_DISPATCH_FLAG_QOS_WRITE_IOPS_THROTTLE | + IMAGE_DISPATCH_FLAG_QOS_WRITE_BPS_THROTTLE), + IMAGE_DISPATCH_FLAG_QOS_MASK = ( + IMAGE_DISPATCH_FLAG_QOS_BPS_MASK | + IMAGE_DISPATCH_FLAG_QOS_IOPS_MASK), +}; + enum ObjectDispatchLayer { OBJECT_DISPATCH_LAYER_NONE = 0, OBJECT_DISPATCH_LAYER_CACHE, diff --git a/src/test/librbd/io/test_mock_ImageRequestWQ.cc b/src/test/librbd/io/test_mock_ImageRequestWQ.cc index 99c8362d887..61929e0f84d 100644 --- a/src/test/librbd/io/test_mock_ImageRequestWQ.cc +++ b/src/test/librbd/io/test_mock_ImageRequestWQ.cc @@ -5,6 +5,7 @@ #include "test/librbd/test_support.h" #include "test/librbd/mock/MockImageCtx.h" #include "test/librbd/mock/exclusive_lock/MockPolicy.h" +#include "test/librbd/mock/io/MockQosImageDispatch.h" #include "librbd/io/ImageDispatchSpec.h" #include "librbd/io/ImageRequestWQ.h" #include "librbd/io/ImageRequest.h" @@ -42,6 +43,8 @@ struct ImageDispatchSpec { AioCompletion *aio_comp = nullptr; bool blocked = false; + std::atomic image_dispatch_flags = {0}; + static ImageDispatchSpec* create_write_request( librbd::MockTestImageCtx &image_ctx, ImageDispatchLayer dispatch_layer, AioCompletion *aio_comp, Extents &&image_extents, bufferlist &&bl, @@ -64,10 +67,6 @@ struct ImageDispatchSpec { MOCK_CONST_METHOD0(start_op, void()); MOCK_CONST_METHOD0(send, void()); MOCK_CONST_METHOD1(fail, void(int)); - MOCK_CONST_METHOD1(was_throttled, bool(uint64_t)); - MOCK_CONST_METHOD0(were_all_throttled, bool()); - MOCK_CONST_METHOD1(set_throttled, void(uint64_t)); - MOCK_CONST_METHOD2(tokens_requested, bool(uint64_t, uint64_t *)); MOCK_CONST_METHOD0(get_image_extents, Extents()); MOCK_CONST_METHOD0(get_aio_completion, AioCompletion*()); MOCK_CONST_METHOD0(get_tid, uint64_t()); @@ -233,27 +232,6 @@ struct TestMockIoImageRequestWQ : public TestMockFixture { })); } - void expect_set_throttled(MockImageDispatchSpec &mock_image_request) { - EXPECT_CALL(mock_image_request, set_throttled(_)).Times(6); - } - - void expect_was_throttled(MockImageDispatchSpec &mock_image_request, bool value) { - EXPECT_CALL(mock_image_request, was_throttled(_)).Times(6).WillRepeatedly(Return(value)); - } - - void expect_tokens_requested(MockImageDispatchSpec &mock_image_request, - uint64_t tokens, bool r) { - EXPECT_CALL(mock_image_request, tokens_requested(_, _)) - .WillOnce(WithArg<1>(Invoke([tokens, r](uint64_t *t) { - *t = tokens; - return r; - }))); - } - - void expect_all_throttled(MockImageDispatchSpec &mock_image_request, bool value) { - EXPECT_CALL(mock_image_request, were_all_throttled()).WillOnce(Return(value)); - } - void expect_start_op(MockImageDispatchSpec &mock_image_request) { EXPECT_CALL(mock_image_request, start_op()).Times(1); } @@ -261,7 +239,7 @@ struct TestMockIoImageRequestWQ : public TestMockFixture { void expect_get_image_extents(MockImageDispatchSpec &mock_image_request, const Extents &extents) { EXPECT_CALL(mock_image_request, get_image_extents()) - .WillOnce(Return(extents)); + .WillRepeatedly(Return(extents)); } void expect_get_tid(MockImageDispatchSpec &mock_image_request, uint64_t tid) { @@ -280,8 +258,6 @@ TEST_F(TestMockIoImageRequestWQ, AcquireLockError) { mock_image_ctx.exclusive_lock = &mock_exclusive_lock; auto mock_queued_image_request = new MockImageDispatchSpec(); - expect_was_throttled(*mock_queued_image_request, false); - expect_set_throttled(*mock_queued_image_request); expect_get_image_extents(*mock_queued_image_request, {}); expect_get_tid(*mock_queued_image_request, 0); @@ -329,8 +305,6 @@ TEST_F(TestMockIoImageRequestWQ, AcquireLockBlacklisted) { mock_image_ctx.exclusive_lock = &mock_exclusive_lock; auto mock_queued_image_request = new MockImageDispatchSpec(); - expect_was_throttled(*mock_queued_image_request, false); - expect_set_throttled(*mock_queued_image_request); expect_get_image_extents(*mock_queued_image_request, {}); expect_get_tid(*mock_queued_image_request, 0); @@ -371,8 +345,6 @@ TEST_F(TestMockIoImageRequestWQ, RefreshError) { MockTestImageCtx mock_image_ctx(*ictx); auto mock_queued_image_request = new MockImageDispatchSpec(); - expect_was_throttled(*mock_queued_image_request, false); - expect_set_throttled(*mock_queued_image_request); expect_get_image_extents(*mock_queued_image_request, {}); expect_get_tid(*mock_queued_image_request, 0); @@ -404,75 +376,5 @@ TEST_F(TestMockIoImageRequestWQ, RefreshError) { aio_comp->release(); } -TEST_F(TestMockIoImageRequestWQ, QosNoLimit) { - librbd::ImageCtx *ictx; - ASSERT_EQ(0, open_image(m_image_name, &ictx)); - - MockTestImageCtx mock_image_ctx(*ictx); - - MockImageDispatchSpec mock_queued_image_request; - expect_was_throttled(mock_queued_image_request, false); - expect_set_throttled(mock_queued_image_request); - - InSequence seq; - MockImageRequestWQ mock_image_request_wq(&mock_image_ctx, "io", 60, nullptr); - - mock_image_request_wq.apply_qos_limit(RBD_QOS_BPS_THROTTLE, 0, 0); - - expect_front(mock_image_request_wq, &mock_queued_image_request); - expect_is_refresh_request(mock_image_ctx, false); - expect_is_write_op(mock_queued_image_request, true); - expect_dequeue(mock_image_request_wq, &mock_queued_image_request); - ASSERT_TRUE(mock_image_request_wq.invoke_dequeue() == &mock_queued_image_request); -} - -TEST_F(TestMockIoImageRequestWQ, BPSQosNoBurst) { - librbd::ImageCtx *ictx; - ASSERT_EQ(0, open_image(m_image_name, &ictx)); - - MockTestImageCtx mock_image_ctx(*ictx); - - MockImageDispatchSpec mock_queued_image_request; - expect_was_throttled(mock_queued_image_request, false); - expect_set_throttled(mock_queued_image_request); - - InSequence seq; - MockImageRequestWQ mock_image_request_wq(&mock_image_ctx, "io", 60, nullptr); - - mock_image_request_wq.apply_qos_limit(RBD_QOS_BPS_THROTTLE, 1, 0); - - expect_front(mock_image_request_wq, &mock_queued_image_request); - expect_tokens_requested(mock_queued_image_request, 2, true); - expect_dequeue(mock_image_request_wq, &mock_queued_image_request); - expect_all_throttled(mock_queued_image_request, true); - expect_requeue_back(mock_image_request_wq); - expect_signal(mock_image_request_wq); - ASSERT_TRUE(mock_image_request_wq.invoke_dequeue() == nullptr); -} - -TEST_F(TestMockIoImageRequestWQ, BPSQosWithBurst) { - librbd::ImageCtx *ictx; - ASSERT_EQ(0, open_image(m_image_name, &ictx)); - - MockTestImageCtx mock_image_ctx(*ictx); - - MockImageDispatchSpec mock_queued_image_request; - expect_was_throttled(mock_queued_image_request, false); - expect_set_throttled(mock_queued_image_request); - - InSequence seq; - MockImageRequestWQ mock_image_request_wq(&mock_image_ctx, "io", 60, nullptr); - - mock_image_request_wq.apply_qos_limit(RBD_QOS_BPS_THROTTLE, 1, 1); - - expect_front(mock_image_request_wq, &mock_queued_image_request); - expect_tokens_requested(mock_queued_image_request, 2, true); - expect_dequeue(mock_image_request_wq, &mock_queued_image_request); - expect_all_throttled(mock_queued_image_request, true); - expect_requeue_back(mock_image_request_wq); - expect_signal(mock_image_request_wq); - ASSERT_TRUE(mock_image_request_wq.invoke_dequeue() == nullptr); -} - } // namespace io } // namespace librbd diff --git a/src/test/librbd/io/test_mock_QosImageDispatch.cc b/src/test/librbd/io/test_mock_QosImageDispatch.cc new file mode 100644 index 00000000000..0ac963a730c --- /dev/null +++ b/src/test/librbd/io/test_mock_QosImageDispatch.cc @@ -0,0 +1,89 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/librbd/test_mock_fixture.h" +#include "test/librbd/test_support.h" +#include "test/librbd/mock/MockImageCtx.h" +#include "test/librbd/mock/exclusive_lock/MockPolicy.h" +#include "librbd/io/ImageDispatchSpec.h" +#include "librbd/io/ImageRequestWQ.h" +#include "librbd/io/ImageRequest.h" + +namespace librbd { +namespace io { + +TEST_F(TestMockIoImageRequestWQ, QosNoLimit) { + librbd::ImageCtx *ictx; + ASSERT_EQ(0, open_image(m_image_name, &ictx)); + + MockTestImageCtx mock_image_ctx(*ictx); + + MockImageDispatchSpec mock_queued_image_request; + expect_was_throttled(mock_queued_image_request, false); + expect_set_throttled(mock_queued_image_request); + + InSequence seq; + MockImageRequestWQ mock_image_request_wq(&mock_image_ctx, "io", 60, nullptr); + + mock_image_request_wq.apply_qos_limit(IMAGE_DISPATCH_FLAG_QOS_BPS_THROTTLE, 0, + 0); + + expect_front(mock_image_request_wq, &mock_queued_image_request); + expect_is_refresh_request(mock_image_ctx, false); + expect_is_write_op(mock_queued_image_request, true); + expect_dequeue(mock_image_request_wq, &mock_queued_image_request); + ASSERT_TRUE(mock_image_request_wq.invoke_dequeue() == &mock_queued_image_request); +} + +TEST_F(TestMockIoImageRequestWQ, BPSQosNoBurst) { + librbd::ImageCtx *ictx; + ASSERT_EQ(0, open_image(m_image_name, &ictx)); + + MockTestImageCtx mock_image_ctx(*ictx); + + MockImageDispatchSpec mock_queued_image_request; + expect_was_throttled(mock_queued_image_request, false); + expect_set_throttled(mock_queued_image_request); + + InSequence seq; + MockImageRequestWQ mock_image_request_wq(&mock_image_ctx, "io", 60, nullptr); + + mock_image_request_wq.apply_qos_limit(IMAGE_DISPATCH_FLAG_QOS_BPS_THROTTLE, 1, + 0); + + expect_front(mock_image_request_wq, &mock_queued_image_request); + expect_tokens_requested(mock_queued_image_request, 2, true); + expect_dequeue(mock_image_request_wq, &mock_queued_image_request); + expect_all_throttled(mock_queued_image_request, true); + expect_requeue_back(mock_image_request_wq); + expect_signal(mock_image_request_wq); + ASSERT_TRUE(mock_image_request_wq.invoke_dequeue() == nullptr); +} + +TEST_F(TestMockIoImageRequestWQ, BPSQosWithBurst) { + librbd::ImageCtx *ictx; + ASSERT_EQ(0, open_image(m_image_name, &ictx)); + + MockTestImageCtx mock_image_ctx(*ictx); + + MockImageDispatchSpec mock_queued_image_request; + expect_was_throttled(mock_queued_image_request, false); + expect_set_throttled(mock_queued_image_request); + + InSequence seq; + MockImageRequestWQ mock_image_request_wq(&mock_image_ctx, "io", 60, nullptr); + + mock_image_request_wq.apply_qos_limit(IMAGE_DISPATCH_FLAG_QOS_BPS_THROTTLE, 1, + 1); + + expect_front(mock_image_request_wq, &mock_queued_image_request); + expect_tokens_requested(mock_queued_image_request, 2, true); + expect_dequeue(mock_image_request_wq, &mock_queued_image_request); + expect_all_throttled(mock_queued_image_request, true); + expect_requeue_back(mock_image_request_wq); + expect_signal(mock_image_request_wq); + ASSERT_TRUE(mock_image_request_wq.invoke_dequeue() == nullptr); +} + +} // namespace io +} // namespace librbd diff --git a/src/test/librbd/mock/io/MockImageDispatcher.h b/src/test/librbd/mock/io/MockImageDispatcher.h index 90b2f072885..bd7b8428e5d 100644 --- a/src/test/librbd/mock/io/MockImageDispatcher.h +++ b/src/test/librbd/mock/io/MockImageDispatcher.h @@ -26,6 +26,9 @@ public: MOCK_METHOD1(send, void(ImageDispatchSpec<>*)); MOCK_METHOD3(finish, void(int r, ImageDispatchLayer, uint64_t)); + + MOCK_METHOD1(apply_qos_schedule_tick_min, void(uint64_t)); + MOCK_METHOD3(apply_qos_limit, void(uint64_t, uint64_t, uint64_t)); }; } // namespace io diff --git a/src/test/librbd/mock/io/MockQosImageDispatch.h b/src/test/librbd/mock/io/MockQosImageDispatch.h new file mode 100644 index 00000000000..e498978165b --- /dev/null +++ b/src/test/librbd/mock/io/MockQosImageDispatch.h @@ -0,0 +1,24 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_TEST_LIBRBD_MOCK_IO_QOS_IMAGE_DISPATCH_H +#define CEPH_TEST_LIBRBD_MOCK_IO_QOS_IMAGE_DISPATCH_H + +#include "gmock/gmock.h" +#include "librbd/io/Types.h" +#include + +struct Context; + +namespace librbd { +namespace io { + +struct MockQosImageDispatch { + MOCK_METHOD4(needs_throttle, bool(bool, const Extents&, + std::atomic*, Context*)); +}; + +} // namespace io +} // namespace librbd + +#endif // CEPH_TEST_LIBRBD_MOCK_IO_QOS_IMAGE_DISPATCH_H -- 2.39.5