From 7c3ca2b528b62d08c4c57fada43159683a7a0f0b Mon Sep 17 00:00:00 2001 From: Jason Dillaman Date: Thu, 15 Feb 2018 15:45:05 -0500 Subject: [PATCH] librbd: switch to object IO requests to dispatcher interface Signed-off-by: Jason Dillaman --- src/librbd/CMakeLists.txt | 1 + src/librbd/image/CloseRequest.cc | 24 ++ src/librbd/image/CloseRequest.h | 20 +- src/librbd/io/ImageRequest.cc | 14 +- src/librbd/io/ObjectDispatch.cc | 141 +++++++ src/librbd/io/ObjectDispatch.h | 104 ++++++ src/librbd/io/ObjectDispatchInterface.h | 86 +++++ src/librbd/io/ObjectDispatcher.cc | 353 +++++++++++++++--- src/librbd/io/ObjectDispatcher.h | 44 ++- src/librbd/io/Types.h | 5 + src/librbd/operation/TrimRequest.cc | 22 +- src/test/librbd/io/test_mock_ImageRequest.cc | 45 --- src/test/librbd/mock/io/MockObjectDispatch.h | 121 ++++++ .../librbd/mock/io/MockObjectDispatcher.h | 15 + .../librbd/operation/test_mock_TrimRequest.cc | 84 ++--- 15 files changed, 916 insertions(+), 163 deletions(-) create mode 100644 src/librbd/io/ObjectDispatch.cc create mode 100644 src/librbd/io/ObjectDispatch.h create mode 100644 src/librbd/io/ObjectDispatchInterface.h create mode 100644 src/test/librbd/mock/io/MockObjectDispatch.h diff --git a/src/librbd/CMakeLists.txt b/src/librbd/CMakeLists.txt index aec82f011413f..8618a19726d34 100644 --- a/src/librbd/CMakeLists.txt +++ b/src/librbd/CMakeLists.txt @@ -60,6 +60,7 @@ set(librbd_internal_srcs io/ImageDispatchSpec.cc io/ImageRequest.cc io/ImageRequestWQ.cc + io/ObjectDispatch.cc io/ObjectDispatchSpec.cc io/ObjectDispatcher.cc io/ObjectRequest.cc diff --git a/src/librbd/image/CloseRequest.cc b/src/librbd/image/CloseRequest.cc index ee1519bf7d5f1..0120c566629e9 100644 --- a/src/librbd/image/CloseRequest.cc +++ b/src/librbd/image/CloseRequest.cc @@ -11,6 +11,7 @@ #include "librbd/ObjectMap.h" #include "librbd/Utils.h" #include "librbd/io/ImageRequestWQ.h" +#include "librbd/io/ObjectDispatcher.h" #define dout_subsys ceph_subsys_rbd #undef dout_prefix @@ -240,6 +241,29 @@ void CloseRequest::handle_shut_down_cache(int r) { if (r < 0) { lderr(cct) << "failed to shut down cache: " << cpp_strerror(r) << dendl; } + send_shut_down_object_dispatcher(); +} + +template +void CloseRequest::send_shut_down_object_dispatcher() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << this << " " << __func__ << dendl; + + m_image_ctx->io_object_dispatcher->shut_down(create_context_callback< + CloseRequest, + &CloseRequest::handle_shut_down_object_dispatcher>(this)); +} + +template +void CloseRequest::handle_shut_down_object_dispatcher(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << this << " " << __func__ << ": r=" << r << dendl; + + save_result(r); + if (r < 0) { + lderr(cct) << "failed to shut down object dispatcher: " + << cpp_strerror(r) << dendl; + } send_flush_op_work_queue(); } diff --git a/src/librbd/image/CloseRequest.h b/src/librbd/image/CloseRequest.h index 812e91d723e18..73c344af4343a 100644 --- a/src/librbd/image/CloseRequest.h +++ b/src/librbd/image/CloseRequest.h @@ -53,13 +53,16 @@ private: * SHUTDOWN_CACHE * | * v - * FLUSH_OP_WORK_QUEUE . . . . . - * | . - * v . - * CLOSE_PARENT . (no parent) - * | . - * v . - * FLUSH_IMAGE_WATCHER < . . . . + * SHUT_DOWN_OBJECT_DISPATCHER + * | + * v + * FLUSH_OP_WORK_QUEUE + * | + * v (skip if no parent) + * CLOSE_PARENT + * | + * v + * FLUSH_IMAGE_WATCHER * | * v * @@ -100,6 +103,9 @@ private: void send_shut_down_cache(); void handle_shut_down_cache(int r); + void send_shut_down_object_dispatcher(); + void handle_shut_down_object_dispatcher(int r); + void send_flush_op_work_queue(); void handle_flush_op_work_queue(int r); diff --git a/src/librbd/io/ImageRequest.cc b/src/librbd/io/ImageRequest.cc index 9ca06eda515e6..df9a3e6941753 100644 --- a/src/librbd/io/ImageRequest.cc +++ b/src/librbd/io/ImageRequest.cc @@ -9,7 +9,7 @@ #include "librbd/Utils.h" #include "librbd/cache/ImageCache.h" #include "librbd/io/AioCompletion.h" -#include "librbd/io/ObjectRequest.h" +#include "librbd/io/ObjectDispatchInterface.h" #include "librbd/io/ObjectDispatchSpec.h" #include "librbd/io/ObjectDispatcher.h" #include "librbd/io/Utils.h" @@ -265,11 +265,11 @@ void ImageReadRequest::send_request() { auto req_comp = new io::ReadResult::C_ObjectReadRequest( aio_comp, extent.offset, extent.length, std::move(extent.buffer_extents), true); - auto req = ObjectReadRequest::create( - &image_ctx, extent.oid.name, extent.objectno, extent.offset, - extent.length, snap_id, m_op_flags, false, this->m_trace, - &req_comp->bl, &req_comp->extent_map, req_comp); - req->send(); + auto req = ObjectDispatchSpec::create_read( + &image_ctx, OBJECT_DISPATCH_LAYER_NONE, extent.oid.name, + extent.objectno, extent.offset, extent.length, snap_id, m_op_flags, + this->m_trace, &req_comp->bl, &req_comp->extent_map, req_comp); + req->send(0); } } @@ -583,7 +583,7 @@ ObjectDispatchSpec *ImageDiscardRequest::create_object_request( auto req = ObjectDispatchSpec::create_discard( &image_ctx, OBJECT_DISPATCH_LAYER_NONE, object_extent.oid.name, object_extent.objectno, object_extent.offset, object_extent.length, snapc, - 0, 0, this->m_trace, on_finish); + OBJECT_DISCARD_FLAG_DISABLE_CLONE_REMOVE, 0, this->m_trace, on_finish); return req; } diff --git a/src/librbd/io/ObjectDispatch.cc b/src/librbd/io/ObjectDispatch.cc new file mode 100644 index 0000000000000..a1055fce55b71 --- /dev/null +++ b/src/librbd/io/ObjectDispatch.cc @@ -0,0 +1,141 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/io/ObjectDispatch.h" +#include "common/dout.h" +#include "common/WorkQueue.h" +#include "librbd/ImageCtx.h" +#include "librbd/io/ObjectRequest.h" + +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "librbd::io::ObjectDispatch: " << this \ + << " " << __func__ << ": " + +namespace librbd { +namespace io { + +template +ObjectDispatch::ObjectDispatch(I* image_ctx) + : m_image_ctx(image_ctx) { +} + +template +void ObjectDispatch::shut_down(Context* on_finish) { + auto cct = m_image_ctx->cct; + ldout(cct, 5) << dendl; + + m_image_ctx->op_work_queue->queue(on_finish, 0); +} + +template +bool ObjectDispatch::read( + const std::string &oid, uint64_t object_no, uint64_t object_off, + uint64_t object_len, librados::snap_t snap_id, int op_flags, + const ZTracer::Trace &parent_trace, ceph::bufferlist* read_data, + ExtentMap* extent_map, int* object_dispatch_flags, + DispatchResult* dispatch_result, Context** on_finish, + Context* on_dispatched) { + auto cct = m_image_ctx->cct; + ldout(cct, 20) << oid << " " << object_off << "~" << object_len << dendl; + + *dispatch_result = DISPATCH_RESULT_COMPLETE; + auto req = new ObjectReadRequest(m_image_ctx, oid, object_no, object_off, + object_len, snap_id, op_flags, + false, parent_trace, read_data, + extent_map, on_dispatched); + req->send(); + return true; +} + +template +bool ObjectDispatch::discard( + const std::string &oid, uint64_t object_no, uint64_t object_off, + uint64_t object_len, const ::SnapContext &snapc, int discard_flags, + const ZTracer::Trace &parent_trace, int* object_dispatch_flags, + uint64_t* journal_tid, DispatchResult* dispatch_result, + Context** on_finish, Context* on_dispatched) { + auto cct = m_image_ctx->cct; + ldout(cct, 20) << oid << " " << object_off << "~" << object_len << dendl; + + bool disable_clone_remove = ( + (discard_flags & OBJECT_DISCARD_FLAG_DISABLE_CLONE_REMOVE) != 0); + bool update_object_map = ( + (discard_flags & OBJECT_DISCARD_FLAG_DISABLE_OBJECT_MAP_UPDATE) == 0); + + *dispatch_result = DISPATCH_RESULT_COMPLETE; + auto req = new ObjectDiscardRequest(m_image_ctx, oid, object_no, + object_off, object_len, snapc, + disable_clone_remove, + update_object_map, parent_trace, + on_dispatched); + req->send(); + return true; +} + +template +bool ObjectDispatch::write( + const std::string &oid, uint64_t object_no, uint64_t object_off, + ceph::bufferlist&& data, const ::SnapContext &snapc, int op_flags, + const ZTracer::Trace &parent_trace, int* object_dispatch_flags, + uint64_t* journal_tid, DispatchResult* dispatch_result, + Context** on_finish, Context* on_dispatched) { + auto cct = m_image_ctx->cct; + ldout(cct, 20) << oid << " " << object_off << "~" << data.length() << dendl; + + *dispatch_result = DISPATCH_RESULT_COMPLETE; + auto req = new ObjectWriteRequest(m_image_ctx, oid, object_no, object_off, + std::move(data), snapc, op_flags, + parent_trace, on_dispatched); + req->send(); + return true; +} + +template +bool ObjectDispatch::write_same( + const std::string &oid, uint64_t object_no, uint64_t object_off, + uint64_t object_len, Extents&& buffer_extents, ceph::bufferlist&& data, + const ::SnapContext &snapc, int op_flags, + const ZTracer::Trace &parent_trace, int* object_dispatch_flags, + uint64_t* journal_tid, DispatchResult* dispatch_result, + Context** on_finish, Context* on_dispatched) { + auto cct = m_image_ctx->cct; + ldout(cct, 20) << oid << " " << object_off << "~" << object_len << dendl; + + *dispatch_result = DISPATCH_RESULT_COMPLETE; + auto req = new ObjectWriteSameRequest(m_image_ctx, oid, object_no, + object_off, object_len, + std::move(data), snapc, op_flags, + parent_trace, on_dispatched); + req->send(); + return true; +} + +template +bool ObjectDispatch::compare_and_write( + const std::string &oid, uint64_t object_no, uint64_t object_off, + ceph::bufferlist&& cmp_data, ceph::bufferlist&& write_data, + const ::SnapContext &snapc, int op_flags, + const ZTracer::Trace &parent_trace, uint64_t* mismatch_offset, + int* object_dispatch_flags, uint64_t* journal_tid, + DispatchResult* dispatch_result, Context** on_finish, + Context* on_dispatched) { + auto cct = m_image_ctx->cct; + ldout(cct, 20) << oid << " " << object_off << "~" << write_data.length() + << dendl; + + *dispatch_result = DISPATCH_RESULT_COMPLETE; + auto req = new ObjectCompareAndWriteRequest(m_image_ctx, oid, object_no, + object_off, + std::move(cmp_data), + std::move(write_data), snapc, + mismatch_offset, op_flags, + parent_trace, on_dispatched); + req->send(); + return true; +} + +} // namespace io +} // namespace librbd + +template class librbd::io::ObjectDispatch; diff --git a/src/librbd/io/ObjectDispatch.h b/src/librbd/io/ObjectDispatch.h new file mode 100644 index 0000000000000..32e8272d739ef --- /dev/null +++ b/src/librbd/io/ObjectDispatch.h @@ -0,0 +1,104 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_IO_OBJECT_DISPATCH_H +#define CEPH_LIBRBD_IO_OBJECT_DISPATCH_H + +#include "include/int_types.h" +#include "include/buffer.h" +#include "include/rados/librados.hpp" +#include "common/snap_types.h" +#include "common/zipkin_trace.h" +#include "librbd/io/Types.h" +#include "librbd/io/ObjectDispatchInterface.h" + +struct Context; + +namespace librbd { + +struct ImageCtx; + +namespace io { + +struct AioCompletion; + +template +class ObjectDispatch : public ObjectDispatchInterface { +public: + ObjectDispatch(ImageCtxT* image_ctx); + + ObjectDispatchLayer get_object_dispatch_layer() const override { + return OBJECT_DISPATCH_LAYER_CORE; + } + + void shut_down(Context* on_finish) override; + + bool read( + const std::string &oid, uint64_t object_no, uint64_t object_off, + uint64_t object_len, librados::snap_t snap_id, int op_flags, + const ZTracer::Trace &parent_trace, ceph::bufferlist* read_data, + ExtentMap* extent_map, int* object_dispatch_flags, + DispatchResult* dispatch_result, Context** on_finish, + Context* on_dispatched) override; + + bool discard( + const std::string &oid, uint64_t object_no, uint64_t object_off, + uint64_t object_len, const ::SnapContext &snapc, int discard_flags, + const ZTracer::Trace &parent_trace, int* object_dispatch_flags, + uint64_t* journal_tid, DispatchResult* dispatch_result, + Context** on_finish, Context* on_dispatched) override; + + bool write( + const std::string &oid, uint64_t object_no, uint64_t object_off, + ceph::bufferlist&& data, const ::SnapContext &snapc, int op_flags, + const ZTracer::Trace &parent_trace, int* object_dispatch_flags, + uint64_t* journal_tid, DispatchResult* dispatch_result, + Context** on_finish, Context* on_dispatched) override; + + bool write_same( + const std::string &oid, uint64_t object_no, uint64_t object_off, + uint64_t object_len, Extents&& buffer_extents, ceph::bufferlist&& data, + const ::SnapContext &snapc, int op_flags, + const ZTracer::Trace &parent_trace, int* object_dispatch_flags, + uint64_t* journal_tid, DispatchResult* dispatch_result, + Context** on_finish, Context* on_dispatched) override; + + bool compare_and_write( + const std::string &oid, uint64_t object_no, uint64_t object_off, + ceph::bufferlist&& cmp_data, ceph::bufferlist&& write_data, + const ::SnapContext &snapc, int op_flags, + const ZTracer::Trace &parent_trace, uint64_t* mismatch_offset, + int* object_dispatch_flags, uint64_t* journal_tid, + DispatchResult* dispatch_result, Context** on_finish, + Context* on_dispatched) override; + + bool flush( + FlushSource flush_source, const ZTracer::Trace &parent_trace, + DispatchResult* dispatch_result, Context** on_finish, + Context* on_dispatched) override { + return false; + } + + bool invalidate_cache(Context* on_finish) override { + return false; + } + bool reset_existence_cache(Context* on_finish) override { + return false; + } + + void extent_overwritten( + uint64_t object_no, uint64_t object_off, uint64_t object_len, + uint64_t journal_tid, uint64_t new_journal_tid) override { + } + +private: + ImageCtxT* m_image_ctx; + +}; + +} // namespace io +} // namespace librbd + +extern template class librbd::io::ObjectDispatch; + +#endif // CEPH_LIBRBD_IO_OBJECT_DISPATCH_H diff --git a/src/librbd/io/ObjectDispatchInterface.h b/src/librbd/io/ObjectDispatchInterface.h new file mode 100644 index 0000000000000..fddf0db1475ae --- /dev/null +++ b/src/librbd/io/ObjectDispatchInterface.h @@ -0,0 +1,86 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_IO_OBJECT_DISPATCH_INTERFACE_H +#define CEPH_LIBRBD_IO_OBJECT_DISPATCH_INTERFACE_H + +#include "include/int_types.h" +#include "include/buffer.h" +#include "include/rados/librados.hpp" +#include "common/snap_types.h" +#include "common/zipkin_trace.h" +#include "librbd/io/Types.h" + +struct Context; +struct RWLock; + +namespace librbd { +namespace io { + +struct AioCompletion; + +struct ObjectDispatchInterface { + virtual ~ObjectDispatchInterface() { + } + + virtual ObjectDispatchLayer get_object_dispatch_layer() const = 0; + + virtual void shut_down(Context* on_finish) = 0; + + virtual bool read( + const std::string &oid, uint64_t object_no, uint64_t object_off, + uint64_t object_len, librados::snap_t snap_id, int op_flags, + const ZTracer::Trace &parent_trace, ceph::bufferlist* read_data, + ExtentMap* extent_map, int* object_dispatch_flags, + DispatchResult* dispatch_result, Context** on_finish, + Context* on_dispatched) = 0; + + virtual bool discard( + const std::string &oid, uint64_t object_no, uint64_t object_off, + uint64_t object_len, const ::SnapContext &snapc, int discard_flags, + const ZTracer::Trace &parent_trace, int* object_dispatch_flags, + uint64_t* journal_tid, DispatchResult* dispatch_result, + Context**on_finish, Context* on_dispatched) = 0; + + virtual bool write( + const std::string &oid, uint64_t object_no, uint64_t object_off, + ceph::bufferlist&& data, const ::SnapContext &snapc, int op_flags, + const ZTracer::Trace &parent_trace, int* object_dispatch_flags, + uint64_t* journal_tid, DispatchResult* dispatch_result, + Context**on_finish, Context* on_dispatched) = 0; + + virtual bool write_same( + const std::string &oid, uint64_t object_no, uint64_t object_off, + uint64_t object_len, Extents&& buffer_extents, ceph::bufferlist&& data, + const ::SnapContext &snapc, int op_flags, + const ZTracer::Trace &parent_trace, int* object_dispatch_flags, + uint64_t* journal_tid, DispatchResult* dispatch_result, + Context**on_finish, Context* on_dispatched) = 0; + + virtual bool compare_and_write( + const std::string &oid, uint64_t object_no, uint64_t object_off, + ceph::bufferlist&& cmp_data, ceph::bufferlist&& write_data, + const ::SnapContext &snapc, int op_flags, + const ZTracer::Trace &parent_trace, uint64_t* mismatch_offset, + int* object_dispatch_flags, uint64_t* journal_tid, + DispatchResult* dispatch_result, Context** on_finish, + Context* on_dispatched) = 0; + + virtual bool flush( + FlushSource flush_source, const ZTracer::Trace &parent_trace, + DispatchResult* dispatch_result, Context** on_finish, + Context* on_dispatched) = 0; + + virtual bool invalidate_cache(Context* on_finish) = 0; + virtual bool reset_existence_cache(Context* on_finish) = 0; + + virtual void extent_overwritten( + uint64_t object_no, uint64_t object_off, uint64_t object_len, + uint64_t journal_tid, uint64_t new_journal_tid) = 0; + +}; + +} // namespace io +} // namespace librbd + +#endif // CEPH_LIBRBD_IO_OBJECT_DISPATCH_INTERFACE_H diff --git a/src/librbd/io/ObjectDispatcher.cc b/src/librbd/io/ObjectDispatcher.cc index 94620b228e518..881e76cb02cc9 100644 --- a/src/librbd/io/ObjectDispatcher.cc +++ b/src/librbd/io/ObjectDispatcher.cc @@ -2,87 +2,350 @@ // vim: ts=8 sw=2 smarttab #include "librbd/io/ObjectDispatcher.h" +#include "include/Context.h" +#include "common/AsyncOpTracker.h" +#include "common/dout.h" +#include "common/WorkQueue.h" #include "librbd/ImageCtx.h" +#include "librbd/Utils.h" +#include "librbd/io/ObjectDispatch.h" #include "librbd/io/ObjectDispatchSpec.h" -#include "librbd/io/ObjectRequest.h" #include +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "librbd::io::ObjectDispatcher: " << this \ + << " " << __func__ << ": " + namespace librbd { namespace io { -// TODO temporary implementation template -struct ObjectDispatcher::SendVisitor - : public boost::static_visitor { +struct ObjectDispatcher::C_LayerIterator : public Context { ObjectDispatcher* object_dispatcher; + Context* on_finish; + + ObjectDispatchLayer object_dispatch_layer = OBJECT_DISPATCH_LAYER_NONE; + + C_LayerIterator(ObjectDispatcher* object_dispatcher, + Context* on_finish) + : object_dispatcher(object_dispatcher), on_finish(on_finish) { + } + + void complete(int r) override { + while (true) { + object_dispatcher->m_lock.get_read(); + auto it = object_dispatcher->m_object_dispatches.upper_bound( + object_dispatch_layer); + if (it == object_dispatcher->m_object_dispatches.end()) { + object_dispatcher->m_lock.put_read(); + Context::complete(r); + return; + } + + auto& object_dispatch_meta = it->second; + auto object_dispatch = object_dispatch_meta.object_dispatch; + + // prevent recursive locking back into the dispatcher while handling IO + object_dispatch_meta.async_op_tracker->start_op(); + object_dispatcher->m_lock.put_read(); + + // next loop should start after current layer + object_dispatch_layer = object_dispatch->get_object_dispatch_layer(); + + auto handled = execute(object_dispatch, this); + object_dispatch_meta.async_op_tracker->finish_op(); + + if (handled) { + break; + } + } + } + + void finish(int r) override { + on_finish->complete(0); + } + + virtual bool execute(ObjectDispatchInterface* object_dispatch, + Context* on_finish) = 0; +}; + +template +struct ObjectDispatcher::C_InvalidateCache : public C_LayerIterator { + C_InvalidateCache(ObjectDispatcher* object_dispatcher, Context* on_finish) + : C_LayerIterator(object_dispatcher, on_finish) { + } + + bool execute(ObjectDispatchInterface* object_dispatch, + Context* on_finish) override { + return object_dispatch->invalidate_cache(on_finish); + } +}; + +template +struct ObjectDispatcher::C_ResetExistenceCache : public C_LayerIterator { + C_ResetExistenceCache(ObjectDispatcher* object_dispatcher, Context* on_finish) + : C_LayerIterator(object_dispatcher, on_finish) { + } + + bool execute(ObjectDispatchInterface* object_dispatch, + Context* on_finish) override { + return object_dispatch->reset_existence_cache(on_finish); + } +}; + +template +struct ObjectDispatcher::SendVisitor : public boost::static_visitor { + ObjectDispatchInterface* object_dispatch; ObjectDispatchSpec* object_dispatch_spec; - SendVisitor(ObjectDispatcher* object_dispatcher, + SendVisitor(ObjectDispatchInterface* object_dispatch, ObjectDispatchSpec* object_dispatch_spec) - : object_dispatcher(object_dispatcher), - object_dispatch_spec(object_dispatch_spec) { + : object_dispatch(object_dispatch), + object_dispatch_spec(object_dispatch_spec) { } - void operator()(ObjectDispatchSpec::ReadRequest& read) const { - auto req = ObjectReadRequest::create( - object_dispatcher->m_image_ctx, read.oid, read.object_no, read.object_off, - read.object_len, read.snap_id, object_dispatch_spec->op_flags, false, - object_dispatch_spec->parent_trace, read.read_data, read.extent_map, + bool operator()(ObjectDispatchSpec::ReadRequest& read) const { + return object_dispatch->read( + read.oid, read.object_no, read.object_off, read.object_len, read.snap_id, + object_dispatch_spec->op_flags, object_dispatch_spec->parent_trace, + read.read_data, read.extent_map, + &object_dispatch_spec->object_dispatch_flags, + &object_dispatch_spec->dispatch_result, + &object_dispatch_spec->dispatcher_ctx.on_finish, &object_dispatch_spec->dispatcher_ctx); - req->send(); } - void operator()(ObjectDispatchSpec::DiscardRequest& discard) const { - auto req = ObjectRequest::create_discard( - object_dispatcher->m_image_ctx, discard.oid, discard.object_no, - discard.object_off, discard.object_len, discard.snapc, true, true, - object_dispatch_spec->parent_trace, + bool operator()(ObjectDispatchSpec::DiscardRequest& discard) const { + return object_dispatch->discard( + discard.oid, discard.object_no, discard.object_off, discard.object_len, + discard.snapc, discard.discard_flags, object_dispatch_spec->parent_trace, + &object_dispatch_spec->object_dispatch_flags, &discard.journal_tid, + &object_dispatch_spec->dispatch_result, + &object_dispatch_spec->dispatcher_ctx.on_finish, &object_dispatch_spec->dispatcher_ctx); - req->send(); } - void operator()(ObjectDispatchSpec::WriteRequest& write) const { - auto req = ObjectRequest::create_write( - object_dispatcher->m_image_ctx, write.oid, write.object_no, - write.object_off, std::move(write.data), write.snapc, - object_dispatch_spec->op_flags, object_dispatch_spec->parent_trace, + bool operator()(ObjectDispatchSpec::WriteRequest& write) const { + return object_dispatch->write( + write.oid, write.object_no, write.object_off, std::move(write.data), + write.snapc, object_dispatch_spec->op_flags, + object_dispatch_spec->parent_trace, + &object_dispatch_spec->object_dispatch_flags, &write.journal_tid, + &object_dispatch_spec->dispatch_result, + &object_dispatch_spec->dispatcher_ctx.on_finish, &object_dispatch_spec->dispatcher_ctx); - req->send(); } - void operator()(ObjectDispatchSpec::WriteSameRequest& write_same) const { - auto req = ObjectRequest::create_write_same( - object_dispatcher->m_image_ctx, write_same.oid, write_same.object_no, - write_same.object_off, write_same.object_len, std::move(write_same.data), - write_same.snapc, object_dispatch_spec->op_flags, - object_dispatch_spec->parent_trace, + bool operator()(ObjectDispatchSpec::WriteSameRequest& write_same) const { + return object_dispatch->write_same( + write_same.oid, write_same.object_no, write_same.object_off, + write_same.object_len, std::move(write_same.buffer_extents), + std::move(write_same.data), write_same.snapc, + object_dispatch_spec->op_flags, object_dispatch_spec->parent_trace, + &object_dispatch_spec->object_dispatch_flags, &write_same.journal_tid, + &object_dispatch_spec->dispatch_result, + &object_dispatch_spec->dispatcher_ctx.on_finish, &object_dispatch_spec->dispatcher_ctx); - req->send(); } - void operator()( + bool operator()( ObjectDispatchSpec::CompareAndWriteRequest& compare_and_write) const { - auto req = ObjectRequest::create_compare_and_write( - object_dispatcher->m_image_ctx, compare_and_write.oid, - compare_and_write.object_no, compare_and_write.object_off, - std::move(compare_and_write.cmp_data), std::move(compare_and_write.data), - compare_and_write.snapc, compare_and_write.mismatch_offset, + return object_dispatch->compare_and_write( + compare_and_write.oid, compare_and_write.object_no, + compare_and_write.object_off, std::move(compare_and_write.cmp_data), + std::move(compare_and_write.data), compare_and_write.snapc, object_dispatch_spec->op_flags, object_dispatch_spec->parent_trace, + compare_and_write.mismatch_offset, + &object_dispatch_spec->object_dispatch_flags, + &compare_and_write.journal_tid, + &object_dispatch_spec->dispatch_result, + &object_dispatch_spec->dispatcher_ctx.on_finish, &object_dispatch_spec->dispatcher_ctx); - req->send(); } - void operator()( - ObjectDispatchSpec::FlushRequest& flush) const { + bool operator()(ObjectDispatchSpec::FlushRequest& flush) const { + return object_dispatch->flush( + flush.flush_source, object_dispatch_spec->parent_trace, + &object_dispatch_spec->dispatch_result, + &object_dispatch_spec->dispatcher_ctx.on_finish, + &object_dispatch_spec->dispatcher_ctx); } }; +template +ObjectDispatcher::ObjectDispatcher(I* image_ctx) + : m_image_ctx(image_ctx), + m_lock(librbd::util::unique_lock_name("librbd::io::ObjectDispatcher::lock", + this)) { + // configure the core object dispatch handler on startup + auto object_dispatch = new ObjectDispatch(image_ctx); + m_object_dispatches[object_dispatch->get_object_dispatch_layer()] = + {object_dispatch, new AsyncOpTracker()}; +} + +template +ObjectDispatcher::~ObjectDispatcher() { + assert(m_object_dispatches.empty()); +} + +template +void ObjectDispatcher::shut_down(Context* on_finish) { + auto cct = m_image_ctx->cct; + ldout(cct, 5) << dendl; + + std::map object_dispatches; + { + RWLock::WLocker locker(m_lock); + std::swap(object_dispatches, m_object_dispatches); + } + + for (auto it : object_dispatches) { + shut_down_object_dispatch(it.second, &on_finish); + } + on_finish->complete(0); +} + +template +void ObjectDispatcher::register_object_dispatch( + ObjectDispatchInterface* object_dispatch) { + auto cct = m_image_ctx->cct; + auto type = object_dispatch->get_object_dispatch_layer(); + ldout(cct, 5) << "object_dispatch_layer=" << type << dendl; + + RWLock::WLocker locker(m_lock); + assert(type < OBJECT_DISPATCH_LAYER_LAST); + + auto result = m_object_dispatches.insert( + {type, {object_dispatch, new AsyncOpTracker()}}); + assert(result.second); +} + +template +void ObjectDispatcher::shut_down_object_dispatch( + ObjectDispatchLayer object_dispatch_layer, Context* on_finish) { + auto cct = m_image_ctx->cct; + ldout(cct, 5) << "object_dispatch_layer=" << object_dispatch_layer << dendl; + assert(object_dispatch_layer + 1 < OBJECT_DISPATCH_LAYER_LAST); + + ObjectDispatchMeta object_dispatch_meta; + { + RWLock::WLocker locker(m_lock); + auto it = m_object_dispatches.find(object_dispatch_layer); + assert(it != m_object_dispatches.end()); + + object_dispatch_meta = it->second; + m_object_dispatches.erase(it); + } + + shut_down_object_dispatch(object_dispatch_meta, &on_finish); + on_finish->complete(0); +} + +template +void ObjectDispatcher::shut_down_object_dispatch( + ObjectDispatchMeta& object_dispatch_meta, Context** on_finish) { + auto object_dispatch = object_dispatch_meta.object_dispatch; + auto async_op_tracker = object_dispatch_meta.async_op_tracker; + + Context* ctx = *on_finish; + ctx = new FunctionContext( + [object_dispatch, async_op_tracker, ctx](int r) { + delete object_dispatch; + delete async_op_tracker; + + ctx->complete(r); + }); + ctx = new FunctionContext([object_dispatch, ctx](int r) { + object_dispatch->shut_down(ctx); + }); + *on_finish = new FunctionContext([async_op_tracker, ctx](int r) { + async_op_tracker->wait_for_ops(ctx); + }); +} + +template +void ObjectDispatcher::invalidate_cache(Context* on_finish) { + auto cct = m_image_ctx->cct; + ldout(cct, 5) << dendl; + + on_finish = util::create_async_context_callback(*m_image_ctx, on_finish); + auto ctx = new C_InvalidateCache(this, on_finish); + ctx->complete(0); +} + +template +void ObjectDispatcher::reset_existence_cache(Context* on_finish) { + auto cct = m_image_ctx->cct; + ldout(cct, 5) << dendl; + + on_finish = util::create_async_context_callback(*m_image_ctx, on_finish); + auto ctx = new C_ResetExistenceCache(this, on_finish); + ctx->complete(0); +} + +template +void ObjectDispatcher::extent_overwritten( + uint64_t object_no, uint64_t object_off, uint64_t object_len, + uint64_t journal_tid, uint64_t new_journal_tid) { + auto cct = m_image_ctx->cct; + ldout(cct, 20) << object_no << " " << object_off << "~" << object_len + << dendl; + + for (auto it : m_object_dispatches) { + auto& object_dispatch_meta = it.second; + auto object_dispatch = object_dispatch_meta.object_dispatch; + object_dispatch->extent_overwritten(object_no, object_off, object_len, + journal_tid, new_journal_tid); + } +} + template void ObjectDispatcher::send(ObjectDispatchSpec* object_dispatch_spec) { - // TODO temporary implementation - object_dispatch_spec->dispatch_result = DISPATCH_RESULT_COMPLETE; - boost::apply_visitor(SendVisitor{this, object_dispatch_spec}, - object_dispatch_spec->request); + auto cct = m_image_ctx->cct; + ldout(cct, 20) << "object_dispatch_spec=" << object_dispatch_spec << dendl; + + auto object_dispatch_layer = object_dispatch_spec->object_dispatch_layer; + assert(object_dispatch_layer + 1 < OBJECT_DISPATCH_LAYER_LAST); + + // apply the IO request to all layers -- this method will be re-invoked + // by the dispatch layer if continuing / restarting the IO + while (true) { + m_lock.get_read(); + object_dispatch_layer = object_dispatch_spec->object_dispatch_layer; + auto it = m_object_dispatches.upper_bound(object_dispatch_layer); + if (it == m_object_dispatches.end()) { + // the request is complete if handled by all layers + object_dispatch_spec->dispatch_result = DISPATCH_RESULT_COMPLETE; + m_lock.put_read(); + break; + } + + auto& object_dispatch_meta = it->second; + auto object_dispatch = object_dispatch_meta.object_dispatch; + object_dispatch_spec->dispatch_result = DISPATCH_RESULT_INVALID; + + // prevent recursive locking back into the dispatcher while handling IO + object_dispatch_meta.async_op_tracker->start_op(); + m_lock.put_read(); + + // advance to next layer in case we skip or continue + object_dispatch_spec->object_dispatch_layer = + object_dispatch->get_object_dispatch_layer(); + + bool handled = boost::apply_visitor( + SendVisitor{object_dispatch, object_dispatch_spec}, + object_dispatch_spec->request); + object_dispatch_meta.async_op_tracker->finish_op(); + + // handled ops will resume when the dispatch ctx is invoked + if (handled) { + return; + } + } + + // skipped through to the last layer + object_dispatch_spec->dispatcher_ctx.complete(0); } } // namespace io diff --git a/src/librbd/io/ObjectDispatcher.h b/src/librbd/io/ObjectDispatcher.h index 763c99625efc2..0370d2684f325 100644 --- a/src/librbd/io/ObjectDispatcher.h +++ b/src/librbd/io/ObjectDispatcher.h @@ -5,7 +5,12 @@ #define CEPH_LIBRBD_IO_OBJECT_DISPATCHER_H #include "include/int_types.h" +#include "common/RWLock.h" #include "librbd/io/Types.h" +#include + +struct AsyncOpTracker; +struct Context; namespace librbd { @@ -13,6 +18,7 @@ struct ImageCtx; namespace io { +struct ObjectDispatchInterface; struct ObjectDispatchSpec; struct ObjectDispatcherInterface { @@ -29,16 +35,50 @@ private: template class ObjectDispatcher : public ObjectDispatcherInterface { public: - ObjectDispatcher(ImageCtxT* image_ctx) : m_image_ctx(image_ctx) { - } + ObjectDispatcher(ImageCtxT* image_ctx); + ~ObjectDispatcher(); + + void shut_down(Context* on_finish); + + void register_object_dispatch(ObjectDispatchInterface* object_dispatch); + void shut_down_object_dispatch(ObjectDispatchLayer object_dispatch_layer, + Context* on_finish); + + void invalidate_cache(Context* on_finish); + void reset_existence_cache(Context* on_finish); + + void extent_overwritten( + uint64_t object_no, uint64_t object_off, uint64_t object_len, + uint64_t journal_tid, uint64_t new_journal_tid); private: + struct ObjectDispatchMeta { + ObjectDispatchInterface* object_dispatch = nullptr; + AsyncOpTracker* async_op_tracker = nullptr; + + ObjectDispatchMeta() { + } + ObjectDispatchMeta(ObjectDispatchInterface* object_dispatch, + AsyncOpTracker* async_op_tracker) + : object_dispatch(object_dispatch), async_op_tracker(async_op_tracker) { + } + }; + + struct C_LayerIterator; + struct C_InvalidateCache; + struct C_ResetExistenceCache; struct SendVisitor; ImageCtxT* m_image_ctx; + RWLock m_lock; + std::map m_object_dispatches; + void send(ObjectDispatchSpec* object_dispatch_spec); + void shut_down_object_dispatch(ObjectDispatchMeta& object_dispatch_meta, + Context** on_finish); + }; } // namespace io diff --git a/src/librbd/io/Types.h b/src/librbd/io/Types.h index 52a85cedbe9ab..b32a5372d66bb 100644 --- a/src/librbd/io/Types.h +++ b/src/librbd/io/Types.h @@ -55,6 +55,11 @@ enum { OBJECT_DISCARD_FLAG_DISABLE_OBJECT_MAP_UPDATE = 1UL << 1 }; +enum { + OBJECT_DISPATCH_FLAG_FLUSH = 1UL << 0, + OBJECT_DISPATCH_FLAG_WILL_RETRY_ON_ERROR = 1UL << 1 +}; + typedef std::vector > Extents; typedef std::map ExtentMap; diff --git a/src/librbd/operation/TrimRequest.cc b/src/librbd/operation/TrimRequest.cc index dcf4956a6060f..7fc6140629393 100644 --- a/src/librbd/operation/TrimRequest.cc +++ b/src/librbd/operation/TrimRequest.cc @@ -8,7 +8,8 @@ #include "librbd/internal.h" #include "librbd/ObjectMap.h" #include "librbd/Utils.h" -#include "librbd/io/ObjectRequest.h" +#include "librbd/io/ObjectDispatchSpec.h" +#include "librbd/io/ObjectDispatcher.h" #include "common/ContextCompletion.h" #include "common/dout.h" #include "common/errno.h" @@ -45,10 +46,11 @@ public: string oid = image_ctx.get_object_name(m_object_no); ldout(image_ctx.cct, 10) << "removing (with copyup) " << oid << dendl; - auto req = io::ObjectRequest::create_discard( - &image_ctx, oid, m_object_no, 0, image_ctx.layout.object_size, m_snapc, - false, false, {}, this); - req->send(); + auto object_dispatch_spec = io::ObjectDispatchSpec::create_discard( + &image_ctx, io::OBJECT_DISPATCH_LAYER_NONE, oid, m_object_no, 0, + image_ctx.layout.object_size, m_snapc, + io::OBJECT_DISCARD_FLAG_DISABLE_OBJECT_MAP_UPDATE, 0, {}, this); + object_dispatch_spec->send(0); return 0; } private: @@ -343,11 +345,11 @@ void TrimRequest::send_clean_boundary() { // treat as a full object delete on the boundary p->length = image_ctx.layout.object_size; } - auto req = io::ObjectRequest::create_discard(&image_ctx, p->oid.name, - p->objectno, p->offset, - p->length, snapc, false, - true, {}, req_comp); - req->send(); + + auto object_dispatch_spec = io::ObjectDispatchSpec::create_discard( + &image_ctx, io::OBJECT_DISPATCH_LAYER_NONE, p->oid.name, p->objectno, + p->offset, p->length, snapc, 0, 0, {}, req_comp); + object_dispatch_spec->send(0); } completion->finish_adding_requests(); } diff --git a/src/test/librbd/io/test_mock_ImageRequest.cc b/src/test/librbd/io/test_mock_ImageRequest.cc index 34d2cd95a1008..1d22396ce72f1 100644 --- a/src/test/librbd/io/test_mock_ImageRequest.cc +++ b/src/test/librbd/io/test_mock_ImageRequest.cc @@ -50,50 +50,6 @@ inline ImageCtx *get_image_ctx(MockTestImageCtx *image_ctx) { } } // namespace util - -namespace io { - -template <> -struct ObjectReadRequest { - typedef std::vector > Extents; - typedef std::map ExtentMap; - - static ObjectReadRequest* s_instance; - Context* on_finish = nullptr; - - static ObjectReadRequest* create(librbd::MockTestImageCtx *ictx, - const std::string &oid, - uint64_t objectno, uint64_t offset, - uint64_t len, Extents &buffer_extents, - librados::snap_t snap_id, int op_flags, - const ZTracer::Trace &parent_trace, - Context *completion) { - assert(s_instance != nullptr); - s_instance->on_finish = completion; - return s_instance; - } - - ObjectReadRequest() { - assert(s_instance == nullptr); - s_instance = this; - } - ~ObjectReadRequest() { - s_instance = nullptr; - } - - MOCK_METHOD0(send, void()); - - MOCK_CONST_METHOD0(get_offset, uint64_t()); - MOCK_CONST_METHOD0(get_length, uint64_t()); - MOCK_METHOD0(data, ceph::bufferlist &()); - MOCK_CONST_METHOD0(get_buffer_extents, const Extents &()); - MOCK_METHOD0(get_extent_map, ExtentMap &()); - -}; - -ObjectReadRequest* ObjectReadRequest::s_instance = nullptr; - -} // namespace io } // namespace librbd #include "librbd/io/ImageRequest.cc" @@ -115,7 +71,6 @@ struct TestMockIoImageRequest : public TestMockFixture { typedef ImageFlushRequest MockImageFlushRequest; typedef ImageWriteSameRequest MockImageWriteSameRequest; typedef ImageCompareAndWriteRequest MockImageCompareAndWriteRequest; - typedef ObjectReadRequest MockObjectReadRequest; void expect_is_journal_appending(MockTestJournal &mock_journal, bool appending) { EXPECT_CALL(mock_journal, is_journal_appending()) diff --git a/src/test/librbd/mock/io/MockObjectDispatch.h b/src/test/librbd/mock/io/MockObjectDispatch.h new file mode 100644 index 0000000000000..5f308dab7bdc7 --- /dev/null +++ b/src/test/librbd/mock/io/MockObjectDispatch.h @@ -0,0 +1,121 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_TEST_LIBRBD_MOCK_IO_OBJECT_DISPATCH_H +#define CEPH_TEST_LIBRBD_MOCK_IO_OBJECT_DISPATCH_H + +#include "gmock/gmock.h" +#include "librbd/io/ObjectDispatchInterface.h" +#include "librbd/io/Types.h" + +class Context; + +namespace librbd { +namespace io { + +struct MockObjectDispatch : public ObjectDispatchInterface { +public: + RWLock lock; + + MockObjectDispatch() : lock("MockObjectDispatch::lock", true, false) { + } + + MOCK_CONST_METHOD0(get_object_dispatch_layer, ObjectDispatchLayer()); + + MOCK_METHOD1(shut_down, void(Context*)); + + MOCK_METHOD8(execute_read, + bool(uint64_t, uint64_t, uint64_t, librados::snap_t, + ceph::bufferlist*, ExtentMap*, DispatchResult*, Context*)); + bool read( + const std::string& oid, uint64_t object_no, uint64_t object_off, + uint64_t object_len, librados::snap_t snap_id, int op_flags, + const ZTracer::Trace& parent_trace, ceph::bufferlist* read_data, + ExtentMap* extent_map, int* dispatch_flags, + DispatchResult* dispatch_result, Context** on_finish, + Context* on_dispatched) { + return execute_read(object_no, object_off, object_len, snap_id, read_data, + extent_map, dispatch_result, on_dispatched); + } + + MOCK_METHOD9(execute_discard, + bool(uint64_t, uint64_t, uint64_t, const ::SnapContext &, int, + int*, uint64_t*, DispatchResult*, Context*)); + bool discard( + const std::string &oid, uint64_t object_no, uint64_t object_off, + uint64_t object_len, const ::SnapContext &snapc, int discard_flags, + const ZTracer::Trace &parent_trace, int* dispatch_flags, + uint64_t* journal_tid, DispatchResult* dispatch_result, + Context** on_finish, Context* on_dispatched) { + return execute_discard(object_no, object_off, object_len, snapc, + discard_flags, dispatch_flags, journal_tid, + dispatch_result, on_dispatched); + } + + MOCK_METHOD8(execute_write, + bool(uint64_t, uint64_t, const ceph::bufferlist&, + const ::SnapContext &, int*, uint64_t*, DispatchResult*, + Context *)); + bool write( + const std::string &oid, uint64_t object_no, uint64_t object_off, + ceph::bufferlist&& data, const ::SnapContext &snapc, int op_flags, + const ZTracer::Trace &parent_trace, int* dispatch_flags, + uint64_t* journal_tid, DispatchResult* dispatch_result, + Context** on_finish, Context* on_dispatched) override { + return execute_write(object_no, object_off, data, snapc, dispatch_flags, + journal_tid, dispatch_result, on_dispatched); + } + + MOCK_METHOD10(execute_write_same, + bool(uint64_t, uint64_t, uint64_t, const Extents&, + const ceph::bufferlist&, const ::SnapContext &, int*, + uint64_t*, DispatchResult*, Context *)); + bool write_same( + const std::string &oid, uint64_t object_no, uint64_t object_off, + uint64_t object_len, Extents&& buffer_extents, ceph::bufferlist&& data, + const ::SnapContext &snapc, int op_flags, + const ZTracer::Trace &parent_trace, int* dispatch_flags, + uint64_t* journal_tid, DispatchResult* dispatch_result, + Context* *on_finish, Context* on_dispatched) override { + return execute_write_same(object_no, object_off, object_len, buffer_extents, + data, snapc, dispatch_flags, journal_tid, + dispatch_result, on_dispatched); + } + + MOCK_METHOD9(execute_compare_and_write, + bool(uint64_t, uint64_t, const ceph::bufferlist&, + const ceph::bufferlist&, uint64_t*, int*, uint64_t*, + DispatchResult*, Context *)); + bool compare_and_write( + const std::string &oid, uint64_t object_no, uint64_t object_off, + ceph::bufferlist&& cmp_data, ceph::bufferlist&& write_data, + const ::SnapContext &snapc, int op_flags, + const ZTracer::Trace &parent_trace, uint64_t* mismatch_offset, + int* dispatch_flags, uint64_t* journal_tid, + DispatchResult* dispatch_result, Context** on_finish, + Context* on_dispatched) override { + return execute_compare_and_write(object_no, object_off, cmp_data, + write_data, mismatch_offset, + dispatch_flags, journal_tid, + dispatch_result, on_dispatched); + } + + MOCK_METHOD3(execute_flush, bool(FlushSource, DispatchResult*, + Context*)); + bool flush(FlushSource flush_source, const ZTracer::Trace &parent_trace, + DispatchResult* dispatch_result, Context** on_finish, + Context* on_dispatched) { + return execute_flush(flush_source, dispatch_result, on_dispatched); + } + + MOCK_METHOD1(invalidate_cache, bool(Context*)); + MOCK_METHOD1(reset_existence_cache, bool(Context*)); + + MOCK_METHOD5(extent_overwritten, void(uint64_t, uint64_t, uint64_t, uint64_t, + uint64_t)); +}; + +} // namespace io +} // namespace librbd + +#endif // CEPH_TEST_LIBRBD_MOCK_IO_OBJECT_DISPATCH_H diff --git a/src/test/librbd/mock/io/MockObjectDispatcher.h b/src/test/librbd/mock/io/MockObjectDispatcher.h index 1c8f9b96a93df..271c30102a322 100644 --- a/src/test/librbd/mock/io/MockObjectDispatcher.h +++ b/src/test/librbd/mock/io/MockObjectDispatcher.h @@ -15,8 +15,23 @@ class Context; namespace librbd { namespace io { +struct ObjectDispatchInterface; + struct MockObjectDispatcher : public ObjectDispatcherInterface { public: + MOCK_METHOD1(shut_down, void(Context*)); + + MOCK_METHOD1(register_object_dispatch, void(ObjectDispatchInterface*)); + MOCK_METHOD2(shut_down_object_dispatch, void(ObjectDispatchLayer, Context*)); + + MOCK_METHOD2(flush, void(FlushSource, Context*)); + + MOCK_METHOD1(invalidate_cache, void(Context*)); + MOCK_METHOD1(reset_existance_cache, void(Context*)); + + MOCK_METHOD5(extent_overwritten, void(uint64_t, uint64_t, uint64_t, uint64_t, + uint64_t)); + MOCK_METHOD1(send, void(ObjectDispatchSpec*)); }; diff --git a/src/test/librbd/operation/test_mock_TrimRequest.cc b/src/test/librbd/operation/test_mock_TrimRequest.cc index d66681005217e..0203395929add 100644 --- a/src/test/librbd/operation/test_mock_TrimRequest.cc +++ b/src/test/librbd/operation/test_mock_TrimRequest.cc @@ -4,16 +4,17 @@ #include "test/librbd/test_mock_fixture.h" #include "test/librbd/test_support.h" #include "test/librbd/mock/MockImageCtx.h" +#include "test/librbd/mock/io/MockObjectDispatch.h" #include "test/librados_test_stub/MockTestMemIoCtxImpl.h" #include "common/bit_vector.hpp" #include "librbd/AsyncRequest.h" #include "librbd/internal.h" #include "librbd/ObjectMap.h" #include "librbd/Utils.h" -#include "librbd/io/ObjectRequest.h" #include "librbd/operation/TrimRequest.h" #include "gmock/gmock.h" #include "gtest/gtest.h" +#include namespace librbd { namespace { @@ -66,38 +67,20 @@ struct AsyncRequest { namespace io { -template <> -struct ObjectRequest { - static ObjectRequest* s_instance; - Context *on_finish = nullptr; - - static ObjectRequest* create_discard(librbd::MockTestImageCtx *ictx, - const std::string &oid, - uint64_t object_no, - uint64_t object_off, - uint64_t object_len, - const ::SnapContext &snapc, - bool disable_remove_on_clone, - bool update_object_map, - const ZTracer::Trace &parent_trace, - Context *completion) { - assert(s_instance != nullptr); - EXPECT_FALSE(disable_remove_on_clone); - s_instance->on_finish = completion; - s_instance->construct(object_off, object_len, update_object_map); - return s_instance; +struct DiscardVisitor + : public boost::static_visitor { + ObjectDispatchSpec::DiscardRequest* + operator()(ObjectDispatchSpec::DiscardRequest& discard) const { + return &discard; } - ObjectRequest() { - s_instance = this; + template + ObjectDispatchSpec::DiscardRequest* + operator()(T& t) const { + return nullptr; } - - MOCK_METHOD3(construct, void(uint64_t, uint64_t, bool)); - MOCK_METHOD0(send, void()); }; -ObjectRequest* ObjectRequest::s_instance = nullptr; - } // namespace io } // namespace librbd @@ -119,7 +102,6 @@ using ::testing::WithArg; class TestMockOperationTrimRequest : public TestMockFixture { public: typedef TrimRequest MockTrimRequest; - typedef librbd::io::ObjectRequest MockObjectRequest; int create_snapshot(const char *snap_name) { librbd::ImageCtx *ictx; @@ -195,15 +177,25 @@ public: } void expect_object_discard(MockImageCtx &mock_image_ctx, - MockObjectRequest &mock_object_request, + io::MockObjectDispatch& mock_io_object_dispatch, uint64_t offset, uint64_t length, - bool update_object_map, int ret_val) { - EXPECT_CALL(mock_object_request, construct(offset, length, - update_object_map)); - EXPECT_CALL(mock_object_request, send()) - .WillOnce(Invoke([&mock_image_ctx, &mock_object_request, ret_val]() { - mock_image_ctx.op_work_queue->queue(mock_object_request.on_finish, ret_val); - })); + bool update_object_map, int r) { + EXPECT_CALL(*mock_image_ctx.io_object_dispatcher, send(_)) + .WillOnce(Invoke([&mock_image_ctx, offset, length, update_object_map, r] + (io::ObjectDispatchSpec* spec) { + auto discard = boost::apply_visitor(io::DiscardVisitor{}, spec->request); + ASSERT_TRUE(discard != nullptr); + ASSERT_EQ(offset, discard->object_off); + ASSERT_EQ(length, discard->object_len); + int flags = 0; + if (!update_object_map) { + flags = io::OBJECT_DISCARD_FLAG_DISABLE_OBJECT_MAP_UPDATE; + } + ASSERT_EQ(flags, discard->discard_flags); + + spec->dispatch_result = io::DISPATCH_RESULT_COMPLETE; + mock_image_ctx.op_work_queue->queue(&spec->dispatcher_ctx, r); + })); } }; @@ -284,11 +276,10 @@ TEST_F(TestMockOperationTrimRequest, SuccessCopyUp) { true, 0); // copy-up + io::MockObjectDispatch mock_io_object_dispatch; expect_get_parent_overlap(mock_image_ctx, ictx->get_object_size()); expect_get_object_name(mock_image_ctx, 0, "object0"); - - MockObjectRequest mock_object_request; - expect_object_discard(mock_image_ctx, mock_object_request, 0, + expect_object_discard(mock_image_ctx, mock_io_object_dispatch, 0, ictx->get_object_size(), false, 0); // remove @@ -329,8 +320,8 @@ TEST_F(TestMockOperationTrimRequest, SuccessBoundary) { EXPECT_CALL(mock_image_ctx, get_stripe_count()).WillOnce(Return(ictx->get_stripe_count())); // boundary - MockObjectRequest mock_object_request; - expect_object_discard(mock_image_ctx, mock_object_request, 1, + io::MockObjectDispatch mock_io_object_dispatch; + expect_object_discard(mock_image_ctx, mock_io_object_dispatch, 1, ictx->get_object_size() - 1, true, 0); C_SaferCond cond_ctx; @@ -429,11 +420,10 @@ TEST_F(TestMockOperationTrimRequest, CopyUpError) { false, 0); // copy-up + io::MockObjectDispatch mock_io_object_dispatch; expect_get_parent_overlap(mock_image_ctx, ictx->get_object_size()); expect_get_object_name(mock_image_ctx, 0, "object0"); - - MockObjectRequest mock_object_request; - expect_object_discard(mock_image_ctx, mock_object_request, 0, + expect_object_discard(mock_image_ctx, mock_io_object_dispatch, 0, ictx->get_object_size(), false, -EINVAL); C_SaferCond cond_ctx; @@ -465,8 +455,8 @@ TEST_F(TestMockOperationTrimRequest, BoundaryError) { EXPECT_CALL(mock_image_ctx, get_stripe_count()).WillOnce(Return(ictx->get_stripe_count())); // boundary - MockObjectRequest mock_object_request; - expect_object_discard(mock_image_ctx, mock_object_request, 1, + io::MockObjectDispatch mock_io_object_dispatch; + expect_object_discard(mock_image_ctx, mock_io_object_dispatch, 1, ictx->get_object_size() - 1, true, -EINVAL); C_SaferCond cond_ctx; -- 2.39.5