From dcdc2241444b8ed31df705ac26924a6bc8895a1a Mon Sep 17 00:00:00 2001 From: Jason Dillaman Date: Fri, 16 Feb 2018 14:07:52 -0500 Subject: [PATCH] librbd: switch to new object dispatcher-based cache hooks Signed-off-by: Jason Dillaman --- .../exclusive_lock/PreReleaseRequest.cc | 29 ++--- src/librbd/exclusive_lock/PreReleaseRequest.h | 2 +- src/librbd/image/CloseRequest.cc | 21 ---- src/librbd/image/CloseRequest.h | 6 - src/librbd/image/OpenRequest.cc | 24 +++- src/librbd/image/OpenRequest.h | 5 + src/librbd/internal.cc | 43 +++++-- src/librbd/io/AioCompletion.h | 10 +- src/librbd/io/ObjectRequest.cc | 4 - src/librbd/operation/ResizeRequest.cc | 23 ++-- .../operation/SnapshotRollbackRequest.cc | 6 +- .../test_mock_PreReleaseRequest.cc | 34 +++--- src/test/librbd/io/test_mock_ImageRequest.cc | 21 +--- src/test/librbd/io/test_mock_ObjectRequest.cc | 113 ------------------ .../operation/test_mock_ResizeRequest.cc | 80 ++++++++++--- .../test_mock_SnapshotRollbackRequest.cc | 10 +- 16 files changed, 175 insertions(+), 256 deletions(-) diff --git a/src/librbd/exclusive_lock/PreReleaseRequest.cc b/src/librbd/exclusive_lock/PreReleaseRequest.cc index 504de0f0b93f9..e5ba42b057e43 100644 --- a/src/librbd/exclusive_lock/PreReleaseRequest.cc +++ b/src/librbd/exclusive_lock/PreReleaseRequest.cc @@ -12,6 +12,7 @@ #include "librbd/ObjectMap.h" #include "librbd/Utils.h" #include "librbd/io/ImageRequestWQ.h" +#include "librbd/io/ObjectDispatcher.h" #define dout_subsys ceph_subsys_rbd #undef dout_prefix @@ -156,25 +157,19 @@ void PreReleaseRequest::handle_wait_for_ops(int r) { CephContext *cct = m_image_ctx.cct; ldout(cct, 10) << dendl; - send_invalidate_cache(false); + send_invalidate_cache(); } template -void PreReleaseRequest::send_invalidate_cache(bool purge_on_error) { - if (m_image_ctx.object_cacher == nullptr) { - send_flush_notifies(); - return; - } - +void PreReleaseRequest::send_invalidate_cache() { CephContext *cct = m_image_ctx.cct; - ldout(cct, 10) << "purge_on_error=" << purge_on_error << dendl; + ldout(cct, 10) << dendl; RWLock::RLocker owner_lock(m_image_ctx.owner_lock); - Context *ctx = create_async_context_callback( - m_image_ctx, create_context_callback< + Context *ctx = create_context_callback< PreReleaseRequest, - &PreReleaseRequest::handle_invalidate_cache>(this)); - m_image_ctx.invalidate_cache(purge_on_error, ctx); + &PreReleaseRequest::handle_invalidate_cache>(this); + m_image_ctx.io_object_dispatcher->invalidate_cache(ctx); } template @@ -182,15 +177,7 @@ void PreReleaseRequest::handle_invalidate_cache(int r) { CephContext *cct = m_image_ctx.cct; ldout(cct, 10) << "r=" << r << dendl; - if (r == -EBLACKLISTED) { - lderr(cct) << "failed to invalidate cache because client is blacklisted" - << dendl; - if (!m_image_ctx.is_cache_empty()) { - // force purge the cache after after being blacklisted - send_invalidate_cache(true); - return; - } - } else if (r < 0 && r != -EBUSY) { + if (r < 0 && r != -EBLACKLISTED && r != -EBUSY) { lderr(cct) << "failed to invalidate cache: " << cpp_strerror(r) << dendl; m_image_ctx.io_work_queue->unblock_writes(); diff --git a/src/librbd/exclusive_lock/PreReleaseRequest.h b/src/librbd/exclusive_lock/PreReleaseRequest.h index 34ac07ab26c85..e5b85a8813b21 100644 --- a/src/librbd/exclusive_lock/PreReleaseRequest.h +++ b/src/librbd/exclusive_lock/PreReleaseRequest.h @@ -87,7 +87,7 @@ private: void send_wait_for_ops(); void handle_wait_for_ops(int r); - void send_invalidate_cache(bool purge_on_error); + void send_invalidate_cache(); void handle_invalidate_cache(int r); void send_flush_notifies(); diff --git a/src/librbd/image/CloseRequest.cc b/src/librbd/image/CloseRequest.cc index 0120c566629e9..cf6813dc06b33 100644 --- a/src/librbd/image/CloseRequest.cc +++ b/src/librbd/image/CloseRequest.cc @@ -220,27 +220,6 @@ void CloseRequest::handle_flush_readahead(int r) { CephContext *cct = m_image_ctx->cct; ldout(cct, 10) << this << " " << __func__ << ": r=" << r << dendl; - send_shut_down_cache(); -} - -template -void CloseRequest::send_shut_down_cache() { - CephContext *cct = m_image_ctx->cct; - ldout(cct, 10) << this << " " << __func__ << dendl; - - m_image_ctx->shut_down_cache(create_context_callback< - CloseRequest, &CloseRequest::handle_shut_down_cache>(this)); -} - -template -void CloseRequest::handle_shut_down_cache(int r) { - CephContext *cct = m_image_ctx->cct; - ldout(cct, 10) << this << " " << __func__ << ": r=" << r << dendl; - - save_result(r); - if (r < 0) { - lderr(cct) << "failed to shut down cache: " << cpp_strerror(r) << dendl; - } send_shut_down_object_dispatcher(); } diff --git a/src/librbd/image/CloseRequest.h b/src/librbd/image/CloseRequest.h index 73c344af4343a..fd101beedb331 100644 --- a/src/librbd/image/CloseRequest.h +++ b/src/librbd/image/CloseRequest.h @@ -50,9 +50,6 @@ private: * FLUSH_READAHEAD * | * v - * SHUTDOWN_CACHE - * | - * v * SHUT_DOWN_OBJECT_DISPATCHER * | * v @@ -100,9 +97,6 @@ private: void send_flush_readahead(); void handle_flush_readahead(int r); - void send_shut_down_cache(); - void handle_shut_down_cache(int r); - void send_shut_down_object_dispatcher(); void handle_shut_down_object_dispatcher(int r); diff --git a/src/librbd/image/OpenRequest.cc b/src/librbd/image/OpenRequest.cc index 73a2273e8a03d..009541e2f2435 100644 --- a/src/librbd/image/OpenRequest.cc +++ b/src/librbd/image/OpenRequest.cc @@ -7,6 +7,7 @@ #include "cls/rbd/cls_rbd_client.h" #include "librbd/ImageCtx.h" #include "librbd/Utils.h" +#include "librbd/cache/ObjectCacherObjectDispatch.h" #include "librbd/image/CloseRequest.h" #include "librbd/image/RefreshRequest.h" #include "librbd/image/SetSnapRequest.h" @@ -441,7 +442,28 @@ Context *OpenRequest::handle_refresh(int *result) { return nullptr; } - m_image_ctx->init_cache(); + return send_init_cache(result); +} + +template +Context *OpenRequest::send_init_cache(int *result) { + // cache is disabled or parent image context + if (!m_image_ctx->cache || m_image_ctx->child != nullptr) { + return send_register_watch(result); + } + + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << this << " " << __func__ << dendl; + + auto cache = cache::ObjectCacherObjectDispatch::create(m_image_ctx); + cache->init(); + + // readahead requires the cache + m_image_ctx->readahead.set_trigger_requests( + m_image_ctx->readahead_trigger_requests); + m_image_ctx->readahead.set_max_readahead_size( + m_image_ctx->readahead_max_bytes); + return send_register_watch(result); } diff --git a/src/librbd/image/OpenRequest.h b/src/librbd/image/OpenRequest.h index 45395f148d3bf..aa8bb79d0a978 100644 --- a/src/librbd/image/OpenRequest.h +++ b/src/librbd/image/OpenRequest.h @@ -58,6 +58,9 @@ private: * V2_GET_DATA_POOL --------------> REFRESH * | * v + * INIT_CACHE + * | + * v * REGISTER_WATCH (skip if * | read-only) * v @@ -111,6 +114,8 @@ private: void send_refresh(); Context *handle_refresh(int *result); + Context *send_init_cache(int *result); + Context *send_register_watch(int *result); Context *handle_register_watch(int *result); diff --git a/src/librbd/internal.cc b/src/librbd/internal.cc index 6141f6e2b6034..cd96a47ce8df7 100644 --- a/src/librbd/internal.cc +++ b/src/librbd/internal.cc @@ -12,9 +12,11 @@ #include "common/errno.h" #include "common/Throttle.h" #include "common/event_socket.h" -#include "cls/lock/cls_lock_client.h" +#include "common/perf_counters.h" +#include "osdc/Striper.h" #include "include/stringify.h" +#include "cls/lock/cls_lock_client.h" #include "cls/rbd/cls_rbd.h" #include "cls/rbd/cls_rbd_types.h" #include "cls/rbd/cls_rbd_client.h" @@ -40,6 +42,8 @@ #include "librbd/io/AioCompletion.h" #include "librbd/io/ImageRequest.h" #include "librbd/io/ImageRequestWQ.h" +#include "librbd/io/ObjectDispatcher.h" +#include "librbd/io/ObjectDispatchSpec.h" #include "librbd/io/ObjectRequest.h" #include "librbd/io/ReadResult.h" #include "librbd/journal/Types.h" @@ -2245,8 +2249,12 @@ bool compare_by_name(const child_info_t& c1, const child_info_t& c2) return r; } - RWLock::RLocker owner_locker(ictx->owner_lock); - r = ictx->invalidate_cache(false); + C_SaferCond ctx; + { + RWLock::RLocker owner_locker(ictx->owner_lock); + ictx->io_object_dispatcher->invalidate_cache(&ctx); + } + r = ctx.wait(); ictx->perfcounter->inc(l_librbd_invalidate_cache); return r; } @@ -2299,10 +2307,18 @@ bool compare_by_name(const child_info_t& c1, const child_info_t& c2) object_t oid; uint64_t offset; uint64_t length; + + bufferlist read_data; + io::ExtentMap extent_map; + C_RBD_Readahead(ImageCtx *ictx, object_t oid, uint64_t offset, uint64_t length) - : ictx(ictx), oid(oid), offset(offset), length(length) { } + : ictx(ictx), oid(oid), offset(offset), length(length) { + ictx->readahead.inc_pending(); + } + void finish(int r) override { - ldout(ictx->cct, 20) << "C_RBD_Readahead on " << oid << ": " << offset << "+" << length << dendl; + ldout(ictx->cct, 20) << "C_RBD_Readahead on " << oid << ": " + << offset << "~" << length << dendl; ictx->readahead.dec_pending(); } }; @@ -2316,7 +2332,7 @@ bool compare_by_name(const child_info_t& c1, const child_info_t& c2) ++p) { total_bytes += p->second; } - + ictx->md_lock.get_write(); bool abort = ictx->readahead_disable_after_bytes != 0 && ictx->total_bytes_read > ictx->readahead_disable_after_bytes; @@ -2327,9 +2343,10 @@ bool compare_by_name(const child_info_t& c1, const child_info_t& c2) ictx->total_bytes_read += total_bytes; ictx->snap_lock.get_read(); uint64_t image_size = ictx->get_image_size(ictx->snap_id); + auto snap_id = ictx->snap_id; ictx->snap_lock.put_read(); ictx->md_lock.put_write(); - + pair readahead_extent = ictx->readahead.update(image_extents, image_size); uint64_t readahead_offset = readahead_extent.first; uint64_t readahead_length = readahead_extent.second; @@ -2343,11 +2360,13 @@ bool compare_by_name(const child_info_t& c1, const child_info_t& c2) for (vector::iterator q = p->second.begin(); q != p->second.end(); ++q) { ldout(ictx->cct, 20) << "(readahead) oid " << q->oid << " " << q->offset << "~" << q->length << dendl; - Context *req_comp = new C_RBD_Readahead(ictx, q->oid, q->offset, q->length); - ictx->readahead.inc_pending(); - ictx->aio_read_from_cache(q->oid, q->objectno, NULL, - q->length, q->offset, - req_comp, 0, nullptr); + auto req_comp = new C_RBD_Readahead(ictx, q->oid, q->offset, + q->length); + auto req = io::ObjectDispatchSpec::create_read( + ictx, io::OBJECT_DISPATCH_LAYER_NONE, q->oid.name, q->objectno, + q->offset, q->length, snap_id, 0, {}, &req_comp->read_data, + &req_comp->extent_map, req_comp); + req->send(0); } } ictx->perfcounter->inc(l_librbd_readahead); diff --git a/src/librbd/io/AioCompletion.h b/src/librbd/io/AioCompletion.h index e0b07cc6ef287..7f3abb3947cd5 100644 --- a/src/librbd/io/AioCompletion.h +++ b/src/librbd/io/AioCompletion.h @@ -90,10 +90,16 @@ struct AioCompletion { } template - static AioCompletion *create_and_start(T *obj, ImageCtx *image_ctx, - aio_type_t type) { + static AioCompletion *create(T *obj, ImageCtx *image_ctx, aio_type_t type) { AioCompletion *comp = create(obj); comp->init_time(image_ctx, type); + return comp; + } + + template + static AioCompletion *create_and_start(T *obj, ImageCtx *image_ctx, + aio_type_t type) { + AioCompletion *comp = create(obj, image_ctx, type); comp->start_op(); return comp; } diff --git a/src/librbd/io/ObjectRequest.cc b/src/librbd/io/ObjectRequest.cc index d2eeb1801df3c..b5537d75af737 100644 --- a/src/librbd/io/ObjectRequest.cc +++ b/src/librbd/io/ObjectRequest.cc @@ -298,10 +298,6 @@ void ObjectReadRequest::handle_read_object(int r) { template void ObjectReadRequest::read_parent() { I *image_ctx = this->m_ictx; - if (m_cache_initiated) { - this->finish(-ENOENT); - return; - } uint64_t object_overlap = 0; Extents parent_extents; diff --git a/src/librbd/operation/ResizeRequest.cc b/src/librbd/operation/ResizeRequest.cc index df9ed2b9b23f3..31827f562f516 100644 --- a/src/librbd/operation/ResizeRequest.cc +++ b/src/librbd/operation/ResizeRequest.cc @@ -7,7 +7,10 @@ #include "librbd/internal.h" #include "librbd/ObjectMap.h" #include "librbd/Utils.h" +#include "librbd/io/AioCompletion.h" +#include "librbd/io/ImageDispatchSpec.h" #include "librbd/io/ImageRequestWQ.h" +#include "librbd/io/ObjectDispatcher.h" #include "librbd/operation/TrimRequest.h" #include "common/dout.h" #include "common/errno.h" @@ -182,18 +185,19 @@ Context *ResizeRequest::handle_trim_image(int *result) { template void ResizeRequest::send_flush_cache() { I &image_ctx = this->m_image_ctx; - if (image_ctx.object_cacher == nullptr) { - send_trim_image(); - return; - } CephContext *cct = image_ctx.cct; ldout(cct, 5) << this << " " << __func__ << dendl; RWLock::RLocker owner_locker(image_ctx.owner_lock); - image_ctx.flush_cache(create_async_context_callback( - image_ctx, create_context_callback< - ResizeRequest, &ResizeRequest::handle_flush_cache>(this))); + auto ctx = create_context_callback< + ResizeRequest, &ResizeRequest::handle_flush_cache>(this); + auto aio_comp = io::AioCompletion::create( + ctx, util::get_image_ctx(&image_ctx), io::AIO_TYPE_FLUSH); + auto req = io::ImageDispatchSpec::create_flush_request( + image_ctx, aio_comp, io::FLUSH_SOURCE_INTERNAL, {}); + req->send(); + delete req; } template @@ -220,9 +224,8 @@ void ResizeRequest::send_invalidate_cache() { // need to invalidate since we're deleting objects, and // ObjectCacher doesn't track non-existent objects RWLock::RLocker owner_locker(image_ctx.owner_lock); - image_ctx.invalidate_cache(false, create_async_context_callback( - image_ctx, create_context_callback< - ResizeRequest, &ResizeRequest::handle_invalidate_cache>(this))); + image_ctx.io_object_dispatcher->invalidate_cache(create_context_callback< + ResizeRequest, &ResizeRequest::handle_invalidate_cache>(this)); } template diff --git a/src/librbd/operation/SnapshotRollbackRequest.cc b/src/librbd/operation/SnapshotRollbackRequest.cc index f224b68fcc4f9..7072e72ce599f 100644 --- a/src/librbd/operation/SnapshotRollbackRequest.cc +++ b/src/librbd/operation/SnapshotRollbackRequest.cc @@ -10,6 +10,7 @@ #include "librbd/ObjectMap.h" #include "librbd/Utils.h" #include "librbd/io/ImageRequestWQ.h" +#include "librbd/io/ObjectDispatcher.h" #include "librbd/operation/ResizeRequest.h" #include "osdc/Striper.h" #include @@ -277,9 +278,6 @@ Context *SnapshotRollbackRequest::send_invalidate_cache() { I &image_ctx = this->m_image_ctx; apply(); - if (image_ctx.object_cacher == NULL) { - return this->create_context_finisher(0); - } CephContext *cct = image_ctx.cct; ldout(cct, 5) << this << " " << __func__ << dendl; @@ -288,7 +286,7 @@ Context *SnapshotRollbackRequest::send_invalidate_cache() { Context *ctx = create_context_callback< SnapshotRollbackRequest, &SnapshotRollbackRequest::handle_invalidate_cache>(this); - image_ctx.invalidate_cache(true, ctx); + image_ctx.io_object_dispatcher->invalidate_cache(ctx); return nullptr; } diff --git a/src/test/librbd/exclusive_lock/test_mock_PreReleaseRequest.cc b/src/test/librbd/exclusive_lock/test_mock_PreReleaseRequest.cc index 44aa012568d12..989e793fc7344 100644 --- a/src/test/librbd/exclusive_lock/test_mock_PreReleaseRequest.cc +++ b/src/test/librbd/exclusive_lock/test_mock_PreReleaseRequest.cc @@ -6,6 +6,7 @@ #include "test/librbd/mock/MockImageCtx.h" #include "test/librbd/mock/MockJournal.h" #include "test/librbd/mock/MockObjectMap.h" +#include "test/librbd/mock/io/MockObjectDispatch.h" #include "test/librados_test_stub/MockTestMemIoCtxImpl.h" #include "common/AsyncOpTracker.h" #include "librbd/exclusive_lock/PreReleaseRequest.h" @@ -94,19 +95,10 @@ public: .WillOnce(CompleteContext(0, mock_image_ctx.image_ctx->op_work_queue)); } - void expect_invalidate_cache(MockImageCtx &mock_image_ctx, bool purge, + void expect_invalidate_cache(MockImageCtx &mock_image_ctx, int r) { - if (mock_image_ctx.object_cacher != nullptr) { - EXPECT_CALL(mock_image_ctx, invalidate_cache(purge, _)) - .WillOnce(WithArg<1>(CompleteContext(r, static_cast(NULL)))); - } - } - - void expect_is_cache_empty(MockImageCtx &mock_image_ctx, bool empty) { - if (mock_image_ctx.object_cacher != nullptr) { - EXPECT_CALL(mock_image_ctx, is_cache_empty()) - .WillOnce(Return(empty)); - } + EXPECT_CALL(*mock_image_ctx.io_object_dispatcher, invalidate_cache(_)) + .WillOnce(CompleteContext(r, mock_image_ctx.image_ctx->op_work_queue)); } void expect_flush_notifies(MockImageCtx &mock_image_ctx) { @@ -142,7 +134,8 @@ TEST_F(TestMockExclusiveLockPreReleaseRequest, Success) { expect_prepare_lock(mock_image_ctx); expect_cancel_op_requests(mock_image_ctx, 0); expect_block_writes(mock_image_ctx, 0); - expect_invalidate_cache(mock_image_ctx, false, 0); + expect_invalidate_cache(mock_image_ctx, 0); + expect_flush_notifies(mock_image_ctx); MockJournal *mock_journal = new MockJournal(); @@ -176,7 +169,8 @@ TEST_F(TestMockExclusiveLockPreReleaseRequest, SuccessJournalDisabled) { InSequence seq; expect_prepare_lock(mock_image_ctx); expect_cancel_op_requests(mock_image_ctx, 0); - expect_invalidate_cache(mock_image_ctx, false, 0); + expect_invalidate_cache(mock_image_ctx, 0); + expect_flush_notifies(mock_image_ctx); MockObjectMap *mock_object_map = new MockObjectMap(); @@ -205,7 +199,8 @@ TEST_F(TestMockExclusiveLockPreReleaseRequest, SuccessObjectMapDisabled) { InSequence seq; expect_cancel_op_requests(mock_image_ctx, 0); - expect_invalidate_cache(mock_image_ctx, false, 0); + expect_invalidate_cache(mock_image_ctx, 0); + expect_flush_notifies(mock_image_ctx); C_SaferCond release_ctx; @@ -229,10 +224,8 @@ TEST_F(TestMockExclusiveLockPreReleaseRequest, Blacklisted) { expect_prepare_lock(mock_image_ctx); expect_cancel_op_requests(mock_image_ctx, 0); expect_block_writes(mock_image_ctx, -EBLACKLISTED); - expect_invalidate_cache(mock_image_ctx, false, -EBLACKLISTED); - expect_is_cache_empty(mock_image_ctx, false); - expect_invalidate_cache(mock_image_ctx, true, -EBLACKLISTED); - expect_is_cache_empty(mock_image_ctx, true); + expect_invalidate_cache(mock_image_ctx, -EBLACKLISTED); + expect_flush_notifies(mock_image_ctx); MockJournal *mock_journal = new MockJournal(); @@ -287,7 +280,8 @@ TEST_F(TestMockExclusiveLockPreReleaseRequest, UnlockError) { InSequence seq; expect_cancel_op_requests(mock_image_ctx, 0); expect_block_writes(mock_image_ctx, 0); - expect_invalidate_cache(mock_image_ctx, false, 0); + expect_invalidate_cache(mock_image_ctx, 0); + expect_flush_notifies(mock_image_ctx); C_SaferCond ctx; diff --git a/src/test/librbd/io/test_mock_ImageRequest.cc b/src/test/librbd/io/test_mock_ImageRequest.cc index 7dd0454f2e455..09970e741422e 100644 --- a/src/test/librbd/io/test_mock_ImageRequest.cc +++ b/src/test/librbd/io/test_mock_ImageRequest.cc @@ -95,11 +95,6 @@ struct TestMockIoImageRequest : public TestMockFixture { })); } - void expect_flush(MockImageCtx &mock_image_ctx, int r) { - EXPECT_CALL(mock_image_ctx, flush_cache(_)) - .WillOnce(CompleteContext(r, mock_image_ctx.image_ctx->op_work_queue)); - } - void expect_flush_async_operations(MockImageCtx &mock_image_ctx, int r) { EXPECT_CALL(mock_image_ctx, flush_async_operations(_)) .WillOnce(CompleteContext(r, mock_image_ctx.image_ctx->op_work_queue)); @@ -118,12 +113,7 @@ TEST_F(TestMockIoImageRequest, AioWriteJournalAppendDisabled) { InSequence seq; expect_is_journal_appending(mock_journal, false); - if (mock_image_ctx.image_ctx->cache) { - expect_write_to_cache(mock_image_ctx, ictx->get_object_name(0), - 0, 1, 0, 0); - } else { - expect_object_request_send(mock_image_ctx, 0); - } + expect_object_request_send(mock_image_ctx, 0); C_SaferCond aio_comp_ctx; AioCompletion *aio_comp = AioCompletion::create_and_start( @@ -184,7 +174,6 @@ TEST_F(TestMockIoImageRequest, AioFlushJournalAppendDisabled) { expect_is_journal_appending(mock_journal, false); expect_flush_async_operations(mock_image_ctx, 0); expect_object_request_send(mock_image_ctx, 0); - expect_flush(mock_image_ctx, 0); C_SaferCond aio_comp_ctx; AioCompletion *aio_comp = AioCompletion::create_and_start( @@ -210,13 +199,7 @@ TEST_F(TestMockIoImageRequest, AioWriteSameJournalAppendDisabled) { InSequence seq; expect_is_journal_appending(mock_journal, false); - if (mock_image_ctx.image_ctx->cache) { - expect_write_to_cache(mock_image_ctx, ictx->get_object_name(0), - 0, 1, 0, 0); - } else { - expect_object_request_send(mock_image_ctx, 0); - } - + expect_object_request_send(mock_image_ctx, 0); C_SaferCond aio_comp_ctx; AioCompletion *aio_comp = AioCompletion::create_and_start( diff --git a/src/test/librbd/io/test_mock_ObjectRequest.cc b/src/test/librbd/io/test_mock_ObjectRequest.cc index ad850ab9dd01d..00d0e42e56ec5 100644 --- a/src/test/librbd/io/test_mock_ObjectRequest.cc +++ b/src/test/librbd/io/test_mock_ObjectRequest.cc @@ -180,22 +180,6 @@ struct TestMockIoObjectRequest : public TestMockFixture { } } - void expect_cache_read(MockTestImageCtx &mock_image_ctx, - const std::string& oid, uint64_t object_no, - uint64_t off, uint64_t len, const std::string& data, - int r) { - bufferlist bl; - bl.append(data); - - EXPECT_CALL(mock_image_ctx, aio_read_from_cache({oid}, object_no, _, len, - off, _, _, _)) - .WillOnce(WithArgs<2, 5>(Invoke([bl, r](bufferlist *out_bl, - Context *on_finish) { - out_bl->append(bl); - on_finish->complete(r); - }))); - } - void expect_aio_read(MockImageRequest& mock_image_request, Extents&& extents, int r) { EXPECT_CALL(mock_image_request, aio_read(_, extents)) @@ -408,60 +392,6 @@ TEST_F(TestMockIoObjectRequest, ReadError) { ASSERT_EQ(-EPERM, ctx.wait()); } -TEST_F(TestMockIoObjectRequest, CacheRead) { - librbd::ImageCtx *ictx; - ASSERT_EQ(0, open_image(m_image_name, &ictx)); - ictx->sparse_read_threshold_bytes = 8096; - - MockTestImageCtx mock_image_ctx(*ictx); - MockObjectMap mock_object_map; - if (ictx->test_features(RBD_FEATURE_OBJECT_MAP)) { - mock_image_ctx.object_map = &mock_object_map; - } - - expect_op_work_queue(mock_image_ctx); - - InSequence seq; - expect_cache_read(mock_image_ctx, ictx->get_object_name(0), 0, 0, 4096, - std::string(4096, '1'), 0); - - bufferlist bl; - ExtentMap extent_map; - C_SaferCond ctx; - auto req = MockObjectReadRequest::create( - &mock_image_ctx, ictx->get_object_name(0), 0, 0, 4096, CEPH_NOSNAP, 0, - false, {}, &bl, &extent_map, &ctx); - req->send(); - ASSERT_EQ(0, ctx.wait()); -} - -TEST_F(TestMockIoObjectRequest, CacheReadError) { - librbd::ImageCtx *ictx; - ASSERT_EQ(0, open_image(m_image_name, &ictx)); - ictx->sparse_read_threshold_bytes = 8096; - - MockTestImageCtx mock_image_ctx(*ictx); - MockObjectMap mock_object_map; - if (ictx->test_features(RBD_FEATURE_OBJECT_MAP)) { - mock_image_ctx.object_map = &mock_object_map; - } - - expect_op_work_queue(mock_image_ctx); - - InSequence seq; - expect_cache_read(mock_image_ctx, ictx->get_object_name(0), 0, 0, 4096, - "", -EPERM); - - bufferlist bl; - ExtentMap extent_map; - C_SaferCond ctx; - auto req = MockObjectReadRequest::create( - &mock_image_ctx, ictx->get_object_name(0), 0, 0, 4096, CEPH_NOSNAP, 0, - false, {}, &bl, &extent_map, &ctx); - req->send(); - ASSERT_EQ(-EPERM, ctx.wait()); -} - TEST_F(TestMockIoObjectRequest, ParentRead) { REQUIRE_FEATURE(RBD_FEATURE_LAYERING); @@ -560,49 +490,6 @@ TEST_F(TestMockIoObjectRequest, ParentReadError) { ASSERT_EQ(-EPERM, ctx.wait()); } -TEST_F(TestMockIoObjectRequest, CacheInitiated) { - REQUIRE_FEATURE(RBD_FEATURE_LAYERING); - - librbd::Image image; - librbd::RBD rbd; - ASSERT_EQ(0, rbd.open(m_ioctx, image, m_image_name.c_str(), NULL)); - ASSERT_EQ(0, image.snap_create("one")); - ASSERT_EQ(0, image.snap_protect("one")); - image.close(); - - std::string clone_name = get_temp_image_name(); - int order = 0; - ASSERT_EQ(0, rbd.clone(m_ioctx, m_image_name.c_str(), "one", m_ioctx, - clone_name.c_str(), RBD_FEATURE_LAYERING, &order)); - - librbd::ImageCtx *ictx; - ASSERT_EQ(0, open_image(clone_name, &ictx)); - ictx->sparse_read_threshold_bytes = 8096; - ictx->clone_copy_on_read = false; - - MockTestImageCtx mock_image_ctx(*ictx); - mock_image_ctx.parent = &mock_image_ctx; - - MockObjectMap mock_object_map; - if (ictx->test_features(RBD_FEATURE_OBJECT_MAP)) { - mock_image_ctx.object_map = &mock_object_map; - } - - InSequence seq; - expect_object_may_exist(mock_image_ctx, 0, true); - expect_get_read_flags(mock_image_ctx, CEPH_NOSNAP, 0); - expect_read(mock_image_ctx, ictx->get_object_name(0), 0, 4096, "", -ENOENT); - - bufferlist bl; - ExtentMap extent_map; - C_SaferCond ctx; - auto req = MockObjectReadRequest::create( - &mock_image_ctx, ictx->get_object_name(0), 0, 0, 4096, CEPH_NOSNAP, 0, true, - {}, &bl, &extent_map, &ctx); - req->send(); - ASSERT_EQ(-ENOENT, ctx.wait()); -} - TEST_F(TestMockIoObjectRequest, CopyOnRead) { REQUIRE_FEATURE(RBD_FEATURE_LAYERING); diff --git a/src/test/librbd/operation/test_mock_ResizeRequest.cc b/src/test/librbd/operation/test_mock_ResizeRequest.cc index 7b70a50e8f48f..85a22610e2bb4 100644 --- a/src/test/librbd/operation/test_mock_ResizeRequest.cc +++ b/src/test/librbd/operation/test_mock_ResizeRequest.cc @@ -4,16 +4,53 @@ #include "test/librbd/test_mock_fixture.h" #include "test/librbd/test_support.h" #include "test/librbd/mock/MockImageCtx.h" +#include "test/librbd/mock/io/MockObjectDispatch.h" #include "test/librados_test_stub/MockTestMemIoCtxImpl.h" #include "common/bit_vector.hpp" #include "librbd/internal.h" #include "librbd/ObjectMap.h" +#include "librbd/io/ImageDispatchSpec.h" #include "librbd/operation/ResizeRequest.h" #include "librbd/operation/TrimRequest.h" #include "gmock/gmock.h" #include "gtest/gtest.h" namespace librbd { + +namespace util { + +inline ImageCtx* get_image_ctx(MockImageCtx* image_ctx) { + return image_ctx->image_ctx; +} + +} // namespace util + +namespace io { + +template <> +struct ImageDispatchSpec { + static ImageDispatchSpec* s_instance; + AioCompletion *aio_comp = nullptr; + + static ImageDispatchSpec* create_flush_request( + MockImageCtx &image_ctx, AioCompletion *aio_comp, + FlushSource flush_source, const ZTracer::Trace &parent_trace) { + assert(s_instance != nullptr); + s_instance->aio_comp = aio_comp; + return s_instance; + } + + MOCK_CONST_METHOD0(send, void()); + + ImageDispatchSpec() { + s_instance = this; + } +}; + +ImageDispatchSpec* ImageDispatchSpec::s_instance = nullptr; + +} // namespace io + namespace operation { template <> @@ -50,6 +87,7 @@ namespace operation { using ::testing::_; using ::testing::DoAll; +using ::testing::Invoke; using ::testing::InSequence; using ::testing::Return; using ::testing::StrEq; @@ -59,6 +97,7 @@ class TestMockOperationResizeRequest : public TestMockFixture { public: typedef ResizeRequest MockResizeRequest; typedef TrimRequest MockTrimRequest; + typedef io::ImageDispatchSpec MockIoImageDispatchSpec; void expect_block_writes(MockImageCtx &mock_image_ctx, int r) { EXPECT_CALL(*mock_image_ctx.io_work_queue, block_writes(_)) @@ -112,21 +151,24 @@ public: .WillOnce(FinishRequest(&mock_trim_request, r, &mock_image_ctx)); } - void expect_flush_cache(MockImageCtx &mock_image_ctx, int r) { - if (!mock_image_ctx.image_ctx->cache) { - return; - } - EXPECT_CALL(mock_image_ctx, flush_cache(_)) - .WillOnce(CompleteContext(r, static_cast(NULL))); - expect_op_work_queue(mock_image_ctx); + void expect_flush_cache(MockImageCtx &mock_image_ctx, + MockIoImageDispatchSpec& mock_io_image_dispatch_spec, + int r) { + EXPECT_CALL(mock_io_image_dispatch_spec, send()) + .WillOnce(Invoke([&mock_image_ctx, &mock_io_image_dispatch_spec, r]() { + auto aio_comp = mock_io_image_dispatch_spec.s_instance->aio_comp; + auto ctx = new FunctionContext([aio_comp](int r) { + aio_comp->get(); + aio_comp->fail(r); + }); + mock_image_ctx.image_ctx->op_work_queue->queue(ctx, r); + })); } - void expect_invalidate_cache(MockImageCtx &mock_image_ctx, int r) { - if (!mock_image_ctx.image_ctx->cache) { - return; - } - EXPECT_CALL(mock_image_ctx, invalidate_cache(false, _)) - .WillOnce(WithArg<1>(CompleteContext(r, static_cast(NULL)))); + void expect_invalidate_cache(MockImageCtx &mock_image_ctx, + int r) { + EXPECT_CALL(*mock_image_ctx.io_object_dispatcher, invalidate_cache(_)) + .WillOnce(CompleteContext(r, mock_image_ctx.image_ctx->op_work_queue)); expect_op_work_queue(mock_image_ctx); } @@ -211,7 +253,8 @@ TEST_F(TestMockOperationResizeRequest, ShrinkSuccess) { expect_unblock_writes(mock_image_ctx); MockTrimRequest mock_trim_request; - expect_flush_cache(mock_image_ctx, 0); + auto mock_io_image_dispatch_spec = new MockIoImageDispatchSpec(); + expect_flush_cache(mock_image_ctx, *mock_io_image_dispatch_spec, 0); expect_invalidate_cache(mock_image_ctx, 0); expect_trim(mock_image_ctx, mock_trim_request, 0); expect_block_writes(mock_image_ctx, 0); @@ -273,7 +316,8 @@ TEST_F(TestMockOperationResizeRequest, TrimError) { expect_unblock_writes(mock_image_ctx); MockTrimRequest mock_trim_request; - expect_flush_cache(mock_image_ctx, 0); + auto mock_io_image_dispatch_spec = new MockIoImageDispatchSpec(); + expect_flush_cache(mock_image_ctx, *mock_io_image_dispatch_spec, 0); expect_invalidate_cache(mock_image_ctx, -EBUSY); expect_trim(mock_image_ctx, mock_trim_request, -EINVAL); expect_commit_op_event(mock_image_ctx, -EINVAL); @@ -298,7 +342,8 @@ TEST_F(TestMockOperationResizeRequest, FlushCacheError) { expect_unblock_writes(mock_image_ctx); MockTrimRequest mock_trim_request; - expect_flush_cache(mock_image_ctx, -EINVAL); + auto mock_io_image_dispatch_spec = new MockIoImageDispatchSpec(); + expect_flush_cache(mock_image_ctx, *mock_io_image_dispatch_spec, -EINVAL); expect_commit_op_event(mock_image_ctx, -EINVAL); ASSERT_EQ(-EINVAL, when_resize(mock_image_ctx, ictx->size / 2, true, 0, false)); } @@ -321,7 +366,8 @@ TEST_F(TestMockOperationResizeRequest, InvalidateCacheError) { expect_unblock_writes(mock_image_ctx); MockTrimRequest mock_trim_request; - expect_flush_cache(mock_image_ctx, 0); + auto mock_io_image_dispatch_spec = new MockIoImageDispatchSpec(); + expect_flush_cache(mock_image_ctx, *mock_io_image_dispatch_spec, 0); expect_invalidate_cache(mock_image_ctx, -EINVAL); expect_commit_op_event(mock_image_ctx, -EINVAL); ASSERT_EQ(-EINVAL, when_resize(mock_image_ctx, ictx->size / 2, true, 0, false)); diff --git a/src/test/librbd/operation/test_mock_SnapshotRollbackRequest.cc b/src/test/librbd/operation/test_mock_SnapshotRollbackRequest.cc index 97cc5f153dc2a..e41b795a1a32f 100644 --- a/src/test/librbd/operation/test_mock_SnapshotRollbackRequest.cc +++ b/src/test/librbd/operation/test_mock_SnapshotRollbackRequest.cc @@ -4,6 +4,7 @@ #include "test/librbd/test_mock_fixture.h" #include "test/librbd/test_support.h" #include "test/librbd/mock/MockImageCtx.h" +#include "test/librbd/mock/io/MockObjectDispatch.h" #include "test/librados_test_stub/MockTestMemIoCtxImpl.h" #include "include/stringify.h" #include "common/bit_vector.hpp" @@ -159,11 +160,10 @@ public: } } - void expect_invalidate_cache(MockOperationImageCtx &mock_image_ctx, int r) { - if (mock_image_ctx.object_cacher != nullptr) { - EXPECT_CALL(mock_image_ctx, invalidate_cache(true, _)) - .WillOnce(WithArg<1>(CompleteContext(r, static_cast(NULL)))); - } + void expect_invalidate_cache(MockOperationImageCtx &mock_image_ctx, + int r) { + EXPECT_CALL(*mock_image_ctx.io_object_dispatcher, invalidate_cache(_)) + .WillOnce(CompleteContext(r, mock_image_ctx.image_ctx->op_work_queue)); } int when_snap_rollback(MockOperationImageCtx &mock_image_ctx, -- 2.39.5