From faf70f3fda11da5db3c43ec008fca7b5b27f5559 Mon Sep 17 00:00:00 2001 From: Jason Dillaman Date: Wed, 2 Sep 2020 19:10:57 -0400 Subject: [PATCH] librbd: generic image-extent list snapshot request Convert image extents to object-extents and issue list snapshot requests against each object. Once all the results are available, assemble the snapshot deltas back into image extents. Signed-off-by: Jason Dillaman --- src/librbd/io/ImageRequest.cc | 177 +++++++++++++++++-- src/librbd/io/ImageRequest.h | 44 ++++- src/test/librbd/io/test_mock_ImageRequest.cc | 68 +++++++ 3 files changed, 267 insertions(+), 22 deletions(-) diff --git a/src/librbd/io/ImageRequest.cc b/src/librbd/io/ImageRequest.cc index 6f444b3c73bda..8869e5b19813c 100644 --- a/src/librbd/io/ImageRequest.cc +++ b/src/librbd/io/ImageRequest.cc @@ -17,10 +17,12 @@ #include "librbd/io/Utils.h" #include "librbd/journal/Types.h" #include "include/rados/librados.hpp" +#include "common/errno.h" #include "common/perf_counters.h" #include "osdc/Striper.h" #include #include +#include #define dout_subsys ceph_subsys_rbd #undef dout_prefix @@ -34,6 +36,81 @@ using librbd::util::get_image_ctx; namespace { +template +struct C_AssembleSnapshotDeltas : public C_AioRequest { + I* image_ctx; + SnapshotDelta* snapshot_delta; + + ceph::mutex lock = ceph::make_mutex( + "librbd::io::C_AssembleSnapshotDeltas::lock", false); + std::map object_snapshot_delta; + + C_AssembleSnapshotDeltas(I* image_ctx, AioCompletion* aio_comp, + SnapshotDelta* snapshot_delta) + : C_AioRequest(aio_comp), + image_ctx(image_ctx), snapshot_delta(snapshot_delta) { + } + + SnapshotDelta* get_snapshot_delta(uint64_t object_no) { + std::unique_lock locker{lock}; + return &object_snapshot_delta[object_no]; + } + + void finish(int r) override { + auto cct = image_ctx->cct; + + if (r < 0) { + lderr(cct) << "C_AssembleSnapshotDeltas: list snaps failed: " + << cpp_strerror(r) << dendl; + C_AioRequest::finish(r); + return; + } + + std::unique_lock locker{lock}; + *snapshot_delta = {}; + for (auto& [object_no, object_snapshot_delta] : object_snapshot_delta) { + SnapshotDelta image_snapshot_delta; + object_to_image_intervals(object_no, object_snapshot_delta, + &image_snapshot_delta, snapshot_delta); + + ldout(cct, 20) << "object_no=" << object_no << ", " + << "object_snapshot_delta=" + << object_snapshot_delta << ", " + << "image_snapshot_delta=" << image_snapshot_delta + << dendl; + } + + ldout(cct, 20) << "snapshot_delta=" << *snapshot_delta << dendl; + C_AioRequest::finish(0); + } + + void object_to_image_intervals( + uint64_t object_no, const SnapshotDelta& object_snapshot_delta, + SnapshotDelta* image_snapshot_delta, + SnapshotDelta* assembled_image_snapshot_delta) { + auto cct = image_ctx->cct; + for (auto& [key, object_extents] : object_snapshot_delta) { + for (auto& object_extent : object_extents) { + Extents image_extents; + Striper::extent_to_file(cct, &image_ctx->layout, object_no, + object_extent.get_off(), + object_extent.get_len(), + image_extents); + + auto& intervals = (*image_snapshot_delta)[key]; + auto& assembled_intervals = (*assembled_image_snapshot_delta)[key]; + for (auto [image_offset, image_length] : image_extents) { + SnapshotExtent snapshot_extent{object_extent.get_val().state, + image_length}; + intervals.insert(image_offset, image_length, snapshot_extent); + assembled_intervals.insert(image_offset, image_length, + snapshot_extent); + } + } + } + } +}; + template struct C_RBD_Readahead : public Context { I *ictx; @@ -269,6 +346,17 @@ int ImageRequest::clip_request() { return 0; } +template +bool ImageRequest::finish_request_early() { + auto total_bytes = get_total_length(); + if (total_bytes == 0) { + auto *aio_comp = this->m_aio_comp; + aio_comp->set_request_count(0); + return true; + } + return false; +} + template uint64_t ImageRequest::get_total_length() const { uint64_t total_bytes = 0; @@ -358,17 +446,6 @@ int ImageReadRequest::clip_request() { return 0; } -template -bool ImageReadRequest::finish_request_early() { - auto total_bytes = this->get_total_length(); - if (total_bytes == 0) { - auto *aio_comp = this->m_aio_comp; - aio_comp->set_request_count(0); - return true; - } - return false; -} - template void ImageReadRequest::send_request() { I &image_ctx = this->m_image_ctx; @@ -440,12 +517,8 @@ bool AbstractImageWriteRequest::finish_request_early() { return true; } } - auto total_bytes = this->get_total_length(); - if (total_bytes == 0) { - aio_comp->set_request_count(0); - return true; - } - return false; + + return ImageRequest::finish_request_early(); } template @@ -903,6 +976,75 @@ int ImageCompareAndWriteRequest::prune_object_extents( return 0; } +template +ImageListSnapsRequest::ImageListSnapsRequest( + I& image_ctx, AioCompletion* aio_comp, Extents&& image_extents, + SnapIds&& snap_ids, int list_snaps_flags, SnapshotDelta* snapshot_delta, + const ZTracer::Trace& parent_trace) + : ImageRequest(image_ctx, aio_comp, std::move(image_extents), + image_ctx.get_data_io_context(), "list-snaps", + parent_trace), + m_snap_ids(std::move(snap_ids)), m_list_snaps_flags(list_snaps_flags), + m_snapshot_delta(snapshot_delta) { + this->set_bypass_image_cache(); +} + +template +int ImageListSnapsRequest::clip_request() { + // permit arbitrary list-snaps requests (internal API) + return 0; +} + +template +void ImageListSnapsRequest::send_request() { + I &image_ctx = this->m_image_ctx; + CephContext *cct = image_ctx.cct; + + // map image extents to object extents + auto &image_extents = this->m_image_extents; + std::map object_number_extents; + for (auto& image_extent : image_extents) { + if (image_extent.second == 0) { + continue; + } + + striper::LightweightObjectExtents object_extents; + Striper::file_to_extents(cct, &image_ctx.layout, image_extent.first, + image_extent.second, 0, 0, &object_extents); + for (auto& object_extent : object_extents) { + object_number_extents[object_extent.object_no].emplace_back( + object_extent.offset, object_extent.length); + } + } + + // reassemble the deltas back into image-extents when complete + auto aio_comp = this->m_aio_comp; + aio_comp->set_request_count(1); + auto assemble_ctx = new C_AssembleSnapshotDeltas( + &image_ctx, aio_comp, m_snapshot_delta); + auto sub_aio_comp = AioCompletion::create_and_start< + Context, &Context::complete>(assemble_ctx, get_image_ctx(&image_ctx), + AIO_TYPE_GENERIC); + + // issue the requests + sub_aio_comp->set_request_count(object_number_extents.size()); + for (auto& oe : object_number_extents) { + ldout(cct, 20) << data_object_name(&image_ctx, oe.first) << " " + << oe.second << dendl; + auto ctx = new C_AioRequest(sub_aio_comp); + auto req = ObjectDispatchSpec::create_list_snaps( + &image_ctx, OBJECT_DISPATCH_LAYER_NONE, oe.first, std::move(oe.second), + SnapIds{m_snap_ids}, m_list_snaps_flags, this->m_trace, + assemble_ctx->get_snapshot_delta(oe.first), ctx); + req->send(); + } +} + +template +void ImageListSnapsRequest::send_image_cache_request() { + ceph_abort(); +} + } // namespace io } // namespace librbd @@ -914,3 +1056,4 @@ template class librbd::io::ImageDiscardRequest; template class librbd::io::ImageFlushRequest; template class librbd::io::ImageWriteSameRequest; template class librbd::io::ImageCompareAndWriteRequest; +template class librbd::io::ImageListSnapsRequest; diff --git a/src/librbd/io/ImageRequest.h b/src/librbd/io/ImageRequest.h index 67c5af2b29ba7..65ac9ca37fc2e 100644 --- a/src/librbd/io/ImageRequest.h +++ b/src/librbd/io/ImageRequest.h @@ -93,10 +93,9 @@ protected: uint64_t get_total_length() const; - virtual bool finish_request_early() { - return false; - } virtual int clip_request(); + virtual bool finish_request_early(); + virtual void update_timestamp(); virtual void send_request() = 0; virtual void send_image_cache_request() = 0; @@ -117,7 +116,6 @@ public: protected: int clip_request() override; - bool finish_request_early() override; void send_request() override; void send_image_cache_request() override; @@ -152,9 +150,10 @@ protected: m_synchronous(false) { } - void send_request() override; bool finish_request_early() override; + void send_request() override; + virtual int prune_object_extents( LightweightObjectExtents* object_extents) const { return 0; @@ -272,6 +271,10 @@ protected: int clip_request() override { return 0; } + bool finish_request_early() override { + return false; + } + void update_timestamp() override { } void send_request() override; @@ -372,6 +375,36 @@ private: int m_op_flags; }; +template +class ImageListSnapsRequest : public ImageRequest { +public: + using typename ImageRequest::Extents; + + ImageListSnapsRequest( + ImageCtxT& image_ctx, AioCompletion* aio_comp, + Extents&& image_extents, SnapIds&& snap_ids, int list_snaps_flags, + SnapshotDelta* snapshot_delta, const ZTracer::Trace& parent_trace); + +protected: + int clip_request() override; + + void update_timestamp() override {} + void send_request() override; + void send_image_cache_request() override; + + aio_type_t get_aio_type() const override { + return AIO_TYPE_GENERIC; + } + const char *get_request_type() const override { + return "list-snaps"; + } + +private: + SnapIds m_snap_ids; + int m_list_snaps_flags; + SnapshotDelta* m_snapshot_delta; +}; + } // namespace io } // namespace librbd @@ -383,5 +416,6 @@ extern template class librbd::io::ImageDiscardRequest; extern template class librbd::io::ImageFlushRequest; extern template class librbd::io::ImageWriteSameRequest; extern template class librbd::io::ImageCompareAndWriteRequest; +extern template class librbd::io::ImageListSnapsRequest; #endif // CEPH_LIBRBD_IO_IMAGE_REQUEST_H diff --git a/src/test/librbd/io/test_mock_ImageRequest.cc b/src/test/librbd/io/test_mock_ImageRequest.cc index 9ef62fcc555fa..196a5c70a9089 100644 --- a/src/test/librbd/io/test_mock_ImageRequest.cc +++ b/src/test/librbd/io/test_mock_ImageRequest.cc @@ -69,6 +69,7 @@ struct TestMockIoImageRequest : public TestMockFixture { typedef ImageFlushRequest MockImageFlushRequest; typedef ImageWriteSameRequest MockImageWriteSameRequest; typedef ImageCompareAndWriteRequest MockImageCompareAndWriteRequest; + typedef ImageListSnapsRequest MockImageListSnapsRequest; void expect_is_journal_appending(MockTestJournal &mock_journal, bool appending) { EXPECT_CALL(mock_journal, is_journal_appending()) @@ -113,6 +114,26 @@ struct TestMockIoImageRequest : public TestMockFixture { mock_image_ctx.image_ctx->op_work_queue->queue(&spec->dispatcher_ctx, r); })); } + + void expect_object_list_snaps_request(MockTestImageCtx &mock_image_ctx, + uint64_t object_no, + const SnapshotDelta& snap_delta, + int r) { + EXPECT_CALL(*mock_image_ctx.io_object_dispatcher, send(_)) + .WillOnce( + Invoke([&mock_image_ctx, object_no, snap_delta, r] + (ObjectDispatchSpec* spec) { + auto request = boost::get< + librbd::io::ObjectDispatchSpec::ListSnapsRequest>( + &spec->request); + ASSERT_TRUE(request != nullptr); + ASSERT_EQ(object_no, request->object_no); + + *request->snapshot_delta = snap_delta; + spec->dispatch_result = io::DISPATCH_RESULT_COMPLETE; + mock_image_ctx.image_ctx->op_work_queue->queue(&spec->dispatcher_ctx, r); + })); + } }; TEST_F(TestMockIoImageRequest, AioWriteModifyTimestamp) { @@ -502,5 +523,52 @@ TEST_F(TestMockIoImageRequest, AioCompareAndWriteJournalAppendDisabled) { ASSERT_EQ(0, aio_comp_ctx.wait()); } +TEST_F(TestMockIoImageRequest, ListSnaps) { + librbd::ImageCtx *ictx; + ASSERT_EQ(0, open_image(m_image_name, &ictx)); + + MockTestImageCtx mock_image_ctx(*ictx); + mock_image_ctx.layout.object_size = 16384; + mock_image_ctx.layout.stripe_unit = 4096; + mock_image_ctx.layout.stripe_count = 2; + + InSequence seq; + + SnapshotDelta object_snapshot_delta; + object_snapshot_delta[{5,6}].insert( + 0, 1024, {SNAPSHOT_EXTENT_STATE_DATA, 1024}); + object_snapshot_delta[{5,5}].insert( + 4096, 4096, {SNAPSHOT_EXTENT_STATE_ZEROED, 4096}); + expect_object_list_snaps_request(mock_image_ctx, 0, object_snapshot_delta, 0); + object_snapshot_delta = {}; + object_snapshot_delta[{5,6}].insert( + 1024, 3072, {SNAPSHOT_EXTENT_STATE_DATA, 3072}); + object_snapshot_delta[{5,5}].insert( + 2048, 2048, {SNAPSHOT_EXTENT_STATE_ZEROED, 2048}); + expect_object_list_snaps_request(mock_image_ctx, 1, object_snapshot_delta, 0); + + SnapshotDelta snapshot_delta; + C_SaferCond aio_comp_ctx; + AioCompletion *aio_comp = AioCompletion::create_and_start( + &aio_comp_ctx, ictx, AIO_TYPE_GENERIC); + MockImageListSnapsRequest mock_image_list_snaps_request( + mock_image_ctx, aio_comp, {{0, 16384}, {16384, 16384}}, {0, CEPH_NOSNAP}, + 0, &snapshot_delta, {}); + { + std::shared_lock owner_locker{mock_image_ctx.owner_lock}; + mock_image_list_snaps_request.send(); + } + ASSERT_EQ(0, aio_comp_ctx.wait()); + + SnapshotDelta expected_snapshot_delta; + expected_snapshot_delta[{5,6}].insert( + 0, 1024, {SNAPSHOT_EXTENT_STATE_DATA, 1024}); + expected_snapshot_delta[{5,6}].insert( + 5120, 3072, {SNAPSHOT_EXTENT_STATE_DATA, 3072}); + expected_snapshot_delta[{5,5}].insert( + 6144, 6144, {SNAPSHOT_EXTENT_STATE_ZEROED, 6144}); + ASSERT_EQ(expected_snapshot_delta, snapshot_delta); +} + } // namespace io } // namespace librbd -- 2.39.5