From 48223589cc91f2462bf712c293022b1927c26efe Mon Sep 17 00:00:00 2001 From: Jason Dillaman Date: Tue, 1 Dec 2020 17:47:38 -0500 Subject: [PATCH] librbd/migration: add snapshot support to the raw format Snapshots can be specified via a new "snapshots" array of objects within the raw format JSON object. Signed-off-by: Jason Dillaman --- src/librbd/migration/RawFormat.cc | 149 +++++++++++++- src/librbd/migration/RawFormat.h | 3 + .../librbd/migration/test_mock_RawFormat.cc | 184 +++++++++++++++++- 3 files changed, 327 insertions(+), 9 deletions(-) diff --git a/src/librbd/migration/RawFormat.cc b/src/librbd/migration/RawFormat.cc index 8e8ecd1b03b..a4e3faf777f 100644 --- a/src/librbd/migration/RawFormat.cc +++ b/src/librbd/migration/RawFormat.cc @@ -15,6 +15,13 @@ namespace librbd { namespace migration { +namespace { + +static const std::string SNAPSHOTS_KEY {"snapshots"}; + + +} // anonymous namespace + #define dout_subsys ceph_subsys_rbd #undef dout_prefix #define dout_prefix *_dout << "librbd::migration::RawFormat: " << this \ @@ -37,8 +44,9 @@ void RawFormat::open(Context* on_finish) { handle_open(r, on_finish); }); // treat the base image as a HEAD-revision snapshot + Snapshots snapshots; int r = m_source_spec_builder->build_snapshot(m_json_object, CEPH_NOSNAP, - &m_snapshots[CEPH_NOSNAP]); + &snapshots[CEPH_NOSNAP]); if (r < 0) { lderr(cct) << "failed to build HEAD revision handler: " << cpp_strerror(r) << dendl; @@ -46,6 +54,36 @@ void RawFormat::open(Context* on_finish) { return; } + auto& snapshots_val = m_json_object[SNAPSHOTS_KEY]; + if (snapshots_val.type() == json_spirit::array_type) { + auto& snapshots_arr = snapshots_val.get_array(); + for (auto& snapshot_val : snapshots_arr) { + uint64_t index = snapshots.size(); + if (snapshot_val.type() != json_spirit::obj_type) { + lderr(cct) << "invalid snapshot " << index << " JSON: " + << cpp_strerror(r) << dendl; + on_finish->complete(-EINVAL); + return; + } + + auto& snapshot_obj = snapshot_val.get_obj(); + r = m_source_spec_builder->build_snapshot(snapshot_obj, index, + &snapshots[index]); + if (r < 0) { + lderr(cct) << "failed to build snapshot " << index << " handler: " + << cpp_strerror(r) << dendl; + on_finish->complete(r); + return; + } + } + } else if (snapshots_val.type() != json_spirit::null_type) { + lderr(cct) << "invalid snapshots array" << dendl; + on_finish->complete(-EINVAL); + return; + } + + m_snapshots = std::move(snapshots); + auto gather_ctx = new C_Gather(cct, on_finish); SnapshotInterface* previous_snapshot = nullptr; for (auto& [_, snapshot] : m_snapshots) { @@ -63,8 +101,16 @@ void RawFormat::handle_open(int r, Context* on_finish) { if (r < 0) { lderr(cct) << "failed to open raw image: " << cpp_strerror(r) << dendl; + + auto gather_ctx = new C_Gather(cct, on_finish); + for (auto& [_, snapshot] : m_snapshots) { + snapshot->close(gather_ctx->new_sub()); + } + m_image_ctx->state->close(new LambdaContext( - [r, on_finish=on_finish](int _) { on_finish->complete(r); })); + [r, on_finish=gather_ctx->new_sub()](int _) { on_finish->complete(r); })); + + gather_ctx->activate(); return; } @@ -147,13 +193,100 @@ void RawFormat::list_snaps(io::Extents&& image_extents, io::SnapshotDelta* snapshot_delta, const ZTracer::Trace &parent_trace, Context* on_finish) { - // raw does support snapshots so list the full IO extent as a delta - auto& snapshot = (*snapshot_delta)[{CEPH_NOSNAP, CEPH_NOSNAP}]; - for (auto& image_extent : image_extents) { - snapshot.insert(image_extent.first, image_extent.second, - {io::SPARSE_EXTENT_STATE_DATA, image_extent.second}); + auto cct = m_image_ctx->cct; + ldout(cct, 20) << "image_extents=" << image_extents << dendl; + + on_finish = new LambdaContext([this, snap_ids=std::move(snap_ids), + snapshot_delta, on_finish](int r) mutable { + handle_list_snaps(r, std::move(snap_ids), snapshot_delta, on_finish); + }); + + auto gather_ctx = new C_Gather(cct, on_finish); + + std::optional previous_size = std::nullopt; + for (auto& [snap_id, snapshot] : m_snapshots) { + auto& sparse_extents = (*snapshot_delta)[{snap_id, snap_id}]; + + // zero out any space between the previous snapshot end and this + // snapshot's end + auto& snap_info = snapshot->get_snap_info(); + if (previous_size && *previous_size > snap_info.size) { + ldout(cct, 20) << "snapshot resize " << *previous_size << " -> " + << snap_info.size << dendl; + interval_set zero_interval; + zero_interval.insert(snap_info.size, *previous_size - snap_info.size); + + for (auto& image_extent : image_extents) { + interval_set image_interval; + image_interval.insert(image_extent.first, image_extent.second); + + image_interval.intersection_of(zero_interval); + for (auto [image_offset, image_length] : image_interval) { + ldout(cct, 20) << "zeroing extent " << image_offset << "~" + << image_length << " at snapshot " << snap_id << dendl; + sparse_extents.insert(image_offset, image_length, + {io::SPARSE_EXTENT_STATE_ZEROED, image_length}); + } + } + } + previous_size = snap_info.size; + + // build set of data/zeroed extents for the current snapshot + snapshot->list_snap(io::Extents{image_extents}, list_snaps_flags, + &sparse_extents, parent_trace, gather_ctx->new_sub()); } - on_finish->complete(0); + + gather_ctx->activate(); +} + +template +void RawFormat::handle_list_snaps(int r, io::SnapIds&& snap_ids, + io::SnapshotDelta* snapshot_delta, + Context* on_finish) { + auto cct = m_image_ctx->cct; + ldout(cct, 20) << "r=" << r << ", " + << "snapshot_delta=" << snapshot_delta << dendl; + + io::SnapshotDelta orig_snapshot_delta = std::move(*snapshot_delta); + snapshot_delta->clear(); + + auto snap_id_it = snap_ids.begin(); + ceph_assert(snap_id_it != snap_ids.end()); + + // merge any snapshot intervals that were not requested + std::list pending_sparse_extents; + for (auto& [snap_key, sparse_extents] : orig_snapshot_delta) { + // advance to next valid requested snap id + while (snap_id_it != snap_ids.end() && *snap_id_it < snap_key.first) { + ++snap_id_it; + } + if (snap_id_it == snap_ids.end()) { + break; + } + + // loop through older write/read snapshot sparse extents to remove any + // overlaps with the current sparse extent + for (auto prev_sparse_extents : pending_sparse_extents) { + for (auto& sparse_extent : sparse_extents) { + prev_sparse_extents->erase(sparse_extent.get_off(), + sparse_extent.get_len()); + } + } + + auto write_read_snap_ids = std::make_pair(*snap_id_it, snap_key.second); + (*snapshot_delta)[write_read_snap_ids] = std::move(sparse_extents); + + if (write_read_snap_ids.first > snap_key.first) { + // the current snapshot wasn't requested so it might need to get + // merged with a later snapshot + pending_sparse_extents.push_back(&(*snapshot_delta)[write_read_snap_ids]); + } else { + // we don't merge results passed a valid requested snapshot + pending_sparse_extents.clear(); + } + } + + on_finish->complete(r); } } // namespace migration diff --git a/src/librbd/migration/RawFormat.h b/src/librbd/migration/RawFormat.h index 0febfe3fb1d..a20c0814f74 100644 --- a/src/librbd/migration/RawFormat.h +++ b/src/librbd/migration/RawFormat.h @@ -65,6 +65,9 @@ private: Snapshots m_snapshots; void handle_open(int r, Context* on_finish); + + void handle_list_snaps(int r, io::SnapIds&& snap_ids, + io::SnapshotDelta* snapshot_delta, Context* on_finish); }; } // namespace migration diff --git a/src/test/librbd/migration/test_mock_RawFormat.cc b/src/test/librbd/migration/test_mock_RawFormat.cc index 0adf6b9395c..d46ca436999 100644 --- a/src/test/librbd/migration/test_mock_RawFormat.cc +++ b/src/test/librbd/migration/test_mock_RawFormat.cc @@ -112,6 +112,19 @@ public: }))); } + void expect_snapshot_list_snap(MockSnapshotInterface& mock_snapshot_interface, + const io::Extents& image_extents, + const io::SparseExtents& sparse_extents, + int r) { + EXPECT_CALL(mock_snapshot_interface, list_snap(image_extents, _, _)) + .WillOnce(WithArgs<1, 2>(Invoke( + [sparse_extents, r](io::SparseExtents* out_sparse_extents, + Context* ctx) { + out_sparse_extents->insert(sparse_extents); + ctx->complete(r); + }))); + } + void expect_close(MockTestImageCtx &mock_image_ctx, int r) { EXPECT_CALL(*mock_image_ctx.state, close(_)) .WillOnce(Invoke([this, r](Context* ctx) { @@ -162,6 +175,7 @@ TEST_F(TestMockMigrationRawFormat, OpenError) { expect_snapshot_open(*mock_snapshot_interface, -ENOENT); + expect_snapshot_close(*mock_snapshot_interface, 0); expect_close(mock_image_ctx, 0); MockRawFormat mock_raw_format(&mock_image_ctx, json_object, @@ -172,6 +186,39 @@ TEST_F(TestMockMigrationRawFormat, OpenError) { ASSERT_EQ(-ENOENT, ctx.wait()); } +TEST_F(TestMockMigrationRawFormat, OpenSnapshotError) { + MockTestImageCtx mock_image_ctx(*m_image_ctx); + + InSequence seq; + MockSourceSpecBuilder mock_source_spec_builder; + + auto mock_snapshot_interface_head = new MockSnapshotInterface(); + expect_build_snapshot(mock_source_spec_builder, CEPH_NOSNAP, + mock_snapshot_interface_head, 0); + + auto mock_snapshot_interface_1 = new MockSnapshotInterface(); + expect_build_snapshot(mock_source_spec_builder, 1, + mock_snapshot_interface_1, 0); + + expect_snapshot_open(*mock_snapshot_interface_1, -ENOENT); + expect_snapshot_open(*mock_snapshot_interface_head, 0); + + expect_snapshot_close(*mock_snapshot_interface_1, 0); + expect_snapshot_close(*mock_snapshot_interface_head, 0); + expect_close(mock_image_ctx, 0); + + json_spirit::mArray snapshots; + snapshots.push_back(json_spirit::mObject{}); + json_object["snapshots"] = snapshots; + + MockRawFormat mock_raw_format(&mock_image_ctx, json_object, + &mock_source_spec_builder); + + C_SaferCond ctx; + mock_raw_format.open(&ctx); + ASSERT_EQ(-ENOENT, ctx.wait()); +} + TEST_F(TestMockMigrationRawFormat, GetSnapshots) { MockTestImageCtx mock_image_ctx(*m_image_ctx); @@ -331,6 +378,12 @@ TEST_F(TestMockMigrationRawFormat, ListSnaps) { SnapInfo snap_info{{}, {}, 123, {}, 0, 0, {}}; expect_snapshot_get_info(*mock_snapshot_interface, snap_info); + expect_snapshot_get_info(*mock_snapshot_interface, snap_info); + io::SparseExtents sparse_extents; + sparse_extents.insert(0, 123, {io::SPARSE_EXTENT_STATE_DATA, 123}); + expect_snapshot_list_snap(*mock_snapshot_interface, {{0, 123}}, + sparse_extents, 0); + expect_snapshot_close(*mock_snapshot_interface, 0); MockRawFormat mock_raw_format(&mock_image_ctx, json_object, @@ -342,9 +395,138 @@ TEST_F(TestMockMigrationRawFormat, ListSnaps) { C_SaferCond ctx2; io::SnapshotDelta snapshot_delta; - mock_raw_format.list_snaps({{0, 123}}, {}, 0, &snapshot_delta, {}, &ctx2); + mock_raw_format.list_snaps({{0, 123}}, {CEPH_NOSNAP}, 0, &snapshot_delta, {}, + &ctx2); ASSERT_EQ(0, ctx2.wait()); + io::SnapshotDelta expected_snapshot_delta; + expected_snapshot_delta[{CEPH_NOSNAP, CEPH_NOSNAP}] = sparse_extents; + ASSERT_EQ(expected_snapshot_delta, snapshot_delta); + + C_SaferCond ctx3; + mock_raw_format.close(&ctx3); + ASSERT_EQ(0, ctx3.wait()); +} + +TEST_F(TestMockMigrationRawFormat, ListSnapsError) { + MockTestImageCtx mock_image_ctx(*m_image_ctx); + + InSequence seq; + MockSourceSpecBuilder mock_source_spec_builder; + + auto mock_snapshot_interface = new MockSnapshotInterface(); + expect_build_snapshot(mock_source_spec_builder, CEPH_NOSNAP, + mock_snapshot_interface, 0); + + + expect_snapshot_open(*mock_snapshot_interface, 0); + SnapInfo snap_info{{}, {}, 123, {}, 0, 0, {}}; + expect_snapshot_get_info(*mock_snapshot_interface, snap_info); + + expect_snapshot_get_info(*mock_snapshot_interface, snap_info); + io::SparseExtents sparse_extents; + sparse_extents.insert(0, 123, {io::SPARSE_EXTENT_STATE_DATA, 123}); + expect_snapshot_list_snap(*mock_snapshot_interface, {{0, 123}}, + sparse_extents, -EINVAL); + + expect_snapshot_close(*mock_snapshot_interface, 0); + + MockRawFormat mock_raw_format(&mock_image_ctx, json_object, + &mock_source_spec_builder); + + C_SaferCond ctx1; + mock_raw_format.open(&ctx1); + ASSERT_EQ(0, ctx1.wait()); + + C_SaferCond ctx2; + io::SnapshotDelta snapshot_delta; + mock_raw_format.list_snaps({{0, 123}}, {CEPH_NOSNAP}, 0, &snapshot_delta, {}, + &ctx2); + ASSERT_EQ(-EINVAL, ctx2.wait()); + + C_SaferCond ctx3; + mock_raw_format.close(&ctx3); + ASSERT_EQ(0, ctx3.wait()); +} + +TEST_F(TestMockMigrationRawFormat, ListSnapsMerge) { + MockTestImageCtx mock_image_ctx(*m_image_ctx); + + InSequence seq; + MockSourceSpecBuilder mock_source_spec_builder; + + auto mock_snapshot_interface_head = new MockSnapshotInterface(); + expect_build_snapshot(mock_source_spec_builder, CEPH_NOSNAP, + mock_snapshot_interface_head, 0); + + auto mock_snapshot_interface_1 = new MockSnapshotInterface(); + expect_build_snapshot(mock_source_spec_builder, 1, + mock_snapshot_interface_1, 0); + + auto mock_snapshot_interface_2 = new MockSnapshotInterface(); + expect_build_snapshot(mock_source_spec_builder, 2, + mock_snapshot_interface_2, 0); + + + expect_snapshot_open(*mock_snapshot_interface_1, 0); + expect_snapshot_open(*mock_snapshot_interface_2, 0); + expect_snapshot_open(*mock_snapshot_interface_head, 0); + + SnapInfo snap_info_head{{}, {}, 256, {}, 0, 0, {}}; + expect_snapshot_get_info(*mock_snapshot_interface_head, snap_info_head); + + SnapInfo snap_info_1{snap_info_head}; + snap_info_1.size = 123; + expect_snapshot_get_info(*mock_snapshot_interface_1, snap_info_1); + io::SparseExtents sparse_extents_1; + sparse_extents_1.insert(0, 123, {io::SPARSE_EXTENT_STATE_DATA, 123}); + expect_snapshot_list_snap(*mock_snapshot_interface_1, {{0, 123}}, + sparse_extents_1, 0); + + SnapInfo snap_info_2{snap_info_head}; + snap_info_2.size = 64; + expect_snapshot_get_info(*mock_snapshot_interface_2, snap_info_2); + io::SparseExtents sparse_extents_2; + sparse_extents_2.insert(0, 32, {io::SPARSE_EXTENT_STATE_DATA, 32}); + expect_snapshot_list_snap(*mock_snapshot_interface_2, {{0, 123}}, + sparse_extents_2, 0); + + expect_snapshot_get_info(*mock_snapshot_interface_head, snap_info_head); + io::SparseExtents sparse_extents_head; + sparse_extents_head.insert(0, 16, {io::SPARSE_EXTENT_STATE_DATA, 16}); + expect_snapshot_list_snap(*mock_snapshot_interface_head, {{0, 123}}, + sparse_extents_head, 0); + + expect_snapshot_close(*mock_snapshot_interface_1, 0); + expect_snapshot_close(*mock_snapshot_interface_2, 0); + expect_snapshot_close(*mock_snapshot_interface_head, 0); + + json_spirit::mArray snapshots; + snapshots.push_back(json_spirit::mObject{}); + snapshots.push_back(json_spirit::mObject{}); + json_object["snapshots"] = snapshots; + + MockRawFormat mock_raw_format(&mock_image_ctx, json_object, + &mock_source_spec_builder); + + C_SaferCond ctx1; + mock_raw_format.open(&ctx1); + ASSERT_EQ(0, ctx1.wait()); + + C_SaferCond ctx2; + io::SnapshotDelta snapshot_delta; + mock_raw_format.list_snaps({{0, 123}}, {1, CEPH_NOSNAP}, 0, &snapshot_delta, + {}, &ctx2); + ASSERT_EQ(0, ctx2.wait()); + + io::SnapshotDelta expected_snapshot_delta; + expected_snapshot_delta[{1, 1}] = sparse_extents_1; + sparse_extents_2.erase(0, 16); + sparse_extents_2.insert(64, 59, {io::SPARSE_EXTENT_STATE_ZEROED, 59}); + expected_snapshot_delta[{CEPH_NOSNAP, 2}] = sparse_extents_2; + expected_snapshot_delta[{CEPH_NOSNAP, CEPH_NOSNAP}] = sparse_extents_head; + ASSERT_EQ(expected_snapshot_delta, snapshot_delta); + C_SaferCond ctx3; mock_raw_format.close(&ctx3); ASSERT_EQ(0, ctx3.wait()); -- 2.39.5