From: Bill Scales Date: Fri, 29 Nov 2024 09:51:46 +0000 (+0000) Subject: osd: EC: helper functions for chunk mapping X-Git-Tag: v20.0.0~288^2~1 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=32f22f4d2540b1adffc10e780f407ec68a69d1de;p=ceph.git osd: EC: helper functions for chunk mapping Tidy up EC code. Extend stripe_info_t to store K, M, a complete mapping for shard->raw_shard and the reverse mapping raw_shard->shard. Signed-off-by: Bill Scales --- diff --git a/src/osd/ECBackend.cc b/src/osd/ECBackend.cc index 3c06268995e4..a1d32c1a5119 100644 --- a/src/osd/ECBackend.cc +++ b/src/osd/ECBackend.cc @@ -132,7 +132,7 @@ ECBackend::ECBackend( rmw_pipeline(cct, ec_impl, this->sinfo, get_parent()->get_eclistener(), *this), recovery_backend(cct, this->coll, ec_impl, this->sinfo, read_pipeline, unstable_hashinfo_registry, get_parent(), this), ec_impl(ec_impl), - sinfo(ec_impl->get_data_chunk_count(), stripe_width), + sinfo(ec_impl, stripe_width), unstable_hashinfo_registry(cct, ec_impl) { ceph_assert((ec_impl->get_data_chunk_count() * ec_impl->get_chunk_size(stripe_width)) == stripe_width); @@ -990,8 +990,7 @@ void ECBackend::handle_sub_write( async); if (!get_parent()->pg_is_undersized() && - (unsigned)get_parent()->whoami_shard().shard >= - ec_impl->get_data_chunk_count()) + (unsigned)get_parent()->whoami_shard().shard >= sinfo.get_k()) op.t.set_fadvise_flag(CEPH_OSD_OP_FLAG_FADVISE_DONTNEED); localt.register_on_commit( diff --git a/src/osd/ECCommon.cc b/src/osd/ECCommon.cc index 99f9a90eea6b..9e981d54b796 100644 --- a/src/osd/ECCommon.cc +++ b/src/osd/ECCommon.cc @@ -322,19 +322,15 @@ void ECCommon::ReadPipeline::get_min_want_to_read_shards( const uint64_t offset, const uint64_t length, const ECUtil::stripe_info_t& sinfo, - const vector& chunk_mapping, set *want_to_read) { const auto [left_chunk_index, right_chunk_index] = sinfo.offset_length_to_data_chunk_indices(offset, length); const auto distance = - std::min(right_chunk_index - left_chunk_index, - sinfo.get_data_chunk_count()); + std::min(right_chunk_index - left_chunk_index, (uint64_t)sinfo.get_k()); for(uint64_t i = 0; i < distance; i++) { - auto raw_chunk = (left_chunk_index + i) % sinfo.get_data_chunk_count(); - auto chunk = chunk_mapping.size() > raw_chunk ? - chunk_mapping[raw_chunk] : static_cast(raw_chunk); - want_to_read->insert(chunk); + auto raw_shard = (left_chunk_index + i) % sinfo.get_k(); + want_to_read->insert(sinfo.get_shard(raw_shard)); } } @@ -343,8 +339,7 @@ void ECCommon::ReadPipeline::get_min_want_to_read_shards( const uint64_t length, set *want_to_read) { - get_min_want_to_read_shards( - offset, length, sinfo, ec_impl->get_chunk_mapping(), want_to_read); + get_min_want_to_read_shards(offset, length, sinfo, want_to_read); dout(20) << __func__ << ": offset " << offset << " length " << length << " want_to_read " << *want_to_read << dendl; } @@ -502,10 +497,8 @@ void ECCommon::ReadPipeline::do_read_op(ReadOp &op) void ECCommon::ReadPipeline::get_want_to_read_shards( std::set *want_to_read) const { - const std::vector &chunk_mapping = ec_impl->get_chunk_mapping(); - for (int i = 0; i < (int)ec_impl->get_data_chunk_count(); ++i) { - int chunk = (int)chunk_mapping.size() > i ? chunk_mapping[i] : i; - want_to_read->insert(chunk); + for (int i = 0; i < (int)sinfo.get_k(); ++i) { + want_to_read->insert(sinfo.get_shard(i)); } } diff --git a/src/osd/ECCommon.h b/src/osd/ECCommon.h index 7ff9cae7646a..7e533e01562e 100644 --- a/src/osd/ECCommon.h +++ b/src/osd/ECCommon.h @@ -460,7 +460,6 @@ struct ECCommon { const uint64_t offset, const uint64_t length, const ECUtil::stripe_info_t& sinfo, - const std::vector& chunk_mapping, std::set *want_to_read); int get_remaining_shards( diff --git a/src/osd/ECUtil.h b/src/osd/ECUtil.h index c84a87ee3804..edaf297bc645 100644 --- a/src/osd/ECUtil.h +++ b/src/osd/ECUtil.h @@ -27,11 +27,70 @@ namespace ECUtil { class stripe_info_t { const uint64_t stripe_width; const uint64_t chunk_size; + const unsigned int k; // Can be calculated with a division from above. Better to cache. + const unsigned int m; + const std::vector chunk_mapping; + const std::vector chunk_mapping_reverse; +private: + static std::vector complete_chunk_mapping( + std::vector _chunk_mapping, unsigned int n) + { + unsigned int size = _chunk_mapping.size(); + std::vector chunk_mapping(n); + for (unsigned int i = 0; i < n; i++) { + if (size > i) { + chunk_mapping.at(i) = _chunk_mapping.at(i); + } else { + chunk_mapping.at(i) = static_cast(i); + } + } + return chunk_mapping; + } + static std::vector reverse_chunk_mapping( + std::vector chunk_mapping) + { + unsigned int size = chunk_mapping.size(); + std::vector reverse(size); + std::vector used(size,false); + for (unsigned int i = 0; i < size; i++) { + int index = chunk_mapping.at(i); + // Mapping must be a bijection and a permutation + ceph_assert(!used.at(index)); + used.at(index) = true; + reverse.at(index) = i; + } + return reverse; + } public: - stripe_info_t(uint64_t stripe_size, uint64_t stripe_width) + stripe_info_t(ErasureCodeInterfaceRef ec_impl, uint64_t stripe_width) + : stripe_width(stripe_width), + chunk_size(stripe_width / ec_impl->get_data_chunk_count()), + k(ec_impl->get_data_chunk_count()), + m(ec_impl->get_coding_chunk_count()), + chunk_mapping(complete_chunk_mapping(ec_impl->get_chunk_mapping(), + k + m)), + chunk_mapping_reverse(reverse_chunk_mapping(chunk_mapping)) { + ceph_assert(stripe_width % k == 0); + } + // Simpler constructors for unit tests + stripe_info_t(unsigned int k, unsigned int m, uint64_t stripe_width) : stripe_width(stripe_width), - chunk_size(stripe_width / stripe_size) { - ceph_assert(stripe_width % stripe_size == 0); + chunk_size(stripe_width / k), + k(k), + m(m), + chunk_mapping(complete_chunk_mapping(std::vector(), k + m)), + chunk_mapping_reverse(reverse_chunk_mapping(chunk_mapping)) { + ceph_assert(stripe_width % k == 0); + } + stripe_info_t(unsigned int k, unsigned int m, uint64_t stripe_width, + std::vector _chunk_mapping) + : stripe_width(stripe_width), + chunk_size(stripe_width / k), + k(k), + m(m), + chunk_mapping(complete_chunk_mapping(_chunk_mapping, k + m)), + chunk_mapping_reverse(reverse_chunk_mapping(chunk_mapping)) { + ceph_assert(stripe_width % k == 0); } bool logical_offset_is_stripe_aligned(uint64_t logical) const { return (logical % stripe_width) == 0; @@ -42,8 +101,20 @@ public: uint64_t get_chunk_size() const { return chunk_size; } - uint64_t get_data_chunk_count() const { - return get_stripe_width() / get_chunk_size(); + unsigned int get_m() const { + return m; + } + unsigned int get_k() const { + return k; + } + unsigned int get_k_plus_m() const { + return k + m; + } + int get_shard(unsigned int raw_shard) const { + return chunk_mapping[raw_shard]; + } + unsigned int get_raw_shard(int shard) const { + return chunk_mapping_reverse[shard]; } uint64_t logical_to_prev_chunk_offset(uint64_t offset) const { return (offset / stripe_width) * chunk_size; diff --git a/src/test/crimson/test_crimson_scrub.cc b/src/test/crimson/test_crimson_scrub.cc index 0b69e2d8ce9a..6548f6d9973d 100644 --- a/src/test/crimson/test_crimson_scrub.cc +++ b/src/test/crimson/test_crimson_scrub.cc @@ -280,7 +280,7 @@ struct test_obj_t : so_builder_t { static test_obj_t make_ec_head(const std::string &desc, Args&&... args) { return make( desc, - ECUtil::stripe_info_t{4, 1<<20}, + ECUtil::stripe_info_t{4, 2, 1<<20}, so_builder_t::make_ec_head(std::forward(args)...)); } @@ -288,7 +288,7 @@ struct test_obj_t : so_builder_t { static test_obj_t make_ec_clone(const std::string &desc, Args&&... args) { return make( desc, - ECUtil::stripe_info_t{4, 1<<20}, + ECUtil::stripe_info_t{4, 2, 1<<20}, so_builder_t::make_ec_clone(std::forward(args)...)); } diff --git a/src/test/osd/TestECBackend.cc b/src/test/osd/TestECBackend.cc index 1421eedfe4fa..c979e84a518e 100644 --- a/src/test/osd/TestECBackend.cc +++ b/src/test/osd/TestECBackend.cc @@ -25,9 +25,10 @@ using namespace std; TEST(ECUtil, stripe_info_t) { const uint64_t swidth = 4096; - const uint64_t ssize = 4; + const unsigned int k = 4; + const unsigned int m = 2; - ECUtil::stripe_info_t s(ssize, swidth); + ECUtil::stripe_info_t s(k, m, swidth); ASSERT_EQ(s.get_stripe_width(), swidth); ASSERT_EQ(s.logical_to_next_chunk_offset(0), 0u); @@ -90,9 +91,10 @@ TEST(ECUtil, offset_length_is_same_stripe) { const uint64_t swidth = 4096; const uint64_t schunk = 1024; - const uint64_t ssize = 4; + const unsigned int k = 4; + const unsigned int m = 2; - ECUtil::stripe_info_t s(ssize, swidth); + ECUtil::stripe_info_t s(k, m, swidth); ASSERT_EQ(s.get_stripe_width(), swidth); ASSERT_EQ(s.get_chunk_size(), schunk); @@ -165,19 +167,18 @@ TEST(ECUtil, offset_length_is_same_stripe) TEST(ECCommon, get_min_want_to_read_shards) { const uint64_t swidth = 4096; - const uint64_t ssize = 4; + const unsigned int k = 4; + const unsigned int m = 2; - ECUtil::stripe_info_t s(ssize, swidth); + ECUtil::stripe_info_t s(k, m, swidth); ASSERT_EQ(s.get_stripe_width(), swidth); ASSERT_EQ(s.get_chunk_size(), 1024); - const std::vector chunk_mapping = {}; // no remapping - // read nothing at the very beginning { std::set want_to_read; ECCommon::ReadPipeline::get_min_want_to_read_shards( - 0, 0, s, chunk_mapping, &want_to_read); + 0, 0, s, &want_to_read); ASSERT_TRUE(want_to_read == std::set{}); } @@ -185,7 +186,7 @@ TEST(ECCommon, get_min_want_to_read_shards) { std::set want_to_read; ECCommon::ReadPipeline::get_min_want_to_read_shards( - 2048, 0, s, chunk_mapping, &want_to_read); + 2048, 0, s, &want_to_read); ASSERT_TRUE(want_to_read == std::set{}); } @@ -193,7 +194,7 @@ TEST(ECCommon, get_min_want_to_read_shards) { std::set want_to_read; ECCommon::ReadPipeline::get_min_want_to_read_shards( - 2048, 42, s, chunk_mapping, &want_to_read); + 2048, 42, s, &want_to_read); ASSERT_TRUE(want_to_read == std::set{2}); } @@ -201,7 +202,7 @@ TEST(ECCommon, get_min_want_to_read_shards) { std::set want_to_read; ECCommon::ReadPipeline::get_min_want_to_read_shards( - 1024, 1024+42, s, chunk_mapping, &want_to_read); + 1024, 1024+42, s, &want_to_read); // extra () due to a language / macro limitation ASSERT_TRUE(want_to_read == (std::set{1, 2})); } @@ -210,7 +211,7 @@ TEST(ECCommon, get_min_want_to_read_shards) { std::set want_to_read; ECCommon::ReadPipeline::get_min_want_to_read_shards( - 0, 3*1024, s, chunk_mapping, &want_to_read); + 0, 3*1024, s, &want_to_read); // extra () due to a language / macro limitation ASSERT_TRUE(want_to_read == (std::set{0, 1, 2})); } @@ -219,7 +220,7 @@ TEST(ECCommon, get_min_want_to_read_shards) { std::set want_to_read; ECCommon::ReadPipeline::get_min_want_to_read_shards( - 1024, swidth-1024, s, chunk_mapping, &want_to_read); + 1024, swidth-1024, s, &want_to_read); // extra () due to a language / macro limitation ASSERT_TRUE(want_to_read == (std::set{1, 2, 3})); } @@ -228,7 +229,7 @@ TEST(ECCommon, get_min_want_to_read_shards) { std::set want_to_read; ECCommon::ReadPipeline::get_min_want_to_read_shards( - 1024, swidth*42, s, chunk_mapping, &want_to_read); + 1024, swidth*42, s, &want_to_read); // extra () due to a language / macro limitation ASSERT_TRUE(want_to_read == (std::set{0, 1, 2, 3})); } @@ -237,7 +238,7 @@ TEST(ECCommon, get_min_want_to_read_shards) { std::set want_to_read; ECCommon::ReadPipeline::get_min_want_to_read_shards( - 0, swidth*42, s, chunk_mapping, &want_to_read); + 0, swidth*42, s, &want_to_read); // extra () due to a language / macro limitation ASSERT_TRUE(want_to_read == (std::set{0, 1, 2, 3})); } @@ -246,24 +247,23 @@ TEST(ECCommon, get_min_want_to_read_shards) TEST(ECCommon, get_min_want_to_read_shards_bug67087) { const uint64_t swidth = 4096; - const uint64_t ssize = 4; + const unsigned int k = 4; + const unsigned int m = 2; - ECUtil::stripe_info_t s(ssize, swidth); + ECUtil::stripe_info_t s(k, m, swidth); ASSERT_EQ(s.get_stripe_width(), swidth); ASSERT_EQ(s.get_chunk_size(), 1024); - const std::vector chunk_mapping = {}; // no remapping - std::set want_to_read; // multitple calls with the same want_to_read can happen during // multi-region reads. { ECCommon::ReadPipeline::get_min_want_to_read_shards( - 512, 512, s, chunk_mapping, &want_to_read); + 512, 512, s, &want_to_read); ASSERT_EQ(want_to_read, std::set{0}); ECCommon::ReadPipeline::get_min_want_to_read_shards( - 512+16*1024, 512, s, chunk_mapping, &want_to_read); + 512+16*1024, 512, s, &want_to_read); ASSERT_EQ(want_to_read, std::set{0}); } } diff --git a/src/test/osd/test_ec_transaction.cc b/src/test/osd/test_ec_transaction.cc index c17df4802ede..82f42bad489e 100644 --- a/src/test/osd/test_ec_transaction.cc +++ b/src/test/osd/test_ec_transaction.cc @@ -37,7 +37,7 @@ TEST(ectransaction, two_writes_separated) b.append_zero(2437120); t->write(h, 669856, b.length(), b, 0); - ECUtil::stripe_info_t sinfo(2, 8192); + ECUtil::stripe_info_t sinfo(2, 2, 8192); auto plan = ECTransaction::get_write_plan( sinfo, *t, @@ -61,7 +61,7 @@ TEST(ectransaction, two_writes_nearby) t->create(h); // two nearby writes, both partly touching the same 8192-byte stripe - ECUtil::stripe_info_t sinfo(2, 8192); + ECUtil::stripe_info_t sinfo(2, 2, 8192); a.append_zero(565760); t->write(h, 0, a.length(), a, 0); b.append_zero(2437120); @@ -91,7 +91,7 @@ TEST(ectransaction, many_writes) b.append_zero(4096); t->create(h); - ECUtil::stripe_info_t sinfo(2, 8192); + ECUtil::stripe_info_t sinfo(2, 2, 8192); // write 2801664~512 // write 2802176~512 // write 2802688~512 diff --git a/src/tools/erasure-code/ceph-erasure-code-tool.cc b/src/tools/erasure-code/ceph-erasure-code-tool.cc index 51343f7d615e..ef6c91902a6c 100644 --- a/src/tools/erasure-code/ceph-erasure-code-tool.cc +++ b/src/tools/erasure-code/ceph-erasure-code-tool.cc @@ -96,7 +96,7 @@ int ec_init(const std::string &profile_str, uint64_t stripe_size = atoi(profile["k"].c_str()); ceph_assert(stripe_size > 0); uint64_t stripe_width = stripe_size * stripe_unit; - sinfo->reset(new ECUtil::stripe_info_t(stripe_size, stripe_width)); + sinfo->reset(new ECUtil::stripe_info_t(*ec_impl, stripe_width)); return 0; }