From: Jon Bailey Date: Mon, 12 May 2025 20:15:06 +0000 (+0100) Subject: erasure-code: Add new flag for whether plugins support encoding/decoding or CRCs X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=a05bb75cf27b0e262c1cbb770719a619e60629ac;p=ceph.git erasure-code: Add new flag for whether plugins support encoding/decoding or CRCs Add new flag for whether plugins support encoding/decoding or CRCs to get the CRC of what the encoded/decoded data would be. Signed-off-by: Jon Bailey --- diff --git a/src/erasure-code/ErasureCodeInterface.h b/src/erasure-code/ErasureCodeInterface.h index eb1651a5dbf53..17c2827d50929 100644 --- a/src/erasure-code/ErasureCodeInterface.h +++ b/src/erasure-code/ErasureCodeInterface.h @@ -679,6 +679,11 @@ namespace ceph { * are irrelevant if this flag is false. */ FLAG_EC_PLUGIN_OPTIMIZED_SUPPORTED = 1<<6, + /* This plugin supports the ability to encode CRCs of data shards to get + * the CRC of a parity shard. This flag also represents the inverse, + * to decode a parity CRC to get the CRC of a data shard. + */ + FLAG_EC_PLUGIN_CRC_ENCODE_DECODE_SUPPORT = 1<<7, }; static const char *get_optimization_flag_name(const plugin_flags flag) { switch (flag) { @@ -689,6 +694,8 @@ namespace ceph { case FLAG_EC_PLUGIN_PARITY_DELTA_OPTIMIZATION: return "paritydelta"; case FLAG_EC_PLUGIN_REQUIRE_SUB_CHUNKS: return "requiresubchunks"; case FLAG_EC_PLUGIN_OPTIMIZED_SUPPORTED: return "optimizedsupport"; + case FLAG_EC_PLUGIN_CRC_ENCODE_DECODE_SUPPORT: + return "crcencodedecode"; default: return "???"; } } diff --git a/src/erasure-code/clay/ErasureCodeClay.h b/src/erasure-code/clay/ErasureCodeClay.h index 0391f614b7679..4024d978c4aab 100644 --- a/src/erasure-code/clay/ErasureCodeClay.h +++ b/src/erasure-code/clay/ErasureCodeClay.h @@ -52,7 +52,8 @@ public: // the corner case of m = 1 return FLAG_EC_PLUGIN_PARTIAL_READ_OPTIMIZATION | FLAG_EC_PLUGIN_PARTIAL_WRITE_OPTIMIZATION | - FLAG_EC_PLUGIN_REQUIRE_SUB_CHUNKS; + FLAG_EC_PLUGIN_REQUIRE_SUB_CHUNKS | + FLAG_EC_PLUGIN_CRC_ENCODE_DECODE_SUPPORT; } return FLAG_EC_PLUGIN_PARTIAL_READ_OPTIMIZATION | FLAG_EC_PLUGIN_REQUIRE_SUB_CHUNKS; diff --git a/src/erasure-code/isa/ErasureCodeIsa.h b/src/erasure-code/isa/ErasureCodeIsa.h index f68fa527d99e1..0b2065cd25edd 100644 --- a/src/erasure-code/isa/ErasureCodeIsa.h +++ b/src/erasure-code/isa/ErasureCodeIsa.h @@ -53,10 +53,10 @@ public: int w; ErasureCodeIsaTableCache &tcache; - const char *technique; + std::string technique; uint64_t flags; - ErasureCodeIsa(const char *_technique, + ErasureCodeIsa(const std::string &_technique, ErasureCodeIsaTableCache &_tcache) : k(0), m(0), @@ -69,9 +69,11 @@ public: FLAG_EC_PLUGIN_ZERO_INPUT_ZERO_OUTPUT_OPTIMIZATION | FLAG_EC_PLUGIN_PARITY_DELTA_OPTIMIZATION; - if (technique == "reed_sol_van"sv || - technique == "default"sv) { - flags |= FLAG_EC_PLUGIN_OPTIMIZED_SUPPORTED; + if (technique == "reed_sol_van"sv) { + flags |= FLAG_EC_PLUGIN_OPTIMIZED_SUPPORTED | + FLAG_EC_PLUGIN_CRC_ENCODE_DECODE_SUPPORT; + } else if (technique == "cauchy"sv && m == 1) { + flags |= FLAG_EC_PLUGIN_CRC_ENCODE_DECODE_SUPPORT; } } @@ -151,15 +153,14 @@ public: unsigned char* encode_tbls; // encoding table ErasureCodeIsaDefault(ErasureCodeIsaTableCache &_tcache, + const std::string& technique, int matrix = kVandermonde) : - - ErasureCodeIsa("default", _tcache), + ErasureCodeIsa(technique, _tcache), encode_coeff(0), encode_tbls(0) { matrixtype = matrix; } - ~ErasureCodeIsaDefault() override { diff --git a/src/erasure-code/isa/ErasureCodePluginIsa.cc b/src/erasure-code/isa/ErasureCodePluginIsa.cc index a3f794e8cfeed..80a4b803de158 100644 --- a/src/erasure-code/isa/ErasureCodePluginIsa.cc +++ b/src/erasure-code/isa/ErasureCodePluginIsa.cc @@ -36,24 +36,22 @@ int ErasureCodePluginIsa::factory(const std::string &directory, std::ostream *ss) { ErasureCodeIsa *interface; - std::string t; - if (profile.find("technique") == profile.end()) - profile["technique"] = "reed_sol_van"; - t = profile.find("technique")->second; - if ((t == "reed_sol_van")) { + std::string technique; + technique = profile.find("technique")->second; + if ((technique == "reed_sol_van")) { interface = new ErasureCodeIsaDefault(tcache, + technique, ErasureCodeIsaDefault::kVandermonde); + } else if ((technique == "cauchy")) { + interface = new ErasureCodeIsaDefault(tcache, + technique, + ErasureCodeIsaDefault::kCauchy); } else { - if ((t == "cauchy")) { - interface = new ErasureCodeIsaDefault(tcache, - ErasureCodeIsaDefault::kCauchy); - } else { - *ss << "technique=" << t << " is not a valid coding technique. " - << " Choose one of the following: " - << "reed_sol_van," - << "cauchy" << std::endl; - return -ENOENT; - } + *ss << "technique=" << technique << " is not a valid coding technique. " + << " Choose one of the following: " + << "reed_sol_van," + << "cauchy" << std::endl; + return -ENOENT; } int r = interface->init(profile, ss); diff --git a/src/erasure-code/jerasure/ErasureCodeJerasure.h b/src/erasure-code/jerasure/ErasureCodeJerasure.h index 3bbd5d0448c10..f7308103e530c 100644 --- a/src/erasure-code/jerasure/ErasureCodeJerasure.h +++ b/src/erasure-code/jerasure/ErasureCodeJerasure.h @@ -54,6 +54,8 @@ public: if (technique == "reed_sol_van"sv) { flags |= FLAG_EC_PLUGIN_OPTIMIZED_SUPPORTED; + } else if (technique != "cauchy_orig"sv) { + flags |= FLAG_EC_PLUGIN_CRC_ENCODE_DECODE_SUPPORT; } } diff --git a/src/erasure-code/shec/ErasureCodeShec.h b/src/erasure-code/shec/ErasureCodeShec.h index 0af840aef583f..9e8838c2f9291 100644 --- a/src/erasure-code/shec/ErasureCodeShec.h +++ b/src/erasure-code/shec/ErasureCodeShec.h @@ -65,7 +65,8 @@ public: return FLAG_EC_PLUGIN_PARTIAL_READ_OPTIMIZATION | FLAG_EC_PLUGIN_PARTIAL_WRITE_OPTIMIZATION | FLAG_EC_PLUGIN_ZERO_INPUT_ZERO_OUTPUT_OPTIMIZATION | - FLAG_EC_PLUGIN_PARITY_DELTA_OPTIMIZATION; + FLAG_EC_PLUGIN_PARITY_DELTA_OPTIMIZATION | + FLAG_EC_PLUGIN_CRC_ENCODE_DECODE_SUPPORT; } unsigned int get_chunk_count() const override { diff --git a/src/osd/ECUtil.h b/src/osd/ECUtil.h index 04ac579d69948..8a228795054c7 100644 --- a/src/osd/ECUtil.h +++ b/src/osd/ECUtil.h @@ -574,6 +574,11 @@ public: ErasureCodeInterface::FLAG_EC_PLUGIN_PARITY_DELTA_OPTIMIZATION) != 0; } + bool supports_encode_decode_crcs() const { + return (plugin_flags & + ErasureCodeInterface::FLAG_EC_PLUGIN_CRC_ENCODE_DECODE_SUPPORT) != 0; + } + uint64_t get_stripe_width() const { return stripe_width; } diff --git a/src/test/erasure-code/TestErasureCodeIsa.cc b/src/test/erasure-code/TestErasureCodeIsa.cc index 797090805f5fb..3112e60495ca8 100644 --- a/src/test/erasure-code/TestErasureCodeIsa.cc +++ b/src/test/erasure-code/TestErasureCodeIsa.cc @@ -49,7 +49,7 @@ void IsaErasureCodeTest::compare_chunks(bufferlist &in, shard_id_map void IsaErasureCodeTest::encode_decode(unsigned object_size) { - ErasureCodeIsaDefault Isa(tcache); + ErasureCodeIsaDefault Isa(tcache, "reed_sol_van"); ErasureCodeProfile profile; profile["k"] = "2"; @@ -190,7 +190,7 @@ TEST_F(IsaErasureCodeTest, encode_decode) TEST_F(IsaErasureCodeTest, minimum_to_decode) { - ErasureCodeIsaDefault Isa(tcache); + ErasureCodeIsaDefault Isa(tcache, "reed_sol_van"); ErasureCodeProfile profile; profile["k"] = "2"; profile["m"] = "2"; @@ -287,7 +287,7 @@ TEST_F(IsaErasureCodeTest, minimum_to_decode) TEST_F(IsaErasureCodeTest, chunk_size) { { - ErasureCodeIsaDefault Isa(tcache); + ErasureCodeIsaDefault Isa(tcache, "reed_sol_van"); ErasureCodeProfile profile; profile["k"] = "2"; profile["m"] = "1"; @@ -299,7 +299,7 @@ TEST_F(IsaErasureCodeTest, chunk_size) ASSERT_EQ(EC_ISA_ADDRESS_ALIGNMENT * 2, Isa.get_chunk_size(EC_ISA_ADDRESS_ALIGNMENT * k + 1)); } { - ErasureCodeIsaDefault Isa(tcache); + ErasureCodeIsaDefault Isa(tcache, "reed_sol_van"); ErasureCodeProfile profile; profile["k"] = "3"; profile["m"] = "1"; @@ -320,7 +320,7 @@ TEST_F(IsaErasureCodeTest, chunk_size) TEST_F(IsaErasureCodeTest, encode) { - ErasureCodeIsaDefault Isa(tcache); + ErasureCodeIsaDefault Isa(tcache, "reed_sol_van"); ErasureCodeProfile profile; profile["k"] = "2"; profile["m"] = "2"; @@ -369,7 +369,7 @@ TEST_F(IsaErasureCodeTest, encode) TEST_F(IsaErasureCodeTest, sanity_check_k) { - ErasureCodeIsaDefault Isa(tcache); + ErasureCodeIsaDefault Isa(tcache, "reed_sol_van"); ErasureCodeProfile profile; profile["k"] = "1"; profile["m"] = "1"; @@ -402,7 +402,7 @@ TEST_F(IsaErasureCodeTest, isa_vandermonde_exhaustive) // Test all possible failure scenarios and reconstruction cases for // a (12,4) configuration using the vandermonde matrix - ErasureCodeIsaDefault Isa(tcache); + ErasureCodeIsaDefault Isa(tcache, "reed_sol_van"); ErasureCodeProfile profile; profile["k"] = "12"; profile["m"] = "4"; @@ -527,7 +527,7 @@ TEST_F(IsaErasureCodeTest, isa_cauchy_exhaustive) { // Test all possible failure scenarios and reconstruction cases for // a (12,4) configuration using the cauchy matrix - ErasureCodeIsaDefault Isa(tcache,ErasureCodeIsaDefault::kCauchy); + ErasureCodeIsaDefault Isa(tcache, "cauchy", ErasureCodeIsaDefault::kCauchy); ErasureCodeProfile profile; profile["k"] = "12"; profile["m"] = "4"; @@ -654,7 +654,7 @@ TEST_F(IsaErasureCodeTest, isa_cauchy_cache_trash) { // Test all possible failure scenarios and reconstruction cases for // a (12,4) configuration using the cauchy matrix - ErasureCodeIsaDefault Isa(tcache,ErasureCodeIsaDefault::kCauchy); + ErasureCodeIsaDefault Isa(tcache, "cauchy", ErasureCodeIsaDefault::kCauchy); ErasureCodeProfile profile; profile["k"] = "16"; profile["m"] = "4"; @@ -782,7 +782,7 @@ TEST_F(IsaErasureCodeTest, isa_xor_codec) // Test all possible failure scenarios and reconstruction cases for // a (4,1) RAID-5 like configuration - ErasureCodeIsaDefault Isa(tcache); + ErasureCodeIsaDefault Isa(tcache, "reed_sol_van"); ErasureCodeProfile profile; profile["k"] = "4"; profile["m"] = "1"; @@ -908,7 +908,7 @@ TEST_F(IsaErasureCodeTest, create_rule) { stringstream ss; - ErasureCodeIsaDefault isa(tcache); + ErasureCodeIsaDefault isa(tcache, "reed_sol_van"); ErasureCodeProfile profile; profile["k"] = "2"; profile["m"] = "2"; @@ -932,7 +932,7 @@ TEST_F(IsaErasureCodeTest, create_rule) } { stringstream ss; - ErasureCodeIsaDefault isa(tcache); + ErasureCodeIsaDefault isa(tcache, "reed_sol_van"); ErasureCodeProfile profile; profile["k"] = "2"; profile["m"] = "2"; @@ -944,7 +944,7 @@ TEST_F(IsaErasureCodeTest, create_rule) } { stringstream ss; - ErasureCodeIsaDefault isa(tcache); + ErasureCodeIsaDefault isa(tcache, "reed_sol_van"); ErasureCodeProfile profile; profile["k"] = "2"; profile["m"] = "2"; diff --git a/src/test/erasure-code/TestErasureCodePlugins.cc b/src/test/erasure-code/TestErasureCodePlugins.cc index 0941a07cf5fa2..e79de0f37a18c 100644 --- a/src/test/erasure-code/TestErasureCodePlugins.cc +++ b/src/test/erasure-code/TestErasureCodePlugins.cc @@ -5,10 +5,18 @@ */ #include #include + +#include +#include + #include "erasure-code/ErasureCodePlugin.h" #include "global/global_context.h" #include "common/config_proxy.h" #include "gtest/gtest.h" +#include "include/buffer.h" +#include "osd/ECTypes.h" +#include "osd/ECUtil.h" + using namespace std; class PluginTest: public ::testing::TestWithParam { public: @@ -78,6 +86,37 @@ public: } bl.append(b); } + uint32_t calculate_crc(const bufferlist& bl, int crc_seed) { + bufferhash hash(crc_seed); + hash << bl; + return hash.digest(); + } + uint32_t calculate_zero_buffer_crc(int crc_seed) { + bufferlist zero_bl; + generate_chunk(zero_bl, 0); + return calculate_crc(zero_bl, crc_seed); + } + bufferptr create_buffer_from_crc(uint32_t crc) { + std::size_t length = sizeof(crc); + char crc_bytes[length]; + for (std::size_t i = 0; i < length; i++) { + crc_bytes[i] = crc >> (8 * i) & 0xFF; + } + ceph::bufferptr buffer = ceph::buffer::create_aligned(chunk_size, 4096); + buffer.zero(true); + buffer.copy_in(0, length, crc_bytes); + return buffer; + } + uint32_t read_crc_from_bufferlist(bufferlist& bl, + uint64_t offset = 0) { + uint32_t crc = 0; + std::size_t length = sizeof(uint32_t); + for (std::size_t i = 0; i < length; i++) { + crc |= ((bl.c_str()[offset + i] & 0xFF) << (8 * i)); + } + + return crc; + } }; TEST_P(PluginTest,Initialize) { @@ -460,6 +499,125 @@ TEST_P(PluginTest,SubChunkSupport) ErasureCodeInterface::FLAG_EC_PLUGIN_REQUIRE_SUB_CHUNKS) != 0); } } +TEST_P(PluginTest, CRCEncodeDecodeSupport) { + initialize(); + + shard_id_set want_to_encode; + for (shard_id_t i = shard_id_t(0); i < get_k_plus_m(); ++i) { + want_to_encode.insert(i); + } + // Generate random data to encode + bufferlist data_bl; + for (unsigned int i = 0; i < get_k(); ++i) { + generate_chunk(data_bl); + } + + int crc_seed = -1; + uint32_t zero_data_crc = calculate_zero_buffer_crc(crc_seed); + + // Calculate CRCs for the random data + bufferlist hashes_bl; + bufferlist unseeded_hashes_bl; + for (unsigned int i = 0; i < get_k(); ++i) { + // Calculate the CRC for the shard at position i + bufferlist data_shard_bl; + data_shard_bl.substr_of(data_bl, i * chunk_size, chunk_size); + uint32_t crc = calculate_crc(data_shard_bl, crc_seed); + + // XOR with the CRC of zeros preseeded with the same preseed + // This undoes the pre-seeding and gives us the CRC as if no seed was + // applied + uint32_t unseeded_crc = crc ^ zero_data_crc; + + // Convert integers to bufferlists + hashes_bl.append(create_buffer_from_crc(crc)); + unseeded_hashes_bl.append(create_buffer_from_crc(unseeded_crc)); + } + + // Encode data and get back data + parity chunks + shard_id_map encoded_data(get_k_plus_m()); + erasure_code->encode(want_to_encode, data_bl, &encoded_data); + + // Encode CRCs to get back the data + parity CRCs + shard_id_map encoded_hashes(get_k_plus_m()); + shard_id_map encoded_unseeded_hashes(get_k_plus_m()); + erasure_code->encode(want_to_encode, hashes_bl, &encoded_hashes); + erasure_code->encode(want_to_encode, unseeded_hashes_bl, + &encoded_unseeded_hashes); + + shard_id_map encoded_data_crcs(get_k_plus_m()); + + // Calculate CRCs for the new data and new CRCs and compare first parity shard + bool different = false; + for (shard_id_t shard_id : want_to_encode) { + // Calculations vary for additional parities, so only first parity supported + if (shard_id < get_k() + 1) { + // Calculate the CRC for the current shard from the encoded data + uint32_t calculated_crc = + calculate_crc(encoded_data.at(shard_id), crc_seed); + encoded_data_crcs[shard_id].append( + create_buffer_from_crc(calculated_crc)); + + // XOR with the CRC of zeros preseeded with the same preseed + // This undoes the pre-seeding and gives us the CRC as if no seed was + // applied + uint32_t calculated_unseeded_crc = calculated_crc ^ zero_data_crc; + + // Calculate integer form of CRCs in bufferlists + uint32_t unseeded_crc = + read_crc_from_bufferlist(encoded_unseeded_hashes.at(shard_id)); + + if (calculated_unseeded_crc != unseeded_crc) { + different = true; + } + } + + ECUtil::stripe_info_t sinfo{get_k(), get_m(), get_k() * chunk_size, + erasure_code->get_chunk_mapping()}; + + // Decode CRCs as if 1 to m-1 data CRCs are missing and assert decoded CRC + // is equal to missing CRC + for (raw_shard_id_t missing_raw_shard_id{0}; missing_raw_shard_id < get_k(); + ++missing_raw_shard_id) { + shard_id_t missing_shard_id = sinfo.get_shard(missing_raw_shard_id); + + shard_id_set need; + need.insert(missing_shard_id); + shard_id_map chunks(get_k_plus_m()); + // Create a map of all buffers except the one (our missing shard) + for (raw_shard_id_t raw_shard_id{0}; raw_shard_id < get_k_plus_m(); + ++raw_shard_id) { + shard_id_t shard_id = sinfo.get_shard(raw_shard_id); + + if (shard_id != missing_shard_id) { + chunks.insert(shard_id, encoded_hashes[shard_id]); + } + } + + // Decode the missing shard + shard_id_map out_bls(get_k_plus_m()); + int r = erasure_code->decode(need, chunks, &out_bls, chunk_size); + + EXPECT_EQ(r, 0); + + // Check the missing shard has been decoded correctly + uint32_t decoded_crc = + read_crc_from_bufferlist(out_bls[missing_shard_id]); + uint32_t original_crc = + read_crc_from_bufferlist(hashes_bl, missing_shard_id.id * chunk_size); + + different = different | (decoded_crc != original_crc); + } + } + + if (erasure_code->get_supported_optimizations() & + ErasureCodeInterface::FLAG_EC_PLUGIN_CRC_ENCODE_DECODE_SUPPORT) { + // Plugin should not have FLAG_EC_PLUGIN_CRC_ENCODE_DECODE_SUPPORT enabled, + // this failure proves that it can cause a data integrity issue + EXPECT_EQ(different, false); + } +} + INSTANTIATE_TEST_SUITE_P( PluginTests, PluginTest,