osd: fixed data digest issue where it was running on shallow scrub

diff --git a/src/osd/ECBackend.cc b/src/osd/ECBackend.cc
index 6567f4fad1db526984306c49802b6d987b04db99..abe6f7c2bbc307ef682a34655d85b6503ea08945 100644 (file)
--- a/src/osd/ECBackend.cc
+++ b/src/osd/ECBackend.cc
@@ -1226,6 +1226,27 @@ int ECBackend::objects_get_attrs(
   return 0;
 }
 
+std::string ECBackend::extract_hex_from_bufferlist(
+    const bufferlist& bl)
+{
+  std::string hex_string;
+  const size_t length = bl.length();
+  for (size_t i = 0; i < length; i++) {
+    hex_string += fmt::format("{:02x}", hex_string[i]);
+  }
+  return hex_string;
+}
+
+std::string ECBackend::extract_crc_from_bufferlist(
+    const bufferlist& crc_buffer) {
+  std::string crc_string;
+  constexpr size_t digest_length = sizeof(uint32_t);
+  for (size_t i = 0; i < digest_length; i++) {
+    crc_string += fmt::format("{:02x}", crc_buffer[digest_length - i]);
+  }
+  return crc_string;
+}
+
 int ECBackend::be_deep_scrub(
   const Scrub::ScrubCounterSet& io_counters,
   const hobject_t &poid,
@@ -1267,6 +1288,7 @@ int ECBackend::be_deep_scrub(
     return 0;
   }
   if (r > 0) {
+    dout(20) << __func__ << fmt::format("{}: Hoid {}, Shard {} ({}~{}) - Read data: {}", __func__, poid, get_parent()->whoami_shard(), pos.data_pos, r, extract_hex_from_bufferlist(bl)) << dendl;
     pos.data_hash << bl;
   }
   perf_logger.inc(io_counters.read_bytes, r);
@@ -1279,6 +1301,7 @@ int ECBackend::be_deep_scrub(
     // We pass the calculated digest here
     // This will be used along with the plugin to verify data consistency
     o.digest = pos.data_hash.digest();
+    dout(20) << __func__ << fmt::format("{}: Hoid {}, Shard {} - Digest: 0x{:x}", __func__, poid, get_parent()->whoami_shard(), o.digest) << dendl;
   }
   else
   {

diff --git a/src/osd/ECBackend.h b/src/osd/ECBackend.h
index c8b9ccdef97138250eccdbecf637e92bb69a9f67..714688fc1aa8cd58665f196ed44f6e7617877cea 100644 (file)
--- a/src/osd/ECBackend.h
+++ b/src/osd/ECBackend.h
@@ -363,6 +363,11 @@ public:
       std::map<std::string, ceph::buffer::list, std::less<>> *out
     );
 
+  std::string extract_hex_from_bufferlist(
+    const bufferlist& crc_buffer);
+  std::string extract_crc_from_bufferlist(
+    const bufferlist& crc_buffer);
+
   bool auto_repair_supported() const { return true; }
 
   int be_deep_scrub(

diff --git a/src/osd/scrubber/scrub_backend.cc b/src/osd/scrubber/scrub_backend.cc
index 2658867b82b9f977df861947b23facc1616f25f9..c60d5b8d9e69d562a7e3e7b178faa8bcbbd8a575 100644 (file)
--- a/src/osd/scrubber/scrub_backend.cc
+++ b/src/osd/scrubber/scrub_backend.cc
@@ -82,7 +82,7 @@ ScrubBackend::ScrubBackend(ScrubBeListener& scrubber,
   m_is_optimized_ec = m_pool.info.allows_ecoptimizations();
 
   // EC-related:
-  if (!m_is_replicated && m_pg.get_ec_supports_crc_encode_decode()) {
+  if (!m_is_replicated && m_pg.get_ec_supports_crc_encode_decode() && m_depth == scrub_level_t::deep) {
     m_ec_digest_map_size = m_pg.get_ec_sinfo().get_k_plus_m();
   }
 
@@ -792,19 +792,26 @@ void ScrubBackend::setup_ec_digest_map(auth_selection_t& auth_selection,
 
   this_chunk->m_ec_digest_map.clear();
 
+  dout(20) << __func__ << ": Hoid:" << ho << ", Beginning setup" << dendl;
+
   if (auth_selection.auth_oi.version != eversion_t() &&
-      m_pg.get_ec_supports_crc_encode_decode()) {
+      m_pg.get_ec_supports_crc_encode_decode() &&
+      m_depth == scrub_level_t::deep) {
     uint64_t auth_length = this_chunk->received_maps[auth_selection.auth_shard]
                                .objects.at(ho)
                                .size;
 
+    dout(20) << __func__ << ": Hoid:" << ho  << ", crc encode/decode supported. Auth length: " << auth_length << dendl;
+
     shard_id_set available_shards;
 
     for (const auto& [srd, smap] : this_chunk->received_maps) {
       if (!m_is_replicated && m_pg.get_ec_supports_crc_encode_decode() &&
-          smap.objects.contains(ho)) {
+          m_depth == scrub_level_t::deep && smap.objects.contains(ho)) {
         uint64_t shard_length = smap.objects.at(ho).size;
 
+        dout(20) << __func__ << ": Hoid:" << ho  << ", crc encode/decode supported again. Auth length: " << auth_length << ". Checking shard " << srd << ". Shard length: " << shard_length << dendl;
+
         available_shards.insert(srd.shard);
 
         uint32_t digest = smap.objects.at(ho).digest;
@@ -815,7 +822,9 @@ void ScrubBackend::setup_ec_digest_map(auth_selection_t& auth_selection,
         // require them to be the same length.
         ceph_assert(auth_length >= shard_length);
         int padding = auth_length - shard_length;
+        dout(20) << __func__ << "Padding length: " << padding << dendl;
         if (padding != 0) {
+          dout(20) << fmt::format("{} hoid: {}, shard: {}, (A) Padding digest with {} zeros, going from {} to {}", __func__, ho, srd, padding, digest, ceph_crc32c_zeros(digest, padding)) << dendl;
           digest = ceph_crc32c_zeros(digest, padding);
         }
 
@@ -882,6 +891,8 @@ void ScrubBackend::setup_ec_digest_map(auth_selection_t& auth_selection,
             -1, logical_to_ondisk_size(auth_selection.auth_oi.size,
                                        auth_selection.auth_shard.shard));
 
+        dout(20) << __func__ << fmt::format("Hoid {} (size {}) - Zero_data_crc: 0x{:x}", ho, logical_to_ondisk_size(auth_selection.auth_oi.size, auth_selection.auth_shard.shard), zero_data_crc) << dendl;
+
         for (const auto& shard_id : m_pg.get_ec_sinfo().get_data_shards()) {
           for (std::size_t i = 0; i < sizeof(zero_data_crc); i++) {
             this_chunk->m_ec_digest_map.at(shard_id).c_str()[i] =
@@ -951,7 +962,10 @@ std::optional<std::string> ScrubBackend::compare_obj_in_maps(
     clog.error() << candidates_errors.str();
   }
 
+  dout(20) << __func__ << ": About to setup" << dendl;
+
   if (!m_is_replicated) {
+    dout(20) << __func__ << ": Not replicated" << dendl;
     setup_ec_digest_map(auth_res, ho);
   }
 
@@ -1209,7 +1223,8 @@ ScrubBackend::auth_and_obj_errs_t ScrubBackend::match_in_shards(
   std::list<pg_shard_t> auth_list;     // out "param" to
   std::set<pg_shard_t> object_errors;  // be returned
   std::size_t digest_size = 0;
-  if (!m_is_replicated && m_pg.get_ec_supports_crc_encode_decode()) {
+  if (!m_is_replicated && m_pg.get_ec_supports_crc_encode_decode() &&
+      m_depth == scrub_level_t::deep) {
     digest_size = m_pg.get_ec_sinfo().get_k_plus_m();
   }
   shard_id_map<bufferlist> digests{digest_size};
@@ -1241,7 +1256,8 @@ ScrubBackend::auth_and_obj_errs_t ScrubBackend::match_in_shards(
                                                      ho.has_snapset(),
                                                      srd);
 
-      if (!m_is_replicated && m_pg.get_ec_supports_crc_encode_decode()) {
+      if (!m_is_replicated && m_pg.get_ec_supports_crc_encode_decode()
+          && m_depth == scrub_level_t::deep) {
         // Create map containing all data shards except current shard and all
         // parity shards Decode the current data shard Add to set<shard_id>
         // incorrectly_decoded_shards if the shard did not decode
@@ -1253,6 +1269,7 @@ ScrubBackend::auth_and_obj_errs_t ScrubBackend::match_in_shards(
         ceph_assert(auth_length >= shard_length);
         int padding = auth_length - shard_length;
         if (padding != 0) {
+          dout(20) << fmt::format("{}: Hoid: {}, shard: {}, (B) Padding digest with {} zeros, going from {} to {}", __func__, ho, srd, padding, digest, ceph_crc32c_zeros(digest, padding)) << dendl;
           digest = ceph_crc32c_zeros(digest, padding);
         }
 
@@ -1353,7 +1370,8 @@ ScrubBackend::auth_and_obj_errs_t ScrubBackend::match_in_shards(
              << dendl;
   }
 
-  if (!m_is_replicated && m_pg.get_ec_supports_crc_encode_decode()) {
+  if (!m_is_replicated && m_pg.get_ec_supports_crc_encode_decode()
+      && m_depth == scrub_level_t::deep) {
     set<shard_id_t> incorrectly_decoded_shards;
 
     shard_id_set shards;
@@ -1374,6 +1392,8 @@ ScrubBackend::auth_and_obj_errs_t ScrubBackend::match_in_shards(
             -1, logical_to_ondisk_size(auth_sel.auth_oi.size,
                                        auth_sel.auth_shard.shard));
 
+        dout(20) << __func__ << fmt::format("Hoid {} (size {}) - Zero_data_crc: 0x{:x}", ho, logical_to_ondisk_size(auth_sel.auth_oi.size, auth_sel.auth_shard.shard), zero_data_crc) << dendl;
+
         for (uint32_t i = 0; i < sizeof(zero_data_crc); i++) {
           bl.c_str()[i] ^= retrieve_byte(zero_data_crc, i);
         }