From: Alex Ainscow Date: Sun, 3 May 2026 16:46:00 +0000 (+0100) Subject: test/osd: Add scrub to osd test fixtures X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=d46aea099ef134fda8f26737d517ddc78886c76f;p=ceph.git test/osd: Add scrub to osd test fixtures Add scrubbing functionality to the OSD test fixtures to enable testing of scrub behavior on erasure-coded pools. This implementation uses as much of the scrub backend as possible, but avoids using pg_scrubber or the interaction with peering due to the complexity of the dependency on PG. Changes: - Add scrub_object() method to PGBackendTestFixture for testing scrub functionality on EC pools - Add create_random_buffer() utility for generating random test data - Add corrupt_shard_data() method to intentionally corrupt shard data for testing scrub detection - Add MockScrubBeListener, MockPgScrubBeListener, MockSnapMapReader, and MockLoggerSinkSet mock classes to support scrub testing - Add three new test cases: ScrubClean, ScrubDetectsCorruption, and ScrubPartialWrite to verify scrub behavior with clean data, corrupted data, and partial writes Signed-off-by: Alex Ainscow # Conflicts: # src/test/osd/TestECFailoverWithPeering.cc --- diff --git a/src/test/osd/PGBackendTestFixture.cc b/src/test/osd/PGBackendTestFixture.cc index a3c26585be7..72b565f908e 100644 --- a/src/test/osd/PGBackendTestFixture.cc +++ b/src/test/osd/PGBackendTestFixture.cc @@ -25,6 +25,12 @@ #include "messages/MOSDPGPush.h" #include "messages/MOSDPGPushReply.h" +void PGBackendTestFixture::initialize_scrub_infra() +{ + scrub_listener = TestScrubBackend::create_scrub_listener(spgid, osdmap); + snap_reader = TestScrubBackend::create_snap_reader(); +} + void PGBackendTestFixture::setup_ec_pool() { CephContext *cct = g_ceph_context; @@ -904,3 +910,162 @@ object_info_t PGBackendTestFixture::read_shard_object_info( oi.decode(p); return oi; } + + +bool PGBackendTestFixture::scrub_object(const std::string& obj_name) +{ + hobject_t hoid = make_test_object(obj_name); + + int total_shards = k + m; + std::map scrub_maps; + + for (int shard = 0; shard < total_shards; ++shard) { + pg_shard_t pg_shard(shard, shard_id_t(shard)); + ScrubMap& smap = scrub_maps[pg_shard]; + + auto backend_it = backends.find(shard); + if (backend_it == backends.end()) { + std::cerr << "ERROR: Backend for shard " << shard << " does not exist" << std::endl; + return true; + } + PGBackend* backend = backend_it->second.get(); + if (!backend) { + std::cerr << "ERROR: Backend pointer for shard " << shard << " is null" << std::endl; + return true; + } + + ScrubMapBuilder pos; + pos.ls.push_back(hoid); + pos.pos = 0; + pos.deep = true; + + const Scrub::ScrubCounterSet& counters = Scrub::io_counters_ec; + + int r = backend->be_scan_list(counters, smap, pos); + while (r == -EINPROGRESS) { + r = backend->be_scan_list(counters, smap, pos); + } + + if (r != 0) { + std::cerr << "ERROR: be_scan_list failed for shard " << shard << ": " << cpp_strerror(r) << std::endl; + return true; + } + + if (!smap.objects.contains(hoid)) { + std::cerr << "ERROR: Object not in scrub map for shard " << shard << std::endl; + return true; + } + } + + if (!scrub_listener || !snap_reader) { + initialize_scrub_infra(); + } + + ceph_assert(scrub_listener); + ceph_assert(snap_reader); + + MockPGBackendListener* primary_listener = get_primary_listener(); + if (!primary_listener) { + std::cerr << "ERROR: No primary listener found" << std::endl; + return true; + } + int primary_shard = primary_listener->pg_whoami.osd; + + PGBackend* primary_backend = get_primary_backend(); + if (!primary_backend) { + std::cerr << "ERROR: No primary backend found" << std::endl; + return true; + } + + const pg_pool_t* pool_info = osdmap->get_pg_pool(pool_id); + ceph_assert(pool_info != nullptr); + + MockPgScrubBeListener pg_scrub_listener(primary_backend); + pg_scrub_listener.pool = std::make_shared( + osdmap, pool_id, *pool_info, "test_pool"); + pg_scrub_listener.primary = primary_shard; + pg_scrub_listener.info.pgid = spgid; + + std::set acting_set; + for (int i = 0; i < k + m; ++i) { + acting_set.insert(pg_shard_t(i, shard_id_t(i))); + } + + TestScrubBackend scrub_backend(*scrub_listener, pg_scrub_listener, + pg_shard_t(primary_shard, shard_id_t(primary_shard)), + false, scrub_level_t::deep, acting_set); + + scrub_backend.new_chunk(); + + for (const auto& [pg_shard, smap] : scrub_maps) { + scrub_backend.insert_faked_smap(pg_shard, smap); + } + + auto result = scrub_backend.scrub_compare_maps(false, *snap_reader); + + return !result.inconsistent_objs.empty(); +} + + +bufferlist PGBackendTestFixture::create_random_buffer(size_t size) +{ + bufferlist bl; + + if (size == 0) { + return bl; + } + + ceph::buffer::ptr bp(size); + + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution dis(0, 255); + + for (size_t i = 0; i < size; ++i) { + bp[i] = dis(gen); + } + + bl.append(std::move(bp)); + return bl; +} + +void PGBackendTestFixture::corrupt_shard_data(const hobject_t& obj, pg_shard_t shard) +{ + auto ch_it = chs.find(shard.osd); + if (ch_it == chs.end()) { + std::cerr << "ERROR: No collection handle for shard " << shard.osd << std::endl; + return; + } + + ghobject_t ghoid(obj, ghobject_t::NO_GEN, shard.shard); + + struct stat st; + int r = store->stat(ch_it->second, ghoid, &st); + if (r < 0) { + std::cerr << "ERROR: Failed to stat object on shard " << shard.osd + << ": " << cpp_strerror(r) << std::endl; + return; + } + + uint64_t size = st.st_size; + if (size == 0) { + std::cerr << "WARNING: Object has zero size on shard " << shard.osd << std::endl; + return; + } + + bufferlist zero_bl; + zero_bl.append_zero(size); + + ObjectStore::Transaction t; + t.write(ch_it->second->cid, ghoid, 0, size, zero_bl); + + r = store->queue_transaction(ch_it->second, std::move(t)); + if (r < 0) { + std::cerr << "ERROR: Failed to corrupt object on shard " << shard.osd + << ": " << cpp_strerror(r) << std::endl; + return; + } + + std::cout << "Corrupted shard " << shard.osd << " data for object " << obj + << " (wrote " << size << " bytes of zeros)" << std::endl; +} diff --git a/src/test/osd/PGBackendTestFixture.h b/src/test/osd/PGBackendTestFixture.h index 0e728f3f38c..cb7bfc155ea 100644 --- a/src/test/osd/PGBackendTestFixture.h +++ b/src/test/osd/PGBackendTestFixture.h @@ -40,6 +40,9 @@ #include "os/ObjectStore.h" #include "erasure-code/ErasureCodePlugin.h" #include "test/osd/OSDMapTestHelpers.h" +#include "test/osd/ScrubTestFixture.h" +#include "osd/scrubber/scrub_backend.h" +#include "osd/scrubber/pg_scrubber.h" // Unified test fixture for EC and Replicated backend tests with ObjectStore. // Uses PoolType to branch between EC (ECSwitch) and Replicated (ReplicatedBackend). @@ -86,7 +89,9 @@ protected: // OpTracker for wrapping messages in OpRequestRef std::shared_ptr op_tracker; - + // Scrub infrastructure - initialized once and reused across scrub operations + std::unique_ptr scrub_listener; + std::unique_ptr snap_reader; ceph::ErasureCodeInterfaceRef ec_impl; std::map> lrus; int k = 4; // data chunks @@ -214,6 +219,9 @@ private: void setup_replicated_pool(); void cleanup_data_dir(); +protected: + void initialize_scrub_infra(); + public: const pg_pool_t& get_pool() const { const pg_pool_t* pool = OSDMapTestHelpers::get_pool(osdmap, pool_id); @@ -505,5 +513,46 @@ public: const std::string& obj_name, int shard); + /** + * Scrub an object and verify it has no corruption. + * + * This utility method: + * 1. Builds scrub maps for all shards using be_scan_list() + * 2. Creates a ScrubBackend with mock listeners + * 3. Calls scrub_compare_maps() to check for inconsistencies + * 4. Returns true if corruption was detected, false otherwise + * + * The scrub infrastructure (mock listeners) is initialized once in SetUp() + * and reused across all scrub operations for efficiency. + * + * @param obj_name Name of the object to scrub + * @return true if corruption detected, false if object is consistent + */ + bool scrub_object(const std::string& obj_name); + + /** + * Corrupt the data for a specific shard of an object. + * + * This utility method directly writes zeros to the stored data for a given + * shard, simulating data corruption at the storage level. This is useful + * for testing scrub detection of corrupted data. + * + * @param obj The hobject_t identifying the object to corrupt + * @param shard The pg_shard_t identifying which shard to corrupt + */ + void corrupt_shard_data(const hobject_t& obj, pg_shard_t shard); + + /** + * Create a bufferlist filled with random data. + * + * This utility method generates a buffer of the specified size filled with + * random bytes. Useful for testing scenarios where random data is needed + * to avoid patterns that might mask bugs (e.g., XOR patterns in EC). + * + * @param size Size of the buffer to create in bytes + * @return A bufferlist containing random data + */ + bufferlist create_random_buffer(size_t size); + }; diff --git a/src/test/osd/ScrubTestFixture.h b/src/test/osd/ScrubTestFixture.h new file mode 100644 index 00000000000..2fb3768a928 --- /dev/null +++ b/src/test/osd/ScrubTestFixture.h @@ -0,0 +1,195 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:nil -*- +// vim: ts=8 sw=2 sts=2 expandtab + +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2026 IBM + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#pragma once + +#include +#include + +#include "include/expected.hpp" +#include "osd/scrubber/scrub_backend.h" +#include "osd/scrubber/pg_scrubber.h" +#include "osd/PGBackend.h" +#include "osd/PeeringState.h" +#include "osd/osd_types.h" +#include "test/osd/scrubber_generators.h" +#include "global/global_context.h" + +// Type alias for compatibility with existing code +using MockLoggerSinkSet = ScrubGenerator::MockLog; + +/** + * MockScrubBeListener - Mock implementation of ScrubBeListener for testing + * + * Provides minimal implementation of the ScrubBeListener interface for + * testing scrub backend functionality. + */ +class MockScrubBeListener : public ScrubBeListener { + public: + MockScrubBeListener() + : pg_whoami(0, shard_id_t::NO_SHARD), + primary_shard(0, shard_id_t::NO_SHARD) {} + + MockScrubBeListener(pg_shard_t whoami, pg_shard_t primary) + : pg_whoami(whoami), primary_shard(primary) {} + + std::ostream& gen_prefix(std::ostream& out) const override { + return out << "scrub: "; + } + CephContext* get_pg_cct() const override { return g_ceph_context; } + LoggerSinkSet& get_logger() const override { return logger; } + bool is_primary() const override { return pg_whoami == primary_shard; } + spg_t get_pgid() const override { return pgid; } + const OSDMapRef& get_osdmap() const override { return osdmap; } + void add_to_stats(const object_stat_sum_t&) override {} + void submit_digest_fixes(const digests_fixes_t&) override {} + + pg_shard_t pg_whoami; // This shard's identity + pg_shard_t primary_shard; // The primary shard's identity + spg_t pgid; + OSDMapRef osdmap; + mutable MockLoggerSinkSet logger; +}; + +/** + * MockPgScrubBeListener - Mock implementation of Scrub::PgScrubBeListener for testing + * + * Provides EC-aware implementation of the PgScrubBeListener interface for + * testing scrub backend functionality with erasure-coded pools. + * + * Key features: + * - Delegates EC operations (encode/decode) to real ECBackend for correctness + * - Implements logical_to_ondisk_size() for proper EC shard size calculation + * - Delegates get_is_nonprimary_shard() to handle multi-zone EC pools correctly + */ +class MockPgScrubBeListener : public Scrub::PgScrubBeListener { + public: + MockPgScrubBeListener(PGBackend* be = nullptr) : backend(be) {} + + const PGPool& get_pgpool() const override { return *pool; } + + pg_shard_t get_primary() const override { + return pg_shard_t(primary, shard_id_t(primary)); + } + + void force_object_missing(ScrubberPasskey, + const std::set& peer, + const hobject_t& oid, + eversion_t version) override {} + + const pg_info_t& get_pg_info(ScrubberPasskey) const override { return info; } + + uint64_t logical_to_ondisk_size(uint64_t logical_size, shard_id_t shard_id, + bool object_is_legacy_ec) const override { + return backend->be_get_ondisk_size(logical_size, shard_id_t(shard_id), + object_is_legacy_ec); + } + + bool ec_can_decode(const shard_id_set& available_shards) const override { + return backend->ec_can_decode(available_shards); + } + + shard_id_map ec_encode_acting_set(const bufferlist& bl) const override { + return backend->ec_encode_acting_set(bl); + } + + shard_id_map ec_decode_acting_set( + const shard_id_map& shard_map, + int chunk_size) const override { + return backend->ec_decode_acting_set(shard_map, chunk_size); + } + + bool get_ec_supports_crc_encode_decode() const override { + return backend->get_ec_supports_crc_encode_decode(); + } + + ECUtil::stripe_info_t get_ec_sinfo() const override { + return backend->ec_get_sinfo(); + } + + bool is_waiting_for_unreadable_object() const override { return false; } + + bool get_is_nonprimary_shard(const pg_shard_t& shard) const override { + return backend->get_is_nonprimary_shard(shard.shard); + } + + bool get_is_hinfo_required() const override { + return backend->get_is_hinfo_required(); + } + + bool get_is_ec_optimized() const override { + return backend->get_is_ec_optimized(); + } + + std::shared_ptr pool; + int primary; + pg_info_t info; + PGBackend* backend; +}; + +/** + * MockSnapMapReader - Mock implementation of Scrub::SnapMapReaderI for testing + * + * Provides minimal implementation that returns empty snapshot sets, + * suitable for testing scrub functionality without snapshot complexity. + */ +class MockSnapMapReader : public Scrub::SnapMapReaderI { + public: + tl::expected, result_t> get_snaps( + const hobject_t&) const override { + return std::set{}; + } + tl::expected, result_t> get_snaps_check_consistency( + const hobject_t&) const override { + return std::set{}; + } +}; + +// Test helper to access ScrubBackend internals +class TestScrubBackend : public ScrubBackend { +public: + TestScrubBackend(ScrubBeListener& scrubber, + Scrub::PgScrubBeListener& pg, + pg_shard_t i_am, + bool repair, + scrub_level_t shallow_or_deep, + const std::set& acting) + : ScrubBackend(scrubber, pg, i_am, repair, shallow_or_deep, acting) + {} + + void insert_faked_smap(pg_shard_t shard, const ScrubMap& smap) { + ceph_assert(this_chunk.has_value()); + this_chunk->received_maps[shard] = smap; + } + + // Factory method to create MockScrubBeListener with proper initialization + static std::unique_ptr create_scrub_listener( + const spg_t& pgid, + const OSDMapRef& osdmap) + { + auto listener = std::make_unique(); + listener->pgid = pgid; + listener->osdmap = osdmap; + return listener; + } + + // Factory method to create MockSnapMapReader + static std::unique_ptr create_snap_reader() + { + return std::make_unique(); + } +}; + +// Made with Bob diff --git a/src/test/osd/TestBackendBasics.cc b/src/test/osd/TestBackendBasics.cc index 8e544c46a5f..36d367fa076 100644 --- a/src/test/osd/TestBackendBasics.cc +++ b/src/test/osd/TestBackendBasics.cc @@ -79,6 +79,7 @@ public: void SetUp() override { PGBackendTestFixture::SetUp(); + initialize_scrub_infra(); } /** @@ -474,6 +475,7 @@ public: void SetUp() override { PGBackendTestFixture::SetUp(); + initialize_scrub_infra(); } void simulate_osd_failure(int failed_osd, int new_primary_instance) diff --git a/src/test/osd/TestECFailoverWithPeering.cc b/src/test/osd/TestECFailoverWithPeering.cc index 9a9cd0f24c9..c9aab2adcf3 100644 --- a/src/test/osd/TestECFailoverWithPeering.cc +++ b/src/test/osd/TestECFailoverWithPeering.cc @@ -16,7 +16,6 @@ #include #include "test/osd/ECPeeringTestFixture.h" #include "test/osd/TestCommon.h" -#include "osd/ECSwitch.h" using namespace std; @@ -843,6 +842,124 @@ TEST_P( set_config("osd_async_recovery_min_cost", "100"); } +TEST_P(TestECFailoverWithPeering, ScrubClean) { + ASSERT_TRUE(all_shards_active()) << "Initial peering must complete"; + + const std::string obj_name = "test_scrub_corruption"; + uint64_t object_size = k * stripe_unit; + + bufferlist bl = create_random_buffer(object_size); + std::string test_data(bl.c_str(), bl.length()); + + std::cout << "Writing full-stripe object (" << object_size << " bytes of random data)" << std::endl; + create_and_write_verify(obj_name, test_data); + + std::cout << "Scrubbing object to verify data integrity" << std::endl; + bool corruption_detected = scrub_object(obj_name); + + ASSERT_FALSE(corruption_detected) + << "scrub_object() should NOT detect corruption when data is valid"; + + std::cout << "=== ScrubDetectsCorruption test completed successfully ===" << std::endl; +} + +TEST_P(TestECFailoverWithPeering, ScrubDetectsCorruption) { + ASSERT_TRUE(all_shards_active()) << "Initial peering must complete"; + + const uint64_t object_size = k * stripe_unit; + const std::vector shard_offsets = {/*0, 1, */k}; + const bool supports_crc = ec_plugin == "isa"; + + for (int zone = 0; zone < 1; ++zone) { + for (int shard_offset : shard_offsets) { + const int absolute_shard = shard_offset; + const std::string obj_name = + "test_obj_zone_" + std::to_string(zone) + + "_shard_" + std::to_string(shard_offset); + + bufferlist bl = create_random_buffer(object_size); + std::string test_data(bl.c_str(), bl.length()); + + std::cout << "\n=== ScrubDetectsCorruption: testing zone " << zone + << ", shard offset " << shard_offset + << " (absolute shard " << absolute_shard << ") ===" << std::endl; + + std::cout << "Writing object " << obj_name << " (" << object_size + << " bytes of random data)" << std::endl; + create_and_write_verify(obj_name, test_data); + + std::cout << "Corrupting object " << obj_name + << " for zone iteration " << zone + << " on relative shard " << shard_offset + << " using absolute shard " << absolute_shard << std::endl; + hobject_t hoid = make_test_object(obj_name); + corrupt_shard_data(hoid, + pg_shard_t(absolute_shard, shard_id_t(absolute_shard))); + + std::cout << "Scrubbing object " << obj_name + << " to verify corruption detection for zone iteration " << zone + << ", shard offset " << shard_offset << std::endl; + bool corruption_detected = scrub_object(obj_name); + + std::cout << "Zone iteration " << zone + << " corruption result for shard offset " << shard_offset + << ": " << (corruption_detected ? "detected" : "not detected") + << " (absolute shard " << absolute_shard + << ", supports_crc=" << (supports_crc ? "true" : "false") + << ")" << std::endl; + + if (supports_crc) { + EXPECT_TRUE(corruption_detected) + << "scrub_object() should detect corruption for object " << obj_name + << " during zone iteration " << zone + << ", shard offset " << shard_offset + << " (absolute shard " << absolute_shard << ")"; + } else { + EXPECT_FALSE(corruption_detected) + << "scrub_object() should not report corruption for object " + << obj_name << " when CRC-based detection is unsupported" + << " during zone iteration " << zone << ", shard offset " + << shard_offset << " (absolute shard " << absolute_shard << ")"; + } + } + } + + std::cout << "=== ScrubDetectsCorruption test completed successfully ===" << std::endl; +} + +TEST_P(TestECFailoverWithPeering, ScrubPartialWrite) { + ASSERT_TRUE(all_shards_active()) << "Initial peering must complete"; + + const std::string obj_name = "test_scrub_partial_write"; + + uint64_t partial_size = stripe_unit / 2; + + std::cout << "Creating partial write object with size " << partial_size + << " bytes (stripe_unit=" << stripe_unit << ", full stripe would be " + << (k * stripe_unit) << " bytes)" << std::endl; + + bufferlist bl = create_random_buffer(partial_size); + std::string test_data(bl.c_str(), bl.length()); + + std::cout << "Writing partial object (" << partial_size << " bytes)" << std::endl; + create_and_write_verify(obj_name, test_data); + + write(obj_name, 0, test_data, test_data.size()); + + // NOTE: Partial writes may expose scrub issues with EC pools + std::cout << "Scrubbing partial write object to test scrub behavior" << std::endl; + bool corruption_detected = scrub_object(obj_name); + + std::cout << "Scrub result for partial write: " + << (corruption_detected ? "corruption detected" : "no corruption detected") + << std::endl; + + EXPECT_FALSE(corruption_detected) + << "scrub_object() should NOT detect corruption on valid partial write"; + + std::cout << "=== ScrubPartialWrite test completed ===" << std::endl; +} + // --------------------------------------------------------------------------- // Instantiate TestECFailoverWithPeering with EC configurations // ---------------------------------------------------------------------------