#include "messages/MOSDPGPush.h"
#include "messages/MOSDPGPushReply.h"
+void PGBackendTestFixture::initialize_scrub_infra()
+{
+ scrub_listener = TestScrubBackend::create_scrub_listener(spgid, osdmap);
+ snap_reader = TestScrubBackend::create_snap_reader();
+}
+
void PGBackendTestFixture::setup_ec_pool()
{
CephContext *cct = g_ceph_context;
oi.decode(p);
return oi;
}
+
+
+bool PGBackendTestFixture::scrub_object(const std::string& obj_name)
+{
+ hobject_t hoid = make_test_object(obj_name);
+
+ int total_shards = k + m;
+ std::map<pg_shard_t, ScrubMap> scrub_maps;
+
+ for (int shard = 0; shard < total_shards; ++shard) {
+ pg_shard_t pg_shard(shard, shard_id_t(shard));
+ ScrubMap& smap = scrub_maps[pg_shard];
+
+ auto backend_it = backends.find(shard);
+ if (backend_it == backends.end()) {
+ std::cerr << "ERROR: Backend for shard " << shard << " does not exist" << std::endl;
+ return true;
+ }
+ PGBackend* backend = backend_it->second.get();
+ if (!backend) {
+ std::cerr << "ERROR: Backend pointer for shard " << shard << " is null" << std::endl;
+ return true;
+ }
+
+ ScrubMapBuilder pos;
+ pos.ls.push_back(hoid);
+ pos.pos = 0;
+ pos.deep = true;
+
+ const Scrub::ScrubCounterSet& counters = Scrub::io_counters_ec;
+
+ int r = backend->be_scan_list(counters, smap, pos);
+ while (r == -EINPROGRESS) {
+ r = backend->be_scan_list(counters, smap, pos);
+ }
+
+ if (r != 0) {
+ std::cerr << "ERROR: be_scan_list failed for shard " << shard << ": " << cpp_strerror(r) << std::endl;
+ return true;
+ }
+
+ if (!smap.objects.contains(hoid)) {
+ std::cerr << "ERROR: Object not in scrub map for shard " << shard << std::endl;
+ return true;
+ }
+ }
+
+ if (!scrub_listener || !snap_reader) {
+ initialize_scrub_infra();
+ }
+
+ ceph_assert(scrub_listener);
+ ceph_assert(snap_reader);
+
+ MockPGBackendListener* primary_listener = get_primary_listener();
+ if (!primary_listener) {
+ std::cerr << "ERROR: No primary listener found" << std::endl;
+ return true;
+ }
+ int primary_shard = primary_listener->pg_whoami.osd;
+
+ PGBackend* primary_backend = get_primary_backend();
+ if (!primary_backend) {
+ std::cerr << "ERROR: No primary backend found" << std::endl;
+ return true;
+ }
+
+ const pg_pool_t* pool_info = osdmap->get_pg_pool(pool_id);
+ ceph_assert(pool_info != nullptr);
+
+ MockPgScrubBeListener pg_scrub_listener(primary_backend);
+ pg_scrub_listener.pool = std::make_shared<PGPool>(
+ osdmap, pool_id, *pool_info, "test_pool");
+ pg_scrub_listener.primary = primary_shard;
+ pg_scrub_listener.info.pgid = spgid;
+
+ std::set<pg_shard_t> acting_set;
+ for (int i = 0; i < k + m; ++i) {
+ acting_set.insert(pg_shard_t(i, shard_id_t(i)));
+ }
+
+ TestScrubBackend scrub_backend(*scrub_listener, pg_scrub_listener,
+ pg_shard_t(primary_shard, shard_id_t(primary_shard)),
+ false, scrub_level_t::deep, acting_set);
+
+ scrub_backend.new_chunk();
+
+ for (const auto& [pg_shard, smap] : scrub_maps) {
+ scrub_backend.insert_faked_smap(pg_shard, smap);
+ }
+
+ auto result = scrub_backend.scrub_compare_maps(false, *snap_reader);
+
+ return !result.inconsistent_objs.empty();
+}
+
+
+bufferlist PGBackendTestFixture::create_random_buffer(size_t size)
+{
+ bufferlist bl;
+
+ if (size == 0) {
+ return bl;
+ }
+
+ ceph::buffer::ptr bp(size);
+
+ std::random_device rd;
+ std::mt19937 gen(rd());
+ std::uniform_int_distribution<unsigned char> dis(0, 255);
+
+ for (size_t i = 0; i < size; ++i) {
+ bp[i] = dis(gen);
+ }
+
+ bl.append(std::move(bp));
+ return bl;
+}
+
+void PGBackendTestFixture::corrupt_shard_data(const hobject_t& obj, pg_shard_t shard)
+{
+ auto ch_it = chs.find(shard.osd);
+ if (ch_it == chs.end()) {
+ std::cerr << "ERROR: No collection handle for shard " << shard.osd << std::endl;
+ return;
+ }
+
+ ghobject_t ghoid(obj, ghobject_t::NO_GEN, shard.shard);
+
+ struct stat st;
+ int r = store->stat(ch_it->second, ghoid, &st);
+ if (r < 0) {
+ std::cerr << "ERROR: Failed to stat object on shard " << shard.osd
+ << ": " << cpp_strerror(r) << std::endl;
+ return;
+ }
+
+ uint64_t size = st.st_size;
+ if (size == 0) {
+ std::cerr << "WARNING: Object has zero size on shard " << shard.osd << std::endl;
+ return;
+ }
+
+ bufferlist zero_bl;
+ zero_bl.append_zero(size);
+
+ ObjectStore::Transaction t;
+ t.write(ch_it->second->cid, ghoid, 0, size, zero_bl);
+
+ r = store->queue_transaction(ch_it->second, std::move(t));
+ if (r < 0) {
+ std::cerr << "ERROR: Failed to corrupt object on shard " << shard.osd
+ << ": " << cpp_strerror(r) << std::endl;
+ return;
+ }
+
+ std::cout << "Corrupted shard " << shard.osd << " data for object " << obj
+ << " (wrote " << size << " bytes of zeros)" << std::endl;
+}
#include "os/ObjectStore.h"
#include "erasure-code/ErasureCodePlugin.h"
#include "test/osd/OSDMapTestHelpers.h"
+#include "test/osd/ScrubTestFixture.h"
+#include "osd/scrubber/scrub_backend.h"
+#include "osd/scrubber/pg_scrubber.h"
// Unified test fixture for EC and Replicated backend tests with ObjectStore.
// Uses PoolType to branch between EC (ECSwitch) and Replicated (ReplicatedBackend).
// OpTracker for wrapping messages in OpRequestRef
std::shared_ptr<OpTracker> op_tracker;
-
+ // Scrub infrastructure - initialized once and reused across scrub operations
+ std::unique_ptr<MockScrubBeListener> scrub_listener;
+ std::unique_ptr<MockSnapMapReader> snap_reader;
ceph::ErasureCodeInterfaceRef ec_impl;
std::map<int, std::unique_ptr<ECExtentCache::LRU>> lrus;
int k = 4; // data chunks
void setup_replicated_pool();
void cleanup_data_dir();
+protected:
+ void initialize_scrub_infra();
+
public:
const pg_pool_t& get_pool() const {
const pg_pool_t* pool = OSDMapTestHelpers::get_pool(osdmap, pool_id);
const std::string& obj_name,
int shard);
+ /**
+ * Scrub an object and verify it has no corruption.
+ *
+ * This utility method:
+ * 1. Builds scrub maps for all shards using be_scan_list()
+ * 2. Creates a ScrubBackend with mock listeners
+ * 3. Calls scrub_compare_maps() to check for inconsistencies
+ * 4. Returns true if corruption was detected, false otherwise
+ *
+ * The scrub infrastructure (mock listeners) is initialized once in SetUp()
+ * and reused across all scrub operations for efficiency.
+ *
+ * @param obj_name Name of the object to scrub
+ * @return true if corruption detected, false if object is consistent
+ */
+ bool scrub_object(const std::string& obj_name);
+
+ /**
+ * Corrupt the data for a specific shard of an object.
+ *
+ * This utility method directly writes zeros to the stored data for a given
+ * shard, simulating data corruption at the storage level. This is useful
+ * for testing scrub detection of corrupted data.
+ *
+ * @param obj The hobject_t identifying the object to corrupt
+ * @param shard The pg_shard_t identifying which shard to corrupt
+ */
+ void corrupt_shard_data(const hobject_t& obj, pg_shard_t shard);
+
+ /**
+ * Create a bufferlist filled with random data.
+ *
+ * This utility method generates a buffer of the specified size filled with
+ * random bytes. Useful for testing scenarios where random data is needed
+ * to avoid patterns that might mask bugs (e.g., XOR patterns in EC).
+ *
+ * @param size Size of the buffer to create in bytes
+ * @return A bufferlist containing random data
+ */
+ bufferlist create_random_buffer(size_t size);
+
};
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:nil -*-
+// vim: ts=8 sw=2 sts=2 expandtab
+
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2026 IBM
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+
+#pragma once
+
+#include <memory>
+#include <set>
+
+#include "include/expected.hpp"
+#include "osd/scrubber/scrub_backend.h"
+#include "osd/scrubber/pg_scrubber.h"
+#include "osd/PGBackend.h"
+#include "osd/PeeringState.h"
+#include "osd/osd_types.h"
+#include "test/osd/scrubber_generators.h"
+#include "global/global_context.h"
+
+// Type alias for compatibility with existing code
+using MockLoggerSinkSet = ScrubGenerator::MockLog;
+
+/**
+ * MockScrubBeListener - Mock implementation of ScrubBeListener for testing
+ *
+ * Provides minimal implementation of the ScrubBeListener interface for
+ * testing scrub backend functionality.
+ */
+class MockScrubBeListener : public ScrubBeListener {
+ public:
+ MockScrubBeListener()
+ : pg_whoami(0, shard_id_t::NO_SHARD),
+ primary_shard(0, shard_id_t::NO_SHARD) {}
+
+ MockScrubBeListener(pg_shard_t whoami, pg_shard_t primary)
+ : pg_whoami(whoami), primary_shard(primary) {}
+
+ std::ostream& gen_prefix(std::ostream& out) const override {
+ return out << "scrub: ";
+ }
+ CephContext* get_pg_cct() const override { return g_ceph_context; }
+ LoggerSinkSet& get_logger() const override { return logger; }
+ bool is_primary() const override { return pg_whoami == primary_shard; }
+ spg_t get_pgid() const override { return pgid; }
+ const OSDMapRef& get_osdmap() const override { return osdmap; }
+ void add_to_stats(const object_stat_sum_t&) override {}
+ void submit_digest_fixes(const digests_fixes_t&) override {}
+
+ pg_shard_t pg_whoami; // This shard's identity
+ pg_shard_t primary_shard; // The primary shard's identity
+ spg_t pgid;
+ OSDMapRef osdmap;
+ mutable MockLoggerSinkSet logger;
+};
+
+/**
+ * MockPgScrubBeListener - Mock implementation of Scrub::PgScrubBeListener for testing
+ *
+ * Provides EC-aware implementation of the PgScrubBeListener interface for
+ * testing scrub backend functionality with erasure-coded pools.
+ *
+ * Key features:
+ * - Delegates EC operations (encode/decode) to real ECBackend for correctness
+ * - Implements logical_to_ondisk_size() for proper EC shard size calculation
+ * - Delegates get_is_nonprimary_shard() to handle multi-zone EC pools correctly
+ */
+class MockPgScrubBeListener : public Scrub::PgScrubBeListener {
+ public:
+ MockPgScrubBeListener(PGBackend* be = nullptr) : backend(be) {}
+
+ const PGPool& get_pgpool() const override { return *pool; }
+
+ pg_shard_t get_primary() const override {
+ return pg_shard_t(primary, shard_id_t(primary));
+ }
+
+ void force_object_missing(ScrubberPasskey,
+ const std::set<pg_shard_t>& peer,
+ const hobject_t& oid,
+ eversion_t version) override {}
+
+ const pg_info_t& get_pg_info(ScrubberPasskey) const override { return info; }
+
+ uint64_t logical_to_ondisk_size(uint64_t logical_size, shard_id_t shard_id,
+ bool object_is_legacy_ec) const override {
+ return backend->be_get_ondisk_size(logical_size, shard_id_t(shard_id),
+ object_is_legacy_ec);
+ }
+
+ bool ec_can_decode(const shard_id_set& available_shards) const override {
+ return backend->ec_can_decode(available_shards);
+ }
+
+ shard_id_map<bufferlist> ec_encode_acting_set(const bufferlist& bl) const override {
+ return backend->ec_encode_acting_set(bl);
+ }
+
+ shard_id_map<bufferlist> ec_decode_acting_set(
+ const shard_id_map<bufferlist>& shard_map,
+ int chunk_size) const override {
+ return backend->ec_decode_acting_set(shard_map, chunk_size);
+ }
+
+ bool get_ec_supports_crc_encode_decode() const override {
+ return backend->get_ec_supports_crc_encode_decode();
+ }
+
+ ECUtil::stripe_info_t get_ec_sinfo() const override {
+ return backend->ec_get_sinfo();
+ }
+
+ bool is_waiting_for_unreadable_object() const override { return false; }
+
+ bool get_is_nonprimary_shard(const pg_shard_t& shard) const override {
+ return backend->get_is_nonprimary_shard(shard.shard);
+ }
+
+ bool get_is_hinfo_required() const override {
+ return backend->get_is_hinfo_required();
+ }
+
+ bool get_is_ec_optimized() const override {
+ return backend->get_is_ec_optimized();
+ }
+
+ std::shared_ptr<PGPool> pool;
+ int primary;
+ pg_info_t info;
+ PGBackend* backend;
+};
+
+/**
+ * MockSnapMapReader - Mock implementation of Scrub::SnapMapReaderI for testing
+ *
+ * Provides minimal implementation that returns empty snapshot sets,
+ * suitable for testing scrub functionality without snapshot complexity.
+ */
+class MockSnapMapReader : public Scrub::SnapMapReaderI {
+ public:
+ tl::expected<std::set<snapid_t>, result_t> get_snaps(
+ const hobject_t&) const override {
+ return std::set<snapid_t>{};
+ }
+ tl::expected<std::set<snapid_t>, result_t> get_snaps_check_consistency(
+ const hobject_t&) const override {
+ return std::set<snapid_t>{};
+ }
+};
+
+// Test helper to access ScrubBackend internals
+class TestScrubBackend : public ScrubBackend {
+public:
+ TestScrubBackend(ScrubBeListener& scrubber,
+ Scrub::PgScrubBeListener& pg,
+ pg_shard_t i_am,
+ bool repair,
+ scrub_level_t shallow_or_deep,
+ const std::set<pg_shard_t>& acting)
+ : ScrubBackend(scrubber, pg, i_am, repair, shallow_or_deep, acting)
+ {}
+
+ void insert_faked_smap(pg_shard_t shard, const ScrubMap& smap) {
+ ceph_assert(this_chunk.has_value());
+ this_chunk->received_maps[shard] = smap;
+ }
+
+ // Factory method to create MockScrubBeListener with proper initialization
+ static std::unique_ptr<MockScrubBeListener> create_scrub_listener(
+ const spg_t& pgid,
+ const OSDMapRef& osdmap)
+ {
+ auto listener = std::make_unique<MockScrubBeListener>();
+ listener->pgid = pgid;
+ listener->osdmap = osdmap;
+ return listener;
+ }
+
+ // Factory method to create MockSnapMapReader
+ static std::unique_ptr<MockSnapMapReader> create_snap_reader()
+ {
+ return std::make_unique<MockSnapMapReader>();
+ }
+};
+
+// Made with Bob
void SetUp() override {
PGBackendTestFixture::SetUp();
+ initialize_scrub_infra();
}
/**
void SetUp() override {
PGBackendTestFixture::SetUp();
+ initialize_scrub_infra();
}
void simulate_osd_failure(int failed_osd, int new_primary_instance)
#include <gtest/gtest.h>
#include "test/osd/ECPeeringTestFixture.h"
#include "test/osd/TestCommon.h"
-#include "osd/ECSwitch.h"
using namespace std;
set_config("osd_async_recovery_min_cost", "100");
}
+TEST_P(TestECFailoverWithPeering, ScrubClean) {
+ ASSERT_TRUE(all_shards_active()) << "Initial peering must complete";
+
+ const std::string obj_name = "test_scrub_corruption";
+ uint64_t object_size = k * stripe_unit;
+
+ bufferlist bl = create_random_buffer(object_size);
+ std::string test_data(bl.c_str(), bl.length());
+
+ std::cout << "Writing full-stripe object (" << object_size << " bytes of random data)" << std::endl;
+ create_and_write_verify(obj_name, test_data);
+
+ std::cout << "Scrubbing object to verify data integrity" << std::endl;
+ bool corruption_detected = scrub_object(obj_name);
+
+ ASSERT_FALSE(corruption_detected)
+ << "scrub_object() should NOT detect corruption when data is valid";
+
+ std::cout << "=== ScrubDetectsCorruption test completed successfully ===" << std::endl;
+}
+
+TEST_P(TestECFailoverWithPeering, ScrubDetectsCorruption) {
+ ASSERT_TRUE(all_shards_active()) << "Initial peering must complete";
+
+ const uint64_t object_size = k * stripe_unit;
+ const std::vector<int> shard_offsets = {/*0, 1, */k};
+ const bool supports_crc = ec_plugin == "isa";
+
+ for (int zone = 0; zone < 1; ++zone) {
+ for (int shard_offset : shard_offsets) {
+ const int absolute_shard = shard_offset;
+ const std::string obj_name =
+ "test_obj_zone_" + std::to_string(zone) +
+ "_shard_" + std::to_string(shard_offset);
+
+ bufferlist bl = create_random_buffer(object_size);
+ std::string test_data(bl.c_str(), bl.length());
+
+ std::cout << "\n=== ScrubDetectsCorruption: testing zone " << zone
+ << ", shard offset " << shard_offset
+ << " (absolute shard " << absolute_shard << ") ===" << std::endl;
+
+ std::cout << "Writing object " << obj_name << " (" << object_size
+ << " bytes of random data)" << std::endl;
+ create_and_write_verify(obj_name, test_data);
+
+ std::cout << "Corrupting object " << obj_name
+ << " for zone iteration " << zone
+ << " on relative shard " << shard_offset
+ << " using absolute shard " << absolute_shard << std::endl;
+ hobject_t hoid = make_test_object(obj_name);
+ corrupt_shard_data(hoid,
+ pg_shard_t(absolute_shard, shard_id_t(absolute_shard)));
+
+ std::cout << "Scrubbing object " << obj_name
+ << " to verify corruption detection for zone iteration " << zone
+ << ", shard offset " << shard_offset << std::endl;
+ bool corruption_detected = scrub_object(obj_name);
+
+ std::cout << "Zone iteration " << zone
+ << " corruption result for shard offset " << shard_offset
+ << ": " << (corruption_detected ? "detected" : "not detected")
+ << " (absolute shard " << absolute_shard
+ << ", supports_crc=" << (supports_crc ? "true" : "false")
+ << ")" << std::endl;
+
+ if (supports_crc) {
+ EXPECT_TRUE(corruption_detected)
+ << "scrub_object() should detect corruption for object " << obj_name
+ << " during zone iteration " << zone
+ << ", shard offset " << shard_offset
+ << " (absolute shard " << absolute_shard << ")";
+ } else {
+ EXPECT_FALSE(corruption_detected)
+ << "scrub_object() should not report corruption for object "
+ << obj_name << " when CRC-based detection is unsupported"
+ << " during zone iteration " << zone << ", shard offset "
+ << shard_offset << " (absolute shard " << absolute_shard << ")";
+ }
+ }
+ }
+
+ std::cout << "=== ScrubDetectsCorruption test completed successfully ===" << std::endl;
+}
+
+TEST_P(TestECFailoverWithPeering, ScrubPartialWrite) {
+ ASSERT_TRUE(all_shards_active()) << "Initial peering must complete";
+
+ const std::string obj_name = "test_scrub_partial_write";
+
+ uint64_t partial_size = stripe_unit / 2;
+
+ std::cout << "Creating partial write object with size " << partial_size
+ << " bytes (stripe_unit=" << stripe_unit << ", full stripe would be "
+ << (k * stripe_unit) << " bytes)" << std::endl;
+
+ bufferlist bl = create_random_buffer(partial_size);
+ std::string test_data(bl.c_str(), bl.length());
+
+ std::cout << "Writing partial object (" << partial_size << " bytes)" << std::endl;
+ create_and_write_verify(obj_name, test_data);
+
+ write(obj_name, 0, test_data, test_data.size());
+
+ // NOTE: Partial writes may expose scrub issues with EC pools
+ std::cout << "Scrubbing partial write object to test scrub behavior" << std::endl;
+ bool corruption_detected = scrub_object(obj_name);
+
+ std::cout << "Scrub result for partial write: "
+ << (corruption_detected ? "corruption detected" : "no corruption detected")
+ << std::endl;
+
+ EXPECT_FALSE(corruption_detected)
+ << "scrub_object() should NOT detect corruption on valid partial write";
+
+ std::cout << "=== ScrubPartialWrite test completed ===" << std::endl;
+}
+
// ---------------------------------------------------------------------------
// Instantiate TestECFailoverWithPeering with EC configurations
// ---------------------------------------------------------------------------
projects / ceph.git / commitdiff