From: Sridhar Seshasayee Date: Wed, 6 May 2026 15:11:33 +0000 (+0530) Subject: osd/PeeringState: add perf counters for PG rebuild times X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=4d29f4e4021081f19a304dea6938b6f4350eb748;p=ceph.git osd/PeeringState: add perf counters for PG rebuild times Track per-OSD PG rebuild duration in prepare_stats_for_publish() (primary OSD only). Three new counters are added to the recoverystate_perf collection: - pg_rebuild_duration: LONGRUNAVG time counter (sum+count pair); This counter internally maintains 'avgcount' that tracks the cumulative number of rebuild events. - pg_rebuild_max_secs: maximum rebuild duration observed in seconds - pg_rebuild_min_secs: minimum rebuild duration observed in seconds The logic uses a per-PG in-memory latch (rebuild_start_time) to capture the redundancy failure entry point. When a PG is first observed to be in a vulnerable state (PG_STATE_DEGRADED, PG_STATE_UNDERSIZED, or num_objects_misplaced/degraded > 0), it latches info.stats.last_change as the start time, provided last_change > last_clean, which ensures only genuine new failures after the prior clean interval are tracked. On recovery, the rebuild duration is computed as (now - rebuild_start_time) and is only recorded if delta num_objects_recovered > 0 or the PG had confirmed redundancy loss at latch time, filtering out spurious state transitions. The latch is cleared after each recorded event. The latch is also cleared in clear_primary_state() so that an interval change or role transition (primary -> replica) does not carry a stale start time or baseline recovered count into a future interval. The new last_degraded is intentionally not used here to retain compatibility with older Ceph branches where the field doesn't exist and requires encoding changes. A future simplification can replace the latch with a direct (last_clean - last_degraded) calculation once last_degraded is consistently available. This interim solution is a close approximation of the PG rebuild time. These counters are scraped per-OSD by ceph-exporter and exposed to Prometheus, enabling durability score calculations over user-defined time windows. Other Changes: 1. Add unit tests to TestPeeringState.cc that exercise the latch logic. 2. Add a standalone integration test to verify that the rebuild perf counters are incremented on the primary OSD after a recovery event. Fixes: https://tracker.ceph.com/issues/77493 Signed-off-by: Sridhar Seshasayee --- diff --git a/qa/standalone/osd/osd-recovery-stats.sh b/qa/standalone/osd/osd-recovery-stats.sh index 6a402d4914f..4d780ccbcb3 100755 --- a/qa/standalone/osd/osd-recovery-stats.sh +++ b/qa/standalone/osd/osd-recovery-stats.sh @@ -716,6 +716,87 @@ function TEST_recovery_last_degraded_undersized() { kill_daemons $dir || return 1 } +# Verify that the rebuild perf counters on the primary OSD increment after a +# real EC shard recovery. Kill one non-primary OSD so the PG goes degraded, +# then let recovery run to completion. After the PG is clean again: +# - pg_rebuild_duration.avgcount must be >= 1 +# - pg_rebuild_duration.sum must be > 0 (real time elapsed) +function TEST_rebuild_perf_ec_increments() { + local dir=$1 + local OSDS=4 + local ecpoolname=ectest + + run_mon $dir a || return 1 + run_mgr $dir x || return 1 + for osd in $(seq 0 $(expr $OSDS - 1)) + do + run_osd $dir $osd --osd-mclock-skip-benchmark=true || return 1 + done + + ceph osd erasure-code-profile set ecprofile \ + plugin=jerasure technique=reed_sol_van k=2 m=1 \ + crush-failure-domain=osd || return 1 + ceph osd pool create $ecpoolname 1 1 erasure ecprofile || return 1 + ceph osd pool set $ecpoolname min_size 2 || return 1 + wait_for_clean || return 1 + + # Write a few objects so the PG has data that must be recovered. + for i in $(seq 1 5) + do + rados -p $ecpoolname put obj$i /etc/hostname || return 1 + done + wait_for_clean || return 1 + + local primary + primary=$(get_primary $ecpoolname obj1) + local replica + replica=$(get_not_primary $ecpoolname obj1) + + # Pause recovery so the PG stays degraded long enough for the latch to + # fire inside prepare_stats_for_publish before recovery completes. + ceph osd set norecover || return 1 + + # Kill one non-primary OSD so the PG becomes degraded. + kill $(cat $dir/osd.${replica}.pid) + ceph osd down osd.${replica} || return 1 + ceph osd out osd.${replica} || return 1 + + # Release the hold and wait for full recovery. + ceph osd unset norecover || return 1 + wait_for_clean || return 1 + + # flush_pg_stats triggers publish_stats_to_osd on every OSD, which calls + # prepare_stats_for_publish and commits the rebuild counters. + flush_pg_stats || return 1 + + # The primary may be the same OSD we started with (we only killed a + # replica), but re-query in case CRUSH remapped the primary shard. + primary=$(get_primary $ecpoolname obj1) + + local dump + dump=$(CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.${primary}) \ + perf dump) || return 1 + + local rebuild_avgcount + rebuild_avgcount=$(jq '.recoverystate_perf.pg_rebuild_duration.avgcount' \ + <<< "$dump") + test "$rebuild_avgcount" -ge 1 || { + echo "FAIL: expected pg_rebuild_duration.avgcount>=1, got $rebuild_avgcount" + return 1 + } + + echo "$dump" | \ + jq -e '.recoverystate_perf.pg_rebuild_duration.sum > 0' > /dev/null || { + local rebuild_sum + rebuild_sum=$(jq '.recoverystate_perf.pg_rebuild_duration.sum' <<< "$dump") + echo "FAIL: expected pg_rebuild_duration.sum>0, got $rebuild_sum" + return 1 + } + + delete_pool $ecpoolname + kill_daemons $dir || return 1 +} + main osd-recovery-stats "$@" # Local Variables: diff --git a/src/osd/PeeringState.cc b/src/osd/PeeringState.cc index ba093f648b4..c3ed6be8107 100644 --- a/src/osd/PeeringState.cc +++ b/src/osd/PeeringState.cc @@ -1062,6 +1062,10 @@ void PeeringState::clear_primary_state() clear_recovery_state(); + rebuild_start_time = utime_t(); + rebuild_base_recovered = 0; + rebuild_had_redundancy_loss = false; + pg_committed_to = eversion_t(); missing_loc.clear(); pl->clear_primary_state(); @@ -4498,6 +4502,95 @@ std::optional PeeringState::prepare_stats_for_publish( if ((info.stats.state & PG_STATE_UNDERSIZED) == 0) info.stats.last_fullsized = now; + /** + * The following block is an interim solution to aggregate PG rebuild stats + * into a set of perf counters. The counterss are set based on the following + * existing pg_stat_t fields: + * - last_clean, last_change + * - num_objects_degraded, num_objects_misplaced, num_objects_recovered + * + * The PG rebuild stats are aggregated into the following recoverystate + * perf counters: + * - rs_pg_rebuild_duration: rebuild duration LONGRUNAVG time counter + * - rs_pg_rebuild_max_secs: maximum rebuild duration (secs) + * - rs_pg_rebuild_min_secs: minimum rebuild duration (secs) + * + * Workflow: + * 1. Only the acting primary OSD of the PG executes the logic to + * determine the rebuild stats. + * 2. The logic uses rebuild_start_time as a per-PG in-memory latch that + * captures the failure entry point. When a PG is considered vulnerable, + * rebuild_start_time latches info.stats.last_change as the start time, + * provided last_change > last_clean, which ensures only genuine new + * failures after the prior clean interval are tracked. + * 3. On recovery, the rebuild duration is computed as + * (now - rebuild_start_time) and is only recorded if delta + * num_objects_recovered > 0 or the PG had confirmed redundancy loss + * at latch time, filtering out spurious state transitions. The latch + * is cleared after each recorded event. + * + * last_degraded is intentionally not used in the interim solution to + * retain compatibility with older Ceph releases where this field doesn't + * exist and requires encoding changes. The interim solution is a + * close approximation of the PG rebuild time. + * + * A future simplification can replace the latch with a direct + * (last_clean - last_degraded) calculation once last_degraded is + * consistently available. + */ + if (is_primary()) { + const int64_t num_degraded = info.stats.stats.sum.num_objects_degraded; + const int64_t num_misplaced = info.stats.stats.sum.num_objects_misplaced; + const int64_t num_recovered = info.stats.stats.sum.num_objects_recovered; + const bool is_vulnerable = + (info.stats.state & (PG_STATE_DEGRADED | PG_STATE_UNDERSIZED)) || + num_degraded > 0 || num_misplaced > 0; + + if (is_vulnerable) { + // Latch the failure entry point on the first publish after a new + // failure; last_change captures when the state transition occurred. + if (rebuild_start_time == utime_t()) { + const bool new_failure = + info.stats.last_clean == utime_t() || + info.stats.last_change > info.stats.last_clean; + if (new_failure) { + rebuild_start_time = info.stats.last_change; + rebuild_base_recovered = num_recovered; + rebuild_had_redundancy_loss = + (num_degraded > 0 || num_misplaced > 0); + psdout(15) << "rebuild-stats: latched failure start for " + << info.pgid << " at " << rebuild_start_time << dendl; + } + } + } else if (rebuild_start_time != utime_t()) { + // PG recovered — record rebuild time if this was a genuine event. + const int64_t delta_recovered = num_recovered - rebuild_base_recovered; + const utime_t rebuild_dur = now - rebuild_start_time; + + if (rebuild_dur.to_msec() > 0 && + (delta_recovered > 0 || rebuild_had_redundancy_loss)) { + PerfCounters &perf = pl->get_peering_perf(); + perf.tinc(rs_pg_rebuild_duration, rebuild_dur); + + const uint64_t rebuild_secs = (uint64_t)rebuild_dur.sec(); + if (rebuild_secs > perf.get(rs_pg_rebuild_max_secs)) { + perf.set(rs_pg_rebuild_max_secs, rebuild_secs); + } + const uint64_t cur_min = perf.get(rs_pg_rebuild_min_secs); + if (cur_min == 0 || rebuild_secs < cur_min) { + perf.set(rs_pg_rebuild_min_secs, rebuild_secs); + } + psdout(15) << "rebuild-stats: recorded rebuild for " << info.pgid + << " duration=" << rebuild_dur + << " delta_recovered=" << delta_recovered << dendl; + } + // reset for the next event + rebuild_start_time = utime_t(); + rebuild_base_recovered = 0; + rebuild_had_redundancy_loss = false; + } + } + // check if the PG is vulnerable if (info.stats.state & (PG_STATE_DEGRADED|PG_STATE_UNDERSIZED)) { // set last_degraded only if we are entering a new diff --git a/src/osd/PeeringState.h b/src/osd/PeeringState.h index ae426060aba..94ee299cdfb 100644 --- a/src/osd/PeeringState.h +++ b/src/osd/PeeringState.h @@ -1579,6 +1579,17 @@ public: bool backfill_reserved = false; bool backfill_reserving = false; + /** + * Per-PG latch state for rebuild time tracking. Cleared after each + * completed rebuild event is recorded in the perf counters. + * The state is also cleared in clear_primary_state() so that an interval + * change or role transition (primary -> replica) does not carry a stale + * start time or baseline recovered count into a future interval. + */ + utime_t rebuild_start_time; + int64_t rebuild_base_recovered = 0; + bool rebuild_had_redundancy_loss = false; + PeeringMachine machine; void update_osdmap_ref(OSDMapRef newmap) { @@ -1680,6 +1691,17 @@ public: } } + // Accessors for the per-PG rebuild latch state. + utime_t get_rebuild_start_time() const { + return rebuild_start_time; + } + int64_t get_rebuild_base_recovered() const { + return rebuild_base_recovered; + } + bool get_rebuild_had_redundancy_loss() const { + return rebuild_had_redundancy_loss; + } + private: bool check_prior_readable_down_osds(const OSDMapRef& map); diff --git a/src/osd/osd_perf_counters.cc b/src/osd/osd_perf_counters.cc index 51d98431713..2fc483131ec 100644 --- a/src/osd/osd_perf_counters.cc +++ b/src/osd/osd_perf_counters.cc @@ -542,6 +542,15 @@ PerfCounters *build_recoverystate_perf(CephContext *cct) { rs_perf.add_u64_counter(rs_update_stats_invalidated, "update_stats_invalidated", "Number of times pg stats received invalidations during stats updates"); rs_perf.add_u64_counter(rs_append_log_stats_invalidated, "append_log_stats_invalidated", "Number of times pg stats received invalidations when appending new log entries"); rs_perf.add_u64_counter(rs_merge_log_stats_invalidated, "merge_log_stats_invalidated", "Number of times pg stats received invalidations during merging of log entries"); + rs_perf.add_time_avg(rs_pg_rebuild_duration, "pg_rebuild_duration", + "Average PG rebuild duration on this OSD (primary role only)", + NULL, PerfCountersBuilder::PRIO_USEFUL); + rs_perf.add_u64(rs_pg_rebuild_max_secs, "pg_rebuild_max_secs", + "Max PG rebuild duration seen on this OSD in seconds (primary role only)", + NULL, PerfCountersBuilder::PRIO_USEFUL); + rs_perf.add_u64(rs_pg_rebuild_min_secs, "pg_rebuild_min_secs", + "Min PG rebuild duration seen on this OSD in seconds (primary role only)", + NULL, PerfCountersBuilder::PRIO_USEFUL); return rs_perf.create_perf_counters(); } diff --git a/src/osd/osd_perf_counters.h b/src/osd/osd_perf_counters.h index 4b56b79215d..c8fccb95152 100644 --- a/src/osd/osd_perf_counters.h +++ b/src/osd/osd_perf_counters.h @@ -263,6 +263,9 @@ enum { rs_update_stats_invalidated, rs_append_log_stats_invalidated, rs_merge_log_stats_invalidated, + rs_pg_rebuild_duration, + rs_pg_rebuild_max_secs, + rs_pg_rebuild_min_secs, rs_last, }; diff --git a/src/test/osd/TestPeeringState.cc b/src/test/osd/TestPeeringState.cc index f77898f16a0..067456f4261 100644 --- a/src/test/osd/TestPeeringState.cc +++ b/src/test/osd/TestPeeringState.cc @@ -29,7 +29,9 @@ * PrimaryLogPG to allow this to be tested. */ +#include #include +#include #include #include "test/osd/MockConnection.h" #include "test/osd/MockECRecPred.h" @@ -1185,6 +1187,15 @@ protected: } } + // Helper - call prepare_stats_for_publish on an OSD, discarding the result. + // Passing nullopt forces the publish branch unconditionally (no last-known + // stat to compare against), which is what we need to drive the latch logic. + void call_prepare_stats(int osd) + { + get_ps(osd)->prepare_stats_for_publish( + std::nullopt, object_stat_collection_t()); + } + // ============================================================================ // GTest - Setup and Teardown // ============================================================================ @@ -2181,6 +2192,337 @@ TEST_F(PeeringStateTest, Issue74218) { verify_logs(); } +// ============================================================================ +// Rebuild Stats Perf Counter Tests +// +// These tests exercise the latch logic in prepare_stats_for_publish() that +// feeds rs_pg_rebuild_duration, rs_pg_rebuild_max_secs, and +// rs_pg_rebuild_min_secs. +// Design notes: +// - call_prepare_stats(osd) passes nullopt so the publish branch always +// runs, which is needed to drive the latch even when stats are unchanged. +// - A 10 ms sleep between the latch call and the clean call ensures +// rebuild_dur.to_msec() > 0 so the record is committed. +// - rebuild_secs = (uint64_t)rebuild_dur.sec(), so sub-second rebuilds +// leave pg_rebuild_max_secs and pg_rebuild_min_secs at 0. Those gauges +// are verified only for their mutual ordering (min <= max); absolute +// values are verified via pg_rebuild_duration.sum which is in nanoseconds. +// ============================================================================ + +// One complete failure+recovery cycle does the following: +// - Swaps acting[slot] from old_osd to new_osd. +// - Peers, latches (via call_prepare_stats while degraded). +// - Sleeps sleep_ms to guarantee non-zero rebuild duration. +// - Recovers, verifies clean, calls prepare_stats to commit the record. +// +// Acting set BEFORE call: [..., old_osd, ...] at position slot. +// Acting set AFTER call: [..., new_osd, ...] at position slot. +// The old_osd PeeringState is left in osd_peeringstate (may go stale). + +// ============================================================================ +// Test 1: Primary OSD records all four counters after a recovery event. +// ============================================================================ +TEST_F(PeeringStateTest, RebuildStatsLatchAndCount) { + dout(0) << "== RebuildStatsLatchAndCount ==" << dendl; + test_create_peering_state(); + test_init(); + test_event_initialize(); + eversion_t v = test_append_log_entry(); + test_peering(); + verify_all_active_clean(v, eversion_t()); + + // Stamp last_clean on the primary so new_failure detection works later. + call_prepare_stats(acting_primary); + + PerfCounters *perf = get_listener(acting_primary)->recoverystate_perf; + + // Introduce a missing replica: swap acting[1] from OSD 1 to OSD 9. + // OSD 9 starts fresh and needs the log entry recovered to it. + modify_up_acting(1, 9); + test_create_peering_state(9, 1); + test_init(9); + test_event_initialize(9); + test_peering(); + // PG is now active+recovering+degraded on the primary. + + // Latch: first prepare_stats call while vulnerable. + // The state-change block sets info.stats.last_change = now and + // rebuild_start_time = last_change inside the latch branch. + call_prepare_stats(acting_primary); + + // Sleep so that rebuild_dur.to_msec() > 0 when the record fires. + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + + // Drive the PG back to active+clean. + test_begin_peer_recover(9, 1); + test_on_peer_recover(9, 1, v); + test_recover_got(9, v); + test_object_recovered(); + test_event_all_replicas_recovered(); + verify_all_active_clean(v, eversion_t()); + + // Record: prepare_stats while clean fires the else branch, commits record. + call_prepare_stats(acting_primary); + + // pg_rebuild_duration: at least one sample with a positive nanosecond sum. + auto [sum_ns, count] = perf->get_tavg_ns(rs_pg_rebuild_duration); + EXPECT_GE(count, 1u); + EXPECT_GT(sum_ns, 0u); + + // max/min gauges track whole seconds; sub-second rebuilds leave both at 0. + // The key invariant is that min never exceeds max. + EXPECT_LE(perf->get(rs_pg_rebuild_min_secs), perf->get(rs_pg_rebuild_max_secs)); +} + +// ============================================================================ +// Test 2: A replica OSD never records anything, even when the PG is degraded. +// ============================================================================ +TEST_F(PeeringStateTest, RebuildStatsReplicaSkips) { + dout(0) << "== RebuildStatsReplicaSkips ==" << dendl; + test_create_peering_state(); + test_init(); + test_event_initialize(); + eversion_t v = test_append_log_entry(); + test_peering(); + verify_all_active_clean(v, eversion_t()); + + call_prepare_stats(acting_primary); + + modify_up_acting(1, 9); + test_create_peering_state(9, 1); + test_init(9); + test_event_initialize(9); + test_peering(); + + // OSD 9 is the recovering replica. prepare_stats_for_publish() is only + // called by the primary. + PerfCounters *replica_perf = get_listener(9)->recoverystate_perf; + + // Drive the primary through the full latch+record cycle. + call_prepare_stats(acting_primary); // latch fires on primary + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + + test_begin_peer_recover(9, 1); + test_on_peer_recover(9, 1, v); + test_recover_got(9, v); + test_object_recovered(); + test_event_all_replicas_recovered(); + verify_all_active_clean(v, eversion_t()); + + call_prepare_stats(acting_primary); // record fires on primary + + // All rebuild counters on the replica must remain at their initial values. + EXPECT_EQ(replica_perf->get(rs_pg_rebuild_max_secs), 0u); + EXPECT_EQ(replica_perf->get(rs_pg_rebuild_min_secs), 0u); + auto [sum_ns, count] = + replica_perf->get_tavg_ns(rs_pg_rebuild_duration); + EXPECT_EQ(count, 0u); + EXPECT_EQ(sum_ns, 0u); + + // Primary must have recorded the event (sanity-check the other side). + auto [primary_sum_ns, primary_count] = + get_listener(acting_primary)->recoverystate_perf->get_tavg_ns(rs_pg_rebuild_duration); + EXPECT_GE(primary_count, 1u); +} + +// ============================================================================ +// Test 3: A second prepare_stats call while still vulnerable does not +// overwrite the already-latched start time or double-record the event. +// ============================================================================ +TEST_F(PeeringStateTest, RebuildStatsNoDoubleLatch) { + dout(0) << "== RebuildStatsNoDoubleLatch ==" << dendl; + test_create_peering_state(); + test_init(); + test_event_initialize(); + eversion_t v = test_append_log_entry(); + test_peering(); + verify_all_active_clean(v, eversion_t()); + + call_prepare_stats(acting_primary); + + modify_up_acting(1, 9); + test_create_peering_state(9, 1); + test_init(9); + test_event_initialize(9); + test_peering(); + + PerfCounters *perf = get_listener(acting_primary)->recoverystate_perf; + + // First vulnerable call — latch fires (rebuild_start_time set). + call_prepare_stats(acting_primary); + + // Second vulnerable call while still degraded. The latch is guarded by + // rebuild_start_time == utime_t(), which is now false, so the start + // time is not overwritten and no record is emitted. + call_prepare_stats(acting_primary); + + // Nothing recorded yet (PG still degraded): duration avgcount must be 0. + { + auto [sum_ns, count] = perf->get_tavg_ns(rs_pg_rebuild_duration); + EXPECT_EQ(count, 0u); + } + + // Now complete the recovery and verify exactly one event is recorded. + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + test_begin_peer_recover(9, 1); + test_on_peer_recover(9, 1, v); + test_recover_got(9, v); + test_object_recovered(); + test_event_all_replicas_recovered(); + verify_all_active_clean(v, eversion_t()); + call_prepare_stats(acting_primary); + + { + auto [sum_ns, count] = perf->get_tavg_ns(rs_pg_rebuild_duration); + EXPECT_EQ(count, 1u); + } +} + +// ============================================================================ +// Test 4: Two sequential failure+recovery cycles accumulate independently. +// +// Cycle 1: acting[1] = 9 (OSD 1 -> OSD 9) +// Cycle 2: acting[2] = 8 (OSD 2 -> OSD 8, while OSD 9 stays in slot 1) +// +// Using distinct slots avoids having to bring OSD 9 stale mid-test while +// still exercising two independent latch+record sequences on the same primary. +// ============================================================================ +TEST_F(PeeringStateTest, RebuildStatsCountAccumulates) { + dout(0) << "== RebuildStatsCountAccumulates ==" << dendl; + test_create_peering_state(); + test_init(); + test_event_initialize(); + eversion_t v = test_append_log_entry(); + test_peering(); + verify_all_active_clean(v, eversion_t()); + + PerfCounters *perf = get_listener(acting_primary)->recoverystate_perf; + + // ---- Cycle 1: replace acting[1] with OSD 9 ---- + call_prepare_stats(acting_primary); // stamp last_clean + + modify_up_acting(1, 9); + test_create_peering_state(9, 1); + test_init(9); + test_event_initialize(9); + test_peering(); + // active+recovering: OSD 9 needs recovery. + + call_prepare_stats(acting_primary); // latch fires + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + + test_begin_peer_recover(9, 1); + test_on_peer_recover(9, 1, v); + test_recover_got(9, v); + test_object_recovered(); + test_event_all_replicas_recovered(); + verify_all_active_clean(v, eversion_t()); + // Flush share_pg_info messages queued by cycle 1's Clean so they are + // delivered to the current replicas now, not stale during cycle 2. + dispatch_all(); + call_prepare_stats(acting_primary); // record fires + + { + auto [sum_ns, count] = perf->get_tavg_ns(rs_pg_rebuild_duration); + EXPECT_EQ(count, 1u); + } + + // ---- Cycle 2: replace acting[2] with OSD 8 ---- + // OSD 9 remains in slot 1; OSD 8 joins as a fresh replica at slot 2. + call_prepare_stats(acting_primary); // stamp last_clean for cycle 2 + + modify_up_acting(2, 8); + test_create_peering_state(8, 2); + test_init(8); + test_event_initialize(8); + test_peering(); + // active+recovering: OSD 8 needs the object recovered to it. + + call_prepare_stats(acting_primary); // second latch fires + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + + test_begin_peer_recover(8, 2); + test_on_peer_recover(8, 2, v); + test_recover_got(8, v); + test_object_recovered(); + test_event_all_replicas_recovered(); + verify_all_active_clean(v, eversion_t()); + call_prepare_stats(acting_primary); // second record fires + + // Both events must appear in the duration avgcount. + auto [sum_ns, count] = perf->get_tavg_ns(rs_pg_rebuild_duration); + EXPECT_EQ(count, 2u); + EXPECT_GT(sum_ns, 0u); + + // min <= max invariant must hold across both events. + EXPECT_LE(perf->get(rs_pg_rebuild_min_secs), perf->get(rs_pg_rebuild_max_secs)); +} + +// ============================================================================ +// Test 5: Latch is discarded when the OSD loses its primary role mid-rebuild. +// +// If a new interval begins while rebuild_start_time is set and the OSD +// transitions from primary to stray, clear_primary_state() must discard the +// stale latch so no spurious rebuild event is emitted for a recovery that the +// OSD no longer owns. +// ============================================================================ +TEST_F(PeeringStateTest, RebuildStatsLatchClearedOnRoleChange) { + dout(0) << "== RebuildStatsLatchClearedOnRoleChange ==" << dendl; + test_create_peering_state(); + test_init(); + test_event_initialize(); + eversion_t v = test_append_log_entry(); + test_peering(); + verify_all_active_clean(v, eversion_t()); + + // Record last_clean so the new_failure guard inside the latch is satisfied. + call_prepare_stats(acting_primary); // acting_primary = OSD 0 + + // --- Phase 1: degrade the PG while OSD 0 is still primary. --- + // Replace acting[1] (OSD 1) with OSD 9; OSD 9 needs recovery. + modify_up_acting(1, 9); + test_create_peering_state(9, 1); + test_init(9); + test_event_initialize(9); + test_peering(); + // PG is now active+recovering+degraded; OSD 0 remains primary. + + PerfCounters *perf_osd0 = get_listener(0)->recoverystate_perf; + + // Latch: first prepare_stats call while vulnerable sets rebuild_start_time + // on OSD 0. + call_prepare_stats(acting_primary); + ASSERT_NE(get_ps(0)->get_rebuild_start_time(), utime_t()) + << "latch must be set before role change"; + + // --- Phase 2: change the primary before recovery completes. --- + // Swap acting[0] from OSD 0 to OSD 7. OSD 0 leaves the acting set + // entirely and becomes a stray; OSD 7 takes over as primary. + modify_up_acting(0, 7); + acting_primary = 7; + up_primary = 7; + test_create_peering_state(7, 0); + test_init(7); + test_event_initialize(7); + + // advance_map on OSD 0: the new acting set excludes OSD 0, so + // should_restart_peering()->start_peering_interval()->clear_primary_state() + // resets the three latch variables. + test_event_advance_map(); + + // --- Phase 3: verify latch is cleared on the old primary. --- + EXPECT_EQ(get_ps(0)->get_rebuild_start_time(), utime_t()); + EXPECT_EQ(get_ps(0)->get_rebuild_base_recovered(), 0); + EXPECT_FALSE(get_ps(0)->get_rebuild_had_redundancy_loss()); + + // No rebuild record must have been emitted: the latch was discarded, not + // fired. A spurious emission here would record a duration spanning a + // recovery that OSD 0 did not complete. + auto [sum_ns, count] = perf_osd0->get_tavg_ns(rs_pg_rebuild_duration); + EXPECT_EQ(count, 0u); + EXPECT_EQ(sum_ns, 0u); +} + // ============================================================================ // Main // ============================================================================