From: Kotresh HR Date: Sun, 21 Jun 2026 17:28:38 +0000 (+0530) Subject: mgr/mirroring: detect stale snapshot mirror sync metrics in omap X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=15e998b865b99b3091054321cf0fdf6df9f327ab;p=ceph.git mgr/mirroring: detect stale snapshot mirror sync metrics in omap Persisted metrics in the cephfs_mirror omap can outlive the writing daemon when cephfs-mirror stops or a directory is reshuffled to another instance; mgr would keep reporting stale progress until the owning daemon writes again. Extend format_and_order_sync_stat_for_display() to compare persisted _instance_id against InstanceWatcher live instances (via FSPolicy.get_live_instance_ids()) and the directory's tracked instance (via Policy.get_tracked_instance_id()). Mark metrics stale when the persisted writer is no longer live (any state), or when it does not match the tracked instance while persisted state is not "idle". Show state "stale" with current_syncing_snap omitted. Pass policy and live instance ids through load_sync_stat_metrics() and fetch_sync_stat_metrics(), and into sync_stat_complete_cache and sync_stat_partial_cache loaders. Cache hits serve already-formatted (stale-marked) entries until TTL expiry without re-checking instance liveness. Fixes: https://tracker.ceph.com/issues/76686 Signed-off-by: Kotresh HR --- diff --git a/src/pybind/mgr/mirroring/fs/dir_map/policy.py b/src/pybind/mgr/mirroring/fs/dir_map/policy.py index aef90b55fe5f..da61bec940b4 100644 --- a/src/pybind/mgr/mirroring/fs/dir_map/policy.py +++ b/src/pybind/mgr/mirroring/fs/dir_map/policy.py @@ -105,6 +105,12 @@ class Policy: 'purging': dir_state.purging} return None + def get_tracked_instance_id(self, dir_path): + lookup = self.lookup(dir_path) + if not lookup: + return None + return lookup.get('instance_id') + def map(self, dir_path, dir_state): log.debug(f'mapping {dir_path}') min_instance_id = None diff --git a/src/pybind/mgr/mirroring/fs/metrics/format.py b/src/pybind/mgr/mirroring/fs/metrics/format.py index b208151c167c..87fc6c4a86ae 100644 --- a/src/pybind/mgr/mirroring/fs/metrics/format.py +++ b/src/pybind/mgr/mirroring/fs/metrics/format.py @@ -75,12 +75,44 @@ def format_peer_status_metrics(metrics, dir_path, peer_uuid, stat): metrics.setdefault(dir_path, {}).setdefault('peer', {})[peer_uuid] = stat +def _mark_sync_stat_stale(stat): + out = dict(stat) + out.pop('current_syncing_snap', None) + out['state'] = 'stale' + return out + + +def _apply_stale_sync_metrics(stat, policy=None, dir_path=None, + live_instance_ids=None): + if not isinstance(stat, dict) or policy is None or dir_path is None: + return stat + + persisted_instance_id = stat.get('_instance_id') + if persisted_instance_id is None: + return stat + + persisted_id = str(persisted_instance_id) + if (live_instance_ids is not None and + persisted_id not in live_instance_ids): + return _mark_sync_stat_stale(stat) + + tracked_id = policy.get_tracked_instance_id(dir_path) + if tracked_id is not None: + tracked_id = str(tracked_id) + if (tracked_id != persisted_id and + stat.get('state') != 'idle'): + return _mark_sync_stat_stale(stat) + return stat + + # to match the output of peer_status -def format_and_order_sync_stat_for_display(stat): +def format_and_order_sync_stat_for_display(stat, policy=None, dir_path=None, + live_instance_ids=None): if not isinstance(stat, dict): return stat - out = dict(stat) + out = dict(_apply_stale_sync_metrics( + stat, policy, dir_path, live_instance_ids)) last_synced_snap = out.get('last_synced_snap') if isinstance(last_synced_snap, dict): snap = dict(last_synced_snap) diff --git a/src/pybind/mgr/mirroring/fs/metrics/load.py b/src/pybind/mgr/mirroring/fs/metrics/load.py index d157b36b7c20..dc7682fa7078 100644 --- a/src/pybind/mgr/mirroring/fs/metrics/load.py +++ b/src/pybind/mgr/mirroring/fs/metrics/load.py @@ -57,7 +57,8 @@ def open_metadata_ioctx(rados_inst, fs_map, filesystem): f'failed to open metadata pool for {filesystem}') -def load_sync_stat_metrics(ioctx, filesystem, peer_uuid=None): +def load_sync_stat_metrics(ioctx, filesystem, peer_uuid=None, policy=None, + live_instance_ids=None): metrics: Dict[str, Any] = {} prefix = f'{SYNC_STAT_KEY_PREFIX}/{filesystem}/' if peer_uuid: @@ -89,7 +90,8 @@ def load_sync_stat_metrics(ioctx, filesystem, peer_uuid=None): continue format_peer_status_metrics( metrics, dir_path, peer, - format_and_order_sync_stat_for_display(stat)) + format_and_order_sync_stat_for_display( + stat, policy, dir_path, live_instance_ids)) start = omap_vals.popitem()[0] except rados.Error as e: log.error(f'failed to read sync stat omap: {e}') @@ -97,7 +99,8 @@ def load_sync_stat_metrics(ioctx, filesystem, peer_uuid=None): return metrics -def fetch_sync_stat_metrics(ioctx, filesystem, peers, mirrored_dir_path, peer_uuid): +def fetch_sync_stat_metrics(ioctx, filesystem, peers, mirrored_dir_path, + peer_uuid, policy=None, live_instance_ids=None): if mirrored_dir_path: dir_path = norm_path(mirrored_dir_path) keys = [sync_stat_omap_key(filesystem, peer, dir_path) for peer in peers] @@ -109,8 +112,10 @@ def fetch_sync_stat_metrics(ioctx, filesystem, peers, mirrored_dir_path, peer_uu if stat is not None: format_peer_status_metrics( metrics, dir_path, peer, - format_and_order_sync_stat_for_display(stat)) + format_and_order_sync_stat_for_display( + stat, policy, dir_path, live_instance_ids)) return metrics, False, dir_path - metrics = load_sync_stat_metrics(ioctx, filesystem, peer_uuid) + metrics = load_sync_stat_metrics( + ioctx, filesystem, peer_uuid, policy, live_instance_ids) return metrics, True, None diff --git a/src/pybind/mgr/mirroring/fs/snapshot_mirror.py b/src/pybind/mgr/mirroring/fs/snapshot_mirror.py index 2b1fa76425e3..5c6a1230d31d 100644 --- a/src/pybind/mgr/mirroring/fs/snapshot_mirror.py +++ b/src/pybind/mgr/mirroring/fs/snapshot_mirror.py @@ -66,6 +66,13 @@ class FSPolicy: def schedule_action(self, dir_paths): self.dir_paths.extend(dir_paths) + def get_live_instance_ids(self): + watcher = self.instance_watcher + if watcher is None: + return None + with watcher.lock: + return frozenset(str(instance_id) for instance_id in watcher.instances) + def init(self, dir_mapping, instances): with self.lock: self.policy.init(dir_mapping) @@ -774,9 +781,12 @@ class FSSnapshotMirror: """ log.debug('sync stat metrics for filesystem %s loaded from omap (complete)', filesystem) + fspolicy = self.pool_policy[filesystem] ioctx = metrics_load.open_metadata_ioctx( self.rados, self.fs_map, filesystem) - return metrics_load.load_sync_stat_metrics(ioctx, filesystem) + return metrics_load.load_sync_stat_metrics( + ioctx, filesystem, None, fspolicy.policy, + fspolicy.get_live_instance_ids()) @lru_cache_timeout( lambda self, *_args, **_kwargs: CACHE_TTL_SECS, @@ -793,10 +803,12 @@ class FSSnapshotMirror: 'loaded from omap', filesystem, dir_path, peer_scope) peers = {peer_id: None for peer_id in peer_ids} + fspolicy = self.pool_policy[filesystem] ioctx = metrics_load.open_metadata_ioctx( self.rados, self.fs_map, filesystem) metrics, _, _ = metrics_load.fetch_sync_stat_metrics( - ioctx, filesystem, peers, dir_path, None) + ioctx, filesystem, peers, dir_path, None, + fspolicy.policy, fspolicy.get_live_instance_ids()) return metrics def metrics_status(self, filesystem, mirrored_dir_path, peer_uuid):