mgr/mirroring: detect stale snapshot mirror sync metrics in omap

Persisted metrics in the cephfs_mirror omap can outlive the writing
daemon when cephfs-mirror stops or a directory is reshuffled to another
instance; mgr would keep reporting stale progress until the owning
daemon writes again.

Extend format_and_order_sync_stat_for_display() to compare persisted
_instance_id against InstanceWatcher live instances (via
FSPolicy.get_live_instance_ids()) and the directory's tracked instance
(via Policy.get_tracked_instance_id()). Mark metrics stale when the
persisted writer is no longer live (any state), or when it does not
match the tracked instance while persisted state is not "idle". Show
state "stale" with current_syncing_snap omitted.

Pass policy and live instance ids through load_sync_stat_metrics() and
fetch_sync_stat_metrics(), and into sync_stat_complete_cache and
sync_stat_partial_cache loaders. Cache hits serve already-formatted
(stale-marked) entries until TTL expiry without re-checking instance
liveness.

Fixes: https://tracker.ceph.com/issues/76686
Signed-off-by: Kotresh HR <khiremat@redhat.com>
(cherry picked from commit 15e998b865b99b3091054321cf0fdf6df9f327ab)

diff --git a/src/pybind/mgr/mirroring/fs/dir_map/policy.py b/src/pybind/mgr/mirroring/fs/dir_map/policy.py
index aef90b55fe5f4afedfeb704754a691a599da077a..da61bec940b414678d9ea8c8f1ca27bfa74dffbe 100644 (file)
--- a/src/pybind/mgr/mirroring/fs/dir_map/policy.py
+++ b/src/pybind/mgr/mirroring/fs/dir_map/policy.py
@@ -105,6 +105,12 @@ class Policy:
                         'purging': dir_state.purging}
             return None
 
+    def get_tracked_instance_id(self, dir_path):
+        lookup = self.lookup(dir_path)
+        if not lookup:
+            return None
+        return lookup.get('instance_id')
+
     def map(self, dir_path, dir_state):
         log.debug(f'mapping {dir_path}')
         min_instance_id = None

diff --git a/src/pybind/mgr/mirroring/fs/metrics/format.py b/src/pybind/mgr/mirroring/fs/metrics/format.py
index b208151c167cc732f580011bf0d3828cba6a45eb..87fc6c4a86aedcdbf4c767d9d7306f623b9fd94a 100644 (file)
--- a/src/pybind/mgr/mirroring/fs/metrics/format.py
+++ b/src/pybind/mgr/mirroring/fs/metrics/format.py
@@ -75,12 +75,44 @@ def format_peer_status_metrics(metrics, dir_path, peer_uuid, stat):
     metrics.setdefault(dir_path, {}).setdefault('peer', {})[peer_uuid] = stat
 
 
+def _mark_sync_stat_stale(stat):
+    out = dict(stat)
+    out.pop('current_syncing_snap', None)
+    out['state'] = 'stale'
+    return out
+
+
+def _apply_stale_sync_metrics(stat, policy=None, dir_path=None,
+                              live_instance_ids=None):
+    if not isinstance(stat, dict) or policy is None or dir_path is None:
+        return stat
+
+    persisted_instance_id = stat.get('_instance_id')
+    if persisted_instance_id is None:
+        return stat
+
+    persisted_id = str(persisted_instance_id)
+    if (live_instance_ids is not None and
+            persisted_id not in live_instance_ids):
+        return _mark_sync_stat_stale(stat)
+
+    tracked_id = policy.get_tracked_instance_id(dir_path)
+    if tracked_id is not None:
+        tracked_id = str(tracked_id)
+    if (tracked_id != persisted_id and
+            stat.get('state') != 'idle'):
+        return _mark_sync_stat_stale(stat)
+    return stat
+
+
 # to match the output of peer_status
-def format_and_order_sync_stat_for_display(stat):
+def format_and_order_sync_stat_for_display(stat, policy=None, dir_path=None,
+                                           live_instance_ids=None):
     if not isinstance(stat, dict):
         return stat
 
-    out = dict(stat)
+    out = dict(_apply_stale_sync_metrics(
+        stat, policy, dir_path, live_instance_ids))
     last_synced_snap = out.get('last_synced_snap')
     if isinstance(last_synced_snap, dict):
         snap = dict(last_synced_snap)

diff --git a/src/pybind/mgr/mirroring/fs/metrics/load.py b/src/pybind/mgr/mirroring/fs/metrics/load.py
index d157b36b7c2086dc12abb802919a4deca9b6dd60..dc7682fa707896043c6fd8a5e14b119529ac769f 100644 (file)
--- a/src/pybind/mgr/mirroring/fs/metrics/load.py
+++ b/src/pybind/mgr/mirroring/fs/metrics/load.py
@@ -57,7 +57,8 @@ def open_metadata_ioctx(rados_inst, fs_map, filesystem):
                               f'failed to open metadata pool for {filesystem}')
 
 
-def load_sync_stat_metrics(ioctx, filesystem, peer_uuid=None):
+def load_sync_stat_metrics(ioctx, filesystem, peer_uuid=None, policy=None,
+                           live_instance_ids=None):
     metrics: Dict[str, Any] = {}
     prefix = f'{SYNC_STAT_KEY_PREFIX}/{filesystem}/'
     if peer_uuid:
@@ -89,7 +90,8 @@ def load_sync_stat_metrics(ioctx, filesystem, peer_uuid=None):
                         continue
                     format_peer_status_metrics(
                         metrics, dir_path, peer,
-                        format_and_order_sync_stat_for_display(stat))
+                        format_and_order_sync_stat_for_display(
+                            stat, policy, dir_path, live_instance_ids))
                 start = omap_vals.popitem()[0]
     except rados.Error as e:
         log.error(f'failed to read sync stat omap: {e}')
@@ -97,7 +99,8 @@ def load_sync_stat_metrics(ioctx, filesystem, peer_uuid=None):
     return metrics
 
 
-def fetch_sync_stat_metrics(ioctx, filesystem, peers, mirrored_dir_path, peer_uuid):
+def fetch_sync_stat_metrics(ioctx, filesystem, peers, mirrored_dir_path,
+                            peer_uuid, policy=None, live_instance_ids=None):
     if mirrored_dir_path:
         dir_path = norm_path(mirrored_dir_path)
         keys = [sync_stat_omap_key(filesystem, peer, dir_path) for peer in peers]
@@ -109,8 +112,10 @@ def fetch_sync_stat_metrics(ioctx, filesystem, peers, mirrored_dir_path, peer_uu
             if stat is not None:
                 format_peer_status_metrics(
                     metrics, dir_path, peer,
-                    format_and_order_sync_stat_for_display(stat))
+                    format_and_order_sync_stat_for_display(
+                        stat, policy, dir_path, live_instance_ids))
         return metrics, False, dir_path
 
-    metrics = load_sync_stat_metrics(ioctx, filesystem, peer_uuid)
+    metrics = load_sync_stat_metrics(
+        ioctx, filesystem, peer_uuid, policy, live_instance_ids)
     return metrics, True, None

diff --git a/src/pybind/mgr/mirroring/fs/snapshot_mirror.py b/src/pybind/mgr/mirroring/fs/snapshot_mirror.py
index 2b1fa76425e37ed3d826ccc04fe93f6fa3d84ecf..5c6a1230d31d1dda00346f7e261a492ac2737f64 100644 (file)
--- a/src/pybind/mgr/mirroring/fs/snapshot_mirror.py
+++ b/src/pybind/mgr/mirroring/fs/snapshot_mirror.py
@@ -66,6 +66,13 @@ class FSPolicy:
     def schedule_action(self, dir_paths):
         self.dir_paths.extend(dir_paths)
 
+    def get_live_instance_ids(self):
+        watcher = self.instance_watcher
+        if watcher is None:
+            return None
+        with watcher.lock:
+            return frozenset(str(instance_id) for instance_id in watcher.instances)
+
     def init(self, dir_mapping, instances):
         with self.lock:
             self.policy.init(dir_mapping)
@@ -774,9 +781,12 @@ class FSSnapshotMirror:
         """
         log.debug('sync stat metrics for filesystem %s loaded from omap (complete)',
                   filesystem)
+        fspolicy = self.pool_policy[filesystem]
         ioctx = metrics_load.open_metadata_ioctx(
             self.rados, self.fs_map, filesystem)
-        return metrics_load.load_sync_stat_metrics(ioctx, filesystem)
+        return metrics_load.load_sync_stat_metrics(
+            ioctx, filesystem, None, fspolicy.policy,
+            fspolicy.get_live_instance_ids())
 
     @lru_cache_timeout(
         lambda self, *_args, **_kwargs: CACHE_TTL_SECS,
@@ -793,10 +803,12 @@ class FSSnapshotMirror:
                   'loaded from omap',
                   filesystem, dir_path, peer_scope)
         peers = {peer_id: None for peer_id in peer_ids}
+        fspolicy = self.pool_policy[filesystem]
         ioctx = metrics_load.open_metadata_ioctx(
             self.rados, self.fs_map, filesystem)
         metrics, _, _ = metrics_load.fetch_sync_stat_metrics(
-            ioctx, filesystem, peers, dir_path, None)
+            ioctx, filesystem, peers, dir_path, None,
+            fspolicy.policy, fspolicy.get_live_instance_ids())
         return metrics
 
     def metrics_status(self, filesystem, mirrored_dir_path, peer_uuid):