cephfs_mirror: update per-directory last sync and summary perf counters

Wire the remaining cephfs_mirror_directory labeled counters that were
registered in the prior commit but not yet refreshed. Live
current_syncing_snap gauges continue to be updated from the per-peer tick
thread; this commit updates last_synced_snap and per-directory snap
summary counters at the points where SnapSyncStat is actually modified.

Depends-on the cephfs_mirror_directory PerfCounters schema (dir_state,
current_*, last_*, snaps_*) added when each mirrored directory is
registered.

Prometheus / counter dump
-------------------------

These counters appear under "cephfs_mirror_directory" in "counter dump" with
the same labels as current-sync metrics (source_fscid, source_filesystem,
peer_uuid, peer_cluster_name, peer_cluster_filesystem, directory). ceph-exporter
exposes them as e.g. ceph_cephfs_mirror_directory_last_sync_bytes and
ceph_cephfs_mirror_directory_snaps_synced.

Unlike cephfs_mirror_peers, values are per (peer_uuid, directory) rather than
aggregated across all directories on the peer. Peer-level counters are
unchanged.

Fixes: https://tracker.ceph.com/issues/73457
Signed-off-by: Kotresh HR <khiremat@redhat.com>

diff --git a/src/tools/cephfs_mirror/PeerReplayer.cc b/src/tools/cephfs_mirror/PeerReplayer.cc
index d532b6a3adfca4833c9699273d7f4761e7213dde..6b370df3c5cd346bb1a83d17743526d4daf01aa9 100644 (file)
--- a/src/tools/cephfs_mirror/PeerReplayer.cc
+++ b/src/tools/cephfs_mirror/PeerReplayer.cc
@@ -273,6 +273,10 @@ uint64_t percent_basis_points(uint64_t num, uint64_t den) {
   return static_cast<uint64_t>((static_cast<double>(num) * 10000.0) / den);
 }
 
+double monotime_to_double(monotime t) {
+  return sec_duration(t.time_since_epoch()).count();
+}
+
 void PeerReplayer::create_directory_perf_counters(const std::string &dir_root) {
   ceph_assert(m_directory_perf_counters.find(dir_root) ==
               m_directory_perf_counters.end());
@@ -475,6 +479,54 @@ void PeerReplayer::update_directory_current_sync_perf_counters(
   }
 }
 
+void PeerReplayer::update_directory_last_sync_perf_counters(
+    PerfCounters *perf, const SnapSyncStat &sync_stat) {
+  if (!perf) {
+    return;
+  }
+
+  if (sync_stat.last_synced_snap) {
+    perf->set(l_cephfs_mirror_directory_last_snap_id,
+              sync_stat.last_synced_snap->first);
+  } else {
+    perf->set(l_cephfs_mirror_directory_last_snap_id, 0);
+  }
+
+  perf->set(l_cephfs_mirror_directory_last_crawl_duration_seconds,
+            sync_stat.last_sync_crawl_duration ?
+              static_cast<uint64_t>(*sync_stat.last_sync_crawl_duration) : 0);
+  perf->set(l_cephfs_mirror_directory_last_datasync_wait_duration_seconds,
+            sync_stat.last_sync_datasync_queue_wait_duration ?
+              static_cast<uint64_t>(*sync_stat.last_sync_datasync_queue_wait_duration) : 0);
+  perf->set(l_cephfs_mirror_directory_last_sync_duration_seconds,
+            sync_stat.last_sync_duration ?
+              static_cast<uint64_t>(*sync_stat.last_sync_duration) : 0);
+
+  utime_t t;
+  if (!clock::is_zero(sync_stat.last_synced)) {
+    t.set_from_double(monotime_to_double(sync_stat.last_synced));
+  } else {
+    t = utime_t();
+  }
+  perf->tset(l_cephfs_mirror_directory_last_sync_timestamp, t);
+
+  perf->set(l_cephfs_mirror_directory_last_sync_bytes,
+            sync_stat.last_sync_bytes ? *sync_stat.last_sync_bytes : 0);
+  perf->set(l_cephfs_mirror_directory_last_sync_files,
+            sync_stat.last_sync_files ? *sync_stat.last_sync_files : 0);
+}
+
+void PeerReplayer::update_directory_summary_perf_counters(
+    PerfCounters *perf, const SnapSyncStat &sync_stat) {
+  if (!perf) {
+    return;
+  }
+
+  perf->set(l_cephfs_mirror_directory_snaps_synced, sync_stat.synced_snap_count);
+  perf->set(l_cephfs_mirror_directory_snaps_deleted, sync_stat.deleted_snap_count);
+  perf->set(l_cephfs_mirror_directory_snaps_renamed, sync_stat.renamed_snap_count);
+}
+
 void PeerReplayer::refresh_directory_current_sync_perf_counters(
     const std::string &dir_root) {
   // caller must hold m_lock
@@ -493,6 +545,11 @@ int PeerReplayer::init() {
   }
   for (auto &dir_root : m_directories) {
     create_directory_perf_counters(dir_root);
+    auto *dir_perf = find_directory_perf_counters(dir_root);
+    update_directory_last_sync_perf_counters(dir_perf,
+                                             m_snap_sync_stats.at(dir_root));
+    update_directory_summary_perf_counters(dir_perf,
+                                           m_snap_sync_stats.at(dir_root));
   }
 
   auto &remote_client = m_peer.remote.client_name;
@@ -642,6 +699,11 @@ void PeerReplayer::add_directory(string_view dir_root) {
       m_directory_perf_counters.end()) {
     create_directory_perf_counters(_dir_root);
   }
+  auto *dir_perf = find_directory_perf_counters(_dir_root);
+  update_directory_last_sync_perf_counters(dir_perf,
+                                           m_snap_sync_stats.at(_dir_root));
+  update_directory_summary_perf_counters(dir_perf,
+                                         m_snap_sync_stats.at(_dir_root));
   m_cond.notify_all();
 }
 
@@ -2610,6 +2672,10 @@ int PeerReplayer::sync_snaps(const std::string &dir_root,
   } else {
     _reset_failed_count(dir_root);
   }
+  if (auto *dir_perf = find_directory_perf_counters(dir_root)) {
+    update_directory_current_sync_perf_counters(dir_perf,
+                                                m_snap_sync_stats.at(dir_root));
+  }
   return r;
 }
 
@@ -2675,6 +2741,10 @@ void PeerReplayer::run(SnapshotReplayerThread *replayer) {
             }
           } else {
             _inc_failed_count(*dir_root);
+            if (auto *dir_perf = find_directory_perf_counters(*dir_root)) {
+              update_directory_current_sync_perf_counters(
+                dir_perf, m_snap_sync_stats.at(*dir_root));
+            }
             if (m_perf_counters) {
               m_perf_counters->inc(l_cephfs_mirror_peer_replayer_snap_sync_failures);
             }

diff --git a/src/tools/cephfs_mirror/PeerReplayer.h b/src/tools/cephfs_mirror/PeerReplayer.h
index 38e54516bdd55a18463464c56496c28877561ba1..8d8d9f42119436980868c52312cb417a9e10f568 100644 (file)
--- a/src/tools/cephfs_mirror/PeerReplayer.h
+++ b/src/tools/cephfs_mirror/PeerReplayer.h
@@ -488,6 +488,10 @@ private:
                             const std::string &snap_name) {
     std::scoped_lock locker(m_lock);
     _set_last_synced_snap(dir_root, snap_id, snap_name);
+    if (auto *dir_perf = find_directory_perf_counters(dir_root)) {
+      update_directory_last_sync_perf_counters(dir_perf,
+                                               m_snap_sync_stats.at(dir_root));
+    }
   }
   void set_current_syncing_snap(const std::string &dir_root, uint64_t snap_id,
                                 const std::string &snap_name) {
@@ -505,11 +509,17 @@ private:
     std::scoped_lock locker(m_lock);
     auto &sync_stat = m_snap_sync_stats.at(dir_root);
     ++sync_stat.deleted_snap_count;
+    if (auto *dir_perf = find_directory_perf_counters(dir_root)) {
+      update_directory_summary_perf_counters(dir_perf, sync_stat);
+    }
   }
   void inc_renamed_snap(const std::string &dir_root) {
     std::scoped_lock locker(m_lock);
     auto &sync_stat = m_snap_sync_stats.at(dir_root);
     ++sync_stat.renamed_snap_count;
+    if (auto *dir_perf = find_directory_perf_counters(dir_root)) {
+      update_directory_summary_perf_counters(dir_perf, sync_stat);
+    }
   }
   void set_last_synced_stat(const std::string &dir_root, uint64_t snap_id,
                             const std::string &snap_name, double duration) {
@@ -526,6 +536,10 @@ private:
     sync_stat.last_sync_files = sync_stat.sync_files;
     ++sync_stat.synced_snap_count;
     _reset_sync_stat(dir_root);
+    if (auto *dir_perf = find_directory_perf_counters(dir_root)) {
+      update_directory_last_sync_perf_counters(dir_perf, sync_stat);
+      update_directory_summary_perf_counters(dir_perf, sync_stat);
+    }
   }
   void set_snapdiff(const std::string &dir_root, bool snapdiff) {
     std::scoped_lock locker(m_lock);
@@ -672,6 +686,10 @@ private:
   PerfCounters *find_directory_perf_counters(const std::string &dir_root);
   void update_directory_current_sync_perf_counters(PerfCounters *perf,
                                                    const SnapSyncStat &sync_stat);
+  void update_directory_last_sync_perf_counters(PerfCounters *perf,
+                                                const SnapSyncStat &sync_stat);
+  void update_directory_summary_perf_counters(PerfCounters *perf,
+                                              const SnapSyncStat &sync_stat);
   void refresh_directory_current_sync_perf_counters(const std::string &dir_root);
 
   void run(SnapshotReplayerThread *replayer);