cephfs-mirror: display peer snapshot sync stats

Signed-off-by: Venky Shankar <vshankar@redhat.com>

diff --git a/src/tools/cephfs_mirror/PeerReplayer.cc b/src/tools/cephfs_mirror/PeerReplayer.cc
index 0d7aca765698fe0820937040769721c2dd089da4..51e1cc2729638a610ff85c6efc41d144154ac9e0 100644 (file)
--- a/src/tools/cephfs_mirror/PeerReplayer.cc
+++ b/src/tools/cephfs_mirror/PeerReplayer.cc
@@ -134,6 +134,9 @@ PeerReplayer::~PeerReplayer() {
 
 int PeerReplayer::init() {
   dout(20) << ": initial dir list=[" << m_directories << "]" << dendl;
+  for (auto &dir_path : m_directories) {
+    m_snap_sync_stats.emplace(dir_path, SnapSyncStat());
+  }
 
   auto &remote_client = m_peer.remote.client_name;
   auto &remote_cluster = m_peer.remote.cluster_name;
@@ -193,17 +196,22 @@ void PeerReplayer::add_directory(string_view dir_path) {
 
   std::scoped_lock locker(m_lock);
   m_directories.emplace_back(dir_path);
+  m_snap_sync_stats.emplace(dir_path, SnapSyncStat());
   m_cond.notify_all();
 }
 
 void PeerReplayer::remove_directory(string_view dir_path) {
   dout(20) << ": dir_path=" << dir_path << dendl;
+  auto _dir_path = std::string(dir_path);
 
   std::scoped_lock locker(m_lock);
-  auto it = std::find(m_directories.begin(), m_directories.end(), dir_path);
+  auto it = std::find(m_directories.begin(), m_directories.end(), _dir_path);
   if (it != m_directories.end()) {
     m_directories.erase(it);
   }
+  if (m_registered.find(_dir_path) == m_registered.end()) {
+    m_snap_sync_stats.erase(_dir_path);
+  }
   m_cond.notify_all();
 }
 
@@ -247,6 +255,9 @@ void PeerReplayer::unregister_directory(const std::string &dir_path) {
 
   unlock_directory(it->first, it->second);
   m_registered.erase(it);
+  if (std::find(m_directories.begin(), m_directories.end(), dir_path) == m_directories.end()) {
+    m_snap_sync_stats.erase(dir_path);
+  }
 }
 
 int PeerReplayer::try_lock_directory(const std::string &dir_path,
@@ -418,6 +429,7 @@ int PeerReplayer::propagate_snap_deletes(const std::string &dir_path,
            << ", snapshot=" << snaps << ": " << cpp_strerror(r) << dendl;
       return r;
     }
+    inc_deleted_snap(dir_path);
   }
 
   return 0;
@@ -440,6 +452,7 @@ int PeerReplayer::propagate_snap_renames(
            << cpp_strerror(r) << dendl;
       return r;
     }
+    inc_renamed_snap(dir_path);
   }
 
   return 0;
@@ -925,7 +938,9 @@ int PeerReplayer::do_sync_snaps(const std::string &dir_path) {
   // start mirroring snapshots from the last snap-id synchronized
   uint64_t last_snap_id = 0;
   if (!remote_snap_map.empty()) {
-    last_snap_id = remote_snap_map.rbegin()->first;
+    auto last = remote_snap_map.rbegin();
+    last_snap_id = last->first;
+    set_last_synced_snap(dir_path, last_snap_id, last->second);
   }
 
   dout(5) << ": last snap-id transferred=" << last_snap_id << dendl;
@@ -940,12 +955,17 @@ int PeerReplayer::do_sync_snaps(const std::string &dir_path) {
 
   dout(10) << ": synzhronizing from snap-id=" << it->first << dendl;
   for (; it != local_snap_map.end(); ++it) {
+    set_current_syncing_snap(dir_path, it->first, it->second);
+    auto start = clock::now();
     r = synchronize(dir_path, it->first, it->second);
     if (r < 0) {
       derr << ": failed to synchronize dir_path=" << dir_path
            << ", snapshot=" << it->second << dendl;
+      clear_current_syncing_snap(dir_path);
       return r;
     }
+    std::chrono::duration<double> duration = clock::now() - start;
+    set_last_synced_stat(dir_path, it->first, it->second, duration.count());
     if (--snaps_per_cycle == 0) {
       break;
     }
@@ -1010,6 +1030,35 @@ void PeerReplayer::run(SnapshotReplayerThread *replayer) {
 }
 
 void PeerReplayer::peer_status(Formatter *f) {
+  std::scoped_lock locker(m_lock);
+  f->open_object_section("stats");
+  for (auto &[dir_path, sync_stat] : m_snap_sync_stats) {
+    f->open_object_section(dir_path);
+    if (!sync_stat.current_syncing_snap) {
+      f->dump_string("state", "idle");
+    } else {
+      f->dump_string("state", "syncing");
+      f->open_object_section("current_sycning_snap");
+      f->dump_unsigned("id", (*sync_stat.current_syncing_snap).first);
+      f->dump_string("name", (*sync_stat.current_syncing_snap).second);
+      f->close_section();
+    }
+    if (sync_stat.last_synced_snap) {
+      f->open_object_section("last_synced_snap");
+      f->dump_unsigned("id", (*sync_stat.last_synced_snap).first);
+      f->dump_string("name", (*sync_stat.last_synced_snap).second);
+      if (sync_stat.last_sync_duration) {
+        f->dump_float("sync_duration", *sync_stat.last_sync_duration);
+        f->dump_stream("sync_time_stamp") << sync_stat.last_synced;
+      }
+      f->close_section();
+    }
+    f->dump_unsigned("snaps_synced", sync_stat.synced_snap_count);
+    f->dump_unsigned("snaps_deleted", sync_stat.deleted_snap_count);
+    f->dump_unsigned("snaps_renamed", sync_stat.renamed_snap_count);
+    f->close_section(); // dir_path
+  }
+  f->close_section(); // stats
 }
 
 } // namespace mirror

diff --git a/src/tools/cephfs_mirror/PeerReplayer.h b/src/tools/cephfs_mirror/PeerReplayer.h
index 69e5fe12560370ecb7d1e1e3d557c7b2c191a295..c46bb13f3a14bd2c350a1739ff7f11ac42622f23 100644 (file)
--- a/src/tools/cephfs_mirror/PeerReplayer.h
+++ b/src/tools/cephfs_mirror/PeerReplayer.h
@@ -91,6 +91,61 @@ private:
     }
   };
 
+  using clock = ceph::coarse_mono_clock;
+  using time = ceph::coarse_mono_time;
+
+  struct SnapSyncStat {
+    boost::optional<std::pair<uint64_t, std::string>> last_synced_snap;
+    boost::optional<std::pair<uint64_t, std::string>> current_syncing_snap;
+    uint64_t synced_snap_count = 0;
+    uint64_t deleted_snap_count = 0;
+    uint64_t renamed_snap_count = 0;
+    time last_synced = clock::zero();
+    boost::optional<double> last_sync_duration;
+  };
+
+  void _set_last_synced_snap(const std::string &dir_path, uint64_t snap_id,
+                            const std::string &snap_name) {
+    auto &sync_stat = m_snap_sync_stats.at(dir_path);
+    sync_stat.last_synced_snap = std::make_pair(snap_id, snap_name);
+    sync_stat.current_syncing_snap = boost::none;
+  }
+  void set_last_synced_snap(const std::string &dir_path, uint64_t snap_id,
+                            const std::string &snap_name) {
+    std::scoped_lock locker(m_lock);
+    _set_last_synced_snap(dir_path, snap_id, snap_name);
+  }
+  void set_current_syncing_snap(const std::string &dir_path, uint64_t snap_id,
+                                const std::string &snap_name) {
+    std::scoped_lock locker(m_lock);
+    auto &sync_stat = m_snap_sync_stats.at(dir_path);
+    sync_stat.current_syncing_snap = std::make_pair(snap_id, snap_name);
+  }
+  void clear_current_syncing_snap(const std::string &dir_path) {
+    std::scoped_lock locker(m_lock);
+    auto &sync_stat = m_snap_sync_stats.at(dir_path);
+    sync_stat.current_syncing_snap = boost::none;
+  }
+  void inc_deleted_snap(const std::string &dir_path) {
+    std::scoped_lock locker(m_lock);
+    auto &sync_stat = m_snap_sync_stats.at(dir_path);
+    ++sync_stat.deleted_snap_count;
+  }
+  void inc_renamed_snap(const std::string &dir_path) {
+    std::scoped_lock locker(m_lock);
+    auto &sync_stat = m_snap_sync_stats.at(dir_path);
+    ++sync_stat.renamed_snap_count;
+  }
+  void set_last_synced_stat(const std::string &dir_path, uint64_t snap_id,
+                            const std::string &snap_name, double duration) {
+    std::scoped_lock locker(m_lock);
+    _set_last_synced_snap(dir_path, snap_id, snap_name);
+    auto &sync_stat = m_snap_sync_stats.at(dir_path);
+    sync_stat.last_synced = clock::now();
+    sync_stat.last_sync_duration = duration;
+    ++sync_stat.synced_snap_count;
+  }
+
   typedef std::vector<std::unique_ptr<SnapshotReplayerThread>> SnapshotReplayers;
 
   CephContext *m_cct;
@@ -99,6 +154,7 @@ private:
   // probably need to be encapsulated when supporting cancelations
   std::map<std::string, DirRegistry> m_registered;
   std::vector<std::string> m_directories;
+  std::map<std::string, SnapSyncStat> m_snap_sync_stats;
   MountRef m_local_mount;
   PeerReplayerAdminSocketHook *m_asok_hook = nullptr;