tools/cephfs_mirror: Add inprogress bytes and files metric

Add following mirroring progress metrics to current_syncing_snap
as below

bytes:
  sync_bytes - bytes synced till now
  total_bytes - total bytes to be synced
  sync_percent - Percentage of bytes synced till now
files:
  total_files - Total files to be synced
  sync_files - files synced till now
  sync_percent - Percentage of files synced till now

sync_files and sync_bytes are also stored in last_synced_snap section
after the snapshot is synced.

The bytes is formatted as below.

Sample output:
--------
{
    "/d0": {
        "state": "syncing",
        "current_syncing_snap": {
            "id": 3,
            "name": "d0_snap1",
            "bytes": {
                "sync_bytes": "120.20 MiB",
                "total_bytes": "149.94 MiB",
                "sync_percent": "80.16%"
            },
            "files": {
                "sync_files": 4032,
                "total_files": 5000,
                "sync_percent": "80.64%"
            }
        },
        "last_synced_snap": {
            "id": 2,
            "name": "d0_snap0",
            "sync_duration": 45,
            "sync_time_stamp": "5642.805770s",
            "sync_bytes": "300.85 MiB",
            "sync_files": 10000
        },
        "snaps_synced": 1,
        "snaps_deleted": 0,
        "snaps_renamed": 0
    }
}

Fixes: https://tracker.ceph.com/issues/73453
Signed-off-by: Kotresh HR <khiremat@redhat.com>

diff --git a/src/tools/cephfs_mirror/PeerReplayer.cc b/src/tools/cephfs_mirror/PeerReplayer.cc
index d584b419790c2c21d467845621de57ec429ef5d2..ebadde544f873c011104d7d39549065d917ce59b 100644 (file)
--- a/src/tools/cephfs_mirror/PeerReplayer.cc
+++ b/src/tools/cephfs_mirror/PeerReplayer.cc
@@ -910,6 +910,7 @@ int PeerReplayer::remote_file_op(std::shared_ptr<SyncMechanism>& syncm, const st
     }
   }
 
+  inc_sync_files(dir_root);
   return 0;
 }
 
@@ -1330,6 +1331,7 @@ PeerReplayer::SyncMechanism::~SyncMechanism() {
 void PeerReplayer::SyncMechanism::push_dataq_entry(SyncEntry e) {
   dout(10) << ": snapshot data replayer dataq pushed" << " syncm=" << this
           << " epath=" << e.epath << dendl;
+  m_peer_replayer.inc_total_bytes_files(std::string(m_dir_root), e.stx.stx_size);
   std::unique_lock lock(sdq_lock);
   m_sync_dataq.push(std::move(e));
   sdq_cv.notify_all();
@@ -2522,6 +2524,32 @@ void PeerReplayer::run_datasync(SnapshotDataSyncThread *data_replayer) {
   } // outer while
 }
 
+std::string PeerReplayer::format_bytes(double bytes) {
+  static constexpr double KiB = 1024.0;
+  static constexpr double MiB = KiB * 1024.0;
+  static constexpr double GiB = MiB * 1024.0;
+  static constexpr double TiB = GiB * 1024.0;
+  static constexpr double PiB = TiB * 1024.0;
+
+  std::ostringstream out;
+  out << std::fixed << std::setprecision(2);
+
+  if (bytes >= PiB) {
+    out << (bytes / PiB) << " PiB";
+  } else if (bytes >= TiB) {
+    out << (bytes / TiB) << " TiB";
+  } else if (bytes >= GiB) {
+    out << (bytes / GiB) << " GiB";
+  } else if (bytes >= MiB) {
+    out << (bytes / MiB) << " MiB";
+  } else if (bytes >= KiB) {
+    out << (bytes / KiB) << " KiB";
+  } else {
+    out << bytes << " B";
+  }
+  return out.str();
+};
+
 void PeerReplayer::peer_status(Formatter *f) {
   std::scoped_lock locker(m_lock);
   f->open_object_section("stats");
@@ -2539,7 +2567,27 @@ void PeerReplayer::peer_status(Formatter *f) {
       f->open_object_section("current_syncing_snap");
       f->dump_unsigned("id", (*sync_stat.current_syncing_snap).first);
       f->dump_string("name", (*sync_stat.current_syncing_snap).second);
-      f->close_section();
+      f->open_object_section("bytes");
+      f->dump_string("sync_bytes", format_bytes(sync_stat.sync_bytes));
+      f->dump_string("total_bytes", format_bytes(sync_stat.total_bytes));
+      if (sync_stat.total_bytes > 0) {
+        double sync_pct = (static_cast<double>(sync_stat.sync_bytes) * 100.0) / sync_stat.total_bytes;
+        std::ostringstream os;
+        os << std::fixed << std::setprecision(2) << sync_pct << "%";
+        f->dump_string("sync_percent", os.str());
+      }
+      f->close_section(); //bytes
+      f->open_object_section("files");
+      f->dump_unsigned("sync_files", sync_stat.sync_files);
+      f->dump_unsigned("total_files", sync_stat.total_files);
+      if (sync_stat.total_files > 0) {
+        double sync_file_pct = (static_cast<double>(sync_stat.sync_files) * 100.0) / sync_stat.total_files;
+        std::ostringstream os;
+        os << std::fixed << std::setprecision(2) << sync_file_pct << "%";
+        f->dump_string("sync_percent", os.str());
+      }
+      f->close_section(); //files
+      f->close_section(); //current_syncing_snap
     }
     if (sync_stat.last_synced_snap) {
       f->open_object_section("last_synced_snap");
@@ -2550,7 +2598,10 @@ void PeerReplayer::peer_status(Formatter *f) {
         f->dump_stream("sync_time_stamp") << sync_stat.last_synced;
       }
       if (sync_stat.last_sync_bytes) {
-       f->dump_unsigned("sync_bytes", *sync_stat.last_sync_bytes);
+           f->dump_string("sync_bytes", format_bytes(*sync_stat.last_sync_bytes));
+      }
+      if (sync_stat.last_sync_files) {
+           f->dump_unsigned("sync_files", *sync_stat.last_sync_files);
       }
       f->close_section();
     }

diff --git a/src/tools/cephfs_mirror/PeerReplayer.h b/src/tools/cephfs_mirror/PeerReplayer.h
index 3609c9926017112f63d47eb6f845c98d42cb044c..bb5e467da53c90afcdbbf69611d3f5e7b135a684 100644 (file)
--- a/src/tools/cephfs_mirror/PeerReplayer.h
+++ b/src/tools/cephfs_mirror/PeerReplayer.h
@@ -385,7 +385,11 @@ private:
     monotime last_synced = clock::zero();
     boost::optional<double> last_sync_duration;
     boost::optional<uint64_t> last_sync_bytes; //last sync bytes for display in status
+    boost::optional<uint64_t> last_sync_files; //last num of sync files for display in status
     uint64_t sync_bytes = 0; //sync bytes counter, independently for each directory sync.
+    uint64_t total_bytes = 0; //total bytes counter, independently for each directory sync.
+    uint64_t sync_files = 0; //sync files counter, independently for each directory sync.
+    uint64_t total_files = 0; //total files counter, independently for each directory sync.
   };
 
   void _inc_failed_count(const std::string &dir_root) {
@@ -415,6 +419,13 @@ private:
     sync_stat.last_failed_reason = boost::none;
   }
 
+  void _reset_sync_stat(const std::string &dir_root) {
+    auto &sync_stat = m_snap_sync_stats.at(dir_root);
+    sync_stat.sync_bytes = 0;
+    sync_stat.total_bytes = 0;
+    sync_stat.sync_files = 0;
+    sync_stat.total_files = 0;
+  }
   void _set_last_synced_snap(const std::string &dir_root, uint64_t snap_id,
                             const std::string &snap_name) {
     auto &sync_stat = m_snap_sync_stats.at(dir_root);
@@ -425,14 +436,13 @@ private:
                             const std::string &snap_name) {
     std::scoped_lock locker(m_lock);
     _set_last_synced_snap(dir_root, snap_id, snap_name);
-    auto &sync_stat = m_snap_sync_stats.at(dir_root);
-    sync_stat.sync_bytes = 0;
   }
   void set_current_syncing_snap(const std::string &dir_root, uint64_t snap_id,
                                 const std::string &snap_name) {
     std::scoped_lock locker(m_lock);
     auto &sync_stat = m_snap_sync_stats.at(dir_root);
     sync_stat.current_syncing_snap = std::make_pair(snap_id, snap_name);
+    _reset_sync_stat(dir_root); //reset counters at the start of every snap sync
   }
   void clear_current_syncing_snap(const std::string &dir_root) {
     std::scoped_lock locker(m_lock);
@@ -457,13 +467,26 @@ private:
     sync_stat.last_synced = clock::now();
     sync_stat.last_sync_duration = duration;
     sync_stat.last_sync_bytes = sync_stat.sync_bytes;
+    sync_stat.last_sync_files = sync_stat.sync_files;
     ++sync_stat.synced_snap_count;
+    _reset_sync_stat(dir_root);
   }
   void inc_sync_bytes(const std::string &dir_root, const uint64_t& b) {
     std::scoped_lock locker(m_lock);
     auto &sync_stat = m_snap_sync_stats.at(dir_root);
     sync_stat.sync_bytes += b;
   }
+  void inc_sync_files(const std::string &dir_root) {
+    std::scoped_lock locker(m_lock);
+    auto &sync_stat = m_snap_sync_stats.at(dir_root);
+    sync_stat.sync_files++;
+  }
+  void inc_total_bytes_files(const std::string &dir_root, const uint64_t& b) {
+    std::scoped_lock locker(m_lock);
+    auto &sync_stat = m_snap_sync_stats.at(dir_root);
+    sync_stat.total_bytes += b;
+    sync_stat.total_files++;
+  }
   bool should_backoff(const std::string &dir_root, int *retval) {
     if (m_fs_mirror->is_blocklisted()) {
       *retval = -EBLOCKLISTED;
@@ -608,6 +631,9 @@ private:
   uint64_t set_datasync_files_per_batch(uint64_t value) {
     return datasync_files_per_batch.exchange(value, std::memory_order_relaxed);
   }
+
+  // format routines for peer_status
+  static std::string format_bytes(double bytes);
 };
 
 } // namespace mirror