tools/cephfs_mirror: Add read/write throughput

The read throughput added measures the bytes
read per second from the source ceph filesystem.
Similarly, the write throughput added measures
the bytes written per second to the remote ceph
filesystem. It's derived from the time spent
in preadv and pwritev calls.

Sample output:
-------------
{
    "/d0": {
        "state": "syncing",
        "current_syncing_snap": {
            "id": 2,
            "name": "d0_snap0",
            "sync-mode": "full",
            "avg_read_throughput_bytes": "12.69 MiB/s",
            "avg_write_throughput_bytes": "54.49 MiB/s",
            "crawl": {
                "state": "completed",
                "duration": "1s"
            },
            "bytes": {
                "sync_bytes": "149.94 MiB",
                "total_bytes": "149.94 MiB",
                "sync_percent": "100.00%"
            },
            "files": {
                "sync_files": 5000,
                "total_files": 5000,
                "sync_percent": "100.00%"
            }
        },
        "snaps_synced": 0,
        "snaps_deleted": 0,
        "snaps_renamed": 0
    }
}
-------------

Fixes: https://tracker.ceph.com/issues/73453
Signed-off-by: Kotresh HR <khiremat@redhat.com>

diff --git a/src/tools/cephfs_mirror/PeerReplayer.cc b/src/tools/cephfs_mirror/PeerReplayer.cc
index 57a6cd7b75d52dba41da872f50396d1eb3535b41..fecc82ea0870a7d2e9ed73d9b14d3a9aea13a71d 100644 (file)
--- a/src/tools/cephfs_mirror/PeerReplayer.cc
+++ b/src/tools/cephfs_mirror/PeerReplayer.cc
@@ -703,6 +703,10 @@ int PeerReplayer::copy_to_remote(const std::string &dir_root,  const std::string
   void *ptr;
   struct iovec iov[NR_IOVECS];
 
+  uint64_t bytes_read = 0;
+  uint64_t bytes_written = 0;
+  sec_duration read_time{0};
+  sec_duration write_time{0};
   int r = ceph_openat(m_local_mount, fh.c_fd, epath.c_str(), O_RDONLY | O_NOFOLLOW, 0);
   if (r < 0) {
     derr << ": failed to open local file path=" << epath << ": "
@@ -765,12 +769,16 @@ int PeerReplayer::copy_to_remote(const std::string &dir_root,  const std::string
         }
       }
 
+      auto rs = clock::now();
       r = ceph_preadv(m_local_mount, l_fd, iov, num_buffers, offset);
+      auto re = clock::now();
+      read_time += sec_duration(re - rs);
       if (r < 0) {
         derr << ": failed to read local file path=" << epath << ": "
              << cpp_strerror(r) << dendl;
         break;
       }
+      bytes_read += r;
       dout(10) << ": read: " << r << " bytes" << dendl;
       if (r == 0) {
         break;
@@ -784,12 +792,16 @@ int PeerReplayer::copy_to_remote(const std::string &dir_root,  const std::string
       }
 
       dout(10) << ": writing to offset: " << offset << dendl;
+      auto ws = clock::now();
       r = ceph_pwritev(m_remote_mount, r_fd, iov, iovs, offset);
+      auto we = clock::now();
+      write_time += sec_duration(we - ws);
       if (r < 0) {
         derr << ": failed to write remote file path=" << epath << ": "
              << cpp_strerror(r) << dendl;
         break;
       }
+      bytes_written += r;
 
       offset += r;
     }
@@ -798,6 +810,9 @@ int PeerReplayer::copy_to_remote(const std::string &dir_root,  const std::string
     ++b;
   }
 
+  //io accounting for metrics
+  add_io(dir_root, bytes_read, bytes_written, read_time.count(), write_time.count());
+
   if (num_blocks == 0 && r >= 0) { // handle blocklist case
     dout(20) << ": truncating epath=" << epath << " to " << stx.stx_size << " bytes"
              << dendl;
@@ -2617,6 +2632,15 @@ void PeerReplayer::peer_status(Formatter *f) {
         f->dump_string("sync-mode", "delta");
       else
         f->dump_string("sync-mode", "full");
+
+      //avg read/write throughput
+      double read_bps = sync_stat.read_time_sec > 0 ?
+          sync_stat.bytes_read / sync_stat.read_time_sec : 0;
+      double write_bps = sync_stat.write_time_sec > 0 ?
+          sync_stat.bytes_written / sync_stat.write_time_sec : 0;
+      f->dump_string("avg_read_throughput_bytes", format_bytes(read_bps) + "/s");
+      f->dump_string("avg_write_throughput_bytes", format_bytes(write_bps) + "/s");
+
       f->open_object_section("crawl");
       if (sync_stat.crawl_finished) {
         f->dump_string("state", "completed");

diff --git a/src/tools/cephfs_mirror/PeerReplayer.h b/src/tools/cephfs_mirror/PeerReplayer.h
index 34786c08f33c37efbe9f91ba669a4bc7a27b7add..c20da7962e071fbe349b3597ee635772b371a253 100644 (file)
--- a/src/tools/cephfs_mirror/PeerReplayer.h
+++ b/src/tools/cephfs_mirror/PeerReplayer.h
@@ -395,6 +395,11 @@ private:
     bool crawl_finished = false; // crawl_state - in-progress/completed
     clock::time_point crawl_start_time; // to show current crawl duration if crawl is in progress
     double crawl_duration = 0.0; // time taken to complete the crawl, includes a few entry operation like mkdir as well
+    // actual io accounting
+    uint64_t bytes_read = 0; //actual bytes read counter, independently for each directory sync.
+    uint64_t bytes_written = 0; //actual bytes written counter, independently for each directory sync.
+    double read_time_sec = 0.0; //actual read time in seconds counter, independently for each directroy sync.
+    double write_time_sec = 0.0; //actual write time in seconds counter, independently for each directroy sync.
   };
 
   void _inc_failed_count(const std::string &dir_root) {
@@ -434,6 +439,10 @@ private:
     sync_stat.crawl_finished = false;
     sync_stat.crawl_start_time = clock::now();
     sync_stat.crawl_duration = 0.0;
+    sync_stat.bytes_read = 0;
+    sync_stat.bytes_written = 0;
+    sync_stat.read_time_sec = 0.0;
+    sync_stat.write_time_sec = 0.0;
   }
   void _set_last_synced_snap(const std::string &dir_root, uint64_t snap_id,
                             const std::string &snap_name) {
@@ -492,6 +501,15 @@ private:
     sync_stat.crawl_finished = state;
     sync_stat.crawl_duration = seconds;
   }
+  void add_io(const std::string &dir_root, const uint64_t& br, const uint64_t bw,
+              const double rt, const double wt) {
+    std::scoped_lock locker(m_lock);
+    auto &sync_stat = m_snap_sync_stats.at(dir_root);
+    sync_stat.bytes_read += br;
+    sync_stat.bytes_written += bw;
+    sync_stat.read_time_sec += rt;
+    sync_stat.write_time_sec += wt;
+  }
   void inc_sync_bytes(const std::string &dir_root, const uint64_t& b) {
     std::scoped_lock locker(m_lock);
     auto &sync_stat = m_snap_sync_stats.at(dir_root);