tools/cephfs_mirror: Do remote fs sync once instead of fsync on each fd

Do remote fs sync once just before taking snapshot
as it's faster than doing fsync on each fd after
file copy.

Moreover, all the datasync threads use the same sinlge libceph
onnection and doing ceph_fsync concurrently on different fds on
a single libcephfs connection could cause hang as observed in
testing as below. This issue is tracked at
https://tracker.ceph.com/issues/75070

-----
Thread 2 (Thread 0xffff644cc400 (LWP 74020) "d_replayer-0"):
0  0x0000ffff8e82656c in __futex_abstimed_wait_cancelable64 () from /lib64/libc.so.6
1  0x0000ffff8e828ff0 [PAC] in pthread_cond_wait@@GLIBC_2.17 () from /lib64/libc.so.6
2  0x0000ffff8fc90fd4 [PAC] in ceph::condition_variable_debug::wait ...
3  0x0000ffff9080fc9c in ceph::condition_variable_debug::wait<Client::wait_on_context_list ...
4  Client::wait_on_context_list ... at /lsandbox/upstream/ceph/src/client/Client.cc:4540
5  0x0000ffff9083fae8 in Client::_fsync ... at /lsandbox/upstream/ceph/src/client/Client.cc:13299
6  0x0000ffff90840278 in Client::_fsync ...
7  0x0000ffff90840514 in Client::fsync ... at /lsandbox/upstream/ceph/src/client/Client.cc:13042
8  0x0000ffff907f06e0 in ceph_fsync ... at /lsandbox/upstream/ceph/src/libcephfs.cc:316
9  0x0000aaaaad5b2f88 in cephfs::mirror::PeerReplayer::copy_to_remote ...
----

Fixes: https://tracker.ceph.com/issues/73452
Signed-off-by: Kotresh HR <khiremat@redhat.com>

diff --git a/src/tools/cephfs_mirror/PeerReplayer.cc b/src/tools/cephfs_mirror/PeerReplayer.cc
index 2ba941319e7fac91f51ce140a4ba54419c529fe7..926fd8aa9928262540f4345f698152f0e600cc64 100644 (file)
--- a/src/tools/cephfs_mirror/PeerReplayer.cc
+++ b/src/tools/cephfs_mirror/PeerReplayer.cc
@@ -806,11 +806,6 @@ int PeerReplayer::copy_to_remote(const std::string &dir_root,  const std::string
            << cpp_strerror(r) << dendl;
       goto freeptr;
     }
-    r = ceph_fsync(m_remote_mount, r_fd, 0);
-    if (r < 0) {
-      derr << ": failed to sync data for file path=" << epath << ": "
-           << cpp_strerror(r) << dendl;
-    }
   }
 
 freeptr:
@@ -2007,7 +2002,14 @@ int PeerReplayer::do_synchronize(const std::string &dir_root, const Snapshot &cu
   bool datasync_err = syncm->wait_for_sync();
 
   if (r == 0 && !datasync_err) {
-    // All good, take the snapshot
+    // All good, fsync remote fs and take the snapshot
+    dout(20) << ": syncing remote filesystem, dir_root=" << dir_root << dendl;
+    r = ceph_sync_fs(m_remote_mount);
+    if (r < 0) {
+      derr << ": failed to sync remote filesystem, dir_root=" << dir_root
+           << ": " << cpp_strerror(r) << dendl;
+      return r;
+    }
     auto cur_snap_id_str{stringify(current.second)};
     snap_metadata snap_meta[] = {{PRIMARY_SNAP_ID_KEY.c_str(), cur_snap_id_str.c_str()}};
     r = ceph_mksnap(m_remote_mount, dir_root.c_str(), current.first.c_str(), 0755,
@@ -2015,6 +2017,7 @@ int PeerReplayer::do_synchronize(const std::string &dir_root, const Snapshot &cu
     if (r < 0) {
       derr << ": failed to snap remote directory dir_root=" << dir_root
            << ": " << cpp_strerror(r) << dendl;
+      return r;
     }
   } else if (datasync_err) {
     r = syncm->get_datasync_errno();