client: adjust `Fb` cap ref count check during synchronous fsync()

cephfs client holds a ref on Fb caps when handing out a write delegation[0].
As fsync from (Ganesha) client holding write delegation will block indefinitely[1]
waiting for cap ref for Fb to drop to 0, which will never happen until the
delegation is returned/recalled.

[0]: https://github.com/ceph/ceph/blob/main/src/client/Delegation.cc#L71
[1]: https://github.com/ceph/ceph/blob/main/src/client/Client.cc#L12438

If an inode has been write delegated, adjust for cap reference count
check in fsync().

Note: This only workls for synchronous fsync() since `client_lock` is
held for the entire duration of the call (at least till the patch leading
upto the reference count check). Asynchronous fsync() needs to be fixed
separately (as that can drop `client_lock`).

Fixes: https://tracker.ceph.com/issues/73298
Signed-off-by: Venky Shankar <vshankar@redhat.com>

diff --git a/src/client/Client.cc b/src/client/Client.cc
index 4d802fa5d60290dc75382bac8abd411d87ffb2c9..899f513ca09ebb785ca54192a371be7fdc979a48 100644 (file)
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -12435,9 +12435,13 @@ int Client::_fsync(Inode *in, bool syncdataonly)
     ldout(cct, 15) << "got " << r << " from flush writeback" << dendl;
   } else {
     // FIXME: this can starve
-    while (in->cap_refs[CEPH_CAP_FILE_BUFFER] > 0) {
+    int nr_refs = 0;
+    if (in->is_write_delegated()) {
+      ++nr_refs;
+    }
+    while (in->cap_refs[CEPH_CAP_FILE_BUFFER] > nr_refs) {
       ldout(cct, 10) << "ino " << in->ino << " has " << in->cap_refs[CEPH_CAP_FILE_BUFFER]
-                    << " uncommitted, waiting" << dendl;
+                    << " uncommitted (nrefs: " << nr_refs << "), waiting" << dendl;
       wait_on_context_list(in->waitfor_commit);
     }
   }

diff --git a/src/client/Delegation.h b/src/client/Delegation.h
index d24a02487e135ccb9c0cfe488c2e13c574331961..85391cb372790419972e7ee02bc6c4aeed45c9aa 100644 (file)
--- a/src/client/Delegation.h
+++ b/src/client/Delegation.h
@@ -28,6 +28,9 @@ public:
   Fh *get_fh() { return fh; }
   unsigned get_type() { return type; }
   bool is_recalled() { return !recall_time.is_zero(); }
+  bool is_write_delegated() {
+    return type == CEPH_DELEGATION_WR;
+  }
 
   void reinit(unsigned _type, ceph_deleg_cb_t _recall_cb, void *_priv);
   void recall(bool skip_read);

diff --git a/src/client/Inode.cc b/src/client/Inode.cc
index 60932d606b522b24f404beaafd6c673b59cfe625..aabed0a3ad3dc2343766c1c4ae2b71dc423c5e0d 100644 (file)
--- a/src/client/Inode.cc
+++ b/src/client/Inode.cc
@@ -640,6 +640,21 @@ bool Inode::has_recalled_deleg()
   return deleg.is_recalled();
 }
 
+bool Inode::is_write_delegated()
+{
+  if (delegations.empty()) {
+    return false;
+  }
+
+  for (auto& deleg : delegations) {
+    if (deleg.is_write_delegated() && !deleg.is_recalled()) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
 void Inode::recall_deleg(bool skip_read)
 {
   if (delegations.empty())

diff --git a/src/client/Inode.h b/src/client/Inode.h
index 5a92df5bea0d5a07fd02dc4b86d1efdccca8123b..1e2e22a2828c74968a4594b7ffbd23cd0b8b012c 100644 (file)
--- a/src/client/Inode.h
+++ b/src/client/Inode.h
@@ -349,6 +349,7 @@ struct Inode : RefCountedObject {
 
   void recall_deleg(bool skip_read);
   bool has_recalled_deleg();
+  bool is_write_delegated();
   int set_deleg(Fh *fh, unsigned type, ceph_deleg_cb_t cb, void *priv);
   void unset_deleg(Fh *fh);