client: void sending mds request while holding cap reference

Client::_write() calls is_quota_bytes_approaching() while holding Fw
cap reference. The later function may send lookup_name request to mds.
This can cause deadlock (mds delays handling the lookup_name request
because subtree is freezing. The subtree stays in freezing state because
mds is revoking the Fw cap and client does not release it)

This patch isn't cherry-picked from master because quota implementation
has been changed in master (introduced quota realm).

Fixes: http://tracker.ceph.com/issues/24369
Signed-off-by: "Yan, Zheng" <zyan@redhat.com>

diff --git a/src/client/Client.cc b/src/client/Client.cc
index 044c29f3a08156818acfde4bd481b4a68903d109..306413f50de1c6a4d6b404b8975d59caac01d6f8 100644 (file)
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -9186,8 +9186,9 @@ int Client::_write(Fh *f, int64_t offset, uint64_t size, const char *buf,
 
   // check quota
   uint64_t endoff = offset + size;
-  if (endoff > in->size && is_quota_bytes_exceeded(in, endoff - in->size,
-                                                  f->actor_perms)) {
+  std::list<InodeRef> quota_roots;
+  if (endoff > in->size &&
+      is_quota_bytes_exceeded(in, endoff - in->size, f->actor_perms, &quota_roots)) {
     return -EDQUOT;
   }
 
@@ -9364,7 +9365,7 @@ success:
     in->size = totalwritten + offset;
     mark_caps_dirty(in, CEPH_CAP_FILE_WR);
 
-    if (is_quota_bytes_approaching(in, f->actor_perms)) {
+    if (is_quota_bytes_approaching(in, quota_roots)) {
       check_caps(in, CHECK_CAPS_NODELAY);
     } else if (is_max_size_approaching(in)) {
       check_caps(in, 0);
@@ -12951,9 +12952,10 @@ int Client::_fallocate(Fh *fh, int mode, int64_t offset, int64_t length)
     return -EBADF;
 
   uint64_t size = offset + length;
+  std::list<InodeRef> quota_roots;
   if (!(mode & (FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE)) &&
       size > in->size &&
-      is_quota_bytes_exceeded(in, size - in->size, fh->actor_perms)) {
+      is_quota_bytes_exceeded(in, size - in->size, fh->actor_perms, &quota_roots)) {
     return -EDQUOT;
   }
 
@@ -13032,7 +13034,7 @@ int Client::_fallocate(Fh *fh, int mode, int64_t offset, int64_t length)
       in->change_attr++;
       mark_caps_dirty(in, CEPH_CAP_FILE_WR);
 
-      if (is_quota_bytes_approaching(in, fh->actor_perms)) {
+      if (is_quota_bytes_approaching(in, quota_roots)) {
         check_caps(in, CHECK_CAPS_NODELAY);
       } else if (is_max_size_approaching(in)) {
        check_caps(in, 0);
@@ -13660,32 +13662,34 @@ bool Client::is_quota_files_exceeded(Inode *in, const UserPerm& perms)
 }
 
 bool Client::is_quota_bytes_exceeded(Inode *in, int64_t new_bytes,
-                                    const UserPerm& perms)
+                                    const UserPerm& perms,
+                                    std::list<InodeRef>* quota_roots)
 {
   return check_quota_condition(in, perms,
-      [&new_bytes](const Inode &in) {
+      [&new_bytes, quota_roots](const Inode &in) {
+       if (quota_roots)
+         quota_roots->emplace_back(const_cast<Inode*>(&in));
         return in.quota.max_bytes && (in.rstat.rbytes + new_bytes)
                > in.quota.max_bytes;
       });
 }
 
-bool Client::is_quota_bytes_approaching(Inode *in, const UserPerm& perms)
+bool Client::is_quota_bytes_approaching(Inode *in, std::list<InodeRef>& quota_roots)
 {
-  return check_quota_condition(in, perms,
-      [](const Inode &in) {
-        if (in.quota.max_bytes) {
-          if (in.rstat.rbytes >= in.quota.max_bytes) {
-            return true;
-          }
-
-          assert(in.size >= in.reported_size);
-          const uint64_t space = in.quota.max_bytes - in.rstat.rbytes;
-          const uint64_t size = in.size - in.reported_size;
-          return (space >> 4) < size;
-        } else {
-          return false;
-        }
-      });
+  assert(in->size >= in->reported_size);
+  const uint64_t size = in->size - in->reported_size;
+
+  for (auto& diri : quota_roots) {
+    if (diri->quota.max_bytes) {
+      if (diri->rstat.rbytes >= diri->quota.max_bytes)
+       return true;
+
+      uint64_t space = diri->quota.max_bytes - diri->rstat.rbytes;
+      if ((space >> 4) < size)
+       return true;
+    }
+  }
+  return false;
 }
 
 enum {

diff --git a/src/client/Client.h b/src/client/Client.h
index 6c131182b8a4c698d277b194d8c7dfd30e07fd22..9a2efb976016f307ad3671a8fcc4002d00007cd4 100644 (file)
--- a/src/client/Client.h
+++ b/src/client/Client.h
@@ -574,8 +574,9 @@ protected:
                             std::function<bool (const Inode &)> test);
   bool is_quota_files_exceeded(Inode *in, const UserPerm& perms);
   bool is_quota_bytes_exceeded(Inode *in, int64_t new_bytes,
-                              const UserPerm& perms);
-  bool is_quota_bytes_approaching(Inode *in, const UserPerm& perms);
+                              const UserPerm& perms,
+                              std::list<InodeRef>* quota_roots=nullptr);
+  bool is_quota_bytes_approaching(Inode *in, std::list<InodeRef>& quota_roots);
 
   std::map<std::pair<int64_t,std::string>, int> pool_perms;
   list<Cond*> waiting_for_pool_perm;