From 5a1447e1d8e4147d216d7336b6c49d0dcbc81254 Mon Sep 17 00:00:00 2001
From: "Yan, Zheng" <zyan@redhat.com>
Date: Fri, 1 Jun 2018 10:21:20 +0800
Subject: [PATCH] client: void sending mds request while holding cap reference

Client::_write() calls is_quota_bytes_approaching() while holding Fw
cap reference. The later function may send lookup_name request to mds.
This can cause deadlock (mds delays handling the lookup_name request
because subtree is freezing. The subtree stays in freezing state because
mds is revoking the Fw cap and client does not release it)

This patch isn't cherry-picked from master because quota implementation
has been changed in master (introduced quota realm).

Fixes: http://tracker.ceph.com/issues/24369
Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
---
 src/client/Client.cc | 50 ++++++++++++++++++++++++--------------------
 src/client/Client.h  |  5 +++--
 2 files changed, 30 insertions(+), 25 deletions(-)

diff --git a/src/client/Client.cc b/src/client/Client.cc
index 044c29f3a081..306413f50de1 100644
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -9186,8 +9186,9 @@ int Client::_write(Fh *f, int64_t offset, uint64_t size, const char *buf,
 
   // check quota
   uint64_t endoff = offset + size;
-  if (endoff > in->size && is_quota_bytes_exceeded(in, endoff - in->size,
-						   f->actor_perms)) {
+  std::list<InodeRef> quota_roots;
+  if (endoff > in->size &&
+      is_quota_bytes_exceeded(in, endoff - in->size, f->actor_perms, &quota_roots)) {
     return -EDQUOT;
   }
 
@@ -9364,7 +9365,7 @@ success:
     in->size = totalwritten + offset;
     mark_caps_dirty(in, CEPH_CAP_FILE_WR);
 
-    if (is_quota_bytes_approaching(in, f->actor_perms)) {
+    if (is_quota_bytes_approaching(in, quota_roots)) {
       check_caps(in, CHECK_CAPS_NODELAY);
     } else if (is_max_size_approaching(in)) {
       check_caps(in, 0);
@@ -12951,9 +12952,10 @@ int Client::_fallocate(Fh *fh, int mode, int64_t offset, int64_t length)
     return -EBADF;
 
   uint64_t size = offset + length;
+  std::list<InodeRef> quota_roots;
   if (!(mode & (FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE)) &&
       size > in->size &&
-      is_quota_bytes_exceeded(in, size - in->size, fh->actor_perms)) {
+      is_quota_bytes_exceeded(in, size - in->size, fh->actor_perms, &quota_roots)) {
     return -EDQUOT;
   }
 
@@ -13032,7 +13034,7 @@ int Client::_fallocate(Fh *fh, int mode, int64_t offset, int64_t length)
       in->change_attr++;
       mark_caps_dirty(in, CEPH_CAP_FILE_WR);
 
-      if (is_quota_bytes_approaching(in, fh->actor_perms)) {
+      if (is_quota_bytes_approaching(in, quota_roots)) {
         check_caps(in, CHECK_CAPS_NODELAY);
       } else if (is_max_size_approaching(in)) {
 	check_caps(in, 0);
@@ -13660,32 +13662,34 @@ bool Client::is_quota_files_exceeded(Inode *in, const UserPerm& perms)
 }
 
 bool Client::is_quota_bytes_exceeded(Inode *in, int64_t new_bytes,
-				     const UserPerm& perms)
+				     const UserPerm& perms,
+				     std::list<InodeRef>* quota_roots)
 {
   return check_quota_condition(in, perms,
-      [&new_bytes](const Inode &in) {
+      [&new_bytes, quota_roots](const Inode &in) {
+	if (quota_roots)
+	  quota_roots->emplace_back(const_cast<Inode*>(&in));
         return in.quota.max_bytes && (in.rstat.rbytes + new_bytes)
                > in.quota.max_bytes;
       });
 }
 
-bool Client::is_quota_bytes_approaching(Inode *in, const UserPerm& perms)
+bool Client::is_quota_bytes_approaching(Inode *in, std::list<InodeRef>& quota_roots)
 {
-  return check_quota_condition(in, perms,
-      [](const Inode &in) {
-        if (in.quota.max_bytes) {
-          if (in.rstat.rbytes >= in.quota.max_bytes) {
-            return true;
-          }
-
-          assert(in.size >= in.reported_size);
-          const uint64_t space = in.quota.max_bytes - in.rstat.rbytes;
-          const uint64_t size = in.size - in.reported_size;
-          return (space >> 4) < size;
-        } else {
-          return false;
-        }
-      });
+  assert(in->size >= in->reported_size);
+  const uint64_t size = in->size - in->reported_size;
+
+  for (auto& diri : quota_roots) {
+    if (diri->quota.max_bytes) {
+      if (diri->rstat.rbytes >= diri->quota.max_bytes)
+	return true;
+
+      uint64_t space = diri->quota.max_bytes - diri->rstat.rbytes;
+      if ((space >> 4) < size)
+	return true;
+    }
+  }
+  return false;
 }
 
 enum {
diff --git a/src/client/Client.h b/src/client/Client.h
index 6c131182b8a4..9a2efb976016 100644
--- a/src/client/Client.h
+++ b/src/client/Client.h
@@ -574,8 +574,9 @@ protected:
 			     std::function<bool (const Inode &)> test);
   bool is_quota_files_exceeded(Inode *in, const UserPerm& perms);
   bool is_quota_bytes_exceeded(Inode *in, int64_t new_bytes,
-			       const UserPerm& perms);
-  bool is_quota_bytes_approaching(Inode *in, const UserPerm& perms);
+			       const UserPerm& perms,
+			       std::list<InodeRef>* quota_roots=nullptr);
+  bool is_quota_bytes_approaching(Inode *in, std::list<InodeRef>& quota_roots);
 
   std::map<std::pair<int64_t,std::string>, int> pool_perms;
   list<Cond*> waiting_for_pool_perm;
-- 
2.47.3