mds: flush mdlog if locked and still has wanted caps not satisfied

In _do_cap_update() if one client is releasing the Fw caps the
relevant client range will be erased, and then new_max will be 0.
It will skip flushing the mdlog after it submitting a journal log,
which will keep holding the wrlock for the filelock.

So when a new client is trying to open the file for reading, since
the wrlock is locked for the filelock the file_eval() is possibly
couldn't changing the lock state and at the same time if the
filelock is in stable state, such as in EXECL, MIX. The mds may
skip flushing the mdlog in the open related code too.

We need to flush the mdlog if there still has any wanted caps
couldn't be satisfied and has any lock for the filelock after the
file_eval().

Fixes: https://tracker.ceph.com/issues/54463
Signed-off-by: Xiubo Li <xiubli@redhat.com>
(cherry picked from commit 52918175f0190581ae46897f19be6e870f24291c)

diff --git a/src/mds/Locker.cc b/src/mds/Locker.cc
index 1d66e6dfeab215abb49380bec2bf416bba52a417..58b3723dc0b2b394a2807b6a0d344205df3eec50 100644 (file)
--- a/src/mds/Locker.cc
+++ b/src/mds/Locker.cc
@@ -2227,7 +2227,11 @@ Capability* Locker::issue_new_caps(CInode *in,
     // [auth] twiddle mode?
     eval(in, CEPH_CAP_LOCKS);
 
-    if (_need_flush_mdlog(in, my_want, true))
+    int all_allowed = -1, loner_allowed = -1, xlocker_allowed = -1;
+    int allowed = get_allowed_caps(in, cap, all_allowed, loner_allowed,
+                                   xlocker_allowed);
+
+    if (_need_flush_mdlog(in, my_want & ~allowed, true))
       mds->mdlog->flush();
 
   } else {
@@ -2266,30 +2270,64 @@ public:
   }
 };
 
-int Locker::issue_caps(CInode *in, Capability *only_cap)
+int Locker::get_allowed_caps(CInode *in, Capability *cap,
+                             int &all_allowed, int &loner_allowed,
+                             int &xlocker_allowed)
 {
+  client_t client = cap->get_client();
+
   // allowed caps are determined by the lock mode.
-  int all_allowed = in->get_caps_allowed_by_type(CAP_ANY);
-  int loner_allowed = in->get_caps_allowed_by_type(CAP_LONER);
-  int xlocker_allowed = in->get_caps_allowed_by_type(CAP_XLOCKER);
+  if (all_allowed == -1)
+    all_allowed = in->get_caps_allowed_by_type(CAP_ANY);
+  if (loner_allowed == -1)
+    loner_allowed = in->get_caps_allowed_by_type(CAP_LONER);
+  if (xlocker_allowed == -1)
+    xlocker_allowed = in->get_caps_allowed_by_type(CAP_XLOCKER);
 
   client_t loner = in->get_loner();
   if (loner >= 0) {
-    dout(7) << "issue_caps loner client." << loner
+    dout(7) << "get_allowed_caps loner client." << loner
            << " allowed=" << ccap_string(loner_allowed) 
            << ", xlocker allowed=" << ccap_string(xlocker_allowed)
            << ", others allowed=" << ccap_string(all_allowed)
            << " on " << *in << dendl;
   } else {
-    dout(7) << "issue_caps allowed=" << ccap_string(all_allowed) 
+    dout(7) << "get_allowed_caps allowed=" << ccap_string(all_allowed) 
            << ", xlocker allowed=" << ccap_string(xlocker_allowed)
            << " on " << *in << dendl;
   }
 
-  ceph_assert(in->is_head());
+  // do not issue _new_ bits when size|mtime is projected
+  int allowed;
+  if (loner == client)
+    allowed = loner_allowed;
+  else
+    allowed = all_allowed;
+
+  // add in any xlocker-only caps (for locks this client is the xlocker for)
+  allowed |= xlocker_allowed & in->get_xlocker_mask(client);
+  if (in->is_dir()) {
+    allowed &= ~CEPH_CAP_ANY_DIR_OPS;
+    if (allowed & CEPH_CAP_FILE_EXCL)
+      allowed |= cap->get_lock_cache_allowed();
+  }
+
+  if ((in->get_inode()->inline_data.version != CEPH_INLINE_NONE &&
+       cap->is_noinline()) ||
+      (!in->get_inode()->layout.pool_ns.empty() &&
+       cap->is_nopoolns()))
+    allowed &= ~(CEPH_CAP_FILE_RD | CEPH_CAP_FILE_WR);
+
+  return allowed;
+}
 
+int Locker::issue_caps(CInode *in, Capability *only_cap)
+{
   // count conflicts with
-  int nissued = 0;        
+  int nissued = 0;
+  int all_allowed = -1, loner_allowed = -1, xlocker_allowed = -1;
+
+  ceph_assert(in->is_head());
 
   // client caps
   map<client_t, Capability>::iterator it;
@@ -2299,28 +2337,8 @@ int Locker::issue_caps(CInode *in, Capability *only_cap)
     it = in->client_caps.begin();
   for (; it != in->client_caps.end(); ++it) {
     Capability *cap = &it->second;
-
-    // do not issue _new_ bits when size|mtime is projected
-    int allowed;
-    if (loner == it->first)
-      allowed = loner_allowed;
-    else
-      allowed = all_allowed;
-
-    // add in any xlocker-only caps (for locks this client is the xlocker for)
-    allowed |= xlocker_allowed & in->get_xlocker_mask(it->first);
-    if (in->is_dir()) {
-      allowed &= ~CEPH_CAP_ANY_DIR_OPS;
-      if (allowed & CEPH_CAP_FILE_EXCL)
-       allowed |= cap->get_lock_cache_allowed();
-    }
-
-    if ((in->get_inode()->inline_data.version != CEPH_INLINE_NONE &&
-        cap->is_noinline()) ||
-       (!in->get_inode()->layout.pool_ns.empty() &&
-        cap->is_nopoolns()))
-      allowed &= ~(CEPH_CAP_FILE_RD | CEPH_CAP_FILE_WR);
-
+    int allowed = get_allowed_caps(in, cap, all_allowed, loner_allowed,
+                                   xlocker_allowed);
     int pending = cap->pending();
     int wanted = cap->wanted();
 

diff --git a/src/mds/Locker.h b/src/mds/Locker.h
index 210a04cc55acde7083e572b578b3ed2a54bcf3a0..78cd2898be245ccfc3cc7fd3f95308d2f101a4d7 100644 (file)
--- a/src/mds/Locker.h
+++ b/src/mds/Locker.h
@@ -164,6 +164,8 @@ public:
   // -- file i/o --
   version_t issue_file_data_version(CInode *in);
   Capability* issue_new_caps(CInode *in, int mode, MDRequestRef& mdr, SnapRealm *conrealm);
+  int get_allowed_caps(CInode *in, Capability *cap, int &all_allowed,
+                       int &loner_allowed, int &xlocker_allowed);
   int issue_caps(CInode *in, Capability *only_cap=0);
   void issue_caps_set(std::set<CInode*>& inset);
   void issue_truncate(CInode *in);