mds: consider client's flushing caps when choosing lock states

Client may flush and drop caps at the same time. If client need to
send cap reconnect before the caps get flushed. The issued caps in
the cap reconnect does not include the flushing caps. When choosing
lock states, MDS only consider the issued caps in cap reconnect, it
may choose wrong states.

Fixes: #11482
Signed-off-by: Yan, Zheng <zyan@redhat.com>

diff --git a/src/mds/CInode.cc b/src/mds/CInode.cc
index 2c384225b89d074e1539e8fdcfee4b4f1badc5c2..489ccf6970c97e684418a70bdb1fc33d8d2c4771 100644 (file)
--- a/src/mds/CInode.cc
+++ b/src/mds/CInode.cc
@@ -2694,9 +2694,9 @@ void CInode::choose_lock_state(SimpleLock *lock, int allissued)
   }
 }
  
-void CInode::choose_lock_states()
+void CInode::choose_lock_states(int dirty_caps)
 {
-  int issued = get_caps_issued();
+  int issued = get_caps_issued() | dirty_caps;
   if (is_auth() && (issued & (CEPH_CAP_ANY_EXCL|CEPH_CAP_ANY_WR)) &&
       choose_ideal_loner() >= 0)
     try_set_loner();

diff --git a/src/mds/CInode.h b/src/mds/CInode.h
index a7abba43c2c9fc2c91395d3c957dc675af9c78b7..53283f73926b137c7147cad0e194246578e30688 100644 (file)
--- a/src/mds/CInode.h
+++ b/src/mds/CInode.h
@@ -812,7 +812,7 @@ public:
 
   // choose new lock state during recovery, based on issued caps
   void choose_lock_state(SimpleLock *lock, int allissued);
-  void choose_lock_states();
+  void choose_lock_states(int dirty_caps);
 
   int count_nonstale_caps() {
     int n = 0;

diff --git a/src/mds/Locker.cc b/src/mds/Locker.cc
index 0cbb5e3982dfbe9543702e92d907b9cfe4948a87..eb83299515e863bb8216b1aca79f667956f07f90 100644 (file)
--- a/src/mds/Locker.cc
+++ b/src/mds/Locker.cc
@@ -2463,6 +2463,11 @@ void Locker::handle_client_caps(MClientCaps *m)
          << " op " << ceph_cap_op_name(m->get_op()) << dendl;
 
   if (!mds->is_clientreplay() && !mds->is_active() && !mds->is_stopping()) {
+    if (mds->is_reconnect() &&
+       m->get_dirty() && m->get_client_tid() > 0 &&
+       session->have_completed_flush(m->get_client_tid())) {
+      mdcache->set_reconnect_dirty_caps(m->get_ino(), m->get_dirty());
+    }
     mds->wait_for_replay(new C_MDS_RetryMessage(mds, m));
     return;
   }

diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index 3ec852a97d136f909ec3a6a82943887219226fda..d7b0bdebfaabb8918feb89637174686a8346e494 100644 (file)
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -5374,7 +5374,11 @@ void MDCache::choose_lock_states_and_reconnect_caps()
     if (in->is_auth() && !in->is_base() && in->inode.is_dirty_rstat())
       in->mark_dirty_rstat();
 
-    in->choose_lock_states();
+    int dirty_caps = 0;
+    map<inodeno_t, int>::iterator it = cap_imports_dirty.find(in->ino());
+    if (it != cap_imports_dirty.end())
+      dirty_caps = it->second;
+    in->choose_lock_states(dirty_caps);
     dout(15) << " chose lock states on " << *in << dendl;
 
     SnapRealm *realm = in->find_snaprealm();
@@ -5520,6 +5524,7 @@ void MDCache::export_remaining_imported_caps()
   }
 
   cap_imports.clear();
+  cap_imports_dirty.clear();
 
   if (warn_str.peek() != EOF) {
     mds->clog->warn() << "failed to reconnect caps for missing inodes:" << "\n";
@@ -5542,7 +5547,11 @@ void MDCache::try_reconnect_cap(CInode *in, Session *session)
     if (in->is_replicated()) {
       mds->locker->try_eval(in, CEPH_CAP_LOCKS);
     } else {
-      in->choose_lock_states();
+      int dirty_caps = 0;
+      map<inodeno_t, int>::iterator it = cap_imports_dirty.find(in->ino());
+      if (it != cap_imports_dirty.end())
+       dirty_caps = it->second;
+      in->choose_lock_states(dirty_caps);
       dout(15) << " chose lock states on " << *in << dendl;
     }
   }

diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h
index c369acdbf8d8cc8bf5c97f5291d44f2e92f48e52..30411b1689cf94b76d20373e5787b6972382caf4 100644 (file)
--- a/src/mds/MDCache.h
+++ b/src/mds/MDCache.h
@@ -492,6 +492,7 @@ protected:
 
   map<inodeno_t,map<client_t,map<mds_rank_t,ceph_mds_cap_reconnect> > > cap_imports;  // ino -> client -> frommds -> capex
   map<inodeno_t,filepath> cap_import_paths;
+  map<inodeno_t,int> cap_imports_dirty;
   set<inodeno_t> cap_imports_missing;
   int cap_imports_num_opening;
   
@@ -549,6 +550,9 @@ public:
     assert(cap_imports[ino][client].size() == 1);
     cap_imports.erase(ino);
   }
+  void set_reconnect_dirty_caps(inodeno_t ino, int dirty) {
+    cap_imports_dirty[ino] |= dirty;
+  }
 
   // [reconnect/rejoin caps]
   map<CInode*,map<client_t, inodeno_t> >  reconnected_caps;   // inode -> client -> realmino