From: Yan, Zheng <zyan@redhat.com>
Date: Fri, 30 Oct 2015 13:04:04 +0000 (+0800)
Subject: mds: consider client's flushing caps when choosing lock states
X-Git-Tag: v10.0.2~151^2~1
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=ce9a596dcaf95dd4af0a3a9e28871462a6bcb930;p=ceph.git

mds: consider client's flushing caps when choosing lock states

Client may flush and drop caps at the same time. If client need to
send cap reconnect before the caps get flushed. The issued caps in
the cap reconnect does not include the flushing caps. When choosing
lock states, MDS only consider the issued caps in cap reconnect, it
may choose wrong states.

Fixes: #11482
Signed-off-by: Yan, Zheng <zyan@redhat.com>
---

diff --git a/src/mds/CInode.cc b/src/mds/CInode.cc
index 2c384225b89d..489ccf6970c9 100644
--- a/src/mds/CInode.cc
+++ b/src/mds/CInode.cc
@@ -2694,9 +2694,9 @@ void CInode::choose_lock_state(SimpleLock *lock, int allissued)
   }
 }
  
-void CInode::choose_lock_states()
+void CInode::choose_lock_states(int dirty_caps)
 {
-  int issued = get_caps_issued();
+  int issued = get_caps_issued() | dirty_caps;
   if (is_auth() && (issued & (CEPH_CAP_ANY_EXCL|CEPH_CAP_ANY_WR)) &&
       choose_ideal_loner() >= 0)
     try_set_loner();
diff --git a/src/mds/CInode.h b/src/mds/CInode.h
index a7abba43c2c9..53283f73926b 100644
--- a/src/mds/CInode.h
+++ b/src/mds/CInode.h
@@ -812,7 +812,7 @@ public:
 
   // choose new lock state during recovery, based on issued caps
   void choose_lock_state(SimpleLock *lock, int allissued);
-  void choose_lock_states();
+  void choose_lock_states(int dirty_caps);
 
   int count_nonstale_caps() {
     int n = 0;
diff --git a/src/mds/Locker.cc b/src/mds/Locker.cc
index 0cbb5e3982df..eb83299515e8 100644
--- a/src/mds/Locker.cc
+++ b/src/mds/Locker.cc
@@ -2463,6 +2463,11 @@ void Locker::handle_client_caps(MClientCaps *m)
 	  << " op " << ceph_cap_op_name(m->get_op()) << dendl;
 
   if (!mds->is_clientreplay() && !mds->is_active() && !mds->is_stopping()) {
+    if (mds->is_reconnect() &&
+	m->get_dirty() && m->get_client_tid() > 0 &&
+	session->have_completed_flush(m->get_client_tid())) {
+      mdcache->set_reconnect_dirty_caps(m->get_ino(), m->get_dirty());
+    }
     mds->wait_for_replay(new C_MDS_RetryMessage(mds, m));
     return;
   }
diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index 3ec852a97d13..d7b0bdebfaab 100644
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -5374,7 +5374,11 @@ void MDCache::choose_lock_states_and_reconnect_caps()
     if (in->is_auth() && !in->is_base() && in->inode.is_dirty_rstat())
       in->mark_dirty_rstat();
 
-    in->choose_lock_states();
+    int dirty_caps = 0;
+    map<inodeno_t, int>::iterator it = cap_imports_dirty.find(in->ino());
+    if (it != cap_imports_dirty.end())
+      dirty_caps = it->second;
+    in->choose_lock_states(dirty_caps);
     dout(15) << " chose lock states on " << *in << dendl;
 
     SnapRealm *realm = in->find_snaprealm();
@@ -5520,6 +5524,7 @@ void MDCache::export_remaining_imported_caps()
   }
 
   cap_imports.clear();
+  cap_imports_dirty.clear();
 
   if (warn_str.peek() != EOF) {
     mds->clog->warn() << "failed to reconnect caps for missing inodes:" << "\n";
@@ -5542,7 +5547,11 @@ void MDCache::try_reconnect_cap(CInode *in, Session *session)
     if (in->is_replicated()) {
       mds->locker->try_eval(in, CEPH_CAP_LOCKS);
     } else {
-      in->choose_lock_states();
+      int dirty_caps = 0;
+      map<inodeno_t, int>::iterator it = cap_imports_dirty.find(in->ino());
+      if (it != cap_imports_dirty.end())
+	dirty_caps = it->second;
+      in->choose_lock_states(dirty_caps);
       dout(15) << " chose lock states on " << *in << dendl;
     }
   }
diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h
index c369acdbf8d8..30411b1689cf 100644
--- a/src/mds/MDCache.h
+++ b/src/mds/MDCache.h
@@ -492,6 +492,7 @@ protected:
 
   map<inodeno_t,map<client_t,map<mds_rank_t,ceph_mds_cap_reconnect> > > cap_imports;  // ino -> client -> frommds -> capex
   map<inodeno_t,filepath> cap_import_paths;
+  map<inodeno_t,int> cap_imports_dirty;
   set<inodeno_t> cap_imports_missing;
   int cap_imports_num_opening;
   
@@ -549,6 +550,9 @@ public:
     assert(cap_imports[ino][client].size() == 1);
     cap_imports.erase(ino);
   }
+  void set_reconnect_dirty_caps(inodeno_t ino, int dirty) {
+    cap_imports_dirty[ino] |= dirty;
+  }
 
   // [reconnect/rejoin caps]
   map<CInode*,map<client_t, inodeno_t> >  reconnected_caps;   // inode -> client -> realmino