From: Yan, Zheng <zyan@redhat.com>
Date: Wed, 29 Jun 2016 03:42:42 +0000 (+0800)
Subject: mds: do files recovery after processing cap flushes
X-Git-Tag: ses5-milestone5~375^2~8
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=dd98448d3d0bbb7d338f03f7ad1e40f217eebe0d;p=ceph.git

mds: do files recovery after processing cap flushes

File recovery may update inode and trigger inode COW. MDS relies on
client caps to setup CInode::client_need_snapflush. But for a given
client, the reconnected caps may not include the flushing caps.
(Before MDS failover, client released and flushed some caps at the
same time. When MDS recovers, client re-send the cap flush and send
cap reconnect to the MDS.) This may cause later snapflush to get
dropped.

Signed-off-by: Yan, Zheng <zyan@redhat.com>
---

diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index 15a0d7cd7ba6..3c7cf077cf8f 100644
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -5166,7 +5166,7 @@ void MDCache::rejoin_gather_finish()
 
   choose_lock_states_and_reconnect_caps();
 
-  identify_files_to_recover(rejoin_recover_q, rejoin_check_q);
+  identify_files_to_recover();
   rejoin_send_acks();
   
   // signal completion of fetches, rejoin_gather_finish, etc.
@@ -5764,7 +5764,6 @@ void MDCache::open_snap_parents()
     dout(10) << "open_snap_parents - all open" << dendl;
     do_delayed_cap_imports();
 
-    start_files_to_recover(rejoin_recover_q, rejoin_check_q);
     assert(rejoin_done != NULL);
     rejoin_done->complete(0);
     rejoin_done = NULL;
@@ -6090,7 +6089,7 @@ void MDCache::_queued_file_recover_cow(CInode *in, MutationRef& mut)
  * called after recovery to recover file sizes for previously opened (for write)
  * files.  that is, those where max_size > size.
  */
-void MDCache::identify_files_to_recover(vector<CInode*>& recover_q, vector<CInode*>& check_q)
+void MDCache::identify_files_to_recover()
 {
   dout(10) << "identify_files_to_recover" << dendl;
   for (ceph::unordered_map<vinodeno_t,CInode*>::iterator p = inode_map.begin();
@@ -6123,27 +6122,30 @@ void MDCache::identify_files_to_recover(vector<CInode*>& recover_q, vector<CInod
     if (recover) {
       in->auth_pin(&in->filelock);
       in->filelock.set_state(LOCK_PRE_SCAN);
-      recover_q.push_back(in);
-      
+      rejoin_recover_q.push_back(in);
+
       // make sure past parents are open/get opened
       SnapRealm *realm = in->find_snaprealm();
       check_realm_past_parents(realm);
     } else {
-      check_q.push_back(in);
+      rejoin_check_q.push_back(in);
     }
   }
 }
 
-void MDCache::start_files_to_recover(vector<CInode*>& recover_q, vector<CInode*>& check_q)
+void MDCache::start_files_to_recover()
 {
-  for (vector<CInode*>::iterator p = check_q.begin(); p != check_q.end(); ++p) {
-    CInode *in = *p;
+  for (CInode *in : rejoin_check_q) {
     mds->locker->check_inode_max_size(in);
   }
-  for (vector<CInode*>::iterator p = recover_q.begin(); p != recover_q.end(); ++p) {
-    CInode *in = *p;
+  rejoin_check_q.clear();
+  for (CInode *in : rejoin_recover_q) {
     mds->locker->file_recover(&in->filelock);
   }
+  if (!rejoin_recover_q.empty()) {
+    rejoin_recover_q.clear();
+    do_file_recover();
+  }
 }
 
 void MDCache::do_file_recover()
diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h
index bf7708ac4631..769a82995916 100644
--- a/src/mds/MDCache.h
+++ b/src/mds/MDCache.h
@@ -630,9 +630,9 @@ public:
   // File size recovery
 private:
   RecoveryQueue recovery_queue;
-  void identify_files_to_recover(vector<CInode*>& recover_q, vector<CInode*>& check_q);
-  void start_files_to_recover(vector<CInode*>& recover_q, vector<CInode*>& check_q);
+  void identify_files_to_recover();
 public:
+  void start_files_to_recover();
   void do_file_recover();
   void queue_file_recover(CInode *in);
   void _queued_file_recover_cow(CInode *in, MutationRef& mut);
diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index 192006fb5aaf..e662f1b1a7b7 100644
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -1239,6 +1239,7 @@ void MDSRank::clientreplay_start()
 {
   dout(1) << "clientreplay_start" << dendl;
   finish_contexts(g_ceph_context, waiting_for_replay);  // kick waiters
+  mdcache->start_files_to_recover();
   queue_one_replay();
 }
 
@@ -1272,6 +1273,7 @@ void MDSRank::active_start()
   mdcache->clean_open_file_lists();
   mdcache->export_remaining_imported_caps();
   finish_contexts(g_ceph_context, waiting_for_replay);  // kick waiters
+  mdcache->start_files_to_recover();
 
   mdcache->reissue_all_caps();