From 57b39f04087f3ba855248950723c949e3b7dcd7c Mon Sep 17 00:00:00 2001
From: "Yan, Zheng" <zyan@redhat.com>
Date: Wed, 29 Jun 2016 11:42:42 +0800
Subject: [PATCH] mds: do files recovery after processing cap flushes

File recovery may update inode and trigger inode COW. MDS relies on
client caps to setup CInode::client_need_snapflush. But for a given
client, the reconnected caps may not include the flushing caps.
(Before MDS failover, client released and flushed some caps at the
same time. When MDS recovers, client re-send the cap flush and send
cap reconnect to the MDS.) This may cause later snapflush to get
dropped.

Signed-off-by: Yan, Zheng <zyan@redhat.com>
(cherry picked from commit dd98448d3d0bbb7d338f03f7ad1e40f217eebe0d)
---
 src/mds/MDCache.cc | 24 +++++++++++++-----------
 src/mds/MDCache.h  |  4 ++--
 src/mds/MDSRank.cc |  2 ++
 3 files changed, 17 insertions(+), 13 deletions(-)

diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index 65d7480ef2a78..fe59bea2e49ca 100644
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -5162,7 +5162,7 @@ void MDCache::rejoin_gather_finish()
 
   choose_lock_states_and_reconnect_caps();
 
-  identify_files_to_recover(rejoin_recover_q, rejoin_check_q);
+  identify_files_to_recover();
   rejoin_send_acks();
   
   // signal completion of fetches, rejoin_gather_finish, etc.
@@ -5760,7 +5760,6 @@ void MDCache::open_snap_parents()
     dout(10) << "open_snap_parents - all open" << dendl;
     do_delayed_cap_imports();
 
-    start_files_to_recover(rejoin_recover_q, rejoin_check_q);
     assert(rejoin_done != NULL);
     rejoin_done->complete(0);
     rejoin_done = NULL;
@@ -6086,7 +6085,7 @@ void MDCache::_queued_file_recover_cow(CInode *in, MutationRef& mut)
  * called after recovery to recover file sizes for previously opened (for write)
  * files.  that is, those where max_size > size.
  */
-void MDCache::identify_files_to_recover(vector<CInode*>& recover_q, vector<CInode*>& check_q)
+void MDCache::identify_files_to_recover()
 {
   dout(10) << "identify_files_to_recover" << dendl;
   for (ceph::unordered_map<vinodeno_t,CInode*>::iterator p = inode_map.begin();
@@ -6119,27 +6118,30 @@ void MDCache::identify_files_to_recover(vector<CInode*>& recover_q, vector<CInod
     if (recover) {
       in->auth_pin(&in->filelock);
       in->filelock.set_state(LOCK_PRE_SCAN);
-      recover_q.push_back(in);
-      
+      rejoin_recover_q.push_back(in);
+
       // make sure past parents are open/get opened
       SnapRealm *realm = in->find_snaprealm();
       check_realm_past_parents(realm);
     } else {
-      check_q.push_back(in);
+      rejoin_check_q.push_back(in);
     }
   }
 }
 
-void MDCache::start_files_to_recover(vector<CInode*>& recover_q, vector<CInode*>& check_q)
+void MDCache::start_files_to_recover()
 {
-  for (vector<CInode*>::iterator p = check_q.begin(); p != check_q.end(); ++p) {
-    CInode *in = *p;
+  for (CInode *in : rejoin_check_q) {
     mds->locker->check_inode_max_size(in);
   }
-  for (vector<CInode*>::iterator p = recover_q.begin(); p != recover_q.end(); ++p) {
-    CInode *in = *p;
+  rejoin_check_q.clear();
+  for (CInode *in : rejoin_recover_q) {
     mds->locker->file_recover(&in->filelock);
   }
+  if (!rejoin_recover_q.empty()) {
+    rejoin_recover_q.clear();
+    do_file_recover();
+  }
 }
 
 void MDCache::do_file_recover()
diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h
index 84229b8de314c..54e8c4199af2d 100644
--- a/src/mds/MDCache.h
+++ b/src/mds/MDCache.h
@@ -629,9 +629,9 @@ public:
   // File size recovery
 private:
   RecoveryQueue recovery_queue;
-  void identify_files_to_recover(vector<CInode*>& recover_q, vector<CInode*>& check_q);
-  void start_files_to_recover(vector<CInode*>& recover_q, vector<CInode*>& check_q);
+  void identify_files_to_recover();
 public:
+  void start_files_to_recover();
   void do_file_recover();
   void queue_file_recover(CInode *in);
   void _queued_file_recover_cow(CInode *in, MutationRef& mut);
diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index 188750a195dc4..68ea33eb0b343 100644
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -1226,6 +1226,7 @@ void MDSRank::clientreplay_start()
 {
   dout(1) << "clientreplay_start" << dendl;
   finish_contexts(g_ceph_context, waiting_for_replay);  // kick waiters
+  mdcache->start_files_to_recover();
   queue_one_replay();
 }
 
@@ -1259,6 +1260,7 @@ void MDSRank::active_start()
   mdcache->clean_open_file_lists();
   mdcache->export_remaining_imported_caps();
   finish_contexts(g_ceph_context, waiting_for_replay);  // kick waiters
+  mdcache->start_files_to_recover();
 
   mdcache->reissue_all_caps();
 
-- 
2.39.5