client: trim unused inodes before reconnecting to recovering MDS

So the recovering MDS does not need to fetch these ununsed inodes during
cache rejoin. This may reduce MDS recovery time.

Signed-off-by: Yan, Zheng <zyan@redhat.com>
(cherry picked from commit 2bd7ceeff53ad0f49d5825b6e7f378683616dffb)

Reviewed-by: Greg Farnum <gfarnum@redhat.com>

diff --git a/src/client/Client.cc b/src/client/Client.cc
index f6886cc983a8c5e3a8c356fd717133f084436194..1d011ad1aae8e6bb1f221d348fd4ad53b5f84b29 100644 (file)
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -506,6 +506,36 @@ void Client::trim_cache()
   }
 }
 
+void Client::trim_cache_for_reconnect(MetaSession *s)
+{
+  int mds = s->mds_num;
+  ldout(cct, 20) << "trim_cache_for_reconnect mds." << mds << dendl;
+
+  int trimmed = 0;
+  list<Dentry*> skipped;
+  while (lru.lru_get_size() > 0) {
+    Dentry *dn = static_cast<Dentry*>(lru.lru_expire());
+    if (!dn)
+      break;
+
+    if ((dn->inode && dn->inode->caps.count(mds)) ||
+       dn->dir->parent_inode->caps.count(mds)) {
+      trim_dentry(dn);
+      trimmed++;
+    } else
+      skipped.push_back(dn);
+  }
+
+  for(list<Dentry*>::iterator p = skipped.begin(); p != skipped.end(); ++p)
+    lru.lru_insert_mid(*p);
+
+  ldout(cct, 20) << "trim_cache_for_reconnect mds." << mds
+                << " trimmed " << trimmed << " dentries" << dendl;
+
+  if (s->caps.size() > 0)
+    _invalidate_kernel_dcache();
+}
+
 void Client::trim_dentry(Dentry *dn)
 {
   ldout(cct, 15) << "trim_dentry unlinking dn " << dn->name 
@@ -2030,8 +2060,13 @@ void Client::handle_mds_map(MMDSMap* m)
     if (!mdsmap->is_up(p->first) ||
        mdsmap->get_inst(p->first) != p->second->inst) {
       p->second->con->mark_down();
-      if (mdsmap->is_up(p->first))
+      if (mdsmap->is_up(p->first)) {
        p->second->inst = mdsmap->get_inst(p->first);
+       // When new MDS starts to take over, notify kernel to trim unused entries
+       // in its dcache/icache. Hopefully, the kernel will release some unused
+       // inodes before the new MDS enters reconnect state.
+       trim_cache_for_reconnect(p->second);
+      }
     } else if (oldstate == newstate)
       continue;  // no change
     
@@ -2069,6 +2104,14 @@ void Client::send_reconnect(MetaSession *session)
   int mds = session->mds_num;
   ldout(cct, 10) << "send_reconnect to mds." << mds << dendl;
 
+  // trim unused caps to reduce MDS's cache rejoin time
+  trim_cache_for_reconnect(session);
+
+  if (session->release) {
+    session->release->put();
+    session->release = NULL;
+  }
+
   MClientReconnect *m = new MClientReconnect;
 
   // i have an open session.
@@ -3187,6 +3230,20 @@ void Client::remove_session_caps(MetaSession *s)
   sync_cond.Signal();
 }
 
+void Client::_invalidate_kernel_dcache()
+{
+  // notify kernel to invalidate top level directory entries. As a side effect,
+  // unused inodes underneath these entries get pruned.
+  if (dentry_invalidate_cb && root->dir) {
+    for (ceph::unordered_map<string, Dentry*>::iterator p = root->dir->dentries.begin();
+        p != root->dir->dentries.end();
+        ++p) {
+      if (p->second->inode)
+       _schedule_invalidate_dentry_callback(p->second, false);
+    }
+  }
+}
+
 void Client::trim_caps(MetaSession *s, int max)
 {
   int mds = s->mds_num;
@@ -3245,23 +3302,8 @@ void Client::trim_caps(MetaSession *s, int max)
   }
   s->s_cap_iterator = NULL;
 
-
-  // notify kernel to invalidate top level directory entries. As a side effect,
-  // unused inodes underneath these entries get pruned.
-  if (dentry_invalidate_cb && s->caps.size() > max) {
-    assert(root);
-    if (root->dir) {
-      for (ceph::unordered_map<string, Dentry*>::iterator p = root->dir->dentries.begin();
-           p != root->dir->dentries.end();
-           ++p) {
-        if (p->second->inode)
-          _schedule_invalidate_dentry_callback(p->second, false);
-      }
-    } else {
-      // This seems unnatural, as long as we are holding caps they must be on
-      // some descendent of the root, so why don't we have the root open?
-    }
-  }
+  if (s->caps.size() > max)
+    _invalidate_kernel_dcache();
 }
 
 void Client::mark_caps_dirty(Inode *in, int caps)

diff --git a/src/client/Client.h b/src/client/Client.h
index 54ccc175ec6af891485f136a08dabc106952571e..2d28a84baa646544d2335f7483c5d1ca1e7bedd5 100644 (file)
--- a/src/client/Client.h
+++ b/src/client/Client.h
@@ -402,8 +402,10 @@ protected:
 
   // trim cache.
   void trim_cache();
+  void trim_cache_for_reconnect(MetaSession *s);
   void trim_dentry(Dentry *dn);
   void trim_caps(MetaSession *s, int max);
+  void _invalidate_kernel_dcache();
   
   void dump_inode(Formatter *f, Inode *in, set<Inode*>& did, bool disconnected);
   void dump_cache(Formatter *f);  // debug