From 02f9720c244eb2c00bb7c0c2dffaab5c5ac9234e Mon Sep 17 00:00:00 2001
From: "Shen, Hang" <shenhang@kuaishou.com>
Date: Thu, 26 Aug 2021 11:47:39 +0800
Subject: [PATCH] mds: make open_ino(...) fetch dentries in batch

Signed-off-by: "Shen, Hang" <shenhang@kuaishou.com>
---
 src/mds/MDCache.cc       | 44 +++++++++++++++++++++++++++++++++++++++-
 src/mds/MDCache.h        | 13 ++++++++----
 src/mds/OpenFileTable.cc |  4 ++++
 3 files changed, 56 insertions(+), 5 deletions(-)

diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index b525601280d..eaa8b6339f9 100644
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -5398,6 +5398,8 @@ bool MDCache::process_imported_caps()
     return true;
   }
 
+  open_ino_batch_start();
+
   for (auto& p : cap_imports) {
     CInode *in = get_inode(p.first);
     if (in) {
@@ -5439,6 +5441,8 @@ bool MDCache::process_imported_caps()
       mds->heartbeat_reset();
   }
 
+  open_ino_batch_submit();
+
   if (cap_imports_num_opening > 0)
     return true;
 
@@ -8904,7 +8908,16 @@ void MDCache::_open_ino_fetch_dir(inodeno_t ino, const cref_t<MMDSOpenIno> &m, b
 {
   if (dir->state_test(CDir::STATE_REJOINUNDEF))
     ceph_assert(dir->get_inode()->dirfragtree.is_leaf(dir->get_frag()));
-  dir->fetch(dname, CEPH_NOSNAP, new C_MDC_OpenInoTraverseDir(this, ino, m, parent));
+
+  auto fin = new C_MDC_OpenInoTraverseDir(this, ino, m, parent);
+  if (open_ino_batch && !dname.empty()) {
+    auto& p = open_ino_batched_fetch[dir];
+    p.first.emplace_back(dname);
+    p.second.emplace_back(fin);
+    return;
+  }
+
+  dir->fetch(dname, CEPH_NOSNAP, fin);
   if (mds->logger)
     mds->logger->inc(l_mds_openino_dir_fetch);
 }
@@ -9208,6 +9221,35 @@ void MDCache::kick_open_ino_peers(mds_rank_t who)
   }
 }
 
+void MDCache::open_ino_batch_start()
+{
+  dout(10) << __func__ << dendl;
+  open_ino_batch = true;
+}
+
+void MDCache::open_ino_batch_submit()
+{
+  dout(10) << __func__ << dendl;
+  open_ino_batch = false;
+
+  for (auto& [dir, p] : open_ino_batched_fetch) {
+    CInode *in = dir->inode;
+    std::vector<dentry_key_t> keys;
+    for (auto& dname : p.first)
+      keys.emplace_back(CEPH_NOSNAP, dname, in->hash_dentry_name(dname));
+    dir->fetch_keys(keys,
+	  new MDSInternalContextWrapper(mds,
+	    new LambdaContext([this, waiters = std::move(p.second)](int r) mutable {
+	      mds->queue_waiters_front(waiters);
+	    })
+	  )
+	);
+    if (mds->logger)
+      mds->logger->inc(l_mds_openino_dir_fetch);
+  }
+  open_ino_batched_fetch.clear();
+}
+
 void MDCache::open_ino(inodeno_t ino, int64_t pool, MDSContext* fin,
 		       bool want_replica, bool want_xlocked,
 		       vector<inode_backpointer_t> *ancestors_hint,
diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h
index 8f3ea5be26f..916fa0322c7 100644
--- a/src/mds/MDCache.h
+++ b/src/mds/MDCache.h
@@ -836,11 +836,13 @@ class MDCache {
 
   void make_trace(std::vector<CDentry*>& trace, CInode *in);
 
-  void kick_open_ino_peers(mds_rank_t who);
   void open_ino(inodeno_t ino, int64_t pool, MDSContext *fin,
 		bool want_replica=true, bool want_xlocked=false,
 		std::vector<inode_backpointer_t> *ancestors_hint=nullptr,
 		mds_rank_t auth_hint=MDS_RANK_NONE);
+  void open_ino_batch_start();
+  void open_ino_batch_submit();
+  void kick_open_ino_peers(mds_rank_t who);
 
   void find_ino_peers(inodeno_t ino, MDSContext *c,
 		      mds_rank_t hint=MDS_RANK_NONE, bool path_locked=false);
@@ -1042,6 +1044,12 @@ class MDCache {
     MDSContext::vec waiters;
   };
 
+  ceph_tid_t open_ino_last_tid = 0;
+  std::map<inodeno_t,open_ino_info_t> opening_inodes;
+
+  bool open_ino_batch = false;
+  std::map<CDir*, std::pair<std::vector<std::string>, MDSContext::vec> > open_ino_batched_fetch;
+
   friend struct C_MDC_OpenInoTraverseDir;
   friend struct C_MDC_OpenInoParentOpened;
   friend struct C_MDC_RetryScanStray;
@@ -1214,9 +1222,6 @@ class MDCache {
   std::unique_ptr<MDSContext> rejoin_done;
   std::unique_ptr<MDSContext> resolve_done;
 
-  ceph_tid_t open_ino_last_tid = 0;
-  std::map<inodeno_t,open_ino_info_t> opening_inodes;
-
   StrayManager stray_manager;
 
  private:
diff --git a/src/mds/OpenFileTable.cc b/src/mds/OpenFileTable.cc
index a438ee1ddca..aae5df6b94f 100644
--- a/src/mds/OpenFileTable.cc
+++ b/src/mds/OpenFileTable.cc
@@ -1129,6 +1129,8 @@ void OpenFileTable::_prefetch_inodes()
       destroyed_inos_set.insert(it.second.begin(), it.second.end());
   }
 
+  mdcache->open_ino_batch_start();
+
   for (auto& [ino, anchor] : loaded_anchor_map) {
     if (destroyed_inos_set.count(ino))
 	continue;
@@ -1168,6 +1170,8 @@ void OpenFileTable::_prefetch_inodes()
       mds->heartbeat_reset();
   }
 
+  mdcache->open_ino_batch_submit();
+
   _open_ino_finish(inodeno_t(0), 0);
 }
 
-- 
2.39.5