mds: fix race of fetching large dirfrag

When a dirfrag contains more than 'mds_dir_keys_per_op' items, MDS
needs to send multiple 'omap-get-vals' requests to fetch the dirfrag
completely. There is a race if MDS commits the dirfrag in the middle
of these 'omap-get-vals' requests.

Re-fetch from the beginning if dirfrag get committed in the middle of
omap-get-vals requests.

Fixes: https://tracker.ceph.com/issues/49617
Signed-off-by: Erqi Chen <chenerqi@kuaishou.com>
(cherry picked from commit 0767336b43155426af8d4ea9f85a7cfbbf032e8c)

Conflicts:
	src/mds/CDir.cc
	src/mds/CDir.h

diff --git a/src/mds/CDir.cc b/src/mds/CDir.cc
index e657654243def26f57737439bca0dcdaab001b80..2a5985ffb49bdbfa76e79c21a5e9a8ac605815f9 100755 (executable)
--- a/src/mds/CDir.cc
+++ b/src/mds/CDir.cc
@@ -1570,14 +1570,21 @@ void CDir::fetch(MDSContext *c, const std::set<dentry_key_t>& keys)
 class C_IO_Dir_OMAP_FetchedMore : public CDirIOContext {
   MDSContext *fin;
 public:
+  const version_t omap_version;
   bufferlist hdrbl;
   bool more = false;
   map<string, bufferlist> omap;      ///< carry-over from before
   map<string, bufferlist> omap_more; ///< new batch
   int ret;
-  C_IO_Dir_OMAP_FetchedMore(CDir *d, MDSContext *f) :
-    CDirIOContext(d), fin(f), ret(0) { }
+  C_IO_Dir_OMAP_FetchedMore(CDir *d, version_t v, MDSContext *f) :
+    CDirIOContext(d), fin(f), omap_version(v), ret(0) { }
   void finish(int r) {
+    if (omap_version < dir->get_committed_version()) {
+      omap.clear();
+      dir->_omap_fetch(fin, {});
+      return;
+    }
+
     // merge results
     if (omap.empty()) {
       omap.swap(omap_more);
@@ -1585,7 +1592,7 @@ public:
       omap.insert(omap_more.begin(), omap_more.end());
     }
     if (more) {
-      dir->_omap_fetch_more(hdrbl, omap, fin);
+      dir->_omap_fetch_more(omap_version, hdrbl, omap, fin);
     } else {
       dir->_omap_fetched(hdrbl, omap, !fin, r);
       if (fin)
@@ -1600,6 +1607,7 @@ public:
 class C_IO_Dir_OMAP_Fetched : public CDirIOContext {
   MDSContext *fin;
 public:
+  const version_t omap_version;
   bufferlist hdrbl;
   bool more = false;
   map<string, bufferlist> omap;
@@ -1607,20 +1615,30 @@ public:
   int ret1, ret2, ret3;
 
   C_IO_Dir_OMAP_Fetched(CDir *d, MDSContext *f) :
-    CDirIOContext(d), fin(f), ret1(0), ret2(0), ret3(0) { }
+    CDirIOContext(d), fin(f),
+    omap_version(d->get_committing_version()),
+    ret1(0), ret2(0), ret3(0) { }
   void finish(int r) override {
     // check the correctness of backtrace
     if (r >= 0 && ret3 != -ECANCELED)
       dir->inode->verify_diri_backtrace(btbl, ret3);
     if (r >= 0) r = ret1;
     if (r >= 0) r = ret2;
+
     if (more) {
-      dir->_omap_fetch_more(hdrbl, omap, fin);
-    } else {
-      dir->_omap_fetched(hdrbl, omap, !fin, r);
-      if (fin)
-       fin->complete(r);
+      if (omap_version < dir->get_committed_version()) {
+        omap.clear();
+        dir->_omap_fetch(fin, {});
+      } else {
+        dir->_omap_fetch_more(omap_version, hdrbl, omap, fin);
+      }
+      return;
     }
+
+    dir->_omap_fetched(hdrbl, omap, !fin, r);
+    if (fin)
+      fin->complete(r);
+
   }
   void print(ostream& out) const override {
     out << "dirfrag_fetch(" << dir->dirfrag() << ")";
@@ -1660,15 +1678,13 @@ void CDir::_omap_fetch(MDSContext *c, const std::set<dentry_key_t>& keys)
                             new C_OnFinisher(fin, cache->mds->finisher));
 }
 
-void CDir::_omap_fetch_more(
-  bufferlist& hdrbl,
-  map<string, bufferlist>& omap,
-  MDSContext *c)
+void CDir::_omap_fetch_more(version_t omap_version, bufferlist& hdrbl,
+                           map<string, bufferlist>& omap, MDSContext *c)
 {
   // we have more omap keys to fetch!
   object_t oid = get_ondisk_object();
   object_locator_t oloc(cache->mds->mdsmap->get_metadata_pool());
-  C_IO_Dir_OMAP_FetchedMore *fin = new C_IO_Dir_OMAP_FetchedMore(this, c);
+  auto fin = new C_IO_Dir_OMAP_FetchedMore(this, omap_version, c);
   fin->hdrbl.claim(hdrbl);
   fin->omap.swap(omap);
   ObjectOperation rd;

diff --git a/src/mds/CDir.h b/src/mds/CDir.h
index 23c94c8bacb011258be0c137bd5fcfd5b1261cb7..9720bd8fe39f299e08c2e7497575ad06c8545430 100644 (file)
--- a/src/mds/CDir.h
+++ b/src/mds/CDir.h
@@ -612,9 +612,8 @@ protected:
   mempool::mds_co::compact_set<mempool::mds_co::string> wanted_items;
 
   void _omap_fetch(MDSContext *fin, const std::set<dentry_key_t>& keys);
-  void _omap_fetch_more(
-    bufferlist& hdrbl, std::map<std::string, bufferlist>& omap,
-    MDSContext *fin);
+  void _omap_fetch_more(version_t omap_version, bufferlist& hdrbl,
+                        map<string, bufferlist>& omap, MDSContext *fin);
   CDentry *_load_dentry(
       std::string_view key,
       std::string_view dname,