From: Zhi Zhang <willzzhang@tencent.com>
Date: Fri, 26 Jul 2019 04:01:08 +0000 (+0800)
Subject: mds: delay exporting directory whose pin value exceeds max rank id
X-Git-Tag: v15.1.0~2032^2
X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=2c312614a7eddda6fc788753db4e2afab4e2b73e;p=ceph-ci.git

mds: delay exporting directory whose pin value exceeds max rank id

Currently we allow to set ceph.dir.pin value to any number. If it is
larger than current max id, this dir will stay in export_pin_queue all
the time and every tick migrator will try to handle it but never export
it successfully.

Fixes: http://tracker.ceph.com/issues/40603

Signed-off-by: Zhi Zhang <zhangz.david@outlook.com>
---

diff --git a/src/mds/CInode.h b/src/mds/CInode.h
index ae641073a9c..c1dd58304a1 100644
--- a/src/mds/CInode.h
+++ b/src/mds/CInode.h
@@ -234,6 +234,7 @@ class CInode : public MDSCacheObject, public InodeStoreBase, public Counter<CIno
   static const int STATE_EVALSTALECAPS		= (1<<16);
   static const int STATE_QUEUEDEXPORTPIN	= (1<<17);
   static const int STATE_TRACKEDBYOFT		= (1<<18);  // tracked by open file table
+  static const int STATE_DELAYEDEXPORTPIN	= (1<<19);
   // orphan inode needs notification of releasing reference
   static const int STATE_ORPHAN =	STATE_NOTIFYREF;
 
@@ -241,7 +242,7 @@ class CInode : public MDSCacheObject, public InodeStoreBase, public Counter<CIno
     (STATE_DIRTY|STATE_NEEDSRECOVER|STATE_DIRTYPARENT|STATE_DIRTYPOOL);
   static const int MASK_STATE_EXPORT_KEPT =
     (STATE_FROZEN|STATE_AMBIGUOUSAUTH|STATE_EXPORTINGCAPS|
-     STATE_QUEUEDEXPORTPIN|STATE_TRACKEDBYOFT);
+     STATE_QUEUEDEXPORTPIN|STATE_TRACKEDBYOFT|STATE_DELAYEDEXPORTPIN);
 
   // -- waiters --
   static const uint64_t WAIT_DIR         = (1<<0);
diff --git a/src/mds/MDBalancer.cc b/src/mds/MDBalancer.cc
index 2cb109a4e11..e7c4fbbdfe3 100644
--- a/src/mds/MDBalancer.cc
+++ b/src/mds/MDBalancer.cc
@@ -102,6 +102,15 @@ void MDBalancer::handle_export_pins(void)
     CInode *in = *cur;
     ceph_assert(in->is_dir());
     mds_rank_t export_pin = in->get_export_pin(false);
+    if (export_pin >= mds->mdsmap->get_max_mds()) {
+      dout(20) << " delay export pin on " << *in << dendl;
+      in->state_clear(CInode::STATE_QUEUEDEXPORTPIN);
+      q.erase(cur);
+
+      in->state_set(CInode::STATE_DELAYEDEXPORTPIN);
+      mds->mdcache->export_pin_delayed_queue.insert(in);
+      continue;
+    }
 
     bool remove = true;
     auto&& dfls = in->get_dirfrags();
@@ -165,7 +174,8 @@ void MDBalancer::handle_export_pins(void)
 		  dendl;
     }
 
-    if (export_pin >= 0 && export_pin != mds->get_nodeid()) {
+    if (export_pin >= 0 && export_pin < mds->mdsmap->get_max_mds() 
+	&& export_pin != mds->get_nodeid()) {
       mds->mdcache->migrator->export_dir(cd, export_pin);
     }
   }
diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index c3bd1ff6f79..cfb6ffa8d04 100644
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -305,6 +305,9 @@ void MDCache::remove_inode(CInode *o)
   if (o->state_test(CInode::STATE_QUEUEDEXPORTPIN))
     export_pin_queue.erase(o);
 
+  if (o->state_test(CInode::STATE_DELAYEDEXPORTPIN))
+    export_pin_delayed_queue.erase(o);
+
   // remove from inode map
   if (o->last == CEPH_NOSNAP) {
     inode_map.erase(o->ino());
@@ -12933,3 +12936,23 @@ bool MDCache::dump_inode(Formatter *f, uint64_t number) {
   f->close_section();
   return true;
 }
+
+void MDCache::handle_mdsmap(const MDSMap &mdsmap) {
+  // process export_pin_delayed_queue whenever a new MDSMap received
+  auto &q = export_pin_delayed_queue;
+  for (auto it = q.begin(); it != q.end(); ) {
+    auto *in = *it;
+    mds_rank_t export_pin = in->get_export_pin(false);
+    dout(10) << " delayed export_pin=" << export_pin << " on " << *in 
+      << " max_mds=" << mdsmap.get_max_mds() << dendl;
+    if (export_pin >= mdsmap.get_max_mds()) {
+      it++;
+      continue;
+    }
+
+    in->state_clear(CInode::STATE_DELAYEDEXPORTPIN);
+    it = q.erase(it);
+    in->maybe_export_pin();
+  }
+}
+
diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h
index 9898fc37c06..f32d5af50ca 100644
--- a/src/mds/MDCache.h
+++ b/src/mds/MDCache.h
@@ -1261,6 +1261,9 @@ public:
   void process_delayed_expire(CDir *dir);
   void discard_delayed_expire(CDir *dir);
 
+  // -- mdsmap --
+  void handle_mdsmap(const MDSMap &mdsmap);
+
 protected:
   int dump_cache(std::string_view fn, Formatter *f);
 public:
@@ -1318,6 +1321,7 @@ public:
 public:
   /* Because exports may fail, this set lets us keep track of inodes that need exporting. */
   std::set<CInode *> export_pin_queue;
+  std::set<CInode *> export_pin_delayed_queue;
 
   OpenFileTable open_file_table;
 };
diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index f683a7bdd11..2be93eab2d5 100644
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -2399,6 +2399,8 @@ void MDSRankDispatcher::handle_mds_map(
   if (oldmap.get_max_mds() != mdsmap->get_max_mds()) {
     purge_queue.update_op_limit(*mdsmap);
   }
+
+  mdcache->handle_mdsmap(*mdsmap);
 }
 
 void MDSRank::handle_mds_recovery(mds_rank_t who)