mds: delay exporting directory whose pin value exceeds max rank id

Currently we allow to set ceph.dir.pin value to any number. If it is
larger than current max id, this dir will stay in export_pin_queue all
the time and every tick migrator will try to handle it but never export
it successfully.

Fixes: http://tracker.ceph.com/issues/40603
Signed-off-by: Zhi Zhang <zhangz.david@outlook.com>

diff --git a/src/mds/CInode.h b/src/mds/CInode.h
index ae641073a9c7fffa5958e05998a8ae83111dcac6..c1dd58304a1753a26972cb707fb993b3ce2e9b30 100644 (file)
--- a/src/mds/CInode.h
+++ b/src/mds/CInode.h
@@ -234,6 +234,7 @@ class CInode : public MDSCacheObject, public InodeStoreBase, public Counter<CIno
   static const int STATE_EVALSTALECAPS         = (1<<16);
   static const int STATE_QUEUEDEXPORTPIN       = (1<<17);
   static const int STATE_TRACKEDBYOFT          = (1<<18);  // tracked by open file table
+  static const int STATE_DELAYEDEXPORTPIN      = (1<<19);
   // orphan inode needs notification of releasing reference
   static const int STATE_ORPHAN =      STATE_NOTIFYREF;
 
@@ -241,7 +242,7 @@ class CInode : public MDSCacheObject, public InodeStoreBase, public Counter<CIno
     (STATE_DIRTY|STATE_NEEDSRECOVER|STATE_DIRTYPARENT|STATE_DIRTYPOOL);
   static const int MASK_STATE_EXPORT_KEPT =
     (STATE_FROZEN|STATE_AMBIGUOUSAUTH|STATE_EXPORTINGCAPS|
-     STATE_QUEUEDEXPORTPIN|STATE_TRACKEDBYOFT);
+     STATE_QUEUEDEXPORTPIN|STATE_TRACKEDBYOFT|STATE_DELAYEDEXPORTPIN);
 
   // -- waiters --
   static const uint64_t WAIT_DIR         = (1<<0);

diff --git a/src/mds/MDBalancer.cc b/src/mds/MDBalancer.cc
index 2cb109a4e11853526040e2eee4f07ec88f79381b..e7c4fbbdfe3cfc1ba41a8e8bb40d9dbb897a858f 100644 (file)
--- a/src/mds/MDBalancer.cc
+++ b/src/mds/MDBalancer.cc
@@ -102,6 +102,15 @@ void MDBalancer::handle_export_pins(void)
     CInode *in = *cur;
     ceph_assert(in->is_dir());
     mds_rank_t export_pin = in->get_export_pin(false);
+    if (export_pin >= mds->mdsmap->get_max_mds()) {
+      dout(20) << " delay export pin on " << *in << dendl;
+      in->state_clear(CInode::STATE_QUEUEDEXPORTPIN);
+      q.erase(cur);
+
+      in->state_set(CInode::STATE_DELAYEDEXPORTPIN);
+      mds->mdcache->export_pin_delayed_queue.insert(in);
+      continue;
+    }
 
     bool remove = true;
     auto&& dfls = in->get_dirfrags();
@@ -165,7 +174,8 @@ void MDBalancer::handle_export_pins(void)
                  dendl;
     }
 
-    if (export_pin >= 0 && export_pin != mds->get_nodeid()) {
+    if (export_pin >= 0 && export_pin < mds->mdsmap->get_max_mds() 
+       && export_pin != mds->get_nodeid()) {
       mds->mdcache->migrator->export_dir(cd, export_pin);
     }
   }

diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index c3bd1ff6f792a6a7942064f0e9e6a3d7ce34685e..cfb6ffa8d0499e7c0bea60a862245c15999b07ac 100644 (file)
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -305,6 +305,9 @@ void MDCache::remove_inode(CInode *o)
   if (o->state_test(CInode::STATE_QUEUEDEXPORTPIN))
     export_pin_queue.erase(o);
 
+  if (o->state_test(CInode::STATE_DELAYEDEXPORTPIN))
+    export_pin_delayed_queue.erase(o);
+
   // remove from inode map
   if (o->last == CEPH_NOSNAP) {
     inode_map.erase(o->ino());
@@ -12933,3 +12936,23 @@ bool MDCache::dump_inode(Formatter *f, uint64_t number) {
   f->close_section();
   return true;
 }
+
+void MDCache::handle_mdsmap(const MDSMap &mdsmap) {
+  // process export_pin_delayed_queue whenever a new MDSMap received
+  auto &q = export_pin_delayed_queue;
+  for (auto it = q.begin(); it != q.end(); ) {
+    auto *in = *it;
+    mds_rank_t export_pin = in->get_export_pin(false);
+    dout(10) << " delayed export_pin=" << export_pin << " on " << *in 
+      << " max_mds=" << mdsmap.get_max_mds() << dendl;
+    if (export_pin >= mdsmap.get_max_mds()) {
+      it++;
+      continue;
+    }
+
+    in->state_clear(CInode::STATE_DELAYEDEXPORTPIN);
+    it = q.erase(it);
+    in->maybe_export_pin();
+  }
+}
+

diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h
index 9898fc37c0667b234474bcbd8f87041509013c83..f32d5af50ca87ea7ad210239296b38bd06aeefef 100644 (file)
--- a/src/mds/MDCache.h
+++ b/src/mds/MDCache.h
@@ -1261,6 +1261,9 @@ public:
   void process_delayed_expire(CDir *dir);
   void discard_delayed_expire(CDir *dir);
 
+  // -- mdsmap --
+  void handle_mdsmap(const MDSMap &mdsmap);
+
 protected:
   int dump_cache(std::string_view fn, Formatter *f);
 public:
@@ -1318,6 +1321,7 @@ public:
 public:
   /* Because exports may fail, this set lets us keep track of inodes that need exporting. */
   std::set<CInode *> export_pin_queue;
+  std::set<CInode *> export_pin_delayed_queue;
 
   OpenFileTable open_file_table;
 };

diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index f683a7bdd1179ea4538040616f2cdc2dfb27e8f0..2be93eab2d520f473a7c6565cee12c3b3853affe 100644 (file)
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -2399,6 +2399,8 @@ void MDSRankDispatcher::handle_mds_map(
   if (oldmap.get_max_mds() != mdsmap->get_max_mds()) {
     purge_queue.update_op_limit(*mdsmap);
   }
+
+  mdcache->handle_mdsmap(*mdsmap);
 }
 
 void MDSRank::handle_mds_recovery(mds_rank_t who)