mds: invalidate lock caches when freezing dirfrag/subtree

Add a list to CDir, which tracks lock caches which hold auth pins on
objects in the CDir. When mds want to freeze dirfrag or subtree, it can
find lock caches and invalidate them

Signed-off-by: "Yan, Zheng" <zyan@redhat.com>

diff --git a/src/mds/CDir.cc b/src/mds/CDir.cc
index 771eb4a890a74f86d1241705b853482729c4b3f5..745b681e73da5ddd4ce628bec5522dd6d7f578f4 100644 (file)
--- a/src/mds/CDir.cc
+++ b/src/mds/CDir.cc
@@ -195,6 +195,7 @@ CDir::CDir(CInode *in, frag_t fg, MDCache *mdcache, bool auth) :
   dirty_rstat_inodes(member_offset(CInode, dirty_rstat_item)),
   dirty_dentries(member_offset(CDentry, item_dir_dirty)),
   item_dirty(this), item_new(this),
+  lock_caches_with_auth_pins(member_offset(MDLockCache::DirItem, item_dir)),
   dir_rep(REP_NONE),
   pop_me(mdcache->decayrate),
   pop_nested(mdcache->decayrate),
@@ -2868,14 +2869,18 @@ bool CDir::freeze_tree()
   // gets decreased. Subtree become 'frozen' when the counter reaches zero.
   freeze_tree_state = std::make_shared<freeze_tree_state_t>(this);
   freeze_tree_state->auth_pins += get_auth_pins() + get_dir_auth_pins();
+  if (!lock_caches_with_auth_pins.empty())
+    cache->mds->locker->invalidate_lock_caches(this);
 
   _walk_tree([this](CDir *dir) {
       if (dir->freeze_tree_state)
        return false;
       dir->freeze_tree_state = freeze_tree_state;
       freeze_tree_state->auth_pins += dir->get_auth_pins() + dir->get_dir_auth_pins();
+      if (!dir->lock_caches_with_auth_pins.empty())
+       cache->mds->locker->invalidate_lock_caches(dir);
       return true;
-     }
+    }
   );
 
   if (is_freezeable(true)) {
@@ -3118,6 +3123,8 @@ bool CDir::freeze_dir()
     return true;
   } else {
     state_set(STATE_FREEZINGDIR);
+    if (!lock_caches_with_auth_pins.empty())
+      cache->mds->locker->invalidate_lock_caches(this);
     dout(10) << "freeze_dir + wait " << *this << dendl;
     return false;
   } 

diff --git a/src/mds/CDir.h b/src/mds/CDir.h
index cba08b2ca68d2a3de2c712439654d9ae20ef0922..c60bf57421f9320ac38eff23cf1fb19030394858 100644 (file)
--- a/src/mds/CDir.h
+++ b/src/mds/CDir.h
@@ -599,6 +599,9 @@ public:
   elist<CDentry*> dirty_dentries;
   elist<CDir*>::item item_dirty, item_new;
 
+  // lock caches that auth-pin me
+  elist<MDLockCache::DirItem*> lock_caches_with_auth_pins;
+
   // all dirfrags within freezing/frozen tree reference the 'state'
   std::shared_ptr<freeze_tree_state_t> freeze_tree_state;
 

diff --git a/src/mds/CInode.h b/src/mds/CInode.h
index 1d29dd700f806e01adf378ff9fbcc30dcdbad60e..f3b6659cd68fe270c9290cfc1f702039f19d769e 100644 (file)
--- a/src/mds/CInode.h
+++ b/src/mds/CInode.h
@@ -893,6 +893,9 @@ class CInode : public MDSCacheObject, public InodeStoreBase, public Counter<CIno
     parent = projected_parent.front();
     projected_parent.pop_front();
   }
+  bool is_parent_projected() const {
+    return !projected_parent.empty();
+  }
 
   void maybe_export_pin(bool update=false);
   void set_export_pin(mds_rank_t rank);

diff --git a/src/mds/Locker.cc b/src/mds/Locker.cc
index 40a9ad0c2405f1a09dcb98a321fc4807eca517d1..fd9311b9fe9602dd05f865cbc00bbdfaeaf338b0 100644 (file)
--- a/src/mds/Locker.cc
+++ b/src/mds/Locker.cc
@@ -797,6 +797,16 @@ void Locker::invalidate_lock_cache(MDLockCache *lock_cache)
   put_lock_cache(lock_cache);
 }
 
+// ask lock caches to release auth pins
+void Locker::invalidate_lock_caches(CDir *dir)
+{
+  dout(10) << "invalidate_lock_caches on " << *dir << dendl;
+  auto &lock_caches = dir->lock_caches_with_auth_pins;
+  while (!lock_caches.empty()) {
+    invalidate_lock_cache(lock_caches.front()->parent);
+  }
+}
+
 // ask lock caches to release locks
 void Locker::invalidate_lock_caches(SimpleLock *lock)
 {
@@ -845,9 +855,18 @@ void Locker::create_lock_cache(MDRequestRef& mdr, CInode *diri)
     if (p.first != diri && !ancestors.count(p.first))
       continue;
     auto& stat = p.second;
-    if (stat.auth_pinned && !p.first->can_auth_pin()) {
-      dout(10) << " can't auth_pin(freezing?) lock parent " << *p.first << ", noop" << dendl;
-      return;
+    if (stat.auth_pinned) {
+      if (!p.first->can_auth_pin()) {
+       dout(10) << " can't auth_pin(freezing?) lock parent " << *p.first << ", noop" << dendl;
+       return;
+      }
+      if (CInode *in = dynamic_cast<CInode*>(p.first); in->is_parent_projected()) {
+       CDir *dir = in->get_projected_parent_dir();
+       if (!dir->can_auth_pin()) {
+         dout(10) << " can't auth_pin(!auth|freezing?) dirfrag " << *dir << ", noop" << dendl;
+         return;
+       }
+      }
     }
   }
 
@@ -886,7 +905,18 @@ void Locker::create_lock_cache(MDRequestRef& mdr, CInode *diri)
       lock_cache->auth_pin(p.first);
     else
       lock_cache->pin(p.first);
+
+    if (CInode *in = dynamic_cast<CInode*>(p.first)) {
+      CDentry *pdn = in->get_projected_parent_dn();
+      if (pdn)
+       dfv.push_back(pdn->get_dir());
+    } else if (CDentry *dn = dynamic_cast<CDentry*>(p.first)) {
+       dfv.push_back(dn->get_dir());
+    } else {
+      ceph_assert(0 == "unknown type of lock parent");
+    }
   }
+  lock_cache->attach_dirfrags(std::move(dfv));
 
   for (auto it = mdr->locks.begin(); it != mdr->locks.end(); ) {
     MDSCacheObject *obj = it->lock->get_parent();

diff --git a/src/mds/Locker.h b/src/mds/Locker.h
index 971b05b6f619523be7ac5a633d18c8e97b4e2865..be4f3bf54c9297003adc5bd8404a7accba1f7331 100644 (file)
--- a/src/mds/Locker.h
+++ b/src/mds/Locker.h
@@ -69,6 +69,7 @@ public:
 
   void create_lock_cache(MDRequestRef& mdr, CInode *diri);
   bool find_and_attach_lock_cache(MDRequestRef& mdr, CInode *diri);
+  void invalidate_lock_caches(CDir *dir);
   void invalidate_lock_caches(SimpleLock *lock);
   void invalidate_lock_cache(MDLockCache *lock_cache);
   void put_lock_cache(MDLockCache* lock_cache);

diff --git a/src/mds/Mutation.cc b/src/mds/Mutation.cc
index c60f685173804581e1ca3a282c42c29fd5e65b2d..f0a5d877cfbccc02420b883490fbc372ac92a0c9 100644 (file)
--- a/src/mds/Mutation.cc
+++ b/src/mds/Mutation.cc
@@ -566,6 +566,23 @@ void MDLockCache::attach_locks()
   }
 }
 
+void MDLockCache::attach_dirfrags(std::vector<CDir*>&& dfv)
+{
+  std::sort(dfv.begin(), dfv.end());
+  auto last = std::unique(dfv.begin(), dfv.end());
+  dfv.erase(last, dfv.end());
+  auth_pinned_dirfrags = std::move(dfv);
+
+  ceph_assert(!items_dir);
+  items_dir.reset(new DirItem[auth_pinned_dirfrags.size()]);
+  int i = 0;
+  for (auto dir : auth_pinned_dirfrags) {
+    items_dir[i].parent = this;
+    dir->lock_caches_with_auth_pins.push_back(&items_dir[i].item_dir);
+    ++i;
+  }
+}
+
 void MDLockCache::detach_all()
 {
   ceph_assert(items_lock);
@@ -576,4 +593,13 @@ void MDLockCache::detach_all()
     ++i;
   }
   items_lock.reset();
+
+  ceph_assert(items_dir);
+  i = 0;
+  for (auto dir : auth_pinned_dirfrags) {
+    (void)dir;
+    items_dir[i].item_dir.remove_myself();
+    ++i;
+  }
+  items_dir.reset();
 }

diff --git a/src/mds/Mutation.h b/src/mds/Mutation.h
index 65a6855070831e6d6fe0d2f3799bb7370fe66572..fd72d6bd8f4bf008fa83a783900df85151cc92b7 100644 (file)
--- a/src/mds/Mutation.h
+++ b/src/mds/Mutation.h
@@ -500,6 +500,14 @@ struct MDLockCache : public MutationImpl {
   // link myself to locked locks
   std::unique_ptr<LockItem[]> items_lock;
 
+  struct DirItem {
+    MDLockCache *parent = nullptr;
+    elist<DirItem*>::item item_dir;
+  };
+  // link myself to auth-pinned dirfrags
+  std::unique_ptr<DirItem[]> items_dir;
+  std::vector<CDir*> auth_pinned_dirfrags;
+
   int ref = 1;
   bool invalidating = false;