From 6d70cee1566d33fc6b7c128c23ba6c680c710d3a Mon Sep 17 00:00:00 2001 From: "Yan, Zheng" Date: Mon, 30 Sep 2019 14:20:17 +0800 Subject: [PATCH] mds: invalidate lock caches when freezing dirfrag/subtree Add a list to CDir, which tracks lock caches which hold auth pins on objects in the CDir. When mds want to freeze dirfrag or subtree, it can find lock caches and invalidate them Signed-off-by: "Yan, Zheng" --- src/mds/CDir.cc | 9 ++++++++- src/mds/CDir.h | 3 +++ src/mds/CInode.h | 3 +++ src/mds/Locker.cc | 36 +++++++++++++++++++++++++++++++++--- src/mds/Locker.h | 1 + src/mds/Mutation.cc | 26 ++++++++++++++++++++++++++ src/mds/Mutation.h | 8 ++++++++ 7 files changed, 82 insertions(+), 4 deletions(-) diff --git a/src/mds/CDir.cc b/src/mds/CDir.cc index 771eb4a890a..745b681e73d 100644 --- a/src/mds/CDir.cc +++ b/src/mds/CDir.cc @@ -195,6 +195,7 @@ CDir::CDir(CInode *in, frag_t fg, MDCache *mdcache, bool auth) : dirty_rstat_inodes(member_offset(CInode, dirty_rstat_item)), dirty_dentries(member_offset(CDentry, item_dir_dirty)), item_dirty(this), item_new(this), + lock_caches_with_auth_pins(member_offset(MDLockCache::DirItem, item_dir)), dir_rep(REP_NONE), pop_me(mdcache->decayrate), pop_nested(mdcache->decayrate), @@ -2868,14 +2869,18 @@ bool CDir::freeze_tree() // gets decreased. Subtree become 'frozen' when the counter reaches zero. freeze_tree_state = std::make_shared(this); freeze_tree_state->auth_pins += get_auth_pins() + get_dir_auth_pins(); + if (!lock_caches_with_auth_pins.empty()) + cache->mds->locker->invalidate_lock_caches(this); _walk_tree([this](CDir *dir) { if (dir->freeze_tree_state) return false; dir->freeze_tree_state = freeze_tree_state; freeze_tree_state->auth_pins += dir->get_auth_pins() + dir->get_dir_auth_pins(); + if (!dir->lock_caches_with_auth_pins.empty()) + cache->mds->locker->invalidate_lock_caches(dir); return true; - } + } ); if (is_freezeable(true)) { @@ -3118,6 +3123,8 @@ bool CDir::freeze_dir() return true; } else { state_set(STATE_FREEZINGDIR); + if (!lock_caches_with_auth_pins.empty()) + cache->mds->locker->invalidate_lock_caches(this); dout(10) << "freeze_dir + wait " << *this << dendl; return false; } diff --git a/src/mds/CDir.h b/src/mds/CDir.h index cba08b2ca68..c60bf57421f 100644 --- a/src/mds/CDir.h +++ b/src/mds/CDir.h @@ -599,6 +599,9 @@ public: elist dirty_dentries; elist::item item_dirty, item_new; + // lock caches that auth-pin me + elist lock_caches_with_auth_pins; + // all dirfrags within freezing/frozen tree reference the 'state' std::shared_ptr freeze_tree_state; diff --git a/src/mds/CInode.h b/src/mds/CInode.h index 1d29dd700f8..f3b6659cd68 100644 --- a/src/mds/CInode.h +++ b/src/mds/CInode.h @@ -893,6 +893,9 @@ class CInode : public MDSCacheObject, public InodeStoreBase, public Counterlock_caches_with_auth_pins; + while (!lock_caches.empty()) { + invalidate_lock_cache(lock_caches.front()->parent); + } +} + // ask lock caches to release locks void Locker::invalidate_lock_caches(SimpleLock *lock) { @@ -845,9 +855,18 @@ void Locker::create_lock_cache(MDRequestRef& mdr, CInode *diri) if (p.first != diri && !ancestors.count(p.first)) continue; auto& stat = p.second; - if (stat.auth_pinned && !p.first->can_auth_pin()) { - dout(10) << " can't auth_pin(freezing?) lock parent " << *p.first << ", noop" << dendl; - return; + if (stat.auth_pinned) { + if (!p.first->can_auth_pin()) { + dout(10) << " can't auth_pin(freezing?) lock parent " << *p.first << ", noop" << dendl; + return; + } + if (CInode *in = dynamic_cast(p.first); in->is_parent_projected()) { + CDir *dir = in->get_projected_parent_dir(); + if (!dir->can_auth_pin()) { + dout(10) << " can't auth_pin(!auth|freezing?) dirfrag " << *dir << ", noop" << dendl; + return; + } + } } } @@ -886,7 +905,18 @@ void Locker::create_lock_cache(MDRequestRef& mdr, CInode *diri) lock_cache->auth_pin(p.first); else lock_cache->pin(p.first); + + if (CInode *in = dynamic_cast(p.first)) { + CDentry *pdn = in->get_projected_parent_dn(); + if (pdn) + dfv.push_back(pdn->get_dir()); + } else if (CDentry *dn = dynamic_cast(p.first)) { + dfv.push_back(dn->get_dir()); + } else { + ceph_assert(0 == "unknown type of lock parent"); + } } + lock_cache->attach_dirfrags(std::move(dfv)); for (auto it = mdr->locks.begin(); it != mdr->locks.end(); ) { MDSCacheObject *obj = it->lock->get_parent(); diff --git a/src/mds/Locker.h b/src/mds/Locker.h index 971b05b6f61..be4f3bf54c9 100644 --- a/src/mds/Locker.h +++ b/src/mds/Locker.h @@ -69,6 +69,7 @@ public: void create_lock_cache(MDRequestRef& mdr, CInode *diri); bool find_and_attach_lock_cache(MDRequestRef& mdr, CInode *diri); + void invalidate_lock_caches(CDir *dir); void invalidate_lock_caches(SimpleLock *lock); void invalidate_lock_cache(MDLockCache *lock_cache); void put_lock_cache(MDLockCache* lock_cache); diff --git a/src/mds/Mutation.cc b/src/mds/Mutation.cc index c60f6851738..f0a5d877cfb 100644 --- a/src/mds/Mutation.cc +++ b/src/mds/Mutation.cc @@ -566,6 +566,23 @@ void MDLockCache::attach_locks() } } +void MDLockCache::attach_dirfrags(std::vector&& dfv) +{ + std::sort(dfv.begin(), dfv.end()); + auto last = std::unique(dfv.begin(), dfv.end()); + dfv.erase(last, dfv.end()); + auth_pinned_dirfrags = std::move(dfv); + + ceph_assert(!items_dir); + items_dir.reset(new DirItem[auth_pinned_dirfrags.size()]); + int i = 0; + for (auto dir : auth_pinned_dirfrags) { + items_dir[i].parent = this; + dir->lock_caches_with_auth_pins.push_back(&items_dir[i].item_dir); + ++i; + } +} + void MDLockCache::detach_all() { ceph_assert(items_lock); @@ -576,4 +593,13 @@ void MDLockCache::detach_all() ++i; } items_lock.reset(); + + ceph_assert(items_dir); + i = 0; + for (auto dir : auth_pinned_dirfrags) { + (void)dir; + items_dir[i].item_dir.remove_myself(); + ++i; + } + items_dir.reset(); } diff --git a/src/mds/Mutation.h b/src/mds/Mutation.h index 65a68550708..fd72d6bd8f4 100644 --- a/src/mds/Mutation.h +++ b/src/mds/Mutation.h @@ -500,6 +500,14 @@ struct MDLockCache : public MutationImpl { // link myself to locked locks std::unique_ptr items_lock; + struct DirItem { + MDLockCache *parent = nullptr; + elist::item item_dir; + }; + // link myself to auth-pinned dirfrags + std::unique_ptr items_dir; + std::vector auth_pinned_dirfrags; + int ref = 1; bool invalidating = false; -- 2.39.5