mds: invalidate lock caches if they hold conflicting locks

Add a list to SimpleLock, which tracks lock caches which hold locks
on SimpleLock itself. When mds want to change lock state, it can find
lock caches and invalidate them.

Signed-off-by: "Yan, Zheng" <zyan@redhat.com>

diff --git a/src/mds/Locker.cc b/src/mds/Locker.cc
index 66d015e9fcfa55acb5eefa321e62265ff6bb54b5..40a9ad0c2405f1a09dcb98a321fc4807eca517d1 100644 (file)
--- a/src/mds/Locker.cc
+++ b/src/mds/Locker.cc
@@ -791,11 +791,21 @@ void Locker::invalidate_lock_cache(MDLockCache *lock_cache)
   ceph_assert(lock_cache->item_cap_lock_cache.is_on_list());
   ceph_assert(!lock_cache->invalidating);
   lock_cache->invalidating = true;
+  lock_cache->detach_all();
   // XXX check issued caps
   lock_cache->item_cap_lock_cache.remove_myself();
   put_lock_cache(lock_cache);
 }
 
+// ask lock caches to release locks
+void Locker::invalidate_lock_caches(SimpleLock *lock)
+{
+  dout(10) << "invalidate_lock_caches " << *lock << " on " << *lock->get_parent() << dendl;
+  while (lock->is_cached()) {
+    invalidate_lock_cache(lock->get_first_cache());
+  }
+}
+
 void Locker::create_lock_cache(MDRequestRef& mdr, CInode *diri)
 {
   if (mdr->lock_cache)
@@ -900,6 +910,7 @@ void Locker::create_lock_cache(MDRequestRef& mdr, CInode *diri)
       ++it;
     }
   }
+  lock_cache->attach_locks();
 
   lock_cache->ref++;
   mdr->lock_cache = lock_cache;
@@ -1685,7 +1696,13 @@ void Locker::wrlock_finish(const MutationImpl::lock_iterator& it, MutationImpl *
   else
     mut->locks.erase(it);
 
-  if (!lock->is_wrlocked()) {
+  if (lock->is_wrlocked()) {
+    // Evaluate unstable lock after scatter_writebehind_finish(). Because
+    // eval_gather() does not change lock's state when lock is flushing.
+    if (!lock->is_stable() && lock->is_flushed() &&
+       lock->get_parent()->is_auth())
+      eval_gather(lock, false, pneed_issue);
+  } else {
     if (!lock->is_stable())
       eval_gather(lock, false, pneed_issue);
     else if (lock->get_parent()->is_auth())
@@ -4216,8 +4233,11 @@ void Locker::handle_simple_lock(SimpleLock *lock, const cref_t<MLock> &m)
     if (lock->is_leased())
       revoke_client_leases(lock);
     eval_gather(lock, true);
-    if (lock->is_unstable_and_locked())
+    if (lock->is_unstable_and_locked()) {
+      if (lock->is_cached())
+       invalidate_lock_caches(lock);
       mds->mdlog->flush();
+    }
     break;
 
 
@@ -4364,8 +4384,11 @@ bool Locker::simple_sync(SimpleLock *lock, bool *need_issue)
     }
 
     int gather = 0;
-    if (lock->is_wrlocked())
+    if (lock->is_wrlocked()) {
       gather++;
+      if (lock->is_cached())
+       invalidate_lock_caches(lock);
+    }
     
     if (lock->get_parent()->is_replicated() && old_state == LOCK_MIX) {
       send_lock_message(lock, LOCK_AC_SYNC);
@@ -4446,6 +4469,8 @@ void Locker::simple_excl(SimpleLock *lock, bool *need_issue)
     gather++;
   if (lock->is_wrlocked())
     gather++;
+  if (gather && lock->is_cached())
+    invalidate_lock_caches(lock);
 
   if (lock->get_parent()->is_replicated() && 
       lock->get_state() != LOCK_LOCK_EXCL &&
@@ -4508,8 +4533,11 @@ void Locker::simple_lock(SimpleLock *lock, bool *need_issue)
     gather++;
     revoke_client_leases(lock);
   }
-  if (lock->is_rdlocked())
+  if (lock->is_rdlocked()) {
+    if (lock->is_cached())
+      invalidate_lock_caches(lock);
     gather++;
+  }
   if (in && in->is_head()) {
     if (in->issued_caps_need_gather(lock)) {
       if (need_issue)
@@ -4590,6 +4618,8 @@ void Locker::simple_xlock(SimpleLock *lock)
     gather++;
   if (lock->is_wrlocked())
     gather++;
+  if (gather && lock->is_cached())
+    invalidate_lock_caches(lock);
   
   if (in && in->is_head()) {
     if (in->issued_caps_need_gather(lock)) {
@@ -4962,8 +4992,11 @@ void Locker::scatter_tempsync(ScatterLock *lock, bool *need_issue)
   }
 
   int gather = 0;
-  if (lock->is_wrlocked())
+  if (lock->is_wrlocked()) {
+    if (lock->is_cached())
+      invalidate_lock_caches(lock);
     gather++;
+  }
 
   if (lock->get_cap_shift() &&
       in->is_head() &&
@@ -5225,8 +5258,11 @@ void Locker::scatter_mix(ScatterLock *lock, bool *need_issue)
     }
 
     int gather = 0;
-    if (lock->is_rdlocked())
+    if (lock->is_rdlocked()) {
+      if (lock->is_cached())
+       invalidate_lock_caches(lock);
       gather++;
+    }
     if (in->is_replicated()) {
       if (lock->get_state() == LOCK_SYNC_MIX) { // for the rest states, replicas are already LOCK
        send_lock_message(lock, LOCK_AC_MIX);
@@ -5302,6 +5338,8 @@ void Locker::file_excl(ScatterLock *lock, bool *need_issue)
     gather++;
   if (lock->is_wrlocked())
     gather++;
+  if (gather && lock->is_cached())
+    invalidate_lock_caches(lock);
 
   if (in->is_replicated() &&
       lock->get_state() != LOCK_LOCK_EXCL &&
@@ -5355,8 +5393,11 @@ void Locker::file_xsyn(SimpleLock *lock, bool *need_issue)
   }
   
   int gather = 0;
-  if (lock->is_wrlocked())
+  if (lock->is_wrlocked()) {
+    if (lock->is_cached())
+      invalidate_lock_caches(lock);
     gather++;
+  }
 
   if (in->is_head() &&
       in->issued_caps_need_gather(lock)) {
@@ -5443,8 +5484,11 @@ void Locker::handle_file_lock(ScatterLock *lock, const cref_t<MLock> &m)
     if (lock->get_state() == LOCK_MIX) {
       lock->set_state(LOCK_MIX_SYNC);
       eval_gather(lock, true);
-      if (lock->is_unstable_and_locked())
+      if (lock->is_unstable_and_locked()) {
+       if (lock->is_cached())
+         invalidate_lock_caches(lock);
        mds->mdlog->flush();
+      }
       break;
     }
 
@@ -5470,8 +5514,11 @@ void Locker::handle_file_lock(ScatterLock *lock, const cref_t<MLock> &m)
     }
 
     eval_gather(lock, true);
-    if (lock->is_unstable_and_locked())
+    if (lock->is_unstable_and_locked()) {
+      if (lock->is_cached())
+       invalidate_lock_caches(lock);
       mds->mdlog->flush();
+    }
 
     break;
 
@@ -5492,8 +5539,11 @@ void Locker::handle_file_lock(ScatterLock *lock, const cref_t<MLock> &m)
       // MIXED
       lock->set_state(LOCK_SYNC_MIX);
       eval_gather(lock, true);
-      if (lock->is_unstable_and_locked())
+      if (lock->is_unstable_and_locked()) {
+       if (lock->is_cached())
+         invalidate_lock_caches(lock);
        mds->mdlog->flush();
+      }
       break;
     } 
 

diff --git a/src/mds/Locker.h b/src/mds/Locker.h
index 8735d92cee81df0027bb4655be399a0a7c9d5fd7..971b05b6f619523be7ac5a633d18c8e97b4e2865 100644 (file)
--- a/src/mds/Locker.h
+++ b/src/mds/Locker.h
@@ -69,6 +69,7 @@ public:
 
   void create_lock_cache(MDRequestRef& mdr, CInode *diri);
   bool find_and_attach_lock_cache(MDRequestRef& mdr, CInode *diri);
+  void invalidate_lock_caches(SimpleLock *lock);
   void invalidate_lock_cache(MDLockCache *lock_cache);
   void put_lock_cache(MDLockCache* lock_cache);
 

diff --git a/src/mds/Mutation.cc b/src/mds/Mutation.cc
index 5d59475b7c0601ea4ec4c4555d9d247fa4650908..c60f685173804581e1ca3a282c42c29fd5e65b2d 100644 (file)
--- a/src/mds/Mutation.cc
+++ b/src/mds/Mutation.cc
@@ -553,3 +553,27 @@ void MDRequestImpl::_dump_op_descriptor_unlocked(ostream& stream) const
     stream << "rejoin:" << reqid;
   }
 }
+
+void MDLockCache::attach_locks()
+{
+  ceph_assert(!items_lock);
+  items_lock.reset(new LockItem[locks.size()]);
+  int i = 0;
+  for (auto& p : locks) {
+    items_lock[i].parent = this;
+    p.lock->add_cache(items_lock[i]);
+    ++i;
+  }
+}
+
+void MDLockCache::detach_all()
+{
+  ceph_assert(items_lock);
+  int i = 0;
+  for (auto& p : locks) {
+    auto& item = items_lock[i];
+    p.lock->remove_cache(item);
+    ++i;
+  }
+  items_lock.reset();
+}

diff --git a/src/mds/Mutation.h b/src/mds/Mutation.h
index 1deb963000895e2eb01cca66de478922e5104fb9..65a6855070831e6d6fe0d2f3799bb7370fe66572 100644 (file)
--- a/src/mds/Mutation.h
+++ b/src/mds/Mutation.h
@@ -484,6 +484,11 @@ struct MDSlaveUpdate {
   }
 };
 
+struct MDLockCacheItem {
+  MDLockCache *parent = nullptr;
+  elist<MDLockCacheItem*>::item item_lock;
+};
+
 struct MDLockCache : public MutationImpl {
   CInode *diri;
   Capability *client_cap;
@@ -491,6 +496,10 @@ struct MDLockCache : public MutationImpl {
 
   elist<MDLockCache*>::item item_cap_lock_cache;
 
+  using LockItem = MDLockCacheItem;
+  // link myself to locked locks
+  std::unique_ptr<LockItem[]> items_lock;
+
   int ref = 1;
   bool invalidating = false;
 
@@ -500,6 +509,9 @@ struct MDLockCache : public MutationImpl {
   }
 
   CInode *get_dir_inode() { return diri; }
+  void attach_locks();
+  void attach_dirfrags(std::vector<CDir*>&& dfv);
+  void detach_all();
 };
 
 #endif

diff --git a/src/mds/SimpleLock.cc b/src/mds/SimpleLock.cc
index c4c0ae0db24737b62b7c7f8b8ae3057f7a0b2a98..38b805d93350ae47ecf481b28c0812ada07e032c 100644 (file)
--- a/src/mds/SimpleLock.cc
+++ b/src/mds/SimpleLock.cc
@@ -41,3 +41,32 @@ void SimpleLock::dump(Formatter *f) const {
   }
   f->close_section();
 }
+
+SimpleLock::unstable_bits_t::unstable_bits_t() :
+  lock_caches(member_offset(MDLockCache::LockItem, item_lock)) {}
+
+void SimpleLock::add_cache(MDLockCacheItem& item)
+{
+  more()->lock_caches.push_back(&item.item_lock);
+  state_flags |= CACHED;
+}
+
+void SimpleLock::remove_cache(MDLockCacheItem& item) {
+  auto& lock_caches = more()->lock_caches;
+  item.item_lock.remove_myself();
+  if (lock_caches.empty()) {
+    state_flags &= ~CACHED;
+    try_clear_more();
+  }
+}
+
+MDLockCache* SimpleLock::get_first_cache()
+{
+  if (have_more()) {
+    auto& lock_caches = more()->lock_caches;
+    if (!lock_caches.empty()) {
+      return lock_caches.front()->parent;
+    }
+  }
+  return nullptr;
+}

diff --git a/src/mds/SimpleLock.h b/src/mds/SimpleLock.h
index 8a3de2977f1758d41f1d2b6f49a7935324bf4928..9266881d9456dd785f0b066a796a04e60cae4567 100644 (file)
--- a/src/mds/SimpleLock.h
+++ b/src/mds/SimpleLock.h
@@ -28,6 +28,9 @@
 struct MutationImpl;
 typedef boost::intrusive_ptr<MutationImpl> MutationRef;
 
+struct MDLockCache;
+struct MDLockCacheItem;
+
 extern "C" {
 #include "locks.h"
 }
@@ -188,6 +191,7 @@ protected:
   enum {
     LEASED             = 1 << 0,
     NEED_RECOVER       = 1 << 1,
+    CACHED             = 1 << 2,
   };
 
 private:
@@ -203,6 +207,8 @@ private:
     client_t xlock_by_client = -1;
     client_t excl_client = -1;
 
+    elist<MDLockCacheItem*> lock_caches;
+
     bool empty() {
       return
        gather_set.empty() &&
@@ -210,10 +216,10 @@ private:
        num_xlock == 0 &&
        xlock_by.get() == NULL &&
        xlock_by_client == -1 &&
-       excl_client == -1;
+       excl_client == -1 &&
+       lock_caches.empty();
     }
-
-    unstable_bits_t() {}
+    unstable_bits_t();
   };
 
   mutable std::unique_ptr<unstable_bits_t> _unstable;
@@ -315,8 +321,13 @@ public:
   bool is_waiter_for(uint64_t mask) const {
     return parent->is_waiter_for(mask << get_wait_shift());
   }
-  
-  
+
+  bool is_cached() const {
+    return state_flags & CACHED;
+  }
+  void add_cache(MDLockCacheItem& item);
+  void remove_cache(MDLockCacheItem& item);
+  MDLockCache* get_first_cache();
 
   // state
   int get_state() const { return state; }