From: Sage Weil <sage@newdream.net>
Date: Wed, 28 May 2008 17:05:06 +0000 (-0700)
Subject: mds: many many scatterlock fixes
X-Git-Tag: v0.3~170^2~38
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=1d17508a32b1f052bb535e23cd92cad9f644b272;p=ceph.git

mds: many many scatterlock fixes
---

diff --git a/src/TODO b/src/TODO
index 3e38bc31be48..1adfd242f31c 100644
--- a/src/TODO
+++ b/src/TODO
@@ -8,10 +8,6 @@ big items
 - client, user authentication
 - cas
 
-bugs
-- -1 on mds file size recovery
-
-- mon needs to trim history
 - meta vs data crush rules
 - use libuuid
 
@@ -92,6 +88,8 @@ mds mustfix
 - rerun destro trace against latest, with various journal lengths
 
 mds
+- bind lease, cap timeouts to session renew, so that they can be _longer_ than the session renew interval
+
 - lease length heuristics
   - mds lock last_change stamp?
 
diff --git a/src/mds/Locker.cc b/src/mds/Locker.cc
index 20247be1de70..c417c4ffeb8b 100644
--- a/src/mds/Locker.cc
+++ b/src/mds/Locker.cc
@@ -1965,15 +1965,17 @@ bool Locker::scatter_wrlock_try(ScatterLock *lock, Mutation *mut)
   dout(7) << "scatter_wrlock_try  on " << *lock
 	  << " on " << *lock->get_parent() << dendl;  
 
-  // pre-twiddle?
+  bool want_scatter = lock->get_parent()->is_auth() &&
+    ((CInode*)lock->get_parent())->has_subtree_root_dirfrag();
+
+  // fast-path?
   if (lock->get_parent()->is_auth() &&
-      !lock->get_parent()->is_replicated() &&
-      !lock->is_rdlocked() &&
-      !lock->is_xlocked() &&
-      lock->get_num_client_lease() == 0 &&
-      lock->get_state() == LOCK_SYNC) 
-    lock->set_state(LOCK_SCATTER);
-  //scatter_scatter(lock);
+      lock->is_stable()) {
+    if (want_scatter) 
+      scatter_scatter_fastpath(lock);
+    else 
+      scatter_lock_fastpath(lock);
+  }
 
   // can wrlock?
   if (lock->can_wrlock()) {
@@ -1986,10 +1988,9 @@ bool Locker::scatter_wrlock_try(ScatterLock *lock, Mutation *mut)
   // initiate scatter or lock?
   if (lock->is_stable()) {
     if (lock->get_parent()->is_auth()) {
-      // auth.  scatter or lock?
-      if (((CInode*)lock->get_parent())->has_subtree_root_dirfrag()) 
+      if (want_scatter) 
 	scatter_scatter(lock);
-      else
+      else 
 	scatter_lock(lock);
     } else {
       // replica.
@@ -2156,6 +2157,19 @@ void Locker::scatter_eval_gather(ScatterLock *lock)
       }
       lock->set_state(LOCK_LOCK);
     }
+
+    if (lock->get_state() == LOCK_GLOCKS &&
+	!lock->is_rdlocked() &&
+	lock->get_num_client_lease() == 0) {
+      dout(10) << "scatter_eval no rdlocks|leases, acking lock" << dendl;
+      int auth = lock->get_parent()->authority().first;
+      if (mds->mdsmap->get_state(auth) >= MDSMap::STATE_REJOIN) {
+	bufferlist data;
+	lock->encode_locked_state(data);
+	mds->send_message_mds(new MLock(lock, LOCK_AC_LOCKACK, mds->get_nodeid(), data), auth);
+      }
+      lock->set_state(LOCK_LOCK);
+    }
     
   } else {
     // AUTH
@@ -2410,6 +2424,40 @@ void Locker::scatter_sync(ScatterLock *lock)
   lock->finish_waiters(ScatterLock::WAIT_RD|ScatterLock::WAIT_STABLE);
 }
 
+
+bool Locker::scatter_scatter_fastpath(ScatterLock *lock)
+{
+  assert(lock->get_parent()->is_auth());
+  assert(lock->is_stable());
+
+  if (lock->get_state() == LOCK_SCATTER)
+    return true;
+  if (!lock->is_rdlocked() &&
+      !lock->is_xlocked() &&
+      !lock->get_num_client_lease() &&
+      (!lock->get_parent()->is_replicated() ||  // if sync
+       lock->get_state() == LOCK_LOCK ||
+       lock->get_state() == LOCK_TEMPSYNC)) {
+    dout(10) << "scatter_scatter_fathpath YES " << *lock
+	     << " on " << *lock->get_parent() << dendl;
+    // do scatter
+    lock->set_last_scatter(g_clock.now());
+
+    if (lock->get_parent()->is_replicated()) {
+      // encode and bcast
+      bufferlist data;
+      lock->encode_locked_state(data);
+      send_lock_message(lock, LOCK_AC_SCATTER, data);
+    } 
+    lock->set_state(LOCK_SCATTER);
+    lock->finish_waiters(ScatterLock::WAIT_WR|ScatterLock::WAIT_STABLE);
+    return true;
+  }
+  dout(20) << "scatter_scatter_fathpath NO " << *lock
+	   << " on " << *lock->get_parent() << dendl;
+  return false;
+}
+
 void Locker::scatter_scatter(ScatterLock *lock)
 {
   dout(10) << "scatter_scatter " << *lock
@@ -2417,52 +2465,47 @@ void Locker::scatter_scatter(ScatterLock *lock)
   assert(lock->get_parent()->is_auth());
   assert(lock->is_stable());
   
-  lock->set_last_scatter(g_clock.now());
-
-  switch (lock->get_state()) {
-  case LOCK_SYNC:
-    if (!lock->is_rdlocked() &&
-	!lock->get_parent()->is_replicated() &&
-	!lock->get_num_client_lease())
-      break; // do it
-    if (lock->get_parent()->is_replicated()) {
-      send_lock_message(lock, LOCK_AC_LOCK);
-      lock->init_gather();
-    }
-    revoke_client_leases(lock);
-    lock->set_state(LOCK_GSCATTERS);
-    lock->get_parent()->auth_pin();
+  if (scatter_scatter_fastpath(lock))
     return;
 
-  case LOCK_LOCK:
-    if (lock->is_xlocked())
-      return;  // sorry
-    break; // do it.
+  if (lock->is_xlocked())
+    return;  // do nothing.
 
-  case LOCK_SCATTER:
-    return; // did it.
+  switch (lock->get_state()) {
+  case LOCK_SYNC: lock->set_state(LOCK_GSCATTERS); break;
+  case LOCK_TEMPSYNC: lock->set_state(LOCK_GSCATTERT); break;
+  default: assert(0);
+  }
 
-  case LOCK_TEMPSYNC:
-    if (lock->is_rdlocked()) {
-      lock->set_state(LOCK_GSCATTERT);
-      lock->get_parent()->auth_pin();
-      return;
-    }
-    break; // do it
+  lock->get_parent()->auth_pin();
 
-  default: 
-    assert(0);
+  if (lock->get_parent()->is_replicated()) {
+    send_lock_message(lock, LOCK_AC_LOCK);
+    lock->init_gather();
   }
+  if (lock->get_num_client_lease())
+    revoke_client_leases(lock);
+}
 
-  // do scatter
-  if (lock->get_parent()->is_replicated()) {
-    // encode and bcast
-    bufferlist data;
-    lock->encode_locked_state(data);
-    send_lock_message(lock, LOCK_AC_SCATTER, data);
-  } 
-  lock->set_state(LOCK_SCATTER);
-  lock->finish_waiters(ScatterLock::WAIT_WR|ScatterLock::WAIT_STABLE);
+bool Locker::scatter_lock_fastpath(ScatterLock *lock)
+{
+  assert(lock->get_parent()->is_auth());
+  assert(lock->is_stable());
+
+  if (lock->get_state() == LOCK_LOCK)
+    return true;
+  if (!lock->is_rdlocked() &&
+      !lock->is_wrlocked() &&
+      !lock->is_xlocked() &&
+      !lock->get_num_client_lease() &&
+      (!lock->get_parent()->is_replicated() ||  // sync | scatter
+       lock->get_state() == LOCK_TEMPSYNC)) {
+    // lock
+    lock->set_state(LOCK_LOCK);
+    lock->finish_waiters(ScatterLock::WAIT_XLOCK|ScatterLock::WAIT_WR|ScatterLock::WAIT_STABLE);
+    return true;
+  }
+  return false;
 }
 
 void Locker::scatter_lock(ScatterLock *lock)
@@ -2472,51 +2515,24 @@ void Locker::scatter_lock(ScatterLock *lock)
   assert(lock->get_parent()->is_auth());
   assert(lock->is_stable());
 
-  switch (lock->get_state()) {
-  case LOCK_SYNC:
-    if (!lock->is_rdlocked() &&
-	!lock->get_parent()->is_replicated() &&
-	!lock->get_num_client_lease())
-      break; // do it.
-
-    if (lock->get_parent()->is_replicated()) {
-      send_lock_message(lock, LOCK_AC_LOCK);
-      lock->init_gather();
-    } 
-    revoke_client_leases(lock);  
-    lock->set_state(LOCK_GLOCKS);
-    lock->get_parent()->auth_pin();
+  if (scatter_lock_fastpath(lock))
     return;
 
-  case LOCK_LOCK:
-    return; // done.
-
-  case LOCK_SCATTER:
-    if (!lock->is_wrlocked() &&
-	!lock->get_parent()->is_replicated()) {
-      break; // do it.
-    }
-
-    if (lock->get_parent()->is_replicated()) {
-      send_lock_message(lock, LOCK_AC_LOCK);
-      lock->init_gather();
-    }
-    lock->set_state(LOCK_GLOCKC);
-    lock->get_parent()->auth_pin();
-    return;
-
-  case LOCK_TEMPSYNC:
-    if (lock->is_rdlocked()) {
-      lock->set_state(LOCK_GLOCKT);
-      lock->get_parent()->auth_pin();
-      return;
-    }
-    break; // do it.
+  switch (lock->get_state()) {
+  case LOCK_SYNC: lock->set_state(LOCK_GLOCKS); break;
+  case LOCK_SCATTER: lock->set_state(LOCK_GLOCKC); break;
+  case LOCK_TEMPSYNC: lock->set_state(LOCK_GLOCKT); break;
+  default: assert(0);
   }
 
-  // do lock
-  lock->set_state(LOCK_LOCK);
-  lock->finish_waiters(ScatterLock::WAIT_XLOCK|ScatterLock::WAIT_WR|ScatterLock::WAIT_STABLE);
+  lock->get_parent()->auth_pin();
+
+  if (lock->get_parent()->is_replicated()) {
+    send_lock_message(lock, LOCK_AC_LOCK);
+    lock->init_gather();
+  } 
+  if (lock->get_num_client_lease())
+    revoke_client_leases(lock);  
 }
 
 void Locker::scatter_tempsync(ScatterLock *lock)
@@ -2599,13 +2615,15 @@ void Locker::handle_scatter_lock(ScatterLock *lock, MLock *m)
       dout(7) << "handle_scatter_lock has wrlocks, waiting on " << *lock
 	      << " on " << *lock->get_parent() << dendl;
       lock->set_state(LOCK_GLOCKC);
-    } else if (lock->is_rdlocked()) {
+    } else if (lock->is_rdlocked() ||
+	       lock->get_num_client_lease()) {
       assert(lock->get_state() == LOCK_SYNC);
-      dout(7) << "handle_scatter_lock has rdlocks, waiting on " << *lock
+      dout(7) << "handle_scatter_lock has rdlocks|leases, waiting on " << *lock
 	      << " on " << *lock->get_parent() << dendl;
+      revoke_client_leases(lock);
       lock->set_state(LOCK_GLOCKS);
     } else {
-      dout(7) << "handle_scatter_lock has no rd|wrlocks, sending lockack for " << *lock
+      dout(7) << "handle_scatter_lock has no rd|wrlocks|leases, sending lockack for " << *lock
 	      << " on " << *lock->get_parent() << dendl;
       
       // encode and reply
diff --git a/src/mds/Locker.h b/src/mds/Locker.h
index c6a9f2bb0d3d..6ac71a19cefd 100644
--- a/src/mds/Locker.h
+++ b/src/mds/Locker.h
@@ -131,12 +131,14 @@ public:
   void scatter_try_unscatter(ScatterLock *lock, Context *c);
   void note_autoscattered(ScatterLock *lock);
 
+  bool scatter_lock_fastpath(ScatterLock *lock);  // called by LogSegment::try_to_expire
   void scatter_lock(ScatterLock *lock);  // called by LogSegment::try_to_expire
 
 protected:
   void handle_scatter_lock(ScatterLock *lock, MLock *m);
   void _scatter_replica_lock(ScatterLock *lock, int auth);
   void scatter_sync(ScatterLock *lock);
+  bool scatter_scatter_fastpath(ScatterLock *lock);
   void scatter_scatter(ScatterLock *lock);
   void scatter_tempsync(ScatterLock *lock);
   bool scatter_rdlock_start(ScatterLock *lock, MDRequest *mut);