From: Sage Weil <sage@newdream.net>
Date: Thu, 12 Mar 2009 20:35:13 +0000 (-0700)
Subject: mds: merge scatter and file lock message handlers
X-Git-Tag: v0.7.1^2~43
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=2e93f030eac2de5c276969b43a1359f10edef4b7;p=ceph.git

mds: merge scatter and file lock message handlers

And fix some issues along the way.
---

diff --git a/src/mds/Locker.cc b/src/mds/Locker.cc
index 6c6b0fa647b3..92052b0e46d4 100644
--- a/src/mds/Locker.cc
+++ b/src/mds/Locker.cc
@@ -482,10 +482,12 @@ void Locker::eval_gather(SimpleLock *lock)
 	  send_lock_message(lock, LOCK_AC_MIX, softdata);
 	}
 	break;
-	
+
+	// to sync
       case LOCK_EXCL_SYNC:
       case LOCK_LOCK_SYNC:
-	{ // bcast data to replicas
+      case LOCK_MIX_SYNC:
+	if (in->is_replicated()) {
 	  bufferlist softdata;
 	  lock->encode_locked_state(softdata);
 	  send_lock_message(lock, LOCK_AC_SYNC, softdata);
@@ -673,10 +675,7 @@ bool Locker::wrlock_start(SimpleLock *lock, MDRequest *mut, bool nowait)
 
     if (in->is_auth()) {
       if (want_scatter) {
-	if (lock->sm == &sm_filelock)
-	  file_mixed((ScatterLock*)lock);
-	else 
-	  scatter_scatter((ScatterLock*)lock, nowait);
+	file_mixed((ScatterLock*)lock);
       } else 
 	simple_lock(lock);
 
@@ -1960,15 +1959,15 @@ void Locker::handle_lock(MLock *m)
     handle_simple_lock(lock, m);
     break;
     
+  case CEPH_LOCK_IDFT:
+  case CEPH_LOCK_INEST:
+    //handle_scatter_lock((ScatterLock*)lock, m);
+    //break;
+
   case CEPH_LOCK_IFILE:
     handle_file_lock((ScatterLock*)lock, m);
     break;
     
-  case CEPH_LOCK_IDFT:
-  case CEPH_LOCK_INEST:
-    handle_scatter_lock((ScatterLock*)lock, m);
-    break;
-
   default:
     dout(7) << "handle_lock got otype " << m->get_lock_type() << dendl;
     assert(0);
@@ -2460,11 +2459,12 @@ void Locker::scatter_eval(ScatterLock *lock)
 
   if (lock->get_parent()->is_frozen()) return;
   
-  if (lock->get_type() == CEPH_LOCK_INEST &&
-      !lock->is_rdlocked() &&
-      lock->get_state() != LOCK_MIX) {
-    scatter_scatter(lock);
-    return;
+  if (lock->get_type() == CEPH_LOCK_INEST) {
+    // in general, we want to keep INEST scattered at all times.
+    if (!lock->is_rdlocked() &&
+	lock->get_state() != LOCK_MIX)
+      file_mixed(lock);
+      return;
   }
 
   CInode *in = (CInode*)lock->get_parent();
@@ -2545,7 +2545,7 @@ void Locker::scatter_nudge(ScatterLock *lock, Context *c)
       case CEPH_LOCK_IDFT:
       case CEPH_LOCK_INEST:
 	if (p->is_replicated() && lock->get_state() != LOCK_MIX)
-	  scatter_scatter(lock);
+	  file_mixed(lock);
 	else // if (lock->get_state() != LOCK_LOCK)
 	  simple_lock(lock);
 	//else
@@ -2598,73 +2598,6 @@ void Locker::scatter_tick()
 }
 
 
-
-bool Locker::scatter_scatter_fastpath(ScatterLock *lock)
-{
-  assert(lock->get_parent()->is_auth());
-  assert(lock->is_stable());
-
-  if (lock->get_state() == LOCK_MIX)
-    return true;
-  if (!lock->is_rdlocked() &&
-      !lock->is_xlocked() &&
-      !lock->get_num_client_lease() &&
-      (!lock->get_parent()->is_replicated() ||  // if sync
-       lock->get_state() == LOCK_LOCK ||
-       lock->get_state() == LOCK_TSYN)) {
-    dout(10) << "scatter_scatter_fastpath YES " << *lock
-	     << " on " << *lock->get_parent() << dendl;
-    // do scatter
-    lock->set_last_scatter(g_clock.now());
-
-    if (lock->get_parent()->is_replicated()) {
-      // encode and bcast
-      bufferlist data;
-      lock->encode_locked_state(data);
-      send_lock_message(lock, LOCK_AC_MIX, data);
-    } 
-    
-    ((CInode *)lock->get_parent())->try_drop_loner();
-
-    lock->set_state(LOCK_MIX);
-    lock->finish_waiters(ScatterLock::WAIT_WR|ScatterLock::WAIT_STABLE);
-    return true;
-  }
-  dout(20) << "scatter_scatter_fastpath NO " << *lock
-	   << " on " << *lock->get_parent() << dendl;
-  return false;
-}
-
-void Locker::scatter_scatter(ScatterLock *lock, bool nowait)
-{
-  dout(10) << "scatter_scatter " << *lock
-	   << " on " << *lock->get_parent() << dendl;
-  assert(lock->get_parent()->is_auth());
-  assert(lock->is_stable());
-  
-  if (scatter_scatter_fastpath(lock) || nowait)
-    return;
-
-  if (lock->is_xlocked())
-    return;  // do nothing.
-
-  switch (lock->get_state()) {
-  case LOCK_SYNC: lock->set_state(LOCK_SYNC_MIX); break;
-  case LOCK_TSYN: lock->set_state(LOCK_TSYN_MIX); break;
-  default: assert(0);
-  }
-
-  lock->get_parent()->auth_pin(lock);
-
-  if (lock->get_parent()->is_replicated()) {
-    send_lock_message(lock, LOCK_AC_LOCK);
-    lock->init_gather();
-  }
-  if (lock->get_num_client_lease())
-    revoke_client_leases(lock);
-}
-
-
 void Locker::scatter_tempsync(ScatterLock *lock)
 {
   dout(10) << "scatter_tempsync " << *lock
@@ -2715,145 +2648,6 @@ void Locker::scatter_tempsync(ScatterLock *lock)
 
 
 
-void Locker::handle_scatter_lock(ScatterLock *lock, MLock *m)
-{
-  int from = m->get_asker();
-  dout(10) << "handle_scatter_lock " << *m << " on " << *lock << " on " << *lock->get_parent() << dendl;
-  
-  if (mds->is_rejoin()) {
-    if (lock->get_parent()->is_rejoining()) {
-      dout(7) << "handle_scatter_lock still rejoining " << *lock->get_parent()
-	      << ", dropping " << *m << dendl;
-      delete m;
-      return;
-    }
-  }
-
-  switch (m->get_action()) {
-    // -- replica --
-  case LOCK_AC_SYNC:
-    assert(lock->get_state() == LOCK_LOCK);
-    lock->set_state(LOCK_SYNC);
-    lock->decode_locked_state(m->get_data());
-    lock->clear_updated();
-    lock->finish_waiters(ScatterLock::WAIT_RD|ScatterLock::WAIT_STABLE);
-    break;
-
-  case LOCK_AC_LOCK:
-    assert(lock->get_state() == LOCK_MIX ||
-	   lock->get_state() == LOCK_SYNC);
-
-    // wait for wrlocks to close?
-    if (lock->is_wrlocked()) {
-      assert(lock->get_state() == LOCK_MIX);
-      dout(7) << "handle_scatter_lock has wrlocks, waiting on " << *lock
-	      << " on " << *lock->get_parent() << dendl;
-      lock->set_state(LOCK_MIX_LOCK);
-    } else if (lock->is_rdlocked() ||
-	       lock->get_num_client_lease()) {
-      assert(lock->get_state() == LOCK_SYNC);
-      dout(7) << "handle_scatter_lock has rdlocks|leases, waiting on " << *lock
-	      << " on " << *lock->get_parent() << dendl;
-      revoke_client_leases(lock);
-      lock->set_state(LOCK_SYNC_LOCK);
-    } else {
-      dout(7) << "handle_scatter_lock has no rd|wrlocks|leases, sending lockack for " << *lock
-	      << " on " << *lock->get_parent() << dendl;
-      
-      // encode and reply
-      bufferlist data;
-      lock->encode_locked_state(data);
-      mds->send_message_mds(new MLock(lock, LOCK_AC_LOCKACK, mds->get_nodeid(), data), from);
-      lock->set_state(LOCK_LOCK);
-    }
-    break;
-
-  case LOCK_AC_MIX:
-    assert(lock->get_state() == LOCK_LOCK);
-    lock->decode_locked_state(m->get_data());
-    lock->clear_updated();
-    lock->set_state(LOCK_MIX);
-    lock->finish_waiters(ScatterLock::WAIT_WR|ScatterLock::WAIT_STABLE);
-    break;
-
-    // -- for auth --
-  case LOCK_AC_LOCKACK:
-    assert(lock->get_state() == LOCK_SYNC_LOCK ||
-	   lock->get_state() == LOCK_MIX_LOCK ||
-	   lock->get_state() == LOCK_SYNC_MIX ||
-	   lock->get_state() == LOCK_MIX_TSYN);
-    assert(lock->is_gathering(from));
-    lock->remove_gather(from);
-    lock->decode_locked_state(m->get_data());
-    
-    if (lock->is_gathering()) {
-      dout(7) << "handle_scatter_lock " << *lock << " on " << *lock->get_parent()
-	      << " from " << from << ", still gathering " << lock->get_gather_set()
-	      << dendl;
-    } else {
-      dout(7) << "handle_scatter_lock " << *lock << " on " << *lock->get_parent()
-	      << " from " << from << ", last one" 
-	      << dendl;
-      eval_gather(lock);
-    }
-    break;
-
-  case LOCK_AC_REQSCATTER:
-    if (lock->is_stable()) {
-      /* NOTE: we can do this _even_ if !can_auth_pin (i.e. freezing)
-       *  because the replica should be holding an auth_pin if they're
-       *  doing this (and thus, we are freezing, not frozen, and indefinite
-       *  starvation isn't an issue).
-       */
-      dout(7) << "handle_scatter_lock got scatter request on " << *lock
-	      << " on " << *lock->get_parent() << dendl;
-      scatter_scatter(lock);
-    } else {
-      dout(7) << "handle_scatter_lock ignoring scatter request on " << *lock
-	      << " on " << *lock->get_parent() << dendl;
-    }
-    break;
-
-    /*
-  case LOCK_AC_REQUNMIX:
-    if (!lock->is_stable()) {
-      dout(7) << "handle_scatter_lock ignoring now-unnecessary unscatter request on " << *lock
-	      << " on " << *lock->get_parent() << dendl;
-    } else if (lock->get_parent()->can_auth_pin()) {
-      dout(7) << "handle_scatter_lock got unscatter request on " << *lock
-	      << " on " << *lock->get_parent() << dendl;
-      scatter_lock(lock);
-    } else {
-      dout(7) << "handle_scatter_lock DROPPING unscatter request on " << *lock
-	      << " on " << *lock->get_parent() << dendl;
-      // FIXME: if we can't auth_pin here, this request is effectively lost... 
-    }
-    break;
-    */
-
-  case LOCK_AC_NUDGE:
-    if (lock->get_parent()->is_auth()) {
-      dout(7) << "handle_scatter_lock trying nudge on " << *lock
-	      << " on " << *lock->get_parent() << dendl;
-      scatter_nudge(lock, 0);
-    } else {
-      dout(7) << "handle_scatter_lock IGNORING nudge on non-auth " << *lock
-	      << " on " << *lock->get_parent() << dendl;
-    }    
-    break;
-
- default: 
-   assert(0);
-  }
-
-  delete m;
-}
-
-
-
-
-
-
 // ==========================================================================
 // local lock
 
@@ -3085,6 +2879,7 @@ void Locker::file_mixed(ScatterLock *lock)
     switch (lock->get_state()) {
     case LOCK_SYNC: lock->set_state(LOCK_SYNC_MIX); break;
     case LOCK_EXCL: lock->set_state(LOCK_EXCL_MIX); break;
+    case LOCK_TSYN: lock->set_state(LOCK_TSYN_MIX); break;
     default: assert(0);
     }
 
@@ -3100,12 +2895,14 @@ void Locker::file_mixed(ScatterLock *lock)
       revoke_client_leases(lock);
       gather++;
     }
-    int loner_issued, other_issued;
-    in->get_caps_issued(&loner_issued, &other_issued, CEPH_CAP_SFILE);
-    if ((loner_issued & ~lock->gcaps_allowed(true)) ||
-	(other_issued & ~lock->gcaps_allowed(false))) {
-      issue_caps(in);
-      gather++;
+    if (lock->get_cap_shift()) {
+      int loner_issued, other_issued;
+      in->get_caps_issued(&loner_issued, &other_issued, lock->get_cap_shift());
+      if ((loner_issued & ~lock->gcaps_allowed(true)) ||
+	  (other_issued & ~lock->gcaps_allowed(false))) {
+	issue_caps(in);
+	gather++;
+      }
     }
     if (in->state_test(CInode::STATE_NEEDSRECOVER)) {
       mds->mdcache->queue_file_recover(in);
@@ -3118,7 +2915,8 @@ void Locker::file_mixed(ScatterLock *lock)
     else {
       in->try_drop_loner();
       lock->set_state(LOCK_MIX);
-      issue_caps(in);
+      if (lock->get_cap_shift())
+	issue_caps(in);
     }
   }
 }
@@ -3192,11 +2990,12 @@ void Locker::handle_file_lock(ScatterLock *lock, MLock *m)
     }
   }
 
-
   dout(7) << "handle_file_lock a=" << get_lock_action_name(m->get_action())
 	  << " on " << *lock
 	  << " from mds" << from << " " 
 	  << *in << dendl;
+
+  bool caps = lock->get_cap_shift();
   
   switch (m->get_action()) {
     // -- replica --
@@ -3238,9 +3037,10 @@ void Locker::handle_file_lock(ScatterLock *lock, MLock *m)
     
     // call back caps?
     int loner_issued, other_issued;
-    in->get_caps_issued(&loner_issued, &other_issued, CEPH_CAP_SFILE);
-    if ((loner_issued & ~lock->gcaps_allowed(true)) ||
-	(other_issued & ~lock->gcaps_allowed(false))) {
+    if (caps)
+      in->get_caps_issued(&loner_issued, &other_issued, CEPH_CAP_SFILE);
+    if (caps && ((loner_issued & ~lock->gcaps_allowed(true)) ||
+		 (other_issued & ~lock->gcaps_allowed(false)))) {
       dout(7) << "handle_file_lock client readers, gathering caps on " << *in << dendl;
       issue_caps(in);
       break;
@@ -3290,7 +3090,8 @@ void Locker::handle_file_lock(ScatterLock *lock, MLock *m)
       // no ack needed.
     }
 
-    issue_caps(in);
+    if (caps)
+      issue_caps(in);
     
     // waiters
     lock->finish_waiters(SimpleLock::WAIT_WR|SimpleLock::WAIT_STABLE);
@@ -3304,37 +3105,39 @@ void Locker::handle_file_lock(ScatterLock *lock, MLock *m)
     assert(lock->get_state() == LOCK_SYNC_LOCK ||
            lock->get_state() == LOCK_MIX_LOCK ||
            lock->get_state() == LOCK_MIX_EXCL ||
-           lock->get_state() == LOCK_SYNC_EXCL);
+           lock->get_state() == LOCK_SYNC_EXCL ||
+           lock->get_state() == LOCK_SYNC_MIX ||
+	   lock->get_state() == LOCK_MIX_TSYN);
     assert(lock->is_gathering(from));
     lock->remove_gather(from);
     
     if (lock->get_state() == LOCK_MIX_LOCK ||
-	lock->get_state() == LOCK_MIX_EXCL)
+	lock->get_state() == LOCK_MIX_EXCL ||
+	lock->get_state() == LOCK_MIX_TSYN)
       lock->decode_locked_state(m->get_data());
 
     if (lock->is_gathering()) {
-      dout(7) << "handle_lock_inode_file " << *in << " from " << from
+      dout(7) << "handle_file_lock " << *in << " from " << from
 	      << ", still gathering " << lock->get_gather_set() << dendl;
     } else {
-      dout(7) << "handle_lock_inode_file " << *in << " from " << from
+      dout(7) << "handle_file_lock " << *in << " from " << from
 	      << ", last one" << dendl;
       eval_gather(lock);
     }
     break;
     
   case LOCK_AC_SYNCACK:
-    assert(lock->get_state() == LOCK_MIX_SYNC ||
-	   lock->get_state() == LOCK_MIX_SYNC2);
+    assert(lock->get_state() == LOCK_MIX_SYNC);
     assert(lock->is_gathering(from));
     lock->remove_gather(from);
     
     lock->decode_locked_state(m->get_data());
 
     if (lock->is_gathering()) {
-      dout(7) << "handle_lock_inode_file " << *in << " from " << from
+      dout(7) << "handle_file_lock " << *in << " from " << from
 	      << ", still gathering " << lock->get_gather_set() << dendl;
     } else {
-      dout(7) << "handle_lock_inode_file " << *in << " from " << from
+      dout(7) << "handle_file_lock " << *in << " from " << from
 	      << ", last one" << dendl;
       eval_gather(lock);
     }
@@ -3346,10 +3149,10 @@ void Locker::handle_file_lock(ScatterLock *lock, MLock *m)
     lock->remove_gather(from);
     
     if (lock->is_gathering()) {
-      dout(7) << "handle_lock_inode_file " << *in << " from " << from
+      dout(7) << "handle_file_lock " << *in << " from " << from
 	      << ", still gathering " << lock->get_gather_set() << dendl;
     } else {
-      dout(7) << "handle_lock_inode_file " << *in << " from " << from
+      dout(7) << "handle_file_lock " << *in << " from " << from
 	      << ", last one" << dendl;
       eval_gather(lock);
     }
diff --git a/src/mds/locks.c b/src/mds/locks.c
index 929e7bce08dd..cc7789ecd6b3 100644
--- a/src/mds/locks.c
+++ b/src/mds/locks.c
@@ -65,6 +65,7 @@ struct sm_state_t scatterlock[30] = {
     [LOCK_MIX]       = { 0,         false, LOCK_MIX,  0,    0,   FW,  ANY, 0,   0,   0,0,0 },
     [LOCK_TSYN_MIX]  = { LOCK_MIX,  false, LOCK_LOCK, 0,    0,   0,   0,   0,   0,   0,0,0 },
     [LOCK_SYNC_MIX]  = { LOCK_MIX,  false, LOCK_LOCK, 0,    0,   0,   0,   0,   0,   0,0,0 },
+    [LOCK_SYNC_MIX2] = { LOCK_MIX,  false, 0,         0,    0,   0,   0,   0,   0,   0,0,0 },
 };
 
 struct sm_t sm_scatterlock = {