mds: send LOCKFLUSHED to trigger finish_flush on replicas

Since f741766a we have triggered start_flush and finish_flush on replicas.
The problem is that the finish_flush didn't always happen for the mix->lock
case: we sould start_flush when we sent the AC_LOCKACK, but could only
finish_flush if/when we got another SYNC or MIX.  If the primary stayed in
the LOCK state, we would keep our flushing flag.  That in turn causes
problems later when we try to eval_gather() (esp if we are auth at that
point?).

Fix this by sending an explicit AC_LOCKFLUSHED message to replicas after
we do a scatter_writebehind.  The replica will only set flushing if it
flushed dirty data, which forces scatter_writebehind, so we will always
get the LOCKFLUSHED to match.  Replicas that didn't flush will also get
it, but oh well.  We'd need to keep track which ones sent dirty data to
do that properly, though.

TODO: still need to verify that this is correct for rejoin.

Signed-off-by: Sage Weil <sage@newdream.net>

diff --git a/src/mds/Locker.cc b/src/mds/Locker.cc
index 1cddcf83f926724b7169e2e29a314adb145863e4..e1e22c36db79011774c7432ef1a2ac8e4102a25f 100644 (file)
--- a/src/mds/Locker.cc
+++ b/src/mds/Locker.cc
@@ -541,6 +541,7 @@ void Locker::eval_gather(SimpleLock *lock, bool first, bool *pneed_issue, list<C
            lock->encode_locked_state(data);
            mds->send_message_mds(new MLock(lock, LOCK_AC_LOCKACK, mds->get_nodeid(), data), auth);
            ((ScatterLock *)lock)->start_flush();
+           // we'll get an AC_LOCKFLUSHED to complete
          }
          break;
 
@@ -3312,6 +3313,17 @@ void Locker::scatter_writebehind_finish(ScatterLock *lock, Mutation *mut)
 
   lock->finish_flush();
 
+  // if replicas may have flushed in a mix->lock state, send another
+  // message so they can finish_flush().
+  if (in->is_replicated()) {
+    switch (lock->get_state()) {
+    case LOCK_MIX_LOCK:
+    case LOCK_MIX_EXCL:
+    case LOCK_MIX_TSYN:
+      send_lock_message(lock, LOCK_AC_LOCKFLUSHED);
+    }
+  }
+
   mut->apply();
   drop_locks(mut);
   mut->cleanup();
@@ -3933,12 +3945,12 @@ void Locker::handle_file_lock(ScatterLock *lock, MLock *m)
       break;
     }
 
+    ((ScatterLock *)lock)->finish_flush();
+
     // ok
     lock->decode_locked_state(m->get_data());
     lock->set_state(LOCK_SYNC);
 
-    ((ScatterLock *)lock)->finish_flush();
-
     lock->get_rdlock();
     lock->finish_waiters(SimpleLock::WAIT_RD|SimpleLock::WAIT_STABLE);
     lock->put_rdlock();
@@ -3953,6 +3965,10 @@ void Locker::handle_file_lock(ScatterLock *lock, MLock *m)
 
     eval_gather(lock, true);
     break;
+
+  case LOCK_AC_LOCKFLUSHED:
+    ((ScatterLock *)lock)->finish_flush();
+    break;
     
   case LOCK_AC_MIX:
     assert(lock->get_state() == LOCK_SYNC ||
@@ -3970,8 +3986,6 @@ void Locker::handle_file_lock(ScatterLock *lock, MLock *m)
     lock->decode_locked_state(m->get_data());
     lock->set_state(LOCK_MIX);
 
-    ((ScatterLock *)lock)->finish_flush();
-
     if (caps)
       issue_caps(in);
     

diff --git a/src/mds/locks.h b/src/mds/locks.h
index 881a4e8eb0af036658d076bba5d19ea6ae033d40..db55104b95d03727d459ba54d53af78826ba57ba 100644 (file)
--- a/src/mds/locks.h
+++ b/src/mds/locks.h
@@ -97,6 +97,7 @@ extern struct sm_t sm_locallock;
 #define LOCK_AC_SYNC        -1
 #define LOCK_AC_MIX         -2
 #define LOCK_AC_LOCK        -3
+#define LOCK_AC_LOCKFLUSHED -4
 
 // for auth
 #define LOCK_AC_SYNCACK      1
@@ -117,6 +118,7 @@ static inline const char *get_lock_action_name(int a) {
   case LOCK_AC_SYNC: return "sync";
   case LOCK_AC_MIX: return "mix";
   case LOCK_AC_LOCK: return "lock";
+  case LOCK_AC_LOCKFLUSHED: return "lockflushed";
 
   case LOCK_AC_SYNCACK: return "syncack";
   case LOCK_AC_MIXACK: return "mixack";