* improved behavior of locker versus migrations and recovery

git-svn-id: https://ceph.svn.sf.net/svnroot/ceph@1371 29311d96-e01e-0410-9327-a35deaab8ce9

diff --git a/branches/sage/cephmds2/TODO b/branches/sage/cephmds2/TODO
index d995f21549e4483186963efda537e3f129ddad34..e7717687cf222151daa8fbe3131f56614174817e 100644 (file)
--- a/branches/sage/cephmds2/TODO
+++ b/branches/sage/cephmds2/TODO
@@ -27,6 +27,10 @@ mds
    - auth -> replica :
 ???
 
+ - failure safe (above)
+ - export safe (ambiguous auth...)
+
+
 - trim_on_rejoin
 
 - more testing of failures + thrashing.

diff --git a/branches/sage/cephmds2/mds/Locker.cc b/branches/sage/cephmds2/mds/Locker.cc
index 3b48d9f3c933b00cee8ed04ef6410f293d27cbca..c9cfb2e6b964012f5806595e4fd7e9c3a9fe7d11 100644 (file)
--- a/branches/sage/cephmds2/mds/Locker.cc
+++ b/branches/sage/cephmds2/mds/Locker.cc
@@ -89,6 +89,8 @@ void Locker::send_lock_message(SimpleLock *lock, int msg)
   for (map<int,int>::iterator it = lock->get_parent()->replicas_begin(); 
        it != lock->get_parent()->replicas_end(); 
        it++) {
+    if (mds->mdsmap->get_state(it->first) < MDSMap::STATE_REJOIN) 
+      continue;
     MLock *m = new MLock(lock, msg, mds->get_nodeid());
     mds->send_message_mds(m, it->first, MDS_PORT_LOCKER);
   }
@@ -99,6 +101,8 @@ void Locker::send_lock_message(SimpleLock *lock, int msg, bufferlist &data)
   for (map<int,int>::iterator it = lock->get_parent()->replicas_begin(); 
        it != lock->get_parent()->replicas_end(); 
        it++) {
+    if (mds->mdsmap->get_state(it->first) < MDSMap::STATE_REJOIN) 
+      continue;
     MLock *m = new MLock(lock, msg, mds->get_nodeid());
     m->set_data(data);
     mds->send_message_mds(m, it->first, MDS_PORT_LOCKER);
@@ -499,9 +503,11 @@ void Locker::request_inode_file_caps(CInode *in)
     assert(!in->is_auth());
 
     in->replica_caps_wanted = wanted;
-    mds->send_message_mds(new MInodeFileCaps(in->ino(), mds->get_nodeid(),
-                                            in->replica_caps_wanted),
-                         auth, MDS_PORT_LOCKER);
+
+    if (mds->mdsmap->get_state(auth) >= MDSMap::STATE_REJOIN)
+      mds->send_message_mds(new MInodeFileCaps(in->ino(), mds->get_nodeid(),
+                                              in->replica_caps_wanted),
+                           auth, MDS_PORT_LOCKER);
   } else {
     in->replica_caps_wanted_keep_until.sec_ref() = 0;
   }
@@ -510,12 +516,20 @@ void Locker::request_inode_file_caps(CInode *in)
 void Locker::handle_inode_file_caps(MInodeFileCaps *m)
 {
   // nobody should be talking to us during recovery.
-  assert(mds->is_active() || mds->is_stopping());
+  assert(mds->is_rejoin() || mds->is_active() || mds->is_stopping());
 
   // ok
   CInode *in = mdcache->get_inode(m->get_ino());
   assert(in);
   assert(in->is_auth());
+
+  if (mds->is_rejoin() &&
+      in->is_rejoining()) {
+    dout(7) << "handle_inode_file_caps still rejoining " << *in << ", dropping " << *m << endl;
+    delete m;
+    return;
+  }
+
   
   dout(7) << "handle_inode_file_caps replica mds" << m->get_from() << " wants caps " << cap_string(m->get_caps()) << " on " << *in << endl;
 
@@ -701,7 +715,7 @@ ALSO:
 void Locker::handle_lock(MLock *m)
 {
   // nobody should be talking to us during recovery.
-  assert(mds->is_active() || mds->is_stopping());
+  assert(mds->is_rejoin() || mds->is_active() || mds->is_stopping());
 
   switch (m->get_otype()) {
   case LOCK_OTYPE_DN:
@@ -778,6 +792,15 @@ void Locker::handle_simple_lock(SimpleLock *lock, MLock *m)
 {
   int from = m->get_asker();
   
+  if (mds->is_rejoin()) {
+    if (lock->get_parent()->is_rejoining()) {
+      dout(7) << "handle_simple_lock still rejoining " << *lock->get_parent()
+             << ", dropping " << *m << endl;
+      delete m;
+      return;
+    }
+  }
+
   switch (m->get_action()) {
     // -- replica --
   case LOCK_AC_SYNC:
@@ -796,15 +819,12 @@ void Locker::handle_simple_lock(SimpleLock *lock, MLock *m)
       dout(7) << "handle_simple_lock has reader, waiting before ack on " << *lock
              << " on " << *lock->get_parent() << endl;
       lock->set_state(LOCK_GLOCKR);
-      lock->add_waiter(SimpleLock::WAIT_NOLOCKS, new C_MDS_RetryMessage(mds, m));
-      return;
+    } else {
+      // update lock and reply
+      lock->set_state(LOCK_LOCK);
+      mds->send_message_mds(new MLock(lock, LOCK_AC_LOCKACK, mds->get_nodeid()), 
+                           from, MDS_PORT_LOCKER);
     }
-
-    // update lock and reply
-    lock->set_state(LOCK_LOCK);
-      
-    mds->send_message_mds(new MLock(lock, LOCK_AC_LOCKACK, mds->get_nodeid()), 
-                         from, MDS_PORT_LOCKER);
     break;
 
 
@@ -815,6 +835,7 @@ void Locker::handle_simple_lock(SimpleLock *lock, MLock *m)
       MDRequest *mdr = mdcache->request_get(m->get_reqid());
       mdr->xlocks.insert(lock);
       mdr->locks.insert(lock);
+      lock->set_state(LOCK_REMOTEXLOCK);
       lock->finish_waiters(SimpleLock::WAIT_REMOTEXLOCK);
     }
     break;
@@ -879,38 +900,66 @@ void Locker::handle_simple_lock(SimpleLock *lock, MLock *m)
 }
 
 
+class C_Locker_SimpleEval : public Context {
+  Locker *locker;
+  SimpleLock *lock;
+public:
+  C_Locker_SimpleEval(Locker *l, SimpleLock *lk) : locker(l), lock(lk) {}
+  void finish(int r) {
+    locker->simple_eval(lock);
+  }
+};
+
 void Locker::simple_eval(SimpleLock *lock)
 {
-  // finished gather?
-  if (lock->get_parent()->is_auth() &&
-      !lock->is_stable() &&
-      !lock->is_gathering()) {
+  // unstable and ambiguous auth?
+  if (!lock->is_stable() &&
+      lock->get_parent()->is_ambiguous_auth()) {
+    dout(7) << "simple_eval not stable and ambiguous auth, waiting on " << *lock->get_parent() << endl;
+    //if (!lock->get_parent()->is_waiter(MDSCacheObject::WAIT_SINGLEAUTH))
+    lock->get_parent()->add_waiter(MDSCacheObject::WAIT_SINGLEAUTH, new C_Locker_SimpleEval(this, lock));
+    return;
+  }
+
+  // finished remote xlock?
+  if (lock->get_state() == LOCK_REMOTEXLOCK &&
+      !lock->is_xlocked()) {
+    // tell auth
+    assert(!lock->get_parent()->is_auth()); // should be auth_pinned on the auth
+    dout(7) << "simple_eval releasing remote xlock on " << *lock->get_parent()  << endl;
+    int auth = lock->get_parent()->authority().first;
+    if (mds->mdsmap->get_state(auth) >= MDSMap::STATE_REJOIN)
+      mds->send_message_mds(new MLock(lock, LOCK_AC_UNXLOCK, mds->get_nodeid()),
+                           auth, MDS_PORT_LOCKER);
+    lock->set_state(LOCK_LOCK);
+  }
+
+  // finished gathering?
+  if (lock->get_state() == LOCK_GLOCKR &&
+      !lock->is_gathering() &&
+      !lock->is_rdlocked()) {
     dout(7) << "simple_eval finished gather on " << *lock << " on " << *lock->get_parent() << endl;
-    switch (lock->get_state()) {
-    case LOCK_GLOCKR:
-      lock->set_state(LOCK_LOCK);
-      lock->finish_waiters(SimpleLock::WAIT_STABLE);
-      break;
-      
-    default:
-      assert(0);
+
+    // replica: tell auth
+    if (!lock->get_parent()->is_auth()) {
+      int auth = lock->get_parent()->authority().first;
+      if (mds->mdsmap->get_state(auth) >= MDSMap::STATE_REJOIN) 
+       mds->send_message_mds(new MLock(lock, LOCK_AC_LOCKACK, mds->get_nodeid()), 
+                             lock->get_parent()->authority().first, MDS_PORT_LOCKER);
     }
-  }
-  if (!lock->is_stable()) return;
-  
-  if (lock->get_parent()->is_auth()) {
     
-    // sync?
-    if (lock->get_state() != LOCK_SYNC &&
-       lock->get_parent()->is_replicated() &&
-       !lock->is_waiter_for(SimpleLock::WAIT_WR)) {
-      dout(7) << "simple_eval stable, syncing " << *lock 
-             << " on " << *lock->get_parent() << endl;
-      simple_sync(lock);
-    }
+    lock->set_state(LOCK_LOCK);
+    lock->finish_waiters(SimpleLock::WAIT_STABLE);
+  }
 
-  } else {
-    // replica
+  // stable -> sync?
+  if (lock->get_parent()->is_auth() &&
+      lock->is_stable() &&
+      lock->get_state() != LOCK_SYNC &&
+      !lock->is_waiter_for(SimpleLock::WAIT_WR)) {
+    dout(7) << "simple_eval stable, syncing " << *lock 
+           << " on " << *lock->get_parent() << endl;
+    simple_sync(lock);
   }
   
 }
@@ -929,13 +978,16 @@ void Locker::simple_sync(SimpleLock *lock)
   if (lock->get_state() == LOCK_GLOCKR) 
     assert(0); // um... hmm!
   assert(lock->get_state() == LOCK_LOCK);
-  
-  // hard data
-  bufferlist data;
-  lock->encode_locked_state(data);
-  
-  // bcast to replicas
-  send_lock_message(lock, LOCK_AC_SYNC, data);
+
+  // sync.
+  if (lock->get_parent()->is_replicated()) {
+    // hard data
+    bufferlist data;
+    lock->encode_locked_state(data);
+    
+    // bcast to replicas
+    send_lock_message(lock, LOCK_AC_SYNC, data);
+  }
   
   // change lock
   lock->set_state(LOCK_SYNC);
@@ -1018,11 +1070,9 @@ void Locker::simple_rdlock_finish(SimpleLock *lock, MDRequest *mdr)
 
   dout(7) << "simple_rdlock_finish on " << *lock << " on " << *lock->get_parent() << endl;
   
-  if (lock->get_state() == LOCK_GLOCKR &&
-      !lock->is_rdlocked()) {
-    lock->set_state(LOCK_SYNC);    // return state to sync, in case the unpinner flails
-    lock->finish_waiters(SimpleLock::WAIT_NOLOCKS);
-  }
+  // last one?
+  if (!lock->is_rdlocked())
+    simple_eval(lock);
 }
 
 bool Locker::simple_xlock_start(SimpleLock *lock, MDRequest *mdr)
@@ -1069,6 +1119,7 @@ bool Locker::simple_xlock_start(SimpleLock *lock, MDRequest *mdr)
                                     new C_MDS_RetryRequest(mdcache, mdr));
       return false;
     }
+    int auth = lock->get_parent()->authority().first;
 
     // wait for sync.
     // (???????????)
@@ -1078,7 +1129,6 @@ bool Locker::simple_xlock_start(SimpleLock *lock, MDRequest *mdr)
     }
 
     // send lock request
-    int auth = lock->get_parent()->authority().first;
     MLock *m = new MLock(lock, LOCK_AC_REQXLOCK, mds->get_nodeid());
     mds->send_message_mds(m, auth, MDS_PORT_LOCKER);
   
@@ -1094,29 +1144,16 @@ void Locker::simple_xlock_finish(SimpleLock *lock, MDRequest *mdr)
   // drop ref
   assert(lock->can_xlock(mdr));
   lock->put_xlock();
+  assert(mdr);
   mdr->xlocks.erase(lock);
   mdr->locks.erase(lock);
   dout(7) << "simple_xlock_finish on " << *lock << " on " << *lock->get_parent() << endl;
 
-  // slave?
-  if (!lock->get_parent()->is_auth()) {
-    mds->send_message_mds(new MLock(lock, LOCK_AC_UNXLOCK, mds->get_nodeid()),
-                         lock->get_parent()->authority().first, MDS_PORT_LOCKER);
-  }
-
   // others waiting?
-  if (lock->is_waiter_for(SimpleLock::WAIT_WR)) {
-    // wake 'em up
-    lock->finish_waiters(SimpleLock::WAIT_WR, 0); 
-  } else {
-    // auto-sync if alone.
-    if (lock->get_parent()->is_auth() &&
-        !lock->get_parent()->is_replicated() &&
-        lock->get_state() != LOCK_SYNC) 
-      lock->set_state(LOCK_SYNC);
-    
-    simple_eval(lock);
-  }
+  lock->finish_waiters(SimpleLock::WAIT_WR, 0); 
+
+  // eval
+  simple_eval(lock);
 }
 
 
@@ -1264,19 +1301,43 @@ void Locker::scatter_wrlock_finish(ScatterLock *lock, MDRequest *mdr)
   scatter_eval(lock);
 }
 
+
+class C_Locker_ScatterEval : public Context {
+  Locker *locker;
+  ScatterLock *lock;
+public:
+  C_Locker_ScatterEval(Locker *l, ScatterLock *lk) : locker(l), lock(lk) {}
+  void finish(int r) {
+    locker->scatter_eval(lock);
+  }
+};
+
 void Locker::scatter_eval(ScatterLock *lock)
 {
+  // unstable and ambiguous auth?
+  if (!lock->is_stable() &&
+      lock->get_parent()->is_ambiguous_auth()) {
+    dout(7) << "scatter_eval not stable and ambiguous auth, waiting on " << *lock->get_parent() << endl;
+    //if (!lock->get_parent()->is_waiter(MDSCacheObject::WAIT_SINGLEAUTH))
+    lock->get_parent()->add_waiter(MDSCacheObject::WAIT_SINGLEAUTH, new C_Locker_ScatterEval(this, lock));
+    return;
+  }
+
   if (!lock->get_parent()->is_auth()) {
     // REPLICA
 
     if (lock->get_state() == LOCK_GSYNCS &&
        !lock->is_wrlocked()) {
       dout(10) << "scatter_eval no wrlocks, acking sync" << endl;
-      bufferlist data;
-      lock->encode_locked_state(data);
-      mds->send_message_mds(new MLock(lock, LOCK_AC_SYNCACK, mds->get_nodeid(), data),
-                           lock->get_parent()->authority().first, MDS_PORT_LOCKER);
+      int auth = lock->get_parent()->authority().first;
+      if (mds->mdsmap->get_state(auth) >= MDSMap::STATE_REJOIN) {
+       bufferlist data;
+       lock->encode_locked_state(data);
+       mds->send_message_mds(new MLock(lock, LOCK_AC_SYNCACK, mds->get_nodeid(), data),
+                             auth, MDS_PORT_LOCKER);
+      }
       lock->set_state(LOCK_SYNC);
+      lock->finish_waiters(ScatterLock::WAIT_STABLE);  // ?
     }
 
   } else {
@@ -1289,7 +1350,7 @@ void Locker::scatter_eval(ScatterLock *lock)
       dout(7) << "scatter_eval finished gather/un-wrlock on " << *lock
              << " on " << *lock->get_parent() << endl;
       lock->set_state(LOCK_SYNC);
-      lock->finish_waiters(SimpleLock::WAIT_STABLE|SimpleLock::WAIT_RD|SimpleLock::WAIT_NOLOCKS);
+      lock->finish_waiters(ScatterLock::WAIT_STABLE|ScatterLock::WAIT_RD);
     }
     
     // gscatters -> scatter?
@@ -1304,13 +1365,15 @@ void Locker::scatter_eval(ScatterLock *lock)
       } 
       
       lock->set_state(LOCK_SCATTER);
-      lock->finish_waiters(SimpleLock::WAIT_WR|SimpleLock::WAIT_STABLE);
+      lock->get_wrlock();
+      lock->finish_waiters(ScatterLock::WAIT_WR|ScatterLock::WAIT_STABLE);
+      lock->put_wrlock();
     }
     
     // waiting for rd?
     if (lock->get_state() == LOCK_SCATTER &&
        !lock->is_wrlocked() &&
-       lock->is_waiter_for(SimpleLock::WAIT_RD)) {
+       lock->is_waiter_for(ScatterLock::WAIT_RD)) {
       dout(10) << "scatter_eval no wrlocks, read waiter, syncing" << endl;
       scatter_sync(lock);
     }
@@ -1342,9 +1405,10 @@ void Locker::scatter_sync(ScatterLock *lock)
   } 
   else if (lock->is_wrlocked()) {
     lock->set_state(LOCK_GSYNCS);
-  } else {    
+  } 
+  else {    
     lock->set_state(LOCK_SYNC);
-    lock->finish_waiters(SimpleLock::WAIT_RD|SimpleLock::WAIT_STABLE);
+    lock->finish_waiters(ScatterLock::WAIT_RD|ScatterLock::WAIT_STABLE);
   }
 }
 
@@ -1368,7 +1432,7 @@ void Locker::scatter_scatter(ScatterLock *lock)
       send_lock_message(lock, LOCK_AC_SCATTER, data);
     } 
     lock->set_state(LOCK_SCATTER);
-    lock->finish_waiters(SimpleLock::WAIT_WR|SimpleLock::WAIT_STABLE);
+    lock->finish_waiters(ScatterLock::WAIT_WR|ScatterLock::WAIT_STABLE);
   }
 }
 
@@ -1378,6 +1442,15 @@ void Locker::handle_scatter_lock(ScatterLock *lock, MLock *m)
 {
   int from = m->get_asker();
   
+  if (mds->is_rejoin()) {
+    if (lock->get_parent()->is_rejoining()) {
+      dout(7) << "handle_scatter_lock still rejoining " << *lock->get_parent()
+             << ", dropping " << *m << endl;
+      delete m;
+      return;
+    }
+  }
+
   switch (m->get_action()) {
     // -- replica --
   case LOCK_AC_SYNC:
@@ -1401,7 +1474,7 @@ void Locker::handle_scatter_lock(ScatterLock *lock, MLock *m)
     assert(lock->get_state() == LOCK_SYNC);
     lock->decode_locked_state(m->get_data());
     lock->set_state(LOCK_SCATTER);
-    lock->finish_waiters(SimpleLock::WAIT_WR|SimpleLock::WAIT_STABLE);
+    lock->finish_waiters(ScatterLock::WAIT_WR|ScatterLock::WAIT_STABLE);
     break;
 
     // -- for auth --
@@ -1419,7 +1492,7 @@ void Locker::handle_scatter_lock(ScatterLock *lock, MLock *m)
       dout(7) << "handle_scatter_lock " << *lock << " on " << *lock->get_parent()
              << " from " << from << ", last one" 
              << endl;
-      simple_eval(lock);
+      scatter_eval(lock);
     }
     break;
   }
@@ -1511,10 +1584,8 @@ void Locker::file_rdlock_finish(FileLock *lock, MDRequest *mdr)
 
   dout(7) << "rdlock_finish on " << *lock << " on " << *lock->get_parent() << endl;
 
-  if (!lock->is_rdlocked()) {
-    lock->finish_waiters(SimpleLock::WAIT_NOLOCKS);
+  if (!lock->is_rdlocked()) 
     file_eval(lock);
-  }
 }
 
 
@@ -1571,10 +1642,13 @@ void Locker::file_xlock_finish(FileLock *lock, MDRequest *mdr)
   dout(7) << "file_xlock_finish on " << *lock << " on " << *lock->get_parent() << endl;
 
   assert(lock->get_parent()->is_auth());  // or implement remote xlocks
-  
-  // drop lock?
-  if (!lock->is_waiter_for(SimpleLock::WAIT_STABLE)) 
-    file_eval(lock);
+
+  // others waiting?
+  lock->finish_waiters(SimpleLock::WAIT_WR, 0); 
+
+  //// drop lock?
+  //if (!lock->is_waiter_for(SimpleLock::WAIT_STABLE)) 
+  file_eval(lock);
 }
 
 
@@ -1585,11 +1659,31 @@ void Locker::file_xlock_finish(FileLock *lock, MDRequest *mdr)
  * - checks if we're in unstable sfot state and can now move on to next state
  * - checks if soft state should change (eg bc last writer closed)
  */
+class C_Locker_FileEval : public Context {
+  Locker *locker;
+  FileLock *lock;
+public:
+  C_Locker_FileEval(Locker *l, FileLock *lk) : locker(l), lock(lk) {}
+  void finish(int r) {
+    locker->file_eval(lock);
+  }
+};
+
 
 void Locker::file_eval(FileLock *lock)
 {
   CInode *in = (CInode*)lock->get_parent();
 
+  // unstable and ambiguous auth?
+  if (!lock->is_stable() &&
+      in->is_ambiguous_auth()) {
+    dout(7) << "file_eval not stable and ambiguous auth, waiting on " << *in << endl;
+    //if (!lock->get_parent()->is_waiter(MDSCacheObject::WAIT_SINGLEAUTH))
+    in->add_waiter(CInode::WAIT_SINGLEAUTH, new C_Locker_FileEval(this, lock));
+    return;
+  }
+
+
   int issued = in->get_caps_issued();
 
   // [auth] finished gather?
@@ -2075,6 +2169,16 @@ void Locker::handle_file_lock(FileLock *lock, MLock *m)
   CInode *in = (CInode*)lock->get_parent();
   int from = m->get_asker();
 
+  if (mds->is_rejoin()) {
+    if (in->is_rejoining()) {
+      dout(7) << "handle_file_lock still rejoining " << *in
+             << ", dropping " << *m << endl;
+      delete m;
+      return;
+    }
+  }
+
+
   dout(7) << "handle_file_lock a=" << m->get_action() << " from " << from << " " 
          << *in << " filelock=" << *lock << endl;  
   
@@ -2109,16 +2213,15 @@ void Locker::handle_file_lock(FileLock *lock, MLock *m)
     }
     if (lock->is_rdlocked()) {
       dout(7) << "handle_file_lock rdlocked, waiting before ack on " << *in << endl;
-      in->add_waiter(SimpleLock::WAIT_NOLOCKS, new C_MDS_RetryMessage(mds, m));
       lock->set_state(LOCK_GLOCKR);
-      assert(0);// i am broken.. why retry message when state captures all the info i need?
-      return;
+      break;
     } 
     if (issued & CAP_FILE_RD) {
+      dout(7) << "handle_file_lock RD cap issued, waiting before ack on " << *in << endl;
       lock->set_state(LOCK_GLOCKR);
       break;
     }
-
+    
     // nothing to wait for, lock and ack.
     {
       lock->set_state(LOCK_LOCK);

diff --git a/branches/sage/cephmds2/mds/MDCache.cc b/branches/sage/cephmds2/mds/MDCache.cc
index e96fdcca7fd798de0036a69597a32af8296da2bd..838ff5c1add1d85a03aee810e084f4073c09a396 100644 (file)
--- a/branches/sage/cephmds2/mds/MDCache.cc
+++ b/branches/sage/cephmds2/mds/MDCache.cc
@@ -1286,6 +1286,7 @@ void MDCache::recalc_auth_bits()
       if (auth) 
        dir->state_set(CDir::STATE_AUTH);
       else {
+       dir->state_set(CDir::STATE_REJOINING);
        dir->state_clear(CDir::STATE_AUTH);
        if (dir->is_dirty()) 
          dir->mark_clean();
@@ -1300,6 +1301,7 @@ void MDCache::recalc_auth_bits()
        if (auth)
          dn->state_set(CDentry::STATE_AUTH);
        else {
+         dn->state_set(CDentry::STATE_REJOINING);
          dn->state_clear(CDentry::STATE_AUTH);
          if (dn->is_dirty()) 
            dn->mark_clean();
@@ -1310,6 +1312,7 @@ void MDCache::recalc_auth_bits()
          if (auth) 
            dn->inode->state_set(CInode::STATE_AUTH);
          else {
+           dn->inode->state_set(CInode::STATE_REJOINING);
            dn->inode->state_clear(CInode::STATE_AUTH);
            if (dn->inode->is_dirty())
              dn->inode->mark_clean();
@@ -1744,6 +1747,7 @@ void MDCache::handle_cache_rejoin_ack(MMDSCacheRejoin *m)
     assert(dir);
 
     dir->set_replica_nonce(p->second.nonce);
+    dir->state_clear(CDir::STATE_REJOINING);
     dout(10) << " got " << *dir << endl;
 
     // dentries
@@ -1754,6 +1758,7 @@ void MDCache::handle_cache_rejoin_ack(MMDSCacheRejoin *m)
       assert(dn);
       dn->set_replica_nonce(q->second.nonce);
       dn->lock.set_state(q->second.lock);
+      dn->state_clear(CDentry::STATE_REJOINING);
       dout(10) << " got " << *dn << endl;
     }
   }
@@ -1770,6 +1775,7 @@ void MDCache::handle_cache_rejoin_ack(MMDSCacheRejoin *m)
     in->dirfragtreelock.set_state(p->second.dirfragtreelock);
     in->filelock.set_state(p->second.filelock);
     in->dirlock.set_state(p->second.dirlock);
+    in->state_clear(CInode::STATE_REJOINING);
     dout(10) << " got " << *in << endl;
   }
 

diff --git a/branches/sage/cephmds2/mds/SimpleLock.h b/branches/sage/cephmds2/mds/SimpleLock.h
index 574961e9fb475cf4c0079ea5554ef1e932060fe4..163e84f7a17f07ab2032049ecbf60cc4ec15e65a 100644 (file)
--- a/branches/sage/cephmds2/mds/SimpleLock.h
+++ b/branches/sage/cephmds2/mds/SimpleLock.h
@@ -41,10 +41,11 @@ inline const char *get_lock_type_name(int t) {
 
 // -- lock states --
 #define LOCK_UNDEF    0
-//                                auth   rep
+//                               auth   rep
 #define LOCK_SYNC     1  // AR   R .    R .
 #define LOCK_LOCK     2  // AR   R W    . .
 #define LOCK_GLOCKR  -3  // AR   R .    . .
+#define LOCK_REMOTEXLOCK  -50    // on NON-auth
 
 inline const char *get_simplelock_state_name(int n) {
   switch (n) {
@@ -52,6 +53,7 @@ inline const char *get_simplelock_state_name(int n) {
   case LOCK_SYNC: return "sync";
   case LOCK_LOCK: return "lock";
   case LOCK_GLOCKR: return "glockr";
+  case LOCK_REMOTEXLOCK: return "remote_xlock";
   default: assert(0);
   }
 }
@@ -62,8 +64,7 @@ class SimpleLock {
 public:
   static const int WAIT_RD          = (1<<0);  // to read
   static const int WAIT_WR          = (1<<1);  // to write
-  static const int WAIT_NOLOCKS     = (1<<2);  // for last rdlock to finish
-  //static const int WAIT_LOCK        = (1<<3);  // for locked state
+  static const int WAIT_SINGLEAUTH  = (1<<2);
   static const int WAIT_STABLE      = (1<<3);  // for a stable state
   static const int WAIT_REMOTEXLOCK = (1<<4);  // for a remote xlock
   static const int WAIT_BITS        = 5;

diff --git a/branches/sage/cephmds2/mds/mdstypes.h b/branches/sage/cephmds2/mds/mdstypes.h
index 41b7f69e2e51bf641754c7517ac7acc485b2cd50..c2cce988e3f940acd1953847e5cb5f29ac97bffb 100644 (file)
--- a/branches/sage/cephmds2/mds/mdstypes.h
+++ b/branches/sage/cephmds2/mds/mdstypes.h
@@ -290,8 +290,9 @@ class MDSCacheObject {
   }
 
   // -- state --
-  const static int STATE_AUTH  = (1<<30);
-  const static int STATE_DIRTY = (1<<29);
+  const static int STATE_AUTH      = (1<<30);
+  const static int STATE_DIRTY     = (1<<29);
+  const static int STATE_REJOINING = (1<<28);  // replica has not joined w/ primary copy
 
   // -- wait --
   const static int WAIT_SINGLEAUTH = (1<<30);
@@ -325,8 +326,9 @@ class MDSCacheObject {
   void state_reset(unsigned s) { state = s; }
 
   bool is_auth() { return state_test(STATE_AUTH); }
-  bool is_dirty() { return state & STATE_DIRTY; }
+  bool is_dirty() { return state_test(STATE_DIRTY); }
   bool is_clean() { return !is_dirty(); }
+  bool is_rejoining() { return state_test(STATE_REJOINING); }
 
   // --------------------------------------------
   // authority