mds: rejoin snap support

diff --git a/src/mds/CDir.h b/src/mds/CDir.h
index bfa27e2cff575d90e8e39cd46a149b0bf54d9608..8f8b735dd7b32c8982d9de03e59bc39a73264827 100644 (file)
--- a/src/mds/CDir.h
+++ b/src/mds/CDir.h
@@ -43,20 +43,6 @@ class MDCache;
 class MDCluster;
 class Context;
 
-
-struct string_snap_t {
-  nstring name;
-  snapid_t snapid;
-  string_snap_t(const string& n, snapid_t s) : name(n), snapid(s) {}
-  string_snap_t(const nstring& n, snapid_t s) : name(n), snapid(s) {}
-  string_snap_t(const char *n, snapid_t s) : name(n), snapid(s) {}
-};
-inline bool operator<(const string_snap_t& l, const string_snap_t& r) {
-  int c = strcmp(l.name.c_str(), r.name.c_str());
-  return c < 0 || (c == 0 && l.snapid < r.snapid);
-}
-
-
 ostream& operator<<(ostream& out, class CDir& dir);
 
 

diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index 8b7b75b2dd7ed7a8b77708718cd7969c0c2d9ec3..fdc3755ec843d3c3b01480827278da3170cd828a 100644 (file)
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -531,18 +531,24 @@ void MDCache::eval_subtree_root(CDir *dir)
 {
   // evaluate subtree inode dirlock?
   //  (we should scatter the dirlock on subtree bounds)
-  if (dir->inode->is_auth() &&
-      dir->inode->dirlock.is_stable()) {
-    // force the issue a bit
-    if (!dir->inode->is_frozen()) {
-      mds->locker->scatter_eval(&dir->inode->dirlock);
-      mds->locker->scatter_eval(&dir->inode->nestlock);
-    } else {
-      mds->locker->try_scatter_eval(&dir->inode->dirlock);  // ** may or may not be auth_pinned **
-      mds->locker->try_scatter_eval(&dir->inode->nestlock);  // ** may or may not be auth_pinned **
+  if (dir->inode->is_auth()) {
+    if (dir->inode->dirlock.is_stable()) {
+      // force the issue a bit
+      if (!dir->inode->is_frozen())
+       mds->locker->scatter_eval(&dir->inode->dirlock);
+      else
+       mds->locker->try_scatter_eval(&dir->inode->dirlock);  // ** may or may not be auth_pinned **
+    }
+    if (dir->inode->nestlock.is_stable()) {
+      // force the issue a bit
+      if (!dir->inode->is_frozen())
+       mds->locker->scatter_eval(&dir->inode->nestlock);
+      else
+       mds->locker->try_scatter_eval(&dir->inode->nestlock);  // ** may or may not be auth_pinned **
     }
   }
   
+  
 }
 
 
@@ -2404,21 +2410,22 @@ void MDCache::rejoin_send_rejoins()
     if (mds->is_rejoin()) {
       // weak
       if (p->first == 0 && root) 
-       p->second->add_weak_inode(root->ino());
-      if (get_inode(MDS_INO_STRAY(p->first))) 
-       p->second->add_weak_inode(MDS_INO_STRAY(p->first));
+       p->second->add_weak_inode(root->vino());
+      CInode *s = get_inode(MDS_INO_STRAY(p->first)); 
+      if (s)
+       p->second->add_weak_inode(s->vino());
     } else {
       // strong
       if (p->first == 0 && root) {
-       p->second->add_weak_inode(root->ino());
-       p->second->add_strong_inode(root->ino(),
+       p->second->add_weak_inode(root->vino());
+       p->second->add_strong_inode(root->vino(),
                                    root->get_caps_wanted(),
                                    root->dirlock.get_state(),
                                    root->nestlock.get_state());
       }
       if (CInode *in = get_inode(MDS_INO_STRAY(p->first))) {
-       p->second->add_weak_inode(in->ino());
-       p->second->add_strong_inode(in->ino(),
+       p->second->add_weak_inode(in->vino());
+       p->second->add_strong_inode(in->vino(),
                                    in->get_caps_wanted(),
                                    in->dirlock.get_state(),
                                    in->nestlock.get_state());
@@ -2445,9 +2452,9 @@ void MDCache::rejoin_send_rejoins()
          MDSCacheObjectInfo i;
          (*q)->set_object_info(i);
          if (i.ino)
-           rejoin->add_inode_authpin(i.ino, p->second->reqid);
+           rejoin->add_inode_authpin(vinodeno_t(i.ino, i.snapid), p->second->reqid);
          else
-           rejoin->add_dentry_authpin(i.dirfrag, i.dname, p->second->reqid);
+           rejoin->add_dentry_authpin(i.dirfrag, i.dname, i.snapid, p->second->reqid);
        }
       }
       // xlocks
@@ -2463,9 +2470,9 @@ void MDCache::rejoin_send_rejoins()
          MDSCacheObjectInfo i;
          (*q)->get_parent()->set_object_info(i);
          if (i.ino)
-           rejoin->add_inode_xlock(i.ino, (*q)->get_type(), p->second->reqid);
+           rejoin->add_inode_xlock(vinodeno_t(i.ino, i.snapid), (*q)->get_type(), p->second->reqid);
          else
-           rejoin->add_dentry_xlock(i.dirfrag, i.dname, p->second->reqid);
+           rejoin->add_dentry_xlock(i.dirfrag, i.dname, i.snapid, p->second->reqid);
        }
       }
     }
@@ -2523,7 +2530,7 @@ void MDCache::rejoin_walk(CDir *dir, MMDSCacheRejoin *rejoin)
       dout(15) << " add_weak_primary_dentry " << *dn << dendl;
       assert(dn->is_primary());
       assert(dn->inode->is_dir());
-      rejoin->add_weak_primary_dentry(dir->dirfrag(), dn->name.c_str(), dn->get_inode()->ino());
+      rejoin->add_weak_primary_dentry(dir->dirfrag(), dn->name.c_str(), dn->first, dn->last, dn->get_inode()->ino());
       dn->get_inode()->get_nested_dirfrags(nested);
     }
   } else {
@@ -2536,7 +2543,7 @@ void MDCache::rejoin_walk(CDir *dir, MMDSCacheRejoin *rejoin)
         ++p) {
       CDentry *dn = p->second;
       dout(15) << " add_strong_dentry " << *dn << dendl;
-      rejoin->add_strong_dentry(dir->dirfrag(), dn->name, 
+      rejoin->add_strong_dentry(dir->dirfrag(), dn->name, dn->first, dn->last,
                                dn->is_primary() ? dn->get_inode()->ino():inodeno_t(0),
                                dn->is_remote() ? dn->get_remote_ino():inodeno_t(0),
                                dn->is_remote() ? dn->get_remote_d_type():0, 
@@ -2545,7 +2552,7 @@ void MDCache::rejoin_walk(CDir *dir, MMDSCacheRejoin *rejoin)
       if (dn->is_primary()) {
        CInode *in = dn->get_inode();
        dout(15) << " add_strong_inode " << *in << dendl;
-       rejoin->add_strong_inode(in->ino(),
+       rejoin->add_strong_inode(in->vino(),
                                 in->get_caps_wanted(),
                                 in->dirlock.get_state(),
                                 in->nestlock.get_state());
@@ -2690,7 +2697,7 @@ void MDCache::handle_cache_rejoin_weak(MMDSCacheRejoin *weak)
   }
   
   // walk weak map
-  for (map<dirfrag_t, map<nstring, MMDSCacheRejoin::dn_weak> >::iterator p = weak->weak.begin();
+  for (map<dirfrag_t, map<string_snap_t, MMDSCacheRejoin::dn_weak> >::iterator p = weak->weak.begin();
        p != weak->weak.end();
        ++p) {
     CDir *dir = get_dirfrag(p->first);
@@ -2703,10 +2710,10 @@ void MDCache::handle_cache_rejoin_weak(MMDSCacheRejoin *weak)
       ack->add_strong_dirfrag(p->first, nonce, dir->dir_rep);
     
     // weak dentries
-    for (map<nstring,MMDSCacheRejoin::dn_weak>::iterator q = p->second.begin();
+    for (map<string_snap_t,MMDSCacheRejoin::dn_weak>::iterator q = p->second.begin();
         q != p->second.end();
         ++q) {
-      CDentry *dn = dir->lookup(q->first);
+      CDentry *dn = dir->lookup(q->first.name, q->first.snapid);
       assert(dn);
       assert(dn->is_primary());
       
@@ -2715,7 +2722,7 @@ void MDCache::handle_cache_rejoin_weak(MMDSCacheRejoin *weak)
       int dnonce = dn->add_replica(from);
       dout(10) << " have " << *dn << dendl;
       if (ack) 
-       ack->add_strong_dentry(p->first, q->first,
+       ack->add_strong_dentry(p->first, dn->name, dn->first, dn->last,
                               dn->get_inode()->ino(), inodeno_t(0), 0, 
                               dnonce, dn->lock.get_replica_state());
 
@@ -2740,7 +2747,7 @@ void MDCache::handle_cache_rejoin_weak(MMDSCacheRejoin *weak)
   }
   
   // weak base inodes?  (root, stray, etc.)
-  for (set<inodeno_t>::iterator p = weak->weak_inodes.begin();
+  for (set<vinodeno_t>::iterator p = weak->weak_inodes.begin();
        p != weak->weak_inodes.end();
        ++p) {
     CInode *in = get_inode(*p);
@@ -2844,7 +2851,7 @@ void MDCache::rejoin_scour_survivor_replicas(int from, MMDSCacheRejoin *ack)
     // inode?
     if (in->is_auth() &&
        in->is_replica(from) &&
-       ack->strong_inodes.count(p->second->ino()) == 0) {
+       ack->strong_inodes.count(p->second->vino()) == 0) {
       inode_remove_replica(in, from);
       dout(10) << " rem " << *in << dendl;
     }
@@ -2873,7 +2880,7 @@ void MDCache::rejoin_scour_survivor_replicas(int from, MMDSCacheRejoin *ack)
        
        if (dn->is_replica(from) &&
            (ack->strong_dentries.count(dir->dirfrag()) == 0 ||
-            ack->strong_dentries[dir->dirfrag()].count(dn->get_name()) == 0)) {
+            ack->strong_dentries[dir->dirfrag()].count(string_snap_t(dn->name, dn->last)) == 0)) {
          dentry_remove_replica(dn, from);
          dout(10) << " rem " << *dn << dendl;
        }
@@ -2883,9 +2890,9 @@ void MDCache::rejoin_scour_survivor_replicas(int from, MMDSCacheRejoin *ack)
 }
 
 
-CInode *MDCache::rejoin_invent_inode(inodeno_t ino)
+CInode *MDCache::rejoin_invent_inode(inodeno_t ino, snapid_t last)
 {
-  CInode *in = new CInode(this);
+  CInode *in = new CInode(this, true, 1, last);
   in->inode.ino = ino;
   in->state_set(CInode::STATE_REJOINUNDEF);
   add_inode(in);
@@ -2912,7 +2919,7 @@ void MDCache::handle_cache_rejoin_strong(MMDSCacheRejoin *strong)
     CDir *dir = get_dirfrag(p->first);
     if (!dir) {
       CInode *in = get_inode(p->first.ino);
-      if (!in) in = rejoin_invent_inode(p->first.ino);
+      if (!in) in = rejoin_invent_inode(p->first.ino, CEPH_NOSNAP);
       if (!in->is_dir()) {
        assert(in->state_test(CInode::STATE_REJOINUNDEF));
        in->inode.mode = S_IFDIR;
@@ -2924,23 +2931,23 @@ void MDCache::handle_cache_rejoin_strong(MMDSCacheRejoin *strong)
     dir->add_replica(from);
     dir->dir_rep = p->second.dir_rep;
 
-    for (map<nstring,MMDSCacheRejoin::dn_strong>::iterator q = strong->strong_dentries[p->first].begin();
+    for (map<string_snap_t,MMDSCacheRejoin::dn_strong>::iterator q = strong->strong_dentries[p->first].begin();
         q != strong->strong_dentries[p->first].end();
         ++q) {
-      CDentry *dn = dir->lookup(q->first);
+      CDentry *dn = dir->lookup(q->first.name, q->first.snapid);
       if (!dn) {
        if (q->second.is_remote()) {
-         dn = dir->add_remote_dentry(q->first, q->second.remote_ino, q->second.remote_d_type);
+         dn = dir->add_remote_dentry(q->first.name, q->second.remote_ino, q->second.remote_d_type, q->second.first, q->first.snapid);
        } else if (q->second.is_null()) {
-         dn = dir->add_null_dentry(q->first);
+         dn = dir->add_null_dentry(q->first.name, q->second.first, q->first.snapid);
        } else {
-         CInode *in = get_inode(q->second.ino);
-         if (!in) in = rejoin_invent_inode(q->second.ino);
-         dn = dir->add_primary_dentry(q->first, in);
+         CInode *in = get_inode(q->second.ino, q->first.snapid);
+         if (!in) in = rejoin_invent_inode(q->second.ino, q->first.snapid);
+         dn = dir->add_primary_dentry(q->first.name, in, q->second.first, q->first.snapid);
 
-         dout(10) << " missing " << q->second.ino << dendl;
+         dout(10) << " missing " << q->second.ino << "." << q->first.snapid << dendl;
          if (!missing) missing = new MMDSCacheRejoin(MMDSCacheRejoin::OP_MISSING);
-         missing->add_weak_inode(q->second.ino);  // we want it back!
+         missing->add_weak_inode(vinodeno_t(q->second.ino, q->first.snapid));  // we want it back!
        }
        dout(10) << " invented " << *dn << dendl;
       }
@@ -2981,8 +2988,8 @@ void MDCache::handle_cache_rejoin_strong(MMDSCacheRejoin *strong)
        CInode *in = dn->get_inode();
        assert(in);
 
-       if (strong->strong_inodes.count(in->ino())) {
-         MMDSCacheRejoin::inode_strong &is = strong->strong_inodes[in->ino()];
+       if (strong->strong_inodes.count(in->vino())) {
+         MMDSCacheRejoin::inode_strong &is = strong->strong_inodes[in->vino()];
 
          // caps_wanted
          if (is.caps_wanted) {
@@ -2997,8 +3004,8 @@ void MDCache::handle_cache_rejoin_strong(MMDSCacheRejoin *strong)
            in->dirlock.set_state(LOCK_SCATTER);
          
          // auth pin?
-         if (strong->authpinned_inodes.count(in->ino())) {
-           metareqid_t ri = strong->authpinned_inodes[in->ino()];
+         if (strong->authpinned_inodes.count(in->vino())) {
+           metareqid_t ri = strong->authpinned_inodes[in->vino()];
            dout(10) << " inode authpin by " << ri << " on " << *in << dendl;
            
            // get/create slave mdrequest
@@ -3011,9 +3018,9 @@ void MDCache::handle_cache_rejoin_strong(MMDSCacheRejoin *strong)
          }
          
          // xlock(s)?
-         if (strong->xlocked_inodes.count(in->ino())) {
-           for (map<int,metareqid_t>::iterator r = strong->xlocked_inodes[in->ino()].begin();
-                r != strong->xlocked_inodes[in->ino()].end();
+         if (strong->xlocked_inodes.count(in->vino())) {
+           for (map<int,metareqid_t>::iterator r = strong->xlocked_inodes[in->vino()].begin();
+                r != strong->xlocked_inodes[in->vino()].end();
                 ++r) {
              SimpleLock *lock = in->get_lock(r->first);
              dout(10) << " inode xlock by " << r->second << " on " << *lock << " on " << *in << dendl;
@@ -3036,7 +3043,7 @@ void MDCache::handle_cache_rejoin_strong(MMDSCacheRejoin *strong)
   }
 
   // base inodes?  (root, stray, etc.)
-  for (set<inodeno_t>::iterator p = strong->weak_inodes.begin();
+  for (set<vinodeno_t>::iterator p = strong->weak_inodes.begin();
        p != strong->weak_inodes.end();
        ++p) {
     CInode *in = get_inode(*p);
@@ -3080,12 +3087,18 @@ void MDCache::handle_cache_rejoin_ack(MMDSCacheRejoin *ack)
     dout(10) << " got " << *dir << dendl;
 
     // dentries
-    for (map<nstring,MMDSCacheRejoin::dn_strong>::iterator q = ack->strong_dentries[p->first].begin();
+    for (map<string_snap_t,MMDSCacheRejoin::dn_strong>::iterator q = ack->strong_dentries[p->first].begin();
         q != ack->strong_dentries[p->first].end();
         ++q) {
-      CDentry *dn = dir->lookup(q->first);
+      CDentry *dn = dir->lookup(q->first.name, q->first.snapid);
       if (!dn) continue;  // must have trimmed?
 
+      assert(dn->last == q->first.snapid);
+      if (dn->first != q->second.first) {
+       dout(10) << " adjust dn.first " << dn->first << " -> " << q->second.first << " on " << *dn << dendl;
+       dn->first = q->second.first;
+      }
+
       // hmm, did we have the proper linkage here?
       if (dn->is_null() &&
          !q->second.is_null()) {
@@ -3093,13 +3106,13 @@ void MDCache::handle_cache_rejoin_ack(MMDSCacheRejoin *ack)
        if (q->second.is_remote()) {
          dn->dir->link_remote_inode(dn, q->second.remote_ino, q->second.remote_d_type);
        } else {
-         CInode *in = get_inode(q->second.ino);
+         CInode *in = get_inode(q->second.ino, q->first.snapid);
          assert(in == 0);  // a rename would have been caught be the resolve stage.
          // barebones inode; the full inode loop below will clean up.
-         in = new CInode(this, false);
+         in = new CInode(this, false, q->second.first, q->first.snapid);
          in->inode.ino = q->second.ino;
          add_inode(in);
-         dn->dir->link_primary_inode(dn, in); 
+         dn->dir->link_primary_inode(dn, in);
        }
       }
       else if (!dn->is_null() &&
@@ -3124,10 +3137,15 @@ void MDCache::handle_cache_rejoin_ack(MMDSCacheRejoin *ack)
   bufferlist::iterator p = ack->inode_base.begin();
   while (!p.end()) {
     inodeno_t ino;
+    snapid_t last;
+    bufferlist basebl;
     ::decode(ino, p);
-    CInode *in = get_inode(ino);
+    ::decode(last, p);
+    ::decode(basebl, p);
+    CInode *in = get_inode(ino, last);
     if (!in) continue;
-    in->_decode_base(p);
+    bufferlist::iterator q = basebl.begin();
+    in->_decode_base(q);
     dout(10) << " got inode content " << *in << dendl;
   }
 
@@ -3135,13 +3153,19 @@ void MDCache::handle_cache_rejoin_ack(MMDSCacheRejoin *ack)
   p = ack->inode_locks.begin();
   while (!p.end()) {
     inodeno_t ino;
-    ::decode(ino, p);
+    snapid_t last;
     int32_t nonce;
+    bufferlist lockbl;
+    ::decode(ino, p);
+    ::decode(last, p);
     ::decode(nonce, p);
-    CInode *in = get_inode(ino);
+    ::decode(lockbl, p);
+
+    CInode *in = get_inode(ino, last);
     if (!in) continue;
     in->set_replica_nonce(nonce);
-    in->_decode_locks_rejoin(p, waiters);
+    bufferlist::iterator q = lockbl.begin();
+    in->_decode_locks_rejoin(q, waiters);
     in->state_clear(CInode::STATE_REJOINING);
     dout(10) << " got " << *in << dendl;
   }
@@ -3169,7 +3193,7 @@ void MDCache::handle_cache_rejoin_missing(MMDSCacheRejoin *missing)
   MMDSCacheRejoin *full = new MMDSCacheRejoin(MMDSCacheRejoin::OP_FULL);
   
   // inodes
-  for (set<inodeno_t>::iterator p = missing->weak_inodes.begin();
+  for (set<vinodeno_t>::iterator p = missing->weak_inodes.begin();
        p != missing->weak_inodes.end();
        ++p) {
     CInode *in = get_inode(*p);
@@ -3194,10 +3218,16 @@ void MDCache::handle_cache_rejoin_full(MMDSCacheRejoin *full)
   bufferlist::iterator p = full->inode_base.begin();
   while (!p.end()) {
     inodeno_t ino;
+    snapid_t last;
+    bufferlist basebl;
     ::decode(ino, p);
+    ::decode(last, p);
+    ::decode(basebl, p);
+
     CInode *in = get_inode(ino);
     assert(in);
-    in->_decode_base(p);
+    bufferlist::iterator pp = basebl.begin();
+    in->_decode_base(pp);
 
     set<CInode*>::iterator q = rejoin_undef_inodes.find(in);
     if (q != rejoin_undef_inodes.end()) {
@@ -3403,7 +3433,7 @@ void MDCache::rejoin_send_acks()
        for (map<int,int>::iterator r = dn->replicas_begin();
             r != dn->replicas_end();
             ++r) 
-         ack[r->first]->add_strong_dentry(dir->dirfrag(), dn->name, 
+         ack[r->first]->add_strong_dentry(dir->dirfrag(), dn->name, dn->first, dn->last,
                                           dn->is_primary() ? dn->get_inode()->ino():inodeno_t(0),
                                           dn->is_remote() ? dn->get_remote_ino():inodeno_t(0),
                                           dn->is_remote() ? dn->get_remote_d_type():0,

diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h
index 13987662a700e4f56b1c0516c26ae9cfd4b9627d..7cae4fdcfb3d25ae64395c95dac0514c7abeb7a4 100644 (file)
--- a/src/mds/MDCache.h
+++ b/src/mds/MDCache.h
@@ -591,7 +591,7 @@ protected:
   void rejoin_walk(CDir *dir, MMDSCacheRejoin *rejoin);
   void handle_cache_rejoin(MMDSCacheRejoin *m);
   void handle_cache_rejoin_weak(MMDSCacheRejoin *m);
-  CInode* rejoin_invent_inode(inodeno_t ino);
+  CInode* rejoin_invent_inode(inodeno_t ino, snapid_t last);
   void handle_cache_rejoin_strong(MMDSCacheRejoin *m);
   void rejoin_scour_survivor_replicas(int from, MMDSCacheRejoin *ack);
   void handle_cache_rejoin_ack(MMDSCacheRejoin *m);

diff --git a/src/mds/mdstypes.h b/src/mds/mdstypes.h
index ad086dd0fd6aa3a846edcc55f64d77bb2263fd4e..edbff08eb80b1b1abe4b62f79f2ebae688a2428d 100644 (file)
--- a/src/mds/mdstypes.h
+++ b/src/mds/mdstypes.h
@@ -436,6 +436,37 @@ inline bool operator<(const dentry_key_t& k1, const dentry_key_t& k2)
 }
 
 
+/*
+ * string_snap_t is a simple (nstring, snapid_t) pair
+ */
+struct string_snap_t {
+  nstring name;
+  snapid_t snapid;
+  string_snap_t() {}
+  string_snap_t(const string& n, snapid_t s) : name(n), snapid(s) {}
+  string_snap_t(const nstring& n, snapid_t s) : name(n), snapid(s) {}
+  string_snap_t(const char *n, snapid_t s) : name(n), snapid(s) {}
+  void encode(bufferlist& bl) const {
+    ::encode(name, bl);
+    ::encode(snapid, bl);
+  }
+  void decode(bufferlist::iterator& bl) {
+    ::decode(name, bl);
+    ::decode(snapid, bl);
+  }
+};
+WRITE_CLASS_ENCODER(string_snap_t)
+
+inline bool operator<(const string_snap_t& l, const string_snap_t& r) {
+  int c = strcmp(l.name.c_str(), r.name.c_str());
+  return c < 0 || (c == 0 && l.snapid < r.snapid);
+}
+
+inline ostream& operator<<(ostream& out, const string_snap_t &k)
+{
+  return out << "(" << k.name << "," << k.snapid << ")";
+}
+
 
 
 // =========

diff --git a/src/messages/MMDSCacheRejoin.h b/src/messages/MMDSCacheRejoin.h
index ded498da77d2ff2acc39c3921bf058ae355aa62e..7df08669d1031684700bfa86487b7259d91db460 100644 (file)
--- a/src/messages/MMDSCacheRejoin.h
+++ b/src/messages/MMDSCacheRejoin.h
@@ -28,7 +28,6 @@ class MMDSCacheRejoin : public Message {
   static const int OP_WEAK    = 1;  // replica -> auth, i exist, + maybe open files.
   static const int OP_STRONG  = 2;  // replica -> auth, i exist, + open files and lock state.
   static const int OP_ACK     = 3;  // auth -> replica, here is your lock state.
-  //static const int OP_PURGE   = 4;  // auth -> replica, remove these items, they are old/obsolete.
   static const int OP_MISSING = 5;  // auth -> replica, i am missing these items
   static const int OP_FULL    = 6;  // replica -> auth, here is the full object.
   static const char *get_opname(int op) {
@@ -80,6 +79,7 @@ class MMDSCacheRejoin : public Message {
   WRITE_CLASS_ENCODER(dirfrag_strong)
 
   struct dn_strong {
+    snapid_t first;
     inodeno_t ino;
     inodeno_t remote_ino;
     unsigned char remote_d_type;
@@ -87,12 +87,13 @@ class MMDSCacheRejoin : public Message {
     int32_t lock;
     dn_strong() : 
       ino(0), remote_ino(0), remote_d_type(0), nonce(0), lock(0) {}
-    dn_strong(inodeno_t pi, inodeno_t ri, unsigned char rdt, int n, int l) : 
-      ino(pi), remote_ino(ri), remote_d_type(rdt), nonce(n), lock(l) {}
+    dn_strong(snapid_t f, inodeno_t pi, inodeno_t ri, unsigned char rdt, int n, int l) : 
+      first(f), ino(pi), remote_ino(ri), remote_d_type(rdt), nonce(n), lock(l) {}
     bool is_primary() { return ino > 0; }
     bool is_remote() { return remote_ino > 0; }
     bool is_null() { return ino == 0 && remote_ino == 0; }
     void encode(bufferlist &bl) const {
+      ::encode(first, bl);
       ::encode(ino, bl);
       ::encode(remote_ino, bl);
       ::encode(remote_d_type, bl);
@@ -100,6 +101,7 @@ class MMDSCacheRejoin : public Message {
       ::encode(lock, bl);
     }
     void decode(bufferlist::iterator &bl) {
+      ::decode(first, bl);
       ::decode(ino, bl);
       ::decode(remote_ino, bl);
       ::decode(remote_d_type, bl);
@@ -110,13 +112,16 @@ class MMDSCacheRejoin : public Message {
   WRITE_CLASS_ENCODER(dn_strong)
 
   struct dn_weak {
+    snapid_t first;
     inodeno_t ino;
     dn_weak() : ino(0) {}
-    dn_weak(inodeno_t pi) : ino(pi) {}
+    dn_weak(snapid_t f, inodeno_t pi) : first(f), ino(pi) {}
     void encode(bufferlist &bl) const {
+      ::encode(first, bl);
       ::encode(ino, bl);
     }
     void decode(bufferlist::iterator &bl) {
+      ::decode(first, bl);
       ::decode(ino, bl);
     }
   };
@@ -127,13 +132,13 @@ class MMDSCacheRejoin : public Message {
 
   // weak
   map<dirfrag_t, pair<frag_info_t, frag_info_t> > dirfrag_stat;
-  map<dirfrag_t, map<nstring, dn_weak> > weak;
-  set<inodeno_t> weak_inodes;
+  map<dirfrag_t, map<string_snap_t, dn_weak> > weak;
+  set<vinodeno_t> weak_inodes;
 
   // strong
   map<dirfrag_t, dirfrag_strong> strong_dirfrags;
-  map<dirfrag_t, map<nstring, dn_strong> > strong_dentries;
-  map<inodeno_t, inode_strong> strong_inodes;
+  map<dirfrag_t, map<string_snap_t, dn_strong> > strong_dentries;
+  map<vinodeno_t, inode_strong> strong_inodes;
 
   // open
   bufferlist cap_export_bl;
@@ -145,10 +150,10 @@ class MMDSCacheRejoin : public Message {
   bufferlist inode_locks;
 
   // authpins, xlocks
-  map<inodeno_t, metareqid_t> authpinned_inodes;
-  map<inodeno_t, map<__s32, metareqid_t> > xlocked_inodes;
-  map<dirfrag_t, map<nstring, metareqid_t> > authpinned_dentries;
-  map<dirfrag_t, map<nstring, metareqid_t> > xlocked_dentries;
+  map<vinodeno_t, metareqid_t> authpinned_inodes;
+  map<vinodeno_t, map<__s32, metareqid_t> > xlocked_inodes;
+  map<dirfrag_t, map<string_snap_t, metareqid_t> > authpinned_dentries;
+  map<dirfrag_t, map<string_snap_t, metareqid_t> > xlocked_dentries;
 
   MMDSCacheRejoin() : Message(MSG_MDS_CACHEREJOIN) {}
   MMDSCacheRejoin(int o) : 
@@ -162,24 +167,30 @@ class MMDSCacheRejoin : public Message {
 
   // -- builders --
   // inodes
-  void add_weak_inode(inodeno_t i) {
+  void add_weak_inode(vinodeno_t i) {
     weak_inodes.insert(i);
   }
-  void add_strong_inode(inodeno_t i, int cw, int dl, int nl) {
+  void add_strong_inode(vinodeno_t i, int cw, int dl, int nl) {
     strong_inodes[i] = inode_strong(cw, dl, nl);
   }
   void add_inode_locks(CInode *in, int nonce) {
     ::encode(in->inode.ino, inode_locks);
-    in->_encode_locks_state(inode_locks);
+    ::encode(in->last, inode_locks);
+    bufferlist bl;
+    in->_encode_locks_state(bl);
+    inode_locks.claim_append(bl);
   }
   void add_inode_base(CInode *in) {
     ::encode(in->inode.ino, inode_base);
-    in->_encode_base(inode_base);
+    ::encode(in->last, inode_base);
+    bufferlist bl;
+    in->_encode_base(bl);
+    inode_base.claim_append(bl);
   }
-  void add_inode_authpin(inodeno_t ino, const metareqid_t& ri) {
+  void add_inode_authpin(vinodeno_t ino, const metareqid_t& ri) {
     authpinned_inodes[ino] = ri;
   }
-  void add_inode_xlock(inodeno_t ino, int lt, const metareqid_t& ri) {
+  void add_inode_xlock(vinodeno_t ino, int lt, const metareqid_t& ri) {
     xlocked_inodes[ino][lt] = ri;
   }
 
@@ -195,7 +206,7 @@ class MMDSCacheRejoin : public Message {
   void add_weak_dirfrag(dirfrag_t df) {
     weak[df];
   }
-  void add_weak_dirfrag(dirfrag_t df, map<nstring,dn_weak>& dnmap) {
+  void add_weak_dirfrag(dirfrag_t df, map<string_snap_t,dn_weak>& dnmap) {
     weak[df] = dnmap;
   }
   void add_strong_dirfrag(dirfrag_t df, int n, int dr) {
@@ -203,20 +214,20 @@ class MMDSCacheRejoin : public Message {
   }
    
   // dentries
-  void add_weak_dentry(dirfrag_t df, const string& dname, dn_weak& dnw) {
-    weak[df][dname] = dnw;
+  void add_weak_dentry(dirfrag_t df, const string& dname, snapid_t last, dn_weak& dnw) {
+    weak[df][string_snap_t(dname, last)] = dnw;
   }
-  void add_weak_primary_dentry(dirfrag_t df, const string& dname, inodeno_t ino) {
-    weak[df][dname] = dn_weak(ino);
+  void add_weak_primary_dentry(dirfrag_t df, const string& dname, snapid_t first, snapid_t last, inodeno_t ino) {
+    weak[df][string_snap_t(dname, last)] = dn_weak(first, ino);
   }
-  void add_strong_dentry(dirfrag_t df, const nstring& dname, inodeno_t pi, inodeno_t ri, unsigned char rdt, int n, int ls) {
-    strong_dentries[df][dname] = dn_strong(pi, ri, rdt, n, ls);
+  void add_strong_dentry(dirfrag_t df, const nstring& dname, snapid_t first, snapid_t last, inodeno_t pi, inodeno_t ri, unsigned char rdt, int n, int ls) {
+    strong_dentries[df][string_snap_t(dname, last)] = dn_strong(first, pi, ri, rdt, n, ls);
   }
-  void add_dentry_authpin(dirfrag_t df, const nstring& dname, const metareqid_t& ri) {
-    authpinned_dentries[df][dname] = ri;
+  void add_dentry_authpin(dirfrag_t df, const nstring& dname, snapid_t last, const metareqid_t& ri) {
+    authpinned_dentries[df][string_snap_t(dname, last)] = ri;
   }
-  void add_dentry_xlock(dirfrag_t df, const nstring& dname, const metareqid_t& ri) {
-    xlocked_dentries[df][dname] = ri;
+  void add_dentry_xlock(dirfrag_t df, const nstring& dname, snapid_t last, const metareqid_t& ri) {
+    xlocked_dentries[df][string_snap_t(dname, last)] = ri;
   }
 
   // -- encoding --