From 1949cce464934165a76a713fed8dd889cf81f6a1 Mon Sep 17 00:00:00 2001
From: Sage Weil <sage@newdream.net>
Date: Fri, 11 Jul 2008 14:43:56 -0700
Subject: [PATCH] client: cache versioned inodes.  use high bits of ino in fuse
 client.

---
 src/client/Client.cc          | 238 +++++++++++++++++-----------------
 src/client/Client.h           |  56 ++++----
 src/client/SyntheticClient.cc |  36 ++---
 src/client/fuse_ll.cc         |  90 +++++++++----
 src/include/ceph_fs.h         |   1 +
 src/mds/mdstypes.h            |   2 +-
 src/messages/MClientReply.h   |   5 +-
 7 files changed, 238 insertions(+), 190 deletions(-)

diff --git a/src/client/Client.cc b/src/client/Client.cc
index 5e2c4fada4c16..b34a723a1d735 100644
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -208,7 +208,7 @@ void Client::dump_cache()
 
   if (root) dump_inode(root, did);
 
-  for (hash_map<inodeno_t, Inode*>::iterator it = inode_map.begin();
+  for (hash_map<vinodeno_t, Inode*>::iterator it = inode_map.begin();
        it != inode_map.end();
        it++) {
     if (did.count(it->second)) continue;
@@ -372,7 +372,8 @@ void Client::update_inode(Inode *in, InodeStat *st, LeaseStat *lease, utime_t fr
   dout(12) << "update_inode mask " << lease->mask << " ttl " << ttl << dendl;
 
   if (lease->mask & CEPH_STAT_MASK_INODE) {
-    in->inode.ino = st->ino;
+    in->inode.ino = st->vino.ino;
+    in->snapid = st->vino.snapid;
     in->inode.layout = st->layout;
     in->inode.rdev = st->rdev;
 
@@ -424,7 +425,7 @@ Inode* Client::insert_dentry_inode(Dir *dir, const string& dname, LeaseStat *dle
   if (dir->dentries.count(dname))
     dn = dir->dentries[dname];
 
-  dout(12) << "insert_dentry_inode " << dname << " ino " << ist->ino
+  dout(12) << "insert_dentry_inode " << dname << " vino " << ist->vino
            << "  size " << ist->size
            << "  mtime " << ist->mtime
 	   << " dmask " << dmask
@@ -432,14 +433,14 @@ Inode* Client::insert_dentry_inode(Dir *dir, const string& dname, LeaseStat *dle
            << dendl;
   
   if (dn) {
-    if (dn->inode->inode.ino == ist->ino) {
+    if (dn->inode->vino() == ist->vino) {
       touch_dn(dn);
       dout(12) << " had dentry " << dname
-               << " with correct ino " << dn->inode->inode.ino
+               << " with correct vino " << dn->inode->vino()
                << dendl;
     } else {
       dout(12) << " had dentry " << dname
-               << " with WRONG ino " << dn->inode->inode.ino
+               << " with WRONG vino " << dn->inode->vino()
                << dendl;
       unlink(dn, true);
       dn = NULL;
@@ -448,18 +449,18 @@ Inode* Client::insert_dentry_inode(Dir *dir, const string& dname, LeaseStat *dle
   
   if (!dn) {
     // have inode linked elsewhere?  -> unlink and relink!
-    if (inode_map.count(ist->ino)) {
-      Inode *in = inode_map[ist->ino];
+    if (inode_map.count(ist->vino)) {
+      Inode *in = inode_map[ist->vino];
       assert(in);
 
       if (in->dn) {
-        dout(12) << " had ino " << in->inode.ino
+        dout(12) << " had vino " << in->vino()
                  << " not linked or linked at the right position, relinking"
                  << dendl;
         dn = relink_inode(dir, dname, in);
       } else {
         // link
-        dout(12) << " had ino " << in->inode.ino
+        dout(12) << " had vino " << in->vino()
                  << " unlinked, linking" << dendl;
         dn = link(dir, dname, in);
       }
@@ -467,10 +468,10 @@ Inode* Client::insert_dentry_inode(Dir *dir, const string& dname, LeaseStat *dle
   }
 
   if (!dn) {
-    Inode *in = new Inode(ist->ino, &ist->layout);
-    inode_map[ist->ino] = in;
+    Inode *in = new Inode(ist->vino, &ist->layout);
+    inode_map[ist->vino] = in;
     dn = link(dir, dname, in);
-    dout(12) << " new dentry+node with ino " << ist->ino << dendl;
+    dout(12) << " new dentry+node with vino " << ist->vino << dendl;
   } 
 
   assert(dn && dn->inode);
@@ -576,16 +577,16 @@ Inode* Client::insert_trace(MClientReply *reply, utime_t from)
   // insert into cache --
   // first inode
   Inode *curi = 0;
-  inodeno_t ino = ist[0].ino;
-  if (!root && ino == 1) {
-    curi = root = new Inode(ino, &ist[0].layout);
+  vinodeno_t vino = ist[0].vino;
+  if (!root && vino.ino == 1) {
+    curi = root = new Inode(vino, &ist[0].layout);
     dout(10) << "insert_trace new root is " << root << dendl;
-    inode_map[ino] = root;
+    inode_map[vino] = root;
     root->dir_auth = 0;
   }
   if (!curi) {
-    assert(inode_map.count(ino));
-    curi = inode_map[ino];
+    assert(inode_map.count(vino));
+    curi = inode_map[vino];
   }
   update_inode(curi, &ist[0], &ilease[0], from);
 
@@ -613,14 +614,15 @@ Inode* Client::insert_trace(MClientReply *reply, utime_t from)
 /*
  * bleh, dentry vs inode semantics here are sloppy
  */
-Dentry *Client::lookup(const filepath& path)
+Dentry *Client::lookup(const filepath& path, snapid_t snapid)
 {
   dout(14) << "lookup " << path << dendl;
 
   Inode *cur;
   if (path.get_ino()) {
-    if (inode_map.count(path.get_ino()))
-      cur = inode_map[path.get_ino()];
+    vinodeno_t vino(path.get_ino(), snapid);
+    if (inode_map.count(vino)) 
+      cur = inode_map[vino];
     else
       return NULL;
   } else
@@ -1186,7 +1188,7 @@ void Client::send_reconnect(int mds)
 
   if (mds_sessions.count(mds)) {
     // i have an open session.
-    for (hash_map<inodeno_t, Inode*>::iterator p = inode_map.begin();
+    for (hash_map<vinodeno_t, Inode*>::iterator p = inode_map.begin();
 	 p != inode_map.end();
 	 p++) {
       Inode *in = p->second;
@@ -1196,14 +1198,14 @@ void Client::send_reconnect(int mds)
 		 << " wants " << cap_string(in->caps_wanted())
 		 << dendl;
 	in->caps[mds]->seq = 0;  // reset seq.
-	m->add_inode_caps(p->first,    // ino
+	m->add_inode_caps(p->first.ino,    // ino
 			  in->caps_wanted(), // wanted
 			  in->caps[mds]->issued,     // issued
 			  in->inode.size, in->inode.mtime, in->inode.atime);
 	filepath path;
 	in->make_path(path);
 	dout(10) << " path on " << p->first << " is " << path << dendl;
-	m->add_inode_path(p->first, path.get_path());
+	m->add_inode_path(p->first.ino, path.get_path());
       }
       if (in->exporting_mds == mds) {
 	dout(10) << " clearing exporting_caps on " << p->first << dendl;
@@ -1253,11 +1255,12 @@ void Client::handle_lease(MClientLease *m)
   mds_sessions[mds].seq++;
 
   Inode *in;
-  if (inode_map.count(m->get_ino()) == 0) {
-    dout(10) << " don't have inode " << m->get_ino() << dendl;
+  vinodeno_t vino(m->get_ino(), CEPH_NOSNAP);
+  if (inode_map.count(vino) == 0) {
+    dout(10) << " don't have vino " << vino << dendl;
     goto revoke;
   }
-  in = inode_map[m->get_ino()];
+  in = inode_map[vino];
 
   if (m->get_mask() & CEPH_LOCK_DN) {
     if (!in->dir || in->dir->dentries.count(m->dname) == 0) {
@@ -1618,8 +1621,9 @@ void Client::handle_snap(MClientSnap *m)
     for (list<inodeno_t>::iterator p = m->split_inos.begin();
 	 p != m->split_inos.end();
 	 p++) {
-      if (inode_map.count(*p)) {
-	Inode *in = inode_map[*p];
+      vinodeno_t vino(*p, CEPH_NOSNAP);
+      if (inode_map.count(vino)) {
+	Inode *in = inode_map[vino];
 	if (!in->snaprealm)
 	  continue;
 	if (in->snaprealm->created > m->snap_created) {
@@ -1663,13 +1667,14 @@ void Client::handle_file_caps(MClientFileCaps *m)
   mds_sessions[mds].seq++;
 
   Inode *in = 0;
-  if (inode_map.count(m->get_ino())) in = inode_map[ m->get_ino() ];
+  vinodeno_t vino(m->get_ino(), CEPH_NOSNAP);
+  if (inode_map.count(vino)) in = inode_map[vino];
   if (!in) {
     /*
      * this can happen if we release caps, and then trim the inode from our cache,
      * but are racing with, e.g., an mds callback.
      */
-    dout(5) << "handle_file_caps don't have ino " << m->get_ino() << dendl;
+    dout(5) << "handle_file_caps don't have vino " << vino << dendl;
     delete m;
     return;
   }
@@ -1999,8 +2004,8 @@ int Client::unmount()
 
   if (g_conf.client_oc) {
     // release any/all caps
-    hash_map<inodeno_t, Inode*>::iterator next;
-    for (hash_map<inodeno_t, Inode*>::iterator p = inode_map.begin();
+    hash_map<vinodeno_t, Inode*>::iterator next;
+    for (hash_map<vinodeno_t, Inode*>::iterator p = inode_map.begin();
          p != inode_map.end(); 
          p = next) {
       next = p;
@@ -2400,6 +2405,7 @@ int Client::fill_stat(Inode *in, struct stat *st, frag_info_t *dirstat)
 	   << " mtime " << in->inode.mtime << " ctime " << in->inode.ctime << dendl;
   memset(st, 0, sizeof(struct stat));
   st->st_ino = in->inode.ino;
+  st->st_dev = in->snapid;
   st->st_mode = in->inode.mode;
   st->st_rdev = in->inode.rdev;
   st->st_nlink = in->inode.nlink;
@@ -3636,7 +3642,7 @@ int Client::statfs(const char *path, struct statvfs *stbuf)
   return _statfs(stbuf);
 }
 
-int Client::ll_statfs(inodeno_t ino, struct statvfs *stbuf)
+int Client::ll_statfs(vinodeno_t vino, struct statvfs *stbuf)
 {
   Mutex::Locker lock(client_lock);
   tout << "ll_statfs" << std::endl;
@@ -3789,12 +3795,12 @@ int Client::_rmsnap(const filepath& path, const char *name, int uid, int gid)
 #define FUSE_SET_ATTR_ATIME	(1 << 4)
 #define FUSE_SET_ATTR_MTIME	(1 << 5)
 
-int Client::ll_lookup(inodeno_t parent, const char *name, struct stat *attr, int uid, int gid)
+int Client::ll_lookup(vinodeno_t parent, const char *name, struct stat *attr, int uid, int gid)
 {
   Mutex::Locker lock(client_lock);
   dout(3) << "ll_lookup " << parent << " " << name << dendl;
   tout << "ll_lookup" << std::endl;
-  tout << parent.val << std::endl;
+  tout << parent.ino.val << std::endl;
   tout << name << std::endl;
 
   string dname = name;
@@ -3871,8 +3877,8 @@ int Client::_ll_put(Inode *in, int num)
 void Client::_ll_drop_pins()
 {
   dout(10) << "_ll_drop_pins" << dendl;
-  hash_map<inodeno_t, Inode*>::iterator next;
-  for (hash_map<inodeno_t, Inode*>::iterator it = inode_map.begin();
+  hash_map<vinodeno_t, Inode*>::iterator next;
+  for (hash_map<vinodeno_t, Inode*>::iterator it = inode_map.begin();
        it != inode_map.end();
        it = next) {
     Inode *in = it->second;
@@ -3883,25 +3889,25 @@ void Client::_ll_drop_pins()
   }
 }
 
-bool Client::ll_forget(inodeno_t ino, int num)
+bool Client::ll_forget(vinodeno_t vino, int num)
 {
   Mutex::Locker lock(client_lock);
-  dout(3) << "ll_forget " << ino << " " << num << dendl;
+  dout(3) << "ll_forget " << vino << " " << num << dendl;
   tout << "ll_forget" << std::endl;
-  tout << ino.val << std::endl;
+  tout << vino.ino.val << std::endl;
   tout << num << std::endl;
 
-  if (ino == 1) return true;  // ignore forget on root.
+  if (vino.ino == 1) return true;  // ignore forget on root.
 
   bool last = false;
-  if (inode_map.count(ino) == 0) {
-    dout(1) << "WARNING: ll_forget on " << ino << " " << num 
+  if (inode_map.count(vino) == 0) {
+    dout(1) << "WARNING: ll_forget on " << vino << " " << num 
 	    << ", which I don't have" << dendl;
   } else {
-    Inode *in = inode_map[ino];
+    Inode *in = inode_map[vino];
     assert(in);
     if (in->ll_ref < num) {
-      dout(1) << "WARNING: ll_forget on " << ino << " " << num << ", which only has ll_ref=" << in->ll_ref << dendl;
+      dout(1) << "WARNING: ll_forget on " << vino << " " << num << ", which only has ll_ref=" << in->ll_ref << dendl;
       _ll_put(in, in->ll_ref);
       last = true;
     } else {
@@ -3912,43 +3918,43 @@ bool Client::ll_forget(inodeno_t ino, int num)
   return last;
 }
 
-Inode *Client::_ll_get_inode(inodeno_t ino)
+Inode *Client::_ll_get_inode(vinodeno_t vino)
 {
-  if (inode_map.count(ino) == 0) {
-    assert(ino == 1);  // must be the root inode.
+  if (inode_map.count(vino) == 0) {
+    assert(vino.ino == 1);  // must be the root inode.
     Inode *in;
     filepath path(1);
     int r = _do_lstat(path, 0, &in);
     assert(r >= 0);
     return in;
   } else {
-    return inode_map[ino];
+    return inode_map[vino];
   }
 }
 
 
-int Client::ll_getattr(inodeno_t ino, struct stat *attr, int uid, int gid)
+int Client::ll_getattr(vinodeno_t vino, struct stat *attr, int uid, int gid)
 {
   Mutex::Locker lock(client_lock);
-  dout(3) << "ll_getattr " << ino << dendl;
+  dout(3) << "ll_getattr " << vino << dendl;
   tout << "ll_getattr" << std::endl;
-  tout << ino.val << std::endl;
+  tout << vino.ino.val << std::endl;
 
-  Inode *in = _ll_get_inode(ino);
+  Inode *in = _ll_get_inode(vino);
   filepath path(in->ino());
   int res = _do_lstat(path, CEPH_STAT_MASK_INODE_ALL, &in, uid, gid);
   if (res == 0)
     fill_stat(in, attr);
-  dout(3) << "ll_getattr " << ino << " = " << res << dendl;
+  dout(3) << "ll_getattr " << vino << " = " << res << dendl;
   return res;
 }
 
-int Client::ll_setattr(inodeno_t ino, struct stat *attr, int mask, int uid, int gid)
+int Client::ll_setattr(vinodeno_t vino, struct stat *attr, int mask, int uid, int gid)
 {
   Mutex::Locker lock(client_lock);
-  dout(3) << "ll_setattr " << ino << " mask " << hex << mask << dec << dendl;
+  dout(3) << "ll_setattr " << vino << " mask " << hex << mask << dec << dendl;
   tout << "ll_setattr" << std::endl;
-  tout << ino.val << std::endl;
+  tout << vino.ino.val << std::endl;
   tout << attr->st_mode << std::endl;
   tout << attr->st_uid << std::endl;
   tout << attr->st_gid << std::endl;
@@ -3957,7 +3963,7 @@ int Client::ll_setattr(inodeno_t ino, struct stat *attr, int mask, int uid, int
   tout << attr->st_atime << std::endl;
   tout << mask << std::endl;
 
-  Inode *in = _ll_get_inode(ino);
+  Inode *in = _ll_get_inode(vino);
 
   filepath path;
   in->make_path(path);
@@ -3985,22 +3991,22 @@ int Client::ll_setattr(inodeno_t ino, struct stat *attr, int mask, int uid, int
   assert(r == 0);
   fill_stat(in, attr);
 
-  dout(3) << "ll_setattr " << ino << " = " << r << dendl;
+  dout(3) << "ll_setattr " << vino << " = " << r << dendl;
   return 0;
 }
 
 // ----------
 // xattrs
 
-int Client::ll_getxattr(inodeno_t ino, const char *name, void *value, size_t size, int uid, int gid)
+int Client::ll_getxattr(vinodeno_t vino, const char *name, void *value, size_t size, int uid, int gid)
 {
   Mutex::Locker lock(client_lock);
-  dout(3) << "ll_getxattr " << ino << " " << name << " size " << size << dendl;
+  dout(3) << "ll_getxattr " << vino << " " << name << " size " << size << dendl;
   tout << "ll_getxattr" << std::endl;
-  tout << ino.val << std::endl;
+  tout << vino.ino.val << std::endl;
   tout << name << std::endl;
 
-  Inode *in = _ll_get_inode(ino);
+  Inode *in = _ll_get_inode(vino);
 
   filepath path;
   in->make_path(path);
@@ -4029,15 +4035,15 @@ int Client::_getxattr(const filepath &path, const char *name, void *value, size_
   return r;
 }
 
-int Client::ll_listxattr(inodeno_t ino, char *names, size_t size, int uid, int gid)
+int Client::ll_listxattr(vinodeno_t vino, char *names, size_t size, int uid, int gid)
 {
   Mutex::Locker lock(client_lock);
-  dout(3) << "ll_listxattr " << ino << " size " << size << dendl;
+  dout(3) << "ll_listxattr " << vino << " size " << size << dendl;
   tout << "ll_listxattr" << std::endl;
-  tout << ino.val << std::endl;
+  tout << vino.ino.val << std::endl;
   tout << size << std::endl;
 
-  Inode *in = _ll_get_inode(ino);
+  Inode *in = _ll_get_inode(vino);
   filepath path;
   in->make_path(path);
   return _listxattr(path, names, size, false, uid, gid);
@@ -4072,19 +4078,19 @@ int Client::_listxattr(const filepath &path, char *name, size_t size,
   return r;
 }
 
-int Client::ll_setxattr(inodeno_t ino, const char *name, const void *value, size_t size, int flags, int uid, int gid)
+int Client::ll_setxattr(vinodeno_t vino, const char *name, const void *value, size_t size, int flags, int uid, int gid)
 {
   Mutex::Locker lock(client_lock);
-  dout(3) << "ll_setxattr " << ino << " " << name << " size " << size << dendl;
+  dout(3) << "ll_setxattr " << vino << " " << name << " size " << size << dendl;
   tout << "ll_setxattr" << std::endl;
-  tout << ino.val << std::endl;
+  tout << vino.ino.val << std::endl;
   tout << name << std::endl;
 
   // only user xattrs, for now
   if (strncmp(name, "user.", 5))
     return -EOPNOTSUPP;
 
-  Inode *in = _ll_get_inode(ino);
+  Inode *in = _ll_get_inode(vino);
   if (in->dn) touch_dn(in->dn);
 
   filepath path;
@@ -4113,19 +4119,19 @@ int Client::_setxattr(const filepath &path, const char *name, const void *value,
   return res;
 }
 
-int Client::ll_removexattr(inodeno_t ino, const char *name, int uid, int gid)
+int Client::ll_removexattr(vinodeno_t vino, const char *name, int uid, int gid)
 {
   Mutex::Locker lock(client_lock);
-  dout(3) << "ll_removexattr " << ino << " " << name << dendl;
+  dout(3) << "ll_removexattr " << vino << " " << name << dendl;
   tout << "ll_removexattr" << std::endl;
-  tout << ino.val << std::endl;
+  tout << vino.ino.val << std::endl;
   tout << name << std::endl;
 
   // only user xattrs, for now
   if (strncmp(name, "user.", 5))
     return -EOPNOTSUPP;
 
-  Inode *in = _ll_get_inode(ino);
+  Inode *in = _ll_get_inode(vino);
   if (in->dn) touch_dn(in->dn);
 
   filepath path;
@@ -4150,14 +4156,14 @@ int Client::_removexattr(const filepath &path, const char *name,
 
 
 
-int Client::ll_readlink(inodeno_t ino, const char **value, int uid, int gid)
+int Client::ll_readlink(vinodeno_t vino, const char **value, int uid, int gid)
 {
   Mutex::Locker lock(client_lock);
-  dout(3) << "ll_readlink " << ino << dendl;
+  dout(3) << "ll_readlink " << vino << dendl;
   tout << "ll_readlink" << std::endl;
-  tout << ino.val << std::endl;
+  tout << vino.ino.val << std::endl;
 
-  Inode *in = _ll_get_inode(ino);
+  Inode *in = _ll_get_inode(vino);
   if (in->dn) touch_dn(in->dn);
 
   int r = 0;
@@ -4167,16 +4173,16 @@ int Client::ll_readlink(inodeno_t ino, const char **value, int uid, int gid)
     *value = "";
     r = -EINVAL;
   }
-  dout(3) << "ll_readlink " << ino << " = " << r << " (" << *value << ")" << dendl;
+  dout(3) << "ll_readlink " << vino << " = " << r << " (" << *value << ")" << dendl;
   return r;
 }
 
-int Client::ll_mknod(inodeno_t parent, const char *name, mode_t mode, dev_t rdev, struct stat *attr, int uid, int gid)
+int Client::ll_mknod(vinodeno_t parent, const char *name, mode_t mode, dev_t rdev, struct stat *attr, int uid, int gid)
 {
   Mutex::Locker lock(client_lock);
   dout(3) << "ll_mknod " << parent << " " << name << dendl;
   tout << "ll_mknod" << std::endl;
-  tout << parent.val << std::endl;
+  tout << parent.ino.val << std::endl;
   tout << name << std::endl;
   tout << mode << std::endl;
   tout << rdev << std::endl;
@@ -4199,12 +4205,12 @@ int Client::ll_mknod(inodeno_t parent, const char *name, mode_t mode, dev_t rdev
   return r;
 }
 
-int Client::ll_mkdir(inodeno_t parent, const char *name, mode_t mode, struct stat *attr, int uid, int gid)
+int Client::ll_mkdir(vinodeno_t parent, const char *name, mode_t mode, struct stat *attr, int uid, int gid)
 {
   Mutex::Locker lock(client_lock);
   dout(3) << "ll_mkdir " << parent << " " << name << dendl;
   tout << "ll_mkdir" << std::endl;
-  tout << parent.val << std::endl;
+  tout << parent.ino.val << std::endl;
   tout << name << std::endl;
   tout << mode << std::endl;
 
@@ -4226,12 +4232,12 @@ int Client::ll_mkdir(inodeno_t parent, const char *name, mode_t mode, struct sta
   return r;
 }
 
-int Client::ll_symlink(inodeno_t parent, const char *name, const char *value, struct stat *attr, int uid, int gid)
+int Client::ll_symlink(vinodeno_t parent, const char *name, const char *value, struct stat *attr, int uid, int gid)
 {
   Mutex::Locker lock(client_lock);
   dout(3) << "ll_symlink " << parent << " " << name << " -> " << value << dendl;
   tout << "ll_symlink" << std::endl;
-  tout << parent.val << std::endl;
+  tout << parent.ino.val << std::endl;
   tout << name << std::endl;
   tout << value << std::endl;
 
@@ -4253,15 +4259,15 @@ int Client::ll_symlink(inodeno_t parent, const char *name, const char *value, st
   return r;
 }
 
-int Client::ll_unlink(inodeno_t ino, const char *name, int uid, int gid)
+int Client::ll_unlink(vinodeno_t vino, const char *name, int uid, int gid)
 {
   Mutex::Locker lock(client_lock);
-  dout(3) << "ll_unlink " << ino << " " << name << dendl;
+  dout(3) << "ll_unlink " << vino << " " << name << dendl;
   tout << "ll_unlink" << std::endl;
-  tout << ino.val << std::endl;
+  tout << vino.ino.val << std::endl;
   tout << name << std::endl;
 
-  Inode *diri = _ll_get_inode(ino);
+  Inode *diri = _ll_get_inode(vino);
 
   filepath path;
   diri->make_path(path);
@@ -4269,15 +4275,15 @@ int Client::ll_unlink(inodeno_t ino, const char *name, int uid, int gid)
   return _unlink(path.c_str(), uid, gid);
 }
 
-int Client::ll_rmdir(inodeno_t ino, const char *name, int uid, int gid)
+int Client::ll_rmdir(vinodeno_t vino, const char *name, int uid, int gid)
 {
   Mutex::Locker lock(client_lock);
-  dout(3) << "ll_rmdir " << ino << " " << name << dendl;
+  dout(3) << "ll_rmdir " << vino << " " << name << dendl;
   tout << "ll_rmdir" << std::endl;
-  tout << ino.val << std::endl;
+  tout << vino.ino.val << std::endl;
   tout << name << std::endl;
 
-  Inode *diri = _ll_get_inode(ino);
+  Inode *diri = _ll_get_inode(vino);
 
   filepath path;
   diri->make_path(path);
@@ -4285,15 +4291,15 @@ int Client::ll_rmdir(inodeno_t ino, const char *name, int uid, int gid)
   return _rmdir(path.c_str(), uid, gid);
 }
 
-int Client::ll_rename(inodeno_t parent, const char *name, inodeno_t newparent, const char *newname, int uid, int gid)
+int Client::ll_rename(vinodeno_t parent, const char *name, vinodeno_t newparent, const char *newname, int uid, int gid)
 {
   Mutex::Locker lock(client_lock);
   dout(3) << "ll_rename " << parent << " " << name << " to "
 	  << newparent << " " << newname << dendl;
   tout << "ll_rename" << std::endl;
-  tout << parent.val << std::endl;
+  tout << parent.ino.val << std::endl;
   tout << name << std::endl;
-  tout << newparent.val << std::endl;
+  tout << newparent.ino.val << std::endl;
   tout << newname << std::endl;
 
   Inode *diri = _ll_get_inode(parent);
@@ -4309,16 +4315,16 @@ int Client::ll_rename(inodeno_t parent, const char *name, inodeno_t newparent, c
   return _rename(path.c_str(), newpath.c_str(), uid, gid);
 }
 
-int Client::ll_link(inodeno_t ino, inodeno_t newparent, const char *newname, struct stat *attr, int uid, int gid)
+int Client::ll_link(vinodeno_t vino, vinodeno_t newparent, const char *newname, struct stat *attr, int uid, int gid)
 {
   Mutex::Locker lock(client_lock);
-  dout(3) << "ll_link " << ino << " to " << newparent << " " << newname << dendl;
+  dout(3) << "ll_link " << vino << " to " << newparent << " " << newname << dendl;
   tout << "ll_link" << std::endl;
-  tout << ino.val << std::endl;
+  tout << vino.ino.val << std::endl;
   tout << newparent << std::endl;
   tout << newname << std::endl;
 
-  Inode *old = _ll_get_inode(ino);
+  Inode *old = _ll_get_inode(vino);
   Inode *diri = _ll_get_inode(newparent);
 
   filepath path;
@@ -4330,21 +4336,21 @@ int Client::ll_link(inodeno_t ino, inodeno_t newparent, const char *newname, str
 
   int r = _link(path.c_str(), newpath.c_str(), uid, gid);
   if (r == 0) {
-    Inode *in = _ll_get_inode(ino);
+    Inode *in = _ll_get_inode(vino);
     fill_stat(in, attr);
     _ll_get(in);
   }
   return r;
 }
 
-int Client::ll_opendir(inodeno_t ino, void **dirpp, int uid, int gid)
+int Client::ll_opendir(vinodeno_t vino, void **dirpp, int uid, int gid)
 {
   Mutex::Locker lock(client_lock);
-  dout(3) << "ll_opendir " << ino << dendl;
+  dout(3) << "ll_opendir " << vino << dendl;
   tout << "ll_opendir" << std::endl;
-  tout << ino.val << std::endl;
+  tout << vino.ino.val << std::endl;
   
-  Inode *diri = inode_map[ino];
+  Inode *diri = inode_map[vino];
   assert(diri);
   filepath path;
   diri->make_path(path);
@@ -4354,7 +4360,7 @@ int Client::ll_opendir(inodeno_t ino, void **dirpp, int uid, int gid)
 
   tout << (unsigned long)*dirpp << std::endl;
 
-  dout(3) << "ll_opendir " << ino << " = " << r << " (" << *dirpp << ")" << dendl;
+  dout(3) << "ll_opendir " << vino << " = " << r << " (" << *dirpp << ")" << dendl;
   return r;
 }
 
@@ -4367,32 +4373,32 @@ void Client::ll_releasedir(void *dirp)
   _closedir((DirResult*)dirp);
 }
 
-int Client::ll_open(inodeno_t ino, int flags, Fh **fhp, int uid, int gid)
+int Client::ll_open(vinodeno_t vino, int flags, Fh **fhp, int uid, int gid)
 {
   Mutex::Locker lock(client_lock);
-  dout(3) << "ll_open " << ino << " " << flags << dendl;
+  dout(3) << "ll_open " << vino << " " << flags << dendl;
   tout << "ll_open" << std::endl;
-  tout << ino.val << std::endl;
+  tout << vino.ino.val << std::endl;
   tout << flags << std::endl;
 
-  Inode *in = _ll_get_inode(ino);
+  Inode *in = _ll_get_inode(vino);
   filepath path;
   in->make_path(path);
 
   int r = _open(path.c_str(), flags, 0, fhp, uid, gid);
 
   tout << (unsigned long)*fhp << std::endl;
-  dout(3) << "ll_open " << ino << " " << flags << " = " << r << " (" << *fhp << ")" << dendl;
+  dout(3) << "ll_open " << vino << " " << flags << " = " << r << " (" << *fhp << ")" << dendl;
   return r;
 }
 
-int Client::ll_create(inodeno_t parent, const char *name, mode_t mode, int flags, 
+int Client::ll_create(vinodeno_t parent, const char *name, mode_t mode, int flags, 
 		      struct stat *attr, Fh **fhp, int uid, int gid)
 {
   Mutex::Locker lock(client_lock);
   dout(3) << "ll_create " << parent << " " << name << " 0" << oct << mode << dec << " " << flags << dendl;
   tout << "ll_create" << std::endl;
-  tout << parent.val << std::endl;
+  tout << parent.ino.val << std::endl;
   tout << name << std::endl;
   tout << mode << std::endl;
   tout << flags << std::endl;
diff --git a/src/client/Client.h b/src/client/Client.h
index cfc94f7f489f4..61582fa60a0ca 100644
--- a/src/client/Client.h
+++ b/src/client/Client.h
@@ -66,7 +66,6 @@ extern class LogType client_logtype;
 extern class Logger  *client_logger;
 
 
-
 // ============================================
 // types for my local metadata cache
 /* basic structure:
@@ -170,6 +169,7 @@ struct InodeCap {
 class Inode {
  public:
   inode_t   inode;    // the actual inode
+  snapid_t  snapid;
   int       lease_mask, lease_mds;
   utime_t   lease_ttl;
 
@@ -235,8 +235,9 @@ class Inode {
     ll_ref -= n;
   }
 
-  Inode(inodeno_t ino, ceph_file_layout *layout) : 
+  Inode(vinodeno_t vino, ceph_file_layout *layout) : 
     //inode(_inode),
+    snapid(vino.snapid),
     lease_mask(0), lease_mds(-1),
     dir_auth(-1), dir_hashed(false), dir_replicated(false), 
     exporting_issued(0), exporting_mds(-1), exporting_mseq(0),
@@ -248,13 +249,14 @@ class Inode {
   {
     memset(&inode, 0, sizeof(inode));
     //memset(open_by_mode, 0, sizeof(int)*CEPH_FILE_MODE_NUM);
-    inode.ino = ino;
+    inode.ino = vino.ino;
   }
   ~Inode() {
     if (symlink) { delete symlink; symlink = 0; }
   }
 
   inodeno_t ino() { return inode.ino; }
+  vinodeno_t vino() { return vinodeno_t(inode.ino, snapid); }
 
   bool is_dir() { return inode.is_dir(); }
 
@@ -598,7 +600,7 @@ protected:
   Objecter              *objecter;     // (non-blocking) osd interface
   
   // cache
-  hash_map<inodeno_t, Inode*> inode_map;
+  hash_map<vinodeno_t, Inode*> inode_map;
   Inode*                 root;
   LRU                    lru;    // lru list of Dentry's in our local metadata cache.
 
@@ -659,7 +661,7 @@ protected:
     in->put(n);
     if (in->ref == 0) {
       //cout << "put_inode deleting " << in << " " << in->inode.ino << std::endl;
-      inode_map.erase(in->inode.ino);
+      inode_map.erase(in->vino());
       if (in == root) root = 0;
       delete in;
     }
@@ -764,7 +766,7 @@ protected:
   void dump_cache();  // debug
   
   // find dentry based on filepath
-  Dentry *lookup(const filepath& path);
+  Dentry *lookup(const filepath& path, snapid_t snap=CEPH_NOSNAP);
 
   int fill_stat(Inode *in, struct stat *st, frag_info_t *dirstat=0);
 
@@ -973,33 +975,33 @@ public:
   int rmsnap(const char *path, const char *name);
 
   // low-level interface
-  int ll_lookup(inodeno_t parent, const char *name, struct stat *attr, int uid = -1, int gid = -1);
-  bool ll_forget(inodeno_t ino, int count);
-  Inode *_ll_get_inode(inodeno_t ino);
-  int ll_getattr(inodeno_t ino, struct stat *st, int uid = -1, int gid = -1);
-  int ll_setattr(inodeno_t ino, struct stat *st, int mask, int uid = -1, int gid = -1);
-  int ll_getxattr(inodeno_t ino, const char *name, void *value, size_t size, int uid=-1, int gid=-1);
-  int ll_setxattr(inodeno_t ino, const char *name, const void *value, size_t size, int flags, int uid=-1, int gid=-1);
-  int ll_removexattr(inodeno_t ino, const char *name, int uid=-1, int gid=-1);
-  int ll_listxattr(inodeno_t ino, char *list, size_t size, int uid=-1, int gid=-1);
-  int ll_opendir(inodeno_t ino, void **dirpp, int uid = -1, int gid = -1);
+  int ll_lookup(vinodeno_t parent, const char *name, struct stat *attr, int uid = -1, int gid = -1);
+  bool ll_forget(vinodeno_t vino, int count);
+  Inode *_ll_get_inode(vinodeno_t vino);
+  int ll_getattr(vinodeno_t vino, struct stat *st, int uid = -1, int gid = -1);
+  int ll_setattr(vinodeno_t vino, struct stat *st, int mask, int uid = -1, int gid = -1);
+  int ll_getxattr(vinodeno_t vino, const char *name, void *value, size_t size, int uid=-1, int gid=-1);
+  int ll_setxattr(vinodeno_t vino, const char *name, const void *value, size_t size, int flags, int uid=-1, int gid=-1);
+  int ll_removexattr(vinodeno_t vino, const char *name, int uid=-1, int gid=-1);
+  int ll_listxattr(vinodeno_t vino, char *list, size_t size, int uid=-1, int gid=-1);
+  int ll_opendir(vinodeno_t vino, void **dirpp, int uid = -1, int gid = -1);
   void ll_releasedir(void *dirp);
-  int ll_readlink(inodeno_t ino, const char **value, int uid = -1, int gid = -1);
-  int ll_mknod(inodeno_t ino, const char *name, mode_t mode, dev_t rdev, struct stat *attr, int uid = -1, int gid = -1);
-  int ll_mkdir(inodeno_t ino, const char *name, mode_t mode, struct stat *attr, int uid = -1, int gid = -1);
-  int ll_symlink(inodeno_t ino, const char *name, const char *value, struct stat *attr, int uid = -1, int gid = -1);
-  int ll_unlink(inodeno_t ino, const char *name, int uid = -1, int gid = -1);
-  int ll_rmdir(inodeno_t ino, const char *name, int uid = -1, int gid = -1);
-  int ll_rename(inodeno_t parent, const char *name, inodeno_t newparent, const char *newname, int uid = -1, int gid = -1);
-  int ll_link(inodeno_t ino, inodeno_t newparent, const char *newname, struct stat *attr, int uid = -1, int gid = -1);
-  int ll_open(inodeno_t ino, int flags, Fh **fh, int uid = -1, int gid = -1);
-  int ll_create(inodeno_t parent, const char *name, mode_t mode, int flags, struct stat *attr, Fh **fh, int uid = -1, int gid = -1);
+  int ll_readlink(vinodeno_t vino, const char **value, int uid = -1, int gid = -1);
+  int ll_mknod(vinodeno_t vino, const char *name, mode_t mode, dev_t rdev, struct stat *attr, int uid = -1, int gid = -1);
+  int ll_mkdir(vinodeno_t vino, const char *name, mode_t mode, struct stat *attr, int uid = -1, int gid = -1);
+  int ll_symlink(vinodeno_t vino, const char *name, const char *value, struct stat *attr, int uid = -1, int gid = -1);
+  int ll_unlink(vinodeno_t vino, const char *name, int uid = -1, int gid = -1);
+  int ll_rmdir(vinodeno_t vino, const char *name, int uid = -1, int gid = -1);
+  int ll_rename(vinodeno_t parent, const char *name, vinodeno_t newparent, const char *newname, int uid = -1, int gid = -1);
+  int ll_link(vinodeno_t vino, vinodeno_t newparent, const char *newname, struct stat *attr, int uid = -1, int gid = -1);
+  int ll_open(vinodeno_t vino, int flags, Fh **fh, int uid = -1, int gid = -1);
+  int ll_create(vinodeno_t parent, const char *name, mode_t mode, int flags, struct stat *attr, Fh **fh, int uid = -1, int gid = -1);
   int ll_read(Fh *fh, loff_t off, loff_t len, bufferlist *bl);
   int ll_write(Fh *fh, loff_t off, loff_t len, const char *data);
   int ll_flush(Fh *fh);
   int ll_fsync(Fh *fh, bool syncdataonly);
   int ll_release(Fh *fh);
-  int ll_statfs(inodeno_t, struct statvfs *stbuf);
+  int ll_statfs(vinodeno_t vino, struct statvfs *stbuf);
 
   // failure
   void ms_handle_failure(Message*, const entity_inst_t& inst);
diff --git a/src/client/SyntheticClient.cc b/src/client/SyntheticClient.cc
index 6a527bca36620..b0079f182f015 100644
--- a/src/client/SyntheticClient.cc
+++ b/src/client/SyntheticClient.cc
@@ -929,7 +929,7 @@ int SyntheticClient::play_trace(Trace& t, string& prefix, bool metadata_only)
   if (prefix.length()) {
     client->mkdir(prefix.c_str(), 0755);
     struct stat attr;
-    if (client->ll_lookup(1, prefix.c_str(), &attr) == 0) {
+    if (client->ll_lookup(vinodeno_t(1, CEPH_NOSNAP), prefix.c_str(), &attr) == 0) {
       ll_inos[1] = attr.st_ino;
       dout(5) << "'root' ino is " << inodeno_t(attr.st_ino) << dendl;
     } else {
@@ -1129,19 +1129,19 @@ int SyntheticClient::play_trace(Trace& t, string& prefix, bool metadata_only)
       int64_t r = t.get_int();
       struct stat attr;
       if (ll_inos.count(i) &&
-	  client->ll_lookup(ll_inos[i], name, &attr) == 0)
+	  client->ll_lookup(vinodeno_t(ll_inos[i],CEPH_NOSNAP), name, &attr) == 0)
 	ll_inos[r] = attr.st_ino;
     } else if (strcmp(op, "ll_forget") == 0) {
       int64_t i = t.get_int();
       int64_t n = t.get_int();
       if (ll_inos.count(i) && 
-	  client->ll_forget(ll_inos[i], n))
+	  client->ll_forget(vinodeno_t(ll_inos[i],CEPH_NOSNAP), n))
 	ll_inos.erase(i);
     } else if (strcmp(op, "ll_getattr") == 0) {
       int64_t i = t.get_int();
       struct stat attr;
       if (ll_inos.count(i))
-	client->ll_getattr(ll_inos[i], &attr);
+	client->ll_getattr(vinodeno_t(ll_inos[i],CEPH_NOSNAP), &attr);
     } else if (strcmp(op, "ll_setattr") == 0) {
       int64_t i = t.get_int();
       struct stat attr;
@@ -1154,12 +1154,12 @@ int SyntheticClient::play_trace(Trace& t, string& prefix, bool metadata_only)
       attr.st_atime = t.get_int();
       int mask = t.get_int();
       if (ll_inos.count(i))
-	client->ll_setattr(ll_inos[i], &attr, mask);
+	client->ll_setattr(vinodeno_t(ll_inos[i],CEPH_NOSNAP), &attr, mask);
     } else if (strcmp(op, "ll_readlink") == 0) {
       int64_t i = t.get_int();
       const char *value;
       if (ll_inos.count(i))
-	client->ll_readlink(ll_inos[i], &value);
+	client->ll_readlink(vinodeno_t(ll_inos[i],CEPH_NOSNAP), &value);
     } else if (strcmp(op, "ll_mknod") == 0) {
       int64_t i = t.get_int();
       const char *n = t.get_string(buf, p);
@@ -1168,7 +1168,7 @@ int SyntheticClient::play_trace(Trace& t, string& prefix, bool metadata_only)
       int64_t ri = t.get_int();
       struct stat attr;
       if (ll_inos.count(i) &&
-	  client->ll_mknod(ll_inos[i], n, m, r, &attr) == 0)
+	  client->ll_mknod(vinodeno_t(ll_inos[i],CEPH_NOSNAP), n, m, r, &attr) == 0)
 	ll_inos[ri] = attr.st_ino;
     } else if (strcmp(op, "ll_mkdir") == 0) {
       int64_t i = t.get_int();
@@ -1177,7 +1177,7 @@ int SyntheticClient::play_trace(Trace& t, string& prefix, bool metadata_only)
       int64_t ri = t.get_int();
       struct stat attr;
       if (ll_inos.count(i) &&
-	  client->ll_mkdir(ll_inos[i], n, m, &attr) == 0)
+	  client->ll_mkdir(vinodeno_t(ll_inos[i],CEPH_NOSNAP), n, m, &attr) == 0)
 	ll_inos[ri] = attr.st_ino;
     } else if (strcmp(op, "ll_symlink") == 0) {
       int64_t i = t.get_int();
@@ -1186,18 +1186,18 @@ int SyntheticClient::play_trace(Trace& t, string& prefix, bool metadata_only)
       int64_t ri = t.get_int();
       struct stat attr;
       if (ll_inos.count(i) &&
-	  client->ll_symlink(ll_inos[i], n, v, &attr) == 0)
+	  client->ll_symlink(vinodeno_t(ll_inos[i],CEPH_NOSNAP), n, v, &attr) == 0)
 	ll_inos[ri] = attr.st_ino;
     } else if (strcmp(op, "ll_unlink") == 0) {
       int64_t i = t.get_int();
       const char *n = t.get_string(buf, p);
       if (ll_inos.count(i))
-	client->ll_unlink(ll_inos[i], n);
+	client->ll_unlink(vinodeno_t(ll_inos[i],CEPH_NOSNAP), n);
     } else if (strcmp(op, "ll_rmdir") == 0) {
       int64_t i = t.get_int();
       const char *n = t.get_string(buf, p);
       if (ll_inos.count(i))
-	client->ll_rmdir(ll_inos[i], n);
+	client->ll_rmdir(vinodeno_t(ll_inos[i],CEPH_NOSNAP), n);
     } else if (strcmp(op, "ll_rename") == 0) {
       int64_t i = t.get_int();
       const char *n = t.get_string(buf, p);
@@ -1205,7 +1205,8 @@ int SyntheticClient::play_trace(Trace& t, string& prefix, bool metadata_only)
       const char *nn = t.get_string(buf2, p);
       if (ll_inos.count(i) &&
 	  ll_inos.count(ni))
-	client->ll_rename(ll_inos[i], n, ll_inos[ni], nn);
+	client->ll_rename(vinodeno_t(ll_inos[i],CEPH_NOSNAP), n, 
+			  vinodeno_t(ll_inos[ni],CEPH_NOSNAP), nn);
     } else if (strcmp(op, "ll_link") == 0) {
       int64_t i = t.get_int();
       int64_t ni = t.get_int();
@@ -1213,13 +1214,14 @@ int SyntheticClient::play_trace(Trace& t, string& prefix, bool metadata_only)
       struct stat attr;
       if (ll_inos.count(i) &&
 	  ll_inos.count(ni))
-      client->ll_link(ll_inos[i], ll_inos[ni], nn, &attr);
+      client->ll_link(vinodeno_t(ll_inos[i],CEPH_NOSNAP),
+		      vinodeno_t(ll_inos[ni],CEPH_NOSNAP), nn, &attr);
     } else if (strcmp(op, "ll_opendir") == 0) {
       int64_t i = t.get_int();
       int64_t r = t.get_int();
       void *dirp;
       if (ll_inos.count(i) &&
-	  client->ll_opendir(ll_inos[i], &dirp) == 0)
+	  client->ll_opendir(vinodeno_t(ll_inos[i],CEPH_NOSNAP), &dirp) == 0)
 	ll_dirs[r] = dirp;
     } else if (strcmp(op, "ll_releasedir") == 0) {
       int64_t f = t.get_int();
@@ -1233,7 +1235,7 @@ int SyntheticClient::play_trace(Trace& t, string& prefix, bool metadata_only)
       int64_t r = t.get_int();
       Fh *fhp;
       if (ll_inos.count(i) &&
-	  client->ll_open(ll_inos[i], f, &fhp) == 0)
+	  client->ll_open(vinodeno_t(ll_inos[i],CEPH_NOSNAP), f, &fhp) == 0)
 	ll_files[r] = fhp;
     } else if (strcmp(op, "ll_create") == 0) {
       int64_t i = t.get_int();
@@ -1245,7 +1247,7 @@ int SyntheticClient::play_trace(Trace& t, string& prefix, bool metadata_only)
       Fh *fhp;
       struct stat attr;
       if (ll_inos.count(i) &&
-	  client->ll_create(ll_inos[i], n, m, f, &attr, &fhp) == 0) {
+	  client->ll_create(vinodeno_t(ll_inos[i],CEPH_NOSNAP), n, m, f, &attr, &fhp) == 0) {
 	ll_inos[ri] = attr.st_ino;
 	ll_files[r] = fhp;
       }
@@ -1292,7 +1294,7 @@ int SyntheticClient::play_trace(Trace& t, string& prefix, bool metadata_only)
     } else if (strcmp(op, "ll_statfs") == 0) {
       int64_t i = t.get_int();
       if (ll_inos.count(i))
-	{} //client->ll_statfs(ll_inos[i]);
+	{} //client->ll_statfs(vinodeno_t(ll_inos[i],CEPH_NOSNAP));
     } 
 
 
diff --git a/src/client/fuse_ll.cc b/src/client/fuse_ll.cc
index 3dd3ac4f9a758..df89412f583bf 100644
--- a/src/client/fuse_ll.cc
+++ b/src/client/fuse_ll.cc
@@ -30,6 +30,40 @@
 
 static Client *client;
 
+Mutex stag_lock;
+int last_stag = 0;
+hash_map<__u64,int> snap_stag_map;
+hash_map<int,__u64> stag_snap_map;
+
+#define FINO_INO(x) ((x) & ((1ull<<40)-1ull))
+#define FINO_STAG(x) ((x) >> 40)
+#define MAKE_FINO(i,s) ((i) | ((s) << 40))
+
+static __u64 fino_snap(__u64 fino)
+{
+  Mutex::Locker l(stag_lock);
+  __u64 stag = FINO_STAG(fino);
+  assert(stag_snap_map.count(stag));
+  return stag_snap_map[stag];
+}
+static vinodeno_t fino_vino(inodeno_t fino)
+{
+  return vinodeno_t(FINO_INO(fino), fino_snap(fino));
+}
+
+
+static __u64 make_fake_ino(inodeno_t ino, snapid_t snapid)
+{
+  Mutex::Locker l(stag_lock);
+  __u64 stag;
+  if (snap_stag_map.count(snapid) == 0) {
+    stag = ++last_stag;
+    snap_stag_map[snapid] = stag;
+    stag_snap_map[stag] = snapid;
+  } else 
+    stag = snap_stag_map[snapid];
+  return MAKE_FINO(ino, stag);
+}
 
 static void ceph_ll_lookup(fuse_req_t req, fuse_ino_t parent, const char *name)
 {
@@ -38,9 +72,9 @@ static void ceph_ll_lookup(fuse_req_t req, fuse_ino_t parent, const char *name)
   int stmask;
 
   memset(&fe, 0, sizeof(fe));
-  stmask = client->ll_lookup(parent, name, &fe.attr, ctx->uid, ctx->gid);
+  stmask = client->ll_lookup(fino_vino(parent), name, &fe.attr, ctx->uid, ctx->gid);
   if (stmask >= 0) {
-    fe.ino = fe.attr.st_ino;
+    fe.ino = make_fake_ino(fe.attr.st_ino, fe.attr.st_dev);
     fuse_reply_entry(req, &fe);
   } else {
     fuse_reply_err(req, ENOENT);
@@ -49,7 +83,7 @@ static void ceph_ll_lookup(fuse_req_t req, fuse_ino_t parent, const char *name)
 
 static void ceph_ll_forget(fuse_req_t req, fuse_ino_t ino, long unsigned nlookup)
 {
-  client->ll_forget(ino, nlookup);
+  client->ll_forget(fino_vino(ino), nlookup);
   fuse_reply_none(req);
 }
 
@@ -61,7 +95,7 @@ static void ceph_ll_getattr(fuse_req_t req, fuse_ino_t ino,
   
   (void) fi;
 
-  if (client->ll_getattr(ino, &stbuf, ctx->uid, ctx->gid) == 0) 
+  if (client->ll_getattr(fino_vino(ino), &stbuf, ctx->uid, ctx->gid) == 0) 
     fuse_reply_attr(req, &stbuf, 0);
   else
     fuse_reply_err(req, ENOENT);
@@ -71,7 +105,7 @@ static void ceph_ll_setattr(fuse_req_t req, fuse_ino_t ino, struct stat *attr,
 			    int to_set, struct fuse_file_info *fi)
 {
   const struct fuse_ctx *ctx = fuse_req_ctx(req);
-  int r = client->ll_setattr(ino, attr, to_set, ctx->uid, ctx->gid);
+  int r = client->ll_setattr(fino_vino(ino), attr, to_set, ctx->uid, ctx->gid);
   if (r == 0)
     fuse_reply_attr(req, attr, 0);
   else
@@ -84,7 +118,7 @@ static void ceph_ll_setxattr(fuse_req_t req, fuse_ino_t ino, const char *name,
 			     const char *value, size_t size, int flags)
 {
   const struct fuse_ctx *ctx = fuse_req_ctx(req);
-  int r = client->ll_setxattr(ino, name, value, size, flags, ctx->uid, ctx->gid);
+  int r = client->ll_setxattr(fino_vino(ino), name, value, size, flags, ctx->uid, ctx->gid);
   fuse_reply_err(req, -r);
 }
 
@@ -92,7 +126,7 @@ static void ceph_ll_listxattr(fuse_req_t req, fuse_ino_t ino, size_t size)
 {
   const struct fuse_ctx *ctx = fuse_req_ctx(req);
   char buf[size];
-  int r = client->ll_listxattr(ino, buf, size, ctx->uid, ctx->gid);
+  int r = client->ll_listxattr(fino_vino(ino), buf, size, ctx->uid, ctx->gid);
   if (size == 0 && r >= 0)
     fuse_reply_xattr(req, r);
   else if (r >= 0) 
@@ -106,7 +140,7 @@ static void ceph_ll_getxattr(fuse_req_t req, fuse_ino_t ino, const char *name,
 {
   const struct fuse_ctx *ctx = fuse_req_ctx(req);
   char buf[size];
-  int r = client->ll_getxattr(ino, name, buf, size, ctx->uid, ctx->gid);
+  int r = client->ll_getxattr(fino_vino(ino), name, buf, size, ctx->uid, ctx->gid);
   if (size == 0 && r >= 0)
     fuse_reply_xattr(req, r);
   else if (r >= 0)
@@ -118,7 +152,7 @@ static void ceph_ll_getxattr(fuse_req_t req, fuse_ino_t ino, const char *name,
 static void ceph_ll_removexattr(fuse_req_t req, fuse_ino_t ino, const char *name)
 {
   const struct fuse_ctx *ctx = fuse_req_ctx(req);
-  int r = client->ll_removexattr(ino, name, ctx->uid, ctx->gid);
+  int r = client->ll_removexattr(fino_vino(ino), name, ctx->uid, ctx->gid);
   fuse_reply_err(req, -r);
 }
 
@@ -128,7 +162,7 @@ static void ceph_ll_opendir(fuse_req_t req, fuse_ino_t ino, struct fuse_file_inf
 {
   const struct fuse_ctx *ctx = fuse_req_ctx(req);
   void *dirp;
-  int r = client->ll_opendir(ino, &dirp, ctx->uid, ctx->gid);
+  int r = client->ll_opendir(fino_vino(ino), &dirp, ctx->uid, ctx->gid);
   if (r >= 0) {
     fi->fh = (long)dirp;
     fuse_reply_open(req, fi);
@@ -141,7 +175,7 @@ static void ceph_ll_readlink(fuse_req_t req, fuse_ino_t ino)
 {
   const struct fuse_ctx *ctx = fuse_req_ctx(req);
   const char *value;
-  int r = client->ll_readlink(ino, &value, ctx->uid, ctx->gid);
+  int r = client->ll_readlink(fino_vino(ino), &value, ctx->uid, ctx->gid);
   if (r == 0) 
     fuse_reply_readlink(req, value);
   else
@@ -155,9 +189,9 @@ static void ceph_ll_mknod(fuse_req_t req, fuse_ino_t parent, const char *name,
   struct fuse_entry_param fe;
   memset(&fe, 0, sizeof(fe));
 
-  int r = client->ll_mknod(parent, name, mode, rdev, &fe.attr, ctx->uid, ctx->gid);
+  int r = client->ll_mknod(fino_vino(parent), name, mode, rdev, &fe.attr, ctx->uid, ctx->gid);
   if (r == 0) {
-    fe.ino = fe.attr.st_ino;
+    fe.ino = make_fake_ino(fe.attr.st_ino, fe.attr.st_dev);
     fuse_reply_entry(req, &fe);
   } else {
     fuse_reply_err(req, -r);
@@ -171,9 +205,9 @@ static void ceph_ll_mkdir(fuse_req_t req, fuse_ino_t parent, const char *name,
   struct fuse_entry_param fe;
   memset(&fe, 0, sizeof(fe));
 
-  int r = client->ll_mkdir(parent, name, mode, &fe.attr, ctx->uid, ctx->gid);
+  int r = client->ll_mkdir(fino_vino(parent), name, mode, &fe.attr, ctx->uid, ctx->gid);
   if (r == 0) {
-    fe.ino = fe.attr.st_ino;
+    fe.ino = make_fake_ino(fe.attr.st_ino, fe.attr.st_dev);
     fuse_reply_entry(req, &fe);
   } else {
     fuse_reply_err(req, -r);
@@ -183,14 +217,14 @@ static void ceph_ll_mkdir(fuse_req_t req, fuse_ino_t parent, const char *name,
 static void ceph_ll_unlink(fuse_req_t req, fuse_ino_t parent, const char *name)
 {
   const struct fuse_ctx *ctx = fuse_req_ctx(req);
-  int r = client->ll_unlink(parent, name, ctx->uid, ctx->gid);
+  int r = client->ll_unlink(fino_vino(parent), name, ctx->uid, ctx->gid);
   fuse_reply_err(req, -r);
 }
 
 static void ceph_ll_rmdir(fuse_req_t req, fuse_ino_t parent, const char *name)
 {
   const struct fuse_ctx *ctx = fuse_req_ctx(req);
-  int r = client->ll_rmdir(parent, name, ctx->uid, ctx->gid);
+  int r = client->ll_rmdir(fino_vino(parent), name, ctx->uid, ctx->gid);
   fuse_reply_err(req, -r);
 }
 
@@ -200,9 +234,9 @@ static void ceph_ll_symlink(fuse_req_t req, const char *existing, fuse_ino_t par
   struct fuse_entry_param fe;
   memset(&fe, 0, sizeof(fe));
 
-  int r = client->ll_symlink(parent, name, existing, &fe.attr, ctx->uid, ctx->gid);
+  int r = client->ll_symlink(fino_vino(parent), name, existing, &fe.attr, ctx->uid, ctx->gid);
   if (r == 0) {
-    fe.ino = fe.attr.st_ino;
+    fe.ino = make_fake_ino(fe.attr.st_ino, fe.attr.st_dev);
     fuse_reply_entry(req, &fe);
   } else {
     fuse_reply_err(req, -r);
@@ -213,7 +247,7 @@ static void ceph_ll_rename(fuse_req_t req, fuse_ino_t parent, const char *name,
 			   fuse_ino_t newparent, const char *newname)
 {
   const struct fuse_ctx *ctx = fuse_req_ctx(req);
-  int r = client->ll_rename(parent, name, newparent, newname, ctx->uid, ctx->gid);
+  int r = client->ll_rename(fino_vino(parent), name, fino_vino(newparent), newname, ctx->uid, ctx->gid);
   fuse_reply_err(req, -r);
 }
 
@@ -224,9 +258,9 @@ static void ceph_ll_link(fuse_req_t req, fuse_ino_t ino, fuse_ino_t newparent,
   struct fuse_entry_param fe;
   memset(&fe, 0, sizeof(fe));
   
-  int r = client->ll_link(ino, newparent, newname, &fe.attr, ctx->uid, ctx->gid);
+  int r = client->ll_link(fino_vino(ino), fino_vino(newparent), newname, &fe.attr, ctx->uid, ctx->gid);
   if (r == 0) {
-    fe.ino = fe.attr.st_ino;
+    fe.ino = make_fake_ino(fe.attr.st_ino, fe.attr.st_dev);
     fuse_reply_entry(req, &fe);
   } else {
     fuse_reply_err(req, -r);
@@ -237,7 +271,7 @@ static void ceph_ll_open(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *
 {
   const struct fuse_ctx *ctx = fuse_req_ctx(req);
   Fh *fh;
-  int r = client->ll_open(ino, fi->flags, &fh, ctx->uid, ctx->gid);
+  int r = client->ll_open(fino_vino(ino), fi->flags, &fh, ctx->uid, ctx->gid);
   if (r == 0) {
     fi->fh = (long)fh;
     fuse_reply_open(req, fi);
@@ -312,10 +346,12 @@ static void ceph_ll_readdir(fuse_req_t req, fuse_ino_t ino, size_t size,
   struct stat st;
   memset(&st, 0, sizeof(st));
 
+  __u64 snap = fino_snap(ino);
+
   while (1) {
     int r = client->readdir_r(dirp, &de);
     if (r < 0) break;
-    st.st_ino = de.d_ino;
+    st.st_ino = make_fake_ino(de.d_ino, snap);
     st.st_mode = DT_TO_MODE(de.d_type);
 
     off_t off = client->telldir(dirp);
@@ -353,10 +389,10 @@ static void ceph_ll_create(fuse_req_t req, fuse_ino_t parent, const char *name,
   struct fuse_entry_param fe;
   memset(&fe, 0, sizeof(fe));
   Fh *fh;
-  int r = client->ll_create(parent, name, mode, fi->flags, &fe.attr, &fh, ctx->uid, ctx->gid);
+  int r = client->ll_create(fino_vino(parent), name, mode, fi->flags, &fe.attr, &fh, ctx->uid, ctx->gid);
   if (r == 0) {
     fi->fh = (long)fh;
-    fe.ino = fe.attr.st_ino;
+    fe.ino = make_fake_ino(fe.attr.st_ino, fe.attr.st_dev);
     fuse_reply_create(req, &fe, fi);
   } else {
     fuse_reply_err(req, -r);
@@ -366,7 +402,7 @@ static void ceph_ll_create(fuse_req_t req, fuse_ino_t parent, const char *name,
 static void ceph_ll_statfs(fuse_req_t req, fuse_ino_t ino)
 {
   struct statvfs stbuf;
-  int r = client->ll_statfs(ino, &stbuf);
+  int r = client->ll_statfs(fino_vino(ino), &stbuf);
   if (r == 0)
     fuse_reply_statfs(req, &stbuf);
   else
diff --git a/src/include/ceph_fs.h b/src/include/ceph_fs.h
index c1e01c14cdcdd..155d69a39f0f4 100644
--- a/src/include/ceph_fs.h
+++ b/src/include/ceph_fs.h
@@ -692,6 +692,7 @@ struct ceph_frag_tree_head {
 
 struct ceph_mds_reply_inode {
 	ceph_ino_t ino;
+	__le64 snapid;
 	__le64 version;
 	struct ceph_file_layout layout;
 	struct ceph_timespec ctime, mtime, atime;
diff --git a/src/mds/mdstypes.h b/src/mds/mdstypes.h
index d70bb3b3c2d24..8504f3b86b24e 100644
--- a/src/mds/mdstypes.h
+++ b/src/mds/mdstypes.h
@@ -153,7 +153,7 @@ inline ostream& operator<<(ostream &out, const frag_info_t &f) {
 struct vinodeno_t {
   inodeno_t ino;
   snapid_t snapid;
-
+  vinodeno_t() {}
   vinodeno_t(inodeno_t i, snapid_t s) : ino(i), snapid(s) {}
 };
 
diff --git a/src/messages/MClientReply.h b/src/messages/MClientReply.h
index 7722584db1b0e..265abd2cf3163 100644
--- a/src/messages/MClientReply.h
+++ b/src/messages/MClientReply.h
@@ -81,7 +81,7 @@ struct DirStat {
 };
 
 struct InodeStat {
-  inodeno_t ino;
+  vinodeno_t vino;
   version_t version;
   ceph_file_layout layout;
   utime_t ctime, mtime, atime;
@@ -104,7 +104,8 @@ struct InodeStat {
   void decode(bufferlist::iterator &p) {
     struct ceph_mds_reply_inode e;
     ::decode(e, p);
-    ino = inodeno_t(e.ino);
+    vino.ino = inodeno_t(e.ino);
+    vino.snapid = snapid_t(e.snapid);
     version = e.version;
     layout = e.layout;
     ctime.decode_timeval(&e.ctime);
-- 
2.39.5