Modified Files:

 	config.cc client/Client.cc client/Client.h include/config.h

- add Client::insert_inode_info and use it in Client::insert_trace and
  Client::getdir UNTESTED
- add g_conf.client_cache_stat_ttl UNTESTED
- all fs ops except mknod, open, read, write, truncate, and fsync now use the
  metadata cache UNTESTED

git-svn-id: https://ceph.svn.sf.net/svnroot/ceph@209 29311d96-e01e-0410-9327-a35deaab8ce9

diff --git a/ceph/client/Client.cc b/ceph/client/Client.cc
index 8ae2045c25b596d3ec24d4cee5b16fcd0e1f557b..9ac16ff645100c8c9a6594a659474ad25ed179f1 100644 (file)
--- a/ceph/client/Client.cc
+++ b/ceph/client/Client.cc
@@ -4,6 +4,7 @@
 
 // unix-ey fs stuff
 #include <sys/types.h>
+#include <time.h>
 #include <utime.h>
 
 
@@ -46,62 +47,76 @@ void Client::shutdown() {
   messenger->shutdown();
 }
 
+// insert inode info into metadata cache
+
+Inode* Client::insert_inode_info(Dir *dir, c_inode_info *in_info)
+{
+  string dname = in_info->ref_dn;
+  Dentry *dn = NULL;
+  if (dir->dentries.count(dname))
+       dn = dir->dentries[dname];
+  dout(12) << "insert_inode_info " << dname << " ino " << in_info->inode.ino << endl;
+  
+  if (dn) {
+       if (dn->inode->inode.ino == in_info->inode.ino) {
+         touch_dn(next);
+         dout(12) << " had dentry " << dname << " with correct ino " << dn->inode->inode.ino << endl;
+       } else {
+         dout(12) << " had dentry " << dname << " with WRONG ino " << dn->inode->inode.ino << endl;
+         unlink(dn);
+         dn = NULL;
+       }
+  }
+  
+  if (!dn) {
+       if (inode_map.count(in_info->inode.ino)) {
+         Inode *in = inode_map[in_info->inode.ino];
+         if (in) {
+               dout(12) << " had ino " << dn->inode->inode.ino << " at wrong position, moving" << endl;
+               if (in->dn) unlink(in->dn);
+               dn = link(dir, dname, in);
+         }
+       }
+  }
+  
+  if (!dn) {
+       dn = link(dir, dname, new Inode());
+       dn->inode->inode = in_info->inode;
+       inode_map[dn->inode->inode.ino] = dn->inode;
+       dout(12) << " new dentry+node with ino " << dn->inode->inode.ino << endl;
+  }
+
+  // take note of latest distribution on mds's
+  dn->inode->mds_contacts = in_info->dist;
+
+  return dn->inode;
+}
+
 // insert trace of reply into metadata cache
 
 void Client::insert_trace(vector<c_inode_info*> trace)
 {
   Inode *cur = root;
+  time_t now = time(NULL);
 
+  if (!trace) return;
+  
   for (int i=0; i<trace.size(); i++) {
-       if (i == 0) {
-         if (!root) {
-               cur = root = new Inode();
-           root->inode = trace[i]->inode;
-               inode_map[root->inode.ino] = root;
-         }
-         dout(12) << "insert_trace trace " << i << " root" << endl;
-       } else {
-         Dir *dir = open_dir( cur );
-         string dname = trace[i]->ref_dn;
-         Dentry *next = NULL;
-         if (dir->dentries.count(dname))
-           next = dir->dentries[dname];
-         dout(12) << "insert_trace trace " << i << " dname " << dname << " ino " << trace[i]->inode.ino << endl;
-         
-         if (next) {
-               if (next->inode->inode.ino == trace[i]->inode.ino) {
-                 touch_dn(next);
-                 dout(12) << " had dentry " << dname << " with correct ino " << next->inode->inode.ino << endl;
-           } else {
-                 dout(12) << " had dentry " << dname << " with WRONG ino " << next->inode->inode.ino << endl;
-                 unlink(next);
-                 next = NULL;
-           }
-         }
-         
-         if (!next) {
-           if (inode_map.count(trace[i]->inode.ino)) {
-             Inode *in = inode_map[trace[i]->inode.ino];
-                 if (in) {
-                   dout(12) << " had ino " << next->inode->inode.ino << " at wrong position, moving" << endl;
-                       if (in->dn) unlink(in->dn);
-                   next = link(dir, dname, in);
-                 }
-               }
-         }
-         
-         if (!next) {
-           next = link(dir, dname, new Inode());
-               next->inode->inode = trace[i]->inode;
-               inode_map[next->inode->inode.ino] = next->inode;
-               dout(12) << " new dentry+node with ino " << next->inode->inode.ino << endl;
-         }
-
-         cur = next->inode;
-       }
-
-       // take note of latest distribution on mds's
-       cur->mds_contacts = trace[i]->dist;
+    if (i == 0) {
+      if (!root) {
+        cur = root = new Inode();
+        root->inode = trace[i]->inode;
+        inode_map[root->inode.ino] = root;
+      }
+         root->mds_contacts = trace[i]->dist;
+         root->last_updated = now;
+      dout(12) << "insert_trace trace " << i << " root" << endl;
+    } else {
+      dout(12) << "insert_trace trace " << i;
+      Dir *dir = open_dir( cur );
+         cur = this->insert_inode_info(dir, trace[i]);
+         cur->last_updated = now;
+    }    
   }
 }
 
@@ -125,6 +140,10 @@ int Client::unlink(const char *path)
    
   MClientReply *reply = (MClientReply*)messenger->sendrecv(req, MSG_ADDR_MDS(0), MDS_PORT_SERVER);
   int res = reply->get_result();
+  if (res == 0) {
+       unlink(lookup(req->get_filepath()));
+  }
+  this->insert_trace(reply->get_trace);
   delete reply;
   dout(10) << "unlink result is " << res << endl;
   return res;
@@ -145,6 +164,7 @@ int Client::rename(const char *from, const char *to)
    
   MClientReply *reply = (MClientReply*)messenger->sendrecv(req, MSG_ADDR_MDS(0), MDS_PORT_SERVER);
   int res = reply->get_result();
+  this->insert_trace(reply->get_trace);
   delete reply;
   dout(10) << "rename result is " << res << endl;
   return res;
@@ -167,6 +187,7 @@ int Client::mkdir(const char *path, mode_t mode)
    
   MClientReply *reply = (MClientReply*)messenger->sendrecv(req, MSG_ADDR_MDS(0), MDS_PORT_SERVER);
   int res = reply->get_result();
+  this->insert_trace(reply->get_trace);
   delete reply;
   dout(10) << "mkdir result is " << res << endl;
   return res;
@@ -186,6 +207,10 @@ int Client::rmdir(const char *path)
    
   MClientReply *reply = (MClientReply*)messenger->sendrecv(req, MSG_ADDR_MDS(0), MDS_PORT_SERVER);
   int res = reply->get_result();
+  if (res == 0) {
+       unlink(lookup(req->get_filepath()));
+  }
+  this->insert_trace(reply->get_trace);  
   delete reply;
   dout(10) << "rmdir result is " << res << endl;
   return res;
@@ -208,6 +233,7 @@ int Client::symlink(const char *target, const char *link)
    
   MClientReply *reply = (MClientReply*)messenger->sendrecv(req, MSG_ADDR_MDS(0), MDS_PORT_SERVER);
   int res = reply->get_result();
+  this->insert_trace(reply->get_trace);  //FIXME assuming trace of link, not of target
   delete reply;
   dout(10) << "symlink result is " << res << endl;
   return res;
@@ -216,7 +242,7 @@ int Client::symlink(const char *target, const char *link)
 int Client::readlink(const char *path, char *buf, size_t size) 
 { 
   dout(3) << "readlink " << path << endl;
-  // stat first  (OR, FIXME, use cached value?) ****
+  // stat first  (FIXME, PERF access cache directly) ****
   struct stat stbuf;
   int r = this->lstat(path, &stbuf);
   if (r != 0) return r;
@@ -238,25 +264,36 @@ int Client::readlink(const char *path, char *buf, size_t size)
 // inode stuff
 
 int Client::lstat(const char *path, struct stat *stbuf)
-// FIXME make sure this implements lstat and not stat
 {
   dout(3) << "lstat " << path << endl;
+  // FIXME, PERF request allocation convenient but not necessary for cache hit
   MClientRequest *req = new MClientRequest(MDS_OP_STAT, whoami);
   req->set_path(path);
   
-  // FIXME where does FUSE maintain user information
-  req->set_caller_uid(getuid());
-  req->set_caller_gid(getgid());
-
-  MClientReply *reply = (MClientReply*)messenger->sendrecv(req, MSG_ADDR_MDS(0), MDS_PORT_SERVER);
-  int res = reply->get_result();
-  dout(10) << "lstat res is " << res << endl;
-  if (res != 0) return res;
-  
-  //Transfer information from reply to stbuf
-  vector<c_inode_info*> trace = reply->get_trace();
-  inode_t inode = trace[trace.size()-1]->inode;
-
+  // check whether cache content is fresh enough
+  Dentry *dn = lookup(req->get_filepath());
+  inode_t inode;
+  if (dn && ((time(NULL) - dn->inode.last_updated) <= g_conf.client_cache_stat_ttl)) {
+       inode = dn->inode->inode;
+       dout(10) << "lstat cache hit" << endl;
+  } else {  
+       // FIXME where does FUSE maintain user information
+       req->set_caller_uid(getuid());
+       req->set_caller_gid(getgid());
+       
+       MClientReply *reply = (MClientReply*)messenger->sendrecv(req, MSG_ADDR_MDS(0), MDS_PORT_SERVER);
+       int res = reply->get_result();
+       dout(10) << "lstat res is " << res << endl;
+       if (res != 0) return res;
+       
+       //Transfer information from reply to stbuf
+       vector<c_inode_info*> trace = reply->get_trace();
+       inode = trace[trace.size()-1]->inode;
+       
+       //Update metadata cache
+       this->insert_trace(trace);
+  }
+     
   memset(stbuf, 0, sizeof(struct stat));
   //stbuf->st_dev = 
   stbuf->st_ino = inode.ino;
@@ -275,11 +312,12 @@ int Client::lstat(const char *path, struct stat *stbuf)
 
   // gross HACK for now.. put an Inode in cache!  
   // for now i just need to put the symlink content somewhere (so that readlink() will work)
-  Inode *in = new Inode;
-  inode_map[inode.ino] = in;   // put in map so subsequent readlink will find it
-  in->inode = inode;
+  //Inode *in = new Inode;
+  //inode_map[inode.ino] = in;   // put in map so subsequent readlink will find it
+  //in->inode = inode;
 
   // symlink?
+  Inode *in = inode_map[inode.ino];
   if (in->inode.mode & INODE_MODE_SYMLINK) {
        if (!in->symlink) in->symlink = new string;
        *(in->symlink) = trace[trace.size()-1]->symlink;
@@ -287,7 +325,6 @@ int Client::lstat(const char *path, struct stat *stbuf)
 
   dout(1) << "stat sez uid = " << inode.uid << " ino = " << stbuf->st_ino << endl;
 
-  // FIXME need to update cache with trace's inode_info
   delete reply;
   return 0;
 }
@@ -307,6 +344,7 @@ int Client::chmod(const char *path, mode_t mode)
   
   MClientReply *reply = (MClientReply*)messenger->sendrecv(req, MSG_ADDR_MDS(0), MDS_PORT_SERVER);
   int res = reply->get_result();
+  this->insert_trace(reply->get_trace);  
   delete reply;
   dout(10) << "chmod result is " << res << endl;
   return res;
@@ -328,6 +366,7 @@ int Client::chown(const char *path, uid_t uid, gid_t gid)
 
   MClientReply *reply = (MClientReply*)messenger->sendrecv(req, MSG_ADDR_MDS(0), MDS_PORT_SERVER);
   int res = reply->get_result();
+  this->insert_trace(reply->get_trace);  
   delete reply;
   dout(10) << "chown result is " << res << endl;
   return res;
@@ -349,6 +388,7 @@ int Client::utime(const char *path, struct utimbuf *buf)
    
   MClientReply *reply = (MClientReply*)messenger->sendrecv(req, MSG_ADDR_MDS(0), MDS_PORT_SERVER);
   int res = reply->get_result();
+  this->insert_trace(reply->get_trace);  
   delete reply;
   dout(10) << "utime result is " << res << endl;
   return res;
@@ -377,18 +417,32 @@ int Client::getdir(const char *path, map<string,inode_t*>& contents)
    
   MClientReply *reply = (MClientReply*)messenger->sendrecv(req, MSG_ADDR_MDS(0), MDS_PORT_SERVER);
   int res = reply->get_result();
+  vector<c_inode_info*> trace = reply->get_trace();
+  this->insert_trace(trace);  
+
+  if (res) return res;
 
-  // dir contents to caller!
+  // dir contents to cache!
+  Dir *dir = open_dir(inode_map[trace[trace.size()-1]->inode.ino]);
+  time_t now = time(NULL);
   vector<c_inode_info*>::iterator it;
   for (it = reply->get_dir_contents().begin(); 
           it != reply->get_dir_contents().end(); 
           it++) {
        // put in cache
-
-    contents[(*it)->ref_dn] = &(*it)->inode;   // FIXME don't use one from reply!
+       Inode *in = this->insert_inode_info(dir, *it);
+       in->last_updated = now;
+  }
+    
+  // dir contents to caller -- some of them might not have been in reply!
+  hash_map<string, *Dentry>::iterator it;
+  for (it = dir->dentries.begin();
+       it != dir->dentries.end();
+          it++) {
+    contents[it->first] = it->second->inode->inode;
   }
 
-  //delete reply;     fix thing above first
+  delete reply;     fix thing above first
   return res;
 }
 

diff --git a/ceph/client/Client.h b/ceph/client/Client.h
index ad7354d1d59a3fe9b86b6a1b0b4d450c3d534590..7f0bd5cc8187f5ff8364e38fd23d7a317956be99 100644 (file)
--- a/ceph/client/Client.h
+++ b/ceph/client/Client.h
@@ -15,6 +15,7 @@
 
 #include "include/types.h"
 #include "include/lru.h"
+#include "include/filepath.h"
 
 // stl
 #include <set>
@@ -122,6 +123,20 @@ class Client : public Dispatcher {
 
 
   // -- metadata cache stuff
+  // find dentry based on filepath
+  Dentry *lookup(filepath* path) {
+    Inode *cur = root;
+       Dentry *dn;
+       for (int i=0; i<path->bits.size(); i++) {
+         if (cur->dir.dentries.count(path->bits[i]))
+           dn = cur->dir.dentries[path->bits[i]];
+         else
+           return NULL;
+         cur = dn->inode;
+       }
+       return dn;
+  }
+               
   // decrease inode ref.  delete if dangling.
   void put_inode(Inode *in) {
        in->put();
@@ -210,6 +225,7 @@ class Client : public Dispatcher {
   }
 
   // metadata cache
+  Inode* insert_inode_info(Dir *dir, c_inode_info *in_info);
   void insert_trace(vector<c_inode_info*> trace);
 
   // ----------------------

diff --git a/ceph/config.cc b/ceph/config.cc
index 31c93108b8c955a7a056892d69b7a55657af7f5f..cc2b29bcf36e933b61e8cfb316da96b12988d2a2 100644 (file)
--- a/ceph/config.cc
+++ b/ceph/config.cc
@@ -20,6 +20,7 @@ md_config_t g_conf = {
 
   client_cache_size: 400,
   client_cache_mid: .5,
+  client_cache_stat_ttl: 10, // seconds until cached stat results become invalid
   client_use_random_mds:  false,
   
   log_messages: true,

diff --git a/ceph/config.h b/ceph/config.h
index c1e3f001b0d4c6f1fbbba99dae34ec7f724220c1..5956da35c9e3a0ddb2234be0ab7076b86a474110 100644 (file)
--- a/ceph/config.h
+++ b/ceph/config.h
@@ -10,6 +10,7 @@ struct md_config_t {
 
   int      client_cache_size;
   float    client_cache_mid;
+  int      client_cache_stat_ttl;
   bool     client_use_random_mds;          // debug flag
 
   bool log_messages;