From bdfe5e08dcba0ba1d3e05a966304f4314a217618 Mon Sep 17 00:00:00 2001 From: carlosm Date: Fri, 6 May 2005 18:17:28 +0000 Subject: [PATCH] Modified Files: config.cc client/Client.cc client/Client.h include/config.h - add Client::insert_inode_info and use it in Client::insert_trace and Client::getdir UNTESTED - add g_conf.client_cache_stat_ttl UNTESTED - all fs ops except mknod, open, read, write, truncate, and fsync now use the metadata cache UNTESTED git-svn-id: https://ceph.svn.sf.net/svnroot/ceph@209 29311d96-e01e-0410-9327-a35deaab8ce9 --- ceph/client/Client.cc | 198 +++++++++++++++++++++++++++--------------- ceph/client/Client.h | 16 ++++ ceph/config.cc | 1 + ceph/config.h | 1 + 4 files changed, 144 insertions(+), 72 deletions(-) diff --git a/ceph/client/Client.cc b/ceph/client/Client.cc index 8ae2045c25b59..9ac16ff645100 100644 --- a/ceph/client/Client.cc +++ b/ceph/client/Client.cc @@ -4,6 +4,7 @@ // unix-ey fs stuff #include +#include #include @@ -46,62 +47,76 @@ void Client::shutdown() { messenger->shutdown(); } +// insert inode info into metadata cache + +Inode* Client::insert_inode_info(Dir *dir, c_inode_info *in_info) +{ + string dname = in_info->ref_dn; + Dentry *dn = NULL; + if (dir->dentries.count(dname)) + dn = dir->dentries[dname]; + dout(12) << "insert_inode_info " << dname << " ino " << in_info->inode.ino << endl; + + if (dn) { + if (dn->inode->inode.ino == in_info->inode.ino) { + touch_dn(next); + dout(12) << " had dentry " << dname << " with correct ino " << dn->inode->inode.ino << endl; + } else { + dout(12) << " had dentry " << dname << " with WRONG ino " << dn->inode->inode.ino << endl; + unlink(dn); + dn = NULL; + } + } + + if (!dn) { + if (inode_map.count(in_info->inode.ino)) { + Inode *in = inode_map[in_info->inode.ino]; + if (in) { + dout(12) << " had ino " << dn->inode->inode.ino << " at wrong position, moving" << endl; + if (in->dn) unlink(in->dn); + dn = link(dir, dname, in); + } + } + } + + if (!dn) { + dn = link(dir, dname, new Inode()); + dn->inode->inode = in_info->inode; + inode_map[dn->inode->inode.ino] = dn->inode; + dout(12) << " new dentry+node with ino " << dn->inode->inode.ino << endl; + } + + // take note of latest distribution on mds's + dn->inode->mds_contacts = in_info->dist; + + return dn->inode; +} + // insert trace of reply into metadata cache void Client::insert_trace(vector trace) { Inode *cur = root; + time_t now = time(NULL); + if (!trace) return; + for (int i=0; iinode = trace[i]->inode; - inode_map[root->inode.ino] = root; - } - dout(12) << "insert_trace trace " << i << " root" << endl; - } else { - Dir *dir = open_dir( cur ); - string dname = trace[i]->ref_dn; - Dentry *next = NULL; - if (dir->dentries.count(dname)) - next = dir->dentries[dname]; - dout(12) << "insert_trace trace " << i << " dname " << dname << " ino " << trace[i]->inode.ino << endl; - - if (next) { - if (next->inode->inode.ino == trace[i]->inode.ino) { - touch_dn(next); - dout(12) << " had dentry " << dname << " with correct ino " << next->inode->inode.ino << endl; - } else { - dout(12) << " had dentry " << dname << " with WRONG ino " << next->inode->inode.ino << endl; - unlink(next); - next = NULL; - } - } - - if (!next) { - if (inode_map.count(trace[i]->inode.ino)) { - Inode *in = inode_map[trace[i]->inode.ino]; - if (in) { - dout(12) << " had ino " << next->inode->inode.ino << " at wrong position, moving" << endl; - if (in->dn) unlink(in->dn); - next = link(dir, dname, in); - } - } - } - - if (!next) { - next = link(dir, dname, new Inode()); - next->inode->inode = trace[i]->inode; - inode_map[next->inode->inode.ino] = next->inode; - dout(12) << " new dentry+node with ino " << next->inode->inode.ino << endl; - } - - cur = next->inode; - } - - // take note of latest distribution on mds's - cur->mds_contacts = trace[i]->dist; + if (i == 0) { + if (!root) { + cur = root = new Inode(); + root->inode = trace[i]->inode; + inode_map[root->inode.ino] = root; + } + root->mds_contacts = trace[i]->dist; + root->last_updated = now; + dout(12) << "insert_trace trace " << i << " root" << endl; + } else { + dout(12) << "insert_trace trace " << i; + Dir *dir = open_dir( cur ); + cur = this->insert_inode_info(dir, trace[i]); + cur->last_updated = now; + } } } @@ -125,6 +140,10 @@ int Client::unlink(const char *path) MClientReply *reply = (MClientReply*)messenger->sendrecv(req, MSG_ADDR_MDS(0), MDS_PORT_SERVER); int res = reply->get_result(); + if (res == 0) { + unlink(lookup(req->get_filepath())); + } + this->insert_trace(reply->get_trace); delete reply; dout(10) << "unlink result is " << res << endl; return res; @@ -145,6 +164,7 @@ int Client::rename(const char *from, const char *to) MClientReply *reply = (MClientReply*)messenger->sendrecv(req, MSG_ADDR_MDS(0), MDS_PORT_SERVER); int res = reply->get_result(); + this->insert_trace(reply->get_trace); delete reply; dout(10) << "rename result is " << res << endl; return res; @@ -167,6 +187,7 @@ int Client::mkdir(const char *path, mode_t mode) MClientReply *reply = (MClientReply*)messenger->sendrecv(req, MSG_ADDR_MDS(0), MDS_PORT_SERVER); int res = reply->get_result(); + this->insert_trace(reply->get_trace); delete reply; dout(10) << "mkdir result is " << res << endl; return res; @@ -186,6 +207,10 @@ int Client::rmdir(const char *path) MClientReply *reply = (MClientReply*)messenger->sendrecv(req, MSG_ADDR_MDS(0), MDS_PORT_SERVER); int res = reply->get_result(); + if (res == 0) { + unlink(lookup(req->get_filepath())); + } + this->insert_trace(reply->get_trace); delete reply; dout(10) << "rmdir result is " << res << endl; return res; @@ -208,6 +233,7 @@ int Client::symlink(const char *target, const char *link) MClientReply *reply = (MClientReply*)messenger->sendrecv(req, MSG_ADDR_MDS(0), MDS_PORT_SERVER); int res = reply->get_result(); + this->insert_trace(reply->get_trace); //FIXME assuming trace of link, not of target delete reply; dout(10) << "symlink result is " << res << endl; return res; @@ -216,7 +242,7 @@ int Client::symlink(const char *target, const char *link) int Client::readlink(const char *path, char *buf, size_t size) { dout(3) << "readlink " << path << endl; - // stat first (OR, FIXME, use cached value?) **** + // stat first (FIXME, PERF access cache directly) **** struct stat stbuf; int r = this->lstat(path, &stbuf); if (r != 0) return r; @@ -238,25 +264,36 @@ int Client::readlink(const char *path, char *buf, size_t size) // inode stuff int Client::lstat(const char *path, struct stat *stbuf) -// FIXME make sure this implements lstat and not stat { dout(3) << "lstat " << path << endl; + // FIXME, PERF request allocation convenient but not necessary for cache hit MClientRequest *req = new MClientRequest(MDS_OP_STAT, whoami); req->set_path(path); - // FIXME where does FUSE maintain user information - req->set_caller_uid(getuid()); - req->set_caller_gid(getgid()); - - MClientReply *reply = (MClientReply*)messenger->sendrecv(req, MSG_ADDR_MDS(0), MDS_PORT_SERVER); - int res = reply->get_result(); - dout(10) << "lstat res is " << res << endl; - if (res != 0) return res; - - //Transfer information from reply to stbuf - vector trace = reply->get_trace(); - inode_t inode = trace[trace.size()-1]->inode; - + // check whether cache content is fresh enough + Dentry *dn = lookup(req->get_filepath()); + inode_t inode; + if (dn && ((time(NULL) - dn->inode.last_updated) <= g_conf.client_cache_stat_ttl)) { + inode = dn->inode->inode; + dout(10) << "lstat cache hit" << endl; + } else { + // FIXME where does FUSE maintain user information + req->set_caller_uid(getuid()); + req->set_caller_gid(getgid()); + + MClientReply *reply = (MClientReply*)messenger->sendrecv(req, MSG_ADDR_MDS(0), MDS_PORT_SERVER); + int res = reply->get_result(); + dout(10) << "lstat res is " << res << endl; + if (res != 0) return res; + + //Transfer information from reply to stbuf + vector trace = reply->get_trace(); + inode = trace[trace.size()-1]->inode; + + //Update metadata cache + this->insert_trace(trace); + } + memset(stbuf, 0, sizeof(struct stat)); //stbuf->st_dev = stbuf->st_ino = inode.ino; @@ -275,11 +312,12 @@ int Client::lstat(const char *path, struct stat *stbuf) // gross HACK for now.. put an Inode in cache! // for now i just need to put the symlink content somewhere (so that readlink() will work) - Inode *in = new Inode; - inode_map[inode.ino] = in; // put in map so subsequent readlink will find it - in->inode = inode; + //Inode *in = new Inode; + //inode_map[inode.ino] = in; // put in map so subsequent readlink will find it + //in->inode = inode; // symlink? + Inode *in = inode_map[inode.ino]; if (in->inode.mode & INODE_MODE_SYMLINK) { if (!in->symlink) in->symlink = new string; *(in->symlink) = trace[trace.size()-1]->symlink; @@ -287,7 +325,6 @@ int Client::lstat(const char *path, struct stat *stbuf) dout(1) << "stat sez uid = " << inode.uid << " ino = " << stbuf->st_ino << endl; - // FIXME need to update cache with trace's inode_info delete reply; return 0; } @@ -307,6 +344,7 @@ int Client::chmod(const char *path, mode_t mode) MClientReply *reply = (MClientReply*)messenger->sendrecv(req, MSG_ADDR_MDS(0), MDS_PORT_SERVER); int res = reply->get_result(); + this->insert_trace(reply->get_trace); delete reply; dout(10) << "chmod result is " << res << endl; return res; @@ -328,6 +366,7 @@ int Client::chown(const char *path, uid_t uid, gid_t gid) MClientReply *reply = (MClientReply*)messenger->sendrecv(req, MSG_ADDR_MDS(0), MDS_PORT_SERVER); int res = reply->get_result(); + this->insert_trace(reply->get_trace); delete reply; dout(10) << "chown result is " << res << endl; return res; @@ -349,6 +388,7 @@ int Client::utime(const char *path, struct utimbuf *buf) MClientReply *reply = (MClientReply*)messenger->sendrecv(req, MSG_ADDR_MDS(0), MDS_PORT_SERVER); int res = reply->get_result(); + this->insert_trace(reply->get_trace); delete reply; dout(10) << "utime result is " << res << endl; return res; @@ -377,18 +417,32 @@ int Client::getdir(const char *path, map& contents) MClientReply *reply = (MClientReply*)messenger->sendrecv(req, MSG_ADDR_MDS(0), MDS_PORT_SERVER); int res = reply->get_result(); + vector trace = reply->get_trace(); + this->insert_trace(trace); + + if (res) return res; - // dir contents to caller! + // dir contents to cache! + Dir *dir = open_dir(inode_map[trace[trace.size()-1]->inode.ino]); + time_t now = time(NULL); vector::iterator it; for (it = reply->get_dir_contents().begin(); it != reply->get_dir_contents().end(); it++) { // put in cache - - contents[(*it)->ref_dn] = &(*it)->inode; // FIXME don't use one from reply! + Inode *in = this->insert_inode_info(dir, *it); + in->last_updated = now; + } + + // dir contents to caller -- some of them might not have been in reply! + hash_map::iterator it; + for (it = dir->dentries.begin(); + it != dir->dentries.end(); + it++) { + contents[it->first] = it->second->inode->inode; } - //delete reply; fix thing above first + delete reply; fix thing above first return res; } diff --git a/ceph/client/Client.h b/ceph/client/Client.h index ad7354d1d59a3..7f0bd5cc8187f 100644 --- a/ceph/client/Client.h +++ b/ceph/client/Client.h @@ -15,6 +15,7 @@ #include "include/types.h" #include "include/lru.h" +#include "include/filepath.h" // stl #include @@ -122,6 +123,20 @@ class Client : public Dispatcher { // -- metadata cache stuff + // find dentry based on filepath + Dentry *lookup(filepath* path) { + Inode *cur = root; + Dentry *dn; + for (int i=0; ibits.size(); i++) { + if (cur->dir.dentries.count(path->bits[i])) + dn = cur->dir.dentries[path->bits[i]]; + else + return NULL; + cur = dn->inode; + } + return dn; + } + // decrease inode ref. delete if dangling. void put_inode(Inode *in) { in->put(); @@ -210,6 +225,7 @@ class Client : public Dispatcher { } // metadata cache + Inode* insert_inode_info(Dir *dir, c_inode_info *in_info); void insert_trace(vector trace); // ---------------------- diff --git a/ceph/config.cc b/ceph/config.cc index 31c93108b8c95..cc2b29bcf36e9 100644 --- a/ceph/config.cc +++ b/ceph/config.cc @@ -20,6 +20,7 @@ md_config_t g_conf = { client_cache_size: 400, client_cache_mid: .5, + client_cache_stat_ttl: 10, // seconds until cached stat results become invalid client_use_random_mds: false, log_messages: true, diff --git a/ceph/config.h b/ceph/config.h index c1e3f001b0d4c..5956da35c9e3a 100644 --- a/ceph/config.h +++ b/ceph/config.h @@ -10,6 +10,7 @@ struct md_config_t { int client_cache_size; float client_cache_mid; + int client_cache_stat_ttl; bool client_use_random_mds; // debug flag bool log_messages; -- 2.39.5