From 435bccfd29009a2abe148bdcc0ef9d694fdf1441 Mon Sep 17 00:00:00 2001 From: sage Date: Fri, 8 Apr 2005 03:08:45 +0000 Subject: [PATCH] *** empty log message *** git-svn-id: https://ceph.svn.sf.net/svnroot/ceph@144 29311d96-e01e-0410-9327-a35deaab8ce9 --- ceph/TODO | 1 + ceph/client/Client.h | 4 +- ceph/include/types.h | 31 +++-- ceph/mds/CFile.h | 21 ++- ceph/mds/CInode.h | 74 +++++------ ceph/mds/MDCache.cc | 7 +- ceph/mds/MDS.cc | 231 +++++++++++++++++---------------- ceph/mds/MDS.h | 8 +- ceph/messages/MClientRequest.h | 23 ++-- ceph/msg/Message.h | 4 + ceph/osd/OSD.cc | 1 + 11 files changed, 210 insertions(+), 195 deletions(-) diff --git a/ceph/TODO b/ceph/TODO index d9d48122e722d..93a3db1ddbc8e 100644 --- a/ceph/TODO +++ b/ceph/TODO @@ -1,4 +1,5 @@ +- file readers - recall messages? - lazy flag? make a table! diff --git a/ceph/client/Client.h b/ceph/client/Client.h index 289aa83d1715e..c17d284b638a4 100644 --- a/ceph/client/Client.h +++ b/ceph/client/Client.h @@ -8,9 +8,11 @@ #include #include -#include using namespace std; +#include +using namespace __gnu_cxx; + // types for my local metadata cache struct Inode { }; diff --git a/ceph/include/types.h b/ceph/include/types.h index 6aeda72685277..1aee4f8d6c2cc 100644 --- a/ceph/include/types.h +++ b/ceph/include/types.h @@ -29,9 +29,7 @@ using namespace std; #define MDS_OP_RMDIR 221 #define MDS_OP_SYMLINK 222 -#define MDS_OP_OPENRD 301 -#define MDS_OP_OPENWR 302 -#define MDS_OP_OPENWRC 303 +#define MDS_OP_OPEN 301 #define OSD_OP_READ 304 #define OSD_OP_WRITE 305 #define MDS_OP_TRUNCATE 306 @@ -40,12 +38,14 @@ using namespace std; - - - - // -- stl crap -- +/* +- this is to make some of the STL types work with 64 bit values, string hash keys, etc. +- added when i was using an old STL.. maybe try taking these out and see if things + compile now? +*/ + namespace __gnu_cxx { template<> struct hash { size_t operator()(unsigned long long __x) const { return __x; } @@ -61,11 +61,13 @@ namespace __gnu_cxx { } -// -- raw inode -- +// -- inode -- typedef __uint64_t inodeno_t; // ino -typedef __uint64_t mdloc_t; // dir locator? +#define INODE_MODE_FILE 0100000 // S_IFREG +#define INODE_MODE_SYMLINK 0120000 // S_IFLNK +#define INODE_MODE_DIR 0040000 // S_IFDIR struct inode_t { // immutable @@ -85,17 +87,14 @@ struct inode_t { unsigned char hash_seed; // 0 if not hashed. }; -#define INODE_MODE_FILE 0100000 // S_IFREG -#define INODE_MODE_SYMLINK 0120000 // S_IFLNK -#define INODE_MODE_DIR 0040000 // S_IFDIR - - - +// misc other types +typedef __uint64_t object_t; +typedef __uint32_t fileh_t; +// dentries #define MAX_DENTRY_LEN 255 -typedef __uint64_t object_t; // -- load balancing stuff -- diff --git a/ceph/mds/CFile.h b/ceph/mds/CFile.h index 97a826dff8ce6..02625011fff31 100644 --- a/ceph/mds/CFile.h +++ b/ceph/mds/CFile.h @@ -1,13 +1,20 @@ +#ifndef __CFILE_H +#define __CFILE_H +#define CFILE_MODE_R 1 +#define CFILE_MODE_RW 2 +#define CFILE_MODE_W 3 -#define FILE_MODE_R 1 -#define FILE_MODE_RW 2 -#define FILE_MODE_W 3 +#define CFILE_LOCK_NONE 0 // e.g. all readers +#define CFILE_LOCK_LOCK 1 // e.g. truncate (all writers/readers stop) +//#define CFILE_LOCK_ASYNC 0 // e.g. writers struct CFile { - client_addr_t client; - fh_t fh; // file handle - byte mode; // mode opened in.. - byte state; // ?? + int client; + fileh_t fh; // file handle + unsigned char mode; // mode opened in + unsigned char state; // lock state }; + +#endif diff --git a/ceph/mds/CInode.h b/ceph/mds/CInode.h index fac36ae0f44d8..0824ae43b1356 100644 --- a/ceph/mds/CInode.h +++ b/ceph/mds/CInode.h @@ -9,6 +9,7 @@ #include #include "CDentry.h" +#include "CFile.h" #include "Lock.h" #include @@ -244,9 +245,8 @@ class CInode : LRUObject { // open file state - // sets of client ids! - multiset open_read; - multiset open_write; + map fh_read; // readers + map fh_write; // writers //MInodeSyncStart *pending_sync_request; @@ -446,49 +446,39 @@ class CInode : LRUObject { // -- open files -- bool is_open() { - if (open_read.empty() && - open_write.empty()) return false; + if (fh_read.empty() && + fh_write.empty()) return false; return true; } - bool is_open_write() { - if (open_write.empty()) return false; - return true; - } - void open_read_add(int c) { - if (open_read.empty()) - get(CINODE_PIN_OPENRD); - open_read.insert(c); - } - void open_read_remove(int c) { - if (open_read.count(c) == 1 && - open_read.size() == 1) - put(CINODE_PIN_OPENRD); - open_read.erase(open_read.find(c)); - } - void open_write_add(int c) { - if (open_write.empty()) - get(CINODE_PIN_OPENWR); - open_write.insert(c); - } - void open_write_remove(int c) { - if (open_write.count(c) == 1 && - open_write.size() == 1) - put(CINODE_PIN_OPENWR); - open_write.erase(open_write.find(c)); + CFile* get_fh(int fh) { + if (fh_read.count(fh)) return fh_read[fh]; + if (fh_write.count(fh)) return fh_write[fh]; } - bool open_remove(int c) { - if (open_read.count(c)) - open_read_remove(c); - else if (open_write.count(c)) - open_write_remove(c); - else return false; - return true; - } - multiset& get_open_write() { - return open_write; + void add_fh(CFile *f) { + if (f->mode == CFILE_MODE_R) { + if (fh_read.empty()) + get(CINODE_PIN_OPENRD); + fh_read[f->fh] = f; + } + else if (f->mode == CFILE_MODE_W) { + if (fh_write.empty()) + get(CINODE_PIN_OPENWR); + fh_write[f->fh] = f; + } + else assert(0); } - multiset& get_open_read() { - return open_read; + void remove_fh(CFile *f) { + if (f->mode == CFILE_MODE_R) { + assert(fh_read.count(f->fh)); + fh_read.erase(fh_read.find(f->fh)); + if (fh_read.empty()) put(CINODE_PIN_OPENRD); + } + else if (f->mode == CFILE_MODE_W) { + assert(fh_write.count(f->fh)); + fh_write.erase(fh_write.find(f->fh)); + if (fh_write.empty()) put(CINODE_PIN_OPENWR); + } + else assert(0); } diff --git a/ceph/mds/MDCache.cc b/ceph/mds/MDCache.cc index a0ec9972c92cd..f44b47f1843af 100644 --- a/ceph/mds/MDCache.cc +++ b/ceph/mds/MDCache.cc @@ -1755,8 +1755,11 @@ INODES: - -modes += modes + - SYNC + Rauth Rreplica Wauth Wreplica + sync + diff --git a/ceph/mds/MDS.cc b/ceph/mds/MDS.cc index 1c557e2fc855d..d0ab648ae5352 100644 --- a/ceph/mds/MDS.cc +++ b/ceph/mds/MDS.cc @@ -34,6 +34,7 @@ #include "events/EInodeUnlink.h" #include +#include #include #include @@ -366,8 +367,9 @@ int MDS::handle_client_request(MClientRequest *req) handle_client_close(req); return 0; - case MDS_OP_OPENWRC: - handle_client_openwrc(req); + case MDS_OP_OPEN: + if (req->get_iarg() & O_CREAT) + handle_client_openc(req); return 0; case MDS_OP_MKDIR: @@ -415,12 +417,8 @@ int MDS::handle_client_request(MClientRequest *req) reply = handle_client_chmod(req, cur); break; - case MDS_OP_OPENRD: - reply = handle_client_openrd(req, cur); - break; - - case MDS_OP_OPENWR: - reply = handle_client_openwr(req, cur); + case MDS_OP_OPEN: + handle_client_open(req, cur); break; case MDS_OP_UNLINK: @@ -709,22 +707,6 @@ MClientReply *MDS::handle_client_readdir(MClientRequest *req, } -MClientReply *MDS::handle_client_openrd(MClientRequest *req, - CInode *cur) -{ - dout(10) << "open (read) on " << *cur << endl; - - // hmm, check permissions or something. - - // add reader - int client = req->get_client(); - cur->open_read_add(client); - - // reply - MClientReply *reply = new MClientReply(req); - reply->set_trace_dist( cur, whoami ); - return reply; -} void MDS::handle_client_close(MClientRequest *req) { @@ -733,14 +715,27 @@ void MDS::handle_client_close(MClientRequest *req) dout(10) << "close on " << *cur << endl; - // hmm, check permissions or something. - // verify on read or write list int client = req->get_client(); - if (!cur->open_remove(client)) { + int fh = req->get_iarg(); + CFile *f = cur->get_fh(fh); + if (!f) { dout(1) << "close on unopen file " << *cur << endl; assert(2+2==5); } + + // update soft metadata + assert(cur->softlock.get_mode() == LOCK_MODE_ASYNC); // otherwise we're toast? + if (!mdcache->inode_soft_write_start(cur, req)) + return; // wait + + // update size, mtime + + // close it. + cur->remove_fh(f); + + // ok we're done + mdcache->inode_soft_write_finish(cur); // reply MClientReply *reply = new MClientReply(req); @@ -753,116 +748,124 @@ void MDS::handle_client_close(MClientRequest *req) delete req; } - -MClientReply *MDS::handle_client_openwr(MClientRequest *req, - CInode *cur) +void MDS::handle_client_open(MClientRequest *req, + CInode *cur) { - if (!cur->is_auth()) { - if (cur->softlock.get_mode() == LOCK_MODE_SYNC) { - int auth = cur->authority(); - assert(auth != whoami); - dout(10) << "open (write) [replica] " << *cur << " on replica, fw to auth " << auth << endl; + int flags = req->get_iarg(); + assert(flags & O_CREAT == 0); + + dout(10) << "open " << flags << " on " << *cur << endl; + + int mode; + if (flags & O_RDONLY) { + mode = CFILE_MODE_R; + } + else if (flags & O_WRONLY) { + mode = CFILE_MODE_W; + + if (!cur->is_auth()) { + if (cur->softlock.get_mode() == LOCK_MODE_SYNC) { + int auth = cur->authority(); + assert(auth != whoami); + dout(10) << "open (write) [replica] " << *cur << " on replica, fw to auth " << auth << endl; + + messenger->send_message(req, + MSG_ADDR_MDS(auth), MDS_PORT_SERVER, + MDS_PORT_SERVER); + return; + } - messenger->send_message(req, - MSG_ADDR_MDS(auth), MDS_PORT_SERVER, - MDS_PORT_SERVER); - return 0; + dout(10) << "open (write) [replica shared write] " << *cur << endl; + assert(0); } - - dout(10) << "open (write) [replica shared write] " << *cur << endl; - assert(0); + + // only writers on auth, for now. + assert(cur->is_auth()); + } + else if (flags & O_RDWR) { + assert(0); // WR not implemented yet + } else { + assert(0); // no mode specified. this should actually be an error code back to client. } - dout(10) << "open (write) [auth] " << *cur << endl; + // hmm, check permissions or something. - // hmm, check permissions! + // create fh + CFile *f = new CFile; + f->mode = mode; + f->client = req->get_client(); + cur->add_fh(f); - // add to writer list. - int client = req->get_client(); - cur->open_write_add(client); - // reply - MClientReply *reply = new MClientReply(req); + MClientReply *reply = new MClientReply(req, f->fh); // fh # is return code reply->set_trace_dist( cur, whoami ); - return reply; + + messenger->send_message(reply, + MSG_ADDR_CLIENT(req->get_client()), 0, + MDS_PORT_SERVER); + + // discard request + delete req; } -/* -int path_depth(string& p) -{ - int d = 0; - for (const char *c = p.c_str(); *c; c++) - if (*c == '/') d++; - return d; -} -*/ -void MDS::handle_client_openwrc(MClientRequest *req) +void MDS::handle_client_openc(MClientRequest *req) { - - // see if file exists + // see if file exists vector trace; int r = mdcache->path_traverse(req->get_filepath(), trace, req, MDS_TRAVERSE_FORWARD); if (r > 0) return; // delayed - if (r < 0) { - // problems: - - if (r == -ENOENT) { - // on the last bit? - filepath path = req->get_path(); - int depth = path.depth(); - if (trace.size() == depth) { // everything but the file - // create dentry, file! - CInode *idir = trace[trace.size()-1]; - assert(idir->dir->is_auth() || idir->dir->is_hashed()); // path_traverse should have fwd if not! - string dname = path.last_bit(); - - dout(7) << "handle_client_openwrc -ENOENT on target file, creating " << dname << endl; - - // verify i am authoritative for this dentry (should have fwd if not) - int auth = idir->dir->dentry_authority(dname); - assert(auth == whoami); - - // create inode and link - CInode *in = mdcache->create_inode(); - mdcache->link_inode( idir->dir, dname, in ); - - in->mark_dirty(); - - // log it - dout(10) << "log for " << *req << " create " << in->ino() << endl; - mdlog->submit_entry(new EInodeUpdate(in), // FIXME should be differnet log entry - new C_MDS_RetryMessage(this, req)); - return; - } else { - dout(7) << "handle_client_openwrc -ENOENT on containing dir; fail!" << endl; - } - } - - // send error response - dout(7) << "handle_client_openwrc error " << r << " replying to client" << endl; - reply_request(req, r); + if (r == 0) { + // it exists, behave normally. + CInode *cur = trace[trace.size()-1]; + handle_client_open(req, cur); return; } - // file exists! do it. + assert (r < 0); - logger->inc("chit"); - CInode *cur = trace[trace.size()-1]; - balancer->hit_inode(cur, MDS_POP_ANY); // bump popularity - - MClientReply *reply = handle_client_openwr(req,cur); - - if (reply) { - messenger->send_message(reply, - MSG_ADDR_CLIENT(req->get_client()), 0, - MDS_PORT_SERVER); - - // discard request - delete req; + // what happened? + if (r == -ENOENT) { + // on the last bit? + filepath path = req->get_path(); + int depth = path.depth(); + + if (trace.size() != depth) { // everything but the file + dout(7) << "handle_client_openc -ENOENT on a parent dir; fail!" << endl; + } else { + assert(trace.size() == depth); + + // create dentry, file! + CInode *idir = trace[trace.size()-1]; + assert(idir->dir->is_auth() || idir->dir->is_hashed()); // path_traverse should have fwd if not! + string dname = path.last_bit(); + + dout(7) << "handle_client_openc -ENOENT on target file, creating " << dname << endl; + + // sanity: verify i am authoritative for this dentry (should have fwd if not) + int auth = idir->dir->dentry_authority(dname); + assert(auth == whoami); + + // create inode and link + CInode *in = mdcache->create_inode(); + mdcache->link_inode( idir->dir, dname, in ); + + in->mark_dirty(); + + // log it + dout(10) << "log for " << *req << " create " << in->ino() << endl; + mdlog->submit_entry(new EInodeUpdate(in), // FIXME should be differnet log entry + new C_MDS_RetryMessage(this, req)); + return; + } } + + // send error response + dout(7) << "handle_client_openwrc error " << r << " replying to client" << endl; + reply_request(req, r); + return; } diff --git a/ceph/mds/MDS.h b/ceph/mds/MDS.h index 06fc720142ce7..0d37632ef0d20 100644 --- a/ceph/mds/MDS.h +++ b/ceph/mds/MDS.h @@ -165,11 +165,9 @@ class MDS : public Dispatcher { MClientReply *reply, CInode *cur); - MClientReply *handle_client_openrd(MClientRequest *req, - CInode *cur); - MClientReply *handle_client_openwr(MClientRequest *req, - CInode *cur); - void handle_client_openwrc(MClientRequest *req); + void handle_client_open(MClientRequest *req, + CInode *cur); + void handle_client_openc(MClientRequest *req); void handle_client_close(MClientRequest *req); void handle_client_unlink(MClientRequest *req, diff --git a/ceph/messages/MClientRequest.h b/ceph/messages/MClientRequest.h index d587713129c29..d80a683773701 100644 --- a/ceph/messages/MClientRequest.h +++ b/ceph/messages/MClientRequest.h @@ -7,9 +7,10 @@ typedef struct { long tid; - int op; int client; + int op; inodeno_t ino; + int iarg; } MClientRequest_st; @@ -17,6 +18,7 @@ class MClientRequest : public Message { MClientRequest_st st; filepath path; string arg; + string arg2; public: MClientRequest() {} @@ -25,12 +27,15 @@ class MClientRequest : public Message { this->st.op = op; this->st.client = client; this->st.ino = 0; + this->st.iarg = 0; } virtual char *get_type_name() { return "creq"; } void set_path(string& p) { path.set_path(p); } void set_ino(inodeno_t ino) { st.ino = ino; } + void set_iarg(int i) { st.iarg = i; } void set_arg(string& arg) { this->arg = arg; } + void set_arg2(string& arg) { this->arg2 = arg; } long get_tid() { return st.tid; } int get_op() { return st.op; } @@ -38,14 +43,18 @@ class MClientRequest : public Message { inodeno_t get_ino() { return st.ino; } string& get_path() { return path.get_path(); } filepath& get_filepath() { return path; } + int get_iarg() { return st.iarg; } string& get_arg() { return arg; } - + string& get_arg2() { return arg2; } virtual void decode_payload(crope& s) { s.copy(0, sizeof(st), (char*)&st); path.set_path( s.c_str() + sizeof(st) ); int off = sizeof(st) + path.length() + 1; arg = s.c_str() + off; + off += arg.length()+1; + arg2 = s.c_str() + off; + off += arg2.length()+1; } virtual void encode_payload(crope& r) { @@ -54,6 +63,8 @@ class MClientRequest : public Message { r.append((char)0); r.append(arg.c_str()); r.append((char)0); + r.append(arg2.c_str()); + r.append((char)0); } }; @@ -70,12 +81,8 @@ inline ostream& operator<<(ostream& out, MClientRequest& req) { out << "stat"; break; case MDS_OP_READDIR: out << "readdir"; break; - case MDS_OP_OPENRD: - out << "openrd"; break; - case MDS_OP_OPENWR: - out << "openwr"; break; - case MDS_OP_OPENWRC: - out << "openwrc"; break; + case MDS_OP_OPEN: + out << "open"; break; case MDS_OP_UNLINK: out << "unlink"; break; case MDS_OP_CLOSE: diff --git a/ceph/msg/Message.h b/ceph/msg/Message.h index cdabbb6ea29ca..71e9748ba4061 100644 --- a/ceph/msg/Message.h +++ b/ceph/msg/Message.h @@ -77,6 +77,10 @@ // address types typedef int msg_addr_t; +// mds's, client's share same (integer) namespace ?????? +// osd's could be separate. + + /* sandwich mds's, then osd's, then clients */ #define MSG_ADDR_MDS(x) (x) #define MSG_ADDR_OSD(x) (g_conf.num_mds+(x)) diff --git a/ceph/osd/OSD.cc b/ceph/osd/OSD.cc index 84512fb8858ae..50ab51c211ceb 100644 --- a/ceph/osd/OSD.cc +++ b/ceph/osd/OSD.cc @@ -68,6 +68,7 @@ void OSD::dispatch(Message *m) write((MOSDWrite*)m); break; + default: dout(1) << " got unknown message " << m->get_type() << endl; } -- 2.39.5