From 6f65592b57342bf5532b412d3f35ed9b984e62ab Mon Sep 17 00:00:00 2001 From: sageweil Date: Mon, 13 Aug 2007 22:46:24 +0000 Subject: [PATCH] client bugfixes, basic statfs implementation (minus mon.pgmon) git-svn-id: https://ceph.svn.sf.net/svnroot/ceph@1631 29311d96-e01e-0410-9327-a35deaab8ce9 --- branches/sage/mds/Makefile | 3 - branches/sage/mds/client/Client.cc | 115 ++++++++++++++-------- branches/sage/mds/client/Client.h | 18 +++- branches/sage/mds/client/fuse_ll.cc | 16 ++- branches/sage/mds/messages/MStatfs.h | 9 +- branches/sage/mds/messages/MStatfsReply.h | 45 +++++++++ branches/sage/mds/mon/Monitor.cc | 6 ++ branches/sage/mds/mon/PGMonitor.cc | 21 +++- branches/sage/mds/msg/Message.cc | 6 ++ branches/sage/mds/msg/Message.h | 5 +- 10 files changed, 182 insertions(+), 62 deletions(-) create mode 100644 branches/sage/mds/messages/MStatfsReply.h diff --git a/branches/sage/mds/Makefile b/branches/sage/mds/Makefile index acf52719b68be..0d02603a3bcbf 100644 --- a/branches/sage/mds/Makefile +++ b/branches/sage/mds/Makefile @@ -174,9 +174,6 @@ msgtestclient: active/msgtestclient.cc client.o osdc.o msg/SimpleMessenger.o com libtrivialtask.so: active/trivial_task.cc client.o osdc.o msg/SimpleMessenger.o common.o ${CC} -fPIC -shared -Wl,-soname,$@.1 ${CFLAGS} ${LIBS} $^ -o $@ -#libhadoopcephfs.so: client/hadoop/CephFSInterface.cc client.o osdc.o msg/SimpleMessenger.o common.o -# ${CC} -fPIC -shared -Wl,-soname,$@.1 ${CFLAGS} ${LIBS} $^ -o $@ - # fake* diff --git a/branches/sage/mds/client/Client.cc b/branches/sage/mds/client/Client.cc index 8a633e5a40013..6f38f2bf8225e 100644 --- a/branches/sage/mds/client/Client.cc +++ b/branches/sage/mds/client/Client.cc @@ -32,6 +32,9 @@ using namespace std; // ceph stuff #include "Client.h" +#include "messages/MStatfs.h" +#include "messages/MStatfsReply.h" + #include "messages/MClientMount.h" #include "messages/MClientUnmount.h" #include "messages/MClientSession.h" @@ -895,7 +898,9 @@ void Client::dispatch(Message *m) handle_file_caps((MClientFileCaps*)m); break; - + case MSG_STATFS_REPLY: + handle_statfs_reply((MStatfsReply*)m); + break; default: cout << "dispatch doesn't recognize message type " << m->get_type() << endl; @@ -2142,6 +2147,8 @@ int Client::_opendir(const char *name, DirResult **dirpp) if (r < 0) { _closedir(*dirpp); *dirpp = 0; + } else { + r = 0; } dout(3) << "_opendir(" << name << ") = " << r << " (" << *dirpp << ")" << endl; @@ -2154,18 +2161,26 @@ void Client::_readdir_add_dirent(DirResult *dirp, const string& name, Inode *in) int stmask = fill_stat(in, &st); frag_t fg = dirp->frag(); dirp->buffer[fg].push_back(DirEntry(name, st, stmask)); - dout(10) << "_readdir_add_dirent added " << name << ", size now " << dirp->buffer[fg].size() << endl; + dout(10) << "_readdir_add_dirent " << dirp << " added '" << name << "' -> " << in->inode.ino + << ", size now " << dirp->buffer[fg].size() << endl; } -void Client::_readdir_add_dirent(DirResult *dirp, const string& name, unsigned char d_type) +//struct dirent { +// ino_t d_ino; /* inode number */ +// off_t d_off; /* offset to the next dirent */ +// unsigned short d_reclen; /* length of this record */ +// unsigned char d_type; /* type of file */ +// char d_name[256]; /* filename */ +//}; +void Client::_readdir_fill_dirent(struct dirent *de, DirEntry *entry, off_t off) { - struct stat st; - memset(&st, 0, sizeof(st)); - st.st_mode = DT_TO_MODE(d_type); - int stmask = STAT_MASK_TYPE; - frag_t fg = dirp->frag(); - dirp->buffer[fg].push_back(DirEntry(name, st, stmask)); - dout(10) << "_readdir_add_dirent added " << name << ", size now " << dirp->buffer[fg].size() << endl; + de->d_ino = entry->st.st_ino; + de->d_off = off + 1; + de->d_reclen = 1; + de->d_type = MODE_TO_DT(entry->st.st_mode); + strncpy(de->d_name, entry->d_name.c_str(), 256); + dout(10) << "_readdir_fill_dirent '" << de->d_name << "' -> " << de->d_ino + << " type " << (int)de->d_type << " at off " << off << endl; } void Client::_readdir_next_frag(DirResult *dirp) @@ -2350,26 +2365,6 @@ int Client::readdirplus_r(DIR *d, struct dirent *de, struct stat *st, int *stmas return 0; } -//struct dirent { -// ino_t d_ino; /* inode number */ -// off_t d_off; /* offset to the next dirent */ -// unsigned short d_reclen; /* length of this record */ -// unsigned char d_type; /* type of file */ -// char d_name[256]; /* filename */ -//}; -void Client::_readdir_fill_dirent(struct dirent *de, DirEntry *entry, off_t off) -{ - if (entry->stmask) - de->d_ino = entry->st.st_ino; - else - de->d_ino = 0; - de->d_off = off + 1; - de->d_reclen = 1; - de->d_type = MODE_TO_DT(entry->st.st_mode); - strncpy(de->d_name, entry->d_name.c_str(), 256); - dout(10) << "_readdir_fill_dirent " << de->d_name << " " << de->d_ino - << " type " << (int)de->d_type << " at off " << off << endl; -} int Client::closedir(DIR *dir) { @@ -3117,20 +3112,54 @@ int Client::statfs(const char *path, struct statvfs *stbuf) { Mutex::Locker lock(client_lock); tout << "statfs" << endl; + return _statfs(stbuf); +} - bzero (stbuf, sizeof (struct statvfs)); - // FIXME - stbuf->f_bsize = 1024; - stbuf->f_frsize = 1024; - stbuf->f_blocks = 1024 * 1024; - stbuf->f_bfree = 1024 * 1024; - stbuf->f_bavail = 1024 * 1024; - stbuf->f_files = 1024 * 1024; - stbuf->f_ffree = 1024 * 1024; - stbuf->f_favail = 1024 * 1024; - stbuf->f_namemax = 1024; +int Client::ll_statfs(inodeno_t ino, struct statvfs *stbuf) +{ + Mutex::Locker lock(client_lock); + tout << "ll_statfs" << endl; + return _statfs(stbuf); +} - return 0; +int Client::_statfs(struct statvfs *stbuf) +{ + dout(3) << "_statfs" << endl; + + Cond cond; + tid_t tid = ++last_tid; + StatfsRequest *req = new StatfsRequest(tid, &cond); + statfs_requests[tid] = req; + + int mon = monmap->pick_mon(); + messenger->send_message(new MStatfs(req->tid), monmap->get_inst(mon)); + + while (req->reply == 0) + cond.Wait(client_lock); + + // yay + memcpy(stbuf, &req->reply->stfs, sizeof(*stbuf)); + + statfs_requests.erase(req->tid); + delete req->reply; + delete req; + + int r = 0; + dout(3) << "_statfs = " << r << endl; + return r; +} + +void Client::handle_statfs_reply(MStatfsReply *reply) +{ + if (statfs_requests.count(reply->tid) && + statfs_requests[reply->tid]->reply == 0) { + dout(10) << "handle_statfs_reply " << *reply << ", kicking waiter" << endl; + statfs_requests[reply->tid]->reply = reply; + statfs_requests[reply->tid]->caller_cond->Signal(); + } else { + dout(10) << "handle_statfs_reply " << *reply << ", dup or old, dropping" << endl; + delete reply; + } } @@ -3676,7 +3705,7 @@ int Client::ll_create(inodeno_t parent, const char *name, mode_t mode, int flags if (r >= 0) { Inode *in = (*fhp)->inode; fill_stat(in, attr); - //_ll_get(in); + _ll_get(in); } tout << (unsigned long)*fhp << endl; tout << attr->st_ino << endl; diff --git a/branches/sage/mds/client/Client.h b/branches/sage/mds/client/Client.h index 5f62b5c9292ac..e7fafccc4b741 100644 --- a/branches/sage/mds/client/Client.h +++ b/branches/sage/mds/client/Client.h @@ -48,6 +48,8 @@ using namespace std; using namespace __gnu_cxx; + +class MStatfsReply; class MClientSession; class MClientRequest; class MClientRequestForward; @@ -430,6 +432,14 @@ class Client : public Dispatcher { tid_t last_tid; map mds_requests; set failed_mds; + + struct StatfsRequest { + tid_t tid; + MStatfsReply *reply; + Cond *caller_cond; + StatfsRequest(tid_t t, Cond *cc) : tid(t), reply(0), caller_cond(cc) {} + }; + map statfs_requests; MClientReply *make_request(MClientRequest *req, int use_auth=-1); int choose_target_mds(MClientRequest *req); @@ -437,6 +447,7 @@ class Client : public Dispatcher { void kick_requests(int mds); void handle_client_request_forward(MClientRequestForward *reply); void handle_client_reply(MClientReply *reply); + void handle_statfs_reply(MStatfsReply *reply); bool mounted; bool unmounting; @@ -667,12 +678,11 @@ private: int _do_lstat(const char *path, int mask, Inode **in); int _opendir(const char *name, DirResult **dirpp); void _readdir_add_dirent(DirResult *dirp, const string& name, Inode *in); - void _readdir_add_dirent(DirResult *dirp, const string& name, unsigned char d_type); + void _readdir_fill_dirent(struct dirent *de, DirEntry *entry, off_t); bool _readdir_have_frag(DirResult *dirp); void _readdir_next_frag(DirResult *dirp); void _readdir_rechoose_frag(DirResult *dirp); int _readdir_get_frag(DirResult *dirp); - void _readdir_fill_dirent(struct dirent *de, DirEntry *entry, off_t); void _closedir(DirResult *dirp); void _ll_get(Inode *in); int _ll_put(Inode *in, int num); @@ -699,6 +709,7 @@ private: int _truncate(const char *file, off_t length); int _ftruncate(Fh *fh, off_t length); int _fsync(Fh *fh, bool syncdataonly); + int _statfs(struct statvfs *stbuf); public: @@ -793,7 +804,8 @@ public: int ll_read(Fh *fh, off_t off, off_t len, bufferlist *bl); int ll_write(Fh *fh, off_t off, off_t len, const char *data); int ll_release(Fh *fh); - + int ll_statfs(inodeno_t, struct statvfs *stbuf); + // failure void ms_handle_failure(Message*, const entity_inst_t& inst); diff --git a/branches/sage/mds/client/fuse_ll.cc b/branches/sage/mds/client/fuse_ll.cc index b44279a42bf03..6ad1a91455796 100644 --- a/branches/sage/mds/client/fuse_ll.cc +++ b/branches/sage/mds/client/fuse_ll.cc @@ -261,10 +261,12 @@ static void ceph_ll_readdir(fuse_req_t req, fuse_ino_t ino, size_t size, size_t entrysize = fuse_add_direntry(req, buf + pos, size - pos, de.d_name, &st, off); + /* cout << "ceph_ll_readdir added " << de.d_name << " at " << pos << " len " << entrysize << " (buffer size is " << size << ")" << " .. off = " << off << endl; + */ if (entrysize > size - pos) break; // didn't fit, done for now. @@ -299,6 +301,16 @@ static void ceph_ll_create(fuse_req_t req, fuse_ino_t parent, const char *name, } } +static void ceph_ll_statfs(fuse_req_t req, fuse_ino_t ino) +{ + struct statvfs stbuf; + int r = client->ll_statfs(ino, &stbuf); + if (r == 0) + fuse_reply_statfs(req, &stbuf); + else + fuse_reply_err(req, -r); +} + static struct fuse_lowlevel_ops ceph_ll_oper = { init: 0, destroy: 0, @@ -324,13 +336,13 @@ static struct fuse_lowlevel_ops ceph_ll_oper = { readdir: ceph_ll_readdir, releasedir: ceph_ll_releasedir, fsyncdir: 0, - statfs: 0, + statfs: ceph_ll_statfs, setxattr: 0, getxattr: 0, listxattr: 0, removexattr: 0, access: 0, - create: 0, //ceph_ll_create, + create: ceph_ll_create, getlk: 0, setlk: 0, bmap: 0 diff --git a/branches/sage/mds/messages/MStatfs.h b/branches/sage/mds/messages/MStatfs.h index 2274707a0e128..66e5847206a7b 100644 --- a/branches/sage/mds/messages/MStatfs.h +++ b/branches/sage/mds/messages/MStatfs.h @@ -20,21 +20,22 @@ class MStatfs : public Message { public: - struct statvfs stfs; + tid_t tid; MStatfs() : Message(MSG_STATFS) {} + MStatfs(tid_t t) : Message(MSG_STATFS), tid(t) {} char *get_type_name() { return "statfs"; } void print(ostream& out) { - out << "statfs" << endl; + out << "statfs(" << tid << ")"; } void encode_payload() { - ::_encode(stfs, payload); + ::_encode(tid, payload); } void decode_payload() { int off = 0; - ::_decode(stfs, payload, off); + ::_decode(tid, payload, off); } }; diff --git a/branches/sage/mds/messages/MStatfsReply.h b/branches/sage/mds/messages/MStatfsReply.h new file mode 100644 index 0000000000000..f8e21ddcc2b31 --- /dev/null +++ b/branches/sage/mds/messages/MStatfsReply.h @@ -0,0 +1,45 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2004-2006 Sage Weil + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + + +#ifndef __MSTATFSREPLY_H +#define __MSTATFSREPLY_H + +#include /* or */ + +class MStatfsReply : public Message { +public: + tid_t tid; + struct statvfs stfs; + + MStatfsReply() : Message(MSG_STATFS_REPLY) {} + MStatfsReply(tid_t t) : Message(MSG_STATFS_REPLY), tid(t) {} + + char *get_type_name() { return "statfs_reply"; } + void print(ostream& out) { + out << "statfs_reply(" << tid << ")"; + } + + void encode_payload() { + ::_encode(tid, payload); + ::_encode(stfs, payload); + } + void decode_payload() { + int off = 0; + ::_decode(tid, payload, off); + ::_decode(stfs, payload, off); + } +}; + +#endif diff --git a/branches/sage/mds/mon/Monitor.cc b/branches/sage/mds/mon/Monitor.cc index 299dbaf2f11c7..bf7e01bfc67e9 100644 --- a/branches/sage/mds/mon/Monitor.cc +++ b/branches/sage/mds/mon/Monitor.cc @@ -282,6 +282,12 @@ void Monitor::dispatch(Message *m) clientmon->dispatch(m); break; + // pg + case MSG_STATFS: + case MSG_PGSTATS: + pgmon->dispatch(m); + break; + // paxos case MSG_MON_PAXOS: diff --git a/branches/sage/mds/mon/PGMonitor.cc b/branches/sage/mds/mon/PGMonitor.cc index 68a75f5f5ee0f..5865229dde145 100644 --- a/branches/sage/mds/mon/PGMonitor.cc +++ b/branches/sage/mds/mon/PGMonitor.cc @@ -20,7 +20,9 @@ #include "MonitorStore.h" #include "messages/MPGStats.h" + #include "messages/MStatfs.h" +#include "messages/MStatfsReply.h" #include "common/Timer.h" @@ -142,13 +144,22 @@ void PGMonitor::handle_statfs(MStatfs *statfs) dout(10) << "handle_statfs " << *statfs << " from " << statfs->get_source() << endl; // fill out stfs - memset(&statfs->stfs, 0, sizeof(statfs->stfs)); - statfs->stfs.f_blocks = pg_map.total_num_blocks; - statfs->stfs.f_fsid = 0; // hmm. - statfs->stfs.f_flag = ST_NOATIME|ST_NODIRATIME; // for now. + MStatfsReply *reply = new MStatfsReply(statfs->tid); + memset(&reply->stfs, 0, sizeof(reply->stfs)); + reply->stfs.f_bsize = 1024; + reply->stfs.f_frsize = 1024; + reply->stfs.f_blocks = 1024 * 1024; //pg_map.total_num_blocks; + reply->stfs.f_bfree = 1024 * 1024; + reply->stfs.f_bavail = 1024 * 1024; + reply->stfs.f_files = 1024 * 1024; + reply->stfs.f_ffree = 1024 * 1024; + reply->stfs.f_favail = 1024 * 1024; + reply->stfs.f_namemax = 1024; + reply->stfs.f_flag = ST_NOATIME|ST_NODIRATIME; // for now. // reply - mon->messenger->send_message(statfs, statfs->get_source_inst()); + mon->messenger->send_message(reply, statfs->get_source_inst()); + delete statfs; } bool PGMonitor::handle_pg_stats(MPGStats *stats) diff --git a/branches/sage/mds/msg/Message.cc b/branches/sage/mds/msg/Message.cc index fa63838a2d51f..4d400735713ee 100644 --- a/branches/sage/mds/msg/Message.cc +++ b/branches/sage/mds/msg/Message.cc @@ -12,7 +12,9 @@ using namespace std; #include "messages/MGenericMessage.h" #include "messages/MPGStats.h" + #include "messages/MStatfs.h" +#include "messages/MStatfsReply.h" #include "messages/MMonCommand.h" #include "messages/MMonCommandAck.h" @@ -112,9 +114,13 @@ decode_message(msg_envelope_t& env, bufferlist& payload) case MSG_PGSTATS: m = new MPGStats; break; + case MSG_STATFS: m = new MStatfs; break; + case MSG_STATFS_REPLY: + m = new MStatfsReply; + break; case MSG_MON_COMMAND: m = new MMonCommand; diff --git a/branches/sage/mds/msg/Message.h b/branches/sage/mds/msg/Message.h index f593f1aac2cf3..693b0ba7f628c 100644 --- a/branches/sage/mds/msg/Message.h +++ b/branches/sage/mds/msg/Message.h @@ -17,8 +17,9 @@ #define MSG_CLOSE 0 -#define MSG_STATFS 1 -#define MSG_PGSTATS 2 +#define MSG_STATFS 1 +#define MSG_STATFS_REPLY 2 +#define MSG_PGSTATS 3 #define MSG_PING 10 #define MSG_PING_ACK 11 -- 2.39.5