From 68a829f73f5d72c34409634c61a66992a9e99450 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 11 Jan 2008 11:42:34 -0800 Subject: [PATCH] file cap ops cleanup --- src/Makefile | 4 +--- src/client/Client.cc | 17 +++++++++-------- src/include/ceph_fs.h | 21 ++++++++++++++++---- src/include/utime.h | 6 ++---- src/kernel/client.c | 18 ++++++----------- src/kernel/client.h | 4 +++- src/kernel/inode.c | 35 ++++++++++++++++++++++++++++++++++ src/kernel/mds_client.c | 2 ++ src/kernel/mds_client.h | 1 + src/kernel/mon_client.c | 3 +-- src/kernel/super.c | 2 +- src/kernel/super.h | 1 + src/mds/Locker.cc | 4 ++-- src/mds/MDCache.cc | 4 ++-- src/mds/Migrator.cc | 8 ++++---- src/mds/Server.cc | 2 +- src/messages/MClientFileCaps.h | 34 ++++++++++++++------------------- 17 files changed, 102 insertions(+), 64 deletions(-) diff --git a/src/Makefile b/src/Makefile index 0d9604ec5b58a..a68c506186be2 100644 --- a/src/Makefile +++ b/src/Makefile @@ -136,8 +136,6 @@ endif all: depend ${TARGETS} -test: depend ${TEST_TARGETS} - # real bits mkmonmap: mkmonmap.cc common.o @@ -300,7 +298,7 @@ osbdb.o: ${OSBDB_OBJS} # handy clean: - rm -f *.o */*.o crush/*.co ${TARGETS} ${TEST_TARGETS} + rm -f *.o */*.o crush/*.co ${TARGETS} count: cat ${SRCS} | wc -l diff --git a/src/client/Client.cc b/src/client/Client.cc index 35745c659da58..5373eab245be2 100644 --- a/src/client/Client.cc +++ b/src/client/Client.cc @@ -1117,8 +1117,8 @@ void Client::handle_file_caps(MClientFileCaps *m) mds_sessions[mds]++; // reap? - if (m->get_op() == MClientFileCaps::OP_IMPORT) { - int other = m->get_mds(); + if (m->get_op() == CEPH_CAP_OP_IMPORT) { + int other = m->get_migrate_mds(); /* * FIXME: there is a race here.. if the caps are exported twice in succession, @@ -1157,7 +1157,7 @@ void Client::handle_file_caps(MClientFileCaps *m) assert(in); // stale? - if (m->get_op() == MClientFileCaps::OP_EXPORT) { + if (m->get_op() == CEPH_CAP_OP_EXPORT) { dout(5) << "handle_file_caps on ino " << m->get_ino() << " seq " << m->get_seq() << " from mds" << mds << " now exported/stale" << dendl; // move to stale list @@ -1172,7 +1172,8 @@ void Client::handle_file_caps(MClientFileCaps *m) // delayed reap? if (cap_reap_queue.count(in->ino()) && cap_reap_queue[in->ino()].count(mds)) { - dout(5) << "handle_file_caps on ino " << m->get_ino() << " from mds" << mds << " delayed reap on mds" << m->get_mds() << dendl; + dout(5) << "handle_file_caps on ino " << m->get_ino() << " from mds" << mds + << " delayed reap on mds?FIXME?" << dendl; /* FIXME */ // process delayed reap handle_file_caps( cap_reap_queue[in->ino()][mds] ); @@ -1186,7 +1187,7 @@ void Client::handle_file_caps(MClientFileCaps *m) } // release? - if (m->get_op() == MClientFileCaps::OP_RELEASE) { + if (m->get_op() == CEPH_CAP_OP_RELEASE) { dout(5) << "handle_file_caps on ino " << m->get_ino() << " from mds" << mds << " release" << dendl; assert(in->caps.count(mds)); in->caps.erase(mds); @@ -1220,7 +1221,7 @@ void Client::handle_file_caps(MClientFileCaps *m) << " seq " << m->get_seq() << " " << cap_string(m->get_caps()) << ", which we don't want caps for, releasing." << dendl; - m->set_op(MClientFileCaps::OP_ACK); + m->set_op(CEPH_CAP_OP_ACK); m->set_caps(0); m->set_wanted(0); messenger->send_message(m, m->get_source_inst()); @@ -1359,7 +1360,7 @@ void Client::release_caps(Inode *in, // release (some of?) these caps it->second.caps = retain & it->second.caps; // note: tell mds _full_ wanted; it'll filter/behave based on what it is allowed to do - MClientFileCaps *m = new MClientFileCaps(MClientFileCaps::OP_ACK, + MClientFileCaps *m = new MClientFileCaps(CEPH_CAP_OP_ACK, in->inode, it->second.seq, it->second.caps, @@ -1384,7 +1385,7 @@ void Client::update_caps_wanted(Inode *in) for (map::iterator it = in->caps.begin(); it != in->caps.end(); it++) { - MClientFileCaps *m = new MClientFileCaps(MClientFileCaps::OP_ACK, + MClientFileCaps *m = new MClientFileCaps(CEPH_CAP_OP_ACK, in->inode, it->second.seq, it->second.caps, diff --git a/src/include/ceph_fs.h b/src/include/ceph_fs.h index e600c594dd486..9be58ffa40164 100644 --- a/src/include/ceph_fs.h +++ b/src/include/ceph_fs.h @@ -50,8 +50,8 @@ struct ceph_object { #define CEPH_INO_ROOT 1 struct ceph_timeval { - __u32 tv_sec; - __u32 tv_usec; + __le32 tv_sec; + __le32 tv_usec; }; /* @@ -419,13 +419,26 @@ struct ceph_mds_reply_dirfrag { } __attribute__ ((packed)); /* client file caps */ +enum { + CEPH_CAP_OP_GRANT, /* mds->client grant */ + CEPH_CAP_OP_ACK, /* client->mds ack (if prior grant was a recall) */ + CEPH_CAP_OP_RELEASE, /* mds->client release (*) */ + CEPH_CAP_OP_EXPORT, /* mds has exported the cap */ + CEPH_CAP_OP_IMPORT /* mds has imported the cap from specified mds */ +}; + /* + * (*) it's a bit counterintuitive, but the mds has to + * close the cap because the client isn't able to tell + * if a concurrent open() would map to the same inode. + */ struct ceph_mds_file_caps { - __le32 op; - __le32 mds; __le64 seq; __le32 caps, wanted; __le64 ino; __le64 size; + __le32 op; + __le32 migrate_mds; + __le32 migrate_seq; struct ceph_timeval mtime, atime; } __attribute__ ((packed)); diff --git a/src/include/utime.h b/src/include/utime.h index 683c148423627..f0dec0ecc4cb5 100644 --- a/src/include/utime.h +++ b/src/include/utime.h @@ -23,10 +23,8 @@ // -------- // utime_t -typedef struct ceph_timeval _utime_t; - class utime_t { - _utime_t tv; + struct ceph_timeval tv; friend class Clock; @@ -42,7 +40,7 @@ class utime_t { utime_t() { tv.tv_sec = 0; tv.tv_usec = 0; normalize(); } //utime_t(time_t s) { tv.tv_sec = s; tv.tv_usec = 0; } utime_t(time_t s, int u) { tv.tv_sec = s; tv.tv_usec = u; normalize(); } - utime_t(const _utime_t &v) : tv(v) {} + utime_t(const struct ceph_timeval &v) : tv(v) {} utime_t(const struct timeval &v) { set_from_timeval(&v); } diff --git a/src/kernel/client.c b/src/kernel/client.c index 9a90ef12ff720..db8a046466445 100644 --- a/src/kernel/client.c +++ b/src/kernel/client.c @@ -175,27 +175,18 @@ static struct ceph_client *get_client_monaddr(struct ceph_entity_addr *monaddr) } */ -struct ceph_client *ceph_get_client(struct ceph_mount_args *args) +struct ceph_client *ceph_create_client(struct ceph_mount_args *args, struct super_block *sb) { struct ceph_client *client = 0; int ret; - /* existing, by fsid? */ - /* - if (args->flags & CEPH_MOUNT_FSID) - client = ceph_get_client_fsid(&args->fsid); - if (client) - return client; - */ - /* existing, by monitors? */ - /* write me. */ - /* create new client */ client = create_client(args); if (IS_ERR(client)) return client; atomic_inc(&client->nref); - + client->sb = sb; + /* request mount */ ret = mount(client, args); if (ret < 0) { @@ -278,6 +269,9 @@ void ceph_dispatch(void *p, struct ceph_msg *msg) case CEPH_MSG_CLIENT_REQUEST_FORWARD: ceph_mdsc_handle_forward(&client->mdsc, msg); break; + case CEPH_MSG_CLIENT_FILECAPS: + ceph_handle_filecaps(&client->mdsc, msg); + break; /* osd client */ case CEPH_MSG_OSD_MAP: diff --git a/src/kernel/client.h b/src/kernel/client.h index 78abde72ec92d..fe0e7b168ea29 100644 --- a/src/kernel/client.h +++ b/src/kernel/client.h @@ -39,6 +39,8 @@ struct ceph_client { struct ceph_fsid fsid; atomic_t nref; + struct super_block *sb; + unsigned long mounting; /* map bitset; 4=mon, 2=mds, 1=osd map */ struct completion mount_completion; @@ -53,7 +55,7 @@ struct ceph_client { struct list_head sb_list; }; -extern struct ceph_client *ceph_get_client(struct ceph_mount_args *args); +extern struct ceph_client *ceph_create_client(struct ceph_mount_args *args, struct super_block *sb); extern void ceph_put_client(struct ceph_client *cl); #endif diff --git a/src/kernel/inode.c b/src/kernel/inode.c index 03745704d362e..3387afb941949 100644 --- a/src/kernel/inode.c +++ b/src/kernel/inode.c @@ -151,6 +151,41 @@ int ceph_get_caps(struct ceph_inode_info *ci) } +/* caps */ + +void ceph_handle_filecaps(struct ceph_mds_client *mdsc, struct ceph_msg *msg) +{ + struct super_block *sb = mdsc->client->sb; + struct ceph_client *client = ceph_sbinfo(sb)->sb_client; + struct inode *inode; + struct ceph_mds_file_caps *h; + int mds = msg->hdr.src.name.num; + int op; + __u64 ino; + + dout(10, "handle_filecaps from mds%d\n", mds); + + /* decode */ + if (msg->front.iov_len != sizeof(*h)) + goto bad; + h = msg->front.iov_base; + op = le32_to_cpu(h->op); + ino = le64_to_cpu(h->ino); + + /* lookup ino */ + inode = ilookup(sb, ino); + dout(20, "op is %d, inode is %llx %p\n", op, ino, inode); + + switch (op) { + + } + + return; +bad: + dout(10, "corrupt filecaps message\n"); +} + + /* diff --git a/src/kernel/mds_client.c b/src/kernel/mds_client.c index 837aba1d08040..235e31154d163 100644 --- a/src/kernel/mds_client.c +++ b/src/kernel/mds_client.c @@ -888,4 +888,6 @@ bad2: return; } + + /* eof */ diff --git a/src/kernel/mds_client.h b/src/kernel/mds_client.h index 15633c65b5875..70ea1ec3c02a4 100644 --- a/src/kernel/mds_client.h +++ b/src/kernel/mds_client.h @@ -106,4 +106,5 @@ extern int ceph_mdsc_parse_reply_info(struct ceph_msg *msg, struct ceph_mds_repl extern void ceph_mdsc_destroy_reply_info(struct ceph_mds_reply_info *info); extern void ceph_mdsc_fill_inode(struct inode *inode, struct ceph_mds_reply_inode *i); + #endif diff --git a/src/kernel/mon_client.c b/src/kernel/mon_client.c index e0a2cb8cfad1a..d642a8fe9b7a5 100644 --- a/src/kernel/mon_client.c +++ b/src/kernel/mon_client.c @@ -74,7 +74,7 @@ void ceph_monc_request_mdsmap(struct ceph_mon_client *monc, __u64 have) /* - * message handlers + * statfs */ void ceph_monc_handle_statfs_reply(struct ceph_mon_client *monc, struct ceph_msg *msg) @@ -148,6 +148,5 @@ int ceph_monc_do_statfs(struct ceph_mon_client *monc, struct ceph_statfs *buf) dout(20, "do_statfs waiting for reply\n"); wait_for_completion(&req.completion); - return 0; } diff --git a/src/kernel/super.c b/src/kernel/super.c index b26d077a23401..15eb752a06203 100644 --- a/src/kernel/super.c +++ b/src/kernel/super.c @@ -499,7 +499,7 @@ static int ceph_get_sb(struct file_system_type *fs_type, /* client */ if (!sbinfo->sb_client) { - sbinfo->sb_client = ceph_get_client(&mount_args); + sbinfo->sb_client = ceph_create_client(&mount_args, sb); if (IS_ERR(sbinfo->sb_client)) { err = PTR_ERR(sbinfo->sb_client); sbinfo->sb_client = 0; diff --git a/src/kernel/super.h b/src/kernel/super.h index 2d9a32584c98b..9034b772af4ca 100644 --- a/src/kernel/super.h +++ b/src/kernel/super.h @@ -139,6 +139,7 @@ extern struct ceph_inode_cap *ceph_find_cap(struct inode *inode, int want); extern struct ceph_inode_cap *ceph_add_cap(struct inode *inode, int mds, u32 cap, u32 seq); extern int ceph_inode_getattr(struct vfsmount *mnt, struct dentry *dentry, struct kstat *stat); +extern void ceph_handle_filecaps(struct ceph_mds_client *mdsc, struct ceph_msg *msg); /* addr.c */ diff --git a/src/mds/Locker.cc b/src/mds/Locker.cc index 94d8773feecc8..b1da8857672a3 100644 --- a/src/mds/Locker.cc +++ b/src/mds/Locker.cc @@ -547,7 +547,7 @@ bool Locker::issue_caps(CInode *in) if (seq > 0 && !it->second.is_suppress()) { dout(7) << " sending MClientFileCaps to client" << it->first << " seq " << it->second.get_last_seq() << " new pending " << cap_string(it->second.pending()) << " was " << cap_string(before) << dendl; - mds->send_message_client(new MClientFileCaps(MClientFileCaps::OP_GRANT, + mds->send_message_client(new MClientFileCaps(CEPH_CAP_OP_GRANT, in->inode, it->second.get_last_seq(), it->second.pending(), @@ -727,7 +727,7 @@ void Locker::handle_client_file_caps(MClientFileCaps *m) request_inode_file_caps(in); // tell client. - MClientFileCaps *r = new MClientFileCaps(MClientFileCaps::OP_RELEASE, + MClientFileCaps *r = new MClientFileCaps(CEPH_CAP_OP_RELEASE, in->inode, 0, 0, 0); mds->send_message_client(r, m->get_source_inst()); diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc index af4dfdec9ce03..b0e76642523e5 100644 --- a/src/mds/MDCache.cc +++ b/src/mds/MDCache.cc @@ -2657,13 +2657,13 @@ void MDCache::rejoin_import_cap(CInode *in, int client, inode_caps_reconnect_t& // send REAP // FIXME client session weirdness. - MClientFileCaps *reap = new MClientFileCaps(MClientFileCaps::OP_IMPORT, + MClientFileCaps *reap = new MClientFileCaps(CEPH_CAP_OP_IMPORT, in->inode, in->client_caps[client].get_last_seq(), in->client_caps[client].pending(), in->client_caps[client].wanted()); - reap->set_mds( frommds ); // reap from whom? + reap->set_migrate_mds(frommds); // reap from whom? mds->messenger->send_message(reap, mds->clientmap.get_inst(client)); } diff --git a/src/mds/Migrator.cc b/src/mds/Migrator.cc index 78cd7fcfcbcc0..058916765b747 100644 --- a/src/mds/Migrator.cc +++ b/src/mds/Migrator.cc @@ -898,7 +898,7 @@ void Migrator::finish_export_inode_caps(CInode *in) it++) { dout(7) << "finish_export_inode telling client" << it->first << " exported caps on " << *in << dendl; - MClientFileCaps *m = new MClientFileCaps(MClientFileCaps::OP_EXPORT, + MClientFileCaps *m = new MClientFileCaps(CEPH_CAP_OP_EXPORT, in->inode, it->second.get_last_seq(), it->second.pending(), @@ -2043,12 +2043,12 @@ void Migrator::finish_import_inode_caps(CInode *in, int from, it != new_caps.end(); it++) { dout(0) << "finish_import_inode_caps for client" << *it << " on " << *in << dendl; - MClientFileCaps *caps = new MClientFileCaps(MClientFileCaps::OP_IMPORT, + MClientFileCaps *caps = new MClientFileCaps(CEPH_CAP_OP_IMPORT, in->inode, in->client_caps[*it].get_last_seq(), in->client_caps[*it].pending(), - in->client_caps[*it].wanted()); - caps->set_mds(from); // from whom? + in->client_caps[*it].wanted(), + from); mds->send_message_client(caps, *it); } } diff --git a/src/mds/Server.cc b/src/mds/Server.cc index ffeee1de8079f..e383a87e1b560 100644 --- a/src/mds/Server.cc +++ b/src/mds/Server.cc @@ -321,7 +321,7 @@ void Server::handle_client_reconnect(MClientReconnect *m) // mark client caps stale. inode_t fake_inode; fake_inode.ino = p->first; - MClientFileCaps *stale = new MClientFileCaps(MClientFileCaps::OP_EXPORT, + MClientFileCaps *stale = new MClientFileCaps(CEPH_CAP_OP_EXPORT, fake_inode, 0, 0, // doesn't matter. diff --git a/src/messages/MClientFileCaps.h b/src/messages/MClientFileCaps.h index 6d552c3271967..b44365ee73c7d 100644 --- a/src/messages/MClientFileCaps.h +++ b/src/messages/MClientFileCaps.h @@ -21,23 +21,13 @@ class MClientFileCaps : public Message { public: - static const int OP_GRANT = 0; // mds->client grant. - static const int OP_ACK = 1; // client->mds ack (if prior grant was a recall) - static const int OP_RELEASE = 2; // mds->client release cap (*) - static const int OP_EXPORT = 3; // mds has exported the cap - static const int OP_IMPORT = 4; // mds has imported the cap from get_mds() - /* - * (*) it's a bit counterintuitive, but the mds has to - * close the cap because the client isn't able to tell - * if a concurrent open() would map to the same inode. - */ static const char* get_opname(int op) { switch (op) { - case OP_GRANT: return "grant"; - case OP_ACK: return "ack"; - case OP_RELEASE: return "release"; - case OP_EXPORT: return "export"; - case OP_IMPORT: return "import"; + case CEPH_CAP_OP_GRANT: return "grant"; + case CEPH_CAP_OP_ACK: return "ack"; + case CEPH_CAP_OP_RELEASE: return "release"; + case CEPH_CAP_OP_EXPORT: return "export"; + case CEPH_CAP_OP_IMPORT: return "import"; default: assert(0); return 0; } } @@ -56,13 +46,15 @@ class MClientFileCaps : public Message { utime_t get_atime() { return utime_t(h.atime); } // for cap migration - int get_mds() { return le32_to_cpu(h.mds); } + int get_migrate_mds() { return le32_to_cpu(h.migrate_mds); } + int get_migrate_seq() { return le32_to_cpu(h.migrate_seq); } int get_op() { return le32_to_cpu(h.op); } void set_caps(int c) { h.caps = cpu_to_le32(c); } void set_wanted(int w) { h.wanted = cpu_to_le32(w); } - void set_mds(int m) { h.mds = cpu_to_le32(m); } + void set_migrate_mds(int m) { h.migrate_mds = cpu_to_le32(m); } + void set_migrate_seq(int m) { h.migrate_seq = cpu_to_le32(m); } void set_op(int o) { h.op = cpu_to_le32(o); } void set_size(loff_t s) { h.size = cpu_to_le64(s); } @@ -75,15 +67,17 @@ class MClientFileCaps : public Message { long seq, int caps, int wanted, - int mds=0) : + int mmds=0, + int mseq=0) : Message(CEPH_MSG_CLIENT_FILECAPS) { - h.op = cpu_to_le32(op); - h.mds = cpu_to_le32(mds); h.seq = cpu_to_le64(seq); h.caps = cpu_to_le32(caps); h.wanted = cpu_to_le32(wanted); h.ino = cpu_to_le64(inode.ino); h.size = cpu_to_le64(inode.size); + h.op = cpu_to_le32(op); + h.migrate_mds = cpu_to_le32(mmds); + h.migrate_seq = cpu_to_le32(mseq); h.mtime = inode.mtime.tv_ref(); h.atime = inode.atime.tv_ref(); } -- 2.39.5