From 95feeb6d6262ba32f1c2c0ee6cbd1eaa065b6739 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Tue, 15 Jan 2008 14:29:47 -0800 Subject: [PATCH] simplify client caps release; track last_open to avoid open race on mds --- src/client/Client.cc | 56 ++++++++++++---------------------- src/include/ceph_fs.h | 8 ++--- src/mds/Capability.h | 5 +++ src/mds/Locker.cc | 37 ++++++++++------------ src/mds/MDCache.cc | 7 +---- src/messages/MClientFileCaps.h | 3 +- 6 files changed, 45 insertions(+), 71 deletions(-) diff --git a/src/client/Client.cc b/src/client/Client.cc index 34ea4971d5d90..7627a8fdfda97 100644 --- a/src/client/Client.cc +++ b/src/client/Client.cc @@ -1205,36 +1205,6 @@ void Client::handle_file_caps(MClientFileCaps *m) return; } - // release? - if (m->get_op() == CEPH_CAP_OP_RELEASE) { - dout(5) << "handle_file_caps on ino " << m->get_ino() << " from mds" << mds << " release" << dendl; - assert(in->caps.count(mds)); - in->caps.erase(mds); - caps_by_mds[mds]--; - for (map::iterator p = in->caps.begin(); - p != in->caps.end(); - p++) - dout(20) << " left cap " << p->first << " " - << cap_string(p->second.caps) << " " - << p->second.seq << dendl; - for (map::iterator p = in->stale_caps.begin(); - p != in->stale_caps.end(); - p++) - dout(20) << " left stale cap " << p->first << " " - << cap_string(p->second.caps) << " " - << p->second.seq << dendl; - - if (in->caps.empty()) { - //dout(0) << "did put_inode" << dendl; - put_inode(in); - } else { - //dout(0) << "didn't put_inode" << dendl; - } - delete m; - return; - } - - // don't want? if (in->file_caps_wanted() == 0) { dout(5) << "handle_file_caps on ino " << m->get_ino() @@ -1366,10 +1336,11 @@ void Client::implemented_caps(MClientFileCaps *m, Inode *in) void Client::release_caps(Inode *in, int retain) { + int wanted = in->file_caps_wanted(); dout(5) << "releasing caps on ino " << in->inode.ino << dec << " had " << cap_string(in->file_caps()) << " retaining " << cap_string(retain) - << " want " << cap_string(in->file_caps_wanted()) + << " want " << cap_string(wanted) << dendl; for (map::iterator it = in->caps.begin(); @@ -1384,11 +1355,17 @@ void Client::release_caps(Inode *in, in->inode, it->second.seq, it->second.caps, - in->file_caps_wanted()); + wanted); messenger->send_message(m, mdsmap->get_inst(it->first)); } + if (wanted == 0) + caps_by_mds[it->first]--; } - + if (wanted == 0 && !in->caps.empty()) { + in->caps.clear(); + put_inode(in); + } + if (in->file_caps() == 0) { in->file_wr_mtime = utime_t(); in->file_wr_size = 0; @@ -1397,8 +1374,9 @@ void Client::release_caps(Inode *in, void Client::update_caps_wanted(Inode *in) { + int wanted = in->file_caps_wanted(); dout(5) << "updating caps wanted on ino " << in->inode.ino - << " to " << cap_string(in->file_caps_wanted()) + << " to " << cap_string(wanted) << dendl; // FIXME: pick a single mds and let the others off the hook.. @@ -1409,11 +1387,17 @@ void Client::update_caps_wanted(Inode *in) in->inode, it->second.seq, it->second.caps, - in->file_caps_wanted()); + wanted); messenger->send_message(m, mdsmap->get_inst(it->first)); + if (wanted == 0) + caps_by_mds[it->first]--; + } + if (wanted == 0 && !in->caps.empty()) { + in->caps.clear(); + put_inode(in); } } - + // ------------------- diff --git a/src/include/ceph_fs.h b/src/include/ceph_fs.h index dadb4253524ff..ca1eee2e5c00b 100644 --- a/src/include/ceph_fs.h +++ b/src/include/ceph_fs.h @@ -434,19 +434,15 @@ struct ceph_mds_reply_dirfrag { #define CEPH_CAP_WRBUFFER 16 /* client can buffer writes */ #define CEPH_CAP_WREXTEND 32 /* client can extend eof */ #define CEPH_CAP_LAZYIO 64 /* client can perform lazy io */ + enum { CEPH_CAP_OP_GRANT, /* mds->client grant */ CEPH_CAP_OP_ACK, /* client->mds ack (if prior grant was a recall) */ CEPH_CAP_OP_REQUEST, /* client->mds request (update wanted bits) */ - CEPH_CAP_OP_RELEASE, /* mds->client release (*) */ CEPH_CAP_OP_EXPORT, /* mds has exported the cap */ CEPH_CAP_OP_IMPORT /* mds has imported the cap from specified mds */ }; - /* - * (*) it's a bit counterintuitive, but the mds has to - * close the cap because the client isn't able to tell - * if a concurrent open() would map to the same inode. - */ + struct ceph_mds_file_caps { __le32 op; __le32 seq; diff --git a/src/mds/Capability.h b/src/mds/Capability.h index 96df0e7b240f1..999a999b3cbf4 100644 --- a/src/mds/Capability.h +++ b/src/mds/Capability.h @@ -63,6 +63,7 @@ private: map cap_history; // seq -> cap capseq_t last_sent, last_recv; + capseq_t last_open; bool suppress; bool stale; @@ -74,10 +75,14 @@ public: wanted_caps(want), last_sent(s), last_recv(s), + last_open(0), suppress(false), stale(false), session_caps_item(this) { } + capseq_t get_last_open() { return last_open; } + void set_last_open() { last_open = last_sent; } + bool is_suppress() { return suppress; } void set_suppress(bool b) { suppress = b; } diff --git a/src/mds/Locker.cc b/src/mds/Locker.cc index aa05e288cf652..0bb2e6bf5c13a 100644 --- a/src/mds/Locker.cc +++ b/src/mds/Locker.cc @@ -484,12 +484,13 @@ Capability* Locker::issue_new_caps(CInode *in, // [replica] tell auth about any new caps wanted request_inode_file_caps(in); } - + // issue caps (pot. incl new one) issue_caps(in); // note: _eval above may have done this already... // re-issue whatever we can cap->issue(cap->pending()); + cap->set_last_open(); // ok, stop suppressing. cap->set_suppress(false); @@ -742,9 +743,13 @@ void Locker::handle_client_file_caps(MClientFileCaps *m) << " on " << *in << dendl; + // confirm caps + int had = cap->confirm_receipt(m->get_seq(), m->get_caps()); + int has = cap->confirmed(); + // update wanted if (cap->wanted() != wanted) { - if (m->get_seq() < cap->get_last_seq()) { + if (m->get_seq() < cap->get_last_open()) { /* this is awkward. client may be trying to release caps (i.e. inode closed, etc.) by setting reducing wanted set. @@ -753,30 +758,20 @@ void Locker::handle_client_file_caps(MClientFileCaps *m) sure the client has seen all the latest caps. */ dout(10) << "handle_client_file_caps ignoring wanted " << cap_string(m->get_wanted()) - << " bc seq " << m->get_seq() << " < " << cap->get_last_seq() << dendl; + << " bc seq " << m->get_seq() << " < last open " << cap->get_last_open() << dendl; + } else if (wanted == 0) { + // outright release? + dout(7) << " cap for client" << client << " is now null, removing from " << *in << dendl; + in->remove_client_cap(client); + if (!in->is_any_caps()) + in->xlist_open_file.remove_myself(); // unpin logsegment + if (!in->is_auth()) + request_inode_file_caps(in); } else { cap->set_wanted(wanted); } } - // confirm caps - int had = cap->confirm_receipt(m->get_seq(), m->get_caps()); - int has = cap->confirmed(); - if (cap->is_null()) { - dout(7) << " cap for client" << client << " is now null, removing from " << *in << dendl; - in->remove_client_cap(client); - if (!in->is_any_caps()) - in->xlist_open_file.remove_myself(); // unpin logsegment - if (!in->is_auth()) - request_inode_file_caps(in); - - // tell client. - MClientFileCaps *r = new MClientFileCaps(CEPH_CAP_OP_RELEASE, - in->inode, - 0, 0, 0); - mds->send_message_client(r, m->get_source_inst()); - } - // merge in atime? if (m->get_atime() > in->inode.atime) { dout(7) << " taking atime " << m->get_atime() << " > " diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc index 44adfa5030944..ee07fefd47b2f 100644 --- a/src/mds/MDCache.cc +++ b/src/mds/MDCache.cc @@ -3562,6 +3562,7 @@ bool MDCache::shutdown_pass() return false; } + // empty stray dir if (!shutdown_export_strays()) { dout(7) << "waiting for strays to migrate" << dendl; return false; @@ -3618,12 +3619,6 @@ bool MDCache::shutdown_pass() assert(!migrator->is_exporting()); assert(!migrator->is_importing()); - - - // empty out stray contents - // FIXME - dout(7) << "FIXME: i need to empty out stray dir contents..." << dendl; - // (only do this once!) if (!mds->mdlog->is_capped()) { dout(7) << "capping the log" << dendl; diff --git a/src/messages/MClientFileCaps.h b/src/messages/MClientFileCaps.h index 705540bc4763b..bf8c97f91f5e2 100644 --- a/src/messages/MClientFileCaps.h +++ b/src/messages/MClientFileCaps.h @@ -24,7 +24,6 @@ class MClientFileCaps : public Message { switch (op) { case CEPH_CAP_OP_GRANT: return "grant"; case CEPH_CAP_OP_ACK: return "ack"; - case CEPH_CAP_OP_RELEASE: return "release"; case CEPH_CAP_OP_EXPORT: return "export"; case CEPH_CAP_OP_IMPORT: return "import"; default: assert(0); return 0; @@ -85,7 +84,7 @@ class MClientFileCaps : public Message { void print(ostream& out) { out << "client_file_caps(" << get_opname(le32_to_cpu(h.op)) << " ino " << inodeno_t(le64_to_cpu(h.ino)) - << " seq " << le32_to_cpu(h.seq) + << " seq " << le32_to_cpu(h.seq) << " caps " << cap_string(le32_to_cpu(h.caps)) << " wanted" << cap_string(le32_to_cpu(h.wanted)) << ")"; -- 2.39.5