From 91085dcb98ce8d32ff3ce870d71cbed99234cc96 Mon Sep 17 00:00:00 2001 From: sageweil Date: Thu, 12 Jul 2007 23:19:12 +0000 Subject: [PATCH] * cleanup in MClientFileCaps * cap reconnect migration appears to work, yay! git-svn-id: https://ceph.svn.sf.net/svnroot/ceph@1491 29311d96-e01e-0410-9327-a35deaab8ce9 --- branches/sage/cephmds2/client/Client.cc | 19 +++-- branches/sage/cephmds2/mds/Capability.h | 16 +++-- branches/sage/cephmds2/mds/Locker.cc | 9 +-- branches/sage/cephmds2/mds/MDCache.cc | 8 +-- branches/sage/cephmds2/mds/Migrator.cc | 12 ++-- branches/sage/cephmds2/mds/Server.cc | 10 +-- .../sage/cephmds2/messages/MClientFileCaps.h | 70 +++++++++++-------- 7 files changed, 83 insertions(+), 61 deletions(-) diff --git a/branches/sage/cephmds2/client/Client.cc b/branches/sage/cephmds2/client/Client.cc index f7aa88a45f882..12c30ecf2368e 100644 --- a/branches/sage/cephmds2/client/Client.cc +++ b/branches/sage/cephmds2/client/Client.cc @@ -512,7 +512,12 @@ int Client::choose_target_mds(MClientRequest *req) // pick mds if (!diri || g_conf.client_use_random_mds) { // no root info, pick a random MDS - mds = 0;//rand() % mdsmap->get_num_mds(); + mds = rand() % mdsmap->get_num_mds(); // FIXME: this isn't really correct. + + if (0) { + dout(0) << "hack: sending all requests to mds0" << endl; + mds = 0; + } } else { if (req->auth_is_best()) { // pick the actual auth (as best we can) @@ -996,7 +1001,7 @@ void Client::handle_file_caps(MClientFileCaps *m) mds_sessions[mds]++; // reap? - if (m->get_special() == MClientFileCaps::OP_REAP) { + if (m->get_op() == MClientFileCaps::OP_REAP) { int other = m->get_mds(); if (in && in->stale_caps.count(other)) { @@ -1025,7 +1030,7 @@ void Client::handle_file_caps(MClientFileCaps *m) assert(in); // stale? - if (m->get_special() == MClientFileCaps::OP_STALE) { + if (m->get_op() == MClientFileCaps::OP_STALE) { dout(5) << "handle_file_caps on ino " << m->get_ino() << " seq " << m->get_seq() << " from mds" << mds << " now stale" << endl; // move to stale list @@ -1054,7 +1059,7 @@ void Client::handle_file_caps(MClientFileCaps *m) } // release? - if (m->get_special() == MClientFileCaps::OP_RELEASE) { + if (m->get_op() == MClientFileCaps::OP_RELEASE) { dout(5) << "handle_file_caps on ino " << m->get_ino() << " from mds" << mds << " release" << endl; assert(in->caps.count(mds)); in->caps.erase(mds); @@ -1221,7 +1226,8 @@ void Client::release_caps(Inode *in, // release (some of?) these caps it->second.caps = retain & it->second.caps; // note: tell mds _full_ wanted; it'll filter/behave based on what it is allowed to do - MClientFileCaps *m = new MClientFileCaps(in->inode, + MClientFileCaps *m = new MClientFileCaps(MClientFileCaps::OP_ACK, + in->inode, it->second.seq, it->second.caps, in->file_caps_wanted()); @@ -1245,7 +1251,8 @@ void Client::update_caps_wanted(Inode *in) for (map::iterator it = in->caps.begin(); it != in->caps.end(); it++) { - MClientFileCaps *m = new MClientFileCaps(in->inode, + MClientFileCaps *m = new MClientFileCaps(MClientFileCaps::OP_ACK, + in->inode, it->second.seq, it->second.caps, in->file_caps_wanted()); diff --git a/branches/sage/cephmds2/mds/Capability.h b/branches/sage/cephmds2/mds/Capability.h index bb3a729838f35..eab6aa84b08bc 100644 --- a/branches/sage/cephmds2/mds/Capability.h +++ b/branches/sage/cephmds2/mds/Capability.h @@ -50,6 +50,8 @@ inline string cap_string(int cap) return s; } +typedef uint32_t capseq_t; + class Capability { public: struct Export { @@ -63,13 +65,13 @@ public: private: int wanted_caps; // what the client wants (ideally) - map cap_history; // seq -> cap - long last_sent, last_recv; + map cap_history; // seq -> cap + capseq_t last_sent, last_recv; bool suppress; public: - Capability(int want=0, long s=0) : + Capability(int want=0, capseq_t s=0) : wanted_caps(want), last_sent(s), last_recv(s), @@ -107,7 +109,7 @@ public: // caps potentially issued int issued() { int c = 0; - for (long seq = last_recv; seq <= last_sent; seq++) { + for (capseq_t seq = last_recv; seq <= last_sent; seq++) { if (cap_history.count(seq)) { c |= cap_history[seq]; dout(10) << " cap issued: seq " << seq << " " << cap_string(cap_history[seq]) << " -> " << cap_string(c) << endl; @@ -143,7 +145,7 @@ public: int issued_conflicts() { return conflicts(issued()); } // issue caps; return seq number. - long issue(int c) { + capseq_t issue(int c) { //int was = pending(); //no! if (c == was && last_sent) return -1; // repeat of previous? @@ -159,7 +161,7 @@ public: */ return last_sent; } - long get_last_seq() { return last_sent; } + capseq_t get_last_seq() { return last_sent; } Export make_export() { return Export(wanted_caps, issued(), pending()); @@ -186,7 +188,7 @@ public: } // confirm receipt of a previous sent/issued seq. - int confirm_receipt(long seq, int caps) { + int confirm_receipt(capseq_t seq, int caps) { int r = 0; // old seqs diff --git a/branches/sage/cephmds2/mds/Locker.cc b/branches/sage/cephmds2/mds/Locker.cc index 574c8393519d7..bfcf990d8e532 100644 --- a/branches/sage/cephmds2/mds/Locker.cc +++ b/branches/sage/cephmds2/mds/Locker.cc @@ -539,7 +539,8 @@ bool Locker::issue_caps(CInode *in) if (seq > 0 && !it->second.is_suppress()) { dout(7) << " sending MClientFileCaps to client" << it->first << " seq " << it->second.get_last_seq() << " new pending " << cap_string(it->second.pending()) << " was " << cap_string(before) << endl; - mds->send_message_client(new MClientFileCaps(in->inode, + mds->send_message_client(new MClientFileCaps(MClientFileCaps::OP_GRANT, + in->inode, it->second.get_last_seq(), it->second.pending(), it->second.wanted()), @@ -684,9 +685,9 @@ void Locker::handle_client_file_caps(MClientFileCaps *m) request_inode_file_caps(in); // tell client. - MClientFileCaps *r = new MClientFileCaps(in->inode, - 0, 0, 0, - MClientFileCaps::OP_RELEASE); + MClientFileCaps *r = new MClientFileCaps(MClientFileCaps::OP_RELEASE, + in->inode, + 0, 0, 0); mds->send_message_client(r, m->get_source_inst()); } diff --git a/branches/sage/cephmds2/mds/MDCache.cc b/branches/sage/cephmds2/mds/MDCache.cc index 8f0515bdf0e0e..7a571ff86b0ad 100644 --- a/branches/sage/cephmds2/mds/MDCache.cc +++ b/branches/sage/cephmds2/mds/MDCache.cc @@ -2032,7 +2032,7 @@ bool MDCache::parallel_fetch(map& pathmap, */ void MDCache::rejoin_scour_survivor_replicas(int from, MMDSCacheRejoin *ack) { - dout(10) << "rejoin_scout_survivor_replicas from mds" << from << endl; + dout(10) << "rejoin_scour_survivor_replicas from mds" << from << endl; // FIXME: what about root and stray inodes. @@ -2485,11 +2485,11 @@ void MDCache::rejoin_import_cap(CInode *in, int client, inode_caps_reconnect_t& // send REAP // FIXME client session weirdness. - MClientFileCaps *reap = new MClientFileCaps(in->inode, + MClientFileCaps *reap = new MClientFileCaps(MClientFileCaps::OP_REAP, + in->inode, in->client_caps[client].get_last_seq(), in->client_caps[client].pending(), - in->client_caps[client].wanted(), - MClientFileCaps::OP_REAP); + in->client_caps[client].wanted()); reap->set_mds( frommds ); // reap from whom? mds->messenger->send_message(reap, diff --git a/branches/sage/cephmds2/mds/Migrator.cc b/branches/sage/cephmds2/mds/Migrator.cc index 8b17893d23b34..09a3bcc205352 100644 --- a/branches/sage/cephmds2/mds/Migrator.cc +++ b/branches/sage/cephmds2/mds/Migrator.cc @@ -769,11 +769,11 @@ void Migrator::encode_export_inode(CInode *in, bufferlist& enc_state, int new_au it != in->client_caps.end(); it++) { dout(7) << "encode_export_inode " << *in << " telling client" << it->first << " stale caps" << endl; - MClientFileCaps *m = new MClientFileCaps(in->inode, + MClientFileCaps *m = new MClientFileCaps(MClientFileCaps::OP_STALE, + in->inode, it->second.get_last_seq(), it->second.pending(), - it->second.wanted(), - MClientFileCaps::OP_STALE); + it->second.wanted()); entity_inst_t inst = mds->clientmap.get_inst(it->first); exported_client_map[it->first] = inst; mds->send_message_client(m, inst); @@ -1816,11 +1816,11 @@ void Migrator::decode_import_inode(CDentry *dn, bufferlist& bl, int& off, int ol for (set::iterator it = merged_client_caps.begin(); it != merged_client_caps.end(); it++) { - MClientFileCaps *caps = new MClientFileCaps(in->inode, + MClientFileCaps *caps = new MClientFileCaps(MClientFileCaps::OP_REAP, + in->inode, in->client_caps[*it].get_last_seq(), in->client_caps[*it].pending(), - in->client_caps[*it].wanted(), - MClientFileCaps::OP_REAP); + in->client_caps[*it].wanted()); caps->set_mds( oldauth ); // reap from whom? mds->send_message_client_maybe_open(caps, imported_client_map[*it]); } diff --git a/branches/sage/cephmds2/mds/Server.cc b/branches/sage/cephmds2/mds/Server.cc index df38ddecb3f30..95d27644ab576 100644 --- a/branches/sage/cephmds2/mds/Server.cc +++ b/branches/sage/cephmds2/mds/Server.cc @@ -251,17 +251,17 @@ void Server::handle_client_reconnect(MClientReconnect *m) if ((in && !in->is_auth()) || !mds->mdcache->path_is_mine(path)) { // not mine. - dout(0) << "missing " << p->first << " " << m->inode_path[p->first] - << " (not mine), will pass off to authority" << endl; + dout(0) << "non-auth " << p->first << " " << m->inode_path[p->first] + << ", will pass off to authority" << endl; // mark client caps stale. inode_t fake_inode; fake_inode.ino = p->first; - MClientFileCaps *stale = new MClientFileCaps(fake_inode, + MClientFileCaps *stale = new MClientFileCaps(MClientFileCaps::OP_STALE, + fake_inode, 0, 0, // doesn't matter. - p->second.wanted, // doesn't matter. - MClientFileCaps::OP_STALE); + p->second.wanted); // doesn't matter. mds->send_message_client(stale, m->get_source_inst()); // add to cap export list. diff --git a/branches/sage/cephmds2/messages/MClientFileCaps.h b/branches/sage/cephmds2/messages/MClientFileCaps.h index 98fee6b29ef11..c584fd63f5a1f 100644 --- a/branches/sage/cephmds2/messages/MClientFileCaps.h +++ b/branches/sage/cephmds2/messages/MClientFileCaps.h @@ -16,22 +16,34 @@ #define __MCLIENTFILECAPS_H #include "msg/Message.h" +#include "mds/Capability.h" class MClientFileCaps : public Message { public: - static const int OP_ACK = 0; // mds->client or client->mds update. FIXME? - static const int OP_RELEASE = 1; // mds closed the cap - static const int OP_STALE = 2; // mds has exported the cap - static const int OP_REAP = 3; // mds has imported the cap from get_mds() + static const int OP_GRANT = 0; // mds->client grant. + static const int OP_ACK = 1; // client->mds ack (if prior grant was a recall) + static const int OP_RELEASE = 2; // mds closed the cap + static const int OP_STALE = 3; // mds has exported the cap + static const int OP_REAP = 4; // mds has imported the cap from get_mds() + static const char* get_opname(int op) { + switch (op) { + case OP_GRANT: return "grant"; + case OP_ACK: return "ack"; + case OP_RELEASE: return "release"; + case OP_STALE: return "stale"; + case OP_REAP: return "reap"; + default: assert(0); return 0; + } + } private: - inode_t inode; - int caps; - long seq; - int wanted; + int32_t op; + inode_t inode; + capseq_t seq; + int32_t caps; + int32_t wanted; - int special; // stale || reap; in conjunction w/ mds value - int mds; + int32_t mds; public: inodeno_t get_ino() { return inode.ino; } @@ -42,33 +54,33 @@ class MClientFileCaps : public Message { // for cap migration int get_mds() { return mds; } - int get_special() { return special; } + int get_op() { return op; } void set_caps(int c) { caps = c; } void set_wanted(int w) { wanted = w; } void set_mds(int m) { mds = m; } - void set_special(int s) { special = s; } + void set_op(int s) { op = s; } MClientFileCaps() {} - MClientFileCaps(inode_t& inode, - long seq, - int caps, - int wanted, - int special = OP_ACK, - int mds=0) : - Message(MSG_CLIENT_FILECAPS) { - this->inode = inode; - this->seq = seq; - this->caps = caps; - this->wanted = wanted; - this->special = special; - this->mds = mds; - } + MClientFileCaps(int op_, + inode_t& inode_, + long seq_, + int caps_, + int wanted_, + int mds_=0) : + Message(MSG_CLIENT_FILECAPS), + op(op_), + inode(inode_), + seq(seq_), + caps(caps_), + wanted(wanted_), + mds(mds_) { } char *get_type_name() { return "Cfcap";} void print(ostream& out) { - out << "client_file_caps(" << inode.ino + out << "client_file_caps(" << get_opname(op) + << " " << inode.ino << " seq " << seq << " caps " << cap_string(caps) << " wanted" << cap_string(wanted) @@ -77,20 +89,20 @@ class MClientFileCaps : public Message { void decode_payload() { int off = 0; + ::_decode(op, payload, off); ::_decode(seq, payload, off); ::_decode(inode, payload, off); ::_decode(caps, payload, off); ::_decode(wanted, payload, off); ::_decode(mds, payload, off); - ::_decode(special, payload, off); } void encode_payload() { + ::_encode(op, payload); ::_encode(seq, payload); ::_encode(inode, payload); ::_encode(caps, payload); ::_encode(wanted, payload); ::_encode(mds, payload); - ::_encode(special, payload); } }; -- 2.39.5