From: Yan, Zheng Date: Fri, 21 Feb 2014 04:51:37 +0000 (+0800) Subject: client: add imported caps when handling cap export message X-Git-Tag: v0.79~179^2~5 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=4bdecf1cd8e73fc56262ed920def8ffd50aea922;p=ceph.git client: add imported caps when handling cap export message Version 3 cap export message includes information about the imported caps. It allows us to add the imported caps if the corresponding cap import message still hasn't been received. This allow us to handle situation that the importer MDS crashes and the cap import message is missing. Signed-off-by: Yan, Zheng --- diff --git a/src/client/Client.cc b/src/client/Client.cc index 9ff6d016d85c..b243ca1e7164 100644 --- a/src/client/Client.cc +++ b/src/client/Client.cc @@ -2012,18 +2012,6 @@ void Client::send_reconnect(MetaSession *session) did_snaprealm.insert(in->snaprealm->ino); } } - if (in->exporting_mds == mds) { - ldout(cct, 10) << " clearing exporting_caps on " << p->first << dendl; - in->exporting_mds = -1; - in->exporting_issued = 0; - in->exporting_mseq = 0; - if (!in->is_any_caps()) { - ldout(cct, 10) << " removing last cap, closing snaprealm" << dendl; - in->snaprealm_item.remove_myself(); - put_snap_realm(in->snaprealm); - in->snaprealm = 0; - } - } } // reset my cap seq number @@ -2898,6 +2886,24 @@ void Client::add_update_cap(Inode *in, MetaSession *mds_session, uint64_t cap_id int mds = mds_session->mds_num; if (in->caps.count(mds)) { cap = in->caps[mds]; + + /* + * auth mds of the inode changed. we received the cap export + * message, but still haven't received the cap import message. + * handle_cap_export() updated the new auth MDS' cap. + * + * "ceph_seq_cmp(seq, cap->seq) <= 0" means we are processing + * a message that was send before the cap import message. So + * don't remove caps. + */ + if (ceph_seq_cmp(seq, cap->seq) <= 0) { + assert(cap == in->auth_cap); + assert(cap->cap_id == cap_id); + seq = cap->seq; + mseq = cap->mseq; + issued |= cap->issued; + flags |= CEPH_CAP_FLAG_AUTH; + } } else { mds_session->num_caps++; if (!in->is_any_caps()) { @@ -2906,12 +2912,6 @@ void Client::add_update_cap(Inode *in, MetaSession *mds_session, uint64_t cap_id in->snaprealm->inodes_with_caps.push_back(&in->snaprealm_item); ldout(cct, 15) << "add_update_cap first one, opened snaprealm " << in->snaprealm << dendl; } - if (in->exporting_mds == mds) { - ldout(cct, 10) << "add_update_cap clearing exporting_caps on " << mds << dendl; - in->exporting_mds = -1; - in->exporting_issued = 0; - in->exporting_mseq = 0; - } in->caps[mds] = cap = new Cap; mds_session->caps.push_back(&cap->cap_item); cap->session = mds_session; @@ -3505,12 +3505,10 @@ void Client::handle_cap_import(MetaSession *session, Inode *in, MClientCaps *m) m->get_caps(), m->get_seq(), m->get_mseq(), m->get_realm(), CEPH_CAP_FLAG_AUTH); - if (m->peer.cap_id) { + if (m->peer.cap_id && in->caps.count(m->peer.mds)) { Cap *cap = in->caps[m->peer.mds]; - if (cap && cap->cap_id == m->peer.cap_id) { - in->exporting_issued = cap->issued; + if (cap && cap->cap_id == m->peer.cap_id) remove_cap(cap, (m->peer.flags & CEPH_CAP_FLAG_RELEASE)); - } } if (in->auth_cap && in->auth_cap->session->mds_num == mds) { @@ -3525,44 +3523,39 @@ void Client::handle_cap_import(MetaSession *session, Inode *in, MClientCaps *m) void Client::handle_cap_export(MetaSession *session, Inode *in, MClientCaps *m) { int mds = session->mds_num; - Cap *cap = NULL; - // note? - bool found_higher_mseq = false; - for (map::iterator p = in->caps.begin(); - p != in->caps.end(); - ++p) { - if (p->first == mds) - cap = p->second; - if (ceph_seq_cmp(p->second->mseq, m->get_mseq()) > 0) { - found_higher_mseq = true; - ldout(cct, 5) << "handle_cap_export ino " << m->get_ino() << " mseq " << m->get_mseq() - << " EXPORT from mds." << mds - << ", but mds." << p->first << " has higher mseq " << p->second->mseq << dendl; - } - } - - if (cap) { - if (!found_higher_mseq) { - ldout(cct, 5) << "handle_cap_export ino " << m->get_ino() << " mseq " << m->get_mseq() - << " EXPORT from mds." << mds - << ", setting exporting_issued " << ccap_string(cap->issued) << dendl; - in->exporting_issued = cap->issued; - in->exporting_mseq = m->get_mseq(); - in->exporting_mds = mds; - } else - ldout(cct, 5) << "handle_cap_export ino " << m->get_ino() << " mseq " << m->get_mseq() - << " EXPORT from mds." << mds - << ", just removing old cap" << dendl; + ldout(cct, 5) << "handle_cap_export ino " << m->get_ino() << " mseq " << m->get_mseq() + << " EXPORT from mds." << mds << dendl; + + if (in->caps.count(mds)) { + Cap *cap = in->caps[mds]; + + if (m->peer.cap_id) { + MetaSession *tsession = _get_or_open_mds_session(m->peer.mds); + if (in->caps.count(m->peer.mds)) { + Cap *tcap = in->caps[m->peer.mds]; + if (tcap->cap_id != m->peer.cap_id || + ceph_seq_cmp(tcap->seq, m->peer.seq) < 0) { + tcap->cap_id = m->peer.cap_id; + tcap->seq = m->peer.seq - 1; + tcap->issue_seq = tcap->seq; + tcap->mseq = m->peer.mseq; + tcap->issued |= cap->issued; + tcap->implemented |= cap->issued; + if (cap == in->auth_cap) + in->auth_cap = tcap; + if (in->auth_cap == tcap && in->flushing_cap_item.is_on_list()) + tsession->flushing_caps.push_back(&in->flushing_cap_item); + } + } else { + add_update_cap(in, tsession, m->peer.cap_id, cap->issued, + m->peer.seq - 1, m->peer.mseq, (uint64_t)-1, + cap == in->auth_cap ? CEPH_CAP_FLAG_AUTH : 0); + } + } remove_cap(cap, false); } - // else we already released it - - // open export targets, so we'll get the matching IMPORT, even if we - // have seen a newer import (or have released the cap entirely), as there - // may be an intervening revocation that will otherwise get blocked up. - connect_mds_targets(mds); m->put(); } @@ -3748,7 +3741,7 @@ void Client::handle_cap_grant(MetaSession *session, Inode *in, Cap *cap, MClient } bool check = false; - if (m->get_op() == CEPH_CAP_OP_IMPORT && m->get_wanted() != wanted) { + if (m->get_op() == CEPH_CAP_OP_IMPORT && m->get_wanted() != wanted) check = true; check_cap_issue(in, cap, issued); diff --git a/src/client/Inode.cc b/src/client/Inode.cc index 4e53e14bd244..0d9d9d9a45ba 100644 --- a/src/client/Inode.cc +++ b/src/client/Inode.cc @@ -137,7 +137,7 @@ bool Inode::put_cap_ref(int cap) bool Inode::is_any_caps() { - return caps.size() || exporting_mds >= 0; + return caps.size(); } bool Inode::cap_is_valid(Cap* cap) @@ -155,7 +155,7 @@ bool Inode::cap_is_valid(Cap* cap) int Inode::caps_issued(int *implemented) { - int c = exporting_issued | snap_caps; + int c = snap_caps; int i = 0; for (map::iterator it = caps.begin(); it != caps.end(); @@ -183,7 +183,7 @@ void Inode::try_touch_cap(int mds) bool Inode::caps_issued_mask(unsigned mask) { - int c = exporting_issued | snap_caps; + int c = snap_caps; if ((c & mask) == mask) return true; // prefer auth cap @@ -390,10 +390,6 @@ void Inode::dump(Formatter *f) const f->dump_int("snap_caps", snap_caps); f->dump_int("snap_cap_refs", snap_cap_refs); } - if (exporting_issued || exporting_mseq) { - f->dump_stream("exporting_issued") << ccap_string(exporting_issued); - f->dump_int("exporting_mseq", exporting_mds); - } f->dump_stream("hold_caps_until") << hold_caps_until; f->dump_unsigned("last_flush_tid", last_flush_tid); diff --git a/src/client/Inode.h b/src/client/Inode.h index b6c7a38aa0be..692e76ab9023 100644 --- a/src/client/Inode.h +++ b/src/client/Inode.h @@ -140,9 +140,6 @@ class Inode { __u16 flushing_cap_tid[CEPH_CAP_BITS]; int shared_gen, cache_gen; int snap_caps, snap_cap_refs; - unsigned exporting_issued; - int exporting_mds; - ceph_seq_t exporting_mseq; utime_t hold_caps_until; xlist::item cap_item, flushing_cap_item; tid_t last_flush_tid; @@ -216,7 +213,6 @@ class Inode { dir_hashed(false), dir_replicated(false), auth_cap(NULL), dirty_caps(0), flushing_caps(0), flushing_cap_seq(0), shared_gen(0), cache_gen(0), snap_caps(0), snap_cap_refs(0), - exporting_issued(0), exporting_mds(-1), exporting_mseq(0), cap_item(this), flushing_cap_item(this), last_flush_tid(0), snaprealm(0), snaprealm_item(this), snapdir_parent(0), oset((void *)this, newlayout->fl_pg_pool, ino),