From: Yan, Zheng Date: Tue, 5 Feb 2019 09:06:48 +0000 (+0800) Subject: mds: optimize revoking stale caps X-Git-Tag: v12.2.12~111^2~4 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=73e814616b7680c4812d6055b47f05c653d3d8cc;p=ceph.git mds: optimize revoking stale caps For caps that are not being revoked and don't have writeable range and don't want exclusive caps or file read/write. there is no need to call Locker::revoke_stale_caps(Capability*). Because these caps don't need recover and don't affect eval_gather()/try_eval(). Signed-off-by: "Yan, Zheng" (cherry picked from commit cb62030f0468fc04735c1b4cff73da779cb11ad8) Conflicts: src/mds/CInode.cc src/mds/Capability.h src/mds/Locker.cc src/mds/Migrator.cc --- diff --git a/src/mds/CInode.cc b/src/mds/CInode.cc index c834fc427e4..b91c47881f5 100644 --- a/src/mds/CInode.cc +++ b/src/mds/CInode.cc @@ -218,7 +218,7 @@ ostream& operator<<(ostream& out, const CInode& in) if (it->second->issued() != it->second->pending()) out << "/" << ccap_string(it->second->issued()); out << "/" << ccap_string(it->second->wanted()) - << "@" << it->second->get_last_sent(); + << "@" << it->second->get_last_seq(); } out << "}"; if (in.get_loner() >= 0 || in.get_wanted_loner() >= 0) { @@ -4260,7 +4260,7 @@ void CInode::dump(Formatter *f) const f->dump_string("pending", ccap_string(it->second->pending())); f->dump_string("issued", ccap_string(it->second->issued())); f->dump_string("wanted", ccap_string(it->second->wanted())); - f->dump_int("last_sent", it->second->get_last_sent()); + f->dump_int("last_sent", it->second->get_last_seq()); f->close_section(); } f->close_section(); diff --git a/src/mds/Capability.cc b/src/mds/Capability.cc index 98be60f079f..8bfa90b1268 100644 --- a/src/mds/Capability.cc +++ b/src/mds/Capability.cc @@ -154,6 +154,7 @@ Capability::Capability(CInode *i, Session *s, uint64_t id) : { if (session) { session->touch_cap_bottom(this); + cap_gen = session->get_cap_gen(); } } @@ -167,6 +168,29 @@ bool Capability::is_stale() const return session ? session->is_stale() : false; } +bool Capability::is_valid() const +{ + return !session || session->get_cap_gen() == cap_gen; +} + +void Capability::revalidate() +{ + if (is_valid()) + return; + + if (_pending & ~CEPH_CAP_PIN) + inc_last_seq(); + + bool was_revoking = _issued & ~_pending; + _pending = _issued = CEPH_CAP_PIN; + _revokes.clear(); + + cap_gen = session->get_cap_gen(); + + if (was_revoking) + maybe_clear_notable(); +} + void Capability::mark_notable() { state |= STATE_NOTABLE; diff --git a/src/mds/Capability.h b/src/mds/Capability.h index 4ff5b811fc8..db3e47d3876 100644 --- a/src/mds/Capability.h +++ b/src/mds/Capability.h @@ -120,11 +120,16 @@ public: const Capability& operator=(const Capability& other); // no copying - int pending() { return _pending; } - int issued() { return _issued; } - bool is_null() { return !_pending && _revokes.empty(); } + int pending() const { + return is_valid() ? _pending : (_pending & CEPH_CAP_PIN); + } + int issued() const { + return is_valid() ? _issued : (_issued & CEPH_CAP_PIN); + } ceph_seq_t issue(unsigned c) { + revalidate(); + if (_pending & ~c) { // revoking (and maybe adding) bits. note caps prior to this revocation _revokes.emplace_back(_pending, last_sent, last_issue); @@ -149,6 +154,8 @@ public: return last_sent; } ceph_seq_t issue_norevoke(unsigned c) { + revalidate(); + _pending |= c; _issued |= c; //check_rdcaps_list(); @@ -204,9 +211,8 @@ public: ceph_seq_t get_mseq() { return mseq; } void inc_mseq() { mseq++; } - ceph_seq_t get_last_sent() { return last_sent; } - utime_t get_last_issue_stamp() { return last_issue_stamp; } - utime_t get_last_revoke_stamp() { return last_revoke_stamp; } + utime_t get_last_issue_stamp() const { return last_issue_stamp; } + utime_t get_last_revoke_stamp() const { return last_revoke_stamp; } void set_last_issue() { last_issue = last_sent; } void set_last_issue_stamp(utime_t t) { last_issue_stamp = t; } @@ -261,8 +267,12 @@ public: void set_wanted(int w); void inc_last_seq() { last_sent++; } - ceph_seq_t get_last_seq() { return last_sent; } - ceph_seq_t get_last_issue() { return last_issue; } + ceph_seq_t get_last_seq() const { + if (!is_valid() && (_pending & ~CEPH_CAP_PIN)) + return last_sent + 1; + return last_sent; + } + ceph_seq_t get_last_issue() const { return last_issue; } void reset_seq() { last_sent = 0; @@ -270,8 +280,8 @@ public: } // -- exports -- - Export make_export() { - return Export(cap_id, _wanted, issued(), pending(), client_follows, last_sent, mseq+1, last_issue_stamp); + Export make_export() const { + return Export(cap_id, wanted(), issued(), pending(), client_follows, get_last_seq(), mseq+1, last_issue_stamp); } void merge(const Export& other, bool auth_cap) { if (!is_stale()) { @@ -337,6 +347,7 @@ private: Session *session; uint64_t cap_id; + uint32_t cap_gen; __u32 _wanted; // what the client wants (ideally) @@ -364,6 +375,9 @@ private: } } + bool is_valid() const; + void revalidate(); + void mark_notable(); void maybe_clear_notable(); }; diff --git a/src/mds/Locker.cc b/src/mds/Locker.cc index e89a1aae5ae..de33e4a8c56 100644 --- a/src/mds/Locker.cc +++ b/src/mds/Locker.cc @@ -2123,19 +2123,27 @@ void Locker::issue_truncate(CInode *in) check_inode_max_size(in); } - -void Locker::revoke_stale_caps(Capability *cap) +void Locker::revoke_stale_caps(Session *session) { - CInode *in = cap->get_inode(); - if (in->state_test(CInode::STATE_EXPORTINGCAPS)) { - // if export succeeds, the cap will be removed. if export fails, we need to - // revoke the cap if it's still stale. - in->state_set(CInode::STATE_EVALSTALECAPS); - return; - } + dout(10) << "revoke_stale_caps for " << session->info.inst.name << dendl; + + std::vector to_eval; + + for (auto p = session->caps.begin(); !p.end(); ) { + Capability *cap = *p; + ++p; + if (!cap->is_notable()) { + // the rest ones are not being revoked and don't have writeable range + // and don't want exclusive caps or want file read/write. They don't + // need recover, they don't affect eval_gather()/try_eval() + break; + } - int issued = cap->issued(); - if (issued & ~CEPH_CAP_PIN) { + int issued = cap->issued(); + if (!(issued & ~CEPH_CAP_PIN)) + continue; + + CInode *in = cap->get_inode(); dout(10) << " revoking " << ccap_string(issued) << " on " << *in << dendl; cap->revoke(); @@ -2143,26 +2151,31 @@ void Locker::revoke_stale_caps(Capability *cap) in->inode.client_ranges.count(cap->get_client())) in->state_set(CInode::STATE_NEEDSRECOVER); - if (!in->filelock.is_stable()) eval_gather(&in->filelock); - if (!in->linklock.is_stable()) eval_gather(&in->linklock); - if (!in->authlock.is_stable()) eval_gather(&in->authlock); - if (!in->xattrlock.is_stable()) eval_gather(&in->xattrlock); - - if (in->is_auth()) { - try_eval(in, CEPH_CAP_LOCKS); - } else { - request_inode_file_caps(in); - } + // eval lock/inode may finish contexts, which may modify other cap's position + // in the session->caps. + to_eval.push_back(in); } -} -void Locker::revoke_stale_caps(Session *session) -{ - dout(10) << "revoke_stale_caps for " << session->info.inst.name << dendl; + // invalidate the rest + session->inc_cap_gen(); - for (xlist::iterator p = session->caps.begin(); !p.end(); ++p) { - Capability *cap = *p; - revoke_stale_caps(cap); + for (auto in : to_eval) { + if (in->state_test(CInode::STATE_EXPORTINGCAPS)) + continue; + + if (!in->filelock.is_stable()) + eval_gather(&in->filelock); + if (!in->linklock.is_stable()) + eval_gather(&in->linklock); + if (!in->authlock.is_stable()) + eval_gather(&in->authlock); + if (!in->xattrlock.is_stable()) + eval_gather(&in->xattrlock); + + if (in->is_auth()) + try_eval(in, CEPH_CAP_LOCKS); + else + request_inode_file_caps(in); } } @@ -2323,7 +2336,7 @@ void Locker::calc_new_client_ranges(CInode *in, uint64_t size, bool update, for (map::iterator p = in->client_caps.begin(); p != in->client_caps.end(); ++p) { - if ((p->second->issued() | p->second->wanted()) & (CEPH_CAP_FILE_WR|CEPH_CAP_FILE_BUFFER)) { + if ((p->second->issued() | p->second->wanted()) & (CEPH_CAP_ANY_FILE_WR)) { client_writeable_range_t& nr = (*new_ranges)[p->first]; nr.range.first = 0; if (latest->client_ranges.count(p->first)) { @@ -3028,7 +3041,7 @@ public: void Locker::kick_issue_caps(CInode *in, client_t client, ceph_seq_t seq) { Capability *cap = in->get_client_cap(client); - if (!cap || cap->get_last_sent() != seq) + if (!cap || cap->get_last_seq() != seq) return; if (in->is_frozen()) { dout(10) << "kick_issue_caps waiting for unfreeze on " << *in << dendl; @@ -3509,18 +3522,20 @@ void Locker::_do_cap_release(client_t client, inodeno_t ino, uint64_t cap_id, eval_cap_gather(in); return; } - remove_client_cap(in, client); + remove_client_cap(in, cap); } -/* This function DOES put the passed message before returning */ - -void Locker::remove_client_cap(CInode *in, client_t client) +void Locker::remove_client_cap(CInode *in, Capability *cap) { + client_t client = cap->get_client(); // clean out any pending snapflush state if (!in->client_need_snapflush.empty()) _do_null_snapflush(in, client); + bool notable = cap->is_notable(); in->remove_client_cap(client); + if (!notable) + return; if (in->is_auth()) { // make sure we clear out the client byte range diff --git a/src/mds/Locker.h b/src/mds/Locker.h index ddbd321f2a7..694c39250d0 100644 --- a/src/mds/Locker.h +++ b/src/mds/Locker.h @@ -184,7 +184,7 @@ public: void kick_cap_releases(MDRequestRef& mdr); void kick_issue_caps(CInode *in, client_t client, ceph_seq_t seq); - void remove_client_cap(CInode *in, client_t client); + void remove_client_cap(CInode *in, Capability *cap); void get_late_revoking_clients(std::list *result, double timeout) const; @@ -244,7 +244,6 @@ public: void issue_caps_set(set& inset); void issue_truncate(CInode *in); void revoke_stale_caps(Session *session); - void revoke_stale_caps(Capability *cap); void resume_stale_caps(Session *session); void remove_stale_leases(Session *session); diff --git a/src/mds/Migrator.cc b/src/mds/Migrator.cc index a1cf8d31a4a..85f6b3b726e 100644 --- a/src/mds/Migrator.cc +++ b/src/mds/Migrator.cc @@ -2128,10 +2128,9 @@ void Migrator::export_reverse(CDir *dir, export_state_t& stat) bool need_issue = false; for (auto& p : in->get_client_caps()) { Capability *cap = p.second; - if (cap->is_stale()) { - mds->locker->revoke_stale_caps(cap); - } else { + if (!cap->is_stale()) { need_issue = true; + break; } } if (need_issue && diff --git a/src/mds/Server.cc b/src/mds/Server.cc index 3a85815251d..38273d6bc93 100644 --- a/src/mds/Server.cc +++ b/src/mds/Server.cc @@ -546,7 +546,7 @@ void Server::_session_logged(Session *session, uint64_t state_seq, bool open, ve Capability *cap = session->caps.front(); CInode *in = cap->get_inode(); dout(20) << " killing capability " << ccap_string(cap->issued()) << " on " << *in << dendl; - mds->locker->remove_client_cap(in, session->info.inst.name.num()); + mds->locker->remove_client_cap(in, cap); } while (!session->leases.empty()) { ClientLease *r = session->leases.front(); diff --git a/src/mds/SessionMap.h b/src/mds/SessionMap.h index 404c456d770..47625baf3a2 100644 --- a/src/mds/SessionMap.h +++ b/src/mds/SessionMap.h @@ -239,6 +239,7 @@ public: // -- caps -- private: + uint32_t cap_gen; version_t cap_push_seq; // cap push seq # map > waitfor_flush; // flush session messages @@ -248,7 +249,9 @@ public: time last_cap_renew = time::min(); time last_seen = time::min(); -public: + void inc_cap_gen() { ++cap_gen; } + uint32_t get_cap_gen() const { return cap_gen; } + version_t inc_push_seq() { return ++cap_push_seq; } version_t get_push_seq() const { return cap_push_seq; } @@ -367,7 +370,7 @@ public: recall_release_count(0), auth_caps(g_ceph_context), connection(NULL), item_session_list(this), requests(0), // member_offset passed to front() manually - cap_push_seq(0), + cap_gen(0), cap_push_seq(0), lease_seq(0), completed_requests_dirty(false), num_trim_flushes_warnings(0),