From: Yan, Zheng Date: Mon, 22 Mar 2021 06:37:49 +0000 (+0800) Subject: mds: reduce Capability size by using elist::item X-Git-Tag: v20.0.0~1306^2~3 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=b6c79ab6480dc769873f66b27e9428bcf42fad74;p=ceph.git mds: reduce Capability size by using elist::item Signed-off-by: "Yan, Zheng" --- diff --git a/src/include/elist.h b/src/include/elist.h index 38be35dbff33..edfb79554944 100644 --- a/src/include/elist.h +++ b/src/include/elist.h @@ -45,6 +45,9 @@ public: bool empty() const { return _prev == this; } bool is_on_list() const { return !empty(); } + bool is_singular() const { + return is_on_list() && _prev == _next; + } bool remove_myself() { if (_next == this) { diff --git a/src/mds/Capability.cc b/src/mds/Capability.cc index de2a16e1ab19..9a3d093f9eb2 100644 --- a/src/mds/Capability.cc +++ b/src/mds/Capability.cc @@ -151,8 +151,7 @@ void Capability::revoke_info::generate_test_instances(std::list::item item_session_caps; - xlist::item item_snaprealm_caps; - xlist::item item_revoking_caps; - xlist::item item_client_revoking_caps; + elist::item item_snaprealm_caps; + elist::item item_revoking_caps; + elist::item item_client_revoking_caps; elist lock_caches; int get_lock_cache_allowed() const { return lock_cache_allowed; } diff --git a/src/mds/Locker.cc b/src/mds/Locker.cc index b84f7b59ee53..e3925e20b475 100644 --- a/src/mds/Locker.cc +++ b/src/mds/Locker.cc @@ -72,7 +72,9 @@ public: }; Locker::Locker(MDSRank *m, MDCache *c) : - need_snapflush_inodes(member_offset(CInode, item_to_flush)), mds(m), mdcache(c) {} + revoking_caps(member_offset(Capability, item_revoking_caps)), + need_snapflush_inodes(member_offset(CInode, item_to_flush)), + mds(m), mdcache(c) {} void Locker::dispatch(const cref_t &m) @@ -2627,9 +2629,11 @@ int Locker::issue_caps(CInode *in, Capability *only_cap) int op = (before & ~after) ? CEPH_CAP_OP_REVOKE : CEPH_CAP_OP_GRANT; if (op == CEPH_CAP_OP_REVOKE) { - if (mds->logger) mds->logger->inc(l_mdss_ceph_cap_op_revoke); + if (mds->logger) mds->logger->inc(l_mdss_ceph_cap_op_revoke); revoking_caps.push_back(&cap->item_revoking_caps); - revoking_caps_by_client[cap->get_client()].push_back(&cap->item_client_revoking_caps); + auto em = revoking_caps_by_client.emplace(cap->get_client(), + member_offset(Capability, item_client_revoking_caps)); + em.first->second.push_back(&cap->item_client_revoking_caps); cap->set_last_revoke_stamp(ceph_clock_now()); cap->reset_num_revoke_warnings(); } else { @@ -4309,42 +4313,33 @@ void Locker::remove_client_cap(CInode *in, Capability *cap, bool kill) try_eval(in, CEPH_CAP_LOCKS); } - -/** - * Return true if any currently revoking caps exceed the - * session_timeout threshold. - */ -bool Locker::any_late_revoking_caps(xlist const &revoking, - double timeout) const +std::set Locker::get_late_revoking_clients(double timeout) { - xlist::const_iterator p = revoking.begin(); - if (p.end()) { + auto any_late_revoking = [timeout](elist &revoking) { + auto p = revoking.begin(); + if (p.end()) // No revoking caps at the moment return false; - } else { - utime_t now = ceph_clock_now(); - utime_t age = now - (*p)->get_last_revoke_stamp(); - if (age <= timeout) { - return false; - } else { - return true; - } - } -} -std::set Locker::get_late_revoking_clients(double timeout) const -{ - std::set result; + utime_t now = ceph_clock_now(); + return now - (*p)->get_last_revoke_stamp() > timeout; + }; - if (any_late_revoking_caps(revoking_caps, timeout)) { + std::set result; + if (!any_late_revoking(revoking_caps)) { + // Fast path: no misbehaving clients, execute in O(1) + } else { // Slow path: execute in O(N_clients) - for (auto &p : revoking_caps_by_client) { - if (any_late_revoking_caps(p.second, timeout)) { - result.insert(p.first); + for (auto it = revoking_caps_by_client.begin(); + it != revoking_caps_by_client.end(); ) { + if (it->second.empty()) { + revoking_caps_by_client.erase(it++); + continue; } + if (any_late_revoking(it->second)) + result.insert(it->first); + ++it; } - } else { - // Fast path: no misbehaving clients, execute in O(1) } return result; } @@ -4376,11 +4371,10 @@ void Locker::caps_tick() } } - dout(20) << __func__ << " " << revoking_caps.size() << " revoking caps" << dendl; now = ceph_clock_now(); int n = 0; - for (xlist::iterator p = revoking_caps.begin(); !p.end(); ++p) { + for (auto p = revoking_caps.begin(); !p.end(); ++p) { Capability *cap = *p; utime_t age = now - cap->get_last_revoke_stamp(); diff --git a/src/mds/Locker.h b/src/mds/Locker.h index aa037ac6abda..eab345984c8b 100644 --- a/src/mds/Locker.h +++ b/src/mds/Locker.h @@ -151,7 +151,7 @@ public: void remove_client_cap(CInode *in, Capability *cap, bool kill=false); - std::set get_late_revoking_clients(double timeout) const; + std::set get_late_revoking_clients(double timeout); void snapflush_nudge(CInode *in); void mark_need_snapflush_inode(CInode *in); @@ -249,9 +249,9 @@ protected: xlist updated_scatterlocks; // Maintain a global list to quickly find if any caps are late revoking - xlist revoking_caps; + elist revoking_caps; // Maintain a per-client list to find clients responsible for late ones quickly - std::map > revoking_caps_by_client; + std::map > revoking_caps_by_client; elist need_snapflush_inodes; @@ -267,7 +267,6 @@ private: void handle_quiesce_failure(const MDRequestRef& mdr, std::string_view& marker); - bool any_late_revoking_caps(xlist const &revoking, double timeout) const; uint64_t calc_new_max_size(const CInode::inode_const_ptr& pi, uint64_t size); __u32 get_xattr_total_length(CInode::mempool_xattr_map &xattr); void decode_new_xattrs(CInode::mempool_inode *inode, diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc index 2ba0d638af0a..1c584e3bf3c0 100644 --- a/src/mds/MDCache.cc +++ b/src/mds/MDCache.cc @@ -5680,7 +5680,7 @@ void MDCache::prepare_realm_merge(SnapRealm *realm, SnapRealm *parent_realm, split_realms.push_back((*p)->inode->ino()); for (const auto& p : realm->client_caps) { - ceph_assert(!p.second->empty()); + ceph_assert(!p.second.empty()); auto em = splits.emplace(std::piecewise_construct, std::forward_as_tuple(p.first), std::forward_as_tuple()); if (em.second) { auto update = make_message(CEPH_SNAP_OP_SPLIT); @@ -10068,7 +10068,7 @@ void MDCache::do_realm_invalidate_and_update_notify(CInode *in, int snapop, bool for (const auto& p : realm->client_caps) { const auto& client = p.first; const auto& caps = p.second; - ceph_assert(!caps->empty()); + ceph_assert(!caps.empty()); auto em = updates.emplace(std::piecewise_construct, std::forward_as_tuple(client), std::forward_as_tuple()); if (em.second) { diff --git a/src/mds/SnapRealm.h b/src/mds/SnapRealm.h index 700c1d81e3b5..b1c38d03fcb3 100644 --- a/src/mds/SnapRealm.h +++ b/src/mds/SnapRealm.h @@ -103,18 +103,18 @@ public: void merge_to(SnapRealm *newparent); void add_cap(client_t client, Capability *cap) { - auto client_caps_entry = client_caps.find(client); - if (client_caps_entry == client_caps.end()) - client_caps_entry = client_caps.emplace(client, - new xlist).first; - client_caps_entry->second->push_back(&cap->item_snaprealm_caps); + auto em = client_caps.emplace(cap->get_client(), + member_offset(Capability, item_snaprealm_caps)); + em.first->second.push_back(&cap->item_snaprealm_caps); } void remove_cap(client_t client, Capability *cap) { + bool last_cap = cap->item_snaprealm_caps.is_singular(); cap->item_snaprealm_caps.remove_myself(); - auto found = client_caps.find(client); - if (found != client_caps.end() && found->second->empty()) { - delete found->second; - client_caps.erase(found); + if (last_cap) { + auto it = client_caps.find(client); + ceph_assert(it != client_caps.end()); + ceph_assert(it->second.empty()); + client_caps.erase(it); } } @@ -129,7 +129,7 @@ public: std::set open_children; // active children that are currently open elist inodes_with_caps; // for efficient realm splits - std::map* > client_caps; // to identify clients who need snap notifications + std::map > client_caps; // to identify clients who need snap notifications protected: void check_cache() const;