mds: reduce Capability size by using elist::item

Signed-off-by: "Yan, Zheng" <yanzheng03@kuaishou.com>

diff --git a/src/include/elist.h b/src/include/elist.h
index 38be35dbff33a0b3a994cf108341d9accfe288a1..edfb79554944b383a64f8e827ce0b9135fa7ce71 100644 (file)
--- a/src/include/elist.h
+++ b/src/include/elist.h
@@ -45,6 +45,9 @@ public:
     
     bool empty() const { return _prev == this; }
     bool is_on_list() const { return !empty(); }
+    bool is_singular() const {
+      return is_on_list() && _prev == _next;
+    }
 
     bool remove_myself() {
       if (_next == this) {

diff --git a/src/mds/Capability.cc b/src/mds/Capability.cc
index de2a16e1ab19870bdcb7dc9d4a94dcfae6cb4168..9a3d093f9eb20205f4877cff2ac3c221e8b76f6b 100644 (file)
--- a/src/mds/Capability.cc
+++ b/src/mds/Capability.cc
@@ -151,8 +151,7 @@ void Capability::revoke_info::generate_test_instances(std::list<Capability::revo
  * Capability
  */
 Capability::Capability(CInode *i, Session *s, uint64_t id) :
-  item_session_caps(this), item_snaprealm_caps(this),
-  item_revoking_caps(this), item_client_revoking_caps(this),
+  item_session_caps(this),
   lock_caches(member_offset(MDLockCache, item_cap_lock_cache)),
   inode(i), session(s), cap_id(id)
 {

diff --git a/src/mds/Capability.h b/src/mds/Capability.h
index ebc626a22949ae07d31cfb101f789d26f1ee2274..9680895a5c80ef41ffbcd13c4654db278b55f420 100644 (file)
--- a/src/mds/Capability.h
+++ b/src/mds/Capability.h
@@ -336,9 +336,9 @@ public:
   int64_t last_rsize = 0;
 
   xlist<Capability*>::item item_session_caps;
-  xlist<Capability*>::item item_snaprealm_caps;
-  xlist<Capability*>::item item_revoking_caps;
-  xlist<Capability*>::item item_client_revoking_caps;
+  elist<Capability*>::item item_snaprealm_caps;
+  elist<Capability*>::item item_revoking_caps;
+  elist<Capability*>::item item_client_revoking_caps;
 
   elist<MDLockCache*> lock_caches;
   int get_lock_cache_allowed() const { return lock_cache_allowed; }

diff --git a/src/mds/Locker.cc b/src/mds/Locker.cc
index b84f7b59ee538dc9aade0fd6df64706ba96eab27..e3925e20b4751f7815f31e534cdf77bb2052f37c 100644 (file)
--- a/src/mds/Locker.cc
+++ b/src/mds/Locker.cc
@@ -72,7 +72,9 @@ public:
 };
 
 Locker::Locker(MDSRank *m, MDCache *c) :
-  need_snapflush_inodes(member_offset(CInode, item_to_flush)), mds(m), mdcache(c) {}
+  revoking_caps(member_offset(Capability, item_revoking_caps)),
+  need_snapflush_inodes(member_offset(CInode, item_to_flush)),
+  mds(m), mdcache(c) {}
 
 
 void Locker::dispatch(const cref_t<Message> &m)
@@ -2627,9 +2629,11 @@ int Locker::issue_caps(CInode *in, Capability *only_cap)
 
       int op = (before & ~after) ? CEPH_CAP_OP_REVOKE : CEPH_CAP_OP_GRANT;
       if (op == CEPH_CAP_OP_REVOKE) {
-        if (mds->logger) mds->logger->inc(l_mdss_ceph_cap_op_revoke);
+       if (mds->logger) mds->logger->inc(l_mdss_ceph_cap_op_revoke);
        revoking_caps.push_back(&cap->item_revoking_caps);
-       revoking_caps_by_client[cap->get_client()].push_back(&cap->item_client_revoking_caps);
+       auto em = revoking_caps_by_client.emplace(cap->get_client(),
+                                                 member_offset(Capability, item_client_revoking_caps));
+       em.first->second.push_back(&cap->item_client_revoking_caps);
        cap->set_last_revoke_stamp(ceph_clock_now());
        cap->reset_num_revoke_warnings();
       } else {
@@ -4309,42 +4313,33 @@ void Locker::remove_client_cap(CInode *in, Capability *cap, bool kill)
   try_eval(in, CEPH_CAP_LOCKS);
 }
 
-
-/**
- * Return true if any currently revoking caps exceed the
- * session_timeout threshold.
- */
-bool Locker::any_late_revoking_caps(xlist<Capability*> const &revoking,
-                                    double timeout) const
+std::set<client_t> Locker::get_late_revoking_clients(double timeout)
 {
-    xlist<Capability*>::const_iterator p = revoking.begin();
-    if (p.end()) {
+  auto any_late_revoking = [timeout](elist<Capability*> &revoking) {
+    auto p = revoking.begin();
+    if (p.end())
       // No revoking caps at the moment
       return false;
-    } else {
-      utime_t now = ceph_clock_now();
-      utime_t age = now - (*p)->get_last_revoke_stamp();
-      if (age <= timeout) {
-          return false;
-      } else {
-          return true;
-      }
-    }
-}
 
-std::set<client_t> Locker::get_late_revoking_clients(double timeout) const
-{
-  std::set<client_t> result;
+    utime_t now = ceph_clock_now();
+    return now - (*p)->get_last_revoke_stamp() > timeout;
+  };
 
-  if (any_late_revoking_caps(revoking_caps, timeout)) {
+  std::set<client_t> result;
+  if (!any_late_revoking(revoking_caps)) {
+    // Fast path: no misbehaving clients, execute in O(1)
+  } else {
     // Slow path: execute in O(N_clients)
-    for (auto &p : revoking_caps_by_client) {
-      if (any_late_revoking_caps(p.second, timeout)) {
-        result.insert(p.first);
+    for (auto it = revoking_caps_by_client.begin();
+        it != revoking_caps_by_client.end(); ) {
+      if (it->second.empty()) {
+       revoking_caps_by_client.erase(it++);
+       continue;
       }
+      if (any_late_revoking(it->second))
+       result.insert(it->first);
+      ++it;
     }
-  } else {
-    // Fast path: no misbehaving clients, execute in O(1)
   }
   return result;
 }
@@ -4376,11 +4371,10 @@ void Locker::caps_tick()
     }
   }
 
-  dout(20) << __func__ << " " << revoking_caps.size() << " revoking caps" << dendl;
 
   now = ceph_clock_now();
   int n = 0;
-  for (xlist<Capability*>::iterator p = revoking_caps.begin(); !p.end(); ++p) {
+  for (auto p = revoking_caps.begin(); !p.end(); ++p) {
     Capability *cap = *p;
 
     utime_t age = now - cap->get_last_revoke_stamp();

diff --git a/src/mds/Locker.h b/src/mds/Locker.h
index aa037ac6abda728f729fdb8d8c0012a6882675b8..eab345984c8bd4c9df86a37eca607b2a61284aa7 100644 (file)
--- a/src/mds/Locker.h
+++ b/src/mds/Locker.h
@@ -151,7 +151,7 @@ public:
 
   void remove_client_cap(CInode *in, Capability *cap, bool kill=false);
 
-  std::set<client_t> get_late_revoking_clients(double timeout) const;
+  std::set<client_t> get_late_revoking_clients(double timeout);
 
   void snapflush_nudge(CInode *in);
   void mark_need_snapflush_inode(CInode *in);
@@ -249,9 +249,9 @@ protected:
   xlist<ScatterLock*> updated_scatterlocks;
 
   // Maintain a global list to quickly find if any caps are late revoking
-  xlist<Capability*> revoking_caps;
+  elist<Capability*> revoking_caps;
   // Maintain a per-client list to find clients responsible for late ones quickly
-  std::map<client_t, xlist<Capability*> > revoking_caps_by_client;
+  std::map<client_t, elist<Capability*> > revoking_caps_by_client;
 
   elist<CInode*> need_snapflush_inodes;
 
@@ -267,7 +267,6 @@ private:
 
   void handle_quiesce_failure(const MDRequestRef& mdr, std::string_view& marker);
 
-  bool any_late_revoking_caps(xlist<Capability*> const &revoking, double timeout) const;
   uint64_t calc_new_max_size(const CInode::inode_const_ptr& pi, uint64_t size);
   __u32 get_xattr_total_length(CInode::mempool_xattr_map &xattr);
   void decode_new_xattrs(CInode::mempool_inode *inode,

diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index 2ba0d638af0a11a868b95c01d394771ca116b9c2..1c584e3bf3c0f58d67f7792ad0713530749cd2aa 100644 (file)
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -5680,7 +5680,7 @@ void MDCache::prepare_realm_merge(SnapRealm *realm, SnapRealm *parent_realm,
     split_realms.push_back((*p)->inode->ino());
 
   for (const auto& p : realm->client_caps) {
-    ceph_assert(!p.second->empty());
+    ceph_assert(!p.second.empty());
     auto em = splits.emplace(std::piecewise_construct, std::forward_as_tuple(p.first), std::forward_as_tuple());
     if (em.second) {
       auto update = make_message<MClientSnap>(CEPH_SNAP_OP_SPLIT);
@@ -10068,7 +10068,7 @@ void MDCache::do_realm_invalidate_and_update_notify(CInode *in, int snapop, bool
       for (const auto& p : realm->client_caps) {
         const auto& client = p.first;
         const auto& caps = p.second;
-       ceph_assert(!caps->empty());
+       ceph_assert(!caps.empty());
 
         auto em = updates.emplace(std::piecewise_construct, std::forward_as_tuple(client), std::forward_as_tuple());
         if (em.second) {

diff --git a/src/mds/SnapRealm.h b/src/mds/SnapRealm.h
index 700c1d81e3b574db992037d8ad0047e23f440529..b1c38d03fcb356b3d3611676bed55629cba68391 100644 (file)
--- a/src/mds/SnapRealm.h
+++ b/src/mds/SnapRealm.h
@@ -103,18 +103,18 @@ public:
   void merge_to(SnapRealm *newparent);
 
   void add_cap(client_t client, Capability *cap) {
-    auto client_caps_entry = client_caps.find(client);
-    if (client_caps_entry == client_caps.end())
-      client_caps_entry = client_caps.emplace(client,
-                                             new xlist<Capability*>).first;
-    client_caps_entry->second->push_back(&cap->item_snaprealm_caps);
+    auto em = client_caps.emplace(cap->get_client(),
+                                 member_offset(Capability, item_snaprealm_caps));
+    em.first->second.push_back(&cap->item_snaprealm_caps);
   }
   void remove_cap(client_t client, Capability *cap) {
+    bool last_cap = cap->item_snaprealm_caps.is_singular();
     cap->item_snaprealm_caps.remove_myself();
-    auto found = client_caps.find(client);
-    if (found != client_caps.end() && found->second->empty()) {
-      delete found->second;
-      client_caps.erase(found);
+    if (last_cap) {
+      auto it = client_caps.find(client);
+      ceph_assert(it != client_caps.end());
+      ceph_assert(it->second.empty());
+      client_caps.erase(it);
     }
   }
 
@@ -129,7 +129,7 @@ public:
   std::set<SnapRealm*> open_children;    // active children that are currently open
 
   elist<CInode*> inodes_with_caps;             // for efficient realm splits
-  std::map<client_t, xlist<Capability*>* > client_caps;   // to identify clients who need snap notifications
+  std::map<client_t, elist<Capability*> > client_caps;   // to identify clients who need snap notifications
 
 protected:
   void check_cache() const;