mds: use single map to track pinned & auth_pinned objects

reduce memory allocation and fragment

Signed-off-by: "Yan, Zheng" <zyan@redhat.com>

diff --git a/src/mds/Locker.cc b/src/mds/Locker.cc
index 991a6217805455e71369de66c28d6e9073a73fa8..e801148d943bb085ff2a69d2394873ee59931dcc 100644 (file)
--- a/src/mds/Locker.cc
+++ b/src/mds/Locker.cc
@@ -371,7 +371,7 @@ bool Locker::acquire_locks(MDRequestRef& mdr,
       dout(10) << " can't auth_pin (freezing?), waiting to authpin " << *object << dendl;
       object->add_waiter(MDSCacheObject::WAIT_UNFREEZE, new C_MDS_RetryRequest(mdcache, mdr));
 
-      if (!mdr->remote_auth_pins.empty())
+      if (mdr->is_any_remote_auth_pin())
        notify_freeze_waiter(object);
 
       return false;
@@ -392,10 +392,12 @@ bool Locker::acquire_locks(MDRequestRef& mdr,
   // request remote auth_pins
   if (!mustpin_remote.empty()) {
     marker.message = "requesting remote authpins";
-    for (const auto& p : mdr->remote_auth_pins) {
+    for (const auto& p : mdr->object_states) {
+      if (p.second.remote_auth_pinned == MDS_RANK_NONE)
+       continue;
       if (mustpin.count(p.first)) {
-       ceph_assert(p.second == p.first->authority().first);
-       map<mds_rank_t, set<MDSCacheObject*> >::iterator q = mustpin_remote.find(p.second);
+       ceph_assert(p.second.remote_auth_pinned == p.first->authority().first);
+       auto q = mustpin_remote.find(p.second.remote_auth_pinned);
        if (q != mustpin_remote.end())
          q->second.insert(p.first);
       }

diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index f5353de0fcde6ea5718d96896650e32a9cb72dac..ea3c4e55de8f263ace5ca3a4512d3b6aca9e262a 100644 (file)
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -4084,11 +4084,14 @@ void MDCache::rejoin_send_rejoins()
       if (mdr->is_slave())
        continue;
       // auth pins
-      for (const auto& q : mdr->remote_auth_pins) {
+      for (const auto& q : mdr->object_states) {
+       if (q.second.remote_auth_pinned == MDS_RANK_NONE)
+         continue;
        if (!q.first->is_auth()) {
-         ceph_assert(q.second == q.first->authority().first);
-         if (rejoins.count(q.second) == 0) continue;
-         const auto& rejoin = rejoins[q.second];
+         mds_rank_t target = q.second.remote_auth_pinned;
+         ceph_assert(target == q.first->authority().first);
+         if (rejoins.count(target) == 0) continue;
+         const auto& rejoin = rejoins[target];
          
          dout(15) << " " << *mdr << " authpin on " << *q.first << dendl;
          MDSCacheObjectInfo i;
@@ -12658,7 +12661,7 @@ void MDCache::repair_dirfrag_stats_work(MDRequestRef& mdr)
 
     mds->locker->drop_locks(mdr.get());
     mdr->drop_local_auth_pins();
-    if (!mdr->remote_auth_pins.empty())
+    if (mdr->is_any_remote_auth_pin())
       mds->locker->notify_freeze_waiter(dir);
     return;
   }

diff --git a/src/mds/Mutation.cc b/src/mds/Mutation.cc
index 5e194301eef8ba71670806d25f67c1e8e46498df..410ac1184c2738e5bd3a2a9978a05f715f9f09bd 100644 (file)
--- a/src/mds/Mutation.cc
+++ b/src/mds/Mutation.cc
@@ -20,17 +20,21 @@
 
 void MutationImpl::pin(MDSCacheObject *o)
 {
-  if (pins.count(o) == 0) {
+  auto& stat = object_states[o];
+  if (!stat.pinned) {
     o->get(MDSCacheObject::PIN_REQUEST);
-    pins.insert(o);
+    stat.pinned = true;
+    ++num_pins;
   }      
 }
 
 void MutationImpl::unpin(MDSCacheObject *o)
 {
-  ceph_assert(pins.count(o));
+  auto& stat = object_states[o];
+  ceph_assert(stat.pinned);
   o->put(MDSCacheObject::PIN_REQUEST);
-  pins.erase(o);
+  stat.pinned = false;
+  --num_pins;
 }
 
 void MutationImpl::set_stickydirs(CInode *in)
@@ -54,9 +58,13 @@ void MutationImpl::put_stickydirs()
 
 void MutationImpl::drop_pins()
 {
-  for (auto& o : pins)
-    o->put(MDSCacheObject::PIN_REQUEST);
-  pins.clear();
+  for (auto& p : object_states) {
+    if (p.second.pinned) {
+      p.first->put(MDSCacheObject::PIN_REQUEST);
+      p.second.pinned = false;
+      --num_pins;
+    }
+  }
 }
 
 void MutationImpl::start_locking(SimpleLock *lock, int target)
@@ -120,31 +128,59 @@ void MutationImpl::LockOpVec::sort_and_merge()
 // auth pins
 bool MutationImpl::is_auth_pinned(MDSCacheObject *object) const
 { 
-  return auth_pins.count(object) || remote_auth_pins.count(object); 
+  auto stat_p = find_object_state(object);
+  if (!stat_p)
+    return false;
+  return stat_p->auth_pinned || stat_p->remote_auth_pinned != MDS_RANK_NONE;
 }
 
 void MutationImpl::auth_pin(MDSCacheObject *object)
 {
-  if (!is_auth_pinned(object)) {
+  auto &stat = object_states[object];
+  if (!stat.auth_pinned) {
     object->auth_pin(this);
-    auth_pins.insert(object);
+    stat.auth_pinned = true;
+    ++num_auth_pins;
   }
 }
 
 void MutationImpl::auth_unpin(MDSCacheObject *object)
 {
-  ceph_assert(auth_pins.count(object));
+  auto &stat = object_states[object];
+  ceph_assert(stat.auth_pinned);
   object->auth_unpin(this);
-  auth_pins.erase(object);
+  stat.auth_pinned = false;
+  --num_auth_pins;
 }
 
 void MutationImpl::drop_local_auth_pins()
 {
-  for (const auto& p : auth_pins) {
-    ceph_assert(p->is_auth());
-    p->auth_unpin(this);
+  for (auto& p : object_states) {
+    if (p.second.auth_pinned) {
+      ceph_assert(p.first->is_auth());
+      p.first->auth_unpin(this);
+      p.second.auth_pinned = false;
+      --num_auth_pins;
+    }
+  }
+}
+
+void MutationImpl::set_remote_auth_pinned(MDSCacheObject *object, mds_rank_t from)
+{
+  auto &stat = object_states[object];
+  if (stat.remote_auth_pinned == MDS_RANK_NONE) {
+    stat.remote_auth_pinned = from;
+    ++num_remote_auth_pins;
+  } else {
+    ceph_assert(stat.remote_auth_pinned == from);
   }
-  auth_pins.clear();
+}
+
+void MutationImpl::_clear_remote_auth_pinned(ObjectState &stat)
+{
+  ceph_assert(stat.remote_auth_pinned != MDS_RANK_NONE);
+  stat.remote_auth_pinned = MDS_RANK_NONE;
+  --num_remote_auth_pins;
 }
 
 void MutationImpl::add_projected_inode(CInode *in)

diff --git a/src/mds/Mutation.h b/src/mds/Mutation.h
index 6e556bec6d164e901f8c9dc2fa93376f2e0d7ef4..6c489e6aae10dc597d5826b527334584e673aeb4 100644 (file)
--- a/src/mds/Mutation.h
+++ b/src/mds/Mutation.h
@@ -52,14 +52,27 @@ public:
   // flag mutation as slave
   mds_rank_t slave_to_mds = MDS_RANK_NONE;  // this is a slave request if >= 0.
 
-  // -- my pins and locks --
+  // -- my pins and auth_pins --
+  struct ObjectState {
+    bool pinned = false;
+    bool auth_pinned = false;
+    mds_rank_t remote_auth_pinned = MDS_RANK_NONE;
+  };
+  ceph::unordered_map<MDSCacheObject*, ObjectState> object_states;
+  int num_pins = 0;
+  int num_auth_pins = 0;
+  int num_remote_auth_pins = 0;
+
+  const ObjectState* find_object_state(MDSCacheObject *obj) const {
+    auto it = object_states.find(obj);
+    return it != object_states.end() ? &it->second : nullptr;
+  }
+
+  bool is_any_remote_auth_pin() const { return num_remote_auth_pins > 0; }
+
   // cache pins (so things don't expire)
-  set< MDSCacheObject* > pins;
   CInode* stickydiri = nullptr;
 
-  // auth pins
-  map<MDSCacheObject*, mds_rank_t> remote_auth_pins;
-  set<MDSCacheObject*> auth_pins;
   
   // held locks
   struct LockOp {
@@ -175,8 +188,8 @@ public:
       slave_to_mds(slave_to) { }
   ~MutationImpl() override {
     ceph_assert(locking == NULL);
-    ceph_assert(pins.empty());
-    ceph_assert(auth_pins.empty());
+    ceph_assert(num_pins == 0);
+    ceph_assert(num_auth_pins == 0);
   }
 
   bool is_master() const { return slave_to_mds == MDS_RANK_NONE; }
@@ -218,6 +231,9 @@ public:
   void auth_pin(MDSCacheObject *object);
   void auth_unpin(MDSCacheObject *object);
   void drop_local_auth_pins();
+  void set_remote_auth_pinned(MDSCacheObject* object, mds_rank_t from);
+  void _clear_remote_auth_pinned(ObjectState& stat);
+
   void add_projected_inode(CInode *in);
   void pop_and_dirty_projected_inodes();
   void add_projected_fnode(CDir *dir);

diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index c2a9c27a4cc10adc98a7dc0331f2369e7926d5bf..6929dd9462ce19c232c064b5199e6f9eb8fcc366 100644 (file)
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -2968,11 +2968,13 @@ void Server::handle_slave_auth_pin(MDRequestRef& mdr)
   auto reply = make_message<MMDSSlaveRequest>(mdr->reqid, mdr->attempt, MMDSSlaveRequest::OP_AUTHPINACK);
   
   // return list of my auth_pins (if any)
-  for (const auto &p : mdr->auth_pins) {
+  for (const auto &p : mdr->object_states) {
+    if (!p.second.auth_pinned)
+      continue;
     MDSCacheObjectInfo info;
-    p->set_object_info(info);
+    p.first->set_object_info(info);
     reply->get_authpins().push_back(info);
-    if (p == (MDSCacheObject*)auth_pin_freeze)
+    if (p.first == (MDSCacheObject*)auth_pin_freeze)
       auth_pin_freeze->set_object_info(reply->get_authpin_freeze());
   }
 
@@ -2999,8 +3001,7 @@ void Server::handle_slave_auth_pin_ack(MDRequestRef& mdr, const cref_t<MMDSSlave
     MDSCacheObject *object = mdcache->get_object(oi);
     ceph_assert(object);  // we pinned it
     dout(10) << " remote has pinned " << *object << dendl;
-    if (!mdr->is_auth_pinned(object))
-      mdr->remote_auth_pins[object] = from;
+    mdr->set_remote_auth_pinned(object, from);
     if (oi == ack->get_authpin_freeze())
       mdr->set_remote_frozen_auth_pin(static_cast<CInode *>(object));
     pinned.insert(object);
@@ -3009,22 +3010,21 @@ void Server::handle_slave_auth_pin_ack(MDRequestRef& mdr, const cref_t<MMDSSlave
   // removed frozen auth pin ?
   if (mdr->more()->is_remote_frozen_authpin &&
       ack->get_authpin_freeze() == MDSCacheObjectInfo()) {
-    auto p = mdr->remote_auth_pins.find(mdr->more()->rename_inode);
-    ceph_assert(p != mdr->remote_auth_pins.end());
-    if (p->second == from) {
+    auto stat_p = mdr->find_object_state(mdr->more()->rename_inode);
+    ceph_assert(stat_p);
+    if (stat_p->remote_auth_pinned == from) {
       mdr->more()->is_remote_frozen_authpin = false;
     }
   }
 
   // removed auth pins?
-  auto p = mdr->remote_auth_pins.begin();
-  while (p != mdr->remote_auth_pins.end()) {
-    MDSCacheObject* object = p->first;
-    if (p->second == from && pinned.count(object) == 0) {
+  for (auto& p : mdr->object_states) {
+    if (p.second.remote_auth_pinned == MDS_RANK_NONE)
+      continue;
+    MDSCacheObject* object = p.first;
+    if (p.second.remote_auth_pinned == from && pinned.count(object) == 0) {
       dout(10) << " remote has unpinned " << *object << dendl;
-      mdr->remote_auth_pins.erase(p++);
-    } else {
-      ++p;
+      mdr->_clear_remote_auth_pinned(p.second);
     }
   }
 
@@ -3487,7 +3487,7 @@ CInode* Server::rdlock_path_pin_ref(MDRequestRef& mdr, int n,
        */
       mds->locker->drop_locks(mdr.get(), NULL);
       mdr->drop_local_auth_pins();
-      if (!mdr->remote_auth_pins.empty())
+      if (mdr->is_any_remote_auth_pin())
        mds->locker->notify_freeze_waiter(ref);
       return 0;
     }