mds: optimize revoking stale caps

For caps that are not being revoked and don't have writeable range
and don't want exclusive caps or file read/write. there is no need
to call Locker::revoke_stale_caps(Capability*). Because these caps
don't need recover and don't affect eval_gather()/try_eval().

Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
(cherry picked from commit cb62030f0468fc04735c1b4cff73da779cb11ad8)

 Conflicts:
	src/mds/CInode.cc
	src/mds/Capability.h
	src/mds/Locker.cc
	src/mds/Migrator.cc

diff --git a/src/mds/CInode.cc b/src/mds/CInode.cc
index c834fc427e46c00ea7ddfa30b0edc2bf4efaee9b..b91c47881f5eb07954cae4b44401fde29c10be68 100644 (file)
--- a/src/mds/CInode.cc
+++ b/src/mds/CInode.cc
@@ -218,7 +218,7 @@ ostream& operator<<(ostream& out, const CInode& in)
       if (it->second->issued() != it->second->pending())
        out << "/" << ccap_string(it->second->issued());
       out << "/" << ccap_string(it->second->wanted())
-         << "@" << it->second->get_last_sent();
+         << "@" << it->second->get_last_seq();
     }
     out << "}";
     if (in.get_loner() >= 0 || in.get_wanted_loner() >= 0) {
@@ -4260,7 +4260,7 @@ void CInode::dump(Formatter *f) const
     f->dump_string("pending", ccap_string(it->second->pending()));
     f->dump_string("issued", ccap_string(it->second->issued()));
     f->dump_string("wanted", ccap_string(it->second->wanted()));
-    f->dump_int("last_sent", it->second->get_last_sent());
+    f->dump_int("last_sent", it->second->get_last_seq());
     f->close_section();
   }
   f->close_section();

diff --git a/src/mds/Capability.cc b/src/mds/Capability.cc
index 98be60f079f4b5c21de86b4aedef740e93b36106..8bfa90b1268b613f0cd72db9e9511f8aba08854c 100644 (file)
--- a/src/mds/Capability.cc
+++ b/src/mds/Capability.cc
@@ -154,6 +154,7 @@ Capability::Capability(CInode *i, Session *s, uint64_t id) :
 {
   if (session) {
     session->touch_cap_bottom(this);
+    cap_gen = session->get_cap_gen();
   }
 }
 
@@ -167,6 +168,29 @@ bool Capability::is_stale() const
   return session ? session->is_stale() : false;
 }
 
+bool Capability::is_valid() const
+{
+  return !session || session->get_cap_gen() == cap_gen;
+}
+
+void Capability::revalidate()
+{
+  if (is_valid())
+    return;
+
+  if (_pending & ~CEPH_CAP_PIN)
+    inc_last_seq();
+
+  bool was_revoking = _issued & ~_pending;
+  _pending = _issued = CEPH_CAP_PIN;
+  _revokes.clear();
+
+  cap_gen = session->get_cap_gen();
+
+  if (was_revoking)
+    maybe_clear_notable();
+}
+
 void Capability::mark_notable()
 {
   state |= STATE_NOTABLE;

diff --git a/src/mds/Capability.h b/src/mds/Capability.h
index 4ff5b811fc852eb91f8c41e8d180eeeb882a4c65..db3e47d3876eb50ff3690f7350c46a7b76db7662 100644 (file)
--- a/src/mds/Capability.h
+++ b/src/mds/Capability.h
@@ -120,11 +120,16 @@ public:
 
   const Capability& operator=(const Capability& other);  // no copying
 
-  int pending() { return _pending; }
-  int issued() { return _issued; }
-  bool is_null() { return !_pending && _revokes.empty(); }
+  int pending() const {
+    return is_valid() ? _pending : (_pending & CEPH_CAP_PIN);
+  }
+  int issued() const {
+    return is_valid() ? _issued : (_issued & CEPH_CAP_PIN);
+  }
 
   ceph_seq_t issue(unsigned c) {
+    revalidate();
+
     if (_pending & ~c) {
       // revoking (and maybe adding) bits.  note caps prior to this revocation
       _revokes.emplace_back(_pending, last_sent, last_issue);
@@ -149,6 +154,8 @@ public:
     return last_sent;
   }
   ceph_seq_t issue_norevoke(unsigned c) {
+    revalidate();
+
     _pending |= c;
     _issued |= c;
     //check_rdcaps_list();
@@ -204,9 +211,8 @@ public:
   ceph_seq_t get_mseq() { return mseq; }
   void inc_mseq() { mseq++; }
 
-  ceph_seq_t get_last_sent() { return last_sent; }
-  utime_t get_last_issue_stamp() { return last_issue_stamp; }
-  utime_t get_last_revoke_stamp() { return last_revoke_stamp; }
+  utime_t get_last_issue_stamp() const { return last_issue_stamp; }
+  utime_t get_last_revoke_stamp() const { return last_revoke_stamp; }
 
   void set_last_issue() { last_issue = last_sent; }
   void set_last_issue_stamp(utime_t t) { last_issue_stamp = t; }
@@ -261,8 +267,12 @@ public:
   void set_wanted(int w);
 
   void inc_last_seq() { last_sent++; }
-  ceph_seq_t get_last_seq() { return last_sent; }
-  ceph_seq_t get_last_issue() { return last_issue; }
+  ceph_seq_t get_last_seq() const {
+    if (!is_valid() && (_pending & ~CEPH_CAP_PIN))
+      return last_sent + 1;
+    return last_sent;
+  }
+  ceph_seq_t get_last_issue() const { return last_issue; }
 
   void reset_seq() {
     last_sent = 0;
@@ -270,8 +280,8 @@ public:
   }
   
   // -- exports --
-  Export make_export() {
-    return Export(cap_id, _wanted, issued(), pending(), client_follows, last_sent, mseq+1, last_issue_stamp);
+  Export make_export() const {
+    return Export(cap_id, wanted(), issued(), pending(), client_follows, get_last_seq(), mseq+1, last_issue_stamp);
   }
   void merge(const Export& other, bool auth_cap) {
     if (!is_stale()) {
@@ -337,6 +347,7 @@ private:
   Session *session;
 
   uint64_t cap_id;
+  uint32_t cap_gen;
 
   __u32 _wanted;     // what the client wants (ideally)
 
@@ -364,6 +375,9 @@ private:
     }
   }
 
+  bool is_valid() const;
+  void revalidate();
+
   void mark_notable();
   void maybe_clear_notable();
 };

diff --git a/src/mds/Locker.cc b/src/mds/Locker.cc
index e89a1aae5ae3582bb5f1fbb6db89f4a555f0902c..de33e4a8c56b6c42a8d021faa7339d8361e70416 100644 (file)
--- a/src/mds/Locker.cc
+++ b/src/mds/Locker.cc
@@ -2123,19 +2123,27 @@ void Locker::issue_truncate(CInode *in)
     check_inode_max_size(in);
 }
 
-
-void Locker::revoke_stale_caps(Capability *cap)
+void Locker::revoke_stale_caps(Session *session)
 {
-  CInode *in = cap->get_inode();
-  if (in->state_test(CInode::STATE_EXPORTINGCAPS)) {
-    // if export succeeds, the cap will be removed. if export fails, we need to
-    // revoke the cap if it's still stale.
-    in->state_set(CInode::STATE_EVALSTALECAPS);
-    return;
-  }
+  dout(10) << "revoke_stale_caps for " << session->info.inst.name << dendl;
+
+  std::vector<CInode*> to_eval;
+
+  for (auto p = session->caps.begin(); !p.end(); ) {
+    Capability *cap = *p;
+    ++p;
+    if (!cap->is_notable()) {
+      // the rest ones are not being revoked and don't have writeable range
+      // and don't want exclusive caps or want file read/write. They don't
+      // need recover, they don't affect eval_gather()/try_eval()
+      break;
+    }
 
-  int issued = cap->issued();
-  if (issued & ~CEPH_CAP_PIN) {
+    int issued = cap->issued();
+    if (!(issued & ~CEPH_CAP_PIN))
+      continue;
+
+    CInode *in = cap->get_inode();
     dout(10) << " revoking " << ccap_string(issued) << " on " << *in << dendl;
     cap->revoke();
 
@@ -2143,26 +2151,31 @@ void Locker::revoke_stale_caps(Capability *cap)
        in->inode.client_ranges.count(cap->get_client()))
       in->state_set(CInode::STATE_NEEDSRECOVER);
 
-    if (!in->filelock.is_stable()) eval_gather(&in->filelock);
-    if (!in->linklock.is_stable()) eval_gather(&in->linklock);
-    if (!in->authlock.is_stable()) eval_gather(&in->authlock);
-    if (!in->xattrlock.is_stable()) eval_gather(&in->xattrlock);
-
-    if (in->is_auth()) {
-      try_eval(in, CEPH_CAP_LOCKS);
-    } else {
-      request_inode_file_caps(in);
-    }
+    // eval lock/inode may finish contexts, which may modify other cap's position
+    // in the session->caps.
+    to_eval.push_back(in);
   }
-}
 
-void Locker::revoke_stale_caps(Session *session)
-{
-  dout(10) << "revoke_stale_caps for " << session->info.inst.name << dendl;
+  // invalidate the rest
+  session->inc_cap_gen();
 
-  for (xlist<Capability*>::iterator p = session->caps.begin(); !p.end(); ++p) {
-    Capability *cap = *p;
-    revoke_stale_caps(cap);
+  for (auto in : to_eval) {
+    if (in->state_test(CInode::STATE_EXPORTINGCAPS))
+      continue;
+
+    if (!in->filelock.is_stable())
+      eval_gather(&in->filelock);
+    if (!in->linklock.is_stable())
+      eval_gather(&in->linklock);
+    if (!in->authlock.is_stable())
+      eval_gather(&in->authlock);
+    if (!in->xattrlock.is_stable())
+      eval_gather(&in->xattrlock);
+
+    if (in->is_auth())
+      try_eval(in, CEPH_CAP_LOCKS);
+    else
+      request_inode_file_caps(in);
   }
 }
 
@@ -2323,7 +2336,7 @@ void Locker::calc_new_client_ranges(CInode *in, uint64_t size, bool update,
   for (map<client_t,Capability*>::iterator p = in->client_caps.begin();
        p != in->client_caps.end();
        ++p) {
-    if ((p->second->issued() | p->second->wanted()) & (CEPH_CAP_FILE_WR|CEPH_CAP_FILE_BUFFER)) {
+    if ((p->second->issued() | p->second->wanted()) & (CEPH_CAP_ANY_FILE_WR)) {
       client_writeable_range_t& nr = (*new_ranges)[p->first];
       nr.range.first = 0;
       if (latest->client_ranges.count(p->first)) {
@@ -3028,7 +3041,7 @@ public:
 void Locker::kick_issue_caps(CInode *in, client_t client, ceph_seq_t seq)
 {
   Capability *cap = in->get_client_cap(client);
-  if (!cap || cap->get_last_sent() != seq)
+  if (!cap || cap->get_last_seq() != seq)
     return;
   if (in->is_frozen()) {
     dout(10) << "kick_issue_caps waiting for unfreeze on " << *in << dendl;
@@ -3509,18 +3522,20 @@ void Locker::_do_cap_release(client_t client, inodeno_t ino, uint64_t cap_id,
     eval_cap_gather(in);
     return;
   }
-  remove_client_cap(in, client);
+  remove_client_cap(in, cap);
 }
 
-/* This function DOES put the passed message before returning */
-
-void Locker::remove_client_cap(CInode *in, client_t client)
+void Locker::remove_client_cap(CInode *in, Capability *cap)
 {
+  client_t client = cap->get_client();
   // clean out any pending snapflush state
   if (!in->client_need_snapflush.empty())
     _do_null_snapflush(in, client);
 
+  bool notable = cap->is_notable();
   in->remove_client_cap(client);
+  if (!notable)
+    return;
 
   if (in->is_auth()) {
     // make sure we clear out the client byte range

diff --git a/src/mds/Locker.h b/src/mds/Locker.h
index ddbd321f2a7603ca2f68dd482ef04f035577ddee..694c39250d0029f883667441dfae58ec4f521d0f 100644 (file)
--- a/src/mds/Locker.h
+++ b/src/mds/Locker.h
@@ -184,7 +184,7 @@ public:
   void kick_cap_releases(MDRequestRef& mdr);
   void kick_issue_caps(CInode *in, client_t client, ceph_seq_t seq);
 
-  void remove_client_cap(CInode *in, client_t client);
+  void remove_client_cap(CInode *in, Capability *cap);
 
   void get_late_revoking_clients(std::list<client_t> *result, double timeout) const;
 
@@ -244,7 +244,6 @@ public:
   void issue_caps_set(set<CInode*>& inset);
   void issue_truncate(CInode *in);
   void revoke_stale_caps(Session *session);
-  void revoke_stale_caps(Capability *cap);
   void resume_stale_caps(Session *session);
   void remove_stale_leases(Session *session);
 

diff --git a/src/mds/Migrator.cc b/src/mds/Migrator.cc
index a1cf8d31a4a326dfafccf8afcd001a231847997d..85f6b3b726ebd9b88f9f49e3a02881d415c65c0b 100644 (file)
--- a/src/mds/Migrator.cc
+++ b/src/mds/Migrator.cc
@@ -2128,10 +2128,9 @@ void Migrator::export_reverse(CDir *dir, export_state_t& stat)
     bool need_issue = false;
     for (auto& p : in->get_client_caps()) {
       Capability *cap = p.second;
-      if (cap->is_stale()) {
-       mds->locker->revoke_stale_caps(cap);
-      } else {
+      if (!cap->is_stale()) {
        need_issue = true;
+       break;
       }
     }
     if (need_issue &&

diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index 3a85815251d4e1b1b800c2fc42d9ed1190498014..38273d6bc934d87f92f0ea5eaa6af8e8ebbb7dda 100644 (file)
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -546,7 +546,7 @@ void Server::_session_logged(Session *session, uint64_t state_seq, bool open, ve
       Capability *cap = session->caps.front();
       CInode *in = cap->get_inode();
       dout(20) << " killing capability " << ccap_string(cap->issued()) << " on " << *in << dendl;
-      mds->locker->remove_client_cap(in, session->info.inst.name.num());
+      mds->locker->remove_client_cap(in, cap);
     }
     while (!session->leases.empty()) {
       ClientLease *r = session->leases.front();

diff --git a/src/mds/SessionMap.h b/src/mds/SessionMap.h
index 404c456d770237e9657206d0d4e90be75e81a49d..47625baf3a2a51fd980f95bf08a3476e721561d2 100644 (file)
--- a/src/mds/SessionMap.h
+++ b/src/mds/SessionMap.h
@@ -239,6 +239,7 @@ public:
 
   // -- caps --
 private:
+  uint32_t cap_gen;
   version_t cap_push_seq;        // cap push seq #
   map<version_t, list<MDSInternalContextBase*> > waitfor_flush; // flush session messages
 
@@ -248,7 +249,9 @@ public:
   time last_cap_renew = time::min();
   time last_seen = time::min();
 
-public:
+  void inc_cap_gen() { ++cap_gen; }
+  uint32_t get_cap_gen() const { return cap_gen; }
+
   version_t inc_push_seq() { return ++cap_push_seq; }
   version_t get_push_seq() const { return cap_push_seq; }
 
@@ -367,7 +370,7 @@ public:
     recall_release_count(0), auth_caps(g_ceph_context),
     connection(NULL), item_session_list(this),
     requests(0),  // member_offset passed to front() manually
-    cap_push_seq(0),
+    cap_gen(0), cap_push_seq(0),
     lease_seq(0),
     completed_requests_dirty(false),
     num_trim_flushes_warnings(0),