mds: optimize resuming stale caps

If client doesn't want any cap, there is no need to re-issue stale
caps.

A special case is that client wants some caps, but skipped updating
'wanted'. For this case, client needs to update 'wanted' when stale
session get renewed.

Fixes: http://tracker.ceph.com/issues/38043
Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
(cherry picked from commit e824b3d2024db36789fdf579f0af9bf3bbe55d51)

 Conflicts:
	src/client/Client.cc
	src/mds/cephfs_features.h

diff --git a/src/client/Client.cc b/src/client/Client.cc
index 8517ecc4b7a77b033c225e6eb4d47b1cfd705b3f..10eb3ac878131e135a6bf6316339231d65e9c46c 100644 (file)
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -3266,10 +3266,8 @@ int Client::get_caps(Inode *in, int need, int want, int *phave, loff_t endoff)
          return ret;
        continue;
       }
-      if ((mds_wanted & file_wanted) ==
-         (file_wanted & (CEPH_CAP_FILE_RD | CEPH_CAP_FILE_WR))) {
+      if (!(file_wanted & ~mds_wanted))
        in->flags &= ~I_CAP_DROPPED;
-      }
     }
 
     if (waitfor_caps)
@@ -3766,6 +3764,14 @@ void Client::wake_up_session_caps(MetaSession *s, bool reconnect)
     if (reconnect) {
       in->requested_max_size = 0;
       in->wanted_max_size = 0;
+    } else {
+      if (cap->gen < s->cap_gen) {
+       // mds did not re-issue stale cap.
+       cap->issued = cap->implemented = CEPH_CAP_PIN;
+       // make sure mds knows what we want.
+       if (in->caps_file_wanted() & ~cap->wanted)
+         in->flags |= I_CAP_DROPPED;
+      }
     }
     signal_cond_list(in->waitfor_caps);
   }
@@ -3949,6 +3955,9 @@ void Client::add_update_cap(Inode *in, MetaSession *mds_session, uint64_t cap_id
   const auto &capem = in->caps.emplace(std::piecewise_construct, std::forward_as_tuple(mds), std::forward_as_tuple(in, mds_session));
   Cap &cap = capem.first->second;
   if (!capem.second) {
+    if (cap.gen < mds_session->cap_gen)
+      cap.issued = cap.implemented = CEPH_CAP_PIN;
+
     /*
      * auth mds of the inode changed. we received the cap export
      * message, but still haven't received the cap import message.
@@ -4071,10 +4080,10 @@ void Client::remove_session_caps(MetaSession *s)
       dirty_caps = in->dirty_caps | in->flushing_caps;
       in->wanted_max_size = 0;
       in->requested_max_size = 0;
-      in->flags |= I_CAP_DROPPED;
     }
+    if (cap->wanted | cap->issued)
+      in->flags |= I_CAP_DROPPED;
     remove_cap(cap, false);
-    signal_cond_list(in->waitfor_caps);
     if (cap_snaps) {
       InodeRef tmp_ref(in);
       in->cap_snaps.clear();
@@ -4089,6 +4098,7 @@ void Client::remove_session_caps(MetaSession *s)
       in->mark_caps_clean();
       put_inode(in);
     }
+    signal_cond_list(in->waitfor_caps);
   }
   s->flushing_caps_tids.clear();
   sync_cond.Signal();
@@ -4872,10 +4882,9 @@ void Client::handle_cap_export(MetaSession *session, Inode *in, MClientCaps *m)
   auto it = in->caps.find(mds);
   if (it != in->caps.end()) {
     Cap &cap = it->second;
-    const mds_rank_t peer_mds = mds_rank_t(m->peer.mds);
-
     if (cap.cap_id == m->get_cap_id()) {
       if (m->peer.cap_id) {
+       const auto peer_mds = mds_rank_t(m->peer.mds);
         MetaSession *tsession = _get_or_open_mds_session(peer_mds);
         auto it = in->caps.find(peer_mds);
         if (it != in->caps.end()) {
@@ -4899,7 +4908,7 @@ void Client::handle_cap_export(MetaSession *session, Inode *in, MClientCaps *m)
                         cap.latest_perms);
         }
       } else {
-        if (&cap == in->auth_cap)
+       if (cap.wanted | cap.issued)
          in->flags |= I_CAP_DROPPED;
       }
 
@@ -5119,16 +5128,21 @@ void Client::handle_cap_grant(MetaSession *session, Inode *in, Cap *cap, MClient
   int used = get_caps_used(in);
   int wanted = in->caps_wanted();
 
-  const int old_caps = cap->issued;
-  const int new_caps = m->get_caps();
+  const unsigned new_caps = m->get_caps();
   const bool was_stale = session->cap_gen > cap->gen;
   ldout(cct, 5) << __func__ << " on in " << m->get_ino() 
                << " mds." << mds << " seq " << m->get_seq()
                << " caps now " << ccap_string(new_caps)
-               << " was " << ccap_string(old_caps) << dendl;
+               << " was " << ccap_string(cap->issued)
+               << (was_stale ? "" : " (stale)") << dendl;
+
+  if (was_stale)
+      cap->issued = cap->implemented = CEPH_CAP_PIN;
   cap->seq = m->get_seq();
   cap->gen = session->cap_gen;
 
+  check_cap_issue(in, cap, new_caps);
+
   // update inode
   int issued;
   in->caps_issued(&issued);
@@ -5207,10 +5221,9 @@ void Client::handle_cap_grant(MetaSession *session, Inode *in, Cap *cap, MClient
     check = true;
   }
 
-  check_cap_issue(in, cap, new_caps);
 
   // update caps
-  int revoked = old_caps & ~new_caps;
+  auto revoked = cap->issued & ~new_caps;
   if (revoked) {
     ldout(cct, 10) << "  revocation of " << ccap_string(revoked) << dendl;
     cap->issued = new_caps;
@@ -5232,10 +5245,10 @@ void Client::handle_cap_grant(MetaSession *session, Inode *in, Cap *cap, MClient
       cap->wanted = 0; // don't let check_caps skip sending a response to MDS
       check = true;
     }
-  } else if (old_caps == new_caps) {
-    ldout(cct, 10) << "  caps unchanged at " << ccap_string(old_caps) << dendl;
+  } else if (cap->issued == new_caps) {
+    ldout(cct, 10) << "  caps unchanged at " << ccap_string(cap->issued) << dendl;
   } else {
-    ldout(cct, 10) << "  grant, new caps are " << ccap_string(new_caps & ~old_caps) << dendl;
+    ldout(cct, 10) << "  grant, new caps are " << ccap_string(new_caps & ~cap->issued) << dendl;
     cap->issued = new_caps;
     cap->implemented |= new_caps;
 

diff --git a/src/mds/Locker.cc b/src/mds/Locker.cc
index e1c2a91fa2c63f966fc707ef0660d01dbcc6ab8b..6126f31931498ba61a0042296febd33a990dbdc4 100644 (file)
--- a/src/mds/Locker.cc
+++ b/src/mds/Locker.cc
@@ -27,6 +27,7 @@
 
 #include "MDLog.h"
 #include "MDSMap.h"
+#include "cephfs_features.h"
 
 #include "events/EUpdate.h"
 #include "events/EOpen.h"
@@ -2219,8 +2220,13 @@ void Locker::resume_stale_caps(Session *session)
 {
   dout(10) << "resume_stale_caps for " << session->info.inst.name << dendl;
 
-  for (xlist<Capability*>::iterator p = session->caps.begin(); !p.end(); ++p) {
+  bool lazy = session->info.has_feature(CEPHFS_FEATURE_LAZY_CAP_WANTED);
+  for (xlist<Capability*>::iterator p = session->caps.begin(); !p.end(); ) {
     Capability *cap = *p;
+    ++p;
+    if (lazy && !cap->is_notable())
+      break; // see revoke_stale_caps()
+
     CInode *in = cap->get_inode();
     ceph_assert(in->is_head());
     dout(10) << " clearing stale flag on " << *in << dendl;

diff --git a/src/mds/cephfs_features.h b/src/mds/cephfs_features.h
index 1c54354b63e9f5b7daf5e16cc232ede67098c418..1114d3cacd670bfa3556ddbac8177301cf95e374 100644 (file)
--- a/src/mds/cephfs_features.h
+++ b/src/mds/cephfs_features.h
@@ -23,6 +23,7 @@
 #define CEPHFS_FEATURE_KRAKEN          6
 #define CEPHFS_FEATURE_LUMINOUS                7
 #define CEPHFS_FEATURE_MIMIC           8
+#define CEPHFS_FEATURE_LAZY_CAP_WANTED  11
 
 #define CEPHFS_FEATURES_ALL {          \
   0, 1, 2, 3, 4,                       \
@@ -30,6 +31,7 @@
   CEPHFS_FEATURE_KRAKEN,               \
   CEPHFS_FEATURE_LUMINOUS,             \
   CEPHFS_FEATURE_MIMIC,                        \
+  CEPHFS_FEATURE_LAZY_CAP_WANTED,      \
 }
 
 #define CEPHFS_FEATURES_MDS_SUPPORTED CEPHFS_FEATURES_ALL