mds: optimize resuming stale caps

If client doesn't want any cap, there is no need to re-issue stale
caps.

A special case is that client wants some caps, but skipped updating
'wanted'. For this case, client needs to update 'wanted' when stale
session get renewed.

Fixes: http://tracker.ceph.com/issues/38043
Signed-off-by: "Yan, Zheng" <zyan@redhat.com>

diff --git a/src/client/Client.cc b/src/client/Client.cc
index 098b0365de82725ff76a915cf27c4f8a2de75eb1..d9e79fa3c3ce86ec51d8713e5d540a048cf52372 100644 (file)
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -3258,10 +3258,8 @@ int Client::get_caps(Inode *in, int need, int want, int *phave, loff_t endoff)
          return ret;
        continue;
       }
-      if ((mds_wanted & file_wanted) ==
-         (file_wanted & (CEPH_CAP_FILE_RD | CEPH_CAP_FILE_WR))) {
+      if (!(file_wanted & ~mds_wanted))
        in->flags &= ~I_CAP_DROPPED;
-      }
     }
 
     if (waitfor_caps)
@@ -3789,6 +3787,14 @@ void Client::wake_up_session_caps(MetaSession *s, bool reconnect)
     if (reconnect) {
       in.requested_max_size = 0;
       in.wanted_max_size = 0;
+    } else {
+      if (cap->gen < s->cap_gen) {
+       // mds did not re-issue stale cap.
+       cap->issued = cap->implemented = CEPH_CAP_PIN;
+       // make sure mds knows what we want.
+       if (in.caps_file_wanted() & ~cap->wanted)
+         in.flags |= I_CAP_DROPPED;
+      }
     }
     signal_cond_list(in.waitfor_caps);
   }
@@ -3972,6 +3978,9 @@ void Client::add_update_cap(Inode *in, MetaSession *mds_session, uint64_t cap_id
   const auto &capem = in->caps.emplace(std::piecewise_construct, std::forward_as_tuple(mds), std::forward_as_tuple(*in, mds_session));
   Cap &cap = capem.first->second;
   if (!capem.second) {
+    if (cap.gen < mds_session->cap_gen)
+      cap.issued = cap.implemented = CEPH_CAP_PIN;
+
     /*
      * auth mds of the inode changed. we received the cap export
      * message, but still haven't received the cap import message.
@@ -4094,10 +4103,10 @@ void Client::remove_session_caps(MetaSession *s)
       dirty_caps = in->dirty_caps | in->flushing_caps;
       in->wanted_max_size = 0;
       in->requested_max_size = 0;
-      in->flags |= I_CAP_DROPPED;
     }
+    if (cap->wanted | cap->issued)
+      in->flags |= I_CAP_DROPPED;
     remove_cap(cap, false);
-    signal_cond_list(in->waitfor_caps);
     if (cap_snaps) {
       in->cap_snaps.clear();
     }
@@ -4111,6 +4120,7 @@ void Client::remove_session_caps(MetaSession *s)
       in->mark_caps_clean();
       put_inode(in.get());
     }
+    signal_cond_list(in->waitfor_caps);
   }
   s->flushing_caps_tids.clear();
   sync_cond.Signal();
@@ -4894,10 +4904,9 @@ void Client::handle_cap_export(MetaSession *session, Inode *in, MClientCaps *m)
   auto it = in->caps.find(mds);
   if (it != in->caps.end()) {
     Cap &cap = it->second;
-    const mds_rank_t peer_mds = mds_rank_t(m->peer.mds);
-
     if (cap.cap_id == m->get_cap_id()) {
       if (m->peer.cap_id) {
+       const auto peer_mds = mds_rank_t(m->peer.mds);
         MetaSession *tsession = _get_or_open_mds_session(peer_mds);
         auto it = in->caps.find(peer_mds);
         if (it != in->caps.end()) {
@@ -4921,7 +4930,7 @@ void Client::handle_cap_export(MetaSession *session, Inode *in, MClientCaps *m)
                         cap.latest_perms);
         }
       } else {
-        if (&cap == in->auth_cap)
+       if (cap.wanted | cap.issued)
          in->flags |= I_CAP_DROPPED;
       }
 
@@ -5138,16 +5147,21 @@ void Client::handle_cap_grant(MetaSession *session, Inode *in, Cap *cap, MClient
   int used = get_caps_used(in);
   int wanted = in->caps_wanted();
 
-  const int old_caps = cap->issued;
-  const int new_caps = m->get_caps();
+  const unsigned new_caps = m->get_caps();
   const bool was_stale = session->cap_gen > cap->gen;
   ldout(cct, 5) << __func__ << " on in " << m->get_ino() 
                << " mds." << mds << " seq " << m->get_seq()
                << " caps now " << ccap_string(new_caps)
-               << " was " << ccap_string(old_caps) << dendl;
+               << " was " << ccap_string(cap->issued)
+               << (was_stale ? "" : " (stale)") << dendl;
+
+  if (was_stale)
+      cap->issued = cap->implemented = CEPH_CAP_PIN;
   cap->seq = m->get_seq();
   cap->gen = session->cap_gen;
 
+  check_cap_issue(in, cap, new_caps);
+
   // update inode
   int issued;
   in->caps_issued(&issued);
@@ -5226,10 +5240,9 @@ void Client::handle_cap_grant(MetaSession *session, Inode *in, Cap *cap, MClient
     check = true;
   }
 
-  check_cap_issue(in, cap, new_caps);
 
   // update caps
-  int revoked = old_caps & ~new_caps;
+  auto revoked = cap->issued & ~new_caps;
   if (revoked) {
     ldout(cct, 10) << "  revocation of " << ccap_string(revoked) << dendl;
     cap->issued = new_caps;
@@ -5252,10 +5265,10 @@ void Client::handle_cap_grant(MetaSession *session, Inode *in, Cap *cap, MClient
       cap->wanted = 0; // don't let check_caps skip sending a response to MDS
       check = true;
     }
-  } else if (old_caps == new_caps) {
-    ldout(cct, 10) << "  caps unchanged at " << ccap_string(old_caps) << dendl;
+  } else if (cap->issued == new_caps) {
+    ldout(cct, 10) << "  caps unchanged at " << ccap_string(cap->issued) << dendl;
   } else {
-    ldout(cct, 10) << "  grant, new caps are " << ccap_string(new_caps & ~old_caps) << dendl;
+    ldout(cct, 10) << "  grant, new caps are " << ccap_string(new_caps & ~cap->issued) << dendl;
     cap->issued = new_caps;
     cap->implemented |= new_caps;
 

diff --git a/src/mds/Locker.cc b/src/mds/Locker.cc
index c9ac1853dc53d287eafc0c77de25f4fffa497eee..1f49db08a76a1862d4f7db1f7d661d5d2456fa63 100644 (file)
--- a/src/mds/Locker.cc
+++ b/src/mds/Locker.cc
@@ -2170,8 +2170,13 @@ void Locker::resume_stale_caps(Session *session)
 {
   dout(10) << "resume_stale_caps for " << session->info.inst.name << dendl;
 
-  for (xlist<Capability*>::iterator p = session->caps.begin(); !p.end(); ++p) {
+  bool lazy = session->info.has_feature(CEPHFS_FEATURE_LAZY_CAP_WANTED);
+  for (xlist<Capability*>::iterator p = session->caps.begin(); !p.end(); ) {
     Capability *cap = *p;
+    ++p;
+    if (lazy && !cap->is_notable())
+      break; // see revoke_stale_caps()
+
     CInode *in = cap->get_inode();
     ceph_assert(in->is_head());
     dout(10) << " clearing stale flag on " << *in << dendl;

diff --git a/src/mds/cephfs_features.h b/src/mds/cephfs_features.h
index 8ad16cc2f8c7710c824b1eaf7968cc4311897a68..06922cd65cfa3a28c76664aee29c93c602594f8b 100644 (file)
--- a/src/mds/cephfs_features.h
+++ b/src/mds/cephfs_features.h
@@ -25,6 +25,7 @@
 #define CEPHFS_FEATURE_MIMIC           8
 #define CEPHFS_FEATURE_REPLY_ENCODING   9
 #define CEPHFS_FEATURE_RECLAIM_CLIENT  10
+#define CEPHFS_FEATURE_LAZY_CAP_WANTED  11
 
 #define CEPHFS_FEATURES_ALL {          \
   0, 1, 2, 3, 4,                       \
@@ -34,6 +35,7 @@
   CEPHFS_FEATURE_MIMIC,                        \
   CEPHFS_FEATURE_REPLY_ENCODING,        \
   CEPHFS_FEATURE_RECLAIM_CLIENT,       \
+  CEPHFS_FEATURE_LAZY_CAP_WANTED,      \
 }
 
 #define CEPHFS_FEATURES_MDS_SUPPORTED CEPHFS_FEATURES_ALL