From: Yan, Zheng Date: Mon, 10 Dec 2018 03:37:32 +0000 (+0800) Subject: mds: optimize resuming stale caps X-Git-Tag: v13.2.7~27^2~5 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=4ebc24890844b65d8cc10bb9436eb39f5d07f126;p=ceph.git mds: optimize resuming stale caps If client doesn't want any cap, there is no need to re-issue stale caps. A special case is that client wants some caps, but skipped updating 'wanted'. For this case, client needs to update 'wanted' when stale session get renewed. Fixes: http://tracker.ceph.com/issues/38043 Signed-off-by: "Yan, Zheng" (cherry picked from commit e824b3d2024db36789fdf579f0af9bf3bbe55d51) Conflicts: src/client/Client.cc src/mds/cephfs_features.h --- diff --git a/src/client/Client.cc b/src/client/Client.cc index 8517ecc4b7a7..10eb3ac87813 100644 --- a/src/client/Client.cc +++ b/src/client/Client.cc @@ -3266,10 +3266,8 @@ int Client::get_caps(Inode *in, int need, int want, int *phave, loff_t endoff) return ret; continue; } - if ((mds_wanted & file_wanted) == - (file_wanted & (CEPH_CAP_FILE_RD | CEPH_CAP_FILE_WR))) { + if (!(file_wanted & ~mds_wanted)) in->flags &= ~I_CAP_DROPPED; - } } if (waitfor_caps) @@ -3766,6 +3764,14 @@ void Client::wake_up_session_caps(MetaSession *s, bool reconnect) if (reconnect) { in->requested_max_size = 0; in->wanted_max_size = 0; + } else { + if (cap->gen < s->cap_gen) { + // mds did not re-issue stale cap. + cap->issued = cap->implemented = CEPH_CAP_PIN; + // make sure mds knows what we want. + if (in->caps_file_wanted() & ~cap->wanted) + in->flags |= I_CAP_DROPPED; + } } signal_cond_list(in->waitfor_caps); } @@ -3949,6 +3955,9 @@ void Client::add_update_cap(Inode *in, MetaSession *mds_session, uint64_t cap_id const auto &capem = in->caps.emplace(std::piecewise_construct, std::forward_as_tuple(mds), std::forward_as_tuple(in, mds_session)); Cap &cap = capem.first->second; if (!capem.second) { + if (cap.gen < mds_session->cap_gen) + cap.issued = cap.implemented = CEPH_CAP_PIN; + /* * auth mds of the inode changed. we received the cap export * message, but still haven't received the cap import message. @@ -4071,10 +4080,10 @@ void Client::remove_session_caps(MetaSession *s) dirty_caps = in->dirty_caps | in->flushing_caps; in->wanted_max_size = 0; in->requested_max_size = 0; - in->flags |= I_CAP_DROPPED; } + if (cap->wanted | cap->issued) + in->flags |= I_CAP_DROPPED; remove_cap(cap, false); - signal_cond_list(in->waitfor_caps); if (cap_snaps) { InodeRef tmp_ref(in); in->cap_snaps.clear(); @@ -4089,6 +4098,7 @@ void Client::remove_session_caps(MetaSession *s) in->mark_caps_clean(); put_inode(in); } + signal_cond_list(in->waitfor_caps); } s->flushing_caps_tids.clear(); sync_cond.Signal(); @@ -4872,10 +4882,9 @@ void Client::handle_cap_export(MetaSession *session, Inode *in, MClientCaps *m) auto it = in->caps.find(mds); if (it != in->caps.end()) { Cap &cap = it->second; - const mds_rank_t peer_mds = mds_rank_t(m->peer.mds); - if (cap.cap_id == m->get_cap_id()) { if (m->peer.cap_id) { + const auto peer_mds = mds_rank_t(m->peer.mds); MetaSession *tsession = _get_or_open_mds_session(peer_mds); auto it = in->caps.find(peer_mds); if (it != in->caps.end()) { @@ -4899,7 +4908,7 @@ void Client::handle_cap_export(MetaSession *session, Inode *in, MClientCaps *m) cap.latest_perms); } } else { - if (&cap == in->auth_cap) + if (cap.wanted | cap.issued) in->flags |= I_CAP_DROPPED; } @@ -5119,16 +5128,21 @@ void Client::handle_cap_grant(MetaSession *session, Inode *in, Cap *cap, MClient int used = get_caps_used(in); int wanted = in->caps_wanted(); - const int old_caps = cap->issued; - const int new_caps = m->get_caps(); + const unsigned new_caps = m->get_caps(); const bool was_stale = session->cap_gen > cap->gen; ldout(cct, 5) << __func__ << " on in " << m->get_ino() << " mds." << mds << " seq " << m->get_seq() << " caps now " << ccap_string(new_caps) - << " was " << ccap_string(old_caps) << dendl; + << " was " << ccap_string(cap->issued) + << (was_stale ? "" : " (stale)") << dendl; + + if (was_stale) + cap->issued = cap->implemented = CEPH_CAP_PIN; cap->seq = m->get_seq(); cap->gen = session->cap_gen; + check_cap_issue(in, cap, new_caps); + // update inode int issued; in->caps_issued(&issued); @@ -5207,10 +5221,9 @@ void Client::handle_cap_grant(MetaSession *session, Inode *in, Cap *cap, MClient check = true; } - check_cap_issue(in, cap, new_caps); // update caps - int revoked = old_caps & ~new_caps; + auto revoked = cap->issued & ~new_caps; if (revoked) { ldout(cct, 10) << " revocation of " << ccap_string(revoked) << dendl; cap->issued = new_caps; @@ -5232,10 +5245,10 @@ void Client::handle_cap_grant(MetaSession *session, Inode *in, Cap *cap, MClient cap->wanted = 0; // don't let check_caps skip sending a response to MDS check = true; } - } else if (old_caps == new_caps) { - ldout(cct, 10) << " caps unchanged at " << ccap_string(old_caps) << dendl; + } else if (cap->issued == new_caps) { + ldout(cct, 10) << " caps unchanged at " << ccap_string(cap->issued) << dendl; } else { - ldout(cct, 10) << " grant, new caps are " << ccap_string(new_caps & ~old_caps) << dendl; + ldout(cct, 10) << " grant, new caps are " << ccap_string(new_caps & ~cap->issued) << dendl; cap->issued = new_caps; cap->implemented |= new_caps; diff --git a/src/mds/Locker.cc b/src/mds/Locker.cc index e1c2a91fa2c6..6126f3193149 100644 --- a/src/mds/Locker.cc +++ b/src/mds/Locker.cc @@ -27,6 +27,7 @@ #include "MDLog.h" #include "MDSMap.h" +#include "cephfs_features.h" #include "events/EUpdate.h" #include "events/EOpen.h" @@ -2219,8 +2220,13 @@ void Locker::resume_stale_caps(Session *session) { dout(10) << "resume_stale_caps for " << session->info.inst.name << dendl; - for (xlist::iterator p = session->caps.begin(); !p.end(); ++p) { + bool lazy = session->info.has_feature(CEPHFS_FEATURE_LAZY_CAP_WANTED); + for (xlist::iterator p = session->caps.begin(); !p.end(); ) { Capability *cap = *p; + ++p; + if (lazy && !cap->is_notable()) + break; // see revoke_stale_caps() + CInode *in = cap->get_inode(); ceph_assert(in->is_head()); dout(10) << " clearing stale flag on " << *in << dendl; diff --git a/src/mds/cephfs_features.h b/src/mds/cephfs_features.h index 1c54354b63e9..1114d3cacd67 100644 --- a/src/mds/cephfs_features.h +++ b/src/mds/cephfs_features.h @@ -23,6 +23,7 @@ #define CEPHFS_FEATURE_KRAKEN 6 #define CEPHFS_FEATURE_LUMINOUS 7 #define CEPHFS_FEATURE_MIMIC 8 +#define CEPHFS_FEATURE_LAZY_CAP_WANTED 11 #define CEPHFS_FEATURES_ALL { \ 0, 1, 2, 3, 4, \ @@ -30,6 +31,7 @@ CEPHFS_FEATURE_KRAKEN, \ CEPHFS_FEATURE_LUMINOUS, \ CEPHFS_FEATURE_MIMIC, \ + CEPHFS_FEATURE_LAZY_CAP_WANTED, \ } #define CEPHFS_FEATURES_MDS_SUPPORTED CEPHFS_FEATURES_ALL