From: Yan, Zheng Date: Mon, 10 Dec 2018 03:37:32 +0000 (+0800) Subject: mds: optimize resuming stale caps X-Git-Tag: v14.1.0~293^2 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=e824b3d2024db36789fdf579f0af9bf3bbe55d51;p=ceph-ci.git mds: optimize resuming stale caps If client doesn't want any cap, there is no need to re-issue stale caps. A special case is that client wants some caps, but skipped updating 'wanted'. For this case, client needs to update 'wanted' when stale session get renewed. Fixes: http://tracker.ceph.com/issues/38043 Signed-off-by: "Yan, Zheng" --- diff --git a/src/client/Client.cc b/src/client/Client.cc index 098b0365de8..d9e79fa3c3c 100644 --- a/src/client/Client.cc +++ b/src/client/Client.cc @@ -3258,10 +3258,8 @@ int Client::get_caps(Inode *in, int need, int want, int *phave, loff_t endoff) return ret; continue; } - if ((mds_wanted & file_wanted) == - (file_wanted & (CEPH_CAP_FILE_RD | CEPH_CAP_FILE_WR))) { + if (!(file_wanted & ~mds_wanted)) in->flags &= ~I_CAP_DROPPED; - } } if (waitfor_caps) @@ -3789,6 +3787,14 @@ void Client::wake_up_session_caps(MetaSession *s, bool reconnect) if (reconnect) { in.requested_max_size = 0; in.wanted_max_size = 0; + } else { + if (cap->gen < s->cap_gen) { + // mds did not re-issue stale cap. + cap->issued = cap->implemented = CEPH_CAP_PIN; + // make sure mds knows what we want. + if (in.caps_file_wanted() & ~cap->wanted) + in.flags |= I_CAP_DROPPED; + } } signal_cond_list(in.waitfor_caps); } @@ -3972,6 +3978,9 @@ void Client::add_update_cap(Inode *in, MetaSession *mds_session, uint64_t cap_id const auto &capem = in->caps.emplace(std::piecewise_construct, std::forward_as_tuple(mds), std::forward_as_tuple(*in, mds_session)); Cap &cap = capem.first->second; if (!capem.second) { + if (cap.gen < mds_session->cap_gen) + cap.issued = cap.implemented = CEPH_CAP_PIN; + /* * auth mds of the inode changed. we received the cap export * message, but still haven't received the cap import message. @@ -4094,10 +4103,10 @@ void Client::remove_session_caps(MetaSession *s) dirty_caps = in->dirty_caps | in->flushing_caps; in->wanted_max_size = 0; in->requested_max_size = 0; - in->flags |= I_CAP_DROPPED; } + if (cap->wanted | cap->issued) + in->flags |= I_CAP_DROPPED; remove_cap(cap, false); - signal_cond_list(in->waitfor_caps); if (cap_snaps) { in->cap_snaps.clear(); } @@ -4111,6 +4120,7 @@ void Client::remove_session_caps(MetaSession *s) in->mark_caps_clean(); put_inode(in.get()); } + signal_cond_list(in->waitfor_caps); } s->flushing_caps_tids.clear(); sync_cond.Signal(); @@ -4894,10 +4904,9 @@ void Client::handle_cap_export(MetaSession *session, Inode *in, MClientCaps *m) auto it = in->caps.find(mds); if (it != in->caps.end()) { Cap &cap = it->second; - const mds_rank_t peer_mds = mds_rank_t(m->peer.mds); - if (cap.cap_id == m->get_cap_id()) { if (m->peer.cap_id) { + const auto peer_mds = mds_rank_t(m->peer.mds); MetaSession *tsession = _get_or_open_mds_session(peer_mds); auto it = in->caps.find(peer_mds); if (it != in->caps.end()) { @@ -4921,7 +4930,7 @@ void Client::handle_cap_export(MetaSession *session, Inode *in, MClientCaps *m) cap.latest_perms); } } else { - if (&cap == in->auth_cap) + if (cap.wanted | cap.issued) in->flags |= I_CAP_DROPPED; } @@ -5138,16 +5147,21 @@ void Client::handle_cap_grant(MetaSession *session, Inode *in, Cap *cap, MClient int used = get_caps_used(in); int wanted = in->caps_wanted(); - const int old_caps = cap->issued; - const int new_caps = m->get_caps(); + const unsigned new_caps = m->get_caps(); const bool was_stale = session->cap_gen > cap->gen; ldout(cct, 5) << __func__ << " on in " << m->get_ino() << " mds." << mds << " seq " << m->get_seq() << " caps now " << ccap_string(new_caps) - << " was " << ccap_string(old_caps) << dendl; + << " was " << ccap_string(cap->issued) + << (was_stale ? "" : " (stale)") << dendl; + + if (was_stale) + cap->issued = cap->implemented = CEPH_CAP_PIN; cap->seq = m->get_seq(); cap->gen = session->cap_gen; + check_cap_issue(in, cap, new_caps); + // update inode int issued; in->caps_issued(&issued); @@ -5226,10 +5240,9 @@ void Client::handle_cap_grant(MetaSession *session, Inode *in, Cap *cap, MClient check = true; } - check_cap_issue(in, cap, new_caps); // update caps - int revoked = old_caps & ~new_caps; + auto revoked = cap->issued & ~new_caps; if (revoked) { ldout(cct, 10) << " revocation of " << ccap_string(revoked) << dendl; cap->issued = new_caps; @@ -5252,10 +5265,10 @@ void Client::handle_cap_grant(MetaSession *session, Inode *in, Cap *cap, MClient cap->wanted = 0; // don't let check_caps skip sending a response to MDS check = true; } - } else if (old_caps == new_caps) { - ldout(cct, 10) << " caps unchanged at " << ccap_string(old_caps) << dendl; + } else if (cap->issued == new_caps) { + ldout(cct, 10) << " caps unchanged at " << ccap_string(cap->issued) << dendl; } else { - ldout(cct, 10) << " grant, new caps are " << ccap_string(new_caps & ~old_caps) << dendl; + ldout(cct, 10) << " grant, new caps are " << ccap_string(new_caps & ~cap->issued) << dendl; cap->issued = new_caps; cap->implemented |= new_caps; diff --git a/src/mds/Locker.cc b/src/mds/Locker.cc index c9ac1853dc5..1f49db08a76 100644 --- a/src/mds/Locker.cc +++ b/src/mds/Locker.cc @@ -2170,8 +2170,13 @@ void Locker::resume_stale_caps(Session *session) { dout(10) << "resume_stale_caps for " << session->info.inst.name << dendl; - for (xlist::iterator p = session->caps.begin(); !p.end(); ++p) { + bool lazy = session->info.has_feature(CEPHFS_FEATURE_LAZY_CAP_WANTED); + for (xlist::iterator p = session->caps.begin(); !p.end(); ) { Capability *cap = *p; + ++p; + if (lazy && !cap->is_notable()) + break; // see revoke_stale_caps() + CInode *in = cap->get_inode(); ceph_assert(in->is_head()); dout(10) << " clearing stale flag on " << *in << dendl; diff --git a/src/mds/cephfs_features.h b/src/mds/cephfs_features.h index 8ad16cc2f8c..06922cd65cf 100644 --- a/src/mds/cephfs_features.h +++ b/src/mds/cephfs_features.h @@ -25,6 +25,7 @@ #define CEPHFS_FEATURE_MIMIC 8 #define CEPHFS_FEATURE_REPLY_ENCODING 9 #define CEPHFS_FEATURE_RECLAIM_CLIENT 10 +#define CEPHFS_FEATURE_LAZY_CAP_WANTED 11 #define CEPHFS_FEATURES_ALL { \ 0, 1, 2, 3, 4, \ @@ -34,6 +35,7 @@ CEPHFS_FEATURE_MIMIC, \ CEPHFS_FEATURE_REPLY_ENCODING, \ CEPHFS_FEATURE_RECLAIM_CLIENT, \ + CEPHFS_FEATURE_LAZY_CAP_WANTED, \ } #define CEPHFS_FEATURES_MDS_SUPPORTED CEPHFS_FEATURES_ALL