From 3aa30e2f8c734a956a10879bbddc4b0f34748053 Mon Sep 17 00:00:00 2001 From: Venky Shankar Date: Tue, 17 Jan 2023 05:45:04 -0500 Subject: [PATCH] mds: record and dump last tid for trimming completed requests (or flushes) CephFS clients include `oldest_tid` as part of the client request to the MDS. This field is the tid of the oldest incomplete mds request (excluding setfilelock request). The MDS uses this to trim completed requests (and flushes). In one case, the ceph cluster had an extremely high completed requests count, meaning, for some reason the client was not advancing its `oldest_tid` field, although, the MDS had successfully "safe replied" the request back to the client. This change adds a debug aid for recording and dumping this field. It might be possible to fetch this from clients (if not, we should add that!), but it makes sense to have this information available from the MDS. Partially-Fixes: http://tracker.ceph.com/issues/57985 Signed-off-by: Venky Shankar (cherry picked from commit d01acd531a0694514f7e7e11087ce7a9d16e94d9) --- doc/cephfs/health-messages.rst | 4 +++- src/mds/SessionMap.cc | 3 +++ src/mds/SessionMap.h | 5 +++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/doc/cephfs/health-messages.rst b/doc/cephfs/health-messages.rst index 7edc1262f5ce..d833d0e6fc68 100644 --- a/doc/cephfs/health-messages.rst +++ b/doc/cephfs/health-messages.rst @@ -130,7 +130,9 @@ other daemons, please see :ref:`health-checks`. from properly cleaning up resources used by client requests. This message appears if a client appears to have more than ``max_completed_requests`` (default 100000) requests that are complete on the MDS side but haven't - yet been accounted for in the client's *oldest tid* value. + yet been accounted for in the client's *oldest tid* value. The last tid + used by the MDS to trim completed client requests (or flush) is included + as part of `session ls` (or `client ls`) command as a debug aid. ``MDS_DAMAGE`` -------------- diff --git a/src/mds/SessionMap.cc b/src/mds/SessionMap.cc index 720396338ca7..9cc2b013847e 100644 --- a/src/mds/SessionMap.cc +++ b/src/mds/SessionMap.cc @@ -622,6 +622,9 @@ void Session::dump(Formatter *f, bool cap_dump) const f->dump_object("session_cache_liveness", session_cache_liveness); f->dump_object("cap_acquisition", cap_acquisition); + f->dump_unsigned("last_trim_completed_requests_tid", last_trim_completed_requests_tid); + f->dump_unsigned("last_trim_completed_flushes_tid", last_trim_completed_flushes_tid); + f->open_array_section("delegated_inos"); for (const auto& [start, len] : delegated_inos) { f->open_object_section("ino_range"); diff --git a/src/mds/SessionMap.h b/src/mds/SessionMap.h index ddf227be9d11..360dd66a27bb 100644 --- a/src/mds/SessionMap.h +++ b/src/mds/SessionMap.h @@ -314,6 +314,7 @@ public: bool trim_completed_requests(ceph_tid_t mintid) { // trim bool erased_any = false; + last_trim_completed_requests_tid = mintid; while (!info.completed_requests.empty() && (mintid == 0 || info.completed_requests.begin()->first < mintid)) { info.completed_requests.erase(info.completed_requests.begin()); @@ -339,6 +340,7 @@ public: } bool trim_completed_flushes(ceph_tid_t mintid) { bool erased_any = false; + last_trim_completed_flushes_tid = mintid; while (!info.completed_flushes.empty() && (mintid == 0 || *info.completed_flushes.begin() < mintid)) { info.completed_flushes.erase(info.completed_flushes.begin()); @@ -493,6 +495,9 @@ private: unsigned num_trim_flushes_warnings = 0; unsigned num_trim_requests_warnings = 0; + + ceph_tid_t last_trim_completed_requests_tid = 0; + ceph_tid_t last_trim_completed_flushes_tid = 0; }; class SessionFilter -- 2.47.3