From: haoyixing <haoyixing@kuaishou.com>
Date: Fri, 25 Mar 2022 03:02:13 +0000 (+0000)
Subject: mds: add a perf counter to record slow replies
X-Git-Tag: v18.0.0~1024^2
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=e8e3b307c87dc9eec2d087b396c0e7a0248b4f1d;p=ceph.git

mds: add a perf counter to record slow replies

Though we have MDS_HEALTH_SLOW_METADATA_IO and MDS_HEALTH_SLOW_REQUEST health alert, but those are not
precise nor accumulated. With slow reply counter compared to reply counter, we can find out the ratio
of slow requests through perf dump.

Fixes: https://tracker.ceph.com/issues/55126
Signed-off-by: haoyixing <haoyixing@kuaishou.com>
---

diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index 053799c0df07..5eccd0acd75c 100644
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -3350,6 +3350,8 @@ void MDSRank::create_logger()
                             "exi", PerfCountersBuilder::PRIO_INTERESTING);
     mds_plb.add_u64_counter(l_mds_imported_inodes, "imported_inodes", "Imported inodes",
                             "imi", PerfCountersBuilder::PRIO_INTERESTING);
+    mds_plb.add_u64_counter(l_mds_slow_reply, "slow_reply", "Slow replies", "slr",
+                              PerfCountersBuilder::PRIO_INTERESTING);
 
     // caps msg stats
     mds_plb.add_u64_counter(l_mdss_handle_client_caps, "handle_client_caps",
diff --git a/src/mds/MDSRank.h b/src/mds/MDSRank.h
index 3df1abb5d13a..f6f9a97bc6b8 100644
--- a/src/mds/MDSRank.h
+++ b/src/mds/MDSRank.h
@@ -53,6 +53,7 @@ enum {
   l_mds_request,
   l_mds_reply,
   l_mds_reply_latency,
+  l_mds_slow_reply,
   l_mds_forward,
   l_mds_dir_fetch_complete,
   l_mds_dir_fetch_keys,
@@ -705,4 +706,3 @@ public:
 };
 
 #endif // MDS_RANK_H_
-
diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index 5b54e8112a41..8a53beac9543 100644
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -2141,6 +2141,9 @@ void Server::early_reply(MDRequestRef& mdr, CInode *tracei, CDentry *tracedn)
   mds->logger->inc(l_mds_reply);
   utime_t lat = ceph_clock_now() - req->get_recv_stamp();
   mds->logger->tinc(l_mds_reply_latency, lat);
+  if (lat >= g_conf()->mds_op_complaint_time) {
+    mds->logger->inc(l_mds_slow_reply);
+  }
   if (client_inst.name.is_client()) {
     mds->sessionmap.hit_session(mdr->session);
   }
@@ -2199,6 +2202,9 @@ void Server::reply_client_request(MDRequestRef& mdr, const ref_t<MClientReply> &
     mds->logger->inc(l_mds_reply);
     utime_t lat = ceph_clock_now() - mdr->client_request->get_recv_stamp();
     mds->logger->tinc(l_mds_reply_latency, lat);
+    if (lat >= g_conf()->mds_op_complaint_time) {
+      mds->logger->inc(l_mds_slow_reply);
+    }
     if (session && client_inst.name.is_client()) {
       mds->sessionmap.hit_session(session);
     }