From: haoyixing <haoyixing@kuaishou.com>
Date: Fri, 25 Mar 2022 03:02:13 +0000 (+0000)
Subject: mds: add a perf counter to record slow replies
X-Git-Tag: v18.0.0~1024^2
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=refs%2Fpull%2F45631%2Fhead;p=ceph.git

mds: add a perf counter to record slow replies

Though we have MDS_HEALTH_SLOW_METADATA_IO and MDS_HEALTH_SLOW_REQUEST health alert, but those are not
precise nor accumulated. With slow reply counter compared to reply counter, we can find out the ratio
of slow requests through perf dump.

Fixes: https://tracker.ceph.com/issues/55126
Signed-off-by: haoyixing <haoyixing@kuaishou.com>
---

diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index 053799c0df0..5eccd0acd75 100644
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -3350,6 +3350,8 @@ void MDSRank::create_logger()
                             "exi", PerfCountersBuilder::PRIO_INTERESTING);
     mds_plb.add_u64_counter(l_mds_imported_inodes, "imported_inodes", "Imported inodes",
                             "imi", PerfCountersBuilder::PRIO_INTERESTING);
+    mds_plb.add_u64_counter(l_mds_slow_reply, "slow_reply", "Slow replies", "slr",
+                              PerfCountersBuilder::PRIO_INTERESTING);
 
     // caps msg stats
     mds_plb.add_u64_counter(l_mdss_handle_client_caps, "handle_client_caps",
diff --git a/src/mds/MDSRank.h b/src/mds/MDSRank.h
index 3df1abb5d13..f6f9a97bc6b 100644
--- a/src/mds/MDSRank.h
+++ b/src/mds/MDSRank.h
@@ -53,6 +53,7 @@ enum {
   l_mds_request,
   l_mds_reply,
   l_mds_reply_latency,
+  l_mds_slow_reply,
   l_mds_forward,
   l_mds_dir_fetch_complete,
   l_mds_dir_fetch_keys,
@@ -705,4 +706,3 @@ public:
 };
 
 #endif // MDS_RANK_H_
-
diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index 5b54e8112a4..8a53beac954 100644
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -2141,6 +2141,9 @@ void Server::early_reply(MDRequestRef& mdr, CInode *tracei, CDentry *tracedn)
   mds->logger->inc(l_mds_reply);
   utime_t lat = ceph_clock_now() - req->get_recv_stamp();
   mds->logger->tinc(l_mds_reply_latency, lat);
+  if (lat >= g_conf()->mds_op_complaint_time) {
+    mds->logger->inc(l_mds_slow_reply);
+  }
   if (client_inst.name.is_client()) {
     mds->sessionmap.hit_session(mdr->session);
   }
@@ -2199,6 +2202,9 @@ void Server::reply_client_request(MDRequestRef& mdr, const ref_t<MClientReply> &
     mds->logger->inc(l_mds_reply);
     utime_t lat = ceph_clock_now() - mdr->client_request->get_recv_stamp();
     mds->logger->tinc(l_mds_reply_latency, lat);
+    if (lat >= g_conf()->mds_op_complaint_time) {
+      mds->logger->inc(l_mds_slow_reply);
+    }
     if (session && client_inst.name.is_client()) {
       mds->sessionmap.hit_session(session);
     }