mds: add a perf counter to record slow replies

Though we have MDS_HEALTH_SLOW_METADATA_IO and MDS_HEALTH_SLOW_REQUEST health alert, but those are not
precise nor accumulated. With slow reply counter compared to reply counter, we can find out the ratio
of slow requests through perf dump.

Fixes: https://tracker.ceph.com/issues/55126
Signed-off-by: haoyixing <haoyixing@kuaishou.com>
(cherry picked from commit e8e3b307c87dc9eec2d087b396c0e7a0248b4f1d)

diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index 886a379dd7ae9cbe1ea4169e19460787dfdde598..b44870f807a2de709851975f3aea72cd2ccf155e 100644 (file)
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -3346,6 +3346,8 @@ void MDSRank::create_logger()
                             "exi", PerfCountersBuilder::PRIO_INTERESTING);
     mds_plb.add_u64_counter(l_mds_imported_inodes, "imported_inodes", "Imported inodes",
                             "imi", PerfCountersBuilder::PRIO_INTERESTING);
+    mds_plb.add_u64_counter(l_mds_slow_reply, "slow_reply", "Slow replies", "slr",
+                              PerfCountersBuilder::PRIO_INTERESTING);
 
     // caps msg stats
     mds_plb.add_u64_counter(l_mdss_handle_client_caps, "handle_client_caps",

diff --git a/src/mds/MDSRank.h b/src/mds/MDSRank.h
index 784aa574158819ee610b6454cc15b066d25deb67..b428f066b97619f1001543edcc6370876b20f100 100644 (file)
--- a/src/mds/MDSRank.h
+++ b/src/mds/MDSRank.h
@@ -53,6 +53,7 @@ enum {
   l_mds_request,
   l_mds_reply,
   l_mds_reply_latency,
+  l_mds_slow_reply,
   l_mds_forward,
   l_mds_dir_fetch,
   l_mds_dir_commit,
@@ -704,4 +705,3 @@ public:
 };
 
 #endif // MDS_RANK_H_
-

diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index e6f7e864fb41075870512325cda987ce7201e0c6..80e5cdf72a3e9663f283e099771fae51d7c9f74f 100644 (file)
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -2137,6 +2137,9 @@ void Server::early_reply(MDRequestRef& mdr, CInode *tracei, CDentry *tracedn)
   mds->logger->inc(l_mds_reply);
   utime_t lat = ceph_clock_now() - req->get_recv_stamp();
   mds->logger->tinc(l_mds_reply_latency, lat);
+  if (lat >= g_conf()->mds_op_complaint_time) {
+    mds->logger->inc(l_mds_slow_reply);
+  }
   if (client_inst.name.is_client()) {
     mds->sessionmap.hit_session(mdr->session);
   }
@@ -2195,6 +2198,9 @@ void Server::reply_client_request(MDRequestRef& mdr, const ref_t<MClientReply> &
     mds->logger->inc(l_mds_reply);
     utime_t lat = ceph_clock_now() - mdr->client_request->get_recv_stamp();
     mds->logger->tinc(l_mds_reply_latency, lat);
+    if (lat >= g_conf()->mds_op_complaint_time) {
+      mds->logger->inc(l_mds_slow_reply);
+    }
     if (session && client_inst.name.is_client()) {
       mds->sessionmap.hit_session(session);
     }