From e8e3b307c87dc9eec2d087b396c0e7a0248b4f1d Mon Sep 17 00:00:00 2001 From: haoyixing Date: Fri, 25 Mar 2022 03:02:13 +0000 Subject: [PATCH] mds: add a perf counter to record slow replies Though we have MDS_HEALTH_SLOW_METADATA_IO and MDS_HEALTH_SLOW_REQUEST health alert, but those are not precise nor accumulated. With slow reply counter compared to reply counter, we can find out the ratio of slow requests through perf dump. Fixes: https://tracker.ceph.com/issues/55126 Signed-off-by: haoyixing --- src/mds/MDSRank.cc | 2 ++ src/mds/MDSRank.h | 2 +- src/mds/Server.cc | 6 ++++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc index 053799c0df07..5eccd0acd75c 100644 --- a/src/mds/MDSRank.cc +++ b/src/mds/MDSRank.cc @@ -3350,6 +3350,8 @@ void MDSRank::create_logger() "exi", PerfCountersBuilder::PRIO_INTERESTING); mds_plb.add_u64_counter(l_mds_imported_inodes, "imported_inodes", "Imported inodes", "imi", PerfCountersBuilder::PRIO_INTERESTING); + mds_plb.add_u64_counter(l_mds_slow_reply, "slow_reply", "Slow replies", "slr", + PerfCountersBuilder::PRIO_INTERESTING); // caps msg stats mds_plb.add_u64_counter(l_mdss_handle_client_caps, "handle_client_caps", diff --git a/src/mds/MDSRank.h b/src/mds/MDSRank.h index 3df1abb5d13a..f6f9a97bc6b8 100644 --- a/src/mds/MDSRank.h +++ b/src/mds/MDSRank.h @@ -53,6 +53,7 @@ enum { l_mds_request, l_mds_reply, l_mds_reply_latency, + l_mds_slow_reply, l_mds_forward, l_mds_dir_fetch_complete, l_mds_dir_fetch_keys, @@ -705,4 +706,3 @@ public: }; #endif // MDS_RANK_H_ - diff --git a/src/mds/Server.cc b/src/mds/Server.cc index 5b54e8112a41..8a53beac9543 100644 --- a/src/mds/Server.cc +++ b/src/mds/Server.cc @@ -2141,6 +2141,9 @@ void Server::early_reply(MDRequestRef& mdr, CInode *tracei, CDentry *tracedn) mds->logger->inc(l_mds_reply); utime_t lat = ceph_clock_now() - req->get_recv_stamp(); mds->logger->tinc(l_mds_reply_latency, lat); + if (lat >= g_conf()->mds_op_complaint_time) { + mds->logger->inc(l_mds_slow_reply); + } if (client_inst.name.is_client()) { mds->sessionmap.hit_session(mdr->session); } @@ -2199,6 +2202,9 @@ void Server::reply_client_request(MDRequestRef& mdr, const ref_t & mds->logger->inc(l_mds_reply); utime_t lat = ceph_clock_now() - mdr->client_request->get_recv_stamp(); mds->logger->tinc(l_mds_reply_latency, lat); + if (lat >= g_conf()->mds_op_complaint_time) { + mds->logger->inc(l_mds_slow_reply); + } if (session && client_inst.name.is_client()) { mds->sessionmap.hit_session(session); } -- 2.47.3