From e941afdac9ab0a5fbed0912079f0f58ccc76f35f Mon Sep 17 00:00:00 2001 From: Samuel Just Date: Thu, 6 Apr 2023 14:15:02 -0700 Subject: [PATCH] osd/: add counters for queue latency for PGRecovery[Context] Signed-off-by: Samuel Just --- src/osd/OSD.h | 1 + src/osd/osd_perf_counters.cc | 9 +++++++++ src/osd/osd_perf_counters.h | 3 +++ src/osd/scheduler/OpSchedulerItem.cc | 6 ++++++ src/osd/scheduler/OpSchedulerItem.h | 4 ++++ 5 files changed, 23 insertions(+) diff --git a/src/osd/OSD.h b/src/osd/OSD.h index 96393b0863d..9f8ac6ea7f6 100644 --- a/src/osd/OSD.h +++ b/src/osd/OSD.h @@ -1567,6 +1567,7 @@ protected: friend class ceph::osd::scheduler::PGOpItem; friend class ceph::osd::scheduler::PGPeeringItem; friend class ceph::osd::scheduler::PGRecovery; + friend class ceph::osd::scheduler::PGRecoveryContext; friend class ceph::osd::scheduler::PGRecoveryMsg; friend class ceph::osd::scheduler::PGDelete; diff --git a/src/osd/osd_perf_counters.cc b/src/osd/osd_perf_counters.cc index b1c1413bf61..1767cbec1ed 100644 --- a/src/osd/osd_perf_counters.cc +++ b/src/osd/osd_perf_counters.cc @@ -187,6 +187,15 @@ PerfCounters *build_osd_logger(CephContext *cct) { "l_osd_recovery_scan_queue_latency", "MOSDPGScan queue latency"); + osd_plb.add_time_avg( + l_osd_recovery_queue_lat, + "l_osd_recovery_queue_latency", + "PGRecovery queue latency"); + osd_plb.add_time_avg( + l_osd_recovery_context_queue_lat, + "l_osd_recovery_context_queue_latency", + "PGRecoveryContext queue latency"); + osd_plb.add_u64(l_osd_loadavg, "loadavg", "CPU load"); osd_plb.add_u64( l_osd_cached_crc, "cached_crc", "Total number getting crc from crc_cache"); diff --git a/src/osd/osd_perf_counters.h b/src/osd/osd_perf_counters.h index f8af01e0e4f..93fef68feb5 100644 --- a/src/osd/osd_perf_counters.h +++ b/src/osd/osd_perf_counters.h @@ -65,6 +65,9 @@ enum { l_osd_recovery_backfill_remove_queue_lat, l_osd_recovery_scan_queue_lat, + l_osd_recovery_queue_lat, + l_osd_recovery_context_queue_lat, + l_osd_loadavg, l_osd_cached_crc, l_osd_cached_crc_adjusted, diff --git a/src/osd/scheduler/OpSchedulerItem.cc b/src/osd/scheduler/OpSchedulerItem.cc index d1df4320057..755bf2d3175 100644 --- a/src/osd/scheduler/OpSchedulerItem.cc +++ b/src/osd/scheduler/OpSchedulerItem.cc @@ -224,6 +224,9 @@ void PGRecovery::run( PGRef& pg, ThreadPool::TPHandle &handle) { + osd->logger->tinc( + l_osd_recovery_queue_lat, + time_queued - ceph_clock_now()); osd->do_recovery(pg.get(), epoch_queued, reserved_pushes, priority, handle); pg->unlock(); } @@ -234,6 +237,9 @@ void PGRecoveryContext::run( PGRef& pg, ThreadPool::TPHandle &handle) { + osd->logger->tinc( + l_osd_recovery_context_queue_lat, + time_queued - ceph_clock_now()); c.release()->complete(handle); pg->unlock(); } diff --git a/src/osd/scheduler/OpSchedulerItem.h b/src/osd/scheduler/OpSchedulerItem.h index df0c2e1320c..3ae176e2b55 100644 --- a/src/osd/scheduler/OpSchedulerItem.h +++ b/src/osd/scheduler/OpSchedulerItem.h @@ -500,6 +500,7 @@ class PGScrubChunkIsFree : public PGScrubItem { }; class PGRecovery : public PGOpQueueable { + utime_t time_queued; epoch_t epoch_queued; uint64_t reserved_pushes; int priority; @@ -510,6 +511,7 @@ public: uint64_t reserved_pushes, int priority) : PGOpQueueable(pg), + time_queued(ceph_clock_now()), epoch_queued(epoch_queued), reserved_pushes(reserved_pushes), priority(priority) {} @@ -530,6 +532,7 @@ public: }; class PGRecoveryContext : public PGOpQueueable { + utime_t time_queued; std::unique_ptr> c; epoch_t epoch; int priority; @@ -538,6 +541,7 @@ public: GenContext *c, epoch_t epoch, int priority) : PGOpQueueable(pgid), + time_queued(ceph_clock_now()), c(c), epoch(epoch), priority(priority) {} std::ostream &print(std::ostream &rhs) const final { return rhs << "PGRecoveryContext(pgid=" << get_pgid() -- 2.39.5