From c826e155538ea29f2bad1b251e51390b75db8496 Mon Sep 17 00:00:00 2001 From: Igor Fedotov Date: Mon, 18 Sep 2023 19:21:18 +0300 Subject: [PATCH] common/perf_counters: track maximum increment value for LONGRUNAVG perf counters This e.g. allows to track maximum observed latencies along with average values which might be helpful for bottleneck detection. Signed-off-by: Igor Fedotov --- src/common/perf_counters.cc | 106 ++++++++++++++++++++++++++++++++---- src/common/perf_counters.h | 25 +++++++-- src/test/perf_counters.cc | 7 +++ 3 files changed, 122 insertions(+), 16 deletions(-) diff --git a/src/common/perf_counters.cc b/src/common/perf_counters.cc index fe9b9b1876534..3b6ee3b0d18f6 100644 --- a/src/common/perf_counters.cc +++ b/src/common/perf_counters.cc @@ -183,6 +183,31 @@ void PerfCounters::inc(int idx, uint64_t amt) } } +void PerfCounters::inc_with_max(int idx, uint64_t amt) +{ +#ifndef WITH_CRIMSON + if (!m_cct->_conf->perf) + return; +#endif + + ceph_assert(idx > m_lower_bound); + ceph_assert(idx < m_upper_bound); + perf_counter_data_any_d& data(m_data[idx - m_lower_bound - 1]); + if (!(data.type & PERFCOUNTER_U64)) + return; + if (data.type & PERFCOUNTER_LONGRUNAVG) { + data.avgcount++; + data.u64 += amt; + uint64_t m; + do { + m = data.max_u64_inc.load(); + } while(amt > m && !data.max_u64_inc.compare_exchange_weak(m, amt)); + data.avgcount2++; + } else { + data.u64 += amt; + } +} + void PerfCounters::dec(int idx, uint64_t amt) { #ifndef WITH_CRIMSON @@ -259,6 +284,32 @@ void PerfCounters::tinc(int idx, utime_t amt) } } +void PerfCounters::tinc_with_max(int idx, utime_t amt) +{ +#ifndef WITH_CRIMSON + if (!m_cct->_conf->perf) + return; +#endif + + ceph_assert(idx > m_lower_bound); + ceph_assert(idx < m_upper_bound); + perf_counter_data_any_d& data(m_data[idx - m_lower_bound - 1]); + if (!(data.type & PERFCOUNTER_TIME)) + return; + if (data.type & PERFCOUNTER_LONGRUNAVG) { + uint64_t new_m = amt.to_nsec(); + data.avgcount++; + data.u64 += new_m; + uint64_t m; + do { + m = data.max_u64_inc.load(); + } while(new_m > m && !data.max_u64_inc.compare_exchange_weak(m, new_m)); + data.avgcount2++; + } else { + data.u64 += amt.to_nsec(); + } +} + void PerfCounters::tinc(int idx, ceph::timespan amt) { #ifndef WITH_CRIMSON @@ -280,6 +331,32 @@ void PerfCounters::tinc(int idx, ceph::timespan amt) } } +void PerfCounters::tinc_with_max(int idx, ceph::timespan amt) +{ +#ifndef WITH_CRIMSON + if (!m_cct->_conf->perf) + return; +#endif + + ceph_assert(idx > m_lower_bound); + ceph_assert(idx < m_upper_bound); + perf_counter_data_any_d& data(m_data[idx - m_lower_bound - 1]); + if (!(data.type & PERFCOUNTER_TIME)) + return; + if (data.type & PERFCOUNTER_LONGRUNAVG) { + uint64_t new_m = amt.count(); + data.avgcount++; + data.u64 += new_m; + uint64_t m; + do { + m = data.max_u64_inc.load(); + } while(new_m > m && !data.max_u64_inc.compare_exchange_weak(m, new_m)); + data.avgcount2++; + } else { + data.u64 += amt.count(); + } +} + void PerfCounters::tset(int idx, utime_t amt) { #ifndef WITH_CRIMSON @@ -361,8 +438,7 @@ pair PerfCounters::get_tavg_ns(int idx) const return make_pair(0, 0); if (!(data.type & PERFCOUNTER_LONGRUNAVG)) return make_pair(0, 0); - pair a = data.read_avg(); - return make_pair(a.second, a.first); + return data.read_avg(); } void PerfCounters::reset() @@ -479,17 +555,27 @@ void PerfCounters::dump_formatted_generic(Formatter *f, bool schema, } else { if (d->type & PERFCOUNTER_LONGRUNAVG) { Formatter::ObjectSection longrunavg_section{*f, d->name}; - pair a = d->read_avg(); + std::tuple a = d->read_avg_ex(); if (d->type & PERFCOUNTER_U64) { - f->dump_unsigned("avgcount", a.second); - f->dump_unsigned("sum", a.first); + f->dump_unsigned("sum", std::get<0>(a)); + f->dump_unsigned("avgcount", std::get<1>(a)); + uint64_t max = std::get<2>(a); + if (max != 0) { + f->dump_unsigned("max_inc", std::get<2>(a)); + } } else if (d->type & PERFCOUNTER_TIME) { - f->dump_unsigned("avgcount", a.second); + uint64_t sum_ns = std::get<0>(a); + uint64_t count = std::get<1>(a); + f->dump_unsigned("avgcount", count); f->dump_format_unquoted("sum", "%" PRId64 ".%09" PRId64, - a.first / 1000000000ull, - a.first % 1000000000ull); - uint64_t count = a.second; - uint64_t sum_ns = a.first; + sum_ns / 1000000000ull, + sum_ns % 1000000000ull); + uint64_t max_ns = std::get<2>(a); + if (max_ns != 0) { + f->dump_format_unquoted("max_inc", "%" PRId64 ".%09" PRId64, + max_ns / 1000000000ull, + max_ns % 1000000000ull); + } if (count) { uint64_t avg_ns = sum_ns / count; f->dump_format_unquoted("avgtime", "%" PRId64 ".%09" PRId64, diff --git a/src/common/perf_counters.h b/src/common/perf_counters.h index 23b32d5a449a4..96cb1e58f777b 100644 --- a/src/common/perf_counters.h +++ b/src/common/perf_counters.h @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -178,12 +179,10 @@ public: description(other.description), nick(other.nick), type(other.type), - unit(other.unit), - u64(other.u64.load()) { - auto a = other.read_avg(); - u64 = a.first; - avgcount = a.second; - avgcount2 = a.second; + unit(other.unit) { + std::tie(u64, avgcount, max_u64_inc) = other.read_avg_ex(); + avgcount2 = avgcount.load(); + if (other.histogram) { histogram.reset(new PerfHistogram<>(*other.histogram)); } @@ -196,6 +195,7 @@ public: enum perfcounter_type_d type; enum unit_t unit; std::atomic u64 = { 0 }; + std::atomic max_u64_inc = { 0 }; std::atomic avgcount = { 0 }; std::atomic avgcount2 = { 0 }; std::unique_ptr> histogram; @@ -204,6 +204,7 @@ public: { if (type != PERFCOUNTER_U64) { u64 = 0; + max_u64_inc = 0; avgcount = 0; avgcount2 = 0; } @@ -223,6 +224,15 @@ public: } while (avgcount != count); return { sum, count }; } + std::tuple read_avg_ex() const { + uint64_t _sum, _count, _max; + do { + _count = avgcount2; + _sum = u64; + _max = max_u64_inc; + } while (avgcount != _count); + return { _sum, _count, _max }; + } }; template @@ -244,6 +254,7 @@ public: ~PerfCounters(); void inc(int idx, uint64_t v = 1); + void inc_with_max(int idx, uint64_t v = 1); void dec(int idx, uint64_t v = 1); void set(int idx, uint64_t v); uint64_t get(int idx) const; @@ -252,6 +263,8 @@ public: void tset(int idx, ceph::timespan v); void tinc(int idx, utime_t v); void tinc(int idx, ceph::timespan v); + void tinc_with_max(int idx, utime_t v); + void tinc_with_max(int idx, ceph::timespan v); utime_t tget(int idx) const; void hinc(int idx, int64_t x, int64_t y); diff --git a/src/test/perf_counters.cc b/src/test/perf_counters.cc index b75e6a508257a..c45bd6fdb31ae 100644 --- a/src/test/perf_counters.cc +++ b/src/test/perf_counters.cc @@ -123,6 +123,13 @@ TEST(PerfCounters, SinglePerfCounters) { ASSERT_EQ(sd("{\"test_perfcounter_1\":{\"element1\":1,\"element2\":0.500000000," "\"element3\":{\"avgcount\":3,\"sum\":120.000000000,\"avgtime\":40.000000000}}}"), msg); + fake_pf->reset(); + fake_pf->tinc_with_max(TEST_PERFCOUNTERS1_ELEMENT_3, utime_t(100, 0)); + fake_pf->tinc_with_max(TEST_PERFCOUNTERS1_ELEMENT_3, utime_t(50, 0)); + ASSERT_EQ("", client.do_request("{ \"prefix\": \"perf dump\", \"format\": \"json\" }", &msg)); + ASSERT_EQ(sd("{\"test_perfcounter_1\":{\"element1\":1," + "\"element2\":0.000000000,\"element3\":{\"avgcount\":2,\"sum\":150.000000000,\"max_inc\":100.000000000,\"avgtime\":75.000000000}}}"), msg); + fake_pf->reset(); msg.clear(); ASSERT_EQ("", client.do_request("{ \"prefix\": \"perf dump\", \"format\": \"json\" }", &msg)); -- 2.39.5