From ff20d130e3397ee69e5f849c05a367465af61f31 Mon Sep 17 00:00:00 2001 From: Ronen Friedman Date: Thu, 14 Dec 2023 08:27:13 -0600 Subject: [PATCH] osd/scrub: add a basic set of performance counters Add a labeled set of performance counters, with the labels selecting one of four copies of the counters - one per each scrub level and pool type combination. Inside the Scrubber, the relevant set of counters is selected when the scrub is initialized. Signed-off-by: Ronen Friedman --- src/osd/osd_perf_counters.cc | 44 ++++++++++++++++++++++++++++++++ src/osd/osd_perf_counters.h | 41 ++++++++++++++++++++++++++++++ src/osd/scrubber/osd_scrub.cc | 40 ++++++++++++++++++++++++++++- src/osd/scrubber/osd_scrub.h | 48 ++++++++++++++++++++++++++++++++++- 4 files changed, 171 insertions(+), 2 deletions(-) diff --git a/src/osd/osd_perf_counters.cc b/src/osd/osd_perf_counters.cc index 3e14244329eb1..af8c62fb30427 100644 --- a/src/osd/osd_perf_counters.cc +++ b/src/osd/osd_perf_counters.cc @@ -319,6 +319,25 @@ PerfCounters *build_osd_logger(CephContext *cct) { osd_plb.add_u64_counter( l_osd_pg_biginfo, "osd_pg_biginfo", "PG updated its biginfo attr"); + /// scrub's replicas reservation time/#replicas histogram + PerfHistogramCommon::axis_config_d rsrv_hist_x_axis_config{ + "number of replicas", + PerfHistogramCommon::SCALE_LINEAR, + 0, ///< Start at 0 + 1, ///< Quantization unit is 1 + 8, ///< 9 OSDs in the active set + }; + PerfHistogramCommon::axis_config_d rsrv_hist_y_axis_config{ + "duration", + PerfHistogramCommon::SCALE_LOG2, ///< Request size in logarithmic scale + 0, ///< Start at 0 + 250'000, ///< 250us granularity + 10, ///< should be enough + }; + osd_plb.add_u64_counter_histogram( + l_osd_scrub_reservation_dur_hist, "scrub_resrv_repnum_vs_duration", + rsrv_hist_x_axis_config, rsrv_hist_y_axis_config, "Histogram of scrub replicas reservation duration"); + return osd_plb.create_perf_counters(); } @@ -360,3 +379,28 @@ PerfCounters *build_recoverystate_perf(CephContext *cct) { return rs_perf.create_perf_counters(); } + + +PerfCounters *build_scrub_labeled_perf(CephContext *cct, std::string label) +{ + // the labels matrix is: + // X // maybe later we'll add + PerfCountersBuilder scrub_perf(cct, label, scrbcnt_first, scrbcnt_last); + + scrub_perf.set_prio_default(PerfCountersBuilder::PRIO_INTERESTING); + + scrub_perf.add_u64_counter(scrbcnt_started, "num_scrubs_started", "scrubs attempted count"); + scrub_perf.add_u64_counter(scrbcnt_active_started, "num_scrubs_past_reservation", "scrubs count"); + scrub_perf.add_u64_counter(scrbcnt_failed, "failed_scrubs", "failed scrubs count"); + scrub_perf.add_u64_counter(scrbcnt_successful, "successful_scrubs", "successful scrubs count"); + scrub_perf.add_time_avg(scrbcnt_failed_elapsed, "failed_scrubs_elapsed", "time to scrub failure"); + scrub_perf.add_time_avg(scrbcnt_successful_elapsed, "successful_scrubs_elapsed", "time to scrub completion"); + + scrub_perf.add_u64_counter(scrbcnt_preempted, "preemptions", "preemptions on scrubs"); + scrub_perf.add_u64_counter(scrbcnt_chunks_selected, "chunk_selected", "chunk selection during scrubs"); + scrub_perf.add_u64_counter(scrbcnt_chunks_busy, "chunk_busy", "chunk busy during scrubs"); + scrub_perf.add_u64_counter(scrbcnt_blocked, "locked_object", "waiting on locked object events"); + scrub_perf.add_u64_counter(scrbcnt_write_blocked, "write_blocked_by_scrub", "write blocked by scrub"); + + return scrub_perf.create_perf_counters(); +} diff --git a/src/osd/osd_perf_counters.h b/src/osd/osd_perf_counters.h index 2445a9dc2c388..1f65e953d3375 100644 --- a/src/osd/osd_perf_counters.h +++ b/src/osd/osd_perf_counters.h @@ -5,6 +5,7 @@ #include "include/common_fwd.h" #include "common/perf_counters.h" +#include "common/perf_counters_key.h" enum { l_osd_first = 10000, @@ -131,6 +132,10 @@ enum { l_osd_pg_fastinfo, l_osd_pg_biginfo, + // scrubber related. Here, as the rest of the scrub counters + // are labeled, and histograms do not fully support labels. + l_osd_scrub_reservation_dur_hist, + l_osd_last, }; @@ -174,3 +179,39 @@ enum { }; PerfCounters *build_recoverystate_perf(CephContext *cct); + +// Scrubber perf counters. There are four sets (shallow vs. deep, +// EC vs. replicated) of these counters: +enum { + scrbcnt_first = 20500, + + // -- basic statistics -- + /// The number of times we started a scrub + scrbcnt_started, + /// # scrubs that got past replicas reservation + scrbcnt_active_started, + /// # successful scrubs + scrbcnt_successful, + /// time to complete a successful scrub + scrbcnt_successful_elapsed, + /// # failed scrubs + scrbcnt_failed, + /// time for a scrub to fail + scrbcnt_failed_elapsed, + + // -- interruptions of various types + /// # preemptions + scrbcnt_preempted, + /// # chunks selection performed + scrbcnt_chunks_selected, + /// # busy chunks + scrbcnt_chunks_busy, + /// # waiting on object events + scrbcnt_blocked, + /// # write blocked by the scrub + scrbcnt_write_blocked, + + scrbcnt_last, +}; + +PerfCounters *build_scrub_labeled_perf(CephContext *cct, std::string label); diff --git a/src/osd/scrubber/osd_scrub.cc b/src/osd/scrubber/osd_scrub.cc index f11ddd2737ca4..5c4eebba29536 100644 --- a/src/osd/scrubber/osd_scrub.cc +++ b/src/osd/scrubber/osd_scrub.cc @@ -4,6 +4,7 @@ #include "./osd_scrub.h" #include "osd/OSD.h" +#include "osd/osd_perf_counters.h" #include "osdc/Objecter.h" #include "pg_scrubber.h" @@ -37,7 +38,14 @@ OsdScrub::OsdScrub( , m_queue{cct, m_osd_svc} , m_log_prefix{fmt::format("osd.{} osd-scrub:", m_osd_svc.get_nodeid())} , m_load_tracker{cct, conf, m_osd_svc.get_nodeid()} -{} +{ + create_scrub_perf_counters(); +} + +OsdScrub::~OsdScrub() +{ + destroy_scrub_perf_counters(); +} std::ostream& OsdScrub::gen_prefix(std::ostream& out, std::string_view fn) const { @@ -377,6 +385,36 @@ std::chrono::milliseconds OsdScrub::scrub_sleep_time( return std::max(extended_sleep, regular_sleep_period); } + +// ////////////////////////////////////////////////////////////////////////// // +// scrub-related performance counters + +void OsdScrub::create_scrub_perf_counters() +{ + auto idx = perf_counters_indices.begin(); + // create a separate set for each pool type & scrub level + for (const auto& label : perf_labels) { + PerfCounters* counters = build_scrub_labeled_perf(cct, label); + cct->get_perfcounters_collection()->add(counters); + m_perf_counters[*idx++] = counters; + } +} + +void OsdScrub::destroy_scrub_perf_counters() +{ + for (const auto& [label, counters] : m_perf_counters) { + std::ignore = label; + cct->get_perfcounters_collection()->remove(counters); + delete counters; + } + m_perf_counters.clear(); +} + +PerfCounters* OsdScrub::get_perf_counters(int pool_type, scrub_level_t level) +{ + return m_perf_counters[pc_index_t{level, pool_type}]; +} + // ////////////////////////////////////////////////////////////////////////// // // forwarders to the queue diff --git a/src/osd/scrubber/osd_scrub.h b/src/osd/scrubber/osd_scrub.h index dae11f860011c..ce7b8524d69cb 100644 --- a/src/osd/scrubber/osd_scrub.h +++ b/src/osd/scrubber/osd_scrub.h @@ -5,6 +5,7 @@ #include #include "osd/osd_types_fmt.h" +#include "osd/osd_perf_counters.h" #include "osd/scrubber/osd_scrub_sched.h" #include "osd/scrubber/scrub_resources.h" #include "osd/scrubber_common.h" @@ -26,7 +27,7 @@ class OsdScrub { Scrub::ScrubSchedListener& osd_svc, const ceph::common::ConfigProxy& config); - ~OsdScrub() = default; + ~OsdScrub(); // note: public, as accessed by the dout macros std::ostream& gen_prefix(std::ostream& out, std::string_view fn) const; @@ -162,6 +163,10 @@ class OsdScrub { */ std::optional update_load_average(); + // the scrub performance counters collections + // --------------------------------------------------------------- + PerfCounters* get_perf_counters(int pool_type, scrub_level_t level); + private: CephContext* cct; Scrub::ScrubSchedListener& m_osd_svc; @@ -238,4 +243,45 @@ class OsdScrub { std::ostream& gen_prefix(std::ostream& out, std::string_view fn) const; }; LoadTracker m_load_tracker; + + // the scrub performance counters collections + // --------------------------------------------------------------- + + // indexed by scrub level & pool type + + using pc_index_t = std::pair; + // easy way to loop over the counter sets. Order must match the + // perf_labels vector + static inline std::array perf_counters_indices = { + pc_index_t{scrub_level_t::shallow, pg_pool_t::TYPE_REPLICATED}, + pc_index_t{scrub_level_t::deep, pg_pool_t::TYPE_REPLICATED}, + pc_index_t{scrub_level_t::shallow, pg_pool_t::TYPE_ERASURE}, + pc_index_t{scrub_level_t::deep, pg_pool_t::TYPE_ERASURE}}; + + std::map m_perf_counters; + + // the labels matrix is: X + static inline std::vector perf_labels = { + ceph::perf_counters::key_create( + "osd_scrub_sh_repl", + {{"level", "shallow"}, {"pooltype", "replicated"}}), + ceph::perf_counters::key_create( + "osd_scrub_dp_repl", + {{"level", "deep"}, {"pooltype", "replicated"}}), + ceph::perf_counters::key_create( + "osd_scrub_sh_ec", + {{"level", "shallow"}, {"pooltype", "ec"}}), + ceph::perf_counters::key_create( + "osd_scrub_dp_ec", + {{"level", "deep"}, {"pooltype", "ec"}})}; + + /** + * create 4 sets of performance counters (for shallow vs. deep, + * replicated vs. erasure pools). Add them to the cct, but also maintain + * a separate map of the counters, indexed by the pool type and scrub level. + */ + void create_scrub_perf_counters(); + + // 'remove' the counters from the cct, and delete them + void destroy_scrub_perf_counters(); }; -- 2.39.5