From: junxiang Mu <1948535941@qq.com> Date: Mon, 25 Mar 2024 08:05:38 +0000 (-0400) Subject: crimson/mgr/client.cc: daemon_health_metrics support X-Git-Tag: v20.0.0~338^2~1 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=34f209f51ff0c2b805a0ba4a6964dc3110f25139;p=ceph.git crimson/mgr/client.cc: daemon_health_metrics support Fixes: https://tracker.ceph.com/issues/63766 Signed-off-by: junxiang Mu <1948535941@qq.com> --- diff --git a/src/crimson/mgr/client.cc b/src/crimson/mgr/client.cc index 3326753ca2e..32366e63754 100644 --- a/src/crimson/mgr/client.cc +++ b/src/crimson/mgr/client.cc @@ -174,10 +174,13 @@ void Client::report() }); } +void Client::update_daemon_health(std::vector&& metrics) +{ + daemon_health_metrics = std::move(metrics); +} + void Client::_send_report() { - // TODO: implement daemon_health_metrics support - // https://tracker.ceph.com/issues/63766 gates.dispatch_in_background(__func__, *this, [this] { if (!conn) { logger().warn("cannot send report; no conn available"); @@ -196,6 +199,7 @@ void Client::_send_report() report->daemon_name = local_conf()->name.get_id(); } report->service_name = service_name; + report->daemon_health_metrics = std::move(daemon_health_metrics); local_conf().get_config_bl(last_config_bl_version, &report->config_bl, &last_config_bl_version); return conn->send(std::move(report)); diff --git a/src/crimson/mgr/client.h b/src/crimson/mgr/client.h index ae489644eaa..5d27bd2c0b3 100644 --- a/src/crimson/mgr/client.h +++ b/src/crimson/mgr/client.h @@ -8,6 +8,7 @@ #include "crimson/common/gated.h" #include "crimson/net/Dispatcher.h" #include "crimson/net/Fwd.h" +#include "mgr/DaemonHealthMetric.h" #include "mon/MgrMap.h" template using Ref = boost::intrusive_ptr; @@ -35,6 +36,7 @@ public: seastar::future<> start(); seastar::future<> stop(); void report(); + void update_daemon_health(std::vector&& metrics); private: std::optional> ms_dispatch( @@ -59,6 +61,8 @@ private: uint64_t last_config_bl_version = 0; std::string service_name, daemon_name; + std::vector daemon_health_metrics; + void _send_report(); }; diff --git a/src/crimson/osd/osd.cc b/src/crimson/osd/osd.cc index 0f19bfd7145..71ec30c1d5b 100644 --- a/src/crimson/osd/osd.cc +++ b/src/crimson/osd/osd.cc @@ -105,6 +105,7 @@ OSD::OSD(int id, uint32_t nonce, std::ignore = update_heartbeat_peers( ).then([this] { update_stats(); + mgrc->update_daemon_health(get_health_metrics()); tick_timer.arm( std::chrono::seconds(TICK_INTERVAL)); }); @@ -976,7 +977,7 @@ void OSD::handle_conf_change( const crimson::common::ConfigProxy& conf, const std::set &changed) { - if (changed.count("osd_beacon_report_interval")) { + if (changed.contains("osd_beacon_report_interval")) { beacon_timer.rearm_periodic( std::chrono::seconds(conf->osd_beacon_report_interval)); } @@ -1415,6 +1416,74 @@ seastar::future<> OSD::handle_recovery_subreq( conn, std::move(m)).second; } +vector OSD::get_health_metrics() +{ + LOG_PREFIX(OSD::get_health_metrics); + vector metrics; + + const utime_t now = ceph_clock_now(); + utime_t oldest_secs = now; + utime_t too_old = now; + too_old -= local_conf()->osd_op_complaint_time; + int slow = 0; + ClientRequest::ICRef oldest_op; + map slow_op_pools; + bool log_aggregated_slow_op = local_conf()->osd_aggregated_slow_ops_logging; + auto count_slow_ops = [&](const ClientRequest& op) { + if (op.get_started() < too_old) { + std::stringstream ss; + ss << "slow request "; + op.print(ss); + ss << " initiated " + << op.get_started(); + WARN("{}", ss.str()); + if (log_aggregated_slow_op) { + uint64_t pool_id = op.get_pgid().pgid.m_pool; + if (pool_id > 0 && pool_id <= (uint64_t) osdmap->get_pool_max()) { + slow_op_pools[pool_id]++; + } + } else { + clog->warn() << ss.str(); + } + ++slow; + if (!oldest_op || op.get_started() < oldest_op->get_started()) { + oldest_op = &op; + } + } + }; + + auto& op_registry = get_shard_services().get_registry(); + op_registry.visit_ops_in_flight(count_slow_ops); + if (slow) { + std::stringstream ss; + ss << __func__ << " reporting " << slow << " slow ops, oldest is "; + ceph_assert(oldest_op); + oldest_op->print(ss); + ERROR("{}", ss.str()); + if (log_aggregated_slow_op && !slow_op_pools.empty()) { + std::stringstream ss; + auto slow_pool_it = std::max_element(slow_op_pools.begin(), slow_op_pools.end(), + [](std::pair p1, std::pair p2) { + return p1.second < p2.second; + }); + if (osdmap->get_pools().find(slow_pool_it->first) != osdmap->get_pools().end()) { + string pool_name = osdmap->get_pool_name(slow_pool_it->first); + ss << "slow requests (most affected pool [ '" + << pool_name + << "' : " + << slow_pool_it->second + << " ])"; + } + WARN("{}", ss.str()); + clog->warn() << ss.str(); + } + oldest_secs = now - oldest_op->get_started(); + metrics.emplace_back(daemon_metric::SLOW_OPS, slow, oldest_secs); + } + + return metrics; +} + bool OSD::should_restart() const { LOG_PREFIX(OSD::should_restart); diff --git a/src/crimson/osd/osd.h b/src/crimson/osd/osd.h index 1a84ccd6a3f..1cf88bea5d9 100644 --- a/src/crimson/osd/osd.h +++ b/src/crimson/osd/osd.h @@ -234,6 +234,8 @@ private: crimson::net::ConnectionRef conn, Ref m); + std::vector get_health_metrics(); + private: crimson::common::gate_per_shard gate; diff --git a/src/crimson/osd/osd_operation.cc b/src/crimson/osd/osd_operation.cc index 8442b605d39..0d7ad313865 100644 --- a/src/crimson/osd/osd_operation.cc +++ b/src/crimson/osd/osd_operation.cc @@ -136,6 +136,17 @@ size_t OSDOperationRegistry::dump_slowest_historic_client_requests(ceph::Formatt return ops_count; } +void OSDOperationRegistry::visit_ops_in_flight(std::function&& visit) +{ + const auto& client_registry = + get_registry(OperationTypeCode::client_request)>(); + auto it = std::begin(client_registry); + for (; it != std::end(client_registry); ++it) { + const auto& fastest_historic_op = static_cast(*it); + visit(fastest_historic_op); + } +} + OperationThrottler::OperationThrottler(ConfigProxy &conf) : scheduler(crimson::osd::scheduler::make_scheduler(conf)) { diff --git a/src/crimson/osd/osd_operation.h b/src/crimson/osd/osd_operation.h index fd8b049c0bf..0e36264c1fa 100644 --- a/src/crimson/osd/osd_operation.h +++ b/src/crimson/osd/osd_operation.h @@ -265,6 +265,7 @@ struct OSDOperationRegistry : OperationRegistryT< size_t dump_historic_client_requests(ceph::Formatter* f) const; size_t dump_slowest_historic_client_requests(ceph::Formatter* f) const; + void visit_ops_in_flight(std::function&& visit); private: size_t num_recent_ops = 0;