From: Radosław Zarzyński Date: Mon, 11 Apr 2022 14:47:54 +0000 (+0200) Subject: crimson/osd: implement dump_ops_in_flight X-Git-Tag: v18.0.0~947^2~33 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=9248607186805c1cffdccd55d71f5d997ea57c03;p=ceph.git crimson/osd: implement dump_ops_in_flight Signed-off-by: Radosław Zarzyński --- diff --git a/src/crimson/admin/osd_admin.cc b/src/crimson/admin/osd_admin.cc index 5bbd27393db16..b29a565056559 100644 --- a/src/crimson/admin/osd_admin.cc +++ b/src/crimson/admin/osd_admin.cc @@ -424,4 +424,32 @@ private: template std::unique_ptr make_asok_hook( crimson::osd::ShardServices&); + +/** + * An InFlightOps admin hook: dump current in-flight operations + */ +class DumpInFlightOpsHook : public AdminSocketHook { +public: + explicit DumpInFlightOpsHook(const crimson::osd::OSDOperationRegistry& op_registry) : + AdminSocketHook{"dump_ops_in_flight", "", "show the ops currently in flight"}, + op_registry(op_registry) + {} + seastar::future call(const cmdmap_t&, + std::string_view format, + ceph::bufferlist&& input) const final + { + logger().warn("{}", __func__); + unique_ptr f{Formatter::create(format, "json-pretty", "json-pretty")}; + f->open_object_section("ops_in_flight"); + op_registry.dump_client_requests(f.get()); + f->close_section(); + f->dump_int("num_ops", 0); + return seastar::make_ready_future(std::move(f)); + } +private: + const crimson::osd::OSDOperationRegistry& op_registry; +}; +template std::unique_ptr +make_asok_hook(const crimson::osd::OSDOperationRegistry& op_registry); + } // namespace crimson::admin diff --git a/src/crimson/admin/osd_admin.h b/src/crimson/admin/osd_admin.h index 61dcfced1ac13..273d17ec23c56 100644 --- a/src/crimson/admin/osd_admin.h +++ b/src/crimson/admin/osd_admin.h @@ -17,6 +17,7 @@ class InjectDataErrorHook; class InjectMDataErrorHook; class OsdStatusHook; class SendBeaconHook; +class DumpInFlightOpsHook; template std::unique_ptr make_asok_hook(Args&&... args); diff --git a/src/crimson/common/operation.cc b/src/crimson/common/operation.cc index 85575a74a0efa..65946b5ad85e1 100644 --- a/src/crimson/common/operation.cc +++ b/src/crimson/common/operation.cc @@ -67,4 +67,31 @@ void AggregateBlocker::dump_detail(ceph::Formatter *f) const f->close_section(); } +namespace detail { +void dump_time_event(const char* name, + const utime_t& timestamp, + ceph::Formatter* f) +{ + assert(f); + f->open_object_section("time_event"); + f->dump_string("name", name); + f->dump_stream("initiated_at") << timestamp; + f->close_section(); +} + +void dump_blocking_event(const char* name, + const utime_t& timestamp, + const Blocker* const blocker, + ceph::Formatter* f) +{ + assert(f); + f->open_object_section("blocking_event"); + f->dump_string("name", name); + f->dump_stream("initiated_at") << timestamp; + if (blocker) { + blocker->dump(f); + } + f->close_section(); } +} // namespace detail +} // namespace crimson diff --git a/src/crimson/common/operation.h b/src/crimson/common/operation.h index 84387356ba76c..a8bcabf918ce8 100644 --- a/src/crimson/common/operation.h +++ b/src/crimson/common/operation.h @@ -123,6 +123,16 @@ make_exception_blocking_future(Exception&& e) { seastar::make_exception_future(e)); } +namespace detail { +void dump_time_event(const char* name, + const utime_t& timestamp, + ceph::Formatter* f); +void dump_blocking_event(const char* name, + const utime_t& timestamp, + const Blocker* blocker, + ceph::Formatter* f); +} // namespace detail + /** * Provides an interface for dumping diagnostic information about * why a particular op is not making progress. @@ -214,9 +224,12 @@ struct TimeEvent : Event { void handle(T&, const Operation&) override { timestamp = ceph_clock_now(); } - private: utime_t timestamp; } internal_backend; + + void dump(ceph::Formatter *f) const { + detail::dump_time_event(typeid(T).name(), internal_backend.timestamp, f); + } }; @@ -304,6 +317,13 @@ public: const OpT& op; }; + + void dump(ceph::Formatter *f) const { + detail::dump_blocking_event(typeid(T).name(), + internal_backend.timestamp, + internal_backend.blocker, + f); + } }; virtual ~BlockerT() = default; diff --git a/src/crimson/osd/osd.cc b/src/crimson/osd/osd.cc index 8f5a7f9d473e3..a1e71de3bbe87 100644 --- a/src/crimson/osd/osd.cc +++ b/src/crimson/osd/osd.cc @@ -545,6 +545,9 @@ seastar::future<> OSD::start_asok_admin() // PG commands asok->register_command(make_asok_hook(*this)); asok->register_command(make_asok_hook(*this)); + // ops commands + asok->register_command(make_asok_hook( + std::as_const(get_shard_services().registry))); }); } diff --git a/src/crimson/osd/osd_operation.cc b/src/crimson/osd/osd_operation.cc index 0ab3c2cf74d14..90370d57b816f 100644 --- a/src/crimson/osd/osd_operation.cc +++ b/src/crimson/osd/osd_operation.cc @@ -3,9 +3,28 @@ #include "osd_operation.h" #include "common/Formatter.h" +#include "crimson/common/log.h" +#include "crimson/osd/osd_operations/client_request.h" + +namespace { + seastar::logger& logger() { + return crimson::get_logger(ceph_subsys_osd); + } +} namespace crimson::osd { +size_t OSDOperationRegistry::dump_client_requests(ceph::Formatter* f) const +{ + const auto& client_registry = + get_registry(ClientRequest::type)>(); + logger().warn("{} num_ops={}", __func__, std::size(client_registry)); + for (const auto& op : client_registry) { + op.dump(f); + } + return std::size(client_registry); +} + OperationThrottler::OperationThrottler(ConfigProxy &conf) : scheduler(crimson::osd::scheduler::make_scheduler(conf)) { diff --git a/src/crimson/osd/osd_operation.h b/src/crimson/osd/osd_operation.h index 6f6c8b04f1449..4b36852017ad3 100644 --- a/src/crimson/osd/osd_operation.h +++ b/src/crimson/osd/osd_operation.h @@ -186,10 +186,11 @@ protected: /** * Maintains a set of lists of all active ops. */ -using OSDOperationRegistry = OperationRegistryT< +struct OSDOperationRegistry : OperationRegistryT< static_cast(OperationTypeCode::last_op) - >; - +> { + size_t dump_client_requests(ceph::Formatter* f) const; +}; /** * Throttles set of currently running operations * diff --git a/src/crimson/osd/osd_operations/client_request.cc b/src/crimson/osd/osd_operations/client_request.cc index 626b82c48406a..bca3952c772d4 100644 --- a/src/crimson/osd/osd_operations/client_request.cc +++ b/src/crimson/osd/osd_operations/client_request.cc @@ -43,6 +43,10 @@ void ClientRequest::print(std::ostream &lhs) const void ClientRequest::dump_detail(Formatter *f) const { + logger().debug("{}: dumping", *this); + std::apply([f] (auto... event) { + (..., event.dump(f)); + }, tracking_events); } ClientRequest::ConnectionPipeline &ClientRequest::cp()