template std::unique_ptr<AdminSocketHook>
make_asok_hook<DumpHistoricOpsHook>(const crimson::osd::OSDOperationRegistry& op_registry);
+
+class DumpSlowestHistoricOpsHook : public AdminSocketHook {
+public:
+ explicit DumpSlowestHistoricOpsHook(const crimson::osd::OSDOperationRegistry& op_registry) :
+ AdminSocketHook{"dump_historic_slow_ops", "", "show slowest recent ops"},
+ op_registry(op_registry)
+ {}
+ seastar::future<tell_result_t> call(const cmdmap_t&,
+ std::string_view format,
+ ceph::bufferlist&& input) const final
+ {
+ logger().warn("{}", __func__);
+ unique_ptr<Formatter> f{Formatter::create(format, "json-pretty", "json-pretty")};
+ f->open_object_section("historic_slow_ops");
+ op_registry.dump_slowest_historic_client_requests(f.get());
+ f->close_section();
+ f->dump_int("num_ops", 0);
+ return seastar::make_ready_future<tell_result_t>(std::move(f));
+ }
+private:
+ const crimson::osd::OSDOperationRegistry& op_registry;
+};
+template std::unique_ptr<AdminSocketHook>
+make_asok_hook<DumpSlowestHistoricOpsHook>(const crimson::osd::OSDOperationRegistry& op_registry);
+
} // namespace crimson::admin
/* add_ref= */ false
};
});
+ last_of_recents = std::end(historic_registry);
// to_ref_down is going off
}
-void OSDOperationRegistry::put_historic(const ClientRequest& op) {
+OSDOperationRegistry::OSDOperationRegistry()
+{
+ constexpr auto historic_reg_index =
+ static_cast<size_t>(OperationTypeCode::historic_client_request);
+ auto& historic_registry = get_registry<historic_reg_index>();
+ last_of_recents = std::begin(historic_registry);
+}
+
+static auto get_duration(const ClientRequest& client_request)
+{
+ // TODO: consider enhancing `CompletionEvent` with computing duration
+ // once -- when it's enetered.
+ return client_request.get_completed() - client_request.get_started();
+}
+
+void OSDOperationRegistry::put_historic(const ClientRequest& op)
+{
// unlink the op from the client request registry. this is a part of
// the re-link procedure. finally it will be in historic registry.
constexpr auto client_reg_index =
static_cast<size_t>(OperationTypeCode::historic_client_request);
auto& client_registry = get_registry<client_reg_index>();
auto& historic_registry = get_registry<historic_reg_index>();
-
historic_registry.splice(std::end(historic_registry),
client_registry,
client_registry.iterator_to(op));
// NOTE: Operation uses the auto-unlink feature of boost::intrusive.
// NOTE: the cleaning happens in OSDOperationRegistry::do_stop()
using crimson::common::local_conf;
- if (historic_registry.size() > local_conf()->osd_op_history_size) {
- const auto& oldest_historic_op =
- static_cast<const ClientRequest&>(historic_registry.front());
+ if (num_recent_ops >= local_conf()->osd_op_history_size) {
+ ++last_of_recents;
+ ++num_slow_ops;
+ } else {
+ ++num_recent_ops;
+ }
+ if (num_slow_ops > local_conf()->osd_op_history_slow_op_size) {
+ // we're interested in keeping slowest ops. if the slow op history
+ // is disabled, the list will have only one element, so the full-blown
+ // search will boil down into `.front()`.
+ const auto fastest_historic_iter = std::min_element(
+ std::cbegin(historic_registry), last_of_recents,
+ [] (const auto& lop, const auto& rop) {
+ const auto& lclient_request = static_cast<const ClientRequest&>(lop);
+ const auto& rclient_request = static_cast<const ClientRequest&>(rop);
+ return get_duration(lclient_request) < get_duration(rclient_request);
+ });
+ assert(fastest_historic_iter != std::end(historic_registry));
+ const auto& fastest_historic_op =
+ static_cast<const ClientRequest&>(*fastest_historic_iter);
// clear a previously "leaked" op
- ClientRequest::ICRef(&oldest_historic_op, /* add_ref= */false);
+ ClientRequest::ICRef(&fastest_historic_op, /* add_ref= */false);
+ --num_slow_ops;
}
}
return ops_count;
}
+size_t OSDOperationRegistry::dump_slowest_historic_client_requests(ceph::Formatter* f) const
+{
+ const auto& historic_client_registry =
+ get_registry<static_cast<size_t>(OperationTypeCode::historic_client_request)>(); //ClientRequest::type)>();
+ f->open_object_section("op_history");
+ f->dump_int("size", historic_client_registry.size());
+ // TODO: f->dump_int("duration", history_duration.load());
+ // the intrusive list is configured to not store the size
+ std::multimap<utime_t,
+ const ClientRequest*,
+ std::greater<utime_t>> sorted_slowest_ops;
+ // iterating over the entire registry as a slow op could be also
+ // in the "recently added" part.
+ std::transform(std::begin(historic_client_registry),
+ std::end(historic_client_registry),
+ std::inserter(sorted_slowest_ops, std::end(sorted_slowest_ops)),
+ [] (const Operation& op) {
+ const auto& cop = static_cast<const ClientRequest&>(op);
+ return std::make_pair(get_duration(cop), &cop);
+ });
+ f->open_array_section("ops");
+ using crimson::common::local_conf;
+ size_t ops_count = 0;
+ for (auto it = std::begin(sorted_slowest_ops);
+ ops_count < local_conf()->osd_op_history_slow_op_size
+ && it != std::end(sorted_slowest_ops);
+ ++it, ++ops_count)
+ {
+ it->second->dump(f);
+ }
+ f->close_section();
+ return ops_count;
+}
+
OperationThrottler::OperationThrottler(ConfigProxy &conf)
: scheduler(crimson::osd::scheduler::make_scheduler(conf))
{
return static_cast<const T*>(this);
}
+protected:
template<class EventT>
decltype(auto) get_event() {
// all out derivates are supposed to define the list of tracking
return std::get<EventT>(that()->tracking_events);
}
-protected:
+ template<class EventT>
+ decltype(auto) get_event() const {
+ return std::get<EventT>(that()->tracking_events);
+ }
+
using OperationT<T>::OperationT;
struct StartEvent : TimeEvent<StartEvent> {};
struct OSDOperationRegistry : OperationRegistryT<
static_cast<size_t>(OperationTypeCode::last_op)
> {
+ OSDOperationRegistry();
+
void do_stop() override;
void put_historic(const class ClientRequest& op);
size_t dump_client_requests(ceph::Formatter* f) const;
size_t dump_historic_client_requests(ceph::Formatter* f) const;
+ size_t dump_slowest_historic_client_requests(ceph::Formatter* f) const;
+private:
+ op_list::const_iterator last_of_recents;
+ size_t num_recent_ops = 0;
+ size_t num_slow_ops = 0;
};
/**
* Throttles set of currently running operations
projects / ceph.git / commitdiff