crimson/osd: implement dump_ops_in_flight

Signed-off-by: Radosław Zarzyński <rzarzyns@redhat.com>

diff --git a/src/crimson/admin/osd_admin.cc b/src/crimson/admin/osd_admin.cc
index 5bbd27393db168fa4b8d0a770a661b7299885c15..b29a56505655993b9f7392f25e61915b3b46a916 100644 (file)
--- a/src/crimson/admin/osd_admin.cc
+++ b/src/crimson/admin/osd_admin.cc
@@ -424,4 +424,32 @@ private:
 template std::unique_ptr<AdminSocketHook> make_asok_hook<InjectMDataErrorHook>(
   crimson::osd::ShardServices&);
 
+
+/**
+ * An InFlightOps admin hook: dump current in-flight operations
+ */
+class DumpInFlightOpsHook : public AdminSocketHook {
+public:
+  explicit DumpInFlightOpsHook(const crimson::osd::OSDOperationRegistry& op_registry) :
+    AdminSocketHook{"dump_ops_in_flight", "", "show the ops currently in flight"},
+    op_registry(op_registry)
+  {}
+  seastar::future<tell_result_t> call(const cmdmap_t&,
+                                     std::string_view format,
+                                     ceph::bufferlist&& input) const final
+  {
+    logger().warn("{}", __func__);
+    unique_ptr<Formatter> f{Formatter::create(format, "json-pretty", "json-pretty")};
+    f->open_object_section("ops_in_flight");
+    op_registry.dump_client_requests(f.get());
+    f->close_section();
+    f->dump_int("num_ops", 0);
+    return seastar::make_ready_future<tell_result_t>(std::move(f));
+  }
+private:
+  const crimson::osd::OSDOperationRegistry& op_registry;
+};
+template std::unique_ptr<AdminSocketHook>
+make_asok_hook<DumpInFlightOpsHook>(const crimson::osd::OSDOperationRegistry& op_registry);
+
 } // namespace crimson::admin

diff --git a/src/crimson/admin/osd_admin.h b/src/crimson/admin/osd_admin.h
index 61dcfced1ac13593949764be17a85b12e8dbe3ab..273d17ec23c564cc5263982fc956e3dc5be61589 100644 (file)
--- a/src/crimson/admin/osd_admin.h
+++ b/src/crimson/admin/osd_admin.h
@@ -17,6 +17,7 @@ class InjectDataErrorHook;
 class InjectMDataErrorHook;
 class OsdStatusHook;
 class SendBeaconHook;
+class DumpInFlightOpsHook;
 
 template<class Hook, class... Args>
 std::unique_ptr<AdminSocketHook> make_asok_hook(Args&&... args);

diff --git a/src/crimson/common/operation.cc b/src/crimson/common/operation.cc
index 85575a74a0efa05cd7b2d8414385fe74600383d3..65946b5ad85e11fdd2903e99ae1781f997e3575d 100644 (file)
--- a/src/crimson/common/operation.cc
+++ b/src/crimson/common/operation.cc
@@ -67,4 +67,31 @@ void AggregateBlocker::dump_detail(ceph::Formatter *f) const
   f->close_section();
 }
 
+namespace detail {
+void dump_time_event(const char* name,
+                    const utime_t& timestamp,
+                    ceph::Formatter* f)
+{
+  assert(f);
+  f->open_object_section("time_event");
+  f->dump_string("name", name);
+  f->dump_stream("initiated_at") << timestamp;
+  f->close_section();
+}
+
+void dump_blocking_event(const char* name,
+                        const utime_t& timestamp,
+                        const Blocker* const blocker,
+                        ceph::Formatter* f)
+{
+  assert(f);
+  f->open_object_section("blocking_event");
+  f->dump_string("name", name);
+  f->dump_stream("initiated_at") << timestamp;
+  if (blocker) {
+    blocker->dump(f);
+  }
+  f->close_section();
 }
+} // namespace detail
+} // namespace crimson

diff --git a/src/crimson/common/operation.h b/src/crimson/common/operation.h
index 84387356ba76c0f15bebef95c00e91c78a955920..a8bcabf918ce8de2cb847938b259e6ee070977c7 100644 (file)
--- a/src/crimson/common/operation.h
+++ b/src/crimson/common/operation.h
@@ -123,6 +123,16 @@ make_exception_blocking_future(Exception&& e) {
     seastar::make_exception_future<V>(e));
 }
 
+namespace detail {
+void dump_time_event(const char* name,
+                    const utime_t& timestamp,
+                    ceph::Formatter* f);
+void dump_blocking_event(const char* name,
+                        const utime_t& timestamp,
+                        const Blocker* blocker,
+                        ceph::Formatter* f);
+} // namespace detail
+
 /**
  * Provides an interface for dumping diagnostic information about
  * why a particular op is not making progress.
@@ -214,9 +224,12 @@ struct TimeEvent : Event<T> {
     void handle(T&, const Operation&) override {
       timestamp = ceph_clock_now();
     }
-  private:
     utime_t timestamp;
   } internal_backend;
+
+  void dump(ceph::Formatter *f) const {
+    detail::dump_time_event(typeid(T).name(), internal_backend.timestamp, f);
+  }
 };
 
 
@@ -304,6 +317,13 @@ public:
 
       const OpT& op;
     };
+
+    void dump(ceph::Formatter *f) const {
+      detail::dump_blocking_event(typeid(T).name(),
+                                 internal_backend.timestamp,
+                                 internal_backend.blocker,
+                                 f);
+    }
   };
 
   virtual ~BlockerT() = default;

diff --git a/src/crimson/osd/osd.cc b/src/crimson/osd/osd.cc
index 8f5a7f9d473e320cd5c4fe712b28d342481f9709..a1e71de3bbe870e7840c4cd4a7adfcc90a490782 100644 (file)
--- a/src/crimson/osd/osd.cc
+++ b/src/crimson/osd/osd.cc
@@ -545,6 +545,9 @@ seastar::future<> OSD::start_asok_admin()
     // PG commands
     asok->register_command(make_asok_hook<pg::QueryCommand>(*this));
     asok->register_command(make_asok_hook<pg::MarkUnfoundLostCommand>(*this));
+    // ops commands
+    asok->register_command(make_asok_hook<DumpInFlightOpsHook>(
+      std::as_const(get_shard_services().registry)));
   });
 }
 

diff --git a/src/crimson/osd/osd_operation.cc b/src/crimson/osd/osd_operation.cc
index 0ab3c2cf74d1442345635cb6ba2281740d76a49c..90370d57b816fe77ffcacb5ba89db11ec5e63b2d 100644 (file)
--- a/src/crimson/osd/osd_operation.cc
+++ b/src/crimson/osd/osd_operation.cc
@@ -3,9 +3,28 @@
 
 #include "osd_operation.h"
 #include "common/Formatter.h"
+#include "crimson/common/log.h"
+#include "crimson/osd/osd_operations/client_request.h"
+
+namespace {
+  seastar::logger& logger() {
+    return crimson::get_logger(ceph_subsys_osd);
+  }
+}
 
 namespace crimson::osd {
 
+size_t OSDOperationRegistry::dump_client_requests(ceph::Formatter* f) const
+{
+  const auto& client_registry =
+    get_registry<static_cast<size_t>(ClientRequest::type)>();
+  logger().warn("{} num_ops={}", __func__, std::size(client_registry));
+  for (const auto& op : client_registry) {
+    op.dump(f);
+  }
+  return std::size(client_registry);
+}
+
 OperationThrottler::OperationThrottler(ConfigProxy &conf)
   : scheduler(crimson::osd::scheduler::make_scheduler(conf))
 {

diff --git a/src/crimson/osd/osd_operation.h b/src/crimson/osd/osd_operation.h
index 6f6c8b04f14495d8daa681b84a776180064f1665..4b36852017ad31f570aa5cf312c69e455845b8f2 100644 (file)
--- a/src/crimson/osd/osd_operation.h
+++ b/src/crimson/osd/osd_operation.h
@@ -186,10 +186,11 @@ protected:
 /**
  * Maintains a set of lists of all active ops.
  */
-using OSDOperationRegistry = OperationRegistryT<
+struct OSDOperationRegistry : OperationRegistryT<
   static_cast<size_t>(OperationTypeCode::last_op)
-  >;
-
+> {
+  size_t dump_client_requests(ceph::Formatter* f) const;
+};
 /**
  * Throttles set of currently running operations
  *

diff --git a/src/crimson/osd/osd_operations/client_request.cc b/src/crimson/osd/osd_operations/client_request.cc
index 626b82c48406a54f7f72bc0b90bb9d987eeb33ec..bca3952c772d44afcc66d840558a1cd9de0b4566 100644 (file)
--- a/src/crimson/osd/osd_operations/client_request.cc
+++ b/src/crimson/osd/osd_operations/client_request.cc
@@ -43,6 +43,10 @@ void ClientRequest::print(std::ostream &lhs) const
 
 void ClientRequest::dump_detail(Formatter *f) const
 {
+  logger().debug("{}: dumping", *this);
+  std::apply([f] (auto... event) {
+    (..., event.dump(f));
+  }, tracking_events);
 }
 
 ClientRequest::ConnectionPipeline &ClientRequest::cp()