osd: support dumping historic slow ops

ceph daemon osd.x dump_historic_slow_ops

Signed-off-by: Zhiqiang Wang <zhiqiang@xsky.com>

diff --git a/src/common/TrackedOp.cc b/src/common/TrackedOp.cc
index ee1f4391568c5b7dcbe6150f39a840f8db9a79eb..58df253262d2e06ef96e847c9ab9a679148eb2ea 100644 (file)
--- a/src/common/TrackedOp.cc
+++ b/src/common/TrackedOp.cc
@@ -174,6 +174,39 @@ bool OpTracker::dump_historic_ops(Formatter *f, bool by_duration)
   return true;
 }
 
+void OpHistory::dump_slow_ops(utime_t now, Formatter *f)
+{
+  Mutex::Locker history_lock(ops_history_lock);
+  cleanup(now);
+  f->open_object_section("OpHistory slow ops");
+  f->dump_int("num to keep", history_slow_op_size);
+  f->dump_int("threshold to keep", history_slow_op_threshold);
+  {
+    f->open_array_section("Ops");
+    for (set<pair<utime_t, TrackedOpRef> >::const_iterator i =
+          slow_op.begin();
+        i != slow_op.end();
+        ++i) {
+      f->open_object_section("Op");
+      i->second->dump(now, f);
+      f->close_section();
+    }
+    f->close_section();
+  }
+  f->close_section();
+}
+
+bool OpTracker::dump_historic_slow_ops(Formatter *f)
+{
+  RWLock::RLocker l(lock);
+  if (!tracking_enabled)
+    return false;
+
+  utime_t now = ceph_clock_now();
+  history.dump_slow_ops(now, f);
+  return true;
+}
+
 bool OpTracker::dump_ops_in_flight(Formatter *f, bool print_only_blocked)
 {
   RWLock::RLocker l(lock);

diff --git a/src/common/TrackedOp.h b/src/common/TrackedOp.h
index 5342e82738168bc0b1e051bc2dd302537d223a3c..c5f6827ec83174a17c9fbdd4b66328d74c9ce934 100644 (file)
--- a/src/common/TrackedOp.h
+++ b/src/common/TrackedOp.h
@@ -54,6 +54,7 @@ public:
   void insert(utime_t now, TrackedOpRef op);
   void dump_ops(utime_t now, Formatter *f);
   void dump_ops_by_duration(utime_t now, Formatter *f);
+  void dump_slow_ops(utime_t now, Formatter *f);
   void on_shutdown();
   void set_size_and_duration(uint32_t new_size, uint32_t new_duration) {
     history_size = new_size;
@@ -97,6 +98,7 @@ public:
   }
   bool dump_ops_in_flight(Formatter *f, bool print_only_blocked=false);
   bool dump_historic_ops(Formatter *f, bool by_duration = false);
+  bool dump_historic_slow_ops(Formatter *f);
   bool register_inflight_op(TrackedOp *i);
   void unregister_inflight_op(TrackedOp *i);
 

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 2eb9e4078d548faa6e57c6b3ea24e58fa9aa463f..b276f0aabbead85b98614d66f894ac5f77098bd2 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -1979,6 +1979,11 @@ bool OSD::asok_command(string admin_command, cmdmap_t& cmdmap, string format,
       ss << "op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. \
        Please enable \"osd_enable_op_tracker\", and the tracker will start to track new ops received afterwards.";
     }
+  } else if (admin_command == "dump_historic_slow_ops") {
+    if (!op_tracker.dump_historic_slow_ops(f)) {
+      ss << "op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. \
+       Please enable \"osd_enable_op_tracker\", and the tracker will start to track new ops received afterwards.";
+    }
   } else if (admin_command == "dump_op_pq_state") {
     f->open_object_section("pq");
     op_shardedwq.dump(f);
@@ -2536,6 +2541,10 @@ void OSD::final_init()
                                     "show the blocked ops currently in flight");
   assert(r == 0);
   r = admin_socket->register_command("dump_historic_ops", "dump_historic_ops",
+                                    asok_hook,
+                                    "show recent ops");
+  assert(r == 0);
+  r = admin_socket->register_command("dump_historic_slow_ops", "dump_historic_slow_ops",
                                     asok_hook,
                                     "show slowest recent ops");
   assert(r == 0);
@@ -2975,6 +2984,7 @@ int OSD::shutdown()
   cct->get_admin_socket()->unregister_command("dump_blocked_ops");
   cct->get_admin_socket()->unregister_command("dump_historic_ops");
   cct->get_admin_socket()->unregister_command("dump_historic_ops_by_duration");
+  cct->get_admin_socket()->unregister_command("dump_historic_slow_ops");
   cct->get_admin_socket()->unregister_command("dump_op_pq_state");
   cct->get_admin_socket()->unregister_command("dump_blacklist");
   cct->get_admin_socket()->unregister_command("dump_watchers");