mds: dump locks when printing mutation ops

For debugging where an operation may be stuck.

Fixes: https://tracker.ceph.com/issues/62086
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
(cherry picked from commit 5c7dea32bb1ee04743f0e4341dcbe06cdb096c2d)

Conflicts:
	src/mds/MDSRank.cc: header conflict

diff --git a/src/mds/MDSDaemon.cc b/src/mds/MDSDaemon.cc
index 877d9132e71ce4c24bccb711149355d1b780bc48..cce7d434c67449bba573e2274a177c1b68e80de9 100644 (file)
--- a/src/mds/MDSDaemon.cc
+++ b/src/mds/MDSDaemon.cc
@@ -258,7 +258,9 @@ void MDSDaemon::set_up_admin_socket()
   r = admin_socket->register_command("dump_ops_in_flight", asok_hook,
                                     "show the ops currently in flight");
   ceph_assert(r == 0);
-  r = admin_socket->register_command("ops", asok_hook,
+  r = admin_socket->register_command("ops "
+                                    "name=flags,type=CephChoices,strings=locks,n=N,req=false ",
+                                     asok_hook,
                                     "show the ops currently in flight");
   ceph_assert(r == 0);
   r = admin_socket->register_command("dump_blocked_ops",

diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index 249868adaed25178b5fc45d4404544023c761087..12a2d5f4a5a640fdf1f0f49a146479357a453f99 100644 (file)
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -38,6 +38,7 @@
 #include "mon/MonClient.h"
 #include "common/HeartbeatMap.h"
 #include "ScrubStack.h"
+#include "Mutation.h"
 
 
 #include "MDSRank.h"
@@ -2610,11 +2611,31 @@ void MDSRankDispatcher::handle_asok_command(
   int r = 0;
   CachedStackStringStream css;
   bufferlist outbl;
-  if (command == "dump_ops_in_flight" ||
-      command == "ops") {
+  dout(10) << __func__ << ": " << command << dendl;
+  if (command == "dump_ops_in_flight") {
     if (!op_tracker.dump_ops_in_flight(f)) {
       *css << "op_tracker disabled; set mds_enable_op_tracker=true to enable";
     }
+  } else if (command == "ops") {
+    vector<string> flags;
+    cmd_getval(cmdmap, "flags", flags);
+    std::unique_lock l(mds_lock, std::defer_lock);
+    auto lambda = OpTracker::default_dumper;
+    if (flags.size()) {
+      /* use std::function if we actually want to capture flags someday */
+      lambda = [](const TrackedOp& op, Formatter* f) {
+        auto* req = dynamic_cast<const MDRequestImpl*>(&op);
+        if (req) {
+          req->dump_with_mds_lock(f);
+        } else {
+          op.dump_type(f);
+        }
+      };
+      l.lock();
+    }
+    if (!op_tracker.dump_ops_in_flight(f, false, {""}, lambda)) {
+      *css << "op_tracker disabled; set mds_enable_op_tracker=true to enable";
+    }
   } else if (command == "dump_blocked_ops") {
     if (!op_tracker.dump_ops_in_flight(f, true)) {
       *css << "op_tracker disabled; set mds_enable_op_tracker=true to enable";

diff --git a/src/mds/Mutation.cc b/src/mds/Mutation.cc
index 2dcc0a694c30083798c9dbbafbf78fd6f0fa2c65..b52e04a63936ef30d7208f1a515e17a8ec27dfe3 100644 (file)
--- a/src/mds/Mutation.cc
+++ b/src/mds/Mutation.cc
@@ -469,12 +469,7 @@ void MDRequestImpl::print(ostream &out) const
   out << ")";
 }
 
-void MDRequestImpl::dump(Formatter *f) const
-{
-  _dump(f);
-}
-
-void MDRequestImpl::_dump(Formatter *f) const
+void MDRequestImpl::_dump(Formatter *f, bool has_mds_lock) const
 {
   std::lock_guard l(lock);
   f->dump_string("flag_point", _get_state_string());
@@ -520,6 +515,27 @@ void MDRequestImpl::_dump(Formatter *f) const
     }
     f->close_section(); // events
   }
+
+  if (has_mds_lock) {
+    f->open_array_section("locks");
+    for (auto& l : locks) {
+      f->open_object_section("lock");
+      {
+        auto* mdsco = l.lock->get_parent();
+        f->dump_object("object", *mdsco);
+        CachedStackStringStream css;
+        *css << *mdsco;
+        f->dump_string("object_string", css->strv());
+        f->dump_object("lock", *l.lock);
+        f->dump_int("flags", l.flags);
+        f->dump_int("wrlock_target", l.wrlock_target);
+      }
+      f->close_section();
+    }
+    f->close_section();
+  } else {
+    f->dump_null("locks");
+  }
 }
 
 void MDRequestImpl::_dump_op_descriptor(ostream& os) const

diff --git a/src/mds/Mutation.h b/src/mds/Mutation.h
index e5fa3c9eb06416ff30c69fb9f6c65187998ee595..6836533b054229f92104a205ebb993ae422152d8 100644 (file)
--- a/src/mds/Mutation.h
+++ b/src/mds/Mutation.h
@@ -396,7 +396,9 @@ struct MDRequestImpl : public MutationImpl {
   std::unique_ptr<BatchOp> release_batch_op();
 
   void print(std::ostream &out) const override;
-  void dump(ceph::Formatter *f) const override;
+  void dump_with_mds_lock(ceph::Formatter* f) const {
+    return _dump(f, true);
+  }
 
   ceph::cref_t<MClientRequest> release_client_request();
   void reset_peer_request(const ceph::cref_t<MMDSPeerRequest>& req=nullptr);
@@ -452,7 +454,10 @@ struct MDRequestImpl : public MutationImpl {
   bool waited_for_osdmap = false;
 
 protected:
-  void _dump(ceph::Formatter *f) const override;
+  void _dump(ceph::Formatter *f) const override {
+    _dump(f, false);
+  }
+  void _dump(ceph::Formatter *f, bool has_mds_lock) const;
   void _dump_op_descriptor(std::ostream& stream) const override;
 };