mds: dump locks when printing mutation ops

For debugging where an operation may be stuck.

Fixes: https://tracker.ceph.com/issues/62086
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>

diff --git a/src/mds/MDSDaemon.cc b/src/mds/MDSDaemon.cc
index e158b123f06d339bed26074f2ce4167e0e37d608..669066fb78022badf63fd5770dc55a565f62e5de 100644 (file)
--- a/src/mds/MDSDaemon.cc
+++ b/src/mds/MDSDaemon.cc
@@ -258,7 +258,9 @@ void MDSDaemon::set_up_admin_socket()
   r = admin_socket->register_command("dump_ops_in_flight", asok_hook,
                                     "show the ops currently in flight");
   ceph_assert(r == 0);
-  r = admin_socket->register_command("ops", asok_hook,
+  r = admin_socket->register_command("ops "
+                                    "name=flags,type=CephChoices,strings=locks,n=N,req=false ",
+                                     asok_hook,
                                     "show the ops currently in flight");
   ceph_assert(r == 0);
   r = admin_socket->register_command("dump_blocked_ops",

diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index 6b437ae04436d83c29a16e9e9a60730b06547d27..31ea7f6710f92af726141b995fab189120c2211f 100644 (file)
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -40,6 +40,7 @@
 #include "ScrubStack.h"
 #include "events/ESubtreeMap.h"
 #include "events/ELid.h"
+#include "Mutation.h"
 
 
 #include "MDSRank.h"
@@ -2620,11 +2621,31 @@ void MDSRankDispatcher::handle_asok_command(
   int r = 0;
   CachedStackStringStream css;
   bufferlist outbl;
-  if (command == "dump_ops_in_flight" ||
-      command == "ops") {
+  dout(10) << __func__ << ": " << command << dendl;
+  if (command == "dump_ops_in_flight") {
     if (!op_tracker.dump_ops_in_flight(f)) {
       *css << "op_tracker disabled; set mds_enable_op_tracker=true to enable";
     }
+  } else if (command == "ops") {
+    vector<string> flags;
+    cmd_getval(cmdmap, "flags", flags);
+    std::unique_lock l(mds_lock, std::defer_lock);
+    auto lambda = OpTracker::default_dumper;
+    if (flags.size()) {
+      /* use std::function if we actually want to capture flags someday */
+      lambda = [](const TrackedOp& op, Formatter* f) {
+        auto* req = dynamic_cast<const MDRequestImpl*>(&op);
+        if (req) {
+          req->dump_with_mds_lock(f);
+        } else {
+          op.dump_type(f);
+        }
+      };
+      l.lock();
+    }
+    if (!op_tracker.dump_ops_in_flight(f, false, {""}, false, lambda)) {
+      *css << "op_tracker disabled; set mds_enable_op_tracker=true to enable";
+    }
   } else if (command == "dump_blocked_ops") {
     if (!op_tracker.dump_ops_in_flight(f, true)) {
       *css << "op_tracker disabled; set mds_enable_op_tracker=true to enable";

diff --git a/src/mds/Mutation.cc b/src/mds/Mutation.cc
index 2dcc0a694c30083798c9dbbafbf78fd6f0fa2c65..b52e04a63936ef30d7208f1a515e17a8ec27dfe3 100644 (file)
--- a/src/mds/Mutation.cc
+++ b/src/mds/Mutation.cc
@@ -469,12 +469,7 @@ void MDRequestImpl::print(ostream &out) const
   out << ")";
 }
 
-void MDRequestImpl::dump(Formatter *f) const
-{
-  _dump(f);
-}
-
-void MDRequestImpl::_dump(Formatter *f) const
+void MDRequestImpl::_dump(Formatter *f, bool has_mds_lock) const
 {
   std::lock_guard l(lock);
   f->dump_string("flag_point", _get_state_string());
@@ -520,6 +515,27 @@ void MDRequestImpl::_dump(Formatter *f) const
     }
     f->close_section(); // events
   }
+
+  if (has_mds_lock) {
+    f->open_array_section("locks");
+    for (auto& l : locks) {
+      f->open_object_section("lock");
+      {
+        auto* mdsco = l.lock->get_parent();
+        f->dump_object("object", *mdsco);
+        CachedStackStringStream css;
+        *css << *mdsco;
+        f->dump_string("object_string", css->strv());
+        f->dump_object("lock", *l.lock);
+        f->dump_int("flags", l.flags);
+        f->dump_int("wrlock_target", l.wrlock_target);
+      }
+      f->close_section();
+    }
+    f->close_section();
+  } else {
+    f->dump_null("locks");
+  }
 }
 
 void MDRequestImpl::_dump_op_descriptor(ostream& os) const

diff --git a/src/mds/Mutation.h b/src/mds/Mutation.h
index c2afb073bb5bdd2f41674453e5741faaba372134..b963dee08420c6d4d51eeda3bb613dad3c7ee458 100644 (file)
--- a/src/mds/Mutation.h
+++ b/src/mds/Mutation.h
@@ -396,7 +396,9 @@ struct MDRequestImpl : public MutationImpl {
   std::unique_ptr<BatchOp> release_batch_op();
 
   void print(std::ostream &out) const override;
-  void dump(ceph::Formatter *f) const override;
+  void dump_with_mds_lock(ceph::Formatter* f) const {
+    return _dump(f, true);
+  }
 
   ceph::cref_t<MClientRequest> release_client_request();
   void reset_peer_request(const ceph::cref_t<MMDSPeerRequest>& req=nullptr);
@@ -453,7 +455,10 @@ struct MDRequestImpl : public MutationImpl {
   bool waited_for_osdmap = false;
 
 protected:
-  void _dump(ceph::Formatter *f) const override;
+  void _dump(ceph::Formatter *f) const override {
+    _dump(f, false);
+  }
+  void _dump(ceph::Formatter *f, bool has_mds_lock) const;
   void _dump_op_descriptor(std::ostream& stream) const override;
 };