From: Patrick Donnelly <pdonnell@redhat.com>
Date: Tue, 18 Jul 2023 01:22:04 +0000 (-0400)
Subject: mds: dump locks when printing mutation ops
X-Git-Tag: v19.0.0~692^2~1
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=5c7dea32bb1ee04743f0e4341dcbe06cdb096c2d;p=ceph.git

mds: dump locks when printing mutation ops

For debugging where an operation may be stuck.

Fixes: https://tracker.ceph.com/issues/62086
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---

diff --git a/src/mds/MDSDaemon.cc b/src/mds/MDSDaemon.cc
index e158b123f06d..669066fb7802 100644
--- a/src/mds/MDSDaemon.cc
+++ b/src/mds/MDSDaemon.cc
@@ -258,7 +258,9 @@ void MDSDaemon::set_up_admin_socket()
   r = admin_socket->register_command("dump_ops_in_flight", asok_hook,
 				     "show the ops currently in flight");
   ceph_assert(r == 0);
-  r = admin_socket->register_command("ops", asok_hook,
+  r = admin_socket->register_command("ops "
+				     "name=flags,type=CephChoices,strings=locks,n=N,req=false ",
+                                     asok_hook,
 				     "show the ops currently in flight");
   ceph_assert(r == 0);
   r = admin_socket->register_command("dump_blocked_ops",
diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index 6b437ae04436..31ea7f6710f9 100644
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -40,6 +40,7 @@
 #include "ScrubStack.h"
 #include "events/ESubtreeMap.h"
 #include "events/ELid.h"
+#include "Mutation.h"
 
 
 #include "MDSRank.h"
@@ -2620,11 +2621,31 @@ void MDSRankDispatcher::handle_asok_command(
   int r = 0;
   CachedStackStringStream css;
   bufferlist outbl;
-  if (command == "dump_ops_in_flight" ||
-      command == "ops") {
+  dout(10) << __func__ << ": " << command << dendl;
+  if (command == "dump_ops_in_flight") {
     if (!op_tracker.dump_ops_in_flight(f)) {
       *css << "op_tracker disabled; set mds_enable_op_tracker=true to enable";
     }
+  } else if (command == "ops") {
+    vector<string> flags;
+    cmd_getval(cmdmap, "flags", flags);
+    std::unique_lock l(mds_lock, std::defer_lock);
+    auto lambda = OpTracker::default_dumper;
+    if (flags.size()) {
+      /* use std::function if we actually want to capture flags someday */
+      lambda = [](const TrackedOp& op, Formatter* f) {
+        auto* req = dynamic_cast<const MDRequestImpl*>(&op);
+        if (req) {
+          req->dump_with_mds_lock(f);
+        } else {
+          op.dump_type(f);
+        }
+      };
+      l.lock();
+    }
+    if (!op_tracker.dump_ops_in_flight(f, false, {""}, false, lambda)) {
+      *css << "op_tracker disabled; set mds_enable_op_tracker=true to enable";
+    }
   } else if (command == "dump_blocked_ops") {
     if (!op_tracker.dump_ops_in_flight(f, true)) {
       *css << "op_tracker disabled; set mds_enable_op_tracker=true to enable";
diff --git a/src/mds/Mutation.cc b/src/mds/Mutation.cc
index 2dcc0a694c30..b52e04a63936 100644
--- a/src/mds/Mutation.cc
+++ b/src/mds/Mutation.cc
@@ -469,12 +469,7 @@ void MDRequestImpl::print(ostream &out) const
   out << ")";
 }
 
-void MDRequestImpl::dump(Formatter *f) const
-{
-  _dump(f);
-}
-
-void MDRequestImpl::_dump(Formatter *f) const
+void MDRequestImpl::_dump(Formatter *f, bool has_mds_lock) const
 {
   std::lock_guard l(lock);
   f->dump_string("flag_point", _get_state_string());
@@ -520,6 +515,27 @@ void MDRequestImpl::_dump(Formatter *f) const
     }
     f->close_section(); // events
   }
+
+  if (has_mds_lock) {
+    f->open_array_section("locks");
+    for (auto& l : locks) {
+      f->open_object_section("lock");
+      {
+        auto* mdsco = l.lock->get_parent();
+        f->dump_object("object", *mdsco);
+        CachedStackStringStream css;
+        *css << *mdsco;
+        f->dump_string("object_string", css->strv());
+        f->dump_object("lock", *l.lock);
+        f->dump_int("flags", l.flags);
+        f->dump_int("wrlock_target", l.wrlock_target);
+      }
+      f->close_section();
+    }
+    f->close_section();
+  } else {
+    f->dump_null("locks");
+  }
 }
 
 void MDRequestImpl::_dump_op_descriptor(ostream& os) const
diff --git a/src/mds/Mutation.h b/src/mds/Mutation.h
index c2afb073bb5b..b963dee08420 100644
--- a/src/mds/Mutation.h
+++ b/src/mds/Mutation.h
@@ -396,7 +396,9 @@ struct MDRequestImpl : public MutationImpl {
   std::unique_ptr<BatchOp> release_batch_op();
 
   void print(std::ostream &out) const override;
-  void dump(ceph::Formatter *f) const override;
+  void dump_with_mds_lock(ceph::Formatter* f) const {
+    return _dump(f, true);
+  }
 
   ceph::cref_t<MClientRequest> release_client_request();
   void reset_peer_request(const ceph::cref_t<MMDSPeerRequest>& req=nullptr);
@@ -453,7 +455,10 @@ struct MDRequestImpl : public MutationImpl {
   bool waited_for_osdmap = false;
 
 protected:
-  void _dump(ceph::Formatter *f) const override;
+  void _dump(ceph::Formatter *f) const override {
+    _dump(f, false);
+  }
+  void _dump(ceph::Formatter *f, bool has_mds_lock) const;
   void _dump_op_descriptor(std::ostream& stream) const override;
 };