From: Igor Golikov <igolikov@redhat.com>
Date: Tue, 30 Dec 2025 14:03:06 +0000 (+0000)
Subject: mds: implement main tracing flows
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=7a538f56d7454338d7c96a73fee09c60a9810c71;p=ceph-ci.git

mds: implement main tracing flows

implement tracing for major flows. currently dumping traces via new asok command

Fixes: https://tracker.ceph.com/issues/73701
Signed-off-by: Igor Golikov <igolikov@redhat.com>
---

diff --git a/src/mds/Locker.cc b/src/mds/Locker.cc
index 58305db6068..ecee5838aec 100644
--- a/src/mds/Locker.cc
+++ b/src/mds/Locker.cc
@@ -20,6 +20,7 @@
 #include "CInode.h"
 #include "common/config.h"
 #include "common/debug.h"
+#include "MDSTracer.h"
 #include "events/EOpen.h"
 #include "events/EUpdate.h"
 #include "MDBalancer.h"
@@ -155,6 +156,7 @@ void Locker::send_lock_message(SimpleLock *lock, int msg, const bufferlist &data
 bool Locker::try_rdlock_snap_layout(CInode *in, const MDRequestRef& mdr,
 				    int n, bool want_layout)
 {
+  ScopedSpan trace_span(mds->tracer, "rdlock_snap_layout", mdr->trace_span, &mdr->trace_data);
   dout(10) << __func__ << " " << *mdr << " " << *in << dendl;
   // rdlock ancestor snaps
   inodeno_t root;
@@ -253,6 +255,9 @@ bool Locker::acquire_locks(const MDRequestRef& mdr,
   }
   dout(20) << " auth_pin_nonblocking=" << auth_pin_nonblocking << dendl;
 
+  // Trace child span: lock acquisition (duration captured automatically)
+  ScopedSpan trace_span(mds->tracer, "acquire_locks", mdr->trace_span, &mdr->trace_data);
+
   MarkEventOnDestruct marker(mdr, "failed to acquire_locks");
 
   client_t client = mdr->get_client();
@@ -1089,6 +1094,7 @@ void Locker::invalidate_lock_caches(SimpleLock *lock)
 
 void Locker::create_lock_cache(const MDRequestRef& mdr, CInode *diri, file_layout_t *dir_layout)
 {
+  ScopedSpan trace_span(mds->tracer, "create_lock_cache", mdr->trace_span, &mdr->trace_data);
   if (mdr->lock_cache)
     return;
 
@@ -1885,6 +1891,7 @@ void Locker::rdlock_finish(const MutationImpl::lock_iterator& it, MutationImpl *
 
 bool Locker::rdlock_try_set(MutationImpl::LockOpVec& lov, const MDRequestRef& mdr)
 {
+  ScopedSpan trace_span(mds->tracer, "rdlock_try_set", mdr->trace_span, &mdr->trace_data);
   dout(10) << __func__  << dendl;
   for (const auto& p : lov) {
     auto lock = p.lock;
@@ -2437,6 +2444,10 @@ Capability* Locker::issue_new_caps(CInode *in,
 				   SnapRealm *realm)
 {
   dout(7) << "issue_new_caps for mode " << mode << " on " << *in << dendl;
+
+  // Trace child span: issuing new caps (duration captured automatically)
+  ScopedSpan trace_span(mds->tracer, "issue_caps", mdr->trace_span, &mdr->trace_data);
+
   Session *session = mdr->session;
   bool new_inode = (mdr->alloc_ino || mdr->used_prealloc_ino);
 
@@ -3698,6 +3709,7 @@ public:
 void Locker::process_request_cap_release(const MDRequestRef& mdr, client_t client, const ceph_mds_request_release& item,
 					 std::string_view dname)
 {
+  ScopedSpan trace_span(mds->tracer, "cap_release", mdr ? mdr->trace_span : jspan_ptr(), mdr ? &mdr->trace_data : nullptr);
   inodeno_t ino = (uint64_t)item.ino;
   uint64_t cap_id = item.cap_id;
   int caps = item.caps;
@@ -4526,6 +4538,7 @@ void Locker::handle_client_lease(const cref_t<MClientLease> &m)
 void Locker::issue_client_lease(CDentry *dn, CInode *in, const MDRequestRef& mdr, utime_t now,
                                 bufferlist &bl)
 {
+  ScopedSpan trace_span(mds->tracer, "issue_lease", mdr->trace_span, &mdr->trace_data);
   client_t client = mdr->get_client();
   Session *session = mdr->session;
 
diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index 623a2e2348a..f7d8ad5f0c3 100644
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -23,6 +23,7 @@
 #include <deque>
 #include <ostream>
 #include <string>
+#include <sstream>
 #include <string_view>
 #include <map>
 #include <memory>
@@ -77,6 +78,8 @@
 #include "msg/Message.h"
 #include "msg/Messenger.h"
 
+#include "MDSTracer.h"
+
 #include "common/debug.h"
 #include "common/errno.h"
 #include "common/perf_counters.h"
@@ -8528,6 +8531,10 @@ int MDCache::path_traverse(const MDRequestRef& mdr, MDSContextFactory& cf,
   bool rdlock_authlock = (flags & MDS_TRAVERSE_RDLOCK_AUTHLOCK);
   bool forimport = (flags & MDS_TRAVERSE_IMPORT);
 
+  // Trace child span: path traversal (duration captured automatically)
+  ScopedSpan trace_span(mds->tracer, "path_traverse", mdr ? mdr->trace_span : jspan_ptr(),
+                        mdr ? &mdr->trace_data : nullptr);
+
   if (forward)
     ceph_assert(mdr);  // forward requires a request
 
@@ -9046,6 +9053,7 @@ void MDCache::open_remote_dirfrag(CInode *diri, frag_t approxfg, MDSContext *fin
  */
 CInode *MDCache::get_dentry_inode(CDentry *dn, const MDRequestRef& mdr, bool projected)
 {
+  ScopedSpan trace_span(mds->tracer, "get_dentry_inode", mdr ? mdr->trace_span : jspan_ptr(), mdr ? &mdr->trace_data : nullptr);
   CDentry::linkage_t *dnl;
   if (projected)
     dnl = dn->get_projected_linkage();
@@ -9979,6 +9987,26 @@ MDRequestRef MDCache::request_start_internal(int op)
     ceph_abort();
   }
   active_requests[mdr->reqid] = mdr;
+
+  // Initialize tracing for internal operations
+  if (mds->tracer.is_enabled()) {
+    const char* op_name = ceph_mds_op_name(op);
+    std::string trace_name = std::string("mds:internal:") + op_name;
+    
+    mdr->trace_span = mds->tracer.start_trace(trace_name.c_str());
+    
+    
+    std::ostringstream reqid_ss;
+    reqid_ss << mdr->reqid;
+    
+    if (mdr->trace_span && mdr->trace_span->IsRecording()) {
+      mdr->trace_span->SetAttribute(MDSTracer::ATTR_OP_TYPE, op);
+      mdr->trace_span->SetAttribute(MDSTracer::ATTR_OP_NAME, op_name);
+      mdr->trace_span->SetAttribute(MDSTracer::ATTR_MDS_RANK, static_cast<int64_t>(mds->get_nodeid()));
+      mdr->trace_span->SetAttribute(MDSTracer::ATTR_INTERNAL_OP, "true");
+    }
+  }
+
   dout(7) << __func__ << " " << *mdr << " op " << op << dendl;
   return mdr;
 }
@@ -10046,6 +10074,7 @@ void MDCache::request_finish(const MDRequestRef& mdr)
 
 void MDCache::request_forward(const MDRequestRef& mdr, mds_rank_t who, int port)
 {
+  ScopedSpan trace_span(mds->tracer, "request_forward", mdr->trace_span, &mdr->trace_data);
   CachedStackStringStream css;
   *css << "forwarding request to mds." << who;
   mdr->mark_event(css->strv());
@@ -10212,6 +10241,19 @@ void MDCache::request_cleanup(const MDRequestRef& mdr)
   if (mds->logger)
     log_stat();
 
+  // Finalize trace span and store in sliding window
+  if (mdr->trace_span && mdr->trace_span->IsRecording()) {
+    mdr->trace_span->AddEvent("request_cleanup");
+    mdr->trace_span->End();
+    
+    // Complete trace_data and store in sliding window
+    if (!mdr->trace_data.trace_id.empty()) {
+      mdr->trace_data.end_time = ceph_clock_now();
+      mdr->trace_data.duration_ms = (mdr->trace_data.end_time - mdr->trace_data.start_time) * 1000.0;
+      mds->tracer.store_trace(mdr->trace_data);
+    }
+  }
+
   mdr->mark_event("cleaned up request");
 }
 
@@ -12406,6 +12448,7 @@ void MDCache::fragment_frozen(const MDRequestRef& mdr, int r)
 
 void MDCache::dispatch_fragment_dir(const MDRequestRef& mdr, bool abort_if_freezing)
 {
+  ScopedSpan trace_span(mds->tracer, "fragment_dir", mdr->trace_span, &mdr->trace_data);
   dirfrag_t basedirfrag = mdr->more()->fragment_base;
   auto it = fragments.find(basedirfrag);
   if (it == fragments.end() || it->second.mdr != mdr) {
@@ -13478,6 +13521,7 @@ void MDCache::enqueue_scrub(
 
 void MDCache::enqueue_scrub_work(const MDRequestRef& mdr)
 {
+  ScopedSpan trace_span(mds->tracer, "enqueue_scrub", mdr->trace_span, &mdr->trace_data);
   CInode *in;
   CF_MDS_RetryRequestFactory cf(this, mdr, true);
   int r = path_traverse(mdr, cf, mdr->get_filepath(),
@@ -13990,6 +14034,7 @@ public:
 
 void MDCache::flush_dentry_work(const MDRequestRef& mdr)
 {
+  ScopedSpan trace_span(mds->tracer, "flush_dentry", mdr->trace_span, &mdr->trace_data);
   MutationImpl::LockOpVec lov;
   CInode *in = mds->server->rdlock_path_pin_ref(mdr, true);
   if (!in)
@@ -14173,6 +14218,7 @@ void MDCache::quiesce_overdrive_fragmenting_async(CDir* dir) {
 
 void MDCache::dispatch_quiesce_inode(const MDRequestRef& mdr)
 {
+  ScopedSpan trace_span(mds->tracer, "quiesce_inode", mdr->trace_span, &mdr->trace_data);
   if (mdr->internal_op_finish == nullptr) {
     dout(20) << __func__ << " " << *mdr << " already finished quiesce" << dendl;
     return;
@@ -14425,6 +14471,7 @@ void MDCache::add_quiesce(CInode* parent, CInode* in)
 
 void MDCache::dispatch_quiesce_path(const MDRequestRef& mdr)
 {
+  ScopedSpan trace_span(mds->tracer, "quiesce_path", mdr->trace_span, &mdr->trace_data);
   if (!mds->is_active()) {
     dout(20) << __func__ << " is not active!" << dendl;
     mds->server->respond_to_request(mdr, -EAGAIN);
@@ -14530,6 +14577,7 @@ MDRequestRef MDCache::quiesce_path(filepath p, C_MDS_QuiescePath* c, Formatter *
 
 void MDCache::dispatch_lock_path(const MDRequestRef& mdr)
 {
+  ScopedSpan trace_span(mds->tracer, "lock_path", mdr->trace_span, &mdr->trace_data);
   CF_MDS_RetryRequestFactory cf(this, mdr, true);
   auto& lps = *static_cast<LockPathState*>(mdr->internal_op_private);
   CInode* in = lps.in;
diff --git a/src/mds/MDSDaemon.cc b/src/mds/MDSDaemon.cc
index ca2a94f7fcd..31ce4c66392 100644
--- a/src/mds/MDSDaemon.cc
+++ b/src/mds/MDSDaemon.cc
@@ -309,6 +309,9 @@ void MDSDaemon::set_up_admin_socket()
   r = admin_socket->register_command("dump_ops_in_flight", asok_hook,
 				     "show the ops currently in flight");
   ceph_assert(r == 0);
+  r = admin_socket->register_command("trace dump", asok_hook,
+				     "dump completed traces from sliding window");
+  ceph_assert(r == 0);
   r = admin_socket->register_command("ops "
 				     "name=flags,type=CephChoices,strings=locks,n=N,req=false "
 				     "name=path,type=CephString,req=false "
diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index 02816d700c5..af4e1927306 100644
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -60,6 +60,7 @@
 
 #include "QuiesceDbManager.h"
 #include "QuiesceAgent.h"
+#include "MDSTracer.h"
 
 #define dout_context g_ceph_context
 #define dout_subsys ceph_subsys_mds
@@ -476,6 +477,7 @@ MDSRank::MDSRank(
     timer(timer_), mdsmap(mdsmap_),
     objecter(new Objecter(g_ceph_context, msgr, monc_, ioc)),
     damage_table(whoami_), sessionmap(this),
+    tracer(cct),
     op_tracker(g_ceph_context, g_conf()->mds_enable_op_tracker,
                g_conf()->osd_num_op_tracker_shard),
     progress_thread(this), whoami(whoami_),
@@ -588,6 +590,9 @@ void MDSRankDispatcher::init()
   update_log_config();
   create_logger();
 
+  // Initialize MDS tracer for distributed tracing
+  tracer.init();
+
   // Expose the OSDMap (already populated during MDS::init) to anyone
   // who is interested in it.
   handle_osd_map();
@@ -818,6 +823,7 @@ void MDSRankDispatcher::shutdown()
   // shutdown metrics handler/updater -- this is ok even if it was not
   // inited.
   metrics_handler.shutdown();
+  tracer.shutdown();
 
   // shutdown metric aggergator
   if (metric_aggregator != nullptr) {
@@ -2675,6 +2681,8 @@ void MDSRankDispatcher::handle_asok_command(
     if (!op_tracker.dump_ops_in_flight(f)) {
       *css << "op_tracker disabled; set mds_enable_op_tracker=true to enable";
     }
+  } else if (command == "trace dump") {
+    tracer.dump_traces(f);
   } else if (command == "ops") {
     vector<string> flags;
     cmd_getval(cmdmap, "flags", flags);
diff --git a/src/mds/MDSRank.h b/src/mds/MDSRank.h
index 80657432c43..426930dc8e8 100644
--- a/src/mds/MDSRank.h
+++ b/src/mds/MDSRank.h
@@ -30,6 +30,7 @@
 #include "SessionMap.h"
 #include "PurgeQueue.h"
 #include "MetricsHandler.h"
+#include "MDSTracer.h"
 
 // Full .h import instead of forward declaration for PerfCounter, for the
 // benefit of those including this header and using MDSRank::logger
@@ -427,6 +428,7 @@ class MDSRank {
     SessionMap sessionmap;
 
     PerfCounters *logger = nullptr, *mlogger = nullptr;
+    MDSTracer tracer;
     OpTracker op_tracker;
 
     std::map<ceph_tid_t, std::unique_ptr<MDSMetaRequest>> internal_client_requests;
diff --git a/src/mds/Migrator.cc b/src/mds/Migrator.cc
index b472637427d..efd035227f5 100644
--- a/src/mds/Migrator.cc
+++ b/src/mds/Migrator.cc
@@ -15,6 +15,7 @@
 
 #include "Migrator.h"
 #include "MDSRank.h"
+#include "MDSTracer.h"
 #include "MDCache.h"
 #include "CInode.h"
 #include "CDir.h"
@@ -1155,6 +1156,7 @@ public:
 
 void Migrator::dispatch_export_dir(const MDRequestRef& mdr, int count)
 {
+  ScopedSpan trace_span(mds->tracer, "export_dir", mdr->trace_span, &mdr->trace_data);
   CDir *dir = mdr->more()->export_dir;
   auto* diri = dir->get_inode();
   dout(7) << *mdr << " " << *dir << dendl;
diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index ea60c5030df..288a1ff78fe 100644
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -72,8 +72,10 @@
 #include "include/compat.h"
 #include "osd/OSDMap.h"
 #include "fscrypt.h"
+#include "MDSTracer.h"
 
 #include <list>
+#include <sstream>
 #include <regex>
 #include <string_view>
 #include <functional>
@@ -171,8 +173,14 @@ protected:
 
   MDRequestRef mdr;
   void pre_finish(int r) override {
-    if (mdr)
+    if (mdr) {
       mdr->mark_event("journal_committed: ");
+      // End the pending journal_wait span (started in journal_and_reply)
+      if (mdr->journal_span_id != 0) {
+        mdr->trace_data.end_pending_span(mdr->journal_span_id);
+        mdr->journal_span_id = 0;
+      }
+    }
   }
 public:
   explicit ServerLogContext(Server *s) : server(s) {
@@ -2116,6 +2124,12 @@ void Server::journal_and_reply(const MDRequestRef& mdr, CInode *in, CDentry *dn,
   dout(10) << "journal_and_reply tracei " << in << " tracedn " << dn << dendl;
   ceph_assert(!mdr->has_completed);
 
+  // Start pending span for async journal wait tracking
+  // This span will be completed in ServerLogContext::pre_finish when journal commits
+  if (!mdr->trace_data.trace_id.empty()) {
+    mdr->journal_span_id = mdr->trace_data.start_pending_span("journal_wait");
+  }
+
   // note trace items for eventual reply.
   mdr->tracei = in;
   if (in)
@@ -2161,7 +2175,13 @@ void Server::submit_mdlog_entry(LogEvent *le, MDSLogContextBase *fin, const MDRe
  */
 void Server::respond_to_request(const MDRequestRef& mdr, int r)
 {
+  // Trace child span: respond to request (duration captured automatically)
+  ScopedSpan trace_span(mds->tracer, "respond", mdr->trace_span, &mdr->trace_data);
+  trace_span.SetAttribute(MDSTracer::ATTR_RESULT, static_cast<int64_t>(r));
+
   mdr->result = r;
+  mdr->trace_data.result = r;
+
   if (mdr->client_request) {
     if (mdr->is_batch_head()) {
       dout(20) << __func__ << ": batch head " << *mdr << dendl;
@@ -2286,6 +2306,9 @@ void Server::perf_gather_op_latency(const cref_t<MClientRequest> &req, utime_t l
 
 void Server::early_reply(const MDRequestRef& mdr, CInode *tracei, CDentry *tracedn)
 {
+  // Trace child span: early reply (duration captured automatically)
+  ScopedSpan trace_span(mds->tracer, "early_reply", mdr->trace_span);
+
   if (!g_conf()->mds_early_reply)
     return;
 
@@ -2692,6 +2715,53 @@ void Server::handle_client_request(const cref_t<MClientRequest> &req)
     return;
   }
 
+  // Initialize tracing for this request
+  if (mds->tracer.is_enabled()) {
+    // Start or continue a trace
+    if (req->otel_trace.IsValid()) {
+      mdr->trace_span = mds->tracer.add_span("mds:client_request", req->otel_trace);
+    } else {
+      mdr->trace_span = mds->tracer.start_trace("mds:client_request");
+    }
+
+    // Set span attributes
+    const char* op_name = ceph_mds_op_name(req->get_op());
+    std::ostringstream reqid_ss;
+    reqid_ss << req->get_reqid();
+    std::string reqid_str = reqid_ss.str();
+    
+    mdr->trace_span->SetAttribute(MDSTracer::ATTR_OP_TYPE, req->get_op());
+    mdr->trace_span->SetAttribute(MDSTracer::ATTR_OP_NAME, op_name);
+    mdr->trace_span->SetAttribute(MDSTracer::ATTR_REQID, reqid_str);
+    mdr->trace_span->SetAttribute(MDSTracer::ATTR_MDS_RANK, static_cast<int64_t>(mds->get_nodeid()));
+
+    if (!req->get_filepath().empty()) {
+      std::string path = req->get_filepath().get_path();
+      mdr->trace_span->SetAttribute(MDSTracer::ATTR_PATH, path);
+    }
+
+    if (req->get_source().is_client()) {
+      mdr->trace_span->SetAttribute(MDSTracer::ATTR_CLIENT_ID, 
+                                    static_cast<int64_t>(req->get_source().num()));
+    }
+
+    // Populate trace_data for sliding window storage
+    mdr->trace_data.trace_id = MDSTracer::get_trace_id(mdr->trace_span);
+    mdr->trace_data.name = "mds:client_request";
+    mdr->trace_data.start_time = ceph_clock_now();
+    mdr->trace_data.attributes[MDSTracer::ATTR_OP_TYPE] = std::to_string(req->get_op());
+    mdr->trace_data.attributes[MDSTracer::ATTR_OP_NAME] = op_name;
+    mdr->trace_data.attributes[MDSTracer::ATTR_REQID] = reqid_str;
+    mdr->trace_data.attributes[MDSTracer::ATTR_MDS_RANK] = std::to_string(mds->get_nodeid());
+    if (!req->get_filepath().empty()) {
+      mdr->trace_data.attributes[MDSTracer::ATTR_PATH] = req->get_filepath().get_path();
+    }
+    if (req->get_source().is_client()) {
+      mdr->trace_data.attributes[MDSTracer::ATTR_CLIENT_ID] = 
+        std::to_string(req->get_source().num());
+    }
+  }
+
   if (session) {
     mdr->session = session;
     session->requests.push_back(&mdr->item_session_request);
@@ -2754,6 +2824,11 @@ void Server::dispatch_client_request(const MDRequestRef& mdr)
   // we shouldn't be waiting on anyone.
   ceph_assert(!mdr->has_more() || mdr->more()->waiting_on_peer.empty());
 
+  // Trace event: request dispatch
+  if (mdr->trace_span && mdr->trace_span->IsRecording()) {
+    mdr->trace_span->AddEvent("dispatch");
+  }
+
   if (mdr->killed) {
     // Should already be reset in request_cleanup().
     ceph_assert(!mdr->is_batch_head());
@@ -3538,6 +3613,7 @@ bool Server::check_dir_max_entries(const MDRequestRef& mdr, CDir *in)
 
 CDentry* Server::prepare_stray_dentry(const MDRequestRef& mdr, CInode *in)
 {
+  ScopedSpan trace_span(mds->tracer, "prepare_stray_dentry", mdr->trace_span, &mdr->trace_data);
   string straydname;
   in->name_stray_dentry(straydname);
 
@@ -3581,6 +3657,7 @@ CDentry* Server::prepare_stray_dentry(const MDRequestRef& mdr, CInode *in)
 CInode* Server::prepare_new_inode(const MDRequestRef& mdr, CDir *dir, inodeno_t useino, unsigned mode,
 				  const file_layout_t *layout, bool referent_inode)
 {
+  ScopedSpan trace_span(mds->tracer, "prepare_new_inode", mdr->trace_span, &mdr->trace_data);
   CInode *in = new CInode(mdcache);
   auto _inode = in->_get_inode();
   
@@ -3838,6 +3915,7 @@ CInode* Server::rdlock_path_pin_ref(const MDRequestRef& mdr,
 				    bool want_auth,
 				    bool no_want_auth)
 {
+  ScopedSpan trace_span(mds->tracer, "rdlock_path_pin_ref", mdr->trace_span, &mdr->trace_data);
   dout(10) << "rdlock_path_pin_ref " << *mdr << " " << refpath << dendl;
 
   if (mdr->locking_state & MutationImpl::PATH_LOCKED)
@@ -3915,6 +3993,7 @@ CDentry* Server::rdlock_path_xlock_dentry(const MDRequestRef& mdr,
 					  bool create, bool okexist, bool authexist,
 					  bool want_layout)
 {
+  ScopedSpan trace_span(mds->tracer, "rdlock_path_xlock_dentry", mdr->trace_span, &mdr->trace_data);
   const filepath& refpath = mdr->get_filepath();
   dout(10) << "rdlock_path_xlock_dentry " << *mdr << " " << refpath << dendl;
 
@@ -4223,6 +4302,7 @@ CDir* Server::try_open_auth_dirfrag(CInode *diri, frag_t fg, const MDRequestRef&
 
 void Server::handle_client_getattr(const MDRequestRef& mdr, bool is_lookup)
 {
+  ScopedSpan trace_span(mds->tracer, is_lookup ? "handle_lookup" : "handle_getattr", mdr->trace_span, &mdr->trace_data);
   const cref_t<MClientRequest> &req = mdr->client_request;
   client_t client = mdr->get_client();
 
@@ -4385,6 +4465,7 @@ struct C_MDS_LookupIno2 : public ServerContext {
 void Server::handle_client_lookup_ino(const MDRequestRef& mdr,
 				      bool want_parent, bool want_dentry)
 {
+  ScopedSpan trace_span(mds->tracer, "handle_lookup_ino", mdr->trace_span, &mdr->trace_data);
   const cref_t<MClientRequest> &req = mdr->client_request;
 
   if ((uint64_t)req->head.args.lookupino.snapid > 0)
@@ -4584,6 +4665,7 @@ void Server::_lookup_ino_2(const MDRequestRef& mdr, int r)
 
 void Server::handle_client_open(const MDRequestRef& mdr)
 {
+  ScopedSpan trace_span(mds->tracer, "handle_open", mdr->trace_span, &mdr->trace_data);
   const cref_t<MClientRequest> &req = mdr->client_request;
   dout(7) << "open on " << req->get_filepath() << dendl;
 
@@ -4840,6 +4922,7 @@ bool Server::can_handle_charmap(const MDRequestRef& mdr, CDentry* dn)
 
 void Server::handle_client_openc(const MDRequestRef& mdr)
 {
+  ScopedSpan trace_span(mds->tracer, "handle_create", mdr->trace_span, &mdr->trace_data);
   const cref_t<MClientRequest> &req = mdr->client_request;
   client_t client = mdr->get_client();
 
@@ -5037,6 +5120,7 @@ void Server::_finalize_readdir(const MDRequestRef& mdr,
 
 void Server::handle_client_readdir(const MDRequestRef& mdr)
 {
+  ScopedSpan trace_span(mds->tracer, "handle_readdir", mdr->trace_span, &mdr->trace_data);
   const cref_t<MClientRequest> &req = mdr->client_request;
   Session *session = mds->get_session(req);
   client_t client = req->get_source().num();
@@ -5326,6 +5410,7 @@ public:
 
 void Server::handle_client_file_setlock(const MDRequestRef& mdr)
 {
+  ScopedSpan trace_span(mds->tracer, "handle_file_setlock", mdr->trace_span, &mdr->trace_data);
   const cref_t<MClientRequest> &req = mdr->client_request;
   MutationImpl::LockOpVec lov;
 
@@ -5429,6 +5514,7 @@ void Server::handle_client_file_setlock(const MDRequestRef& mdr)
 
 void Server::handle_client_file_readlock(const MDRequestRef& mdr)
 {
+  ScopedSpan trace_span(mds->tracer, "handle_file_readlock", mdr->trace_span, &mdr->trace_data);
   const cref_t<MClientRequest> &req = mdr->client_request;
   MutationImpl::LockOpVec lov;
 
@@ -5482,6 +5568,7 @@ void Server::handle_client_file_readlock(const MDRequestRef& mdr)
 
 void Server::handle_client_setattr(const MDRequestRef& mdr)
 {
+  ScopedSpan trace_span(mds->tracer, "handle_setattr", mdr->trace_span, &mdr->trace_data);
   const cref_t<MClientRequest> &req = mdr->client_request;
   MutationImpl::LockOpVec lov;
   CInode *cur = rdlock_path_pin_ref(mdr, true);
@@ -5753,6 +5840,7 @@ void Server::do_open_truncate(const MDRequestRef& mdr, int cmode)
 /* This function cleans up the passed mdr */
 void Server::handle_client_setlayout(const MDRequestRef& mdr)
 {
+  ScopedSpan trace_span(mds->tracer, "handle_setlayout", mdr->trace_span, &mdr->trace_data);
   const cref_t<MClientRequest> &req = mdr->client_request;
   CInode *cur = rdlock_path_pin_ref(mdr, true);
   if (!cur) return;
@@ -5876,6 +5964,7 @@ CInode* Server::try_get_auth_inode(const MDRequestRef& mdr, inodeno_t ino)
 
 void Server::handle_client_setdirlayout(const MDRequestRef& mdr)
 {
+  ScopedSpan trace_span(mds->tracer, "handle_setdirlayout", mdr->trace_span, &mdr->trace_data);
   const cref_t<MClientRequest> &req = mdr->client_request;
 
   // can't use rdlock_path_pin_ref because we need to xlock snaplock/policylock
@@ -6247,6 +6336,7 @@ int Server::check_layout_vxattr(const MDRequestRef& mdr,
 
 void Server::handle_client_setvxattr(const MDRequestRef& mdr, CInode *cur)
 {
+  ScopedSpan trace_span(mds->tracer, "handle_setvxattr", mdr->trace_span, &mdr->trace_data);
   const cref_t<MClientRequest> &req = mdr->client_request;
   bool is_rmxattr = (req->get_op() == CEPH_MDS_OP_RMXATTR);
   MutationImpl::LockOpVec lov;
@@ -7052,6 +7142,7 @@ void Server::mirror_info_removexattr_handler(CInode *cur, InodeStoreBase::xattr_
 
 void Server::handle_client_setxattr(const MDRequestRef& mdr)
 {
+  ScopedSpan trace_span(mds->tracer, "handle_setxattr", mdr->trace_span, &mdr->trace_data);
   const cref_t<MClientRequest> &req = mdr->client_request;
   string name(req->get_path2());
 
@@ -7149,6 +7240,7 @@ void Server::handle_client_setxattr(const MDRequestRef& mdr)
 
 void Server::handle_client_removexattr(const MDRequestRef& mdr)
 {
+  ScopedSpan trace_span(mds->tracer, "handle_removexattr", mdr->trace_span, &mdr->trace_data);
   const cref_t<MClientRequest> &req = mdr->client_request;
   std::string name(req->get_path2());
 
@@ -7218,6 +7310,7 @@ void Server::handle_client_removexattr(const MDRequestRef& mdr)
 
 void Server::handle_client_getvxattr(const MDRequestRef& mdr)
 {
+  ScopedSpan trace_span(mds->tracer, "handle_getvxattr", mdr->trace_span, &mdr->trace_data);
   const auto& req = mdr->client_request;
   string xattr_name{req->get_path2()};
 
@@ -7483,6 +7576,7 @@ public:
 
 void Server::handle_client_mknod(const MDRequestRef& mdr)
 {
+  ScopedSpan trace_span(mds->tracer, "handle_mknod", mdr->trace_span, &mdr->trace_data);
   const cref_t<MClientRequest> &req = mdr->client_request;
   client_t client = mdr->get_client();
 
@@ -7589,6 +7683,7 @@ void Server::handle_client_mknod(const MDRequestRef& mdr)
 // MKDIR
 void Server::handle_client_mkdir(const MDRequestRef& mdr)
 {
+  ScopedSpan trace_span(mds->tracer, "handle_mkdir", mdr->trace_span, &mdr->trace_data);
   const cref_t<MClientRequest> &req = mdr->client_request;
 
   mdr->disable_lock_cache();
@@ -7693,6 +7788,7 @@ void Server::handle_client_mkdir(const MDRequestRef& mdr)
 
 void Server::handle_client_symlink(const MDRequestRef& mdr)
 {
+  ScopedSpan trace_span(mds->tracer, "handle_symlink", mdr->trace_span, &mdr->trace_data);
   const auto& req = mdr->client_request;
 
   mdr->disable_lock_cache();
@@ -7765,6 +7861,7 @@ void Server::handle_client_symlink(const MDRequestRef& mdr)
 
 void Server::handle_client_link(const MDRequestRef& mdr)
 {
+  ScopedSpan trace_span(mds->tracer, "handle_link", mdr->trace_span, &mdr->trace_data);
   const cref_t<MClientRequest> &req = mdr->client_request;
 
   dout(7) << "handle_client_link " << req->get_filepath()
@@ -8628,6 +8725,7 @@ void Server::handle_peer_link_prep_ack(const MDRequestRef& mdr, const cref_t<MMD
 
 void Server::handle_client_unlink(const MDRequestRef& mdr)
 {
+  ScopedSpan trace_span(mds->tracer, "handle_unlink", mdr->trace_span, &mdr->trace_data);
   const cref_t<MClientRequest> &req = mdr->client_request;
   client_t client = mdr->get_client();
 
@@ -9445,6 +9543,7 @@ public:
  */
 void Server::handle_client_rename(const MDRequestRef& mdr)
 {
+  ScopedSpan trace_span(mds->tracer, "handle_rename", mdr->trace_span, &mdr->trace_data);
   const auto& req = mdr->client_request;
   dout(7) << "handle_client_rename " << *req << dendl;
 
@@ -11985,6 +12084,7 @@ void Server::_peer_rename_sessions_flushed(const MDRequestRef& mdr)
 // snaps
 void Server::handle_client_lssnap(const MDRequestRef& mdr)
 {
+  ScopedSpan trace_span(mds->tracer, "handle_lssnap", mdr->trace_span, &mdr->trace_data);
   const cref_t<MClientRequest> &req = mdr->client_request;
 
   // traverse to path
@@ -12094,6 +12194,7 @@ struct C_MDS_mksnap_finish : public ServerLogContext {
 
 void Server::handle_client_mksnap(const MDRequestRef& mdr)
 {
+  ScopedSpan trace_span(mds->tracer, "handle_mksnap", mdr->trace_span, &mdr->trace_data);
   const cref_t<MClientRequest> &req = mdr->client_request;
   // make sure we have as new a map as the client
   if (req->get_mdsmap_epoch() > mds->mdsmap->get_epoch()) {
@@ -12290,6 +12391,7 @@ struct C_MDS_rmsnap_finish : public ServerLogContext {
 
 void Server::handle_client_rmsnap(const MDRequestRef& mdr)
 {
+  ScopedSpan trace_span(mds->tracer, "handle_rmsnap", mdr->trace_span, &mdr->trace_data);
   const cref_t<MClientRequest> &req = mdr->client_request;
 
   CInode *diri = try_get_auth_inode(mdr, req->get_filepath().get_ino());
@@ -12419,6 +12521,7 @@ struct C_MDS_renamesnap_finish : public ServerLogContext {
 
 void Server::handle_client_renamesnap(const MDRequestRef& mdr)
 {
+  ScopedSpan trace_span(mds->tracer, "handle_renamesnap", mdr->trace_span, &mdr->trace_data);
   const cref_t<MClientRequest> &req = mdr->client_request;
   if (req->get_filepath().get_ino() != req->get_filepath2().get_ino()) {
     respond_to_request(mdr, -EINVAL);
@@ -12570,6 +12673,7 @@ public:
 
 void Server::handle_client_file_blockdiff(const MDRequestRef& mdr)
 {
+  ScopedSpan trace_span(mds->tracer, "handle_file_blockdiff", mdr->trace_span, &mdr->trace_data);
   const cref_t<MClientRequest>& req = mdr->client_request;
 
   dout(10) << __func__ << dendl;
diff --git a/src/mds/StrayManager.cc b/src/mds/StrayManager.cc
index af5bc1cf80e..f1faa298516 100644
--- a/src/mds/StrayManager.cc
+++ b/src/mds/StrayManager.cc
@@ -16,6 +16,7 @@
 #include "StrayManager.h"
 #include "BatchOp.h"
 #include "MDSRank.h"
+#include "MDSTracer.h"
 #include "Mutation.h"
 
 #include "common/debug.h"