From 5d4c5e65be94a346e62b9f8ac63241779c4c7a6a Mon Sep 17 00:00:00 2001
From: Omri Zeneva <ozeneva@redhat.com>
Date: Wed, 24 Aug 2022 09:57:11 -0400
Subject: [PATCH] tracer/osd/librados/build/rgw: rgw and osd end2end tracing
 using opentelemetry

* build: add opentelemetry to cmake system
crimson targets that uses Message.cc/h are built before opentelemetry (o-tel), so we need to build o-tel eralier so we also add the library to the include path earlier
this shoud work for WITH_JAEGER flag both the ON/OFF cases, and for librados where the compilation flag is ignored

* msg/tracer: add o-tel trace to Messages with decode/encode function in tracer.h
some files that uses Message.cc/h just need the encode/decode functions  and not all others functions.
some crimson targets does not link with ceph_context (common) which is required for tracer.cc file. so we just need to include that functions

* librados: Add opentelemtry trace param for aio_operate and operate methods
in order to propagate the trace info I added the otel-trace as an extra param.
in some places, there already was a blkin trace info, and since it is not used in other places we can safely change it to o-tel trace info.
this will be done in another commit, so the cleanup of blkin trace will be in a dedicated commit

* osd: use the o-tel trace of the msg as a parent span of the osd trace
if there is a valid span in the msg, we will add this op to the request
trace, otherwise it will start a new trace for the OSD op

* rgw: pass put obj trace info to librados
in order to make it possible, I saved the trace info inside the sal::Object, so we can use it later when writing the object to rados
it could be used also later for read ops.
note the trace field of req_state is initalized only in rgw_process, so it's also required in librgw request flow

Signed-off-by: Omri Zeneva <ozeneva@redhat.com>
---
 cmake/modules/BuildOpentelemetry.cmake       |  1 +
 src/CMakeLists.txt                           | 15 +++--
 src/common/tracer.cc                         | 37 +-----------
 src/common/tracer.h                          | 60 +++++++++++++++++---
 src/include/rados/librados.hpp               |  4 +-
 src/include/rados/librados_fwd.hpp           | 12 ++++
 src/librados/IoCtxImpl.cc                    |  8 +--
 src/librados/IoCtxImpl.h                     |  4 +-
 src/librados/librados_asio.h                 |  6 +-
 src/librados/librados_cxx.cc                 |  8 +--
 src/messages/MOSDOp.h                        | 16 +++++-
 src/msg/Message.cc                           |  9 +++
 src/msg/Message.h                            |  6 ++
 src/osd/OSD.cc                               |  7 ++-
 src/osdc/Objecter.cc                         |  4 ++
 src/osdc/Objecter.h                          | 12 ++--
 src/rgw/driver/rados/rgw_putobj_processor.cc | 10 ++--
 src/rgw/driver/rados/rgw_putobj_processor.h  | 24 ++++----
 src/rgw/driver/rados/rgw_rados.cc            | 31 ++++++----
 src/rgw/driver/rados/rgw_rados.h             |  7 ++-
 src/rgw/driver/rados/rgw_sal_rados.cc        | 15 ++---
 src/rgw/driver/rados/rgw_sal_rados.h         | 14 +++--
 src/rgw/driver/rados/rgw_tools.cc            |  8 +--
 src/rgw/driver/rados/rgw_tools.h             |  4 +-
 src/rgw/rgw_aio.cc                           | 23 ++++----
 src/rgw/rgw_aio.h                            |  2 +-
 src/rgw/rgw_lib.cc                           |  2 +
 src/rgw/rgw_op.cc                            |  2 +
 src/rgw/rgw_sal.h                            |  5 +-
 src/rgw/rgw_sal_filter.h                     |  5 +-
 src/rgw/rgw_sal_store.h                      |  5 +-
 src/rgw/services/svc_rados.cc                |  4 +-
 src/rgw/services/svc_rados.h                 |  2 +-
 33 files changed, 233 insertions(+), 139 deletions(-)

diff --git a/cmake/modules/BuildOpentelemetry.cmake b/cmake/modules/BuildOpentelemetry.cmake
index ba2edaa09329d..48b219e9c0fc2 100644
--- a/cmake/modules/BuildOpentelemetry.cmake
+++ b/cmake/modules/BuildOpentelemetry.cmake
@@ -82,4 +82,5 @@ function(build_opentelemetry)
     PROPERTIES
       INTERFACE_LINK_LIBRARIES "${opentelemetry_deps}"
       INTERFACE_INCLUDE_DIRECTORIES "${opentelemetry_include_dir}")
+  include_directories(SYSTEM "${opentelemetry_include_dir}")
 endfunction()
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 0a841c5db37ab..44b1185673e29 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -299,6 +299,15 @@ if (WITH_BLKIN)
   add_subdirectory(blkin/blkin-lib)
 endif(WITH_BLKIN)
 
+if(WITH_JAEGER)
+  find_package(thrift 0.13.0 REQUIRED)
+  include(BuildOpentelemetry)
+  build_opentelemetry()
+  add_library(jaeger_base INTERFACE)
+  target_link_libraries(jaeger_base INTERFACE opentelemetry::libopentelemetry
+    thrift::libthrift)
+endif()
+
 set(mds_files)
 list(APPEND mds_files
   mds/MDSMap.cc
@@ -432,12 +441,6 @@ target_compile_definitions(common-objs PRIVATE
 add_dependencies(common-objs legacy-option-headers)
 
 if(WITH_JAEGER)
-  find_package(thrift 0.13.0 REQUIRED)
-  include(BuildOpentelemetry)
-  build_opentelemetry()
-  add_library(jaeger_base INTERFACE)
-  target_link_libraries(jaeger_base INTERFACE opentelemetry::libopentelemetry
-    thrift::libthrift)
   add_dependencies(common-objs jaeger_base)
   target_link_libraries(common-objs jaeger_base)
 endif()
diff --git a/src/common/tracer.cc b/src/common/tracer.cc
index ffabc0b20a111..2e02474ad0684 100644
--- a/src/common/tracer.cc
+++ b/src/common/tracer.cc
@@ -52,7 +52,7 @@ jspan Tracer::start_trace(opentelemetry::nostd::string_view trace_name, bool tra
 }
 
 jspan Tracer::add_span(opentelemetry::nostd::string_view span_name, const jspan& parent_span) {
-  if (is_enabled() && parent_span->IsRecording()) {
+  if (is_enabled() && parent_span && parent_span->IsRecording()) {
     opentelemetry::trace::StartSpanOptions span_opts;
     span_opts.parent = parent_span->GetContext();
     return tracer->StartSpan(span_name, span_opts);
@@ -73,41 +73,6 @@ bool Tracer::is_enabled() const {
   return g_ceph_context->_conf->jaeger_tracing_enable;
 }
 
-void encode(const jspan_context& span_ctx, bufferlist& bl, uint64_t f) {
-  ENCODE_START(1, 1, bl);
-  using namespace opentelemetry;
-  using namespace trace;
-  auto is_valid = span_ctx.IsValid();
-  encode(is_valid, bl);
-  if (is_valid) {
-    encode_nohead(std::string_view(reinterpret_cast<const char*>(span_ctx.trace_id().Id().data()), TraceId::kSize), bl);
-    encode_nohead(std::string_view(reinterpret_cast<const char*>(span_ctx.span_id().Id().data()), SpanId::kSize), bl);
-    encode(span_ctx.trace_flags().flags(), bl);
-  }
-  ENCODE_FINISH(bl);
-}
-
-void decode(jspan_context& span_ctx, bufferlist::const_iterator& bl) {
-  using namespace opentelemetry;
-  using namespace trace;
-  DECODE_START(1, bl);
-  bool is_valid;
-  decode(is_valid, bl);
-  if (is_valid) {
-    std::array<uint8_t, TraceId::kSize> trace_id;
-    std::array<uint8_t, SpanId::kSize> span_id;
-    uint8_t flags;
-    decode(trace_id, bl);
-    decode(span_id, bl);
-    decode(flags, bl);
-    span_ctx = SpanContext(
-      TraceId(nostd::span<uint8_t, TraceId::kSize>(trace_id)),
-      SpanId(nostd::span<uint8_t, SpanId::kSize>(span_id)),
-      TraceFlags(flags),
-      true);
-  }
-  DECODE_FINISH(bl);
-}
 } // namespace tracing
 
 #endif // HAVE_JAEGER
diff --git a/src/common/tracer.h b/src/common/tracer.h
index 9d13c78aafcee..f0a4df5120776 100644
--- a/src/common/tracer.h
+++ b/src/common/tracer.h
@@ -4,7 +4,7 @@
 #pragma once
 
 #include "acconfig.h"
-#include "include/buffer.h"
+#include "include/encoding.h"
 
 #ifdef HAVE_JAEGER
 #include "opentelemetry/trace/provider.h"
@@ -18,10 +18,11 @@ namespace tracing {
 class Tracer {
  private:
   const static opentelemetry::nostd::shared_ptr<opentelemetry::trace::Tracer> noop_tracer;
-  const static jspan noop_span;
   opentelemetry::nostd::shared_ptr<opentelemetry::trace::Tracer> tracer;
 
  public:
+  const static jspan noop_span;
+
   Tracer() = default;
   Tracer(opentelemetry::nostd::string_view service_name);
 
@@ -45,8 +46,41 @@ class Tracer {
 
 };
 
-void encode(const jspan_context& span, ceph::buffer::list& bl, uint64_t f = 0);
-void decode(jspan_context& span_ctx, ceph::buffer::list::const_iterator& bl);
+inline void encode(const jspan_context& span_ctx, bufferlist& bl, uint64_t f = 0) {
+  ENCODE_START(1, 1, bl);
+  using namespace opentelemetry;
+  using namespace trace;
+  auto is_valid = span_ctx.IsValid();
+  encode(is_valid, bl);
+  if (is_valid) {
+    encode_nohead(std::string_view(reinterpret_cast<const char*>(span_ctx.trace_id().Id().data()), TraceId::kSize), bl);
+    encode_nohead(std::string_view(reinterpret_cast<const char*>(span_ctx.span_id().Id().data()), SpanId::kSize), bl);
+    encode(span_ctx.trace_flags().flags(), bl);
+  }
+  ENCODE_FINISH(bl);
+}
+
+inline void decode(jspan_context& span_ctx, bufferlist::const_iterator& bl) {
+  using namespace opentelemetry;
+  using namespace trace;
+  DECODE_START(1, bl);
+  bool is_valid;
+  decode(is_valid, bl);
+  if (is_valid) {
+    std::array<uint8_t, TraceId::kSize> trace_id;
+    std::array<uint8_t, SpanId::kSize> span_id;
+    uint8_t flags;
+    decode(trace_id, bl);
+    decode(span_id, bl);
+    decode(flags, bl);
+    span_ctx = SpanContext(
+      TraceId(nostd::span<uint8_t, TraceId::kSize>(trace_id)),
+      SpanId(nostd::span<uint8_t, SpanId::kSize>(span_id)),
+      TraceFlags(flags),
+      true);
+  }
+  DECODE_FINISH(bl);
+}
 
 } // namespace tracing
 
@@ -62,10 +96,20 @@ class Value {
 
 using jspan_attribute = Value;
 
-struct jspan_context {
-  jspan_context() {}
-  jspan_context(bool sampled_flag, bool is_remote) {}
+namespace opentelemetry {
+inline namespace v1 {
+namespace trace {
+class SpanContext {
+public:
+  SpanContext() = default;
+  SpanContext(bool sampled_flag, bool is_remote) {}
+  bool IsValid() const { return false;}
 };
+} // namespace trace
+} // namespace v1
+} // namespace opentelemetry
+
+using jspan_context = opentelemetry::v1::trace::SpanContext;
 
 struct span_stub {
   jspan_context _ctx;
@@ -74,7 +118,7 @@ struct span_stub {
   void AddEvent(std::string_view) {}
   void AddEvent(std::string_view, std::initializer_list<std::pair<std::string_view, jspan_attribute>> fields) {}
   template <typename T> void AddEvent(std::string_view name, const T& fields = {}) {}
-  const jspan_context& GetContext() { return _ctx; }
+  jspan_context GetContext() const { return _ctx; }
   void UpdateName(std::string_view) {}
   bool IsRecording() { return false; }
 };
diff --git a/src/include/rados/librados.hpp b/src/include/rados/librados.hpp
index cb8261af12d24..3e216b85f2e04 100644
--- a/src/include/rados/librados.hpp
+++ b/src/include/rados/librados.hpp
@@ -1168,11 +1168,11 @@ inline namespace v14_2_0 {
 
     // compound object operations
     int operate(const std::string& oid, ObjectWriteOperation *op);
-    int operate(const std::string& oid, ObjectWriteOperation *op, int flags);
+    int operate(const std::string& oid, ObjectWriteOperation *op, int flags, const jspan_context *trace_info = nullptr);
     int operate(const std::string& oid, ObjectReadOperation *op, bufferlist *pbl);
     int operate(const std::string& oid, ObjectReadOperation *op, bufferlist *pbl, int flags);
     int aio_operate(const std::string& oid, AioCompletion *c, ObjectWriteOperation *op);
-    int aio_operate(const std::string& oid, AioCompletion *c, ObjectWriteOperation *op, int flags);
+    int aio_operate(const std::string& oid, AioCompletion *c, ObjectWriteOperation *op, int flags, const jspan_context *trace_info = nullptr);
     /**
      * Schedule an async write operation with explicit snapshot parameters
      *
diff --git a/src/include/rados/librados_fwd.hpp b/src/include/rados/librados_fwd.hpp
index 396f3a8387575..d9a455adb38a7 100644
--- a/src/include/rados/librados_fwd.hpp
+++ b/src/include/rados/librados_fwd.hpp
@@ -3,6 +3,18 @@
 
 struct blkin_trace_info;
 
+namespace opentelemetry {
+inline namespace v1 {
+namespace trace {
+
+class SpanContext;
+
+} // namespace trace
+} // inline namespace v1
+} // namespace opentelemetry
+
+using jspan_context = opentelemetry::v1::trace::SpanContext;
+
 namespace libradosstriper {
 
 class RadosStriper;
diff --git a/src/librados/IoCtxImpl.cc b/src/librados/IoCtxImpl.cc
index b9a73c87ca861..9a087c6eeb7b1 100644
--- a/src/librados/IoCtxImpl.cc
+++ b/src/librados/IoCtxImpl.cc
@@ -637,7 +637,7 @@ int librados::IoCtxImpl::writesame(const object_t& oid, bufferlist& bl,
 }
 
 int librados::IoCtxImpl::operate(const object_t& oid, ::ObjectOperation *o,
-				 ceph::real_time *pmtime, int flags)
+				 ceph::real_time *pmtime, int flags, const jspan_context* otel_trace)
 {
   ceph::real_time ut = (pmtime ? *pmtime :
     ceph::real_clock::now());
@@ -664,7 +664,7 @@ int librados::IoCtxImpl::operate(const object_t& oid, ::ObjectOperation *o,
     oid, oloc,
     *o, snapc, ut,
     flags | extra_op_flags,
-    oncommit, &ver);
+    oncommit, &ver, osd_reqid_t(), nullptr, otel_trace);
   objecter->op_submit(objecter_op);
 
   {
@@ -752,7 +752,7 @@ int librados::IoCtxImpl::aio_operate_read(const object_t &oid,
 int librados::IoCtxImpl::aio_operate(const object_t& oid,
 				     ::ObjectOperation *o, AioCompletionImpl *c,
 				     const SnapContext& snap_context, int flags,
-                                     const blkin_trace_info *trace_info)
+                                     const blkin_trace_info *trace_info, const jspan_context *otel_trace)
 {
   FUNCTRACE(client->cct);
   OID_EVENT_TRACE(oid.name.c_str(), "RADOS_WRITE_OP_BEGIN");
@@ -778,7 +778,7 @@ int librados::IoCtxImpl::aio_operate(const object_t& oid,
   trace.event("init root span");
   Objecter::Op *op = objecter->prepare_mutate_op(
     oid, oloc, *o, snap_context, ut, flags | extra_op_flags,
-    oncomplete, &c->objver, osd_reqid_t(), &trace);
+    oncomplete, &c->objver, osd_reqid_t(), &trace, otel_trace);
   objecter->op_submit(op, &c->tid);
   trace.event("rados operate op submitted");
 
diff --git a/src/librados/IoCtxImpl.h b/src/librados/IoCtxImpl.h
index 4ab8a5b747b07..19a13b2d631b3 100644
--- a/src/librados/IoCtxImpl.h
+++ b/src/librados/IoCtxImpl.h
@@ -154,11 +154,11 @@ struct librados::IoCtxImpl {
   int getxattrs(const object_t& oid, std::map<std::string, bufferlist>& attrset);
   int rmxattr(const object_t& oid, const char *name);
 
-  int operate(const object_t& oid, ::ObjectOperation *o, ceph::real_time *pmtime, int flags=0);
+  int operate(const object_t& oid, ::ObjectOperation *o, ceph::real_time *pmtime, int flags=0, const jspan_context *otel_trace = nullptr);
   int operate_read(const object_t& oid, ::ObjectOperation *o, bufferlist *pbl, int flags=0);
   int aio_operate(const object_t& oid, ::ObjectOperation *o,
 		  AioCompletionImpl *c, const SnapContext& snap_context,
-		  int flags, const blkin_trace_info *trace_info = nullptr);
+		  int flags, const blkin_trace_info *trace_info = nullptr, const jspan_context *otel_trace = nullptr);
   int aio_operate_read(const object_t& oid, ::ObjectOperation *o,
 		       AioCompletionImpl *c, int flags, bufferlist *pbl, const blkin_trace_info *trace_info = nullptr);
 
diff --git a/src/librados/librados_asio.h b/src/librados/librados_asio.h
index bd672d951f730..2eae1c268f6cc 100644
--- a/src/librados/librados_asio.h
+++ b/src/librados/librados_asio.h
@@ -152,7 +152,7 @@ auto async_write(ExecutionContext& ctx, IoCtx& io, const std::string& oid,
 template <typename ExecutionContext, typename CompletionToken>
 auto async_operate(ExecutionContext& ctx, IoCtx& io, const std::string& oid,
                    ObjectReadOperation *read_op, int flags,
-                   CompletionToken&& token)
+                   CompletionToken&& token, const jspan_context* trace_ctx = nullptr)
 {
   using Op = detail::AsyncOp<bufferlist>;
   using Signature = typename Op::Signature;
@@ -176,7 +176,7 @@ auto async_operate(ExecutionContext& ctx, IoCtx& io, const std::string& oid,
 template <typename ExecutionContext, typename CompletionToken>
 auto async_operate(ExecutionContext& ctx, IoCtx& io, const std::string& oid,
                    ObjectWriteOperation *write_op, int flags,
-                   CompletionToken &&token)
+                   CompletionToken &&token, const jspan_context* trace_ctx = nullptr)
 {
   using Op = detail::AsyncOp<void>;
   using Signature = typename Op::Signature;
@@ -184,7 +184,7 @@ auto async_operate(ExecutionContext& ctx, IoCtx& io, const std::string& oid,
   auto p = Op::create(ctx.get_executor(), init.completion_handler);
   auto& op = p->user_data;
 
-  int ret = io.aio_operate(oid, op.aio_completion.get(), write_op, flags);
+  int ret = io.aio_operate(oid, op.aio_completion.get(), write_op, flags, trace_ctx);
   if (ret < 0) {
     auto ec = boost::system::error_code{-ret, librados::detail::err_category()};
     ceph::async::post(std::move(p), ec);
diff --git a/src/librados/librados_cxx.cc b/src/librados/librados_cxx.cc
index a14519b57becf..0661f08fd51bd 100644
--- a/src/librados/librados_cxx.cc
+++ b/src/librados/librados_cxx.cc
@@ -1528,12 +1528,12 @@ int librados::IoCtx::operate(const std::string& oid, librados::ObjectWriteOperat
   return io_ctx_impl->operate(obj, &o->impl->o, (ceph::real_time *)o->impl->prt);
 }
 
-int librados::IoCtx::operate(const std::string& oid, librados::ObjectWriteOperation *o, int flags)
+int librados::IoCtx::operate(const std::string& oid, librados::ObjectWriteOperation *o, int flags, const jspan_context* otel_trace)
 {
   object_t obj(oid);
   if (unlikely(!o->impl))
     return -EINVAL;
-  return io_ctx_impl->operate(obj, &o->impl->o, (ceph::real_time *)o->impl->prt, translate_flags(flags));
+  return io_ctx_impl->operate(obj, &o->impl->o, (ceph::real_time *)o->impl->prt, translate_flags(flags), otel_trace);
 }
 
 int librados::IoCtx::operate(const std::string& oid, librados::ObjectReadOperation *o, bufferlist *pbl)
@@ -1562,14 +1562,14 @@ int librados::IoCtx::aio_operate(const std::string& oid, AioCompletion *c,
 				  io_ctx_impl->snapc, 0);
 }
 int librados::IoCtx::aio_operate(const std::string& oid, AioCompletion *c,
-				 ObjectWriteOperation *o, int flags)
+				 ObjectWriteOperation *o, int flags, const jspan_context* otel_trace)
 {
   object_t obj(oid);
   if (unlikely(!o->impl))
     return -EINVAL;
   return io_ctx_impl->aio_operate(obj, &o->impl->o, c->pc,
 				  io_ctx_impl->snapc,
-				  translate_flags(flags));
+				  translate_flags(flags), nullptr, otel_trace);
 }
 
 int librados::IoCtx::aio_operate(const std::string& oid, AioCompletion *c,
diff --git a/src/messages/MOSDOp.h b/src/messages/MOSDOp.h
index 2275c45883229..61a35ba2f06ba 100644
--- a/src/messages/MOSDOp.h
+++ b/src/messages/MOSDOp.h
@@ -36,7 +36,7 @@ namespace _mosdop {
 template<typename V>
 class MOSDOp final : public MOSDFastDispatchOp {
 private:
-  static constexpr int HEAD_VERSION = 8;
+  static constexpr int HEAD_VERSION = 9;
   static constexpr int COMPAT_VERSION = 3;
 
 private:
@@ -361,8 +361,7 @@ struct ceph_osd_request_head {
       encode(retry_attempt, payload);
       encode(features, payload);
     } else {
-      // latest v8 encoding with hobject_t hash separate from pgid, no
-      // reassert version
+      // latest v9 opentelemetry trace
       header.version = HEAD_VERSION;
 
       encode(pgid, payload);
@@ -371,6 +370,7 @@ struct ceph_osd_request_head {
       encode(flags, payload);
       encode(reqid, payload);
       encode_trace(payload, features);
+      encode_otel_trace(payload, features);
 
       // -- above decoded up front; below decoded post-dispatch thread --
 
@@ -400,6 +400,16 @@ struct ceph_osd_request_head {
 
     // Always keep here the newest version of decoding order/rule
     if (header.version == HEAD_VERSION) {
+      decode(pgid, p);
+      uint32_t hash;
+      decode(hash, p);
+      hobj.set_hash(hash);
+      decode(osdmap_epoch, p);
+      decode(flags, p);
+      decode(reqid, p);
+      decode_trace(p);
+      decode_otel_trace(p);
+    } else if (header.version == 8) {
       decode(pgid, p);      // actual pgid
       uint32_t hash;
       decode(hash, p); // raw hash value
diff --git a/src/msg/Message.cc b/src/msg/Message.cc
index e10981794b4e8..3f91b07706a19 100644
--- a/src/msg/Message.cc
+++ b/src/msg/Message.cc
@@ -1023,6 +1023,15 @@ void Message::decode_trace(ceph::bufferlist::const_iterator &p, bool create)
 #endif
 }
 
+void Message::encode_otel_trace(ceph::bufferlist &bl, uint64_t features) const
+{
+  tracing::encode(otel_trace, bl);
+}
+
+void Message::decode_otel_trace(ceph::bufferlist::const_iterator &p, bool create)
+{
+  tracing::decode(otel_trace, p);
+}
 
 // This routine is not used for ordinary messages, but only when encapsulating a message
 // for forwarding and routing.  It's also used in a backward compatibility test, which only
diff --git a/src/msg/Message.h b/src/msg/Message.h
index 6ebf06346c3f0..ede8d4121d167 100644
--- a/src/msg/Message.h
+++ b/src/msg/Message.h
@@ -32,6 +32,7 @@
 #include "common/ref.h"
 #include "common/debug.h"
 #include "common/zipkin_trace.h"
+#include "common/tracer.h"
 #include "include/ceph_assert.h" // Because intrusive_ptr clobbers our assert...
 #include "include/buffer.h"
 #include "include/types.h"
@@ -288,6 +289,11 @@ public:
   void encode_trace(ceph::buffer::list &bl, uint64_t features) const;
   void decode_trace(ceph::buffer::list::const_iterator &p, bool create = false);
 
+  // otel tracing
+  jspan_context otel_trace{false, false};
+  void encode_otel_trace(ceph::buffer::list &bl, uint64_t features) const;
+  void decode_otel_trace(ceph::buffer::list::const_iterator &p, bool create = false);
+
   class CompletionHook : public Context {
   protected:
     Message *m;
diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 6a66b73e3e732..35bf0e6409a94 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -7372,7 +7372,12 @@ void OSD::ms_fast_dispatch(Message *m)
     tracepoint(osd, ms_fast_dispatch, reqid.name._type,
         reqid.name._num, reqid.tid, reqid.inc);
   }
-  op->osd_parent_span = tracing::osd::tracer.start_trace("op-request-created");
+
+  if (m->otel_trace.IsValid()) {
+    op->osd_parent_span = tracing::osd::tracer.add_span("op-request-created", m->otel_trace);
+  } else {
+    op->osd_parent_span = tracing::osd::tracer.start_trace("op-request-created");
+  }
 
   if (m->trace)
     op->osd_trace.init("osd op", &trace_endpoint, &m->trace);
diff --git a/src/osdc/Objecter.cc b/src/osdc/Objecter.cc
index f006597e82739..2af40138cf6dc 100644
--- a/src/osdc/Objecter.cc
+++ b/src/osdc/Objecter.cc
@@ -3232,6 +3232,10 @@ Objecter::MOSDOp *Objecter::_prepare_osd_op(Op *op)
     m->set_reqid(op->reqid);
   }
 
+  if (op->otel_trace && op->otel_trace->IsValid()) {
+     m->otel_trace = jspan_context(*op->otel_trace);
+  }
+
   logger->inc(l_osdc_op_send);
   ssize_t sum = 0;
   for (unsigned i = 0; i < m->ops.size(); i++) {
diff --git a/src/osdc/Objecter.h b/src/osdc/Objecter.h
index d9d723dca747a..0cdd93e43716a 100644
--- a/src/osdc/Objecter.h
+++ b/src/osdc/Objecter.h
@@ -49,6 +49,7 @@
 #include "common/config_obs.h"
 #include "common/shunique_lock.h"
 #include "common/zipkin_trace.h"
+#include "common/tracer.h"
 #include "common/Throttle.h"
 
 #include "mon/MonClient.h"
@@ -1957,6 +1958,7 @@ public:
 
     osd_reqid_t reqid; // explicitly setting reqid
     ZTracer::Trace trace;
+    const jspan_context* otel_trace = nullptr;
 
     static bool has_completion(decltype(onfinish)& f) {
       return std::visit([](auto&& arg) { return bool(arg);}, f);
@@ -2006,7 +2008,7 @@ public:
 
     Op(const object_t& o, const object_locator_t& ol, osdc_opvec&& _ops,
        int f, Context* fin, version_t *ov, int *offset = nullptr,
-       ZTracer::Trace *parent_trace = nullptr) :
+       ZTracer::Trace *parent_trace = nullptr, const jspan_context *otel_trace = nullptr) :
       target(o, ol, f),
       ops(std::move(_ops)),
       out_bl(ops.size(), nullptr),
@@ -2015,7 +2017,8 @@ public:
       out_ec(ops.size(), nullptr),
       onfinish(fin),
       objver(ov),
-      data_offset(offset) {
+      data_offset(offset),
+      otel_trace(otel_trace) {
       if (target.base_oloc.key == o)
 	target.base_oloc.key.clear();
       if (parent_trace && parent_trace->valid()) {
@@ -2917,10 +2920,11 @@ public:
     ceph::real_time mtime, int flags,
     Context *oncommit, version_t *objver = NULL,
     osd_reqid_t reqid = osd_reqid_t(),
-    ZTracer::Trace *parent_trace = nullptr) {
+    ZTracer::Trace *parent_trace = nullptr,
+    const jspan_context *otel_trace = nullptr) {
     Op *o = new Op(oid, oloc, std::move(op.ops), flags | global_op_flags |
 		   CEPH_OSD_FLAG_WRITE, oncommit, objver,
-		   nullptr, parent_trace);
+		   nullptr, nullptr, otel_trace);
     o->priority = op.priority;
     o->mtime = mtime;
     o->snapc = snapc;
diff --git a/src/rgw/driver/rados/rgw_putobj_processor.cc b/src/rgw/driver/rados/rgw_putobj_processor.cc
index d52e303291d1a..106eb8ff86f88 100644
--- a/src/rgw/driver/rados/rgw_putobj_processor.cc
+++ b/src/rgw/driver/rados/rgw_putobj_processor.cc
@@ -111,7 +111,7 @@ int RadosWriter::process(bufferlist&& bl, uint64_t offset)
   }
   constexpr uint64_t id = 0; // unused
   auto& ref = stripe_obj.get_ref();
-  auto c = aio->get(ref.obj, Aio::librados_op(ref.pool.ioctx(), std::move(op), y), cost, id);
+  auto c = aio->get(ref.obj, Aio::librados_op(ref.pool.ioctx(), std::move(op), y, &trace), cost, id);
   return process_completed(c, &written);
 }
 
@@ -126,7 +126,7 @@ int RadosWriter::write_exclusive(const bufferlist& data)
 
   constexpr uint64_t id = 0; // unused
   auto& ref = stripe_obj.get_ref();
-  auto c = aio->get(ref.obj, Aio::librados_op(ref.pool.ioctx(), std::move(op), y), cost, id);
+  auto c = aio->get(ref.obj, Aio::librados_op(ref.pool.ioctx(), std::move(op), y, &trace), cost, id);
   auto d = aio->drain();
   c.splice(c.end(), d);
   return process_completed(c, &written);
@@ -341,7 +341,7 @@ int AtomicObjectProcessor::complete(size_t accounted_size,
   obj_op.meta.zones_trace = zones_trace;
   obj_op.meta.modify_tail = true;
 
-  r = obj_op.write_meta(dpp, actual_size, accounted_size, attrs, y);
+  r = obj_op.write_meta(dpp, actual_size, accounted_size, attrs, y, writer.get_trace());
   if (r < 0) {
     if (r == -ETIMEDOUT) {
       // The head object write may eventually succeed, clear the set of objects for deletion. if it
@@ -469,7 +469,7 @@ int MultipartObjectProcessor::complete(size_t accounted_size,
   obj_op.meta.zones_trace = zones_trace;
   obj_op.meta.modify_tail = true;
 
-  r = obj_op.write_meta(dpp, actual_size, accounted_size, attrs, y);
+  r = obj_op.write_meta(dpp, actual_size, accounted_size, attrs, y, writer.get_trace());
   if (r < 0)
     return r;
 
@@ -707,7 +707,7 @@ int AppendObjectProcessor::complete(size_t accounted_size, const string &etag, c
   }
   r = obj_op.write_meta(dpp, actual_size + cur_size,
 			accounted_size + *cur_accounted_size,
-			attrs, y);
+			attrs, y, writer.get_trace());
   if (r < 0) {
     return r;
   }
diff --git a/src/rgw/driver/rados/rgw_putobj_processor.h b/src/rgw/driver/rados/rgw_putobj_processor.h
index fa9200f32dae7..b1df0a33c7362 100644
--- a/src/rgw/driver/rados/rgw_putobj_processor.h
+++ b/src/rgw/driver/rados/rgw_putobj_processor.h
@@ -74,14 +74,15 @@ class RadosWriter : public rgw::sal::DataProcessor {
   RawObjSet written; // set of written objects for deletion
   const DoutPrefixProvider *dpp;
   optional_yield y;
+  jspan_context& trace;
 
  public:
   RadosWriter(Aio *aio, RGWRados *store,
               const RGWBucketInfo& bucket_info,
               RGWObjectCtx& obj_ctx, const rgw_obj& _head_obj,
-              const DoutPrefixProvider *dpp, optional_yield y)
+              const DoutPrefixProvider *dpp, optional_yield y, jspan_context& _trace)
     : aio(aio), store(store), bucket_info(bucket_info),
-      obj_ctx(obj_ctx), head_obj(_head_obj), dpp(dpp), y(y)
+      obj_ctx(obj_ctx), head_obj(_head_obj), dpp(dpp), y(y), trace(_trace)
   {}
   ~RadosWriter();
 
@@ -103,6 +104,7 @@ class RadosWriter : public rgw::sal::DataProcessor {
   // so they aren't deleted on destruction
   void clear_written() { written.clear(); }
 
+  jspan_context& get_trace() { return trace; }
 };
 
 
@@ -133,12 +135,14 @@ class ManifestObjectProcessor : public HeadObjectProcessor,
                           const rgw_placement_rule *ptail_placement_rule,
                           const rgw_user& owner, RGWObjectCtx& _obj_ctx,
                           const rgw_obj& _head_obj,
-                          const DoutPrefixProvider* dpp, optional_yield y)
+                          const DoutPrefixProvider* dpp,
+                          optional_yield y,
+                          jspan_context& trace)
     : HeadObjectProcessor(0),
       store(store), bucket_info(bucket_info),
       owner(owner),
       obj_ctx(_obj_ctx), head_obj(_head_obj),
-      writer(aio, store, bucket_info, obj_ctx, head_obj, dpp, y),
+      writer(aio, store, bucket_info, obj_ctx, head_obj, dpp, y, trace),
       chunk(&writer, 0), stripe(&chunk, this, 0), dpp(dpp) {
         if (ptail_placement_rule) {
           tail_placement_rule = *ptail_placement_rule;
@@ -175,9 +179,9 @@ class AtomicObjectProcessor : public ManifestObjectProcessor {
                         RGWObjectCtx& obj_ctx, const rgw_obj& _head_obj,
                         std::optional<uint64_t> olh_epoch,
                         const std::string& unique_tag,
-                        const DoutPrefixProvider *dpp, optional_yield y)
+                        const DoutPrefixProvider *dpp, optional_yield y, jspan_context& trace)
     : ManifestObjectProcessor(aio, store, bucket_info, ptail_placement_rule,
-                              owner, obj_ctx, _head_obj, dpp, y),
+                              owner, obj_ctx, _head_obj, dpp, y, trace),
       olh_epoch(olh_epoch), unique_tag(unique_tag)
   {}
 
@@ -219,9 +223,9 @@ class MultipartObjectProcessor : public ManifestObjectProcessor {
                            const rgw_obj& _head_obj,
                            const std::string& upload_id, uint64_t part_num,
                            const std::string& part_num_str,
-                           const DoutPrefixProvider *dpp, optional_yield y)
+                           const DoutPrefixProvider *dpp, optional_yield y, jspan_context& trace)
     : ManifestObjectProcessor(aio, store, bucket_info, ptail_placement_rule,
-                              owner, obj_ctx, _head_obj, dpp, y),
+                              owner, obj_ctx, _head_obj, dpp, y, trace),
       target_obj(head_obj), upload_id(upload_id),
       part_num(part_num), part_num_str(part_num_str),
       mp(head_obj.key.name, upload_id)
@@ -262,9 +266,9 @@ class MultipartObjectProcessor : public ManifestObjectProcessor {
                           const rgw_obj& _head_obj,
                           const std::string& unique_tag, uint64_t position,
                           uint64_t *cur_accounted_size,
-                          const DoutPrefixProvider *dpp, optional_yield y)
+                          const DoutPrefixProvider *dpp, optional_yield y, jspan_context& trace)
             : ManifestObjectProcessor(aio, store, bucket_info, ptail_placement_rule,
-                                      owner, obj_ctx, _head_obj, dpp, y),
+                                      owner, obj_ctx, _head_obj, dpp, y, trace),
               position(position), cur_size(0), cur_accounted_size(cur_accounted_size),
               unique_tag(unique_tag), cur_manifest(nullptr)
     {}
diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc
index bd487989370bc..006eddb00ad89 100644
--- a/src/rgw/driver/rados/rgw_rados.cc
+++ b/src/rgw/driver/rados/rgw_rados.cc
@@ -2876,6 +2876,8 @@ int RGWRados::swift_versioning_copy(RGWObjectCtx& obj_ctx,
 
   rgw_zone_id no_zone;
 
+  jspan_context no_trace{false, false};
+
   r = copy_obj(obj_ctx,
                user,
                NULL, /* req_info *info */
@@ -2904,7 +2906,8 @@ int RGWRados::swift_versioning_copy(RGWObjectCtx& obj_ctx,
                NULL, /* void (*progress_cb)(off_t, void *) */
                NULL, /* void *progress_data */
                dpp,
-               null_yield);
+               null_yield,
+               no_trace);
   if (r == -ECANCELED || r == -ENOENT) {
     /* Has already been overwritten, meaning another rgw process already
      * copied it out */
@@ -2970,6 +2973,8 @@ int RGWRados::swift_versioning_restore(RGWObjectCtx& obj_ctx,
     obj_ctx.set_atomic(archive_obj);
     obj_ctx.set_atomic(obj);
 
+    jspan_context no_trace{false, false};
+
     int ret = copy_obj(obj_ctx,
                        user,
                        nullptr,       /* req_info *info */
@@ -2998,7 +3003,8 @@ int RGWRados::swift_versioning_restore(RGWObjectCtx& obj_ctx,
                        nullptr,       /* void (*progress_cb)(off_t, void *) */
                        nullptr,       /* void *progress_data */
                        dpp,
-                       null_yield);
+                       null_yield,
+                       no_trace);
     if (ret == -ECANCELED || ret == -ENOENT) {
       /* Has already been overwritten, meaning another rgw process already
        * copied it out */
@@ -3028,7 +3034,7 @@ int RGWRados::Object::Write::_do_write_meta(const DoutPrefixProvider *dpp,
                                            uint64_t size, uint64_t accounted_size,
                                            map<string, bufferlist>& attrs,
                                            bool assume_noent, bool modify_tail,
-                                           void *_index_op, optional_yield y)
+                                           void *_index_op, optional_yield y, jspan_context& trace)
 {
   RGWRados::Bucket::UpdateIndex *index_op = static_cast<RGWRados::Bucket::UpdateIndex *>(_index_op);
   RGWRados *store = target->get_store();
@@ -3203,7 +3209,7 @@ int RGWRados::Object::Write::_do_write_meta(const DoutPrefixProvider *dpp,
   auto& ioctx = ref.pool.ioctx();
 
   tracepoint(rgw_rados, operate_enter, req_id.c_str());
-  r = rgw_rados_operate(dpp, ref.pool.ioctx(), ref.obj.oid, &op, null_yield);
+  r = rgw_rados_operate(dpp, ref.pool.ioctx(), ref.obj.oid, &op, null_yield, 0, &trace);
   tracepoint(rgw_rados, operate_exit, req_id.c_str());
   if (r < 0) { /* we can expect to get -ECANCELED if object was replaced under,
                 or -ENOENT if was removed, or -EEXIST if it did not exist
@@ -3318,7 +3324,7 @@ done_cancel:
 }
 
 int RGWRados::Object::Write::write_meta(const DoutPrefixProvider *dpp, uint64_t size, uint64_t accounted_size,
-                                           map<string, bufferlist>& attrs, optional_yield y)
+                                           map<string, bufferlist>& attrs, optional_yield y, jspan_context& trace)
 {
   RGWBucketInfo& bucket_info = target->get_bucket_info();
 
@@ -3329,13 +3335,13 @@ int RGWRados::Object::Write::write_meta(const DoutPrefixProvider *dpp, uint64_t
   bool assume_noent = (meta.if_match == NULL && meta.if_nomatch == NULL);
   int r;
   if (assume_noent) {
-    r = _do_write_meta(dpp, size, accounted_size, attrs, assume_noent, meta.modify_tail, (void *)&index_op, y);
+    r = _do_write_meta(dpp, size, accounted_size, attrs, assume_noent, meta.modify_tail, (void *)&index_op, y, trace);
     if (r == -EEXIST) {
       assume_noent = false;
     }
   }
   if (!assume_noent) {
-    r = _do_write_meta(dpp, size, accounted_size, attrs, assume_noent, meta.modify_tail, (void *)&index_op, y);
+    r = _do_write_meta(dpp, size, accounted_size, attrs, assume_noent, meta.modify_tail, (void *)&index_op, y, trace);
   }
   return r;
 }
@@ -3904,9 +3910,10 @@ int RGWRados::fetch_remote_obj(RGWObjectCtx& obj_ctx,
 
   rgw::BlockingAioThrottle aio(cct->_conf->rgw_put_obj_min_window_size);
   using namespace rgw::putobj;
+  jspan_context no_trace{false, false};
   AtomicObjectProcessor processor(&aio, this, dest_bucket_info, nullptr,
                                   user_id, obj_ctx, dest_obj, olh_epoch,
-				  tag, dpp, null_yield);
+				  tag, dpp, null_yield, no_trace);
   RGWRESTConn *conn;
   auto& zone_conn_map = svc.zone->get_zone_conn_map();
   auto& zonegroup_conn_map = svc.zone->get_zonegroup_conn_map();
@@ -4289,7 +4296,8 @@ int RGWRados::copy_obj(RGWObjectCtx& obj_ctx,
                void (*progress_cb)(off_t, void *),
                void *progress_data,
                const DoutPrefixProvider *dpp,
-               optional_yield y)
+               optional_yield y,
+               jspan_context& trace)
 {
   int ret;
   uint64_t obj_size;
@@ -4553,7 +4561,7 @@ int RGWRados::copy_obj(RGWObjectCtx& obj_ctx,
   write_op.meta.delete_at = delete_at;
   write_op.meta.modify_tail = !copy_itself;
 
-  ret = write_op.write_meta(dpp, obj_size, astate->accounted_size, attrs, y);
+  ret = write_op.write_meta(dpp, obj_size, astate->accounted_size, attrs, y, trace);
   if (ret < 0) {
     goto done_ret;
   }
@@ -4621,9 +4629,10 @@ int RGWRados::copy_obj_data(RGWObjectCtx& obj_ctx,
 
   auto aio = rgw::make_throttle(cct->_conf->rgw_put_obj_min_window_size, y);
   using namespace rgw::putobj;
+  jspan_context no_trace{false, false};
   AtomicObjectProcessor processor(aio.get(), this, dest_bucket_info,
                                   &dest_placement, dest_bucket_info.owner,
-                                  obj_ctx, dest_obj, olh_epoch, tag, dpp, y);
+                                  obj_ctx, dest_obj, olh_epoch, tag, dpp, y, no_trace);
   int ret = processor.prepare(y);
   if (ret < 0)
     return ret;
diff --git a/src/rgw/driver/rados/rgw_rados.h b/src/rgw/driver/rados/rgw_rados.h
index 96244b15a4ab2..692ea1df7ce9a 100644
--- a/src/rgw/driver/rados/rgw_rados.h
+++ b/src/rgw/driver/rados/rgw_rados.h
@@ -814,9 +814,9 @@ public:
                      uint64_t size, uint64_t accounted_size,
                      std::map<std::string, bufferlist>& attrs,
                      bool modify_tail, bool assume_noent,
-                     void *index_op, optional_yield y);
+                     void *index_op, optional_yield y, jspan_context& trace);
       int write_meta(const DoutPrefixProvider *dpp, uint64_t size, uint64_t accounted_size,
-                     std::map<std::string, bufferlist>& attrs, optional_yield y);
+                     std::map<std::string, bufferlist>& attrs, optional_yield y, jspan_context& trace);
       int write_data(const char *data, uint64_t ofs, uint64_t len, bool exclusive);
       const req_state* get_req_state() {
         return nullptr;  /* XXX dang Only used by LTTng, and it handles null anyway */
@@ -1182,7 +1182,8 @@ public:
                void (*progress_cb)(off_t, void *),
                void *progress_data,
                const DoutPrefixProvider *dpp,
-               optional_yield y);
+               optional_yield y,
+               jspan_context& trace);
 
   int copy_obj_data(RGWObjectCtx& obj_ctx,
                RGWBucketInfo& dest_bucket_info,
diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc
index c5527e2b3bf67..845ccf999b1d3 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.cc
+++ b/src/rgw/driver/rados/rgw_sal_rados.cc
@@ -1695,7 +1695,7 @@ std::unique_ptr<Writer> RadosStore::get_append_writer(const DoutPrefixProvider *
 				 this, std::move(aio), owner,
 				 ptail_placement_rule,
 				 unique_tag, position,
-				 cur_accounted_size);
+				 cur_accounted_size, obj->get_trace());
 }
 
 std::unique_ptr<Writer> RadosStore::get_atomic_writer(const DoutPrefixProvider *dpp,
@@ -1713,7 +1713,7 @@ std::unique_ptr<Writer> RadosStore::get_atomic_writer(const DoutPrefixProvider *
 				 bucket_info, obj_ctx, obj->get_obj(),
 				 this, std::move(aio), owner,
 				 ptail_placement_rule,
-				 olh_epoch, unique_tag);
+				 olh_epoch, unique_tag, obj->get_trace());
 }
 
 const std::string& RadosStore::get_compression_type(const rgw_placement_rule& rule)
@@ -2119,7 +2119,7 @@ int RadosObject::write_cloud_tier(const DoutPrefixProvider* dpp,
   attrs.erase(RGW_ATTR_ID_TAG);
   attrs.erase(RGW_ATTR_TAIL_TAG);
 
-  return obj_op.write_meta(dpp, 0, 0, attrs, y);
+  return obj_op.write_meta(dpp, 0, 0, attrs, y, head_obj->get_trace());
 }
 
 int RadosObject::get_max_chunk_size(const DoutPrefixProvider* dpp, rgw_placement_rule placement_rule, uint64_t* max_chunk_size, uint64_t* alignment)
@@ -2362,7 +2362,8 @@ int RadosObject::copy_object(User* user,
 				     progress_cb,
 				     progress_data,
 				     dpp,
-				     y);
+				     y,
+                                     dest_object->get_trace());
 }
 
 int RadosObject::RadosReadOp::iterate(const DoutPrefixProvider* dpp, int64_t ofs, int64_t end, RGWGetDataCB* cb, optional_yield y)
@@ -2565,7 +2566,7 @@ int RadosMultipartUpload::init(const DoutPrefixProvider *dpp, optional_yield y,
     encode(upload_info, bl);
     obj_op.meta.data = &bl;
 
-    ret = obj_op.write_meta(dpp, bl.length(), 0, attrs, y);
+    ret = obj_op.write_meta(dpp, bl.length(), 0, attrs, y, get_trace());
   } while (ret == -EEXIST);
 
   return ret;
@@ -2848,7 +2849,7 @@ int RadosMultipartUpload::complete(const DoutPrefixProvider *dpp,
   obj_op.meta.completeMultipart = true;
   obj_op.meta.olh_epoch = olh_epoch;
 
-  ret = obj_op.write_meta(dpp, ofs, accounted_size, attrs, y);
+  ret = obj_op.write_meta(dpp, ofs, accounted_size, attrs, y, get_trace());
   if (ret < 0)
     return ret;
 
@@ -2946,7 +2947,7 @@ std::unique_ptr<Writer> RadosMultipartUpload::get_writer(
   return std::make_unique<RadosMultipartWriter>(dpp, y, get_upload_id(),
 				 bucket_info, obj_ctx,
 				 obj->get_obj(), store, std::move(aio), owner,
-				 ptail_placement_rule, part_num, part_num_str);
+				 ptail_placement_rule, part_num, part_num_str, obj->get_trace());
 }
 
 MPRadosSerializer::MPRadosSerializer(const DoutPrefixProvider *dpp, RadosStore* store, RadosObject* obj, const std::string& lock_name) :
diff --git a/src/rgw/driver/rados/rgw_sal_rados.h b/src/rgw/driver/rados/rgw_sal_rados.h
index 917addaa442e3..72c7fe6ce0b05 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.h
+++ b/src/rgw/driver/rados/rgw_sal_rados.h
@@ -779,7 +779,8 @@ public:
 		    const rgw_user& owner,
 		    const rgw_placement_rule *ptail_placement_rule,
 		    uint64_t olh_epoch,
-		    const std::string& unique_tag) :
+		    const std::string& unique_tag,
+                    jspan_context& trace) :
 			StoreWriter(dpp, y),
 			store(_store),
 			aio(std::move(_aio)),
@@ -787,7 +788,7 @@ public:
 			processor(&*aio, store->getRados(), bucket_info,
 				  ptail_placement_rule, owner, obj_ctx,
 				  obj, olh_epoch, unique_tag,
-				  dpp, y)
+				  dpp, y, trace)
   {}
   ~RadosAtomicWriter() = default;
 
@@ -826,7 +827,8 @@ public:
 		    const rgw_placement_rule *ptail_placement_rule,
 		    const std::string& unique_tag,
 		    uint64_t position,
-		    uint64_t *cur_accounted_size) :
+		    uint64_t *cur_accounted_size,
+                    jspan_context& trace) :
 			StoreWriter(dpp, y),
 			store(_store),
 			aio(std::move(_aio)),
@@ -834,7 +836,7 @@ public:
 			processor(&*aio, store->getRados(), bucket_info,
 				  ptail_placement_rule, owner, obj_ctx,
 				  obj, unique_tag, position,
-				  cur_accounted_size, dpp, y)
+				  cur_accounted_size, dpp, y, trace)
   {}
   ~RadosAppendWriter() = default;
 
@@ -871,7 +873,7 @@ public:
 		       RadosStore* _store, std::unique_ptr<Aio> _aio,
 		       const rgw_user& owner,
 		       const rgw_placement_rule *ptail_placement_rule,
-		       uint64_t part_num, const std::string& part_num_str) :
+		       uint64_t part_num, const std::string& part_num_str, jspan_context& trace) :
 			StoreWriter(dpp, y),
 			store(_store),
 			aio(std::move(_aio)),
@@ -879,7 +881,7 @@ public:
 			processor(&*aio, store->getRados(), bucket_info,
 				  ptail_placement_rule, owner, obj_ctx,
 				  obj, upload_id,
-				  part_num, part_num_str, dpp, y)
+				  part_num, part_num_str, dpp, y, trace)
   {}
   ~RadosMultipartWriter() = default;
 
diff --git a/src/rgw/driver/rados/rgw_tools.cc b/src/rgw/driver/rados/rgw_tools.cc
index 1c2951c4bcd17..b2f73359b0599 100644
--- a/src/rgw/driver/rados/rgw_tools.cc
+++ b/src/rgw/driver/rados/rgw_tools.cc
@@ -165,7 +165,7 @@ int rgw_delete_system_obj(const DoutPrefixProvider *dpp,
 
 int rgw_rados_operate(const DoutPrefixProvider *dpp, librados::IoCtx& ioctx, const std::string& oid,
                       librados::ObjectReadOperation *op, bufferlist* pbl,
-                      optional_yield y, int flags)
+                      optional_yield y, int flags, const jspan_context* trace_info)
 {
   // given a yield_context, call async_operate() to yield the coroutine instead
   // of blocking
@@ -192,13 +192,13 @@ int rgw_rados_operate(const DoutPrefixProvider *dpp, librados::IoCtx& ioctx, con
 
 int rgw_rados_operate(const DoutPrefixProvider *dpp, librados::IoCtx& ioctx, const std::string& oid,
                       librados::ObjectWriteOperation *op, optional_yield y,
-		      int flags)
+		      int flags, const jspan_context* trace_info)
 {
   if (y) {
     auto& context = y.get_io_context();
     auto& yield = y.get_yield_context();
     boost::system::error_code ec;
-    librados::async_operate(context, ioctx, oid, op, flags, yield[ec]);
+    librados::async_operate(context, ioctx, oid, op, flags, yield[ec], trace_info);
     return -ec.value();
   }
   if (is_asio_thread) {
@@ -207,7 +207,7 @@ int rgw_rados_operate(const DoutPrefixProvider *dpp, librados::IoCtx& ioctx, con
     ldpp_dout(dpp, 20) << "BACKTRACE: " << __func__ << ": " << ClibBackTrace(0) << dendl;
 #endif
   }
-  return ioctx.operate(oid, op, flags);
+  return ioctx.operate(oid, op, flags, trace_info);
 }
 
 int rgw_rados_notify(const DoutPrefixProvider *dpp, librados::IoCtx& ioctx, const std::string& oid,
diff --git a/src/rgw/driver/rados/rgw_tools.h b/src/rgw/driver/rados/rgw_tools.h
index d96912cb866be..63c36db82748e 100644
--- a/src/rgw/driver/rados/rgw_tools.h
+++ b/src/rgw/driver/rados/rgw_tools.h
@@ -95,10 +95,10 @@ extern thread_local bool is_asio_thread;
 /// perform the rados operation, using the yield context when given
 int rgw_rados_operate(const DoutPrefixProvider *dpp, librados::IoCtx& ioctx, const std::string& oid,
                       librados::ObjectReadOperation *op, bufferlist* pbl,
-                      optional_yield y, int flags = 0);
+                      optional_yield y, int flags = 0, const jspan_context* trace_info = nullptr);
 int rgw_rados_operate(const DoutPrefixProvider *dpp, librados::IoCtx& ioctx, const std::string& oid,
                       librados::ObjectWriteOperation *op, optional_yield y,
-		      int flags = 0);
+		      int flags = 0, const jspan_context* trace_info = nullptr);
 int rgw_rados_notify(const DoutPrefixProvider *dpp, librados::IoCtx& ioctx, const std::string& oid,
                      bufferlist& bl, uint64_t timeout_ms, bufferlist* pbl,
                      optional_yield y);
diff --git a/src/rgw/rgw_aio.cc b/src/rgw/rgw_aio.cc
index 0bbc1f7bfa8bd..c90841dbb7b0c 100644
--- a/src/rgw/rgw_aio.cc
+++ b/src/rgw/rgw_aio.cc
@@ -49,15 +49,16 @@ void cb(librados::completion_t, void* arg) {
 }
 
 template <typename Op>
-Aio::OpFunc aio_abstract(librados::IoCtx ctx, Op&& op) {
-  return [ctx = std::move(ctx), op = std::move(op)] (Aio* aio, AioResult& r) mutable {
+Aio::OpFunc aio_abstract(librados::IoCtx ctx, Op&& op, jspan_context* trace_ctx = nullptr) {
+  return [ctx = std::move(ctx), op = std::move(op), trace_ctx] (Aio* aio, AioResult& r) mutable {
       constexpr bool read = std::is_same_v<std::decay_t<Op>, librados::ObjectReadOperation>;
       // use placement new to construct the rados state inside of user_data
       auto s = new (&r.user_data) state(aio, ctx, r);
       if constexpr (read) {
+        (void)trace_ctx; // suppress unused trace_ctx warning. until we will support the read op trace
         r.result = ctx.aio_operate(r.obj.oid, s->c, &op, &r.data);
       } else {
-        r.result = ctx.aio_operate(r.obj.oid, s->c, &op);
+        r.result = ctx.aio_operate(r.obj.oid, s->c, &op, trace_ctx);
       }
       if (r.result < 0) {
         // cb() won't be called, so release everything here
@@ -88,8 +89,8 @@ struct Handler {
 template <typename Op>
 Aio::OpFunc aio_abstract(librados::IoCtx ctx, Op&& op,
                          boost::asio::io_context& context,
-                         yield_context yield) {
-  return [ctx = std::move(ctx), op = std::move(op), &context, yield] (Aio* aio, AioResult& r) mutable {
+                         yield_context yield, jspan_context* trace_ctx = nullptr) {
+  return [ctx = std::move(ctx), op = std::move(op), &context, yield, trace_ctx] (Aio* aio, AioResult& r) mutable {
       // arrange for the completion Handler to run on the yield_context's strand
       // executor so it can safely call back into Aio without locking
       using namespace boost::asio;
@@ -97,7 +98,7 @@ Aio::OpFunc aio_abstract(librados::IoCtx ctx, Op&& op,
       auto ex = get_associated_executor(init.completion_handler);
 
       librados::async_operate(context, ctx, r.obj.oid, &op, 0,
-                              bind_executor(ex, Handler{aio, ctx, r}));
+                              bind_executor(ex, Handler{aio, ctx, r}), trace_ctx);
     };
 }
 
@@ -114,15 +115,15 @@ Aio::OpFunc d3n_cache_aio_abstract(const DoutPrefixProvider *dpp, optional_yield
 
 
 template <typename Op>
-Aio::OpFunc aio_abstract(librados::IoCtx ctx, Op&& op, optional_yield y) {
+Aio::OpFunc aio_abstract(librados::IoCtx ctx, Op&& op, optional_yield y, jspan_context *trace_ctx = nullptr) {
   static_assert(std::is_base_of_v<librados::ObjectOperation, std::decay_t<Op>>);
   static_assert(!std::is_lvalue_reference_v<Op>);
   static_assert(!std::is_const_v<Op>);
   if (y) {
     return aio_abstract(std::move(ctx), std::forward<Op>(op),
-                        y.get_io_context(), y.get_yield_context());
+                        y.get_io_context(), y.get_yield_context(), trace_ctx);
   }
-  return aio_abstract(std::move(ctx), std::forward<Op>(op));
+  return aio_abstract(std::move(ctx), std::forward<Op>(op), null_yield, trace_ctx);
 }
 
 } // anonymous namespace
@@ -134,8 +135,8 @@ Aio::OpFunc Aio::librados_op(librados::IoCtx ctx,
 }
 Aio::OpFunc Aio::librados_op(librados::IoCtx ctx,
                              librados::ObjectWriteOperation&& op,
-                             optional_yield y) {
-  return aio_abstract(std::move(ctx), std::move(op), y);
+                             optional_yield y, jspan_context *trace_ctx) {
+  return aio_abstract(std::move(ctx), std::move(op), y, trace_ctx);
 }
 
 Aio::OpFunc Aio::d3n_cache_op(const DoutPrefixProvider *dpp, optional_yield y,
diff --git a/src/rgw/rgw_aio.h b/src/rgw/rgw_aio.h
index 0070346327b71..9be144f607f2d 100644
--- a/src/rgw/rgw_aio.h
+++ b/src/rgw/rgw_aio.h
@@ -96,7 +96,7 @@ class Aio {
                             optional_yield y);
   static OpFunc librados_op(librados::IoCtx ctx,
                             librados::ObjectWriteOperation&& op,
-                            optional_yield y);
+                            optional_yield y, jspan_context *trace_ctx = nullptr);
   static OpFunc d3n_cache_op(const DoutPrefixProvider *dpp, optional_yield y,
                              off_t read_ofs, off_t read_len, std::string& location);
 };
diff --git a/src/rgw/rgw_lib.cc b/src/rgw/rgw_lib.cc
index f449cce21c026..16a66ef9f7aaf 100644
--- a/src/rgw/rgw_lib.cc
+++ b/src/rgw/rgw_lib.cc
@@ -218,6 +218,8 @@ namespace rgw {
       goto done;
     }
 
+    s->trace = tracing::rgw::tracer.start_trace(op->name());
+
     /* req is-a RGWOp, currently initialized separately */
     ret = req->op_init();
     if (ret < 0) {
diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index 513cb20bc69a2..821568d7396ef 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -4027,6 +4027,8 @@ void RGWPutObj::execute(optional_yield y)
 
   rgw_placement_rule *pdest_placement = &s->dest_placement;
 
+  s->object->set_trace(s->trace->GetContext());
+
   if (multipart) {
     std::unique_ptr<rgw::sal::MultipartUpload> upload;
     upload = s->bucket->get_multipart_upload(s->object->get_name(),
diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h
index 2caf02f87efc1..03bd92dcd5dc6 100644
--- a/src/rgw/rgw_sal.h
+++ b/src/rgw/rgw_sal.h
@@ -1099,6 +1099,9 @@ class Object {
     /** Get a unique copy of this object */
     virtual std::unique_ptr<Object> clone() = 0;
 
+    virtual jspan_context& get_trace() = 0;
+    virtual void set_trace (jspan_context&& _trace_ctx) = 0;
+
     /* dang - This is temporary, until the API is completed */
     /** Get the key for this object */
     virtual rgw_obj_key& get_key() = 0;
@@ -1178,7 +1181,7 @@ public:
   virtual std::map<uint32_t, std::unique_ptr<MultipartPart>>& get_parts() = 0;
 
   /** Get the trace context of this upload */
-  virtual const jspan_context& get_trace() = 0;
+  virtual jspan_context& get_trace() = 0;
 
   /** Get the Object that represents this upload */
   virtual std::unique_ptr<rgw::sal::Object> get_meta_obj() = 0;
diff --git a/src/rgw/rgw_sal_filter.h b/src/rgw/rgw_sal_filter.h
index 6260f3acae37f..ba57963fc9258 100644
--- a/src/rgw/rgw_sal_filter.h
+++ b/src/rgw/rgw_sal_filter.h
@@ -687,6 +687,9 @@ public:
     return std::make_unique<FilterObject>(*this);
   }
 
+  virtual jspan_context& get_trace() { return next->get_trace(); }
+  virtual void set_trace (jspan_context&& _trace_ctx) { next->set_trace(std::move(_trace_ctx)); }
+
   virtual void print(std::ostream& out) const override { return next->print(out); }
 
   /* Internal to Filters */
@@ -726,7 +729,7 @@ public:
 
   virtual std::map<uint32_t, std::unique_ptr<MultipartPart>>& get_parts() override { return parts; }
 
-  virtual const jspan_context& get_trace() override { return next->get_trace(); }
+  virtual jspan_context& get_trace() override { return next->get_trace(); }
 
   virtual std::unique_ptr<rgw::sal::Object> get_meta_obj() override;
 
diff --git a/src/rgw/rgw_sal_store.h b/src/rgw/rgw_sal_store.h
index e4822a6cb6e5a..0f4726d01c3a3 100644
--- a/src/rgw/rgw_sal_store.h
+++ b/src/rgw/rgw_sal_store.h
@@ -171,6 +171,7 @@ class StoreObject : public Object {
     RGWObjState state;
     Bucket* bucket = nullptr;
     bool delete_marker{false};
+    jspan_context trace_ctx{false, false};
 
   public:
     StoreObject() = default;
@@ -240,6 +241,8 @@ class StoreObject : public Object {
        * work with lifecycle */
       return -1;
     }
+    jspan_context& get_trace() override { return trace_ctx; }
+    void set_trace (jspan_context&& _trace_ctx) override { trace_ctx = std::move(_trace_ctx); }
 
     virtual int get_torrent_info(const DoutPrefixProvider* dpp,
                                  optional_yield y, bufferlist& bl) override {
@@ -277,7 +280,7 @@ public:
 
   virtual std::map<uint32_t, std::unique_ptr<MultipartPart>>& get_parts() override { return parts; }
 
-  virtual const jspan_context& get_trace() override { return trace_ctx; }
+  virtual jspan_context& get_trace() override { return trace_ctx; }
 
   virtual void print(std::ostream& out) const override {
     out << get_meta();
diff --git a/src/rgw/services/svc_rados.cc b/src/rgw/services/svc_rados.cc
index 99f400f42b021..dd2b23bb931eb 100644
--- a/src/rgw/services/svc_rados.cc
+++ b/src/rgw/services/svc_rados.cc
@@ -148,9 +148,9 @@ int RGWSI_RADOS::Obj::operate(const DoutPrefixProvider *dpp, librados::ObjectRea
   return rgw_rados_operate(dpp, ref.pool.ioctx(), ref.obj.oid, op, pbl, y, flags);
 }
 
-int RGWSI_RADOS::Obj::aio_operate(librados::AioCompletion *c, librados::ObjectWriteOperation *op)
+int RGWSI_RADOS::Obj::aio_operate(librados::AioCompletion *c, librados::ObjectWriteOperation *op, const jspan_context *trace_ctx)
 {
-  return ref.pool.ioctx().aio_operate(ref.obj.oid, c, op);
+  return ref.pool.ioctx().aio_operate(ref.obj.oid, c, op, 0, trace_ctx);
 }
 
 int RGWSI_RADOS::Obj::aio_operate(librados::AioCompletion *c, librados::ObjectReadOperation *op,
diff --git a/src/rgw/services/svc_rados.h b/src/rgw/services/svc_rados.h
index ede029aa897b2..23931c461636d 100644
--- a/src/rgw/services/svc_rados.h
+++ b/src/rgw/services/svc_rados.h
@@ -176,7 +176,7 @@ public:
 		int flags = 0);
     int operate(const DoutPrefixProvider *dpp, librados::ObjectReadOperation *op, bufferlist *pbl,
                 optional_yield y, int flags = 0);
-    int aio_operate(librados::AioCompletion *c, librados::ObjectWriteOperation *op);
+    int aio_operate(librados::AioCompletion *c, librados::ObjectWriteOperation *op, const jspan_context *trace_ctx = nullptr);
     int aio_operate(librados::AioCompletion *c, librados::ObjectReadOperation *op,
                     bufferlist *pbl);
 
-- 
2.39.5