tracer/osd/librados/build/rgw: rgw and osd end2end tracing using opentelemetry

* build: add opentelemetry to cmake system
crimson targets that uses Message.cc/h are built before opentelemetry (o-tel), so we need to build o-tel eralier so we also add the library to the include path earlier
this shoud work for WITH_JAEGER flag both the ON/OFF cases, and for librados where the compilation flag is ignored

* msg/tracer: add o-tel trace to Messages with decode/encode function in tracer.h
some files that uses Message.cc/h just need the encode/decode functions  and not all others functions.
some crimson targets does not link with ceph_context (common) which is required for tracer.cc file. so we just need to include that functions

* librados: Add opentelemtry trace param for aio_operate and operate methods
in order to propagate the trace info I added the otel-trace as an extra param.
in some places, there already was a blkin trace info, and since it is not used in other places we can safely change it to o-tel trace info.
this will be done in another commit, so the cleanup of blkin trace will be in a dedicated commit

* osd: use the o-tel trace of the msg as a parent span of the osd trace
if there is a valid span in the msg, we will add this op to the request
trace, otherwise it will start a new trace for the OSD op

* rgw: pass put obj trace info to librados
in order to make it possible, I saved the trace info inside the sal::Object, so we can use it later when writing the object to rados
it could be used also later for read ops.
note the trace field of req_state is initalized only in rgw_process, so it's also required in librgw request flow

* prevent breaking channges to kSize. make sure that changes between components built with
different versions of OTEL do not break message compatibility

Signed-off-by: Omri Zeneva <ozeneva@redhat.com>

diff --git a/cmake/modules/BuildOpentelemetry.cmake b/cmake/modules/BuildOpentelemetry.cmake
index ba2edaa09329d3de76bad76ee780770c48127678..48b219e9c0fc2f67152054aceaff28e483d9fd42 100644 (file)
--- a/cmake/modules/BuildOpentelemetry.cmake
+++ b/cmake/modules/BuildOpentelemetry.cmake
@@ -82,4 +82,5 @@ function(build_opentelemetry)
     PROPERTIES
       INTERFACE_LINK_LIBRARIES "${opentelemetry_deps}"
       INTERFACE_INCLUDE_DIRECTORIES "${opentelemetry_include_dir}")
+  include_directories(SYSTEM "${opentelemetry_include_dir}")
 endfunction()

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index afa2442d5709de7f635a897cacb8635d20963ac6..0b88801d3b15a5a2ea19f872762fc3b00d1a700e 100644 (file)
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -301,6 +301,15 @@ if (WITH_BLKIN)
   add_subdirectory(blkin/blkin-lib)
 endif(WITH_BLKIN)
 
+if(WITH_JAEGER)
+  find_package(thrift 0.13.0 REQUIRED)
+  include(BuildOpentelemetry)
+  build_opentelemetry()
+  add_library(jaeger_base INTERFACE)
+  target_link_libraries(jaeger_base INTERFACE opentelemetry::libopentelemetry
+    thrift::libthrift)
+endif()
+
 set(mds_files)
 list(APPEND mds_files
   mds/MDSMap.cc
@@ -441,12 +450,6 @@ target_compile_definitions(common-objs PRIVATE
 add_dependencies(common-objs legacy-option-headers)
 
 if(WITH_JAEGER)
-  find_package(thrift 0.13.0 REQUIRED)
-  include(BuildOpentelemetry)
-  build_opentelemetry()
-  add_library(jaeger_base INTERFACE)
-  target_link_libraries(jaeger_base INTERFACE opentelemetry::libopentelemetry
-    thrift::libthrift)
   add_dependencies(common-objs jaeger_base)
   target_link_libraries(common-objs jaeger_base)
 endif()

diff --git a/src/common/tracer.cc b/src/common/tracer.cc
index e98053735b48d88ea41d5a0af2ee08559c777648..6a84480d60b5efb7dcf660be4fe435e52e10b88b 100644 (file)
--- a/src/common/tracer.cc
+++ b/src/common/tracer.cc
@@ -60,8 +60,7 @@ jspan_ptr Tracer::start_trace(opentelemetry::nostd::string_view trace_name, bool
 }
 
 jspan_ptr Tracer::add_span(opentelemetry::nostd::string_view span_name, const jspan_ptr& parent_span) {
-  if (parent_span && parent_span->IsRecording()) {
-    ceph_assert(tracer);
+  if (is_enabled() && parent_span && parent_span->IsRecording()) {
     opentelemetry::trace::StartSpanOptions span_opts;
     span_opts.parent = parent_span->GetContext();
     ldout(cct, 20) << "adding span " << span_name << " " << dendl;
@@ -85,41 +84,6 @@ bool Tracer::is_enabled() const {
   return cct->_conf->jaeger_tracing_enable;
 }
 
-void encode(const jspan_context& span_ctx, bufferlist& bl, uint64_t f) {
-  ENCODE_START(1, 1, bl);
-  using namespace opentelemetry;
-  using namespace trace;
-  auto is_valid = span_ctx.IsValid();
-  encode(is_valid, bl);
-  if (is_valid) {
-    encode_nohead(std::string_view(reinterpret_cast<const char*>(span_ctx.trace_id().Id().data()), TraceId::kSize), bl);
-    encode_nohead(std::string_view(reinterpret_cast<const char*>(span_ctx.span_id().Id().data()), SpanId::kSize), bl);
-    encode(span_ctx.trace_flags().flags(), bl);
-  }
-  ENCODE_FINISH(bl);
-}
-
-void decode(jspan_context& span_ctx, bufferlist::const_iterator& bl) {
-  using namespace opentelemetry;
-  using namespace trace;
-  DECODE_START(1, bl);
-  bool is_valid;
-  decode(is_valid, bl);
-  if (is_valid) {
-    std::array<uint8_t, TraceId::kSize> trace_id;
-    std::array<uint8_t, SpanId::kSize> span_id;
-    uint8_t flags;
-    decode(trace_id, bl);
-    decode(span_id, bl);
-    decode(flags, bl);
-    span_ctx = SpanContext(
-      TraceId(nostd::span<uint8_t, TraceId::kSize>(trace_id)),
-      SpanId(nostd::span<uint8_t, SpanId::kSize>(span_id)),
-      TraceFlags(flags),
-      true);
-  }
-  DECODE_FINISH(bl);
-}
 } // namespace tracing
 
 #endif // HAVE_JAEGER

diff --git a/src/common/tracer.h b/src/common/tracer.h
index 291ff9de25a7becc77b8b881ebc736bde3d0f1e8..b039d304a79a8e45b66486945cdf315deabb3c51 100644 (file)
--- a/src/common/tracer.h
+++ b/src/common/tracer.h
@@ -4,7 +4,7 @@
 #pragma once
 
 #include "acconfig.h"
-#include "include/buffer.h"
+#include "include/encoding.h"
 
 #ifdef HAVE_JAEGER
 #include "opentelemetry/trace/provider.h"
@@ -16,6 +16,11 @@ using jspan_attribute = opentelemetry::common::AttributeValue;
 
 namespace tracing {
 
+static constexpr int TraceIdkSize = 16;
+static constexpr int SpanIdkSize = 8;
+static_assert(TraceIdkSize == opentelemetry::trace::TraceId::kSize);
+static_assert(SpanIdkSize == opentelemetry::trace::SpanId::kSize);
+
 class Tracer {
  private:
   const static opentelemetry::nostd::shared_ptr<opentelemetry::trace::Tracer> noop_tracer;
@@ -24,6 +29,7 @@ class Tracer {
   opentelemetry::nostd::shared_ptr<opentelemetry::trace::Tracer> tracer;
 
  public:
+
   Tracer() = default;
 
   void init(CephContext* _cct, opentelemetry::nostd::string_view service_name);
@@ -46,8 +52,41 @@ class Tracer {
 
 };
 
-void encode(const jspan_context& span, ceph::buffer::list& bl, uint64_t f = 0);
-void decode(jspan_context& span_ctx, ceph::buffer::list::const_iterator& bl);
+inline void encode(const jspan_context& span_ctx, bufferlist& bl, uint64_t f = 0) {
+  ENCODE_START(1, 1, bl);
+  using namespace opentelemetry;
+  using namespace trace;
+  auto is_valid = span_ctx.IsValid();
+  encode(is_valid, bl);
+  if (is_valid) {
+    encode_nohead(std::string_view(reinterpret_cast<const char*>(span_ctx.trace_id().Id().data()), TraceIdkSize), bl);
+    encode_nohead(std::string_view(reinterpret_cast<const char*>(span_ctx.span_id().Id().data()), SpanIdkSize), bl);
+    encode(span_ctx.trace_flags().flags(), bl);
+  }
+  ENCODE_FINISH(bl);
+}
+
+inline void decode(jspan_context& span_ctx, bufferlist::const_iterator& bl) {
+  using namespace opentelemetry;
+  using namespace trace;
+  DECODE_START(1, bl);
+  bool is_valid;
+  decode(is_valid, bl);
+  if (is_valid) {
+    std::array<uint8_t, TraceIdkSize> trace_id;
+    std::array<uint8_t, SpanIdkSize> span_id;
+    uint8_t flags;
+    decode(trace_id, bl);
+    decode(span_id, bl);
+    decode(flags, bl);
+    span_ctx = SpanContext(
+      TraceId(nostd::span<uint8_t, TraceIdkSize>(trace_id)),
+      SpanId(nostd::span<uint8_t, SpanIdkSize>(span_id)),
+      TraceFlags(flags),
+      true);
+  }
+  DECODE_FINISH(bl);
+}
 
 } // namespace tracing
 
@@ -63,10 +102,20 @@ class Value {
 
 using jspan_attribute = Value;
 
-struct jspan_context {
-  jspan_context() {}
-  jspan_context(bool sampled_flag, bool is_remote) {}
+namespace opentelemetry {
+inline namespace v1 {
+namespace trace {
+class SpanContext {
+public:
+  SpanContext() = default;
+  SpanContext(bool sampled_flag, bool is_remote) {}
+  bool IsValid() const { return false;}
 };
+} // namespace trace
+} // namespace v1
+} // namespace opentelemetry
+
+using jspan_context = opentelemetry::v1::trace::SpanContext;
 
 class jspan {
   jspan_context _ctx;
@@ -76,7 +125,7 @@ public:
   void AddEvent(std::string_view) {}
   void AddEvent(std::string_view, std::initializer_list<std::pair<std::string_view, jspan_attribute>> fields) {}
   template <typename T> void AddEvent(std::string_view name, const T& fields = {}) {}
-  const jspan_context& GetContext() { return _ctx; }
+  jspan_context GetContext() const { return _ctx; }
   void UpdateName(std::string_view) {}
   bool IsRecording() { return false; }
 };

diff --git a/src/include/rados/librados.hpp b/src/include/rados/librados.hpp
index 2cd418627be956fafe00255640b9e1d7c89fde0a..4a7ac3ea6e0a7fe5789fa3665810bf7c04eed9ad 100644 (file)
--- a/src/include/rados/librados.hpp
+++ b/src/include/rados/librados.hpp
@@ -1169,10 +1169,12 @@ inline namespace v14_2_0 {
     // compound object operations
     int operate(const std::string& oid, ObjectWriteOperation *op);
     int operate(const std::string& oid, ObjectWriteOperation *op, int flags);
+    int operate(const std::string& oid, ObjectWriteOperation *op, int flags, const jspan_context *trace_info);
     int operate(const std::string& oid, ObjectReadOperation *op, bufferlist *pbl);
     int operate(const std::string& oid, ObjectReadOperation *op, bufferlist *pbl, int flags);
     int aio_operate(const std::string& oid, AioCompletion *c, ObjectWriteOperation *op);
     int aio_operate(const std::string& oid, AioCompletion *c, ObjectWriteOperation *op, int flags);
+    int aio_operate(const std::string& oid, AioCompletion *c, ObjectWriteOperation *op, int flags, const jspan_context *trace_info);
     /**
      * Schedule an async write operation with explicit snapshot parameters
      *

diff --git a/src/include/rados/librados_fwd.hpp b/src/include/rados/librados_fwd.hpp
index 396f3a8387575948ec912751d1f227484f66a049..d9a455adb38a71fc3f6a507fca9a3e39f3d402e4 100644 (file)
--- a/src/include/rados/librados_fwd.hpp
+++ b/src/include/rados/librados_fwd.hpp
@@ -3,6 +3,18 @@
 
 struct blkin_trace_info;
 
+namespace opentelemetry {
+inline namespace v1 {
+namespace trace {
+
+class SpanContext;
+
+} // namespace trace
+} // inline namespace v1
+} // namespace opentelemetry
+
+using jspan_context = opentelemetry::v1::trace::SpanContext;
+
 namespace libradosstriper {
 
 class RadosStriper;

diff --git a/src/librados/IoCtxImpl.cc b/src/librados/IoCtxImpl.cc
index d66b56560f9c3bdba853b5f1bb475d43a40a6fcc..b6be9050b1a9b792c2276a0ca80fdcfbf1a4d3a1 100644 (file)
--- a/src/librados/IoCtxImpl.cc
+++ b/src/librados/IoCtxImpl.cc
@@ -637,7 +637,7 @@ int librados::IoCtxImpl::writesame(const object_t& oid, bufferlist& bl,
 }
 
 int librados::IoCtxImpl::operate(const object_t& oid, ::ObjectOperation *o,
-                                ceph::real_time *pmtime, int flags)
+                                ceph::real_time *pmtime, int flags, const jspan_context* otel_trace)
 {
   ceph::real_time ut = (pmtime ? *pmtime :
     ceph::real_clock::now());
@@ -664,7 +664,7 @@ int librados::IoCtxImpl::operate(const object_t& oid, ::ObjectOperation *o,
     oid, oloc,
     *o, snapc, ut,
     flags | extra_op_flags,
-    oncommit, &ver);
+    oncommit, &ver, osd_reqid_t(), nullptr, otel_trace);
   objecter->op_submit(objecter_op);
 
   {
@@ -753,7 +753,7 @@ int librados::IoCtxImpl::aio_operate(const object_t& oid,
                                     ::ObjectOperation *o, AioCompletionImpl *c,
                                     const SnapContext& snap_context,
                                     const ceph::real_time *pmtime, int flags,
-                                     const blkin_trace_info *trace_info)
+                                     const blkin_trace_info *trace_info, const jspan_context *otel_trace)
 {
   FUNCTRACE(client->cct);
   OID_EVENT_TRACE(oid.name.c_str(), "RADOS_WRITE_OP_BEGIN");
@@ -779,7 +779,7 @@ int librados::IoCtxImpl::aio_operate(const object_t& oid,
   trace.event("init root span");
   Objecter::Op *op = objecter->prepare_mutate_op(
     oid, oloc, *o, snap_context, ut, flags | extra_op_flags,
-    oncomplete, &c->objver, osd_reqid_t(), &trace);
+    oncomplete, &c->objver, osd_reqid_t(), &trace, otel_trace);
   objecter->op_submit(op, &c->tid);
   trace.event("rados operate op submitted");
 

diff --git a/src/librados/IoCtxImpl.h b/src/librados/IoCtxImpl.h
index 477768ef78a4f13159b5f834dfb32e5a2c26a4cb..23c402d7b5dfde883042b277dd05383f6363b3da 100644 (file)
--- a/src/librados/IoCtxImpl.h
+++ b/src/librados/IoCtxImpl.h
@@ -154,12 +154,12 @@ struct librados::IoCtxImpl {
   int getxattrs(const object_t& oid, std::map<std::string, bufferlist>& attrset);
   int rmxattr(const object_t& oid, const char *name);
 
-  int operate(const object_t& oid, ::ObjectOperation *o, ceph::real_time *pmtime, int flags=0);
+  int operate(const object_t& oid, ::ObjectOperation *o, ceph::real_time *pmtime, int flags=0, const jspan_context *otel_trace = nullptr);
   int operate_read(const object_t& oid, ::ObjectOperation *o, bufferlist *pbl, int flags=0);
   int aio_operate(const object_t& oid, ::ObjectOperation *o,
                  AioCompletionImpl *c, const SnapContext& snap_context,
                  const ceph::real_time *pmtime, int flags,
-                 const blkin_trace_info *trace_info = nullptr);
+                 const blkin_trace_info *trace_info = nullptr, const jspan_context *otel_trace = nullptr);
   int aio_operate_read(const object_t& oid, ::ObjectOperation *o,
                       AioCompletionImpl *c, int flags, bufferlist *pbl, const blkin_trace_info *trace_info = nullptr);
 

diff --git a/src/librados/librados_asio.h b/src/librados/librados_asio.h
index bd672d951f7302f971eea914fb360ca6c52d1d99..2eae1c268f6cc2ba4933e25ac1dfa48d5b4f6993 100644 (file)
--- a/src/librados/librados_asio.h
+++ b/src/librados/librados_asio.h
@@ -152,7 +152,7 @@ auto async_write(ExecutionContext& ctx, IoCtx& io, const std::string& oid,
 template <typename ExecutionContext, typename CompletionToken>
 auto async_operate(ExecutionContext& ctx, IoCtx& io, const std::string& oid,
                    ObjectReadOperation *read_op, int flags,
-                   CompletionToken&& token)
+                   CompletionToken&& token, const jspan_context* trace_ctx = nullptr)
 {
   using Op = detail::AsyncOp<bufferlist>;
   using Signature = typename Op::Signature;
@@ -176,7 +176,7 @@ auto async_operate(ExecutionContext& ctx, IoCtx& io, const std::string& oid,
 template <typename ExecutionContext, typename CompletionToken>
 auto async_operate(ExecutionContext& ctx, IoCtx& io, const std::string& oid,
                    ObjectWriteOperation *write_op, int flags,
-                   CompletionToken &&token)
+                   CompletionToken &&token, const jspan_context* trace_ctx = nullptr)
 {
   using Op = detail::AsyncOp<void>;
   using Signature = typename Op::Signature;
@@ -184,7 +184,7 @@ auto async_operate(ExecutionContext& ctx, IoCtx& io, const std::string& oid,
   auto p = Op::create(ctx.get_executor(), init.completion_handler);
   auto& op = p->user_data;
 
-  int ret = io.aio_operate(oid, op.aio_completion.get(), write_op, flags);
+  int ret = io.aio_operate(oid, op.aio_completion.get(), write_op, flags, trace_ctx);
   if (ret < 0) {
     auto ec = boost::system::error_code{-ret, librados::detail::err_category()};
     ceph::async::post(std::move(p), ec);

diff --git a/src/librados/librados_cxx.cc b/src/librados/librados_cxx.cc
index 926ddf86dab444f24bcae8f3226909207643f291..f9bc3b8fd0443232a4ebee5b5d7df2ede4d47090 100644 (file)
--- a/src/librados/librados_cxx.cc
+++ b/src/librados/librados_cxx.cc
@@ -1525,6 +1525,14 @@ int librados::IoCtx::operate(const std::string& oid, librados::ObjectWriteOperat
   return io_ctx_impl->operate(obj, &o->impl->o, (ceph::real_time *)o->impl->prt, translate_flags(flags));
 }
 
+int librados::IoCtx::operate(const std::string& oid, librados::ObjectWriteOperation *o, int flags, const jspan_context* otel_trace)
+{
+  object_t obj(oid);
+  if (unlikely(!o->impl))
+    return -EINVAL;
+  return io_ctx_impl->operate(obj, &o->impl->o, (ceph::real_time *)o->impl->prt, translate_flags(flags), otel_trace);
+}
+
 int librados::IoCtx::operate(const std::string& oid, librados::ObjectReadOperation *o, bufferlist *pbl)
 {
   object_t obj(oid);
@@ -1550,6 +1558,7 @@ int librados::IoCtx::aio_operate(const std::string& oid, AioCompletion *c,
   return io_ctx_impl->aio_operate(obj, &o->impl->o, c->pc,
                                  io_ctx_impl->snapc, o->impl->prt, 0);
 }
+
 int librados::IoCtx::aio_operate(const std::string& oid, AioCompletion *c,
                                 ObjectWriteOperation *o, int flags)
 {
@@ -1558,7 +1567,18 @@ int librados::IoCtx::aio_operate(const std::string& oid, AioCompletion *c,
     return -EINVAL;
   return io_ctx_impl->aio_operate(obj, &o->impl->o, c->pc,
                                  io_ctx_impl->snapc, o->impl->prt,
-                                 translate_flags(flags));
+                                 translate_flags(flags), nullptr);
+}
+
+int librados::IoCtx::aio_operate(const std::string& oid, AioCompletion *c,
+                                ObjectWriteOperation *o, int flags, const jspan_context* otel_trace)
+{
+  object_t obj(oid);
+  if (unlikely(!o->impl))
+    return -EINVAL;
+  return io_ctx_impl->aio_operate(obj, &o->impl->o, c->pc,
+                                 io_ctx_impl->snapc, o->impl->prt,
+                                 translate_flags(flags), nullptr, otel_trace);
 }
 
 int librados::IoCtx::aio_operate(const std::string& oid, AioCompletion *c,

diff --git a/src/messages/MOSDOp.h b/src/messages/MOSDOp.h
index 61908d802cd7873d3a2b772643b32e00e750f58a..0153fdbb27231b206abd4d68001599ddeb49b57a 100644 (file)
--- a/src/messages/MOSDOp.h
+++ b/src/messages/MOSDOp.h
@@ -36,7 +36,7 @@ namespace _mosdop {
 template<typename V>
 class MOSDOp final : public MOSDFastDispatchOp {
 private:
-  static constexpr int HEAD_VERSION = 8;
+  static constexpr int HEAD_VERSION = 9;
   static constexpr int COMPAT_VERSION = 3;
 
 private:
@@ -364,9 +364,38 @@ struct ceph_osd_request_head {
 
       encode(retry_attempt, payload);
       encode(features, payload);
-    } else {
-      // latest v8 encoding with hobject_t hash separate from pgid, no
+    } else if (!HAVE_FEATURE(features, SERVER_SQUID)) {
+      // v8 encoding with hobject_t hash separate from pgid, no
       // reassert version
+      header.version = 8;
+
+      encode(pgid, payload);
+      encode(hobj.get_hash(), payload);
+      encode(osdmap_epoch, payload);
+      encode(flags, payload);
+      encode(reqid, payload);
+      encode_trace(payload, features);
+
+      // -- above decoded up front; below decoded post-dispatch thread --
+
+      encode(client_inc, payload);
+      encode(mtime, payload);
+      encode(get_object_locator(), payload);
+      encode(hobj.oid, payload);
+
+      __u16 num_ops = ops.size();
+      encode(num_ops, payload);
+      for (unsigned i = 0; i < ops.size(); i++)
+       encode(ops[i].op, payload);
+
+      encode(hobj.snap, payload);
+      encode(snap_seq, payload);
+      encode(snaps, payload);
+
+      encode(retry_attempt, payload);
+      encode(features, payload);
+    } else {
+      // latest v9 opentelemetry trace
       header.version = HEAD_VERSION;
 
       encode(pgid, payload);
@@ -375,6 +404,7 @@ struct ceph_osd_request_head {
       encode(flags, payload);
       encode(reqid, payload);
       encode_trace(payload, features);
+      encode_otel_trace(payload, features);
 
       // -- above decoded up front; below decoded post-dispatch thread --
 
@@ -404,6 +434,16 @@ struct ceph_osd_request_head {
 
     // Always keep here the newest version of decoding order/rule
     if (header.version == HEAD_VERSION) {
+      decode(pgid, p);
+      uint32_t hash;
+      decode(hash, p);
+      hobj.set_hash(hash);
+      decode(osdmap_epoch, p);
+      decode(flags, p);
+      decode(reqid, p);
+      decode_trace(p);
+      decode_otel_trace(p);
+    } else if (header.version == 8) {
       decode(pgid, p);      // actual pgid
       uint32_t hash;
       decode(hash, p); // raw hash value

diff --git a/src/msg/Message.cc b/src/msg/Message.cc
index 70ac4ad133894b03f5d5a352ed19ffdba88f49b3..1faadb22a1cd7c852d313042921012a4e598f6bc 100644 (file)
--- a/src/msg/Message.cc
+++ b/src/msg/Message.cc
@@ -1033,6 +1033,15 @@ void Message::decode_trace(ceph::bufferlist::const_iterator &p, bool create)
 #endif
 }
 
+void Message::encode_otel_trace(ceph::bufferlist &bl, uint64_t features) const
+{
+  tracing::encode(otel_trace, bl);
+}
+
+void Message::decode_otel_trace(ceph::bufferlist::const_iterator &p, bool create)
+{
+  tracing::decode(otel_trace, p);
+}
 
 // This routine is not used for ordinary messages, but only when encapsulating a message
 // for forwarding and routing.  It's also used in a backward compatibility test, which only

diff --git a/src/msg/Message.h b/src/msg/Message.h
index 40833744b67d1c337fd37e52f97a78e48112e043..e56e4bb1b6d0636510c749c223a506656224e871 100644 (file)
--- a/src/msg/Message.h
+++ b/src/msg/Message.h
@@ -32,6 +32,7 @@
 #include "common/ref.h"
 #include "common/debug.h"
 #include "common/zipkin_trace.h"
+#include "common/tracer.h"
 #include "include/ceph_assert.h" // Because intrusive_ptr clobbers our assert...
 #include "include/buffer.h"
 #include "include/types.h"
@@ -282,6 +283,11 @@ public:
   void encode_trace(ceph::buffer::list &bl, uint64_t features) const;
   void decode_trace(ceph::buffer::list::const_iterator &p, bool create = false);
 
+  // otel tracing
+  jspan_context otel_trace{false, false};
+  void encode_otel_trace(ceph::buffer::list &bl, uint64_t features) const;
+  void decode_otel_trace(ceph::buffer::list::const_iterator &p, bool create = false);
+
   class CompletionHook : public Context {
   protected:
     Message *m;

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index b0ff7883fabf193c15bef45bbf3107044e651180..5913dd9fde08eb476f200ff5852a06f91731c070 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -7566,7 +7566,12 @@ void OSD::ms_fast_dispatch(Message *m)
     tracepoint(osd, ms_fast_dispatch, reqid.name._type,
         reqid.name._num, reqid.tid, reqid.inc);
   }
-  op->osd_parent_span = tracing::osd::tracer.start_trace("op-request-created");
+
+  if (m->otel_trace.IsValid()) {
+    op->osd_parent_span = tracing::osd::tracer.add_span("op-request-created", m->otel_trace);
+  } else {
+    op->osd_parent_span = tracing::osd::tracer.start_trace("op-request-created");
+  }
 
   if (m->trace)
     op->osd_trace.init("osd op", &trace_endpoint, &m->trace);

diff --git a/src/osdc/Objecter.cc b/src/osdc/Objecter.cc
index 969b486d8997f554c63f91b4b10714b3b0c155db..d881c6e1dc3869380ac5487f01a21ab3f7828d3f 100644 (file)
--- a/src/osdc/Objecter.cc
+++ b/src/osdc/Objecter.cc
@@ -3239,6 +3239,10 @@ Objecter::MOSDOp *Objecter::_prepare_osd_op(Op *op)
     m->set_reqid(op->reqid);
   }
 
+  if (op->otel_trace && op->otel_trace->IsValid()) {
+     m->otel_trace = jspan_context(*op->otel_trace);
+  }
+
   logger->inc(l_osdc_op_send);
   ssize_t sum = 0;
   for (unsigned i = 0; i < m->ops.size(); i++) {

diff --git a/src/osdc/Objecter.h b/src/osdc/Objecter.h
index 6daf57b928d3296a47c2c334978d679e8423c30e..68bd76268ae94fd9819c016ba54725e1522a6cc6 100644 (file)
--- a/src/osdc/Objecter.h
+++ b/src/osdc/Objecter.h
@@ -56,6 +56,7 @@
 #include "common/config_obs.h"
 #include "common/shunique_lock.h"
 #include "common/zipkin_trace.h"
+#include "common/tracer.h"
 #include "common/Throttle.h"
 
 #include "mon/MonClient.h"
@@ -2057,6 +2058,7 @@ public:
 
     osd_reqid_t reqid; // explicitly setting reqid
     ZTracer::Trace trace;
+    const jspan_context* otel_trace = nullptr;
 
     static bool has_completion(decltype(onfinish)& f) {
       return std::visit([](auto&& arg) { return bool(arg);}, f);
@@ -2107,7 +2109,7 @@ public:
 
     Op(const object_t& o, const object_locator_t& ol, osdc_opvec&& _ops,
        int f, Context* fin, version_t *ov, int *offset = nullptr,
-       ZTracer::Trace *parent_trace = nullptr) :
+       ZTracer::Trace *parent_trace = nullptr, const jspan_context *otel_trace = nullptr) :
       target(o, ol, f),
       ops(std::move(_ops)),
       out_bl(ops.size(), nullptr),
@@ -2116,7 +2118,8 @@ public:
       out_ec(ops.size(), nullptr),
       onfinish(fin),
       objver(ov),
-      data_offset(offset) {
+      data_offset(offset),
+      otel_trace(otel_trace) {
       if (target.base_oloc.key == o)
        target.base_oloc.key.clear();
       if (parent_trace && parent_trace->valid()) {
@@ -3041,10 +3044,11 @@ public:
     ceph::real_time mtime, int flags,
     Context *oncommit, version_t *objver = NULL,
     osd_reqid_t reqid = osd_reqid_t(),
-    ZTracer::Trace *parent_trace = nullptr) {
+    ZTracer::Trace *parent_trace = nullptr,
+    const jspan_context *otel_trace = nullptr) {
     Op *o = new Op(oid, oloc, std::move(op.ops), flags | global_op_flags |
                   CEPH_OSD_FLAG_WRITE, oncommit, objver,
-                  nullptr, parent_trace);
+                  nullptr, nullptr, otel_trace);
     o->priority = op.priority;
     o->mtime = mtime;
     o->snapc = snapc;

diff --git a/src/rgw/driver/rados/rgw_putobj_processor.cc b/src/rgw/driver/rados/rgw_putobj_processor.cc
index d7462587c87cb3f44d82de05d97c9fd89964aaff..8a97e08248affe30c22a905051f85cad5d0d9186 100644 (file)
--- a/src/rgw/driver/rados/rgw_putobj_processor.cc
+++ b/src/rgw/driver/rados/rgw_putobj_processor.cc
@@ -146,7 +146,7 @@ int RadosWriter::process(bufferlist&& bl, uint64_t offset)
   }
   constexpr uint64_t id = 0; // unused
   auto c = aio->get(stripe_obj.obj, Aio::librados_op(stripe_obj.ioctx,
-                                                    std::move(op), y),
+                                                    std::move(op), y, &trace),
                    cost, id);
   return process_completed(c, &written);
 }
@@ -162,7 +162,7 @@ int RadosWriter::write_exclusive(const bufferlist& data)
 
   constexpr uint64_t id = 0; // unused
   auto c = aio->get(stripe_obj.obj, Aio::librados_op(stripe_obj.ioctx,
-                                                    std::move(op), y),
+                                                    std::move(op), y, &trace),
                    cost, id);
   auto d = aio->drain();
   c.splice(c.end(), d);
@@ -383,7 +383,7 @@ int AtomicObjectProcessor::complete(size_t accounted_size,
   read_cloudtier_info_from_attrs(attrs, obj_op.meta.category, manifest);
 
   r = obj_op.write_meta(actual_size, accounted_size, attrs, rctx,
-                        flags & rgw::sal::FLAG_LOG_OP);
+                        writer.get_trace(), flags & rgw::sal::FLAG_LOG_OP);
   if (r < 0) {
     if (r == -ETIMEDOUT) {
       // The head object write may eventually succeed, clear the set of objects for deletion. if it
@@ -514,7 +514,7 @@ int MultipartObjectProcessor::complete(size_t accounted_size,
   obj_op.meta.modify_tail = true;
 
   r = obj_op.write_meta(actual_size, accounted_size, attrs, rctx,
-                        flags & rgw::sal::FLAG_LOG_OP);
+                        writer.get_trace(), flags & rgw::sal::FLAG_LOG_OP);
   if (r < 0)
     return r;
 
@@ -753,7 +753,7 @@ int AppendObjectProcessor::complete(size_t accounted_size, const string &etag, c
   }
   r = obj_op.write_meta(actual_size + cur_size,
                        accounted_size + *cur_accounted_size,
-                       attrs, rctx, flags & rgw::sal::FLAG_LOG_OP);
+                       attrs, rctx, writer.get_trace(), flags & rgw::sal::FLAG_LOG_OP);
   if (r < 0) {
     return r;
   }

diff --git a/src/rgw/driver/rados/rgw_putobj_processor.h b/src/rgw/driver/rados/rgw_putobj_processor.h
index 53b14c41ee08ec92b84de8a0ed748953d24a0b9f..62be10870144af4ced7eb23424250c794d2d01d0 100644 (file)
--- a/src/rgw/driver/rados/rgw_putobj_processor.h
+++ b/src/rgw/driver/rados/rgw_putobj_processor.h
@@ -73,14 +73,15 @@ class RadosWriter : public rgw::sal::DataProcessor {
   RawObjSet written; // set of written objects for deletion
   const DoutPrefixProvider *dpp;
   optional_yield y;
+  jspan_context& trace;
 
  public:
   RadosWriter(Aio *aio, RGWRados *store,
               const RGWBucketInfo& bucket_info,
               RGWObjectCtx& obj_ctx, const rgw_obj& _head_obj,
-              const DoutPrefixProvider *dpp, optional_yield y)
+              const DoutPrefixProvider *dpp, optional_yield y, jspan_context& _trace)
     : aio(aio), store(store), bucket_info(bucket_info),
-      obj_ctx(obj_ctx), head_obj(_head_obj), dpp(dpp), y(y)
+      obj_ctx(obj_ctx), head_obj(_head_obj), dpp(dpp), y(y), trace(_trace)
   {}
   ~RadosWriter();
 
@@ -102,6 +103,7 @@ class RadosWriter : public rgw::sal::DataProcessor {
   // so they aren't deleted on destruction
   void clear_written() { written.clear(); }
 
+  jspan_context& get_trace() { return trace; }
 };
 
 
@@ -132,12 +134,14 @@ class ManifestObjectProcessor : public HeadObjectProcessor,
                           const rgw_placement_rule *ptail_placement_rule,
                           const rgw_user& owner, RGWObjectCtx& _obj_ctx,
                           const rgw_obj& _head_obj,
-                          const DoutPrefixProvider* dpp, optional_yield y)
+                          const DoutPrefixProvider* dpp,
+                          optional_yield y,
+                          jspan_context& trace)
     : HeadObjectProcessor(0),
       store(store), bucket_info(bucket_info),
       owner(owner),
       obj_ctx(_obj_ctx), head_obj(_head_obj),
-      writer(aio, store, bucket_info, obj_ctx, head_obj, dpp, y),
+      writer(aio, store, bucket_info, obj_ctx, head_obj, dpp, y, trace),
       chunk(&writer, 0), stripe(&chunk, this, 0), dpp(dpp) {
         if (ptail_placement_rule) {
           tail_placement_rule = *ptail_placement_rule;
@@ -174,9 +178,9 @@ class AtomicObjectProcessor : public ManifestObjectProcessor {
                         RGWObjectCtx& obj_ctx, const rgw_obj& _head_obj,
                         std::optional<uint64_t> olh_epoch,
                         const std::string& unique_tag,
-                        const DoutPrefixProvider *dpp, optional_yield y)
+                        const DoutPrefixProvider *dpp, optional_yield y, jspan_context& trace)
     : ManifestObjectProcessor(aio, store, bucket_info, ptail_placement_rule,
-                              owner, obj_ctx, _head_obj, dpp, y),
+                              owner, obj_ctx, _head_obj, dpp, y, trace),
       olh_epoch(olh_epoch), unique_tag(unique_tag)
   {}
 
@@ -219,9 +223,9 @@ class MultipartObjectProcessor : public ManifestObjectProcessor {
                            const rgw_obj& _head_obj,
                            const std::string& upload_id, uint64_t part_num,
                            const std::string& part_num_str,
-                           const DoutPrefixProvider *dpp, optional_yield y)
+                           const DoutPrefixProvider *dpp, optional_yield y, jspan_context& trace)
     : ManifestObjectProcessor(aio, store, bucket_info, ptail_placement_rule,
-                              owner, obj_ctx, _head_obj, dpp, y),
+                              owner, obj_ctx, _head_obj, dpp, y, trace),
       target_obj(head_obj), upload_id(upload_id),
       part_num(part_num), part_num_str(part_num_str),
       mp(head_obj.key.name, upload_id)
@@ -263,9 +267,9 @@ class MultipartObjectProcessor : public ManifestObjectProcessor {
                           const rgw_obj& _head_obj,
                           const std::string& unique_tag, uint64_t position,
                           uint64_t *cur_accounted_size,
-                          const DoutPrefixProvider *dpp, optional_yield y)
+                          const DoutPrefixProvider *dpp, optional_yield y, jspan_context& trace)
             : ManifestObjectProcessor(aio, store, bucket_info, ptail_placement_rule,
-                                      owner, obj_ctx, _head_obj, dpp, y),
+                                      owner, obj_ctx, _head_obj, dpp, y, trace),
               position(position), cur_size(0), cur_accounted_size(cur_accounted_size),
               unique_tag(unique_tag), cur_manifest(nullptr)
     {}

diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc
index b802bb114bd7fcc443a715c7c023b902b7c0c2f8..09342951592ab71be3a29131d230f26d69e89ea8 100644 (file)
--- a/src/rgw/driver/rados/rgw_rados.cc
+++ b/src/rgw/driver/rados/rgw_rados.cc
@@ -2897,6 +2897,8 @@ int RGWRados::swift_versioning_copy(RGWObjectCtx& obj_ctx,
 
   rgw_zone_id no_zone;
 
+  jspan_context no_trace{false, false};
+
   r = copy_obj(obj_ctx,
                user,
                NULL, /* req_info *info */
@@ -2925,7 +2927,8 @@ int RGWRados::swift_versioning_copy(RGWObjectCtx& obj_ctx,
                NULL, /* void (*progress_cb)(off_t, void *) */
                NULL, /* void *progress_data */
                dpp,
-               y);
+               y,
+               no_trace);
   if (r == -ECANCELED || r == -ENOENT) {
     /* Has already been overwritten, meaning another rgw process already
      * copied it out */
@@ -2991,6 +2994,8 @@ int RGWRados::swift_versioning_restore(RGWObjectCtx& obj_ctx,
     obj_ctx.set_atomic(archive_obj);
     obj_ctx.set_atomic(obj);
 
+    jspan_context no_trace{false, false};
+
     int ret = copy_obj(obj_ctx,
                        user,
                        nullptr,       /* req_info *info */
@@ -3019,7 +3024,8 @@ int RGWRados::swift_versioning_restore(RGWObjectCtx& obj_ctx,
                        nullptr,       /* void (*progress_cb)(off_t, void *) */
                        nullptr,       /* void *progress_data */
                        dpp,
-                       y);
+                       y,
+                       no_trace);
     if (ret == -ECANCELED || ret == -ENOENT) {
       /* Has already been overwritten, meaning another rgw process already
        * copied it out */
@@ -3049,7 +3055,7 @@ int RGWRados::Object::Write::_do_write_meta(uint64_t size, uint64_t accounted_si
                                            map<string, bufferlist>& attrs,
                                            bool assume_noent, bool modify_tail,
                                            void *_index_op, const req_context& rctx,
-                                           bool log_op)
+                                           jspan_context& trace, bool log_op)
 {
   RGWRados::Bucket::UpdateIndex *index_op = static_cast<RGWRados::Bucket::UpdateIndex *>(_index_op);
   RGWRados *store = target->get_store();
@@ -3224,7 +3230,7 @@ int RGWRados::Object::Write::_do_write_meta(uint64_t size, uint64_t accounted_si
   auto& ioctx = ref.ioctx;
 
   tracepoint(rgw_rados, operate_enter, req_id.c_str());
-  r = rgw_rados_operate(rctx.dpp, ref.ioctx, ref.obj.oid, &op, rctx.y);
+  r = rgw_rados_operate(rctx.dpp, ref.ioctx, ref.obj.oid, &op, rctx.y, 0, &trace);
   tracepoint(rgw_rados, operate_exit, req_id.c_str());
   if (r < 0) { /* we can expect to get -ECANCELED if object was replaced under,
                 or -ENOENT if was removed, or -EEXIST if it did not exist
@@ -3340,7 +3346,7 @@ done_cancel:
 
 int RGWRados::Object::Write::write_meta(uint64_t size, uint64_t accounted_size,
                                         map<string, bufferlist>& attrs, const req_context& rctx,
-                                        bool log_op)
+                                        jspan_context& trace, bool log_op)
 {
   RGWBucketInfo& bucket_info = target->get_bucket_info();
 
@@ -3351,13 +3357,13 @@ int RGWRados::Object::Write::write_meta(uint64_t size, uint64_t accounted_size,
   bool assume_noent = (meta.if_match == NULL && meta.if_nomatch == NULL);
   int r;
   if (assume_noent) {
-    r = _do_write_meta(size, accounted_size, attrs, assume_noent, meta.modify_tail, (void *)&index_op, rctx, log_op);
+    r = _do_write_meta(size, accounted_size, attrs, assume_noent, meta.modify_tail, (void *)&index_op, rctx, trace, log_op);
     if (r == -EEXIST) {
       assume_noent = false;
     }
   }
   if (!assume_noent) {
-    r = _do_write_meta(size, accounted_size, attrs, assume_noent, meta.modify_tail, (void *)&index_op, rctx, log_op);
+    r = _do_write_meta(size, accounted_size, attrs, assume_noent, meta.modify_tail, (void *)&index_op, rctx, trace, log_op);
   }
   return r;
 }
@@ -4166,9 +4172,10 @@ int RGWRados::fetch_remote_obj(RGWObjectCtx& obj_ctx,
 
   rgw::BlockingAioThrottle aio(cct->_conf->rgw_put_obj_min_window_size);
   using namespace rgw::putobj;
+  jspan_context no_trace{false, false};
   AtomicObjectProcessor processor(&aio, this, dest_bucket_info, nullptr,
                                   user_id, obj_ctx, dest_obj, olh_epoch,
-                                 tag, rctx.dpp, rctx.y);
+                                 tag, rctx.dpp, rctx.y, no_trace);
   RGWRESTConn *conn;
   auto& zone_conn_map = svc.zone->get_zone_conn_map();
   auto& zonegroup_conn_map = svc.zone->get_zonegroup_conn_map();
@@ -4601,7 +4608,8 @@ int RGWRados::copy_obj(RGWObjectCtx& obj_ctx,
                void (*progress_cb)(off_t, void *),
                void *progress_data,
                const DoutPrefixProvider *dpp,
-               optional_yield y)
+               optional_yield y,
+               jspan_context& trace)
 {
   int ret;
   uint64_t obj_size;
@@ -4876,7 +4884,7 @@ int RGWRados::copy_obj(RGWObjectCtx& obj_ctx,
   write_op.meta.delete_at = delete_at;
   write_op.meta.modify_tail = !copy_itself;
 
-  ret = write_op.write_meta(obj_size, astate->accounted_size, attrs, rctx);
+  ret = write_op.write_meta(obj_size, astate->accounted_size, attrs, rctx, trace);
   if (ret < 0) {
     goto done_ret;
   }
@@ -4946,9 +4954,10 @@ int RGWRados::copy_obj_data(RGWObjectCtx& obj_ctx,
 
   auto aio = rgw::make_throttle(cct->_conf->rgw_put_obj_min_window_size, y);
   using namespace rgw::putobj;
+  jspan_context no_trace{false, false};
   AtomicObjectProcessor processor(aio.get(), this, dest_bucket_info,
                                   &dest_placement, dest_bucket_info.owner,
-                                  obj_ctx, dest_obj, olh_epoch, tag, dpp, y);
+                                  obj_ctx, dest_obj, olh_epoch, tag, dpp, y, no_trace);
   int ret = processor.prepare(y);
   if (ret < 0)
     return ret;

diff --git a/src/rgw/driver/rados/rgw_rados.h b/src/rgw/driver/rados/rgw_rados.h
index 9245dc1512bd90857705a194593964282f405c7b..264f5eb4f33a1ef3f9e3963e74f81a3206f2c460 100644 (file)
--- a/src/rgw/driver/rados/rgw_rados.h
+++ b/src/rgw/driver/rados/rgw_rados.h
@@ -823,10 +823,11 @@ public:
                      std::map<std::string, bufferlist>& attrs,
                      bool modify_tail, bool assume_noent,
                      void *index_op, const req_context& rctx,
+                     jspan_context& trace,
                      bool log_op = true);
       int write_meta(uint64_t size, uint64_t accounted_size,
                      std::map<std::string, bufferlist>& attrs,
-                     const req_context& rctx, bool log_op = true);
+                     const req_context& rctx, jspan_context& trace, bool log_op = true);
       int write_data(const char *data, uint64_t ofs, uint64_t len, bool exclusive);
       const req_state* get_req_state() {
         return nullptr;  /* XXX dang Only used by LTTng, and it handles null anyway */
@@ -1199,7 +1200,8 @@ public:
                void (*progress_cb)(off_t, void *),
                void *progress_data,
                const DoutPrefixProvider *dpp,
-               optional_yield y);
+               optional_yield y,
+               jspan_context& trace);
 
   int copy_obj_data(RGWObjectCtx& obj_ctx,
                RGWBucketInfo& dest_bucket_info,

diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc
index 299f46e3c9df50ecab7c2df439be252b1ffab845..24aba37d430cb3859cb06c37155aeaa08c1fce9e 100644 (file)
--- a/src/rgw/driver/rados/rgw_sal_rados.cc
+++ b/src/rgw/driver/rados/rgw_sal_rados.cc
@@ -1427,7 +1427,7 @@ std::unique_ptr<Writer> RadosStore::get_append_writer(const DoutPrefixProvider *
                                 this, std::move(aio), owner,
                                 ptail_placement_rule,
                                 unique_tag, position,
-                                cur_accounted_size);
+                                cur_accounted_size, obj->get_trace());
 }
 
 std::unique_ptr<Writer> RadosStore::get_atomic_writer(const DoutPrefixProvider *dpp,
@@ -1445,7 +1445,7 @@ std::unique_ptr<Writer> RadosStore::get_atomic_writer(const DoutPrefixProvider *
                                 bucket_info, obj_ctx, obj->get_obj(),
                                 this, std::move(aio), owner,
                                 ptail_placement_rule,
-                                olh_epoch, unique_tag);
+                                olh_epoch, unique_tag, obj->get_trace());
 }
 
 const std::string& RadosStore::get_compression_type(const rgw_placement_rule& rule)
@@ -1854,7 +1854,7 @@ int RadosObject::write_cloud_tier(const DoutPrefixProvider* dpp,
   attrs.erase(RGW_ATTR_TAIL_TAG);
 
   const req_context rctx{dpp, y, nullptr};
-  return obj_op.write_meta(0, 0, attrs, rctx);
+  return obj_op.write_meta(0, 0, attrs, rctx, head_obj->get_trace());
 }
 
 int RadosObject::get_max_chunk_size(const DoutPrefixProvider* dpp, rgw_placement_rule placement_rule, uint64_t* max_chunk_size, uint64_t* alignment)
@@ -2100,7 +2100,8 @@ int RadosObject::copy_object(User* user,
                                     progress_cb,
                                     progress_data,
                                     dpp,
-                                    y);
+                                    y,
+                                     dest_object->get_trace());
 }
 
 int RadosObject::RadosReadOp::iterate(const DoutPrefixProvider* dpp, int64_t ofs, int64_t end, RGWGetDataCB* cb, optional_yield y)
@@ -2304,7 +2305,7 @@ int RadosMultipartUpload::init(const DoutPrefixProvider *dpp, optional_yield y,
     encode(upload_info, bl);
     obj_op.meta.data = &bl;
 
-    ret = obj_op.write_meta(bl.length(), 0, attrs, rctx, false);
+    ret = obj_op.write_meta(bl.length(), 0, attrs, rctx, get_trace(), false);
   } while (ret == -EEXIST);
 
   return ret;
@@ -2588,7 +2589,7 @@ int RadosMultipartUpload::complete(const DoutPrefixProvider *dpp,
   obj_op.meta.olh_epoch = olh_epoch;
 
   const req_context rctx{dpp, y, nullptr};
-  ret = obj_op.write_meta(ofs, accounted_size, attrs, rctx);
+  ret = obj_op.write_meta(ofs, accounted_size, attrs, rctx, get_trace());
   if (ret < 0)
     return ret;
 
@@ -2686,7 +2687,7 @@ std::unique_ptr<Writer> RadosMultipartUpload::get_writer(
   return std::make_unique<RadosMultipartWriter>(dpp, y, get_upload_id(),
                                 bucket_info, obj_ctx,
                                 obj->get_obj(), store, std::move(aio), owner,
-                                ptail_placement_rule, part_num, part_num_str);
+                                ptail_placement_rule, part_num, part_num_str, obj->get_trace());
 }
 
 MPRadosSerializer::MPRadosSerializer(const DoutPrefixProvider *dpp, RadosStore* store, RadosObject* obj, const std::string& lock_name) :

diff --git a/src/rgw/driver/rados/rgw_sal_rados.h b/src/rgw/driver/rados/rgw_sal_rados.h
index d4bd19a55fe1a23b833f49d44482b7dd2d34305f..a4f4809a2f054f15e5d156c4d09fe8650762bc89 100644 (file)
--- a/src/rgw/driver/rados/rgw_sal_rados.h
+++ b/src/rgw/driver/rados/rgw_sal_rados.h
@@ -739,7 +739,8 @@ public:
                    const rgw_user& owner,
                    const rgw_placement_rule *ptail_placement_rule,
                    uint64_t olh_epoch,
-                   const std::string& unique_tag) :
+                   const std::string& unique_tag,
+                    jspan_context& trace) :
                        StoreWriter(dpp, y),
                        store(_store),
                        aio(std::move(_aio)),
@@ -747,7 +748,7 @@ public:
                        processor(&*aio, store->getRados(), bucket_info,
                                  ptail_placement_rule, owner, obj_ctx,
                                  obj, olh_epoch, unique_tag,
-                                 dpp, y)
+                                 dpp, y, trace)
   {}
   ~RadosAtomicWriter() = default;
 
@@ -787,7 +788,8 @@ public:
                    const rgw_placement_rule *ptail_placement_rule,
                    const std::string& unique_tag,
                    uint64_t position,
-                   uint64_t *cur_accounted_size) :
+                   uint64_t *cur_accounted_size,
+                    jspan_context& trace) :
                        StoreWriter(dpp, y),
                        store(_store),
                        aio(std::move(_aio)),
@@ -795,7 +797,7 @@ public:
                        processor(&*aio, store->getRados(), bucket_info,
                                  ptail_placement_rule, owner, obj_ctx,
                                  obj, unique_tag, position,
-                                 cur_accounted_size, dpp, y)
+                                 cur_accounted_size, dpp, y, trace)
   {}
   ~RadosAppendWriter() = default;
 
@@ -833,7 +835,7 @@ public:
                       RadosStore* _store, std::unique_ptr<Aio> _aio,
                       const rgw_user& owner,
                       const rgw_placement_rule *ptail_placement_rule,
-                      uint64_t part_num, const std::string& part_num_str) :
+                      uint64_t part_num, const std::string& part_num_str, jspan_context& trace) :
                        StoreWriter(dpp, y),
                        store(_store),
                        aio(std::move(_aio)),
@@ -841,7 +843,7 @@ public:
                        processor(&*aio, store->getRados(), bucket_info,
                                  ptail_placement_rule, owner, obj_ctx,
                                  obj, upload_id,
-                                 part_num, part_num_str, dpp, y)
+                                 part_num, part_num_str, dpp, y, trace)
   {}
   ~RadosMultipartWriter() = default;
 

diff --git a/src/rgw/driver/rados/rgw_tools.cc b/src/rgw/driver/rados/rgw_tools.cc
index e4bd34aa868fd33516fd5e61cf2639a3bf7b80b4..c143875538aadc2c3b1f0268e399293c8993859e 100644 (file)
--- a/src/rgw/driver/rados/rgw_tools.cc
+++ b/src/rgw/driver/rados/rgw_tools.cc
@@ -198,7 +198,7 @@ int rgw_delete_system_obj(const DoutPrefixProvider *dpp,
 
 int rgw_rados_operate(const DoutPrefixProvider *dpp, librados::IoCtx& ioctx, const std::string& oid,
                       librados::ObjectReadOperation *op, bufferlist* pbl,
-                      optional_yield y, int flags)
+                      optional_yield y, int flags, const jspan_context* trace_info)
 {
   // given a yield_context, call async_operate() to yield the coroutine instead
   // of blocking
@@ -225,13 +225,13 @@ int rgw_rados_operate(const DoutPrefixProvider *dpp, librados::IoCtx& ioctx, con
 
 int rgw_rados_operate(const DoutPrefixProvider *dpp, librados::IoCtx& ioctx, const std::string& oid,
                       librados::ObjectWriteOperation *op, optional_yield y,
-                     int flags)
+                     int flags, const jspan_context* trace_info)
 {
   if (y) {
     auto& context = y.get_io_context();
     auto& yield = y.get_yield_context();
     boost::system::error_code ec;
-    librados::async_operate(context, ioctx, oid, op, flags, yield[ec]);
+    librados::async_operate(context, ioctx, oid, op, flags, yield[ec], trace_info);
     return -ec.value();
   }
   if (is_asio_thread) {
@@ -240,7 +240,7 @@ int rgw_rados_operate(const DoutPrefixProvider *dpp, librados::IoCtx& ioctx, con
     ldpp_dout(dpp, 20) << "BACKTRACE: " << __func__ << ": " << ClibBackTrace(0) << dendl;
 #endif
   }
-  return ioctx.operate(oid, op, flags);
+  return ioctx.operate(oid, op, flags, trace_info);
 }
 
 int rgw_rados_notify(const DoutPrefixProvider *dpp, librados::IoCtx& ioctx, const std::string& oid,

diff --git a/src/rgw/driver/rados/rgw_tools.h b/src/rgw/driver/rados/rgw_tools.h
index 27bc6f0c4daa3bf1efe86e91ac59bbebe7a26a2b..aa365deb42aab9b9ee581a9cd6f4266eb9a45ded 100644 (file)
--- a/src/rgw/driver/rados/rgw_tools.h
+++ b/src/rgw/driver/rados/rgw_tools.h
@@ -97,10 +97,10 @@ extern thread_local bool is_asio_thread;
 /// perform the rados operation, using the yield context when given
 int rgw_rados_operate(const DoutPrefixProvider *dpp, librados::IoCtx& ioctx, const std::string& oid,
                       librados::ObjectReadOperation *op, bufferlist* pbl,
-                      optional_yield y, int flags = 0);
+                      optional_yield y, int flags = 0, const jspan_context* trace_info = nullptr);
 int rgw_rados_operate(const DoutPrefixProvider *dpp, librados::IoCtx& ioctx, const std::string& oid,
                       librados::ObjectWriteOperation *op, optional_yield y,
-                     int flags = 0);
+                     int flags = 0, const jspan_context* trace_info = nullptr);
 int rgw_rados_notify(const DoutPrefixProvider *dpp, librados::IoCtx& ioctx, const std::string& oid,
                      bufferlist& bl, uint64_t timeout_ms, bufferlist* pbl,
                      optional_yield y);

diff --git a/src/rgw/rgw_aio.cc b/src/rgw/rgw_aio.cc
index c70acae79e959933dce17a2dac709316c5abe4bc..1bc50c89229c032f22c1805a87c7e53979f999a4 100644 (file)
--- a/src/rgw/rgw_aio.cc
+++ b/src/rgw/rgw_aio.cc
@@ -50,15 +50,16 @@ void cb(librados::completion_t, void* arg) {
 }
 
 template <typename Op>
-Aio::OpFunc aio_abstract(librados::IoCtx ctx, Op&& op) {
-  return [ctx = std::move(ctx), op = std::move(op)] (Aio* aio, AioResult& r) mutable {
+Aio::OpFunc aio_abstract(librados::IoCtx ctx, Op&& op, jspan_context* trace_ctx = nullptr) {
+  return [ctx = std::move(ctx), op = std::move(op), trace_ctx] (Aio* aio, AioResult& r) mutable {
       constexpr bool read = std::is_same_v<std::decay_t<Op>, librados::ObjectReadOperation>;
       // use placement new to construct the rados state inside of user_data
       auto s = new (&r.user_data) state(aio, ctx, r);
       if constexpr (read) {
+        (void)trace_ctx; // suppress unused trace_ctx warning. until we will support the read op trace
         r.result = ctx.aio_operate(r.obj.oid, s->c, &op, &r.data);
       } else {
-        r.result = ctx.aio_operate(r.obj.oid, s->c, &op);
+        r.result = ctx.aio_operate(r.obj.oid, s->c, &op, trace_ctx);
       }
       if (r.result < 0) {
         // cb() won't be called, so release everything here
@@ -89,8 +90,8 @@ struct Handler {
 template <typename Op>
 Aio::OpFunc aio_abstract(librados::IoCtx ctx, Op&& op,
                          boost::asio::io_context& context,
-                         spawn::yield_context yield) {
-  return [ctx = std::move(ctx), op = std::move(op), &context, yield] (Aio* aio, AioResult& r) mutable {
+                         spawn::yield_context yield, jspan_context* trace_ctx = nullptr) {
+  return [ctx = std::move(ctx), op = std::move(op), &context, yield, trace_ctx] (Aio* aio, AioResult& r) mutable {
       // arrange for the completion Handler to run on the yield_context's strand
       // executor so it can safely call back into Aio without locking
       using namespace boost::asio;
@@ -98,7 +99,7 @@ Aio::OpFunc aio_abstract(librados::IoCtx ctx, Op&& op,
       auto ex = get_associated_executor(init.completion_handler);
 
       librados::async_operate(context, ctx, r.obj.oid, &op, 0,
-                              bind_executor(ex, Handler{aio, ctx, r}));
+                              bind_executor(ex, Handler{aio, ctx, r}), trace_ctx);
     };
 }
 
@@ -115,15 +116,15 @@ Aio::OpFunc d3n_cache_aio_abstract(const DoutPrefixProvider *dpp, optional_yield
 
 
 template <typename Op>
-Aio::OpFunc aio_abstract(librados::IoCtx ctx, Op&& op, optional_yield y) {
+Aio::OpFunc aio_abstract(librados::IoCtx ctx, Op&& op, optional_yield y, jspan_context *trace_ctx = nullptr) {
   static_assert(std::is_base_of_v<librados::ObjectOperation, std::decay_t<Op>>);
   static_assert(!std::is_lvalue_reference_v<Op>);
   static_assert(!std::is_const_v<Op>);
   if (y) {
     return aio_abstract(std::move(ctx), std::forward<Op>(op),
-                        y.get_io_context(), y.get_yield_context());
+                        y.get_io_context(), y.get_yield_context(), trace_ctx);
   }
-  return aio_abstract(std::move(ctx), std::forward<Op>(op));
+  return aio_abstract(std::move(ctx), std::forward<Op>(op), null_yield, trace_ctx);
 }
 
 } // anonymous namespace
@@ -135,8 +136,8 @@ Aio::OpFunc Aio::librados_op(librados::IoCtx ctx,
 }
 Aio::OpFunc Aio::librados_op(librados::IoCtx ctx,
                              librados::ObjectWriteOperation&& op,
-                             optional_yield y) {
-  return aio_abstract(std::move(ctx), std::move(op), y);
+                             optional_yield y, jspan_context *trace_ctx) {
+  return aio_abstract(std::move(ctx), std::move(op), y, trace_ctx);
 }
 
 Aio::OpFunc Aio::d3n_cache_op(const DoutPrefixProvider *dpp, optional_yield y,

diff --git a/src/rgw/rgw_aio.h b/src/rgw/rgw_aio.h
index 0070346327b71b5074ff3f182aa79b37c9cacbcc..9be144f607f2dd1b62bc38a501715fef22d455b1 100644 (file)
--- a/src/rgw/rgw_aio.h
+++ b/src/rgw/rgw_aio.h
@@ -96,7 +96,7 @@ class Aio {
                             optional_yield y);
   static OpFunc librados_op(librados::IoCtx ctx,
                             librados::ObjectWriteOperation&& op,
-                            optional_yield y);
+                            optional_yield y, jspan_context *trace_ctx = nullptr);
   static OpFunc d3n_cache_op(const DoutPrefixProvider *dpp, optional_yield y,
                              off_t read_ofs, off_t read_len, std::string& location);
 };

diff --git a/src/rgw/rgw_lib.cc b/src/rgw/rgw_lib.cc
index 3168abadb0b4e10f1dde789287cdeec8133ca4bc..5a8fc14a80471cb4db74b5f6855bc6db6132de62 100644 (file)
--- a/src/rgw/rgw_lib.cc
+++ b/src/rgw/rgw_lib.cc
@@ -218,6 +218,8 @@ namespace rgw {
       goto done;
     }
 
+    s->trace = tracing::rgw::tracer.start_trace(op->name());
+
     /* req is-a RGWOp, currently initialized separately */
     ret = req->op_init();
     if (ret < 0) {

diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index f6011dbe5c9dd309e98b63ba0ece6115adae4edb..b7e50e3421f7d471d784beab538d98aa0ad3052f 100644 (file)
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -4230,6 +4230,8 @@ void RGWPutObj::execute(optional_yield y)
 
   rgw_placement_rule *pdest_placement = &s->dest_placement;
 
+  s->object->set_trace(s->trace->GetContext());
+
   if (multipart) {
     std::unique_ptr<rgw::sal::MultipartUpload> upload;
     upload = s->bucket->get_multipart_upload(s->object->get_name(),

diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h
index 6ee02be0b67f5457f8e1244aa32299577e8ca199..278ed1965bb47a796103a6733cdf904f149a6353 100644 (file)
--- a/src/rgw/rgw_sal.h
+++ b/src/rgw/rgw_sal.h
@@ -1056,6 +1056,9 @@ class Object {
     /** Get a unique copy of this object */
     virtual std::unique_ptr<Object> clone() = 0;
 
+    virtual jspan_context& get_trace() = 0;
+    virtual void set_trace (jspan_context&& _trace_ctx) = 0;
+
     /* dang - This is temporary, until the API is completed */
     /** Get the key for this object */
     virtual rgw_obj_key& get_key() = 0;
@@ -1135,7 +1138,7 @@ public:
   virtual std::map<uint32_t, std::unique_ptr<MultipartPart>>& get_parts() = 0;
 
   /** Get the trace context of this upload */
-  virtual const jspan_context& get_trace() = 0;
+  virtual jspan_context& get_trace() = 0;
 
   /** Get the Object that represents this upload */
   virtual std::unique_ptr<rgw::sal::Object> get_meta_obj() = 0;

diff --git a/src/rgw/rgw_sal_filter.h b/src/rgw/rgw_sal_filter.h
index 80e236a25d94ca94b026139b10479a0d068da7dc..b5c4c4dfc681394492d9b09632bc5df16704c0c3 100644 (file)
--- a/src/rgw/rgw_sal_filter.h
+++ b/src/rgw/rgw_sal_filter.h
@@ -649,6 +649,9 @@ public:
     return std::make_unique<FilterObject>(*this);
   }
 
+  virtual jspan_context& get_trace() { return next->get_trace(); }
+  virtual void set_trace (jspan_context&& _trace_ctx) { next->set_trace(std::move(_trace_ctx)); }
+
   virtual void print(std::ostream& out) const override { return next->print(out); }
 
   /* Internal to Filters */
@@ -688,7 +691,7 @@ public:
 
   virtual std::map<uint32_t, std::unique_ptr<MultipartPart>>& get_parts() override { return parts; }
 
-  virtual const jspan_context& get_trace() override { return next->get_trace(); }
+  virtual jspan_context& get_trace() override { return next->get_trace(); }
 
   virtual std::unique_ptr<rgw::sal::Object> get_meta_obj() override;
 

diff --git a/src/rgw/rgw_sal_store.h b/src/rgw/rgw_sal_store.h
index b1592413d782f4c5a283f8be3592572585563aff..7c35258dd5dc149d16b8556ba17e4ce3d9a5ba1b 100644 (file)
--- a/src/rgw/rgw_sal_store.h
+++ b/src/rgw/rgw_sal_store.h
@@ -145,6 +145,7 @@ class StoreObject : public Object {
     RGWObjState state;
     Bucket* bucket = nullptr;
     bool delete_marker{false};
+    jspan_context trace_ctx{false, false};
 
   public:
     StoreObject() = default;
@@ -217,6 +218,8 @@ class StoreObject : public Object {
        * work with lifecycle */
       return -1;
     }
+    jspan_context& get_trace() override { return trace_ctx; }
+    void set_trace (jspan_context&& _trace_ctx) override { trace_ctx = std::move(_trace_ctx); }
 
     virtual int get_torrent_info(const DoutPrefixProvider* dpp,
                                  optional_yield y, bufferlist& bl) override {
@@ -254,7 +257,7 @@ public:
 
   virtual std::map<uint32_t, std::unique_ptr<MultipartPart>>& get_parts() override { return parts; }
 
-  virtual const jspan_context& get_trace() override { return trace_ctx; }
+  virtual jspan_context& get_trace() override { return trace_ctx; }
 
   virtual void print(std::ostream& out) const override {
     out << get_meta();