From 05d2e5f1aa46b621792f71bb4cfe21be3f5c63bb Mon Sep 17 00:00:00 2001 From: Omri Zeneva Date: Sun, 1 Aug 2021 15:00:05 +0300 Subject: [PATCH] Tracer implementation tracing for RGWOps Signed-off-by: Omri Zeneva --- src/common/options/rgw.yaml.in | 8 +++ src/rgw/CMakeLists.txt | 5 +- src/rgw/rgw_common.h | 3 ++ src/rgw/rgw_op.cc | 4 ++ src/rgw/rgw_process.cc | 29 ++++++++++- src/rgw/rgw_tracer.cc | 55 ++++++++++++++++++++ src/rgw/rgw_tracer.h | 93 ++++++++++++++++++++++++++++++++++ 7 files changed, 193 insertions(+), 4 deletions(-) create mode 100644 src/rgw/rgw_tracer.cc create mode 100644 src/rgw/rgw_tracer.h diff --git a/src/common/options/rgw.yaml.in b/src/common/options/rgw.yaml.in index cde28e0b64fb..961d22b9e6d7 100644 --- a/src/common/options/rgw.yaml.in +++ b/src/common/options/rgw.yaml.in @@ -796,6 +796,14 @@ options: services: - rgw with_legacy: true +- name: rgw_jaeger_enable + type: bool + level: advanced + desc: should RGW use jaeger tracing system + default: false + services: + - rgw + with_legacy: true - name: rgw_s3_auth_use_keystone type: bool level: advanced diff --git a/src/rgw/CMakeLists.txt b/src/rgw/CMakeLists.txt index 9d5741149c39..231e6ac5c6f4 100644 --- a/src/rgw/CMakeLists.txt +++ b/src/rgw/CMakeLists.txt @@ -156,7 +156,8 @@ set(librgw_common_srcs rgw_lua_utils.cc rgw_lua.cc rgw_lua_request.cc - rgw_bucket_encryption.cc) + rgw_bucket_encryption.cc + rgw_tracer.cc) if(WITH_RADOSGW_AMQP_ENDPOINT) list(APPEND librgw_common_srcs rgw_amqp.cc) @@ -229,7 +230,7 @@ if(WITH_LTTNG) endif() if(WITH_JAEGER) - add_dependencies(rgw_common jaegertracing::libjaegertracing) + add_dependencies(rgw_common jaeger-base) endif() if(WITH_RADOSGW_DBSTORE) diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h index 1a24707c358f..f1768672165e 100644 --- a/src/rgw/rgw_common.h +++ b/src/rgw/rgw_common.h @@ -38,6 +38,7 @@ #include "cls/rgw/cls_rgw_types.h" #include "include/rados/librados.hpp" #include "rgw_public_access.h" +#include "rgw_tracer.h" namespace ceph { class Formatter; @@ -1647,6 +1648,8 @@ struct req_state : DoutPrefixProvider { std::vector session_policies; + jspan trace; + req_state(CephContext* _cct, RGWEnv* e, uint64_t id); ~req_state(); diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index 4cb9bed0197c..e7c15f5fc7a0 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -51,6 +51,7 @@ #include "rgw_notify_event_type.h" #include "rgw_sal.h" #include "rgw_sal_rados.h" +#include "rgw_tracer.h" #include "services/svc_zone.h" #include "services/svc_quota.h" @@ -3739,6 +3740,7 @@ void RGWPutObj::execute(optional_yield y) rgw_placement_rule *pdest_placement = &s->dest_placement; if (multipart) { + s->trace->SetTag(tracing::UPLOAD_ID, multipart_upload_id); std::unique_ptr upload; upload = store->get_multipart_upload(s->bucket.get(), s->object->get_name(), multipart_upload_id); @@ -5919,6 +5921,7 @@ void RGWInitMultipart::execute(optional_yield y) if (op_ret == 0) { upload_id = upload->get_upload_id(); } + s->trace->SetTag(tracing::UPLOAD_ID, upload_id); } @@ -6038,6 +6041,7 @@ void RGWCompleteMultipart::execute(optional_yield y) upload = store->get_multipart_upload(s->bucket.get(), s->object->get_name(), upload_id); + s->trace->SetTag(tracing::UPLOAD_ID, upload_id); RGWCompressionInfo cs_info; bool compressed = false; diff --git a/src/rgw/rgw_process.cc b/src/rgw/rgw_process.cc index b5e886b6915b..9cf78fe8809b 100644 --- a/src/rgw/rgw_process.cc +++ b/src/rgw/rgw_process.cc @@ -144,7 +144,12 @@ int rgw_process_authenticated(RGWHandler_REST * const handler, } ldpp_dout(op, 2) << "verifying op permissions" << dendl; - ret = op->verify_permission(y); + { + auto span = rgw_tracer.start_span("verify_permission", s->trace); + std::swap(span, s->trace); + ret = op->verify_permission(y); + std::swap(span, s->trace); + } if (ret < 0) { if (s->system_request) { dout(2) << "overriding permissions due to system operation" << dendl; @@ -165,7 +170,12 @@ int rgw_process_authenticated(RGWHandler_REST * const handler, op->pre_exec(); ldpp_dout(op, 2) << "executing" << dendl; - op->execute(y); + { + auto span = rgw_tracer.start_span("execute", s->trace); + std::swap(span, s->trace); + op->execute(y); + std::swap(span, s->trace); + } ldpp_dout(op, 2) << "completing" << dendl; op->complete(); @@ -297,6 +307,11 @@ int process_request(rgw::sal::Store* const store, goto done; } + const auto trace_name = std::string(op->name()) + " " + s->trans_id; + s->trace = rgw_tracer.start_trace(trace_name); + s->trace->SetTag(tracing::OP, op->name()); + s->trace->SetTag(tracing::TYPE, tracing::REQUEST); + ret = rgw_process_authenticated(handler, op, req, s, yield); if (ret < 0) { abort_early(s, op, ret, handler, yield); @@ -309,6 +324,16 @@ int process_request(rgw::sal::Store* const store, done: if (op) { + s->trace->SetTag(tracing::RETURN, op->get_ret()); + if (s->user) { + s->trace->SetTag(tracing::USER_ID, s->user->get_id().id); + } + if (s->bucket) { + s->trace->SetTag(tracing::BUCKET_NAME, s->bucket->get_name()); + } + if (s->object) { + s->trace->SetTag(tracing::OBJECT_NAME, s->object->get_name()); + } std::string script; auto rc = rgw::lua::read_script(s, store, s->bucket_tenant, s->yield, rgw::lua::context::postRequest, script); if (rc == -ENOENT) { diff --git a/src/rgw/rgw_tracer.cc b/src/rgw/rgw_tracer.cc new file mode 100644 index 000000000000..b0b251728577 --- /dev/null +++ b/src/rgw/rgw_tracer.cc @@ -0,0 +1,55 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp +#include "common/ceph_context.h" +#include "global/global_context.h" +#include "rgw_tracer.h" + +#ifdef HAVE_JAEGER + +thread_local tracing::Tracer rgw_tracer(jaeger_configuration::jaeger_default_config); + +namespace tracing { + +const std::shared_ptr Tracer::noop_tracer = opentracing::MakeNoopTracer(); + +Tracer::Tracer(jaegertracing::Config& conf):open_tracer(jaegertracing::Tracer::make(conf)) {} + +std::unique_ptr Tracer::start_trace(opentracing::string_view trace_name) { + if(is_enabled()) { + return open_tracer->StartSpan(trace_name); + } + return noop_tracer->StartSpan(trace_name); +} + +std::unique_ptr Tracer::start_span(opentracing::string_view span_name, std::unique_ptr& parent_span) { + if(is_enabled()) { + return open_tracer->StartSpan(span_name, { opentracing::ChildOf(&parent_span->context()) }); + } + return noop_tracer->StartSpan(span_name); +} + +bool Tracer::is_enabled() const { + return g_ceph_context->_conf->rgw_jaeger_enable; +} +} // namespace tracing + +namespace jaeger_configuration { + +jaegertracing::samplers::Config const_sampler("const", 1, "", 0, jaegertracing::samplers::Config::defaultSamplingRefreshInterval()); + +jaegertracing::reporters::Config reporter_default_config(jaegertracing::reporters::Config::kDefaultQueueSize, jaegertracing::reporters::Config::defaultBufferFlushInterval(), true, jaegertracing::reporters::Config::kDefaultLocalAgentHostPort, ""); + +jaegertracing::propagation::HeadersConfig headers_config("","","",""); + +jaegertracing::baggage::RestrictionsConfig baggage_config(false, "", std::chrono::steady_clock::duration()); + +jaegertracing::Config jaeger_default_config(false, const_sampler, reporter_default_config, headers_config, baggage_config, "rgw", std::vector()); + +} + +#else // !HAVE_JAEGER + +tracing::Tracer rgw_tracer; + +#endif + \ No newline at end of file diff --git a/src/rgw/rgw_tracer.h b/src/rgw/rgw_tracer.h new file mode 100644 index 000000000000..5c70bc7e55ab --- /dev/null +++ b/src/rgw/rgw_tracer.h @@ -0,0 +1,93 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +#ifndef RGW_TRACER_H +#define RGW_TRACER_H + +#ifdef HAVE_JAEGER + +#define SIGNED_RIGHT_SHIFT_IS 1 +#define ARITHMETIC_RIGHT_SHIFT 1 +#include + +typedef std::unique_ptr jspan; + +#else // !HAVE_JAEGER + +struct span_stub { + template + void SetTag(std::string_view key, const T& value) const noexcept {} + void Log(std::initializer_list> fields) {} +}; + +class jspan { + span_stub span; + public: + span_stub& operator*() { return span; } + const span_stub& operator*() const { return span; } + + span_stub* operator->() { return &span; } + const span_stub* operator->() const { return &span; } +}; + +#endif // !HAVE_JAEGER + +namespace tracing { + +const auto OP = "op"; +const auto BUCKET_NAME = "bucket_name"; +const auto USER_ID = "user_id"; +const auto OBJECT_NAME = "object_name"; +const auto RETURN = "return"; +const auto UPLOAD_ID = "upload_id"; +const auto TYPE = "type"; +const auto REQUEST = "request"; + +#ifdef HAVE_JAEGER + +class Tracer { +private: + const static std::shared_ptr noop_tracer; + std::shared_ptr open_tracer; + +public: + Tracer(jaegertracing::Config& conf); + bool is_enabled() const; + // creates and returns a new span with `trace_name` + // this span represents a trace, since it has no parent. + jspan start_trace(opentracing::string_view trace_name); + // creates and returns a new span with `trace_name` which parent span is `parent_span' + jspan start_span(opentracing::string_view span_name, jspan& parent_span); +}; + +#else // !HAVE_JAEGER + +struct Tracer { + bool is_enabled() const { return false; } + jspan start_trace(std::string_view) { return {}; } + jspan start_span(std::string_view, const jspan&) { return {}; } +}; + +#endif + +} // namespace tracing + + +#ifdef HAVE_JAEGER + +extern thread_local tracing::Tracer rgw_tracer; + +namespace jaeger_configuration { + +extern jaegertracing::Config jaeger_default_config; + +} + +#else // !HAVE_JAEGER + +extern tracing::Tracer rgw_tracer; + +#endif + +#endif // RGW_TRACER_H + \ No newline at end of file -- 2.47.3