From: Cory Snyder Date: Fri, 15 Jul 2022 01:40:22 +0000 (-0400) Subject: rgw: add hook for RGWOp subclasses to write custom metadata to ops logs X-Git-Tag: v17.2.6~210^2~2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=eaef2c4f55fbf7b468b9046d8230da5f7d1cf336;p=ceph.git rgw: add hook for RGWOp subclasses to write custom metadata to ops logs Adds a write_ops_log_entry method to the RGWOp class to allow it's subclasses an opportunity to modify the request ops log entry. Signed-off-by: Cory Snyder (cherry picked from commit be33b5d11e3b64356a0ef10a9795e6c33b7133dc) Conflicts: src/rgw/librgw.cc src/rgw/rgw_lib.cc src/rgw/rgw_log.cc src/rgw/rgw_log.h src/rgw/rgw_lua_request.cc src/rgw/rgw_lua_request.h src/test/rgw/test_rgw_lua.cc Cherry-pick notes: - Minor conflicts with due to whitespace and code movement --- diff --git a/src/rgw/librgw.cc b/src/rgw/librgw.cc index 18c4140e60b3..6f030f8e5016 100644 --- a/src/rgw/librgw.cc +++ b/src/rgw/librgw.cc @@ -340,7 +340,7 @@ namespace rgw { << e.what() << dendl; } if (should_log) { - rgw_log_op(nullptr /* !rest */, s, (op ? op->name() : "unknown"), olog); + rgw_log_op(nullptr /* !rest */, s, op, olog); } int http_ret = s->err.http_ret; diff --git a/src/rgw/rgw_log.cc b/src/rgw/rgw_log.cc index c51eecded53d..eed24e82647b 100644 --- a/src/rgw/rgw_log.cc +++ b/src/rgw/rgw_log.cc @@ -515,10 +515,11 @@ int OpsLogRados::log(struct req_state* s, struct rgw_log_entry& entry) return 0; } -int rgw_log_op(RGWREST* const rest, struct req_state *s, const string& op_name, OpsLogSink *olog) +int rgw_log_op(RGWREST* const rest, struct req_state *s, const RGWOp* op, OpsLogSink *olog) { struct rgw_log_entry entry; string bucket_id; + string op_name = (op ? op->name() : "unknown"); if (s->enable_usage_log) log_usage(s, op_name); @@ -594,6 +595,9 @@ int rgw_log_op(RGWREST* const rest, struct req_state *s, const string& op_name, entry.uri = std::move(uri); entry.op = op_name; + if (op) { + op->write_ops_log_entry(entry); + } if (s->auth.identity) { entry.identity_type = s->auth.identity->get_identity_type(); diff --git a/src/rgw/rgw_log.h b/src/rgw/rgw_log.h index acbf441f47b3..114c5757dee5 100644 --- a/src/rgw/rgw_log.h +++ b/src/rgw/rgw_log.h @@ -16,6 +16,8 @@ namespace rgw { namespace sal { class Store; } } +class RGWOp; + struct rgw_log_entry { using headers_map = boost::container::flat_map; @@ -223,7 +225,7 @@ public: class RGWREST; int rgw_log_op(RGWREST* const rest, struct req_state* s, - const std::string& op_name, OpsLogSink* olog); + const RGWOp* op, OpsLogSink* olog); void rgw_log_usage_init(CephContext* cct, rgw::sal::Store* store); void rgw_log_usage_finalize(); void rgw_format_ops_log_entry(struct rgw_log_entry& entry, diff --git a/src/rgw/rgw_lua_request.cc b/src/rgw/rgw_lua_request.cc index 2e971c329c87..f1aa5e5b34f8 100644 --- a/src/rgw/rgw_lua_request.cc +++ b/src/rgw/rgw_lua_request.cc @@ -27,9 +27,9 @@ int RequestLog(lua_State* L) const auto rest = reinterpret_cast(lua_touserdata(L, lua_upvalueindex(1))); const auto olog = reinterpret_cast(lua_touserdata(L, lua_upvalueindex(2))); const auto s = reinterpret_cast(lua_touserdata(L, lua_upvalueindex(3))); - const std::string op_name(reinterpret_cast(lua_touserdata(L, lua_upvalueindex(4)))); + const auto op(reinterpret_cast(lua_touserdata(L, lua_upvalueindex(4)))); if (s) { - const auto rc = rgw_log_op(rest, s, op_name, olog); + const auto rc = rgw_log_op(rest, s, op, olog); lua_pushinteger(L, rc); } else { ldpp_dout(s, 1) << "Lua ERROR: missing request state, cannot use ops log" << dendl; @@ -793,11 +793,11 @@ int execute( RGWREST* rest, OpsLogSink* olog, req_state* s, - const char* op_name, + RGWOp* op, const std::string& script) - { auto L = luaL_newstate(); + const char* op_name = op ? op->name() : "Unknown"; lua_state_guard lguard(L); open_standard_libs(L); @@ -816,7 +816,7 @@ int execute( lua_pushlightuserdata(L, rest); lua_pushlightuserdata(L, olog); lua_pushlightuserdata(L, s); - lua_pushlightuserdata(L, const_cast(op_name)); + lua_pushlightuserdata(L, op); lua_pushcclosure(L, RequestLog, FOUR_UPVALS); lua_rawset(L, -3); diff --git a/src/rgw/rgw_lua_request.h b/src/rgw/rgw_lua_request.h index fcf0673f3ba7..17884069c032 100644 --- a/src/rgw/rgw_lua_request.h +++ b/src/rgw/rgw_lua_request.h @@ -18,8 +18,8 @@ int execute( RGWREST* rest, OpsLogSink* olog, req_state *s, - const char* op_name, + RGWOp* op, const std::string& script); -} +} // namespace rgw::lua::request diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h index c5bd7f62a3d5..250c6679eb22 100644 --- a/src/rgw/rgw_op.h +++ b/src/rgw/rgw_op.h @@ -278,6 +278,7 @@ public: virtual dmc::client_id dmclock_client() { return dmc::client_id::metadata; } virtual dmc::Cost dmclock_cost() { return 1; } + virtual void write_ops_log_entry(rgw_log_entry& entry) const {}; }; class RGWDefaultResponseOp : public RGWOp { diff --git a/src/rgw/rgw_process.cc b/src/rgw/rgw_process.cc index f8d43a3258d2..413c1150dd2c 100644 --- a/src/rgw/rgw_process.cc +++ b/src/rgw/rgw_process.cc @@ -338,7 +338,7 @@ int process_request(rgw::sal::Store* const store, } else if (rc < 0) { ldpp_dout(op, 5) << "WARNING: failed to read pre request script. error: " << rc << dendl; } else { - rc = rgw::lua::request::execute(store, rest, olog, s, op->name(), script); + rc = rgw::lua::request::execute(store, rest, olog, s, op, script); if (rc < 0) { ldpp_dout(op, 5) << "WARNING: failed to execute pre request script. error: " << rc << dendl; } @@ -420,7 +420,7 @@ done: } else if (rc < 0) { ldpp_dout(op, 5) << "WARNING: failed to read post request script. error: " << rc << dendl; } else { - rc = rgw::lua::request::execute(store, rest, olog, s, op->name(), script); + rc = rgw::lua::request::execute(store, rest, olog, s, op, script); if (rc < 0) { ldpp_dout(op, 5) << "WARNING: failed to execute post request script. error: " << rc << dendl; } @@ -434,7 +434,7 @@ done: << e.what() << dendl; } if (should_log) { - rgw_log_op(rest, s, (op ? op->name() : "unknown"), olog); + rgw_log_op(rest, s, op, olog); } if (http_ret != nullptr) { diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index 9b5128d08450..ff33756cea47 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -4122,11 +4122,11 @@ void RGWDeleteMultiObj_ObjStore_S3::send_partial_response(rgw_obj_key& key, s->formatter->open_object_section("Deleted"); s->formatter->dump_string("Key", key.name); if (!key.instance.empty()) { - s->formatter->dump_string("VersionId", key.instance); + s->formatter->dump_string("VersionId", key.instance); } if (delete_marker) { - s->formatter->dump_bool("DeleteMarker", true); - s->formatter->dump_string("DeleteMarkerVersionId", marker_version_id); + s->formatter->dump_bool("DeleteMarker", true); + s->formatter->dump_string("DeleteMarkerVersionId", marker_version_id); } s->formatter->close_section(); } else if (ret < 0) { diff --git a/src/test/rgw/test_rgw_lua.cc b/src/test/rgw/test_rgw_lua.cc index 501144d5a34d..0b5d1a72f6cd 100644 --- a/src/test/rgw/test_rgw_lua.cc +++ b/src/test/rgw/test_rgw_lua.cc @@ -164,7 +164,7 @@ TEST(TestRGWLua, EmptyScript) uint64_t id = 0; req_state s(cct, &e, id); - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, 0); } @@ -179,7 +179,7 @@ TEST(TestRGWLua, SyntaxError) DEFINE_REQ_STATE; - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, -1); } @@ -191,7 +191,7 @@ TEST(TestRGWLua, Hello) DEFINE_REQ_STATE; - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, 0); } @@ -203,7 +203,7 @@ TEST(TestRGWLua, RGWDebugLogNumber) DEFINE_REQ_STATE; - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, 0); } @@ -215,7 +215,7 @@ TEST(TestRGWLua, RGWDebugNil) DEFINE_REQ_STATE; - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, -1); } @@ -229,7 +229,7 @@ TEST(TestRGWLua, URI) DEFINE_REQ_STATE; s.decoded_uri = "http://hello.world/"; - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, 0); } @@ -248,7 +248,7 @@ TEST(TestRGWLua, Response) s.err.err_code = "Bad Request"; s.err.message = "This is a bad request"; - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, 0); } @@ -263,7 +263,7 @@ TEST(TestRGWLua, SetResponse) DEFINE_REQ_STATE; s.err.message = "this is a bad request"; - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, 0); } @@ -277,7 +277,7 @@ TEST(TestRGWLua, SetRGWId) DEFINE_REQ_STATE; s.host_id = "foo"; - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_NE(rc, 0); } @@ -290,7 +290,7 @@ TEST(TestRGWLua, InvalidField) DEFINE_REQ_STATE; s.host_id = "foo"; - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "kaboom", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, -1); } @@ -302,7 +302,7 @@ TEST(TestRGWLua, InvalidSubField) DEFINE_REQ_STATE; - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "kaboom", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, -1); } @@ -336,7 +336,7 @@ TEST(TestRGWLua, Bucket) b.bucket_id = "myid"; s.bucket.reset(new sal::RadosBucket(nullptr, b)); - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, 0); } @@ -359,7 +359,7 @@ TEST(TestRGWLua, GenericAttributes) s.generic_attrs["goodbye"] = "cruel world"; s.generic_attrs["ka"] = "boom"; - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, 0); } @@ -381,7 +381,7 @@ TEST(TestRGWLua, Environment) s.env.emplace("goodbye", "cruel world"); s.env.emplace("ka", "boom"); - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, 0); } @@ -402,7 +402,7 @@ TEST(TestRGWLua, Tags) s.tagset.add_tag("goodbye", "cruel world"); s.tagset.add_tag("ka", "boom"); - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, 0); } @@ -415,7 +415,7 @@ TEST(TestRGWLua, TagsNotWriteable) DEFINE_REQ_STATE; s.tagset.add_tag("hello", "world"); - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_NE(rc, 0); } @@ -441,7 +441,7 @@ TEST(TestRGWLua, Metadata) s.info.x_meta_map["foo"] = "bar"; s.info.x_meta_map["ka"] = "boom"; - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, 0); } @@ -492,7 +492,7 @@ TEST(TestRGWLua, Acl) s.user_acl->get_acl().add_grant(&grant3); s.user_acl->get_acl().add_grant(&grant4); s.user_acl->get_acl().add_grant(&grant5); - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, 0); } @@ -511,7 +511,7 @@ TEST(TestRGWLua, User) u.id = "myid"; s.user.reset(new sal::RadosUser(nullptr, u)); - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, 0); } @@ -556,7 +556,7 @@ TEST(TestRGWLua, UseFunction) s.object_acl->get_owner().set_name("user five"); s.object_acl->get_owner().set_id(rgw_user("tenant5", "user5")); - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, 0); } @@ -577,7 +577,7 @@ TEST(TestRGWLua, WithLib) b.name = "my-bucket-name-is-fish"; s.bucket.reset(new sal::RadosBucket(nullptr, b)); - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, 0); } @@ -590,7 +590,7 @@ TEST(TestRGWLua, NotAllowedInLib) DEFINE_REQ_STATE; - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_NE(rc, 0); } #include @@ -669,11 +669,11 @@ TEST(TestRGWLua, OpsLog) s.auth.identity = std::unique_ptr( new FakeIdentity()); - auto rc = lua::request::execute(store.get(), nullptr, olog.get(), &s, "put_obj", script); + auto rc = lua::request::execute(store.get(), nullptr, olog.get(), &s, nullptr, script); EXPECT_EQ(rc, 0); s.err.http_ret = 400; - rc = lua::request::execute(store.get(), nullptr, olog.get(), &s, "put_obj", script); + rc = lua::request::execute(store.get(), nullptr, olog.get(), &s, nullptr, script); EXPECT_EQ(rc, 0); // give the socket client time to read