From: Cory Snyder Date: Fri, 15 Jul 2022 01:40:22 +0000 (-0400) Subject: rgw: add hook for RGWOp subclasses to write custom metadata to ops logs X-Git-Tag: v18.1.0~1085^2~2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=be33b5d11e3b64356a0ef10a9795e6c33b7133dc;p=ceph.git rgw: add hook for RGWOp subclasses to write custom metadata to ops logs Adds a write_ops_log_entry method to the RGWOp class to allow it's subclasses an opportunity to modify the request ops log entry. Signed-off-by: Cory Snyder --- diff --git a/src/rgw/librgw.cc b/src/rgw/librgw.cc index 38e43a59da16..bf6fc50d70c4 100644 --- a/src/rgw/librgw.cc +++ b/src/rgw/librgw.cc @@ -36,7 +36,6 @@ namespace rgw { bool global_stop = false; - static std::mutex librgw_mtx; static RGWLib rgwlib; diff --git a/src/rgw/rgw_lib.cc b/src/rgw/rgw_lib.cc index ebb69a26d143..24e7dd900d1a 100644 --- a/src/rgw/rgw_lib.cc +++ b/src/rgw/rgw_lib.cc @@ -307,7 +307,7 @@ namespace rgw { << e.what() << dendl; } if (should_log) { - rgw_log_op(nullptr /* !rest */, s, (op ? op->name() : "unknown"), olog); + rgw_log_op(nullptr /* !rest */, s, op, olog); } int http_ret = s->err.http_ret; diff --git a/src/rgw/rgw_log.cc b/src/rgw/rgw_log.cc index 75b1d9d8f30a..2618a8845c7a 100644 --- a/src/rgw/rgw_log.cc +++ b/src/rgw/rgw_log.cc @@ -519,10 +519,11 @@ int OpsLogRados::log(req_state* s, struct rgw_log_entry& entry) return 0; } -int rgw_log_op(RGWREST* const rest, req_state *s, const string& op_name, OpsLogSink *olog) +int rgw_log_op(RGWREST* const rest, req_state *s, const RGWOp* op, OpsLogSink *olog) { struct rgw_log_entry entry; string bucket_id; + string op_name = (op ? op->name() : "unknown"); if (s->enable_usage_log) log_usage(s, op_name); @@ -598,6 +599,9 @@ int rgw_log_op(RGWREST* const rest, req_state *s, const string& op_name, OpsLogS entry.uri = std::move(uri); entry.op = op_name; + if (op) { + op->write_ops_log_entry(entry); + } if (s->auth.identity) { entry.identity_type = s->auth.identity->get_identity_type(); diff --git a/src/rgw/rgw_log.h b/src/rgw/rgw_log.h index 7cceadcf6cdf..8acb433489ac 100644 --- a/src/rgw/rgw_log.h +++ b/src/rgw/rgw_log.h @@ -11,6 +11,8 @@ #include #include "rgw_sal_fwd.h" +class RGWOp; + struct rgw_log_entry { using headers_map = boost::container::flat_map; @@ -217,8 +219,8 @@ public: class RGWREST; -int rgw_log_op(RGWREST* const rest, req_state* s, - const std::string& op_name, OpsLogSink* olog); +int rgw_log_op(RGWREST* const rest, struct req_state* s, + const RGWOp* op, OpsLogSink* olog); void rgw_log_usage_init(CephContext* cct, rgw::sal::Store* store); void rgw_log_usage_finalize(); void rgw_format_ops_log_entry(struct rgw_log_entry& entry, diff --git a/src/rgw/rgw_lua_request.cc b/src/rgw/rgw_lua_request.cc index a6f56b5f96d5..869c8f886272 100644 --- a/src/rgw/rgw_lua_request.cc +++ b/src/rgw/rgw_lua_request.cc @@ -29,9 +29,9 @@ int RequestLog(lua_State* L) const auto rest = reinterpret_cast(lua_touserdata(L, lua_upvalueindex(FIRST_UPVAL))); const auto olog = reinterpret_cast(lua_touserdata(L, lua_upvalueindex(SECOND_UPVAL))); const auto s = reinterpret_cast(lua_touserdata(L, lua_upvalueindex(THIRD_UPVAL))); - const std::string op_name(reinterpret_cast(lua_touserdata(L, lua_upvalueindex(FOURTH_UPVAL)))); + const auto op(reinterpret_cast(lua_touserdata(L, lua_upvalueindex(FOURTH_UPVAL)))); if (s) { - const auto rc = rgw_log_op(rest, s, op_name, olog); + const auto rc = rgw_log_op(rest, s, op, olog); lua_pushinteger(L, rc); } else { ldpp_dout(s, 1) << "Lua ERROR: missing request state, cannot use ops log" << dendl; @@ -850,11 +850,11 @@ int execute( RGWREST* rest, OpsLogSink* olog, req_state* s, - const char* op_name, + RGWOp* op, const std::string& script) - { auto L = luaL_newstate(); + const char* op_name = op ? op->name() : "Unknown"; lua_state_guard lguard(L); open_standard_libs(L); @@ -874,7 +874,7 @@ int execute( lua_pushlightuserdata(L, rest); lua_pushlightuserdata(L, olog); lua_pushlightuserdata(L, s); - lua_pushlightuserdata(L, const_cast(op_name)); + lua_pushlightuserdata(L, op); lua_pushcclosure(L, RequestLog, FOUR_UPVALS); lua_rawset(L, -3); diff --git a/src/rgw/rgw_lua_request.h b/src/rgw/rgw_lua_request.h index 908d160eb3e9..c52fbab8f836 100644 --- a/src/rgw/rgw_lua_request.h +++ b/src/rgw/rgw_lua_request.h @@ -20,8 +20,7 @@ int execute( RGWREST* rest, OpsLogSink* olog, req_state *s, - const char* op_name, + RGWOp* op, const std::string& script); - } // namespace rgw::lua::request diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h index 9b6f110e5556..e6db5862e71a 100644 --- a/src/rgw/rgw_op.h +++ b/src/rgw/rgw_op.h @@ -277,6 +277,7 @@ public: virtual dmc::client_id dmclock_client() { return dmc::client_id::metadata; } virtual dmc::Cost dmclock_cost() { return 1; } + virtual void write_ops_log_entry(rgw_log_entry& entry) const {}; }; class RGWDefaultResponseOp : public RGWOp { diff --git a/src/rgw/rgw_process.cc b/src/rgw/rgw_process.cc index 25090baa6850..2c115444c41d 100644 --- a/src/rgw/rgw_process.cc +++ b/src/rgw/rgw_process.cc @@ -339,7 +339,7 @@ int process_request(rgw::sal::Store* const store, } else if (rc < 0) { ldpp_dout(op, 5) << "WARNING: failed to read pre request script. error: " << rc << dendl; } else { - rc = rgw::lua::request::execute(store, rest, olog, s, op->name(), script); + rc = rgw::lua::request::execute(store, rest, olog, s, op, script); if (rc < 0) { ldpp_dout(op, 5) << "WARNING: failed to execute pre request script. error: " << rc << dendl; } @@ -424,7 +424,7 @@ done: } else if (rc < 0) { ldpp_dout(op, 5) << "WARNING: failed to read post request script. error: " << rc << dendl; } else { - rc = rgw::lua::request::execute(store, rest, olog, s, op->name(), script); + rc = rgw::lua::request::execute(store, rest, olog, s, op, script); if (rc < 0) { ldpp_dout(op, 5) << "WARNING: failed to execute post request script. error: " << rc << dendl; } @@ -438,7 +438,7 @@ done: << e.what() << dendl; } if (should_log) { - rgw_log_op(rest, s, (op ? op->name() : "unknown"), olog); + rgw_log_op(rest, s, op, olog); } if (http_ret != nullptr) { diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index 8f7036eea2d0..dcb9edec8096 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -4147,11 +4147,11 @@ void RGWDeleteMultiObj_ObjStore_S3::send_partial_response(rgw_obj_key& key, s->formatter->open_object_section("Deleted"); s->formatter->dump_string("Key", key.name); if (!key.instance.empty()) { - s->formatter->dump_string("VersionId", key.instance); + s->formatter->dump_string("VersionId", key.instance); } if (delete_marker) { - s->formatter->dump_bool("DeleteMarker", true); - s->formatter->dump_string("DeleteMarkerVersionId", marker_version_id); + s->formatter->dump_bool("DeleteMarker", true); + s->formatter->dump_string("DeleteMarkerVersionId", marker_version_id); } s->formatter->close_section(); } else if (ret < 0) { diff --git a/src/test/rgw/test_rgw_lua.cc b/src/test/rgw/test_rgw_lua.cc index fe39e915fab0..41c3c7fdd53c 100644 --- a/src/test/rgw/test_rgw_lua.cc +++ b/src/test/rgw/test_rgw_lua.cc @@ -173,7 +173,7 @@ TEST(TestRGWLua, EmptyScript) DEFINE_REQ_STATE; - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, 0); } @@ -186,7 +186,7 @@ TEST(TestRGWLua, SyntaxError) DEFINE_REQ_STATE; - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, -1); } @@ -198,7 +198,7 @@ TEST(TestRGWLua, Hello) DEFINE_REQ_STATE; - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, 0); } @@ -210,7 +210,7 @@ TEST(TestRGWLua, RGWDebugLogNumber) DEFINE_REQ_STATE; - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, 0); } @@ -222,7 +222,7 @@ TEST(TestRGWLua, RGWDebugNil) DEFINE_REQ_STATE; - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, -1); } @@ -236,7 +236,7 @@ TEST(TestRGWLua, URI) DEFINE_REQ_STATE; s.decoded_uri = "http://hello.world/"; - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, 0); } @@ -255,7 +255,7 @@ TEST(TestRGWLua, Response) s.err.err_code = "Bad Request"; s.err.message = "This is a bad request"; - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, 0); } @@ -270,7 +270,7 @@ TEST(TestRGWLua, SetResponse) DEFINE_REQ_STATE; s.err.message = "this is a bad request"; - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, 0); } @@ -284,7 +284,7 @@ TEST(TestRGWLua, RGWIdNotWriteable) DEFINE_REQ_STATE; s.host_id = "foo"; - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_NE(rc, 0); } @@ -297,7 +297,7 @@ TEST(TestRGWLua, InvalidField) DEFINE_REQ_STATE; s.host_id = "foo"; - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "kaboom", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, -1); } @@ -309,7 +309,7 @@ TEST(TestRGWLua, InvalidSubField) DEFINE_REQ_STATE; - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "kaboom", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, -1); } @@ -343,7 +343,7 @@ TEST(TestRGWLua, Bucket) b.bucket_id = "myid"; s.bucket.reset(new sal::RadosBucket(nullptr, b)); - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, 0); } @@ -399,7 +399,7 @@ TEST(TestRGWLua, GenericAttributes) s.generic_attrs["goodbye"] = "cruel world"; s.generic_attrs["ka"] = "boom"; - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, 0); } @@ -421,7 +421,7 @@ TEST(TestRGWLua, Environment) s.env.emplace("goodbye", "cruel world"); s.env.emplace("ka", "boom"); - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, 0); } @@ -442,7 +442,7 @@ TEST(TestRGWLua, Tags) s.tagset.add_tag("goodbye", "cruel world"); s.tagset.add_tag("ka", "boom"); - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, 0); } @@ -455,7 +455,7 @@ TEST(TestRGWLua, TagsNotWriteable) DEFINE_REQ_STATE; s.tagset.add_tag("hello", "world"); - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_NE(rc, 0); } @@ -481,7 +481,7 @@ TEST(TestRGWLua, Metadata) s.info.x_meta_map["foo"] = "bar"; s.info.x_meta_map["ka"] = "boom"; - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, 0); } @@ -532,7 +532,7 @@ TEST(TestRGWLua, Acl) s.user_acl->get_acl().add_grant(&grant3); s.user_acl->get_acl().add_grant(&grant4); s.user_acl->get_acl().add_grant(&grant5); - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, 0); } @@ -596,7 +596,7 @@ TEST(TestRGWLua, UseFunction) s.object_acl->get_owner().set_name("user five"); s.object_acl->get_owner().set_id(rgw_user("tenant5", "user5")); - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, 0); } @@ -617,7 +617,7 @@ TEST(TestRGWLua, WithLib) b.name = "my-bucket-name-is-fish"; s.bucket.reset(new sal::RadosBucket(nullptr, b)); - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, 0); } @@ -630,7 +630,7 @@ TEST(TestRGWLua, NotAllowedInLib) DEFINE_REQ_STATE; - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_NE(rc, 0); } @@ -678,12 +678,12 @@ TEST(TestRGWLua, OpsLog) s.auth.identity = std::unique_ptr( new FakeIdentity()); - auto rc = lua::request::execute(store.get(), nullptr, &olog, &s, "put_obj", script); + auto rc = lua::request::execute(store.get(), nullptr, &olog, &s, nullptr, script); EXPECT_EQ(rc, 0); EXPECT_FALSE(olog.logged); // don't log http_ret=200 s.err.http_ret = 400; - rc = lua::request::execute(store.get(), nullptr, &olog, &s, "put_obj", script); + rc = lua::request::execute(store.get(), nullptr, &olog, &s, nullptr, script); EXPECT_EQ(rc, 0); EXPECT_TRUE(olog.logged); }