From: Cory Snyder Date: Fri, 15 Jul 2022 01:40:22 +0000 (-0400) Subject: rgw: add hook for RGWOp subclasses to write custom metadata to ops logs X-Git-Tag: v16.2.11~258^2~2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=f079de22e54e5efe95a04a7d057cb35d301ec45c;p=ceph.git rgw: add hook for RGWOp subclasses to write custom metadata to ops logs Adds a write_ops_log_entry method to the RGWOp class to allow it's subclasses an opportunity to modify the request ops log entry. Signed-off-by: Cory Snyder (cherry picked from commit be33b5d11e3b64356a0ef10a9795e6c33b7133dc) Conflicts: src/rgw/librgw.cc src/rgw/rgw_lib.cc src/rgw/rgw_log.cc src/rgw/rgw_log.h src/rgw/rgw_lua_request.cc src/rgw/rgw_lua_request.h src/test/rgw/test_rgw_lua.cc Cherry-pick notes: - minor conflicts due to formatting and param changes --- diff --git a/src/rgw/librgw.cc b/src/rgw/librgw.cc index 54f68085c0eb..4ed2a2b9985e 100644 --- a/src/rgw/librgw.cc +++ b/src/rgw/librgw.cc @@ -341,7 +341,7 @@ namespace rgw { << e.what() << dendl; } if (should_log) { - rgw_log_op(nullptr /* !rest */, s, (op ? op->name() : "unknown"), olog); + rgw_log_op(nullptr /* !rest */, s, op, olog); } int http_ret = s->err.http_ret; diff --git a/src/rgw/rgw_log.cc b/src/rgw/rgw_log.cc index c10f63c08b97..60fb5bf1263c 100644 --- a/src/rgw/rgw_log.cc +++ b/src/rgw/rgw_log.cc @@ -525,10 +525,11 @@ done: return ret; } -int rgw_log_op(RGWREST* const rest, struct req_state *s, const string& op_name, OpsLogSink *olog) +int rgw_log_op(RGWREST* const rest, req_state *s, const RGWOp* op, OpsLogSink *olog) { struct rgw_log_entry entry; string bucket_id; + string op_name = (op ? op->name() : "unknown"); if (s->enable_usage_log) log_usage(s, op_name); @@ -603,6 +604,9 @@ int rgw_log_op(RGWREST* const rest, struct req_state *s, const string& op_name, entry.uri = std::move(uri); entry.op = op_name; + if (op) { + op->write_ops_log_entry(entry); + } if (s->auth.identity) { entry.identity_type = s->auth.identity->get_identity_type(); diff --git a/src/rgw/rgw_log.h b/src/rgw/rgw_log.h index d91e27c2d0bf..557724588689 100644 --- a/src/rgw/rgw_log.h +++ b/src/rgw/rgw_log.h @@ -14,6 +14,8 @@ class RGWRados; +class RGWOp; + struct rgw_log_entry { using headers_map = boost::container::flat_map; @@ -220,7 +222,7 @@ public: class RGWREST; int rgw_log_op(RGWREST* const rest, struct req_state* s, - const std::string& op_name, OpsLogSink* olog); + const RGWOp* op, OpsLogSink* olog); void rgw_log_usage_init(CephContext *cct, RGWRados *store); void rgw_log_usage_finalize(); void rgw_format_ops_log_entry(struct rgw_log_entry& entry, diff --git a/src/rgw/rgw_lua_request.cc b/src/rgw/rgw_lua_request.cc index 6e5ab86daeed..a770f0cf7df6 100644 --- a/src/rgw/rgw_lua_request.cc +++ b/src/rgw/rgw_lua_request.cc @@ -27,9 +27,9 @@ int RequestLog(lua_State* L) const auto rest = reinterpret_cast(lua_touserdata(L, lua_upvalueindex(1))); const auto olog = reinterpret_cast(lua_touserdata(L, lua_upvalueindex(2))); const auto s = reinterpret_cast(lua_touserdata(L, lua_upvalueindex(3))); - const std::string op_name(reinterpret_cast(lua_touserdata(L, lua_upvalueindex(4)))); + const auto op(reinterpret_cast(lua_touserdata(L, lua_upvalueindex(4)))); if (s) { - const auto rc = rgw_log_op(rest, s, op_name, olog); + const auto rc = rgw_log_op(rest, s, op, olog); lua_pushinteger(L, rc); } else { ldpp_dout(s, 1) << "Lua ERROR: missing request state, cannot use ops log" << dendl; @@ -772,11 +772,11 @@ int execute( RGWREST* rest, OpsLogSink* olog, req_state* s, - const char* op_name, + RGWOp* op, const std::string& script) - { auto L = luaL_newstate(); + const char* op_name = op ? op->name() : "Unknown"; lua_state_guard lguard(L); open_standard_libs(L); @@ -795,7 +795,7 @@ int execute( lua_pushlightuserdata(L, rest); lua_pushlightuserdata(L, olog); lua_pushlightuserdata(L, s); - lua_pushlightuserdata(L, const_cast(op_name)); + lua_pushlightuserdata(L, op); lua_pushcclosure(L, RequestLog, FOUR_UPVALS); lua_rawset(L, -3); diff --git a/src/rgw/rgw_lua_request.h b/src/rgw/rgw_lua_request.h index f736b9dbbb20..b017464280b3 100644 --- a/src/rgw/rgw_lua_request.h +++ b/src/rgw/rgw_lua_request.h @@ -18,7 +18,7 @@ int execute( RGWREST* rest, OpsLogSink* olog, req_state *s, - const char* op_name, + RGWOp* op, const std::string& script); } diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h index 828d59e7b5f7..89879b6280e1 100644 --- a/src/rgw/rgw_op.h +++ b/src/rgw/rgw_op.h @@ -210,6 +210,7 @@ public: virtual dmc::client_id dmclock_client() { return dmc::client_id::metadata; } virtual dmc::Cost dmclock_cost() { return 1; } + virtual void write_ops_log_entry(rgw_log_entry& entry) const {}; }; class RGWDefaultResponseOp : public RGWOp { diff --git a/src/rgw/rgw_process.cc b/src/rgw/rgw_process.cc index 203f5a4547e4..67d0a67f9ece 100644 --- a/src/rgw/rgw_process.cc +++ b/src/rgw/rgw_process.cc @@ -250,7 +250,7 @@ int process_request(rgw::sal::RGWRadosStore* const store, } else if (rc < 0) { ldpp_dout(op, 5) << "WARNING: failed to read pre request script. error: " << rc << dendl; } else { - rc = rgw::lua::request::execute(store, rest, olog, s, op->name(), script); + rc = rgw::lua::request::execute(store, rest, olog, s, op, script); if (rc < 0) { ldpp_dout(op, 5) << "WARNING: failed to execute pre request script. error: " << rc << dendl; } @@ -318,7 +318,7 @@ done: } else if (rc < 0) { ldpp_dout(op, 5) << "WARNING: failed to read post request script. error: " << rc << dendl; } else { - rc = rgw::lua::request::execute(store, rest, olog, s, op->name(), script); + rc = rgw::lua::request::execute(store, rest, olog, s, op, script); if (rc < 0) { ldpp_dout(op, 5) << "WARNING: failed to execute post request script. error: " << rc << dendl; } @@ -333,7 +333,7 @@ done: } if (should_log) { - rgw_log_op(rest, s, (op ? op->name() : "unknown"), olog); + rgw_log_op(rest, s, op, olog); } if (http_ret != nullptr) { diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index 8945e929fcb0..b82854548965 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -3935,11 +3935,11 @@ void RGWDeleteMultiObj_ObjStore_S3::send_partial_response(rgw_obj_key& key, s->formatter->open_object_section("Deleted"); s->formatter->dump_string("Key", key.name); if (!key.instance.empty()) { - s->formatter->dump_string("VersionId", key.instance); + s->formatter->dump_string("VersionId", key.instance); } if (delete_marker) { - s->formatter->dump_bool("DeleteMarker", true); - s->formatter->dump_string("DeleteMarkerVersionId", marker_version_id); + s->formatter->dump_bool("DeleteMarker", true); + s->formatter->dump_string("DeleteMarkerVersionId", marker_version_id); } s->formatter->close_section(); } else if (ret < 0) { diff --git a/src/test/rgw/test_rgw_lua.cc b/src/test/rgw/test_rgw_lua.cc index db1eab8cf468..a39370885345 100644 --- a/src/test/rgw/test_rgw_lua.cc +++ b/src/test/rgw/test_rgw_lua.cc @@ -125,7 +125,7 @@ TEST(TestRGWLua, EmptyScript) uint64_t id = 0; req_state s(cct, &e, id); - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, 0); } @@ -140,7 +140,7 @@ TEST(TestRGWLua, SyntaxError) DEFINE_REQ_STATE; - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, -1); } @@ -152,7 +152,7 @@ TEST(TestRGWLua, Hello) DEFINE_REQ_STATE; - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, 0); } @@ -164,7 +164,7 @@ TEST(TestRGWLua, RGWDebugLogNumber) DEFINE_REQ_STATE; - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, 0); } @@ -176,7 +176,7 @@ TEST(TestRGWLua, RGWDebugNil) DEFINE_REQ_STATE; - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, -1); } @@ -190,7 +190,7 @@ TEST(TestRGWLua, URI) DEFINE_REQ_STATE; s.decoded_uri = "http://hello.world/"; - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, 0); } @@ -209,7 +209,7 @@ TEST(TestRGWLua, Response) s.err.err_code = "Bad Request"; s.err.message = "This is a bad request"; - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, 0); } @@ -224,7 +224,7 @@ TEST(TestRGWLua, SetResponse) DEFINE_REQ_STATE; s.err.message = "this is a bad request"; - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, 0); } @@ -238,7 +238,7 @@ TEST(TestRGWLua, SetRGWId) DEFINE_REQ_STATE; s.host_id = "foo"; - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_NE(rc, 0); } @@ -251,7 +251,7 @@ TEST(TestRGWLua, InvalidField) DEFINE_REQ_STATE; s.host_id = "foo"; - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "kaboom", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, -1); } @@ -263,7 +263,7 @@ TEST(TestRGWLua, InvalidSubField) DEFINE_REQ_STATE; - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "kaboom", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, -1); } @@ -297,7 +297,7 @@ TEST(TestRGWLua, Bucket) b.bucket_id = "myid"; s.bucket.reset(new sal::RGWRadosBucket(nullptr, b)); - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, 0); } @@ -320,7 +320,7 @@ TEST(TestRGWLua, GenericAttributes) s.generic_attrs["goodbye"] = "cruel world"; s.generic_attrs["ka"] = "boom"; - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, 0); } @@ -343,7 +343,7 @@ TEST(TestRGWLua, Environment) s.env["goodbye"] = "cruel world"; s.env["ka"] = "boom"; - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, 0); } @@ -364,7 +364,7 @@ TEST(TestRGWLua, Tags) s.tagset.add_tag("goodbye", "cruel world"); s.tagset.add_tag("ka", "boom"); - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, 0); } @@ -377,7 +377,7 @@ TEST(TestRGWLua, TagsNotWriteable) DEFINE_REQ_STATE; s.tagset.add_tag("hello", "world"); - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_NE(rc, 0); } @@ -403,7 +403,7 @@ TEST(TestRGWLua, Metadata) s.info.x_meta_map["foo"] = "bar"; s.info.x_meta_map["ka"] = "boom"; - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, 0); } @@ -454,7 +454,7 @@ TEST(TestRGWLua, Acl) s.user_acl->get_acl().add_grant(&grant3); s.user_acl->get_acl().add_grant(&grant4); s.user_acl->get_acl().add_grant(&grant5); - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, 0); } @@ -498,7 +498,7 @@ TEST(TestRGWLua, UseFunction) s.object_acl->get_owner().set_name("user five"); s.object_acl->get_owner().set_id(rgw_user("tenant5", "user5")); - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, 0); } @@ -519,7 +519,7 @@ TEST(TestRGWLua, WithLib) b.name = "my-bucket-name-is-fish"; s.bucket.reset(new sal::RGWRadosBucket(nullptr, b)); - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_EQ(rc, 0); } @@ -532,7 +532,7 @@ TEST(TestRGWLua, NotAllowedInLib) DEFINE_REQ_STATE; - const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script); + const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script); ASSERT_NE(rc, 0); } #include @@ -611,11 +611,11 @@ TEST(TestRGWLua, OpsLog) s.auth.identity = std::unique_ptr( new FakeIdentity()); - auto rc = lua::request::execute(store.get(), nullptr, olog.get(), &s, "put_obj", script); + auto rc = lua::request::execute(store.get(), nullptr, olog.get(), &s, nullptr, script); EXPECT_EQ(rc, 0); s.err.http_ret = 400; - rc = lua::request::execute(store.get(), nullptr, olog.get(), &s, "put_obj", script); + rc = lua::request::execute(store.get(), nullptr, olog.get(), &s, nullptr, script); EXPECT_EQ(rc, 0); // give the socket client time to read