rgw: add hook for RGWOp subclasses to write custom metadata to ops logs

Adds a write_ops_log_entry method to the RGWOp class to allow it's
subclasses an opportunity to modify the request ops log entry.

Signed-off-by: Cory Snyder <csnyder@iland.com>
(cherry picked from commit be33b5d11e3b64356a0ef10a9795e6c33b7133dc)

Conflicts:
	src/rgw/librgw.cc
	src/rgw/rgw_lib.cc
	src/rgw/rgw_log.cc
	src/rgw/rgw_log.h
	src/rgw/rgw_lua_request.cc
	src/rgw/rgw_lua_request.h
	src/test/rgw/test_rgw_lua.cc

Cherry-pick notes:
- minor conflicts due to formatting and param changes

diff --git a/src/rgw/librgw.cc b/src/rgw/librgw.cc
index 54f68085c0eb83c8133b1df6059c2b69fec40efe..4ed2a2b9985e19df957b31bae89578f8cfdb237c 100644 (file)
--- a/src/rgw/librgw.cc
+++ b/src/rgw/librgw.cc
@@ -341,7 +341,7 @@ namespace rgw {
               << e.what() << dendl;
     }
     if (should_log) {
-      rgw_log_op(nullptr /* !rest */, s, (op ? op->name() : "unknown"), olog);
+      rgw_log_op(nullptr /* !rest */, s, op, olog);
     }
 
     int http_ret = s->err.http_ret;

diff --git a/src/rgw/rgw_log.cc b/src/rgw/rgw_log.cc
index c10f63c08b97c783f8a4f83abaf438dcd543bc26..60fb5bf1263cbac3dd266e3e9f136054ab60ba72 100644 (file)
--- a/src/rgw/rgw_log.cc
+++ b/src/rgw/rgw_log.cc
@@ -525,10 +525,11 @@ done:
   return ret;
 }
 
-int rgw_log_op(RGWREST* const rest, struct req_state *s, const string& op_name, OpsLogSink *olog)
+int rgw_log_op(RGWREST* const rest, req_state *s, const RGWOp* op, OpsLogSink *olog)
 {
   struct rgw_log_entry entry;
   string bucket_id;
+  string op_name = (op ? op->name() : "unknown");
 
   if (s->enable_usage_log)
     log_usage(s, op_name);
@@ -603,6 +604,9 @@ int rgw_log_op(RGWREST* const rest, struct req_state *s, const string& op_name,
   entry.uri = std::move(uri);
 
   entry.op = op_name;
+  if (op) {
+    op->write_ops_log_entry(entry);
+  }
 
   if (s->auth.identity) {
     entry.identity_type = s->auth.identity->get_identity_type();

diff --git a/src/rgw/rgw_log.h b/src/rgw/rgw_log.h
index d91e27c2d0bf4b191a7f0713459dbaa37a9dc5d6..557724588689fa83caa52ad79c3c16f369db5ec0 100644 (file)
--- a/src/rgw/rgw_log.h
+++ b/src/rgw/rgw_log.h
@@ -14,6 +14,8 @@
 
 class RGWRados;
 
+class RGWOp;
+
 struct rgw_log_entry {
 
   using headers_map = boost::container::flat_map<std::string, std::string>;
@@ -220,7 +222,7 @@ public:
 class RGWREST;
 
 int rgw_log_op(RGWREST* const rest, struct req_state* s,
-              const std::string& op_name, OpsLogSink* olog);
+                    const RGWOp* op, OpsLogSink* olog);
 void rgw_log_usage_init(CephContext *cct, RGWRados *store);
 void rgw_log_usage_finalize();
 void rgw_format_ops_log_entry(struct rgw_log_entry& entry,

diff --git a/src/rgw/rgw_lua_request.cc b/src/rgw/rgw_lua_request.cc
index 6e5ab86daeed57153dffd0a7853956cf52c156be..a770f0cf7df6027fc89910dd8d29b5b9c1c1f2d4 100644 (file)
--- a/src/rgw/rgw_lua_request.cc
+++ b/src/rgw/rgw_lua_request.cc
@@ -27,9 +27,9 @@ int RequestLog(lua_State* L)
   const auto rest = reinterpret_cast<RGWREST*>(lua_touserdata(L, lua_upvalueindex(1)));
   const auto olog = reinterpret_cast<OpsLogSink*>(lua_touserdata(L, lua_upvalueindex(2)));
   const auto s = reinterpret_cast<req_state*>(lua_touserdata(L, lua_upvalueindex(3)));
-  const std::string op_name(reinterpret_cast<const char*>(lua_touserdata(L, lua_upvalueindex(4))));
+  const auto op(reinterpret_cast<RGWOp*>(lua_touserdata(L, lua_upvalueindex(4))));
   if (s) {
-    const auto rc = rgw_log_op(rest, s, op_name, olog);
+    const auto rc = rgw_log_op(rest, s, op, olog);
     lua_pushinteger(L, rc);
   } else {
     ldpp_dout(s, 1) << "Lua ERROR: missing request state, cannot use ops log"  << dendl;
@@ -772,11 +772,11 @@ int execute(
     RGWREST* rest,
     OpsLogSink* olog,
     req_state* s, 
-    const char* op_name,
+    RGWOp* op,
     const std::string& script)
-
 {
   auto L = luaL_newstate();
+  const char* op_name = op ? op->name() : "Unknown";
   lua_state_guard lguard(L);
 
   open_standard_libs(L);
@@ -795,7 +795,7 @@ int execute(
   lua_pushlightuserdata(L, rest);
   lua_pushlightuserdata(L, olog);
   lua_pushlightuserdata(L, s);
-  lua_pushlightuserdata(L, const_cast<char*>(op_name));
+  lua_pushlightuserdata(L, op);
   lua_pushcclosure(L, RequestLog, FOUR_UPVALS);
   lua_rawset(L, -3);
 

diff --git a/src/rgw/rgw_lua_request.h b/src/rgw/rgw_lua_request.h
index f736b9dbbb20670f94e9d48a22eef099edc1416b..b017464280b369c6cb8efdd95270541b1d19e7e4 100644 (file)
--- a/src/rgw/rgw_lua_request.h
+++ b/src/rgw/rgw_lua_request.h
@@ -18,7 +18,7 @@ int execute(
     RGWREST* rest,
     OpsLogSink* olog,
     req_state *s, 
-    const char* op_name,
+    RGWOp* op,
     const std::string& script);
 
 }

diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h
index 828d59e7b5f7978b1da55339b558b2926330bf21..89879b6280e153e99e5667d7cd047219653de777 100644 (file)
--- a/src/rgw/rgw_op.h
+++ b/src/rgw/rgw_op.h
@@ -210,6 +210,7 @@ public:
 
   virtual dmc::client_id dmclock_client() { return dmc::client_id::metadata; }
   virtual dmc::Cost dmclock_cost() { return 1; }
+  virtual void write_ops_log_entry(rgw_log_entry& entry) const {};
 };
 
 class RGWDefaultResponseOp : public RGWOp {

diff --git a/src/rgw/rgw_process.cc b/src/rgw/rgw_process.cc
index 203f5a4547e47c308c801caa490cabfa2e9cccf8..67d0a67f9ece101dfec24dcee53a2a6481277542 100644 (file)
--- a/src/rgw/rgw_process.cc
+++ b/src/rgw/rgw_process.cc
@@ -250,7 +250,7 @@ int process_request(rgw::sal::RGWRadosStore* const store,
     } else if (rc < 0) {
       ldpp_dout(op, 5) << "WARNING: failed to read pre request script. error: " << rc << dendl;
     } else {
-      rc = rgw::lua::request::execute(store, rest, olog, s, op->name(), script);
+      rc = rgw::lua::request::execute(store, rest, olog, s, op, script);
       if (rc < 0) {
         ldpp_dout(op, 5) << "WARNING: failed to execute pre request script. error: " << rc << dendl;
       }
@@ -318,7 +318,7 @@ done:
     } else if (rc < 0) {
       ldpp_dout(op, 5) << "WARNING: failed to read post request script. error: " << rc << dendl;
     } else {
-      rc = rgw::lua::request::execute(store, rest, olog, s, op->name(), script);
+      rc = rgw::lua::request::execute(store, rest, olog, s, op, script);
       if (rc < 0) {
         ldpp_dout(op, 5) << "WARNING: failed to execute post request script. error: " << rc << dendl;
       }
@@ -333,7 +333,7 @@ done:
   }
 
   if (should_log) {
-    rgw_log_op(rest, s, (op ? op->name() : "unknown"), olog);
+    rgw_log_op(rest, s, op, olog);
   }
 
   if (http_ret != nullptr) {

diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc
index 8945e929fcb083a1ab295f89813f0f63b5913b7d..b82854548965db62f3408084c1e7dcc0de93b52f 100644 (file)
--- a/src/rgw/rgw_rest_s3.cc
+++ b/src/rgw/rgw_rest_s3.cc
@@ -3935,11 +3935,11 @@ void RGWDeleteMultiObj_ObjStore_S3::send_partial_response(rgw_obj_key& key,
       s->formatter->open_object_section("Deleted");
       s->formatter->dump_string("Key", key.name);
       if (!key.instance.empty()) {
-       s->formatter->dump_string("VersionId", key.instance);
+          s->formatter->dump_string("VersionId", key.instance);
       }
       if (delete_marker) {
-       s->formatter->dump_bool("DeleteMarker", true);
-       s->formatter->dump_string("DeleteMarkerVersionId", marker_version_id);
+          s->formatter->dump_bool("DeleteMarker", true);
+          s->formatter->dump_string("DeleteMarkerVersionId", marker_version_id);
       }
       s->formatter->close_section();
     } else if (ret < 0) {

diff --git a/src/test/rgw/test_rgw_lua.cc b/src/test/rgw/test_rgw_lua.cc
index db1eab8cf468a9197f59bbbc309b406cd231cf40..a393708853458602673b5ddf633679fe5a6674a6 100644 (file)
--- a/src/test/rgw/test_rgw_lua.cc
+++ b/src/test/rgw/test_rgw_lua.cc
@@ -125,7 +125,7 @@ TEST(TestRGWLua, EmptyScript)
   uint64_t id = 0;
   req_state s(cct, &e, id); 
 
-  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "", script);
+  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script);
   ASSERT_EQ(rc, 0);
 }
 
@@ -140,7 +140,7 @@ TEST(TestRGWLua, SyntaxError)
 
   DEFINE_REQ_STATE;
 
-  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "", script);
+  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script);
   ASSERT_EQ(rc, -1);
 }
 
@@ -152,7 +152,7 @@ TEST(TestRGWLua, Hello)
 
   DEFINE_REQ_STATE;
 
-  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "", script);
+  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script);
   ASSERT_EQ(rc, 0);
 }
 
@@ -164,7 +164,7 @@ TEST(TestRGWLua, RGWDebugLogNumber)
 
   DEFINE_REQ_STATE;
 
-  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "", script);
+  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script);
   ASSERT_EQ(rc, 0);
 }
 
@@ -176,7 +176,7 @@ TEST(TestRGWLua, RGWDebugNil)
 
   DEFINE_REQ_STATE;
 
-  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "", script);
+  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script);
   ASSERT_EQ(rc, -1);
 }
 
@@ -190,7 +190,7 @@ TEST(TestRGWLua, URI)
   DEFINE_REQ_STATE;
   s.decoded_uri = "http://hello.world/";
 
-  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "", script);
+  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script);
   ASSERT_EQ(rc, 0);
 }
 
@@ -209,7 +209,7 @@ TEST(TestRGWLua, Response)
   s.err.err_code = "Bad Request";
   s.err.message = "This is a bad request";
 
-  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script);
+  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script);
   ASSERT_EQ(rc, 0);
 }
 
@@ -224,7 +224,7 @@ TEST(TestRGWLua, SetResponse)
   DEFINE_REQ_STATE;
   s.err.message = "this is a bad request";
 
-  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script);
+  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script);
   ASSERT_EQ(rc, 0);
 }
 
@@ -238,7 +238,7 @@ TEST(TestRGWLua, SetRGWId)
   DEFINE_REQ_STATE;
   s.host_id = "foo";
 
-  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script);
+  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script);
   ASSERT_NE(rc, 0);
 }
 
@@ -251,7 +251,7 @@ TEST(TestRGWLua, InvalidField)
   DEFINE_REQ_STATE;
   s.host_id = "foo";
 
-  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "kaboom", script);
+  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script);
   ASSERT_EQ(rc, -1);
 }
 
@@ -263,7 +263,7 @@ TEST(TestRGWLua, InvalidSubField)
 
   DEFINE_REQ_STATE;
 
-  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "kaboom", script);
+  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script);
   ASSERT_EQ(rc, -1);
 }
 
@@ -297,7 +297,7 @@ TEST(TestRGWLua, Bucket)
   b.bucket_id = "myid"; 
   s.bucket.reset(new sal::RGWRadosBucket(nullptr, b));
 
-  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script);
+  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script);
   ASSERT_EQ(rc, 0);
 }
 
@@ -320,7 +320,7 @@ TEST(TestRGWLua, GenericAttributes)
   s.generic_attrs["goodbye"] = "cruel world";
   s.generic_attrs["ka"] = "boom";
 
-  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script);
+  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script);
   ASSERT_EQ(rc, 0);
 }
 
@@ -343,7 +343,7 @@ TEST(TestRGWLua, Environment)
   s.env["goodbye"] = "cruel world";
   s.env["ka"] = "boom";
 
-  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script);
+  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script);
   ASSERT_EQ(rc, 0);
 }
 
@@ -364,7 +364,7 @@ TEST(TestRGWLua, Tags)
   s.tagset.add_tag("goodbye", "cruel world");
   s.tagset.add_tag("ka", "boom");
 
-  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script);
+  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script);
   ASSERT_EQ(rc, 0);
 }
 
@@ -377,7 +377,7 @@ TEST(TestRGWLua, TagsNotWriteable)
   DEFINE_REQ_STATE;
   s.tagset.add_tag("hello", "world");
 
-  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script);
+  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script);
   ASSERT_NE(rc, 0);
 }
 
@@ -403,7 +403,7 @@ TEST(TestRGWLua, Metadata)
   s.info.x_meta_map["foo"] = "bar";
   s.info.x_meta_map["ka"] = "boom";
 
-  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script);
+  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script);
   ASSERT_EQ(rc, 0);
 }
 
@@ -454,7 +454,7 @@ TEST(TestRGWLua, Acl)
   s.user_acl->get_acl().add_grant(&grant3);
   s.user_acl->get_acl().add_grant(&grant4);
   s.user_acl->get_acl().add_grant(&grant5);
-  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script);
+  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script);
   ASSERT_EQ(rc, 0);
 }
 
@@ -498,7 +498,7 @@ TEST(TestRGWLua, UseFunction)
   s.object_acl->get_owner().set_name("user five");
   s.object_acl->get_owner().set_id(rgw_user("tenant5", "user5"));
 
-  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script);
+  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script);
   ASSERT_EQ(rc, 0);
 }
 
@@ -519,7 +519,7 @@ TEST(TestRGWLua, WithLib)
   b.name = "my-bucket-name-is-fish";
   s.bucket.reset(new sal::RGWRadosBucket(nullptr, b));
 
-  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script);
+  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script);
   ASSERT_EQ(rc, 0);
 }
 
@@ -532,7 +532,7 @@ TEST(TestRGWLua, NotAllowedInLib)
 
   DEFINE_REQ_STATE;
 
-  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script);
+  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, nullptr, script);
   ASSERT_NE(rc, 0);
 }
 #include <sys/socket.h>
@@ -611,11 +611,11 @@ TEST(TestRGWLua, OpsLog)
   s.auth.identity = std::unique_ptr<rgw::auth::Identity>(
                         new FakeIdentity());
 
-  auto rc = lua::request::execute(store.get(), nullptr, olog.get(), &s, "put_obj", script);
+  auto rc = lua::request::execute(store.get(), nullptr, olog.get(), &s, nullptr, script);
   EXPECT_EQ(rc, 0);
  
        s.err.http_ret = 400;
-  rc = lua::request::execute(store.get(), nullptr, olog.get(), &s, "put_obj", script);
+  rc = lua::request::execute(store.get(), nullptr, olog.get(), &s, nullptr, script);
   EXPECT_EQ(rc, 0);
 
        // give the socket client time to read