rgw: log deletion status of individual objects in multi object delete request

Provides ops log visibility for individual objects that are deleted as part
of a multi-object delete request.

Fixes: https://tracker.ceph.com/issues/56645
Signed-off-by: Cory Snyder <csnyder@iland.com>
 Conflicts:
	src/rgw/rgw_op.h
	src/test/rgw/test_rgw_lua.cc

Cherry-pick notes:
- minor conflicts due to formatting changes and added tests

diff --git a/src/rgw/rgw_log.cc b/src/rgw/rgw_log.cc
index 60fb5bf1263cbac3dd266e3e9f136054ab60ba72..915134056fbe63a03ebae735a2abf7fcba41e7a8 100644 (file)
--- a/src/rgw/rgw_log.cc
+++ b/src/rgw/rgw_log.cc
@@ -321,6 +321,29 @@ void rgw_format_ops_log_entry(struct rgw_log_entry& entry, Formatter *formatter)
     formatter->dump_string("subuser", entry.subuser);
   }
   formatter->dump_bool("temp_url", entry.temp_url);
+
+  if (entry.op == "multi_object_delete") {
+    formatter->open_object_section("op_data");
+    formatter->dump_bool("num_ok", entry.delete_multi_obj_meta.num_ok);
+    formatter->dump_bool("num_err", entry.delete_multi_obj_meta.num_err);
+    formatter->open_array_section("objects");
+    for (const auto& iter: entry.delete_multi_obj_meta.objects) {
+      formatter->open_object_section("");
+      formatter->dump_string("key", iter.key);
+      formatter->dump_string("version_id", iter.version_id);
+      formatter->dump_int("http_status", iter.http_status);
+      formatter->dump_bool("error", iter.error);
+      if (iter.error) {
+        formatter->dump_string("error_message", iter.error_message);
+      } else {
+        formatter->dump_bool("delete_marker", iter.delete_marker);
+        formatter->dump_string("marker_version_id", iter.marker_version_id);
+      }
+      formatter->close_section();
+    }
+    formatter->close_section();
+    formatter->close_section();
+  }
   formatter->close_section();
 }
 

diff --git a/src/rgw/rgw_log.h b/src/rgw/rgw_log.h
index 557724588689fa83caa52ad79c3c16f369db5ec0..591f6c48964bd5e0ed8a48f3a96f23535db66585 100644 (file)
--- a/src/rgw/rgw_log.h
+++ b/src/rgw/rgw_log.h
@@ -16,6 +16,59 @@ class RGWRados;
 
 class RGWOp;
 
+struct delete_multi_obj_entry {
+  std::string key, version_id, error_message, marker_version_id;
+  uint32_t http_status;
+  bool error, delete_marker;
+
+  void encode(bufferlist &bl) const {
+    ENCODE_START(1, 1, bl);
+    encode(key, bl);
+    encode(version_id, bl);
+    encode(error_message, bl);
+    encode(marker_version_id, bl);
+    encode(http_status, bl);
+    encode(error, bl);
+    encode(delete_marker, bl);
+    ENCODE_FINISH(bl);
+  }
+
+  void decode(bufferlist::const_iterator &p) {
+    DECODE_START_LEGACY_COMPAT_LEN(1, 1, 1, p);
+    decode(key, p);
+    decode(version_id, p);
+    decode(error_message, p);
+    decode(marker_version_id, p);
+    decode(http_status, p);
+    decode(error, p);
+    decode(delete_marker, p);
+    DECODE_FINISH(p);
+  }
+};
+WRITE_CLASS_ENCODER(delete_multi_obj_entry)
+
+struct delete_multi_obj_op_meta {
+  uint32_t num_ok, num_err;
+  std::vector<delete_multi_obj_entry> objects;
+
+  void encode(bufferlist &bl) const {
+    ENCODE_START(1, 1, bl);
+    encode(num_ok, bl);
+    encode(num_err, bl);
+    encode(objects, bl);
+    ENCODE_FINISH(bl);
+  }
+
+  void decode(bufferlist::const_iterator &p) {
+    DECODE_START_LEGACY_COMPAT_LEN(1, 1, 1, p);
+    decode(num_ok, p);
+    decode(num_err, p);
+    decode(objects, p);
+    DECODE_FINISH(p);
+  }
+};
+WRITE_CLASS_ENCODER(delete_multi_obj_op_meta)
+
 struct rgw_log_entry {
 
   using headers_map = boost::container::flat_map<std::string, std::string>;
@@ -46,9 +99,10 @@ struct rgw_log_entry {
   std::string access_key_id;
   std::string subuser;
   bool temp_url {false};
+  delete_multi_obj_op_meta delete_multi_obj_meta;
 
   void encode(bufferlist &bl) const {
-    ENCODE_START(13, 5, bl);
+    ENCODE_START(14, 5, bl);
     encode(object_owner.id, bl);
     encode(bucket_owner.id, bl);
     encode(bucket, bl);
@@ -77,10 +131,11 @@ struct rgw_log_entry {
     encode(access_key_id, bl);
     encode(subuser, bl);
     encode(temp_url, bl);
+    encode(delete_multi_obj_meta, bl);
     ENCODE_FINISH(bl);
   }
   void decode(bufferlist::const_iterator &p) {
-    DECODE_START_LEGACY_COMPAT_LEN(13, 5, 5, p);
+    DECODE_START_LEGACY_COMPAT_LEN(14, 5, 5, p);
     decode(object_owner.id, p);
     if (struct_v > 3)
       decode(bucket_owner.id, p);
@@ -140,6 +195,9 @@ struct rgw_log_entry {
       decode(subuser, p);
       decode(temp_url, p);
     }
+    if (struct_v >= 14) {
+      decode(delete_multi_obj_meta, p);
+    }
     DECODE_FINISH(p);
   }
   void dump(ceph::Formatter *f) const;

diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index 04f94f60f3f8d10f0888f95a4bf96beb857fbf14..2d1356159a0171dbfb863ea514110f3abc82fc6b 100644 (file)
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -6773,6 +6773,23 @@ void RGWDeleteMultiObj::pre_exec()
   rgw_bucket_object_pre_exec(s);
 }
 
+void RGWDeleteMultiObj::write_ops_log_entry(rgw_log_entry& entry) const {
+  int num_err = 0;
+  int num_ok = 0;
+  for (auto iter = ops_log_entries.begin();
+       iter != ops_log_entries.end();
+       ++iter) {
+    if (iter->error) {
+      num_err++;
+    } else {
+      num_ok++;
+    }
+  }
+  entry.delete_multi_obj_meta.num_err = num_err;
+  entry.delete_multi_obj_meta.num_ok = num_ok;
+  entry.delete_multi_obj_meta.objects = std::move(ops_log_entries);
+}
+
 void RGWDeleteMultiObj::execute(optional_yield y)
 {
   RGWMultiDelDelete *multi_delete;

diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h
index 89879b6280e153e99e5667d7cd047219653de777..92a4afe7789c68f6ee7f67b3cc29f810bf407e24 100644 (file)
--- a/src/rgw/rgw_op.h
+++ b/src/rgw/rgw_op.h
@@ -44,6 +44,7 @@
 #include "rgw_putobj.h"
 #include "rgw_multi.h"
 #include "rgw_sal.h"
+#include "rgw_log.h"
 
 #include "rgw_lc.h"
 #include "rgw_torrent.h"
@@ -1907,6 +1908,7 @@ public:
 
 class RGWDeleteMultiObj : public RGWOp {
 protected:
+  std::vector<delete_multi_obj_entry> ops_log_entries;
   bufferlist data;
   rgw::sal::RGWBucket* bucket;
   bool quiet;
@@ -1936,6 +1938,8 @@ public:
   const char* name() const override { return "multi_object_delete"; }
   RGWOpType get_type() override { return RGW_OP_DELETE_MULTI_OBJ; }
   uint32_t op_mask() override { return RGW_OP_TYPE_DELETE; }
+
+  void write_ops_log_entry(rgw_log_entry& entry) const override;
 };
 
 class RGWInfo: public RGWOp {

diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc
index b82854548965db62f3408084c1e7dcc0de93b52f..f5ee6095287a41859080a73879a36d93cda6be06 100644 (file)
--- a/src/rgw/rgw_rest_s3.cc
+++ b/src/rgw/rgw_rest_s3.cc
@@ -3931,17 +3931,28 @@ void RGWDeleteMultiObj_ObjStore_S3::send_partial_response(rgw_obj_key& key,
                                                          const string& marker_version_id, int ret)
 {
   if (!key.empty()) {
-    if (ret == 0 && !quiet) {
-      s->formatter->open_object_section("Deleted");
-      s->formatter->dump_string("Key", key.name);
-      if (!key.instance.empty()) {
-          s->formatter->dump_string("VersionId", key.instance);
-      }
+    delete_multi_obj_entry ops_log_entry;
+    ops_log_entry.key = key.name;
+    ops_log_entry.version_id = key.instance;
+    if (ret == 0) {
+      ops_log_entry.error = false;
+      ops_log_entry.http_status = 200;
+      ops_log_entry.delete_marker = delete_marker;
       if (delete_marker) {
-          s->formatter->dump_bool("DeleteMarker", true);
-          s->formatter->dump_string("DeleteMarkerVersionId", marker_version_id);
+        ops_log_entry.marker_version_id = marker_version_id;
+      }
+      if (!quiet) {
+        s->formatter->open_object_section("Deleted");
+        s->formatter->dump_string("Key", key.name);
+        if (!key.instance.empty()) {
+            s->formatter->dump_string("VersionId", key.instance);
+        }
+        if (delete_marker) {
+            s->formatter->dump_bool("DeleteMarker", true);
+            s->formatter->dump_string("DeleteMarkerVersionId", marker_version_id);
+        }
+        s->formatter->close_section();
       }
-      s->formatter->close_section();
     } else if (ret < 0) {
       struct rgw_http_error r;
       int err_no;
@@ -3951,6 +3962,10 @@ void RGWDeleteMultiObj_ObjStore_S3::send_partial_response(rgw_obj_key& key,
       err_no = -ret;
       rgw_get_errno_s3(&r, err_no);
 
+      ops_log_entry.error = true;
+      ops_log_entry.http_status = r.http_ret;
+      ops_log_entry.error_message = r.s3_code;
+
       s->formatter->dump_string("Key", key.name);
       s->formatter->dump_string("VersionId", key.instance);
       s->formatter->dump_string("Code", r.s3_code);
@@ -3958,6 +3973,7 @@ void RGWDeleteMultiObj_ObjStore_S3::send_partial_response(rgw_obj_key& key,
       s->formatter->close_section();
     }
 
+    ops_log_entries.push_back(std::move(ops_log_entry));
     rgw_flush_formatter(s, s->formatter);
   }
 }

diff --git a/src/test/rgw/test_rgw_lua.cc b/src/test/rgw/test_rgw_lua.cc
index a393708853458602673b5ddf633679fe5a6674a6..80e249a686f1fa1b14058cc4a12387200c04b986 100644 (file)
--- a/src/test/rgw/test_rgw_lua.cc
+++ b/src/test/rgw/test_rgw_lua.cc
@@ -622,5 +622,6 @@ TEST(TestRGWLua, OpsLog)
        std::this_thread::sleep_for(std::chrono::seconds(5));
        unix_socket_thread.detach(); // read is stuck there, so we cannot join
   EXPECT_TRUE(unix_socket_client_ended_ok);
+
 }