rgw: log deletion status of individual objects in multi object delete request

Provides ops log visibility for individual objects that are deleted as part
of a multi-object delete request.

Fixes: https://tracker.ceph.com/issues/56645
Signed-off-by: Cory Snyder <csnyder@iland.com>
(cherry picked from commit fbd2858b610e552253fd35c80c5d52c1d7b1bfa5)

Conflicts:
	src/test/rgw/test_rgw_lua.cc

Cherry-pick notes:
- rgw lua tests have been refactored some since quincy

diff --git a/src/rgw/rgw_log.cc b/src/rgw/rgw_log.cc
index eed24e82647b1e0b2b1f61f2b0699e006e623c45..9f79ca0051548b7c77e7434cbd53c7d7e7969135 100644 (file)
--- a/src/rgw/rgw_log.cc
+++ b/src/rgw/rgw_log.cc
@@ -323,6 +323,29 @@ void rgw_format_ops_log_entry(struct rgw_log_entry& entry, Formatter *formatter)
     formatter->dump_string("subuser", entry.subuser);
   }
   formatter->dump_bool("temp_url", entry.temp_url);
+
+  if (entry.op == "multi_object_delete") {
+    formatter->open_object_section("op_data");
+    formatter->dump_bool("num_ok", entry.delete_multi_obj_meta.num_ok);
+    formatter->dump_bool("num_err", entry.delete_multi_obj_meta.num_err);
+    formatter->open_array_section("objects");
+    for (const auto& iter: entry.delete_multi_obj_meta.objects) {
+      formatter->open_object_section("");
+      formatter->dump_string("key", iter.key);
+      formatter->dump_string("version_id", iter.version_id);
+      formatter->dump_int("http_status", iter.http_status);
+      formatter->dump_bool("error", iter.error);
+      if (iter.error) {
+        formatter->dump_string("error_message", iter.error_message);
+      } else {
+        formatter->dump_bool("delete_marker", iter.delete_marker);
+        formatter->dump_string("marker_version_id", iter.marker_version_id);
+      }
+      formatter->close_section();
+    }
+    formatter->close_section();
+    formatter->close_section();
+  }
   formatter->close_section();
 }
 

diff --git a/src/rgw/rgw_log.h b/src/rgw/rgw_log.h
index 114c5757dee53cb0db0b7151ff88e05bdeb6c183..cbb5fc0226f9e16850e1fa5a7e88efc2e7a79b2a 100644 (file)
--- a/src/rgw/rgw_log.h
+++ b/src/rgw/rgw_log.h
@@ -18,6 +18,59 @@ namespace rgw { namespace sal {
 
 class RGWOp;
 
+struct delete_multi_obj_entry {
+  std::string key, version_id, error_message, marker_version_id;
+  uint32_t http_status;
+  bool error, delete_marker;
+
+  void encode(bufferlist &bl) const {
+    ENCODE_START(1, 1, bl);
+    encode(key, bl);
+    encode(version_id, bl);
+    encode(error_message, bl);
+    encode(marker_version_id, bl);
+    encode(http_status, bl);
+    encode(error, bl);
+    encode(delete_marker, bl);
+    ENCODE_FINISH(bl);
+  }
+
+  void decode(bufferlist::const_iterator &p) {
+    DECODE_START_LEGACY_COMPAT_LEN(1, 1, 1, p);
+    decode(key, p);
+    decode(version_id, p);
+    decode(error_message, p);
+    decode(marker_version_id, p);
+    decode(http_status, p);
+    decode(error, p);
+    decode(delete_marker, p);
+    DECODE_FINISH(p);
+  }
+};
+WRITE_CLASS_ENCODER(delete_multi_obj_entry)
+
+struct delete_multi_obj_op_meta {
+  uint32_t num_ok, num_err;
+  std::vector<delete_multi_obj_entry> objects;
+
+  void encode(bufferlist &bl) const {
+    ENCODE_START(1, 1, bl);
+    encode(num_ok, bl);
+    encode(num_err, bl);
+    encode(objects, bl);
+    ENCODE_FINISH(bl);
+  }
+
+  void decode(bufferlist::const_iterator &p) {
+    DECODE_START_LEGACY_COMPAT_LEN(1, 1, 1, p);
+    decode(num_ok, p);
+    decode(num_err, p);
+    decode(objects, p);
+    DECODE_FINISH(p);
+  }
+};
+WRITE_CLASS_ENCODER(delete_multi_obj_op_meta)
+
 struct rgw_log_entry {
 
   using headers_map = boost::container::flat_map<std::string, std::string>;
@@ -48,9 +101,10 @@ struct rgw_log_entry {
   std::string access_key_id;
   std::string subuser;
   bool temp_url {false};
+  delete_multi_obj_op_meta delete_multi_obj_meta;
 
   void encode(bufferlist &bl) const {
-    ENCODE_START(13, 5, bl);
+    ENCODE_START(14, 5, bl);
     encode(object_owner.id, bl);
     encode(bucket_owner.id, bl);
     encode(bucket, bl);
@@ -79,10 +133,11 @@ struct rgw_log_entry {
     encode(access_key_id, bl);
     encode(subuser, bl);
     encode(temp_url, bl);
+    encode(delete_multi_obj_meta, bl);
     ENCODE_FINISH(bl);
   }
   void decode(bufferlist::const_iterator &p) {
-    DECODE_START_LEGACY_COMPAT_LEN(13, 5, 5, p);
+    DECODE_START_LEGACY_COMPAT_LEN(14, 5, 5, p);
     decode(object_owner.id, p);
     if (struct_v > 3)
       decode(bucket_owner.id, p);
@@ -142,6 +197,9 @@ struct rgw_log_entry {
       decode(subuser, p);
       decode(temp_url, p);
     }
+    if (struct_v >= 14) {
+      decode(delete_multi_obj_meta, p);
+    }
     DECODE_FINISH(p);
   }
   void dump(ceph::Formatter *f) const;

diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index 0fa2ffc85aef9d3d324787a08163af1a898add92..91cea06a42f08ceed63f8478a8a4d84838c51afe 100644 (file)
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -6766,6 +6766,23 @@ void RGWDeleteMultiObj::pre_exec()
   rgw_bucket_object_pre_exec(s);
 }
 
+void RGWDeleteMultiObj::write_ops_log_entry(rgw_log_entry& entry) const {
+  int num_err = 0;
+  int num_ok = 0;
+  for (auto iter = ops_log_entries.begin();
+       iter != ops_log_entries.end();
+       ++iter) {
+    if (iter->error) {
+      num_err++;
+    } else {
+      num_ok++;
+    }
+  }
+  entry.delete_multi_obj_meta.num_err = num_err;
+  entry.delete_multi_obj_meta.num_ok = num_ok;
+  entry.delete_multi_obj_meta.objects = std::move(ops_log_entries);
+}
+
 void RGWDeleteMultiObj::execute(optional_yield y)
 {
   RGWMultiDelDelete *multi_delete;

diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h
index 250c6679eb22b78187dcf9d4d0e8da2bc9213928..fa246fe26a25d4cf13a874b5cc91a4bd58da54de 100644 (file)
--- a/src/rgw/rgw_op.h
+++ b/src/rgw/rgw_op.h
@@ -44,6 +44,7 @@
 #include "rgw_putobj.h"
 #include "rgw_sal.h"
 #include "rgw_compression_types.h"
+#include "rgw_log.h"
 
 #include "rgw_lc.h"
 #include "rgw_torrent.h"
@@ -2022,6 +2023,7 @@ public:
 
 class RGWDeleteMultiObj : public RGWOp {
 protected:
+  std::vector<delete_multi_obj_entry> ops_log_entries;
   bufferlist data;
   rgw::sal::Bucket* bucket;
   bool quiet;
@@ -2051,6 +2053,8 @@ public:
   const char* name() const override { return "multi_object_delete"; }
   RGWOpType get_type() override { return RGW_OP_DELETE_MULTI_OBJ; }
   uint32_t op_mask() override { return RGW_OP_TYPE_DELETE; }
+
+  void write_ops_log_entry(rgw_log_entry& entry) const override;
 };
 
 class RGWInfo: public RGWOp {

diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc
index ff33756cea47758c1605f9f194775f2dccc1e53a..8fe87ef193aeb36251306650450f3aa7a6d0042f 100644 (file)
--- a/src/rgw/rgw_rest_s3.cc
+++ b/src/rgw/rgw_rest_s3.cc
@@ -4118,17 +4118,28 @@ void RGWDeleteMultiObj_ObjStore_S3::send_partial_response(rgw_obj_key& key,
                                                          const string& marker_version_id, int ret)
 {
   if (!key.empty()) {
-    if (ret == 0 && !quiet) {
-      s->formatter->open_object_section("Deleted");
-      s->formatter->dump_string("Key", key.name);
-      if (!key.instance.empty()) {
-          s->formatter->dump_string("VersionId", key.instance);
-      }
+    delete_multi_obj_entry ops_log_entry;
+    ops_log_entry.key = key.name;
+    ops_log_entry.version_id = key.instance;
+    if (ret == 0) {
+      ops_log_entry.error = false;
+      ops_log_entry.http_status = 200;
+      ops_log_entry.delete_marker = delete_marker;
       if (delete_marker) {
-          s->formatter->dump_bool("DeleteMarker", true);
-          s->formatter->dump_string("DeleteMarkerVersionId", marker_version_id);
+        ops_log_entry.marker_version_id = marker_version_id;
+      }
+      if (!quiet) {
+        s->formatter->open_object_section("Deleted");
+        s->formatter->dump_string("Key", key.name);
+        if (!key.instance.empty()) {
+            s->formatter->dump_string("VersionId", key.instance);
+        }
+        if (delete_marker) {
+            s->formatter->dump_bool("DeleteMarker", true);
+            s->formatter->dump_string("DeleteMarkerVersionId", marker_version_id);
+        }
+        s->formatter->close_section();
       }
-      s->formatter->close_section();
     } else if (ret < 0) {
       struct rgw_http_error r;
       int err_no;
@@ -4138,6 +4149,10 @@ void RGWDeleteMultiObj_ObjStore_S3::send_partial_response(rgw_obj_key& key,
       err_no = -ret;
       rgw_get_errno_s3(&r, err_no);
 
+      ops_log_entry.error = true;
+      ops_log_entry.http_status = r.http_ret;
+      ops_log_entry.error_message = r.s3_code;
+
       s->formatter->dump_string("Key", key.name);
       s->formatter->dump_string("VersionId", key.instance);
       s->formatter->dump_string("Code", r.s3_code);
@@ -4145,6 +4160,7 @@ void RGWDeleteMultiObj_ObjStore_S3::send_partial_response(rgw_obj_key& key,
       s->formatter->close_section();
     }
 
+    ops_log_entries.push_back(std::move(ops_log_entry));
     rgw_flush_formatter(s, s->formatter);
   }
 }