From b2a299ff628c0124fa866f1dcae2710843399daf Mon Sep 17 00:00:00 2001 From: Cory Snyder Date: Thu, 14 Jul 2022 22:51:48 -0400 Subject: [PATCH] rgw: log deletion status of individual objects in multi object delete request Provides ops log visibility for individual objects that are deleted as part of a multi-object delete request. Fixes: https://tracker.ceph.com/issues/56645 Signed-off-by: Cory Snyder (cherry picked from commit fbd2858b610e552253fd35c80c5d52c1d7b1bfa5) Conflicts: src/test/rgw/test_rgw_lua.cc Cherry-pick notes: - rgw lua tests have been refactored some since quincy --- src/rgw/rgw_log.cc | 23 ++++++++++++++++ src/rgw/rgw_log.h | 62 ++++++++++++++++++++++++++++++++++++++++-- src/rgw/rgw_op.cc | 17 ++++++++++++ src/rgw/rgw_op.h | 4 +++ src/rgw/rgw_rest_s3.cc | 34 +++++++++++++++++------ 5 files changed, 129 insertions(+), 11 deletions(-) diff --git a/src/rgw/rgw_log.cc b/src/rgw/rgw_log.cc index eed24e82647b1..9f79ca0051548 100644 --- a/src/rgw/rgw_log.cc +++ b/src/rgw/rgw_log.cc @@ -323,6 +323,29 @@ void rgw_format_ops_log_entry(struct rgw_log_entry& entry, Formatter *formatter) formatter->dump_string("subuser", entry.subuser); } formatter->dump_bool("temp_url", entry.temp_url); + + if (entry.op == "multi_object_delete") { + formatter->open_object_section("op_data"); + formatter->dump_bool("num_ok", entry.delete_multi_obj_meta.num_ok); + formatter->dump_bool("num_err", entry.delete_multi_obj_meta.num_err); + formatter->open_array_section("objects"); + for (const auto& iter: entry.delete_multi_obj_meta.objects) { + formatter->open_object_section(""); + formatter->dump_string("key", iter.key); + formatter->dump_string("version_id", iter.version_id); + formatter->dump_int("http_status", iter.http_status); + formatter->dump_bool("error", iter.error); + if (iter.error) { + formatter->dump_string("error_message", iter.error_message); + } else { + formatter->dump_bool("delete_marker", iter.delete_marker); + formatter->dump_string("marker_version_id", iter.marker_version_id); + } + formatter->close_section(); + } + formatter->close_section(); + formatter->close_section(); + } formatter->close_section(); } diff --git a/src/rgw/rgw_log.h b/src/rgw/rgw_log.h index 114c5757dee53..cbb5fc0226f9e 100644 --- a/src/rgw/rgw_log.h +++ b/src/rgw/rgw_log.h @@ -18,6 +18,59 @@ namespace rgw { namespace sal { class RGWOp; +struct delete_multi_obj_entry { + std::string key, version_id, error_message, marker_version_id; + uint32_t http_status; + bool error, delete_marker; + + void encode(bufferlist &bl) const { + ENCODE_START(1, 1, bl); + encode(key, bl); + encode(version_id, bl); + encode(error_message, bl); + encode(marker_version_id, bl); + encode(http_status, bl); + encode(error, bl); + encode(delete_marker, bl); + ENCODE_FINISH(bl); + } + + void decode(bufferlist::const_iterator &p) { + DECODE_START_LEGACY_COMPAT_LEN(1, 1, 1, p); + decode(key, p); + decode(version_id, p); + decode(error_message, p); + decode(marker_version_id, p); + decode(http_status, p); + decode(error, p); + decode(delete_marker, p); + DECODE_FINISH(p); + } +}; +WRITE_CLASS_ENCODER(delete_multi_obj_entry) + +struct delete_multi_obj_op_meta { + uint32_t num_ok, num_err; + std::vector objects; + + void encode(bufferlist &bl) const { + ENCODE_START(1, 1, bl); + encode(num_ok, bl); + encode(num_err, bl); + encode(objects, bl); + ENCODE_FINISH(bl); + } + + void decode(bufferlist::const_iterator &p) { + DECODE_START_LEGACY_COMPAT_LEN(1, 1, 1, p); + decode(num_ok, p); + decode(num_err, p); + decode(objects, p); + DECODE_FINISH(p); + } +}; +WRITE_CLASS_ENCODER(delete_multi_obj_op_meta) + struct rgw_log_entry { using headers_map = boost::container::flat_map; @@ -48,9 +101,10 @@ struct rgw_log_entry { std::string access_key_id; std::string subuser; bool temp_url {false}; + delete_multi_obj_op_meta delete_multi_obj_meta; void encode(bufferlist &bl) const { - ENCODE_START(13, 5, bl); + ENCODE_START(14, 5, bl); encode(object_owner.id, bl); encode(bucket_owner.id, bl); encode(bucket, bl); @@ -79,10 +133,11 @@ struct rgw_log_entry { encode(access_key_id, bl); encode(subuser, bl); encode(temp_url, bl); + encode(delete_multi_obj_meta, bl); ENCODE_FINISH(bl); } void decode(bufferlist::const_iterator &p) { - DECODE_START_LEGACY_COMPAT_LEN(13, 5, 5, p); + DECODE_START_LEGACY_COMPAT_LEN(14, 5, 5, p); decode(object_owner.id, p); if (struct_v > 3) decode(bucket_owner.id, p); @@ -142,6 +197,9 @@ struct rgw_log_entry { decode(subuser, p); decode(temp_url, p); } + if (struct_v >= 14) { + decode(delete_multi_obj_meta, p); + } DECODE_FINISH(p); } void dump(ceph::Formatter *f) const; diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index 0fa2ffc85aef9..91cea06a42f08 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -6766,6 +6766,23 @@ void RGWDeleteMultiObj::pre_exec() rgw_bucket_object_pre_exec(s); } +void RGWDeleteMultiObj::write_ops_log_entry(rgw_log_entry& entry) const { + int num_err = 0; + int num_ok = 0; + for (auto iter = ops_log_entries.begin(); + iter != ops_log_entries.end(); + ++iter) { + if (iter->error) { + num_err++; + } else { + num_ok++; + } + } + entry.delete_multi_obj_meta.num_err = num_err; + entry.delete_multi_obj_meta.num_ok = num_ok; + entry.delete_multi_obj_meta.objects = std::move(ops_log_entries); +} + void RGWDeleteMultiObj::execute(optional_yield y) { RGWMultiDelDelete *multi_delete; diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h index 250c6679eb22b..fa246fe26a25d 100644 --- a/src/rgw/rgw_op.h +++ b/src/rgw/rgw_op.h @@ -44,6 +44,7 @@ #include "rgw_putobj.h" #include "rgw_sal.h" #include "rgw_compression_types.h" +#include "rgw_log.h" #include "rgw_lc.h" #include "rgw_torrent.h" @@ -2022,6 +2023,7 @@ public: class RGWDeleteMultiObj : public RGWOp { protected: + std::vector ops_log_entries; bufferlist data; rgw::sal::Bucket* bucket; bool quiet; @@ -2051,6 +2053,8 @@ public: const char* name() const override { return "multi_object_delete"; } RGWOpType get_type() override { return RGW_OP_DELETE_MULTI_OBJ; } uint32_t op_mask() override { return RGW_OP_TYPE_DELETE; } + + void write_ops_log_entry(rgw_log_entry& entry) const override; }; class RGWInfo: public RGWOp { diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index ff33756cea477..8fe87ef193aeb 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -4118,17 +4118,28 @@ void RGWDeleteMultiObj_ObjStore_S3::send_partial_response(rgw_obj_key& key, const string& marker_version_id, int ret) { if (!key.empty()) { - if (ret == 0 && !quiet) { - s->formatter->open_object_section("Deleted"); - s->formatter->dump_string("Key", key.name); - if (!key.instance.empty()) { - s->formatter->dump_string("VersionId", key.instance); - } + delete_multi_obj_entry ops_log_entry; + ops_log_entry.key = key.name; + ops_log_entry.version_id = key.instance; + if (ret == 0) { + ops_log_entry.error = false; + ops_log_entry.http_status = 200; + ops_log_entry.delete_marker = delete_marker; if (delete_marker) { - s->formatter->dump_bool("DeleteMarker", true); - s->formatter->dump_string("DeleteMarkerVersionId", marker_version_id); + ops_log_entry.marker_version_id = marker_version_id; + } + if (!quiet) { + s->formatter->open_object_section("Deleted"); + s->formatter->dump_string("Key", key.name); + if (!key.instance.empty()) { + s->formatter->dump_string("VersionId", key.instance); + } + if (delete_marker) { + s->formatter->dump_bool("DeleteMarker", true); + s->formatter->dump_string("DeleteMarkerVersionId", marker_version_id); + } + s->formatter->close_section(); } - s->formatter->close_section(); } else if (ret < 0) { struct rgw_http_error r; int err_no; @@ -4138,6 +4149,10 @@ void RGWDeleteMultiObj_ObjStore_S3::send_partial_response(rgw_obj_key& key, err_no = -ret; rgw_get_errno_s3(&r, err_no); + ops_log_entry.error = true; + ops_log_entry.http_status = r.http_ret; + ops_log_entry.error_message = r.s3_code; + s->formatter->dump_string("Key", key.name); s->formatter->dump_string("VersionId", key.instance); s->formatter->dump_string("Code", r.s3_code); @@ -4145,6 +4160,7 @@ void RGWDeleteMultiObj_ObjStore_S3::send_partial_response(rgw_obj_key& key, s->formatter->close_section(); } + ops_log_entries.push_back(std::move(ops_log_entry)); rgw_flush_formatter(s, s->formatter); } } -- 2.47.3