From 12c201b2f94f807013e7705dd8f9f1dbb3c299f6 Mon Sep 17 00:00:00 2001 From: Yuval Lifshitz Date: Mon, 12 May 2025 15:43:52 +0000 Subject: [PATCH] rgw/logging: support object metadata changes in journal mode Fixes: https://tracker.ceph.com/issues/71255 Signed-off-by: Yuval Lifshitz (cherry picked from commit 94016c7786f29f3061a181c132abd4069337f5c9) --- doc/radosgw/bucket_logging.rst | 34 ++++++++++++-- doc/radosgw/s3/bucketops.rst | 2 +- src/rgw/rgw_op.cc | 84 ++++++++++++++++++++++++++++++++++ src/rgw/rgw_rest.h | 4 ++ src/rgw/rgw_rest_s3.h | 1 + 5 files changed, 120 insertions(+), 5 deletions(-) diff --git a/doc/radosgw/bucket_logging.rst b/doc/radosgw/bucket_logging.rst index 8364020ccbc6d..b0b95d8e1e681 100644 --- a/doc/radosgw/bucket_logging.rst +++ b/doc/radosgw/bucket_logging.rst @@ -11,7 +11,7 @@ log data can be used to monitor bucket activity, detect unauthorized access, get insights into the bucket usage and use the logs as a journal for bucket changes. The log records are stored in objects in a separate bucket and can be analyzed later. Logging configuration is done at the bucket level and can be enabled or disabled at any time. -The log bucket can accumulate logs from multiple buckets. It is recommended to configured +The log bucket can accumulate logs from multiple buckets. It is recommended to configured a different "prefix" for each bucket, so that the logs of different buckets will be stored in different objects in the log bucket. @@ -50,12 +50,38 @@ This means that there are the logging operation may fail, with no indication to Journal ``````` -If logging type is set to "Journal", the records are written to the log bucket before the bucket operation is completed. -This means that if the logging action fails, the operation will not be executed, and an error will be returned to the client. -An exception to the above are "multi/delete" log records: if writing these log records fail, the operation continues and may still be successful. +If logging type is set to "Journal", the records are written to the log bucket before the bucket operation is completed. +This means that if the logging action fails, the operation will not be executed, and an error will be returned to the client +Some exceptions exists to that rule, the "Fails Operation" columns in the table below indicate by "No" which operations will not fail even if logging failed. Journal mode supports filtering out records based on matches of the prefixes and suffixes of the logged object keys. Regular-expression matching can also be used on these to create filters. Note that it may happen that the log records were successfully written, but the bucket operation failed, since the logs are written. +The following operation are supported in journal mode: + ++-------------------------------+-------------------------------------+-----------------+ +| Operation | Operation Name | Fails Operation | ++===============================+=====================================+=================+ +| ``PutObject`` | ``REST.PUT.OBJECT`` | Yes | ++-------------------------------+-------------------------------------+-----------------+ +| ``DeleteObject`` | ``REST.DELETE.OBJECT`` | No | ++-------------------------------+-------------------------------------+-----------------+ +| ``DeleteObjects`` | ``REST.POST.DELETE_MULTI_OBJECT`` | No | ++-------------------------------+-------------------------------------+-----------------+ +| ``CompleteMultipartUpload`` | ``REST.POST.UPLOAD`` | Yes | ++-------------------------------+-------------------------------------+-----------------+ +| ``CopyObject`` | ``REST.PUT.OBJECT`` | Yes | ++-------------------------------+-------------------------------------+-----------------+ +| ``PutObjectAcl`` | ``REST.PUT.ACL`` | Yes | ++-------------------------------+-------------------------------------+-----------------+ +| ``PutObjectLegalHold`` | ``REST.PUT.LEGAL_HOLD`` | Yes | ++-------------------------------+-------------------------------------+-----------------+ +| ``PutObjectRetention`` | ``REST.PUT.RETENTION`` | Yes | ++-------------------------------+-------------------------------------+-----------------+ +| ``PutObjectTagging`` | ``REST.PUT.OBJECT_TAGGING`` | Yes | ++-------------------------------+-------------------------------------+-----------------+ +| ``DeleteObjectTagging`` | ``REST.DELETE.OBJECT_TAGGING`` | No | ++-------------------------------+-------------------------------------+-----------------+ + Bucket Logging Policy --------------------- diff --git a/doc/radosgw/s3/bucketops.rst b/doc/radosgw/s3/bucketops.rst index 2d3ff1729e159..f15c8329c2614 100644 --- a/doc/radosgw/s3/bucketops.rst +++ b/doc/radosgw/s3/bucketops.rst @@ -788,7 +788,7 @@ Parameters are XML encoded in the body of the request, in the following format: | ``LoggingType`` | String | The type of logging. Valid values are: | No | | | | ``Standard`` (default) all bucket operations are logged after being perfomed. | | | | | The log record will contain all fields. | | -| | | ``Journal`` only PUT, COPY, MULTI/DELETE and MPU operations are logged. | | +| | | ``Journal`` only operations that modify and object are logged. | | | | | Will record the minimum subset of fields in the log record that is needed | | | | | for journaling. | | +-------------------------------+-----------+--------------------------------------------------------------------------------------+----------+ diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index 81c32f5449b86..859ade5a7998a 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -1295,6 +1295,25 @@ void RGWPutObjTags::execute(optional_yield y) return; } + op_ret = s->object->get_obj_attrs(y, this); + if (op_ret < 0) { + ldpp_dout(this, 0) << "ERROR: failed to get obj attrs, obj=" << s->object + << " ret=" << op_ret << dendl; + return; + } + const auto etag = s->object->get_attrs()[RGW_ATTR_ETAG].to_str(); + op_ret = rgw::bucketlogging::log_record(driver, + rgw::bucketlogging::LoggingType::Journal, + s->object.get(), + s, + canonical_name(), + etag, + s->object->get_size(), + this, y, false, false); + if (op_ret < 0) { + return; + } + s->object->set_atomic(true); op_ret = s->object->modify_obj_attrs(RGW_ATTR_TAGS, tags_bl, y, this); if (op_ret == -ECANCELED){ @@ -1329,6 +1348,25 @@ void RGWDeleteObjTags::execute(optional_yield y) if (rgw::sal::Object::empty(s->object.get())) return; + op_ret = s->object->get_obj_attrs(y, this); + if (op_ret < 0) { + ldpp_dout(this, 0) << "ERROR: failed to get obj attrs, obj=" << s->object + << " ret=" << op_ret << dendl; + return; + } + const auto etag = s->object->get_attrs()[RGW_ATTR_ETAG].to_str(); + op_ret = rgw::bucketlogging::log_record(driver, + rgw::bucketlogging::LoggingType::Journal, + s->object.get(), + s, + canonical_name(), + etag, + s->object->get_size(), + this, y, false, false); + if (op_ret < 0) { + return; + } + op_ret = s->object->delete_obj_attrs(this, RGW_ATTR_TAGS, y); } @@ -6317,6 +6355,19 @@ void RGWPutACLs::execute(optional_yield y) return; } + const auto etag = s->object->get_attrs()[RGW_ATTR_ETAG].to_str(); + op_ret = rgw::bucketlogging::log_record(driver, + rgw::bucketlogging::LoggingType::Journal, + s->object.get(), + s, + canonical_name(), + etag, + s->object->get_size(), + this, y, false, false); + if (op_ret < 0) { + return; + } + bufferlist bl; new_policy.encode(bl); map attrs; @@ -8991,6 +9042,19 @@ void RGWPutObjRetention::execute(optional_yield y) } } + const auto etag = s->object->get_attrs()[RGW_ATTR_ETAG].to_str(); + op_ret = rgw::bucketlogging::log_record(driver, + rgw::bucketlogging::LoggingType::Journal, + s->object.get(), + s, + canonical_name(), + etag, + s->object->get_size(), + this, y, false, false); + if (op_ret < 0) { + return; + } + op_ret = s->object->modify_obj_attrs(RGW_ATTR_OBJECT_RETENTION, bl, s->yield, this); return; @@ -9093,6 +9157,26 @@ void RGWPutObjLegalHold::execute(optional_yield y) { op_ret = -ERR_MALFORMED_XML; return; } + + op_ret = s->object->get_obj_attrs(y, this); + if (op_ret < 0) { + ldpp_dout(this, 0) << "ERROR: failed to get obj attrs, obj=" << s->object + << " ret=" << op_ret << dendl; + return; + } + const auto etag = s->object->get_attrs()[RGW_ATTR_ETAG].to_str(); + op_ret = rgw::bucketlogging::log_record(driver, + rgw::bucketlogging::LoggingType::Journal, + s->object.get(), + s, + canonical_name(), + etag, + s->object->get_size(), + this, y, false, false); + if (op_ret < 0) { + return; + } + bufferlist bl; obj_legal_hold.encode(bl); //if instance is empty, we should modify the latest object diff --git a/src/rgw/rgw_rest.h b/src/rgw/rgw_rest.h index a302257e428bc..d0c1e49c065a1 100644 --- a/src/rgw/rgw_rest.h +++ b/src/rgw/rgw_rest.h @@ -580,12 +580,14 @@ class RGWPutObjRetention_ObjStore : public RGWPutObjRetention { public: RGWPutObjRetention_ObjStore() = default; ~RGWPutObjRetention_ObjStore() override = default; + virtual std::string canonical_name() const override { return fmt::format("REST.{}.RETENTION", s->info.method); } }; class RGWGetObjRetention_ObjStore : public RGWGetObjRetention { public: RGWGetObjRetention_ObjStore() = default; ~RGWGetObjRetention_ObjStore() = default; + virtual std::string canonical_name() const override { return fmt::format("REST.{}.RETENTION", s->info.method); } }; class RGWPutObjLegalHold_ObjStore : public RGWPutObjLegalHold { @@ -593,11 +595,13 @@ public: RGWPutObjLegalHold_ObjStore() = default; ~RGWPutObjLegalHold_ObjStore() override = default; int get_params(optional_yield y) override; + virtual std::string canonical_name() const override { return fmt::format("REST.{}.LEGAL_HOLD", s->info.method); } }; class RGWGetObjLegalHold_ObjStore : public RGWGetObjLegalHold { public: RGWGetObjLegalHold_ObjStore() = default; + virtual std::string canonical_name() const override { return fmt::format("REST.{}.LEGAL_HOLD", s->info.method); } ~RGWGetObjLegalHold_ObjStore() = default; }; diff --git a/src/rgw/rgw_rest_s3.h b/src/rgw/rgw_rest_s3.h index 62445b032707f..a1a1132ca7faf 100644 --- a/src/rgw/rgw_rest_s3.h +++ b/src/rgw/rgw_rest_s3.h @@ -83,6 +83,7 @@ class RGWDeleteObjTags_ObjStore_S3 : public RGWDeleteObjTags public: ~RGWDeleteObjTags_ObjStore_S3() override {} void send_response() override; + virtual std::string canonical_name() const override { return fmt::format("REST.{}.OBJECT_TAGGING", s->info.method); } }; class RGWGetBucketTags_ObjStore_S3 : public RGWGetBucketTags_ObjStore -- 2.39.5