From 9a9e14db35048c4327ea8aeae54fe574ef76e0e9 Mon Sep 17 00:00:00 2001 From: Ali Masarwa Date: Mon, 11 Nov 2024 15:16:30 +0200 Subject: [PATCH] RGW/logging: add rest API to flush logging bucket Signed-off-by: Ali Masarwa --- doc/radosgw/bucket_logging.rst | 2 + doc/radosgw/s3/bucketops.rst | 24 ++++++ examples/rgw/boto3/post_bucket_logging.py | 23 ++++++ examples/rgw/boto3/service-2.sdk-extras.json | 23 ++++++ src/rgw/rgw_admin.cc | 2 +- src/rgw/rgw_iam_policy.cc | 4 + src/rgw/rgw_iam_policy.h | 2 + src/rgw/rgw_op_type.h | 1 + src/rgw/rgw_rest_bucket_logging.cc | 78 ++++++++++++++++++++ src/rgw/rgw_rest_bucket_logging.h | 1 + src/rgw/rgw_rest_s3.cc | 7 +- 11 files changed, 165 insertions(+), 2 deletions(-) create mode 100644 examples/rgw/boto3/post_bucket_logging.py diff --git a/doc/radosgw/bucket_logging.rst b/doc/radosgw/bucket_logging.rst index d96ffbe2758a2..b156e8f6b2510 100644 --- a/doc/radosgw/bucket_logging.rst +++ b/doc/radosgw/bucket_logging.rst @@ -27,6 +27,8 @@ This time (in seconds) could be set per source bucket via a Ceph extension to th or globally via the `rgw_bucket_logging_obj_roll_time` configuration option. If not set, the default time is 5 minutes. Adding a log object to the log bucket is done "lazily", meaning, that if no more records are written to the object, it may remain outside of the log bucket even after the configured time has passed. +To counter that, you can flush all logging objects on a given source bucket to log them, +regardless if enough time passed or if no more records are written to the object. Standard ```````` diff --git a/doc/radosgw/s3/bucketops.rst b/doc/radosgw/s3/bucketops.rst index 2e5cc48646dee..d3f81b6975233 100644 --- a/doc/radosgw/s3/bucketops.rst +++ b/doc/radosgw/s3/bucketops.rst @@ -894,3 +894,27 @@ HTTP Response | ``404`` | NoSuchBucket | The bucket does not exist | +---------------+-----------------------+----------------------------------------------------------+ +Flush Bucket Logging +-------------------- + +Flushes all logging objects for a given source bucket (logging bucket are written lazily). + +Syntax +~~~~~~ + +:: + + POST /{bucket}?logging HTTP/1.1 + + +HTTP Response +~~~~~~~~~~~~~ + ++---------------+-----------------------+----------------------------------------------------------+ +| HTTP Status | Status Code | Description | ++===============+=======================+==========================================================+ +| ``201`` | Created | Flushed all logging objects successfully | ++---------------+-----------------------+----------------------------------------------------------+ +| ``404`` | NoSuchBucket | The bucket does not exist | ++---------------+-----------------------+----------------------------------------------------------+ + diff --git a/examples/rgw/boto3/post_bucket_logging.py b/examples/rgw/boto3/post_bucket_logging.py new file mode 100644 index 0000000000000..130fc53b50a12 --- /dev/null +++ b/examples/rgw/boto3/post_bucket_logging.py @@ -0,0 +1,23 @@ +import boto3 +import sys + + +if len(sys.argv) == 2: + # bucket name as first argument + bucketname = sys.argv[1] +else: + print('Usage: ' + sys.argv[0] + ' ') + sys.exit(1) + +# endpoint and keys from vstart +endpoint = 'http://127.0.0.1:8000/'+bucketname +access_key='0555b35654ad1656d804' +secret_key='h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q==' + +client = boto3.client('s3', + endpoint_url=endpoint, + aws_access_key_id=access_key, + aws_secret_access_key=secret_key) + +# flushing the logs for bucket logging +print(client.post_bucket_logging(Bucket=bucketname)) diff --git a/examples/rgw/boto3/service-2.sdk-extras.json b/examples/rgw/boto3/service-2.sdk-extras.json index 5c22ee9f2487b..eeb71ae6b878c 100644 --- a/examples/rgw/boto3/service-2.sdk-extras.json +++ b/examples/rgw/boto3/service-2.sdk-extras.json @@ -13,6 +13,17 @@ "documentationUrl":"https://docs.ceph.com/docs/master/radosgw/s3/bucketops/#delete-notification", "documentation":"

Deletes the notification configuration from the bucket.

" }, + "PostBucketLogging":{ + "name":"PostBucketLogging", + "http":{ + "method":"POST", + "requestUri":"/{Bucket}?logging", + "responseCode":201 + }, + "input":{"shape":"PostBucketLoggingRequest"}, + "documentationUrl":"https://docs.ceph.com/docs/master/radosgw/s3/bucketops/#post-bucket-logging", + "documentation":"

Flushes the logging objects of the buckets.

" + }, "GetUsageStats":{ "name":"GetUsageStats", "http":{ @@ -146,6 +157,18 @@ } } }, + "PostBucketLoggingRequest":{ + "type":"structure", + "required":["Bucket"], + "members":{ + "Bucket":{ + "shape":"BucketName", + "documentation":"

Name of the bucket to flush its logging objects.

", + "location":"uri", + "locationName":"Bucket" + } + } + }, "FilterRule":{ "type":"structure", "members":{ diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc index 8fd930c907ee6..6b1ae38a62d20 100644 --- a/src/rgw/rgw_admin.cc +++ b/src/rgw/rgw_admin.cc @@ -7599,7 +7599,7 @@ int main(int argc, const char **argv) << "' to target bucket '" << configuration.target_bucket << "'" << std::endl; return -ret; } - cerr << "flushed pending logging object '" << obj_name + cout << "flushed pending logging object '" << obj_name << "' to target bucket '" << configuration.target_bucket << "'" << std::endl; return 0; } diff --git a/src/rgw/rgw_iam_policy.cc b/src/rgw/rgw_iam_policy.cc index ce76ed4c3c3e8..2a5c9cd313e82 100644 --- a/src/rgw/rgw_iam_policy.cc +++ b/src/rgw/rgw_iam_policy.cc @@ -113,6 +113,7 @@ static const actpair actpairs[] = { "s3:PutBucketCORS", s3PutBucketCORS }, { "s3:PutBucketEncryption", s3PutBucketEncryption }, { "s3:PutBucketLogging", s3PutBucketLogging }, + { "s3:PostBucketLogging", s3PostBucketLogging }, { "s3:PutBucketNotification", s3PutBucketNotification }, { "s3:PutBucketOwnershipControls", s3PutBucketOwnershipControls }, { "s3:PutBucketPolicy", s3PutBucketPolicy }, @@ -1406,6 +1407,9 @@ const char* action_bit_string(uint64_t action) { case s3PutBucketLogging: return "s3:PutBucketLogging"; + case s3PostBucketLogging: + return "s3:PostBucketLogging"; + case s3GetBucketTagging: return "s3:GetBucketTagging"; diff --git a/src/rgw/rgw_iam_policy.h b/src/rgw/rgw_iam_policy.h index 1494cbf0b81c3..0476926143f16 100644 --- a/src/rgw/rgw_iam_policy.h +++ b/src/rgw/rgw_iam_policy.h @@ -81,6 +81,7 @@ enum { s3PutBucketNotification, s3GetBucketLogging, s3PutBucketLogging, + s3PostBucketLogging, s3GetBucketTagging, s3PutBucketTagging, s3GetBucketWebsite, @@ -298,6 +299,7 @@ inline int op_to_perm(std::uint64_t op) { case s3PutBucketCORS: case s3PutBucketEncryption: case s3PutBucketLogging: + case s3PostBucketLogging: case s3PutBucketNotification: case s3PutBucketPolicy: case s3PutBucketRequestPayment: diff --git a/src/rgw/rgw_op_type.h b/src/rgw/rgw_op_type.h index a36274add4075..49faea6403d48 100644 --- a/src/rgw/rgw_op_type.h +++ b/src/rgw/rgw_op_type.h @@ -117,6 +117,7 @@ enum RGWOpType { RGW_OP_DETACH_GROUP_POLICY, RGW_OP_LIST_ATTACHED_GROUP_POLICIES, RGW_OP_PUT_BUCKET_LOGGING, + RGW_OP_POST_BUCKET_LOGGING, /* rgw specific */ RGW_OP_ADMIN_SET_METADATA, RGW_OP_GET_OBJ_LAYOUT, diff --git a/src/rgw/rgw_rest_bucket_logging.cc b/src/rgw/rgw_rest_bucket_logging.cc index 8d17114d80476..ed12ce855a92e 100644 --- a/src/rgw/rgw_rest_bucket_logging.cc +++ b/src/rgw/rgw_rest_bucket_logging.cc @@ -209,6 +209,84 @@ class RGWPutBucketLoggingOp : public RGWDefaultResponseOp { } }; +// Post //?logging +// actual configuration is XML encoded in the body of the message +class RGWPostBucketLoggingOp : public RGWDefaultResponseOp { + int verify_permission(optional_yield y) override { + auto [has_s3_existing_tag, has_s3_resource_tag] = rgw_check_policy_condition(this, s, false); + if (has_s3_resource_tag) + rgw_iam_add_buckettags(this, s); + + if (!verify_bucket_permission(this, s, rgw::IAM::s3PostBucketLogging)) { + return -EACCES; + } + + return 0; + } + + const char* name() const override { return "post_bucket_logging"; } + RGWOpType get_type() override { return RGW_OP_POST_BUCKET_LOGGING; } + uint32_t op_mask() override { return RGW_OP_TYPE_WRITE; } + + void execute(optional_yield y) override { + op_ret = verify_bucket_logging_params(this, s); + if (op_ret < 0) { + return; + } + + std::unique_ptr bucket; + op_ret = driver->load_bucket(this, rgw_bucket(s->bucket_tenant, s->bucket_name), + &bucket, y); + if (op_ret < 0) { + ldpp_dout(this, 1) << "ERROR: failed to get bucket '" << s->bucket_name << "', ret = " << op_ret << dendl; + return; + } + const auto& bucket_attrs = bucket->get_attrs(); + auto iter = bucket_attrs.find(RGW_ATTR_BUCKET_LOGGING); + if (iter == bucket_attrs.end()) { + ldpp_dout(this, 1) << "WARNING: no logging configured on bucket" << dendl; + return; + } + rgw::bucketlogging::configuration configuration; + try { + configuration.enabled = true; + decode(configuration, iter->second); + } catch (buffer::error& err) { + ldpp_dout(this, 1) << "ERROR: failed to decode logging attribute '" << RGW_ATTR_BUCKET_LOGGING + << "'. error: " << err.what() << dendl; + op_ret = -EINVAL; + return; + } + + std::unique_ptr target_bucket; + op_ret = driver->load_bucket(this, rgw_bucket(s->bucket_tenant, configuration.target_bucket), + &target_bucket, y); + if (op_ret < 0) { + ldpp_dout(this, 1) << "ERROR: failed to get target bucket '" << configuration.target_bucket << "', ret = " << op_ret << dendl; + return; + } + std::string obj_name; + RGWObjVersionTracker objv_tracker; + op_ret = target_bucket->get_logging_object_name(obj_name, configuration.target_prefix, null_yield, this, &objv_tracker); + if (op_ret < 0) { + ldpp_dout(this, 1) << "ERROR: failed to get pending logging object name from target bucket '" << configuration.target_bucket << "'" << dendl; + return; + } + op_ret = rgw::bucketlogging::rollover_logging_object(configuration, target_bucket, obj_name, this, null_yield, true, &objv_tracker); + if (op_ret < 0) { + ldpp_dout(this, 1) << "ERROR: failed to flush pending logging object '" << obj_name + << "' to target bucket '" << configuration.target_bucket << "'" << dendl; + return; + } + ldpp_dout(this, 20) << "flushed pending logging object '" << obj_name + << "' to target bucket '" << configuration.target_bucket << "'" << dendl; + } +}; + +RGWOp* RGWHandler_REST_BucketLogging_S3::create_post_op() { + return new RGWPostBucketLoggingOp(); +} + RGWOp* RGWHandler_REST_BucketLogging_S3::create_put_op() { return new RGWPutBucketLoggingOp(); } diff --git a/src/rgw/rgw_rest_bucket_logging.h b/src/rgw/rgw_rest_bucket_logging.h index ca2220084bb2c..0b31d88dad823 100644 --- a/src/rgw/rgw_rest_bucket_logging.h +++ b/src/rgw/rgw_rest_bucket_logging.h @@ -14,5 +14,6 @@ public: virtual ~RGWHandler_REST_BucketLogging_S3() = default; static RGWOp* create_get_op(); static RGWOp* create_put_op(); + static RGWOp* create_post_op(); }; diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index ecc6b7f3a94e6..416acfb5db6d2 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -4912,6 +4912,10 @@ RGWOp *RGWHandler_REST_Bucket_S3::op_post() return new RGWDeleteMultiObj_ObjStore_S3; } + if (s->info.args.exists("logging")) { + return RGWHandler_REST_BucketLogging_S3::create_post_op(); + } + if (s->info.args.exists("mdsearch")) { if (!s->cct->_conf->rgw_enable_mdsearch) { return NULL; @@ -6069,7 +6073,8 @@ AWSGeneralAbstractor::get_auth_data_v4(const req_state* const s, case RGW_OP_GET_BUCKET_PUBLIC_ACCESS_BLOCK: case RGW_OP_DELETE_BUCKET_PUBLIC_ACCESS_BLOCK: case RGW_OP_GET_OBJ://s3select its post-method(payload contain the query) , the request is get-object - case RGW_OP_PUT_BUCKET_LOGGING: + case RGW_OP_PUT_BUCKET_LOGGING: + case RGW_OP_POST_BUCKET_LOGGING: case RGW_OP_GET_BUCKET_LOGGING: break; default: -- 2.39.5