RGW/logging: add rest API to flush logging bucket

Signed-off-by: Ali Masarwa <amasarwa@redhat.com>

diff --git a/doc/radosgw/bucket_logging.rst b/doc/radosgw/bucket_logging.rst
index d96ffbe2758a2eafe9a208de093f7553f5ab9bf4..b156e8f6b2510935bf6092f5a0e10ec61b89b0ea 100644 (file)
--- a/doc/radosgw/bucket_logging.rst
+++ b/doc/radosgw/bucket_logging.rst
@@ -27,6 +27,8 @@ This time (in seconds) could be set per source bucket via a Ceph extension to th
 or globally via the `rgw_bucket_logging_obj_roll_time` configuration option. If not set, the default time is 5 minutes.
 Adding a log object to the log bucket is done "lazily", meaning, that if no more records are written to the object, it may
 remain outside of the log bucket even after the configured time has passed.
+To counter that, you can flush all logging objects on a given source bucket to log them,
+regardless if enough time passed or if no more records are written to the object.
 
 Standard
 ````````

diff --git a/doc/radosgw/s3/bucketops.rst b/doc/radosgw/s3/bucketops.rst
index 2e5cc48646dee9913caba9fe989c066769f5588f..d3f81b6975233418f1999eb9609f12592ff93180 100644 (file)
--- a/doc/radosgw/s3/bucketops.rst
+++ b/doc/radosgw/s3/bucketops.rst
@@ -894,3 +894,27 @@ HTTP Response
 | ``404``       | NoSuchBucket          | The bucket does not exist                                |
 +---------------+-----------------------+----------------------------------------------------------+
 
+Flush Bucket Logging
+--------------------
+
+Flushes all logging objects for a given source bucket (logging bucket are written lazily).
+
+Syntax
+~~~~~~
+
+::
+
+    POST /{bucket}?logging HTTP/1.1
+
+
+HTTP Response
+~~~~~~~~~~~~~
+
++---------------+-----------------------+----------------------------------------------------------+
+| HTTP Status   | Status Code           | Description                                              |
++===============+=======================+==========================================================+
+| ``201``       | Created               | Flushed all logging objects successfully                 |
++---------------+-----------------------+----------------------------------------------------------+
+| ``404``       | NoSuchBucket          | The bucket does not exist                                |
++---------------+-----------------------+----------------------------------------------------------+
+

diff --git a/examples/rgw/boto3/post_bucket_logging.py b/examples/rgw/boto3/post_bucket_logging.py
new file mode 100644 (file)
index 0000000..130fc53
--- /dev/null
+++ b/examples/rgw/boto3/post_bucket_logging.py
@@ -0,0 +1,23 @@
+import boto3
+import sys
+
+
+if len(sys.argv) == 2:
+    # bucket name as first argument
+    bucketname = sys.argv[1]
+else:
+    print('Usage: ' + sys.argv[0] + ' <bucket>')
+    sys.exit(1)
+
+# endpoint and keys from vstart
+endpoint = 'http://127.0.0.1:8000/'+bucketname
+access_key='0555b35654ad1656d804'
+secret_key='h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q=='
+
+client = boto3.client('s3',
+                      endpoint_url=endpoint,
+                      aws_access_key_id=access_key,
+                      aws_secret_access_key=secret_key)
+
+# flushing the logs for bucket logging
+print(client.post_bucket_logging(Bucket=bucketname))

diff --git a/examples/rgw/boto3/service-2.sdk-extras.json b/examples/rgw/boto3/service-2.sdk-extras.json
index 5c22ee9f2487b6ae76d2376fef2c806c2adff889..eeb71ae6b878ce3c76281be59567067533b1bd91 100644 (file)
--- a/examples/rgw/boto3/service-2.sdk-extras.json
+++ b/examples/rgw/boto3/service-2.sdk-extras.json
@@ -13,6 +13,17 @@
             "documentationUrl":"https://docs.ceph.com/docs/master/radosgw/s3/bucketops/#delete-notification",
             "documentation":"<p>Deletes the notification configuration from the bucket.</p>"
         },
+        "PostBucketLogging":{
+            "name":"PostBucketLogging",
+            "http":{
+                "method":"POST",
+                "requestUri":"/{Bucket}?logging",
+                "responseCode":201
+            },
+            "input":{"shape":"PostBucketLoggingRequest"},
+            "documentationUrl":"https://docs.ceph.com/docs/master/radosgw/s3/bucketops/#post-bucket-logging",
+            "documentation":"<p>Flushes the logging objects of the buckets.</p>"
+        },
         "GetUsageStats":{
             "name":"GetUsageStats",
             "http":{
@@ -146,6 +157,18 @@
                 }
             }
         },
+        "PostBucketLoggingRequest":{
+            "type":"structure",
+            "required":["Bucket"],
+            "members":{
+                "Bucket":{
+                    "shape":"BucketName",
+                    "documentation":"<p>Name of the bucket to flush its logging objects.</p>",
+                    "location":"uri",
+                    "locationName":"Bucket"
+                }
+            }
+        },
         "FilterRule":{
             "type":"structure",
             "members":{

diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index 8fd930c907ee6cfe78ce623298b319a57244b0b6..6b1ae38a62d20f33d5d83f7b313170c61f0afa26 100644 (file)
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -7599,7 +7599,7 @@ int main(int argc, const char **argv)
         << "' to target bucket '" << configuration.target_bucket << "'" << std::endl;
       return -ret;
     }
-    cerr << "flushed pending logging object '" << obj_name
+    cout << "flushed pending logging object '" << obj_name
       << "' to target bucket '" << configuration.target_bucket << "'" << std::endl;
     return 0;
   }

diff --git a/src/rgw/rgw_iam_policy.cc b/src/rgw/rgw_iam_policy.cc
index ce76ed4c3c3e80603fb1b64f8c834e22c6563826..2a5c9cd313e8222c405df961b9831ff0d36057b5 100644 (file)
--- a/src/rgw/rgw_iam_policy.cc
+++ b/src/rgw/rgw_iam_policy.cc
@@ -113,6 +113,7 @@ static const actpair actpairs[] =
  { "s3:PutBucketCORS", s3PutBucketCORS },
  { "s3:PutBucketEncryption", s3PutBucketEncryption },
  { "s3:PutBucketLogging", s3PutBucketLogging },
+ { "s3:PostBucketLogging", s3PostBucketLogging },
  { "s3:PutBucketNotification", s3PutBucketNotification },
  { "s3:PutBucketOwnershipControls", s3PutBucketOwnershipControls },
  { "s3:PutBucketPolicy", s3PutBucketPolicy },
@@ -1406,6 +1407,9 @@ const char* action_bit_string(uint64_t action) {
   case s3PutBucketLogging:
     return "s3:PutBucketLogging";
 
+    case s3PostBucketLogging:
+      return "s3:PostBucketLogging";
+
   case s3GetBucketTagging:
     return "s3:GetBucketTagging";
 

diff --git a/src/rgw/rgw_iam_policy.h b/src/rgw/rgw_iam_policy.h
index 1494cbf0b81c38112803a146f01d593c5a2546cb..0476926143f16893827fae31e383bde0aabca35b 100644 (file)
--- a/src/rgw/rgw_iam_policy.h
+++ b/src/rgw/rgw_iam_policy.h
@@ -81,6 +81,7 @@ enum {
   s3PutBucketNotification,
   s3GetBucketLogging,
   s3PutBucketLogging,
+  s3PostBucketLogging,
   s3GetBucketTagging,
   s3PutBucketTagging,
   s3GetBucketWebsite,
@@ -298,6 +299,7 @@ inline int op_to_perm(std::uint64_t op) {
   case s3PutBucketCORS:
   case s3PutBucketEncryption:
   case s3PutBucketLogging:
+  case s3PostBucketLogging:
   case s3PutBucketNotification:
   case s3PutBucketPolicy:
   case s3PutBucketRequestPayment:

diff --git a/src/rgw/rgw_op_type.h b/src/rgw/rgw_op_type.h
index a36274add407591a26e122061f639c21ae8ebf7e..49faea6403d48c4ca609e8c4ac8723c92c8ca51d 100644 (file)
--- a/src/rgw/rgw_op_type.h
+++ b/src/rgw/rgw_op_type.h
@@ -117,6 +117,7 @@ enum RGWOpType {
   RGW_OP_DETACH_GROUP_POLICY,
   RGW_OP_LIST_ATTACHED_GROUP_POLICIES,
   RGW_OP_PUT_BUCKET_LOGGING,
+  RGW_OP_POST_BUCKET_LOGGING,
   /* rgw specific */
   RGW_OP_ADMIN_SET_METADATA,
   RGW_OP_GET_OBJ_LAYOUT,

diff --git a/src/rgw/rgw_rest_bucket_logging.cc b/src/rgw/rgw_rest_bucket_logging.cc
index 8d17114d80476c95919934fb32c49d8f2643157c..ed12ce855a92ea82d3c2f214ed74e3cde2c49dba 100644 (file)
--- a/src/rgw/rgw_rest_bucket_logging.cc
+++ b/src/rgw/rgw_rest_bucket_logging.cc
@@ -209,6 +209,84 @@ class RGWPutBucketLoggingOp : public RGWDefaultResponseOp {
   }
 };
 
+// Post /<bucket name>/?logging
+// actual configuration is XML encoded in the body of the message
+class RGWPostBucketLoggingOp : public RGWDefaultResponseOp {
+  int verify_permission(optional_yield y) override {
+    auto [has_s3_existing_tag, has_s3_resource_tag] = rgw_check_policy_condition(this, s, false);
+    if (has_s3_resource_tag)
+      rgw_iam_add_buckettags(this, s);
+
+    if (!verify_bucket_permission(this, s, rgw::IAM::s3PostBucketLogging)) {
+      return -EACCES;
+    }
+
+    return 0;
+  }
+
+  const char* name() const override { return "post_bucket_logging"; }
+  RGWOpType get_type() override { return RGW_OP_POST_BUCKET_LOGGING; }
+  uint32_t op_mask() override { return RGW_OP_TYPE_WRITE; }
+
+  void execute(optional_yield y) override {
+    op_ret = verify_bucket_logging_params(this, s);
+    if (op_ret < 0) {
+      return;
+    }
+
+    std::unique_ptr<rgw::sal::Bucket> bucket;
+    op_ret = driver->load_bucket(this, rgw_bucket(s->bucket_tenant, s->bucket_name),
+                                 &bucket, y);
+    if (op_ret < 0) {
+      ldpp_dout(this, 1) << "ERROR: failed to get bucket '" << s->bucket_name << "', ret = " << op_ret << dendl;
+      return;
+    }
+    const auto& bucket_attrs = bucket->get_attrs();
+    auto iter = bucket_attrs.find(RGW_ATTR_BUCKET_LOGGING);
+    if (iter == bucket_attrs.end()) {
+      ldpp_dout(this, 1) << "WARNING: no logging configured on bucket" << dendl;
+      return;
+    }
+    rgw::bucketlogging::configuration configuration;
+    try {
+      configuration.enabled = true;
+      decode(configuration, iter->second);
+    } catch (buffer::error& err) {
+      ldpp_dout(this, 1) << "ERROR: failed to decode logging attribute '" << RGW_ATTR_BUCKET_LOGGING
+                         << "'. error: " << err.what() << dendl;
+      op_ret = -EINVAL;
+      return;
+    }
+
+    std::unique_ptr<rgw::sal::Bucket> target_bucket;
+    op_ret = driver->load_bucket(this, rgw_bucket(s->bucket_tenant, configuration.target_bucket),
+                                 &target_bucket, y);
+    if (op_ret < 0) {
+      ldpp_dout(this, 1) << "ERROR: failed to get target bucket '" << configuration.target_bucket << "', ret = " << op_ret << dendl;
+      return;
+    }
+    std::string obj_name;
+    RGWObjVersionTracker objv_tracker;
+    op_ret = target_bucket->get_logging_object_name(obj_name, configuration.target_prefix, null_yield, this, &objv_tracker);
+    if (op_ret < 0) {
+      ldpp_dout(this, 1) << "ERROR: failed to get pending logging object name from target bucket '" << configuration.target_bucket << "'" << dendl;
+      return;
+    }
+    op_ret = rgw::bucketlogging::rollover_logging_object(configuration, target_bucket, obj_name, this, null_yield, true, &objv_tracker);
+    if (op_ret < 0) {
+      ldpp_dout(this, 1) << "ERROR: failed to flush pending logging object '" << obj_name
+               << "' to target bucket '" << configuration.target_bucket << "'" << dendl;
+      return;
+    }
+    ldpp_dout(this, 20) << "flushed pending logging object '" << obj_name
+                << "' to target bucket '" << configuration.target_bucket << "'" << dendl;
+  }
+};
+
+RGWOp* RGWHandler_REST_BucketLogging_S3::create_post_op() {
+  return new RGWPostBucketLoggingOp();
+}
+
 RGWOp* RGWHandler_REST_BucketLogging_S3::create_put_op() {
   return new RGWPutBucketLoggingOp();
 }

diff --git a/src/rgw/rgw_rest_bucket_logging.h b/src/rgw/rgw_rest_bucket_logging.h
index ca2220084bb2c7b85bcd661d384fd5569237b6cf..0b31d88dad82341080ca86725e4ede62003a4e8b 100644 (file)
--- a/src/rgw/rgw_rest_bucket_logging.h
+++ b/src/rgw/rgw_rest_bucket_logging.h
@@ -14,5 +14,6 @@ public:
   virtual ~RGWHandler_REST_BucketLogging_S3() = default;
   static RGWOp* create_get_op();
   static RGWOp* create_put_op();
+  static RGWOp* create_post_op();
 };
 

diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc
index ecc6b7f3a94e6076af3f76614f565c4cdfd874fa..416acfb5db6d2020d6eeb90950f16b4a19403231 100644 (file)
--- a/src/rgw/rgw_rest_s3.cc
+++ b/src/rgw/rgw_rest_s3.cc
@@ -4912,6 +4912,10 @@ RGWOp *RGWHandler_REST_Bucket_S3::op_post()
     return new RGWDeleteMultiObj_ObjStore_S3;
   }
 
+  if (s->info.args.exists("logging")) {
+    return RGWHandler_REST_BucketLogging_S3::create_post_op();
+  }
+
   if (s->info.args.exists("mdsearch")) {
     if (!s->cct->_conf->rgw_enable_mdsearch) {
       return NULL;
@@ -6069,7 +6073,8 @@ AWSGeneralAbstractor::get_auth_data_v4(const req_state* const s,
         case RGW_OP_GET_BUCKET_PUBLIC_ACCESS_BLOCK:
         case RGW_OP_DELETE_BUCKET_PUBLIC_ACCESS_BLOCK:
        case RGW_OP_GET_OBJ://s3select its post-method(payload contain the query) , the request is get-object
-        case RGW_OP_PUT_BUCKET_LOGGING: 
+        case RGW_OP_PUT_BUCKET_LOGGING:
+        case RGW_OP_POST_BUCKET_LOGGING:
         case RGW_OP_GET_BUCKET_LOGGING: 
           break;
         default: