rgw: add configurable max aio limit for multi-object delete requests

Adds the 'rgw_multi_obj_del_max_aio' configuration parameter to
bound aio on multi-object delete requests.

Signed-off-by: Cory Snyder <csnyder@iland.com>
(cherry picked from commit 63e41fb8a58717d404b436096079d4fc09f30682)

Conflicts:
	src/common/options/rgw.yaml.in
	src/rgw/rgw_op.cc

Cherry-pick notes:
- Conflicts due to change in configuration declaration post-pacfic

diff --git a/src/common/legacy_config_opts.h b/src/common/legacy_config_opts.h
index 1301b649eeed50d2c5777b57a866bd7798b10b42..d9a9872784d32527bd9b7b68438f38d1c71f2bc3 100644 (file)
--- a/src/common/legacy_config_opts.h
+++ b/src/common/legacy_config_opts.h
@@ -1264,6 +1264,11 @@ OPTION(rgw_override_bucket_index_max_shards, OPT_U32)
  */
 OPTION(rgw_bucket_index_max_aio, OPT_U32)
 
+/**
+ * Represents the maximum AIO pending requests for multi object delete requests.
+ */
+OPTION(rgw_multi_obj_del_max_aio, OPT_U32)
+
 /**
  * whether or not the quota/gc threads should be started
  */

diff --git a/src/common/options.cc b/src/common/options.cc
index b0239e24aa4d3ebe94f2f9700e09edbfd3ec1a01..9fe5a1a5d5f7f267a5e2af995b7de46e396e0152 100644 (file)
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -5868,6 +5868,10 @@ std::vector<Option> get_rgw_options() {
     .set_default(128)
     .set_description("Max number of concurrent RADOS requests when handling bucket shards."),
 
+    Option("rgw_multi_obj_del_max_aio", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
+    .set_default(128)
+    .set_description("Max number of concurrent RADOS requests per multi-object delete request."),
+
     Option("rgw_enable_quota_threads", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
     .set_default(true)
     .set_description("Enables the quota maintenance thread.")

diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index 134e785f74e39620297aeff22691f507a6c7727d..5e2b3eb6927406c54bfb39890495fa8423c5716e 100644 (file)
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -7035,21 +7035,14 @@ void RGWDeleteMultiObj::write_ops_log_entry(rgw_log_entry& entry) const {
   entry.delete_multi_obj_meta.objects = std::move(ops_log_entries);
 }
 
-void RGWDeleteMultiObj::wait_flush(optional_yield y, size_t n)
+void RGWDeleteMultiObj::wait_flush(optional_yield y, std::function<bool()> predicate)
 {
   if (y) {
-    if (ops_log_entries.size() == n) {
-      rgw_flush_formatter(s, s->formatter);
-      return;
-    }
     auto yc = y.get_yield_context();
-    for (;;) {
+    while (!predicate()) {
       boost::system::error_code error;
       formatter_flush_cond->async_wait(yc[error]);
       rgw_flush_formatter(s, s->formatter);
-      if (ops_log_entries.size() == n) {
-        break;
-      }
     }
   }
 }
@@ -7057,6 +7050,7 @@ void RGWDeleteMultiObj::wait_flush(optional_yield y, size_t n)
 void RGWDeleteMultiObj::handle_individual_object(const rgw_obj_key *o, optional_yield y)
 {
   std::string version_id;
+  RGWObjectCtx *obj_ctx = static_cast<RGWObjectCtx *>(s->obj_ctx);
   std::unique_ptr<rgw::sal::RGWObject> obj = bucket->get_object(*o);
   if (s->iam_policy || ! s->iam_user_policies.empty() || !s->session_policies.empty()) {
     auto identity_policy_res = eval_identity_or_session_policies(s->iam_user_policies, s->env,
@@ -7192,7 +7186,8 @@ void RGWDeleteMultiObj::execute(optional_yield y)
   RGWMultiDelDelete *multi_delete;
   vector<rgw_obj_key>::iterator iter;
   RGWMultiDelXMLParser parser;
-  RGWObjectCtx *obj_ctx = static_cast<RGWObjectCtx *>(s->obj_ctx);
+  uint32_t aio_count = 0;
+  uint32_t max_aio = s->cct->_conf->rgw_multi_obj_del_max_aio;
   char* buf;
   if (y) {
     formatter_flush_cond = std::make_unique<boost::asio::deadline_timer>(y.get_io_context());  
@@ -7258,16 +7253,22 @@ void RGWDeleteMultiObj::execute(optional_yield y)
         iter != multi_delete->objects.end();
         ++iter) {
     rgw_obj_key* obj_key = &*iter;
-    if (y) {
-      spawn::spawn(y.get_yield_context(), [this, &y, obj_key] (yield_context yield) {
+    if (y && max_aio > 1) {
+      wait_flush(y, [&aio_count, max_aio] {
+        return aio_count < max_aio;
+      });
+      aio_count++;
+      spawn::spawn(y.get_yield_context(), [this, &y, &aio_count, obj_key] (yield_context yield) {
         handle_individual_object(obj_key, optional_yield { y.get_io_context(), yield }); 
+        aio_count--;
       }); 
     } else {
       handle_individual_object(obj_key, y);
     }
   }
-
-  wait_flush(y, multi_delete->objects.size());
+  wait_flush(y, [this, n=multi_delete->objects.size()] {
+    return n == ops_log_entries.size();
+  });
 
   /*  set the return code to zero, errors at this point will be
   dumped to the response */

diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h
index 3e49148e437e3e0c48457201aa6048b24866b7c2..fbf97817164a77b64490bb08808be215cd7b6c9d 100644 (file)
--- a/src/rgw/rgw_op.h
+++ b/src/rgw/rgw_op.h
@@ -1924,7 +1924,7 @@ class RGWDeleteMultiObj : public RGWOp {
    * and saved on the req_state vs. one that is passed on the stack.
    * This is a no-op in the case where we're not executing as a coroutine.
    */
-  void wait_flush(optional_yield y, size_t n);
+  void wait_flush(optional_yield y, std::function<bool()> predicate);
 
 protected:
   std::vector<delete_multi_obj_entry> ops_log_entries;