rgw: add configurable max aio limit for multi-object delete requests

Adds the 'rgw_multi_obj_del_max_aio' configuration parameter to
bound aio on multi-object delete requests.

Signed-off-by: Cory Snyder <csnyder@iland.com>
(cherry picked from commit 63e41fb8a58717d404b436096079d4fc09f30682)

Conflicts:
	src/rgw/rgw_op.cc

diff --git a/src/common/options/rgw.yaml.in b/src/common/options/rgw.yaml.in
index ac140fce7976945d7b5764342547707c6a411357..414e82a9e67e46df25896f498a5ba93441efef87 100644 (file)
--- a/src/common/options/rgw.yaml.in
+++ b/src/common/options/rgw.yaml.in
@@ -175,6 +175,14 @@ options:
   services:
   - rgw
   with_legacy: true
+- name: rgw_multi_obj_del_max_aio
+  type: uint
+  level: advanced
+  desc: Max number of concurrent RADOS requests per multi-object delete request.
+  default: 128
+  services:
+  - rgw
+  with_legacy: true
 # whether or not the quota/gc threads should be started
 - name: rgw_enable_quota_threads
   type: bool

diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index 606ee2e2a1ed29f3f34ccec69dee18ba81a14acf..c5230d67ae6585caff5507525e435b77481ce569 100644 (file)
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -6784,27 +6784,21 @@ void RGWDeleteMultiObj::write_ops_log_entry(rgw_log_entry& entry) const {
   entry.delete_multi_obj_meta.objects = std::move(ops_log_entries);
 }
 
-void RGWDeleteMultiObj::wait_flush(optional_yield y, size_t n)
+void RGWDeleteMultiObj::wait_flush(optional_yield y, std::function<bool()> predicate)
 {
   if (y) {
-    if (ops_log_entries.size() == n) {
-      rgw_flush_formatter(s, s->formatter);
-      return;
-    }
     auto yc = y.get_yield_context();
-    for (;;) {
+    while (!predicate()) {
       boost::system::error_code error;
       formatter_flush_cond->async_wait(yc[error]);
       rgw_flush_formatter(s, s->formatter);
-      if (ops_log_entries.size() == n) {
-        break;
-      }
     }
   }
 }
 
 void RGWDeleteMultiObj::handle_individual_object(const rgw_obj_key *o, optional_yield y)
 {
+  RGWObjectCtx *obj_ctx = static_cast<RGWObjectCtx *>(s->obj_ctx);
   std::string version_id;
   std::unique_ptr<rgw::sal::Object> obj = bucket->get_object(*o);
   if (s->iam_policy || ! s->iam_user_policies.empty() || !s->session_policies.empty()) {
@@ -6947,7 +6941,8 @@ void RGWDeleteMultiObj::execute(optional_yield y)
   RGWMultiDelDelete *multi_delete;
   vector<rgw_obj_key>::iterator iter;
   RGWMultiDelXMLParser parser;
-  RGWObjectCtx *obj_ctx = static_cast<RGWObjectCtx *>(s->obj_ctx);
+  uint32_t aio_count = 0;
+  uint32_t max_aio = s->cct->_conf->rgw_multi_obj_del_max_aio;
   char* buf;
   if (y) {
     formatter_flush_cond = std::make_unique<boost::asio::deadline_timer>(y.get_io_context());  
@@ -7013,16 +7008,22 @@ void RGWDeleteMultiObj::execute(optional_yield y)
         iter != multi_delete->objects.end();
         ++iter) {
     rgw_obj_key* obj_key = &*iter;
-    if (y) {
-      spawn::spawn(y.get_yield_context(), [this, &y, obj_key] (yield_context yield) {
+    if (y && max_aio > 1) {
+      wait_flush(y, [&aio_count, max_aio] {
+        return aio_count < max_aio;
+      });
+      aio_count++;
+      spawn::spawn(y.get_yield_context(), [this, &y, &aio_count, obj_key] (yield_context yield) {
         handle_individual_object(obj_key, optional_yield { y.get_io_context(), yield }); 
+        aio_count--;
       }); 
     } else {
       handle_individual_object(obj_key, y);
     }
   }
-
-  wait_flush(y, multi_delete->objects.size());
+  wait_flush(y, [this, n=multi_delete->objects.size()] {
+    return n == ops_log_entries.size();
+  });
 
   /*  set the return code to zero, errors at this point will be
   dumped to the response */

diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h
index 030a77b91e2ee6e7244587a40a1c8a7efcc23f20..be89e6365904f5672222626d41767c5c01cd0cac 100644 (file)
--- a/src/rgw/rgw_op.h
+++ b/src/rgw/rgw_op.h
@@ -2039,7 +2039,7 @@ class RGWDeleteMultiObj : public RGWOp {
    * and saved on the req_state vs. one that is passed on the stack.
    * This is a no-op in the case where we're not executing as a coroutine.
    */
-  void wait_flush(optional_yield y, size_t n);
+  void wait_flush(optional_yield y, std::function<bool()> predicate);
 
 protected:
   std::vector<delete_multi_obj_entry> ops_log_entries;