Merge pull request #57659 from mkogan1/wip-graceful-shutdown

rgw: delay the RGW process exit until all actice requests have completed

diff --cc src/common/options/rgw.yaml.in
Simple merge

diff --cc src/rgw/rgw_asio_frontend.cc
index f80e40c70b621967b0e8547ba6728d5a99861eff,855b9650f0ca5fa87aa97139909a7eb1de1cf09e..30e1e77fd151ea45f494e1a0d1fdc49dd4d061c8
--- 1/src/rgw/rgw_asio_frontend.cc
--- 2/src/rgw/rgw_asio_frontend.cc
+++ b/src/rgw/rgw_asio_frontend.cc
@@@ -1153,9 -1085,21 +1153,23 @@@ void AsioFrontend::stop(
    // close all listeners
    for (auto& listener : listeners) {
      listener.acceptor.close(ec);
 +    // signal cancellation of accept()
 +    listener.signal.emit(boost::asio::cancellation_type::terminal);
    }
+ 
+   const bool graceful_stop{ g_ceph_context->_conf->rgw_graceful_stop };
+   if (graceful_stop) {
+     ldout(ctx(), 4) << "frontend pausing and waiting for outstanding requests to complete..." << dendl;
+     pause_mutex.lock(ec);
+     if (ec) {
+       ldout(ctx(), 1) << "frontend failed to pause: " << ec.message() << dendl;
+     } else {
+       ldout(ctx(), 4) << "frontend paused" << dendl;
+     }
+     ldout(ctx(), 4) << "frontend outstanding requests have completed" << dendl;
+     pause_mutex.unlock();
+   }
+ 
    // close all connections
    connections.close(ec);
    pause_mutex.cancel();