From: Shilpa Jagannath <smanjara@redhat.com>
Date: Mon, 23 Mar 2026 16:38:34 +0000 (+0000)
Subject: rgw/multisite: use aio_operate for  RGWSimpleRadosLockCR/UnlockCR
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=573dc5f4aba16790a80989ba252ef17fbcb6e29a;p=ceph.git

rgw/multisite: use aio_operate for  RGWSimpleRadosLockCR/UnlockCR

RGWContinuousLeaseCR renews the sync lock every interval/2
by calling RGWSimpleRadosLockCR, which previously queued an
RGWAsyncLockSystemObj request via async_rados->queue().
At a large scale, when the async thread pool is fully saturated,
the cr thread can block, stalling lock renewal for extended periods
of time, eventually expiring.

Fix this by allowing RGWSimpleRadosLockCR and RGWSimpleRadosUnlockCR
to use aio_operate without having to queue behind other async requests

Signed-off-by: Shilpa Jagannath <smanjara@redhat.com>
---

diff --git a/src/rgw/driver/rados/rgw_cr_rados.cc b/src/rgw/driver/rados/rgw_cr_rados.cc
index fe44103a649d..c1a3f1c763c8 100644
--- a/src/rgw/driver/rados/rgw_cr_rados.cc
+++ b/src/rgw/driver/rados/rgw_cr_rados.cc
@@ -522,33 +522,43 @@ RGWSimpleRadosLockCR::RGWSimpleRadosLockCR(RGWAsyncRadosProcessor *_async_rados,
 			     lock_name(_lock_name),
 			     cookie(_cookie),
            duration(_duration),
-			     obj(_obj),
-			     req(nullptr)
+			     obj(_obj)
 {
   set_description() << "rados lock dest=" << obj << " lock=" << lock_name << " cookie=" << cookie << " duration=" << duration;
 }
 
 void RGWSimpleRadosLockCR::request_cleanup()
 {
-  if (req) {
-    req->finish();
-    req = NULL;
-  }
 }
 
 int RGWSimpleRadosLockCR::send_request(const DoutPrefixProvider *dpp)
 {
+  int r = store->getRados()->get_raw_obj_ref(dpp, obj, &ref);
+  if (r < 0) {
+    ldpp_dout(dpp, -1) << "ERROR: failed to get ref for (" << obj << ") ret=" << r << dendl;
+    return r;
+  }
+
   set_status() << "sending request";
-  req = new RGWAsyncLockSystemObj(this, stack->create_completion_notifier(),
-                                 store, NULL, obj, lock_name, cookie, duration);
-  async_rados->queue(req);
-  return 0;
+
+  rados::cls::lock::Lock l(lock_name);
+  utime_t dur(duration, 0);
+  l.set_duration(dur);
+  l.set_cookie(cookie);
+  l.set_may_renew(true);
+
+  librados::ObjectWriteOperation op;
+  l.lock_exclusive(&op);
+
+  cn = stack->create_completion_notifier();
+  return ref.ioctx.aio_operate(ref.obj.oid, cn->completion(), &op);
 }
 
 int RGWSimpleRadosLockCR::request_complete()
 {
-  set_status() << "request complete; ret=" << req->get_ret_status();
-  return req->get_ret_status();
+  int r = cn->completion()->get_return_value();
+  set_status() << "request complete; ret=" << r;
+  return r;
 }
 
 RGWSimpleRadosUnlockCR::RGWSimpleRadosUnlockCR(RGWAsyncRadosProcessor *_async_rados, rgw::sal::RadosStore* _store,
@@ -559,34 +569,40 @@ RGWSimpleRadosUnlockCR::RGWSimpleRadosUnlockCR(RGWAsyncRadosProcessor *_async_ra
                                                 store(_store),
                                                 lock_name(_lock_name),
                                                 cookie(_cookie),
-                                                obj(_obj),
-                                                req(NULL)
+                                                obj(_obj)
 {
   set_description() << "rados unlock dest=" << obj << " lock=" << lock_name << " cookie=" << cookie;
 }
 
 void RGWSimpleRadosUnlockCR::request_cleanup()
 {
-  if (req) {
-    req->finish();
-    req = NULL;
-  }
 }
 
 int RGWSimpleRadosUnlockCR::send_request(const DoutPrefixProvider *dpp)
 {
+  int r = store->getRados()->get_raw_obj_ref(dpp, obj, &ref);
+  if (r < 0) {
+    ldpp_dout(dpp, -1) << "ERROR: failed to get ref for (" << obj << ") ret=" << r << dendl;
+    return r;
+  }
+
   set_status() << "sending request";
 
-  req = new RGWAsyncUnlockSystemObj(this, stack->create_completion_notifier(),
-                                 store, NULL, obj, lock_name, cookie);
-  async_rados->queue(req);
-  return 0;
+  rados::cls::lock::Lock l(lock_name);
+  l.set_cookie(cookie);
+
+  librados::ObjectWriteOperation op;
+  l.unlock(&op);
+
+  cn = stack->create_completion_notifier();
+  return ref.ioctx.aio_operate(ref.obj.oid, cn->completion(), &op);
 }
 
 int RGWSimpleRadosUnlockCR::request_complete()
 {
-  set_status() << "request complete; ret=" << req->get_ret_status();
-  return req->get_ret_status();
+  int r = cn->completion()->get_return_value();
+  set_status() << "request complete; ret=" << r;
+  return r;
 }
 
 int RGWOmapAppend::operate(const DoutPrefixProvider *dpp) {
diff --git a/src/rgw/driver/rados/rgw_cr_rados.h b/src/rgw/driver/rados/rgw_cr_rados.h
index df40db5dbc46..f49523c11645 100644
--- a/src/rgw/driver/rados/rgw_cr_rados.h
+++ b/src/rgw/driver/rados/rgw_cr_rados.h
@@ -784,8 +784,8 @@ class RGWSimpleRadosLockCR : public RGWSimpleCoroutine {
     uint32_t duration;
 
     rgw_raw_obj obj;
-
-    RGWAsyncLockSystemObj *req;
+    rgw_rados_ref ref;
+    boost::intrusive_ptr<RGWAioCompletionNotifier> cn;
 
 public:
   RGWSimpleRadosLockCR(RGWAsyncRadosProcessor *_async_rados, rgw::sal::RadosStore* _store,
@@ -816,8 +816,8 @@ class RGWSimpleRadosUnlockCR : public RGWSimpleCoroutine {
   std::string cookie;
 
   rgw_raw_obj obj;
-
-  RGWAsyncUnlockSystemObj *req;
+  rgw_rados_ref ref;
+  boost::intrusive_ptr<RGWAioCompletionNotifier> cn;
 
 public:
   RGWSimpleRadosUnlockCR(RGWAsyncRadosProcessor *_async_rados, rgw::sal::RadosStore* _store,