rgw: config options for mp lock error/delay injection

rgw_mp_lock_inject_delay can inject delays in multipart complete to make
sure lock renewal requests get issued

rgw_mp_lock_inject_renewal_error can inject lock renewal errors to test
the error handling

Signed-off-by: Casey Bodley <cbodley@redhat.com>

diff --git a/src/common/options/rgw.yaml.in b/src/common/options/rgw.yaml.in
index 6ca27bc188ef2ca2e9fb67fa35386f5a513be99e..67d04781a37ae29905e3db1f09fbc4ab4d67be4f 100644 (file)
--- a/src/common/options/rgw.yaml.in
+++ b/src/common/options/rgw.yaml.in
@@ -573,6 +573,20 @@ options:
   min: 2_min
   services:
   - rgw
+- name: rgw_mp_lock_inject_delay
+  type: int
+  level: dev
+  desc: Injected delay after acquiring the multipart lock for renewal testing
+  default: 0
+  services:
+  - rgw
+- name: rgw_mp_lock_inject_renewal_error
+  type: int
+  level: dev
+  desc: Injected error code for multipart lock renewal testing
+  default: 0
+  services:
+  - rgw
 - name: rgw_script_uri
   type: str
   level: dev

diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc
index 2fa4095750d8cab8e28650cb6777b0b7cd344bf0..5f3a8237456786fca38819e05d2693cb38ed7181 100644 (file)
--- a/src/rgw/driver/rados/rgw_sal_rados.cc
+++ b/src/rgw/driver/rados/rgw_sal_rados.cc
@@ -4680,10 +4680,20 @@ static void renewal(const DoutPrefixProvider* dpp,
       break;
     }
 
+    // for renewal testing, inject an error from lock renewal
+    ceph_assert(dpp->get_cct());
+    int ret = dpp->get_cct()->_conf.get_val<int64_t>("rgw_mp_lock_inject_renewal_error");
+    if (ret < 0) {
+      ldpp_dout(dpp, 0) << "MPSerializer lock renewal on "
+          << oid << " failed with injected error " << ret << dendl;
+      serializer.clear_locked();
+      return;
+    }
+
     librados::ObjectWriteOperation op;
     op.assert_exists();
     lock.lock_exclusive(&op);
-    int ret = rgw_rados_operate(dpp, ioctx, oid, std::move(op), yield);
+    ret = rgw_rados_operate(dpp, ioctx, oid, std::move(op), yield);
     if (ret < 0) {
       ldpp_dout(dpp, 0) << "ERROR: MPSerializer lock renewal on "
           << oid << " failed with " << ret << ". If this upload completes, "
@@ -4757,6 +4767,20 @@ int MPRadosSerializer::try_lock(const DoutPrefixProvider *dpp, ceph::timespan du
     locked = true;
 
     start_renewal(dur);
+
+    // for renewal testing, inject a delay after lock acquisition
+    ceph_assert(dpp->get_cct());
+    const auto inject_delay = dpp->get_cct()->_conf.get_val<int64_t>("rgw_mp_lock_inject_delay");
+    if (inject_delay) {
+      ldpp_dout(dpp, 10) << "MPSerializer injecting delay after lock..." << dendl;
+      auto timer = Timer{ex, std::chrono::seconds(inject_delay)};
+      if (y) {
+        timer.async_wait(y.get_yield_context());
+      } else {
+        timer.wait();
+      }
+      ldpp_dout(dpp, 10) << "MPSerializer waking up after injected delay" << dendl;
+    }
   }
   return ret;
 }