rgw: RGWSI_SysObj_Cache::remove() invalidates after successful delete

invalidating the cache before the librados delete means that a racing call
to `RGWSI_SysObj_Cache::read()` may succeed and repopulate the cache. in
that case, subsequent reads will continue to return cached data even after
the librados delete succeeds

Fixes: https://tracker.ceph.com/issues/64480
Signed-off-by: Casey Bodley <cbodley@redhat.com>
(cherry picked from commit 0eebbc3f02ad153d41d23de227bd989da50e54cf)

Conflicts: RGWSysObjectCtxBase argument not present on reef/squid
	src/rgw/services/svc_sys_obj_cache.cc

diff --git a/src/rgw/services/svc_sys_obj_cache.cc b/src/rgw/services/svc_sys_obj_cache.cc
index e7c595bd2a81f358283a9c756e6dc51c5a0e5ee7..9a29f49d3471ba40f830429dce73257968dbf0e1 100644 (file)
--- a/src/rgw/services/svc_sys_obj_cache.cc
+++ b/src/rgw/services/svc_sys_obj_cache.cc
@@ -91,6 +91,11 @@ int RGWSI_SysObj_Cache::remove(const DoutPrefixProvider *dpp,
                                optional_yield y)
 
 {
+  int r = RGWSI_SysObj_Core::remove(dpp, obj_ctx, objv_tracker, obj, y);
+  if (r < 0) {
+    return r;
+  }
+
   rgw_pool pool;
   string oid;
   normalize_pool_and_obj(obj.pool, obj.oid, pool, oid);
@@ -99,12 +104,12 @@ int RGWSI_SysObj_Cache::remove(const DoutPrefixProvider *dpp,
   cache.invalidate_remove(dpp, name);
 
   ObjectCacheInfo info;
-  int r = distribute_cache(dpp, name, obj, info, INVALIDATE_OBJ, y);
+  r = distribute_cache(dpp, name, obj, info, INVALIDATE_OBJ, y);
   if (r < 0) {
     ldpp_dout(dpp, 0) << "ERROR: " << __func__ << "(): failed to distribute cache: r=" << r << dendl;
-  }
+  } // not fatal
 
-  return RGWSI_SysObj_Core::remove(dpp, obj_ctx, objv_tracker, obj, y);
+  return 0;
 }
 
 int RGWSI_SysObj_Cache::read(const DoutPrefixProvider *dpp,