rgw/rgw_file: Fix the incorrect lru object eviction

In func lookup_fh, when RGWFileHandle not be found in fh_cache, it
need to recycle an object and create an new RGWFileHandle. When there
are multi threads use lookup_fh to find and create RGWFileHandle concurrently,
it must to make sure evict lru object from the partiton of fh_cache which new
RGWFileHandle will be inserted to.

Fixes: https://tracker.ceph.com/issues/47235
Signed-off-by: luo rixin <luorixin@huawei.com>
(cherry picked from commit f2097338722d7f2526bb815da47695f2da17fcce)

diff --git a/src/common/cohort_lru.h b/src/common/cohort_lru.h
index 3c789582f58174eb775dd475c568745863644c4d..2383fc95d7a3b48307896e6b720b11de0f45f472 100644 (file)
--- a/src/common/cohort_lru.h
+++ b/src/common/cohort_lru.h
@@ -42,6 +42,8 @@ namespace cohort {
 
     typedef bi::link_mode<bi::safe_link> link_mode;
 
+    class ObjectFactory; // Forward declaration
+
     class Object
     {
     private:
@@ -70,7 +72,7 @@ namespace cohort {
 
       uint32_t get_refcnt() const { return lru_refcnt; }
 
-      virtual bool reclaim() = 0;
+      virtual bool reclaim(const ObjectFactory* newobj_fac) = 0;
 
       virtual ~Object() {}
 
@@ -132,7 +134,7 @@ namespace cohort {
                (!(o->lru_flags & FLAG_EVICTING)));
       }
 
-      Object* evict_block() {
+      Object* evict_block(const ObjectFactory* newobj_fac) {
        uint32_t lane_ix = next_evict_lane();
        for (int ix = 0; ix < n_lanes; ++ix,
               lane_ix = next_evict_lane()) {
@@ -144,7 +146,7 @@ namespace cohort {
            ++(o->lru_refcnt);
            o->lru_flags |= FLAG_EVICTING;
            lane.lock.unlock();
-           if (o->reclaim()) {
+           if (o->reclaim(newobj_fac)) {
              lane.lock.lock();
              --(o->lru_refcnt);
              /* assertions that o state has not changed across
@@ -236,7 +238,7 @@ namespace cohort {
       Object* insert(ObjectFactory* fac, Edge edge, uint32_t& flags) {
        /* use supplied functor to re-use an evicted object, or
         * allocate a new one of the descendant type */
-       Object* o = evict_block();
+       Object* o = evict_block(fac);
        if (o) {
          fac->recycle(o); /* recycle existing object */
          flags |= FLAG_RECYCLE;
@@ -425,7 +427,9 @@ namespace cohort {
          lat.lock->unlock();
        return v;
       } /* find_latch */
-
+      bool is_same_partition(uint64_t lhs, uint64_t rhs) {
+        return ((lhs % n_part) == (rhs % n_part));
+      }
       void insert_latched(T* v, Latch& lat, uint32_t flags) {
        (void) lat.p->tr.insert_unique_commit(*v, lat.commit_data);
        if (flags & FLAG_UNLOCK)

diff --git a/src/rgw/rgw_file.cc b/src/rgw/rgw_file.cc
index d2aec4e71647ea762418f4bb0820465c250d219a..5ccc01c3e4b95e4724232bfb3390b3d800d33e2e 100644 (file)
--- a/src/rgw/rgw_file.cc
+++ b/src/rgw/rgw_file.cc
@@ -1201,10 +1201,19 @@ namespace rgw {
     return dar;
   } /* RGWFileHandle::decode_attrs */
 
-  bool RGWFileHandle::reclaim() {
+  bool RGWFileHandle::reclaim(const cohort::lru::ObjectFactory* newobj_fac) {
     lsubdout(fs->get_context(), rgw, 17)
       << __func__ << " " << *this
       << dendl;
+    auto factory = dynamic_cast<const RGWFileHandle::Factory*>(newobj_fac);
+    if (factory == nullptr) {
+      return false;
+    }
+    /* make sure the reclaiming object is the same partiton with newobject factory,
+     * then we can recycle the object, and replace with newobject */
+    if (!fs->fh_cache.is_same_partition(factory->fhk.fh_hk.object, fh.fh_hk.object)) {
+      return false;
+    }
     /* in the non-delete case, handle may still be in handle table */
     if (fh_hook.is_linked()) {
       /* in this case, we are being called from a context which holds

diff --git a/src/rgw/rgw_file.h b/src/rgw/rgw_file.h
index 0bf488a579b46ec3ab3fcf8b1ee91c2a587bfdc7..46565fb14c11d006b13d335ec3880512a2a66fce 100644 (file)
--- a/src/rgw/rgw_file.h
+++ b/src/rgw/rgw_file.h
@@ -721,7 +721,7 @@ namespace rgw {
 
     void invalidate();
 
-    bool reclaim() override;
+    bool reclaim(const cohort::lru::ObjectFactory* newobj_fac) override;
 
     typedef cohort::lru::LRU<std::mutex> FhLRU;