rgw_file: alternate fix deadlock on lru eviction

This change is an alternate fix for two problems found and fixed
by Yao Zongyou <yaozongyou@vip.qq.com>.

The deadlock can be avoided just by not taking it in the recycle
case, which invariantly holds the lock.

The invalidation of the insert iterator by the recyle-path unlink
we'd like to handle as a condition in order to preserve the cached
insertion point optimization we get in the common case.  (The
original behavior was, indeed, incorrect.)

Based on feedback from Yao, removed the RGWFileHandle dtor version
of the unlink check, which I think happened twice.

Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>

diff --git a/src/common/cohort_lru.h b/src/common/cohort_lru.h
index bd4fa95709785cb45f7f0a396e40b757230fe136..edd3e38828a9bded70b03a708f387944effcaaf8 100644 (file)
--- a/src/common/cohort_lru.h
+++ b/src/common/cohort_lru.h
@@ -32,6 +32,7 @@ namespace cohort {
     /* public flag values */
     constexpr uint32_t FLAG_NONE = 0x0000;
     constexpr uint32_t FLAG_INITIAL = 0x0001;
+    constexpr uint32_t FLAG_RECYCLE = 0x0002;
 
     enum class Edge : std::uint8_t
     {
@@ -232,12 +233,14 @@ namespace cohort {
          delete tdo;
       } /* unref */
 
-      Object* insert(ObjectFactory* fac, Edge edge, uint32_t flags) {
+      Object* insert(ObjectFactory* fac, Edge edge, uint32_t& flags) {
        /* use supplied functor to re-use an evicted object, or
         * allocate a new one of the descendant type */
        Object* o = evict_block();
-       if (o)
+       if (o) {
          fac->recycle(o); /* recycle existing object */
+         flags |= FLAG_RECYCLE;
+       }
        else
          o = fac->alloc(); /* get a new one */
 

diff --git a/src/rgw/rgw_file.cc b/src/rgw/rgw_file.cc
index d37f05be30c69b6ef1c7089bab68aa5429e2cead..b35630bc9a125c78feb578c64a681212e6e6c535 100644 (file)
--- a/src/rgw/rgw_file.cc
+++ b/src/rgw/rgw_file.cc
@@ -954,10 +954,6 @@ namespace rgw {
   }
 
   RGWFileHandle::~RGWFileHandle() {
-    /* in the non-delete case, handle may still be in handle table */
-    if (fh_hook.is_linked()) {
-      fs->fh_cache.remove(fh.fh_hk.object, this, FHCache::FLAG_LOCK);
-    }
     /* cond-unref parent */
     if (parent && (! parent->is_mount())) {
       /* safe because if parent->unref causes its deletion,
@@ -1007,9 +1003,11 @@ namespace rgw {
     lsubdout(fs->get_context(), rgw, 17)
       << __func__ << " " << *this
       << dendl;
-    /* remove if still in fh_cache */
+    /* in the non-delete case, handle may still be in handle table */
     if (fh_hook.is_linked()) {
-      fs->fh_cache.remove(fh.fh_hk.object, this, FHCache::FLAG_LOCK);
+      /* in this case, we are being called from a context which holds
+       * the partition lock */
+      fs->fh_cache.remove(fh.fh_hk.object, this, FHCache::FLAG_NONE);
     }
     return true;
   } /* RGWFileHandle::reclaim */

diff --git a/src/rgw/rgw_file.h b/src/rgw/rgw_file.h
index 4843c95dc987786705604caca9c2356c83113c66..d4e5333e0100789578ff0ea2cc41c464599d1f65 100644 (file)
--- a/src/rgw/rgw_file.h
+++ b/src/rgw/rgw_file.h
@@ -1061,16 +1061,25 @@ namespace rgw {
        /* make or re-use handle */
        RGWFileHandle::Factory prototype(this, parent, fhk,
                                         obj_name, CREATE_FLAGS(flags));
+       uint32_t iflags{cohort::lru::FLAG_INITIAL};
        fh = static_cast<RGWFileHandle*>(
          fh_lru.insert(&prototype,
                        cohort::lru::Edge::MRU,
-                       cohort::lru::FLAG_INITIAL));
+                       iflags));
        if (fh) {
          /* lock fh (LATCHED) */
          if (flags & RGWFileHandle::FLAG_LOCK)
            fh->mtx.lock();
-         /* inserts, releasing latch */
-         fh_cache.insert_latched(fh, lat, RGWFileHandle::FHCache::FLAG_UNLOCK);
+         if (likely(! (iflags & cohort::lru::FLAG_RECYCLE))) {
+           /* inserts at cached insert iterator, releasing latch */
+           fh_cache.insert_latched(
+             fh, lat, RGWFileHandle::FHCache::FLAG_UNLOCK);
+         } else {
+           /* recycle step invalidates Latch */
+           fh_cache.insert(
+             fhk.fh_hk.object, fh, RGWFileHandle::FHCache::FLAG_NONE);
+           lat.lock->unlock(); /* !LATCHED */
+         }
          get<1>(fhr) |= RGWFileHandle::FLAG_CREATE;
          /* ref parent (non-initial ref cannot fail on valid object) */
          if (! parent->is_mount()) {