os/bluestore: fix race between remove_collection and object removals.

Fixes: https://tracker.ceph.com/issues/25077
Signed-off-by: Igor Fedotov <ifedotov@suse.com>

diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index 42318ab811143e929b08c648c39ea803b36eaa0a..bfb35ac594f5e271c5ab14331f615656a7767a2d 100644 (file)
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -3196,6 +3196,11 @@ void BlueStore::Collection::flush()
   osr->flush();
 }
 
+void BlueStore::Collection::flush_all_but_last()
+{
+  osr->flush_all_but_last();
+}
+
 void BlueStore::Collection::open_shared_blob(uint64_t sbid, BlobRef b)
 {
   assert(!b->shared_blob);
@@ -8616,8 +8621,7 @@ void BlueStore::_txc_finish_io(TransContext *txc)
   } while (p != osr->q.end() &&
           p->state == TransContext::STATE_IO_DONE);
 
-  if (osr->kv_submitted_waiters &&
-      osr->_is_all_kv_submitted()) {
+  if (osr->kv_submitted_waiters) {
     osr->qcond.notify_all();
   }
 }
@@ -9131,9 +9135,7 @@ void BlueStore::_kv_sync_thread()
          txc->state = TransContext::STATE_KV_SUBMITTED;
          if (txc->osr->kv_submitted_waiters) {
            std::lock_guard<std::mutex> l(txc->osr->qlock);
-           if (txc->osr->_is_all_kv_submitted()) {
-             txc->osr->qcond.notify_all();
-           }
+           txc->osr->qcond.notify_all();
          }
 
        } else {
@@ -11395,7 +11397,9 @@ int BlueStore::_remove(TransContext *txc,
                       CollectionRef& c,
                       OnodeRef &o)
 {
-  dout(15) << __func__ << " " << c->cid << " " << o->oid << dendl;
+  dout(15) << __func__ << " " << c->cid << " " << o->oid
+          << " onode " << o.get()
+          << " txc "<< txc << dendl;
   int r = _do_remove(txc, c, o);
   dout(10) << __func__ << " " << c->cid << " " << o->oid << " = " << r << dendl;
   return r;
@@ -11929,6 +11933,7 @@ int BlueStore::_remove_collection(TransContext *txc, const coll_t &cid,
   dout(15) << __func__ << " " << cid << dendl;
   int r;
 
+  (*c)->flush_all_but_last();
   {
     RWLock::WLocker l(coll_lock);
     if (!*c) {
@@ -11967,7 +11972,9 @@ int BlueStore::_remove_collection(TransContext *txc, const coll_t &cid,
         exists = !onode || onode->exists;
         if (exists) {
           dout(10) << __func__ << " " << *it
-                   << " exists in db" << dendl;
+                  << " exists in db, "
+                  << (!onode ? "not present in ram" : "present in ram")
+                  << dendl;
         }
       }
       if (!exists) {

diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h
index 5e915d3e045bc0f2a44f0720925d51f2667e1a24..6361bd4f9efeb93754b7f5f22d634f1476f68554 100644 (file)
--- a/src/os/bluestore/BlueStore.h
+++ b/src/os/bluestore/BlueStore.h
@@ -1400,6 +1400,7 @@ public:
 
     bool flush_commit(Context *c) override;
     void flush() override;
+    void flush_all_but_last();
 
     Collection(BlueStore *ns, Cache *ca, coll_t c);
   };
@@ -1747,6 +1748,29 @@ public:
       }
     }
 
+    void flush_all_but_last() {
+      std::unique_lock<std::mutex> l(qlock);
+      assert (q.size() >= 1);
+      while (true) {
+       // set flag before the check because the condition
+       // may become true outside qlock, and we need to make
+       // sure those threads see waiters and signal qcond.
+       ++kv_submitted_waiters;
+       if (q.size() <= 1) {
+         --kv_submitted_waiters;
+         return;
+       } else {
+         auto it = q.rbegin();
+         it++;
+         if (it->state >= TransContext::STATE_KV_SUBMITTED) {
+           return;
+          }
+       }
+       qcond.wait(l);
+       --kv_submitted_waiters;
+      }
+    }
+
     bool flush_commit(Context *c) {
       std::lock_guard<std::mutex> l(qlock);
       if (q.empty()) {