os/bluestore: prevent throttle deadlock due to deferred writes

Kick off deferred IOs if we pass the throttle midpoint or if we would
block during submission.

Signed-off-by: Sage Weil <sage@redhat.com>

diff --git a/src/common/Throttle.h b/src/common/Throttle.h
index b07c9e47923c440803b1012dc991689bf71188e4..e4119a3b97c8feb616cf9bc369e2f225b28362b0 100644 (file)
--- a/src/common/Throttle.h
+++ b/src/common/Throttle.h
@@ -66,6 +66,13 @@ public:
    */
   int64_t get_max() const { return max.read(); }
 
+  /**
+   * return true if past midpoint
+   */
+  bool past_midpoint() const {
+    return count.read() >= max.read() / 2;
+  }
+
   /**
    * set the new max number, and wait until the number of taken slots drains
    * and drops below this limit.

diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index 39803670b1ff835d8b8672f1bcd57d50afcf83eb..2419d2e2dbb22ff0557793525ca75bc49667931e 100644 (file)
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -7665,7 +7665,9 @@ void BlueStore::_kv_sync_thread()
 
       if (!deferred_aggressive) {
        std::lock_guard<std::mutex> l(deferred_lock);
-       if (deferred_queue_size >= (int)g_conf->bluestore_deferred_batch_ops) {
+       if (deferred_queue_size >= (int)g_conf->bluestore_deferred_batch_ops ||
+           throttle_deferred_ops.past_midpoint() ||
+           throttle_deferred_bytes.past_midpoint()) {
          _deferred_try_submit();
        }
       }
@@ -7958,8 +7960,15 @@ int BlueStore::queue_transactions(
   throttle_ops.get(txc->ops);
   throttle_bytes.get(txc->bytes);
   if (txc->deferred_txn) {
-    throttle_deferred_ops.get(txc->ops);
-    throttle_deferred_bytes.get(txc->bytes);
+    // ensure we do not block here because of deferred writes
+    if (!throttle_deferred_ops.get_or_fail(txc->ops)) {
+      deferred_try_submit();
+      throttle_deferred_ops.get(txc->ops);
+    }
+    if (!throttle_deferred_bytes.get_or_fail(txc->bytes)) {
+      deferred_try_submit();
+      throttle_deferred_bytes.get(txc->bytes);
+    }
   }
 
   if (handle)

diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h
index 106e9ccf38f80185bc487dab47c4ca861c01ef57..39913a98b3461e94ca59cb8572f8408038ed7737 100644 (file)
--- a/src/os/bluestore/BlueStore.h
+++ b/src/os/bluestore/BlueStore.h
@@ -1856,6 +1856,10 @@ private:
 
   bluestore_deferred_op_t *_get_deferred_op(TransContext *txc, OnodeRef o);
   void _deferred_queue(TransContext *txc);
+  void deferred_try_submit() {
+    std::lock_guard<std::mutex> l(deferred_lock);
+    _deferred_try_submit();
+  }
   void _deferred_try_submit();
   void _deferred_try_submit(OpSequencer *osr);
   int _deferred_finish(TransContext *txc);