From 2b6a9d1ec0778f66b6e9d9f216df21a00caeba3f Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Thu, 31 Aug 2017 16:43:39 -0400 Subject: [PATCH] os/bluestore: separate finisher for deferred_try_submit Reusing finishers[0], which is used for completions back into the OSD, is deadlock-prone: the OSD code might block trying to submit new IO or while waiting for some other bluestore work to complete. Fixes: http://tracker.ceph.com/issues/21207 Signed-off-by: Sage Weil --- src/os/bluestore/BlueStore.cc | 7 ++++++- src/os/bluestore/BlueStore.h | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index d274874ea1a..43e7bf7ad8c 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -3477,6 +3477,7 @@ BlueStore::BlueStore(CephContext *cct, const string& path) throttle_deferred_bytes(cct, "bluestore_throttle_deferred_bytes", cct->_conf->bluestore_throttle_bytes + cct->_conf->bluestore_throttle_deferred_bytes), + deferred_finisher(cct, "defered_finisher", "dfin"), kv_sync_thread(this), kv_finalize_thread(this), mempool_thread(this) @@ -3495,6 +3496,7 @@ BlueStore::BlueStore(CephContext *cct, throttle_deferred_bytes(cct, "bluestore_throttle_deferred_bytes", cct->_conf->bluestore_throttle_bytes + cct->_conf->bluestore_throttle_deferred_bytes), + deferred_finisher(cct, "defered_finisher", "dfin"), kv_sync_thread(this), kv_finalize_thread(this), min_alloc_size(_min_alloc_size), @@ -8234,6 +8236,7 @@ void BlueStore::_kv_start() finishers.push_back(f); } + deferred_finisher.start(); for (auto f : finishers) { f->start(); } @@ -8271,6 +8274,8 @@ void BlueStore::_kv_stop() kv_finalize_stop = false; } dout(10) << __func__ << " stopping finishers" << dendl; + deferred_finisher.wait_for_empty(); + deferred_finisher.stop(); for (auto f : finishers) { f->wait_for_empty(); f->stop(); @@ -8710,7 +8715,7 @@ void BlueStore::_deferred_aio_finish(OpSequencer *osr) deferred_queue.erase(q); } else if (deferred_aggressive) { dout(20) << __func__ << " queuing async deferred_try_submit" << dendl; - finishers[0]->queue(new FunctionContext([&](int) { + deferred_finisher.queue(new FunctionContext([&](int) { deferred_try_submit(); })); } diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h index 8c5eb4a02b4..d0de456d422 100644 --- a/src/os/bluestore/BlueStore.h +++ b/src/os/bluestore/BlueStore.h @@ -1846,6 +1846,7 @@ private: deferred_osr_queue_t deferred_queue; ///< osr's with deferred io pending int deferred_queue_size = 0; ///< num txc's queued across all osrs atomic_int deferred_aggressive = {0}; ///< aggressive wakeup of kv thread + Finisher deferred_finisher; int m_finisher_num = 1; vector finishers; -- 2.39.5