From 028471899e0f15bae701eec5088ac35145c86749 Mon Sep 17 00:00:00 2001 From: Ilsoo Byun Date: Thu, 18 Aug 2016 14:29:38 -0400 Subject: [PATCH] os/bluestore: add multiple finishers to bluestore - The single finisher of a bluestore can be a bottleneck when using an SSD as a backend device. If too much load is given to the single finisher, client-side IO latency increases. So we add multiple finishers to the bluestore, which shows better performance. - 'bluestore_shard_finishers' option is added to be able to configure wheather finsihers is multiple or not. - a finisher is selected according to the shard id of a sequencer. - the number of finishers is decided by osd_op_num_shards. Signed-off-by: Ilsoo Byun --- src/common/config_opts.h | 1 + src/os/ObjectStore.h | 3 ++- src/os/bluestore/BlueStore.cc | 46 ++++++++++++++++++++++++++--------- src/os/bluestore/BlueStore.h | 3 ++- src/osd/PG.cc | 1 + 5 files changed, 41 insertions(+), 13 deletions(-) diff --git a/src/common/config_opts.h b/src/common/config_opts.h index f2a50165fcb9..65e5878c8fd8 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -1011,6 +1011,7 @@ OPTION(bluestore_debug_freelist, OPT_BOOL, false) OPTION(bluestore_debug_prefill, OPT_FLOAT, 0) OPTION(bluestore_debug_prefragment_max, OPT_INT, 1048576) OPTION(bluestore_inject_wal_apply_delay, OPT_FLOAT, 0) +OPTION(bluestore_shard_finishers, OPT_BOOL, false) OPTION(kstore_max_ops, OPT_U64, 512) OPTION(kstore_max_bytes, OPT_U64, 64*1024*1024) diff --git a/src/os/ObjectStore.h b/src/os/ObjectStore.h index 98260880f6ec..ab200ea1b2cd 100644 --- a/src/os/ObjectStore.h +++ b/src/os/ObjectStore.h @@ -179,10 +179,11 @@ public: */ struct Sequencer { string name; + spg_t shard_hint; Sequencer_implRef p; explicit Sequencer(string n) - : name(n), p(NULL) {} + : name(n), shard_hint(spg_t()), p(NULL) {} ~Sequencer() { } diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index eac416d0e8e6..e3de7b96f2a6 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -1400,7 +1400,7 @@ BlueStore::BlueStore(CephContext *cct, const string& path) cct->_conf->bluestore_wal_thread_timeout, cct->_conf->bluestore_wal_thread_suicide_timeout, &wal_tp), - finisher(cct), + m_finisher_num(1), kv_sync_thread(this), kv_stop(false), logger(NULL), @@ -1410,10 +1410,26 @@ BlueStore::BlueStore(CephContext *cct, const string& path) _init_logger(); g_ceph_context->_conf->add_observer(this); set_cache_shards(1); + + if (cct->_conf->bluestore_shard_finishers) { + m_finisher_num = cct->_conf->osd_op_num_shards; + } + + for (int i = 0; i < m_finisher_num; ++i) { + ostringstream oss; + oss << "finisher-" << i; + Finisher *f = new Finisher(cct, oss.str(), "finisher"); + finishers.push_back(f); + } } BlueStore::~BlueStore() { + for (auto f : finishers) { + delete f; + f = NULL; + } + g_ceph_context->_conf->remove_observer(this); _shutdown_logger(); assert(!mounted); @@ -2824,7 +2840,9 @@ int BlueStore::mount() goto out_coll; } - finisher.start(); + for (auto f : finishers) { + f->start(); + } wal_tp.start(); kv_sync_thread.create("bstore_kv_sync"); @@ -2842,8 +2860,10 @@ int BlueStore::mount() _kv_stop(); wal_wq.drain(); wal_tp.stop(); - finisher.wait_for_empty(); - finisher.stop(); + for (auto f : finishers) { + f->wait_for_empty(); + f->stop(); + } out_coll: coll_map.clear(); out_alloc: @@ -2876,10 +2896,12 @@ int BlueStore::umount() wal_wq.drain(); dout(20) << __func__ << " stopping wal_tp" << dendl; wal_tp.stop(); - dout(20) << __func__ << " draining finisher" << dendl; - finisher.wait_for_empty(); - dout(20) << __func__ << " stopping finisher" << dendl; - finisher.stop(); + for (auto f : finishers) { + dout(20) << __func__ << " draining finisher" << dendl; + f->wait_for_empty(); + dout(20) << __func__ << " stopping finisher" << dendl; + f->stop(); + } dout(20) << __func__ << " closing" << dendl; mounted = false; @@ -4944,16 +4966,18 @@ void BlueStore::_txc_finish_kv(TransContext *txc) txc->onreadable_sync->complete(0); txc->onreadable_sync = NULL; } + unsigned n = txc->osr->parent->shard_hint.hash_to_shard(m_finisher_num); if (txc->onreadable) { - finisher.queue(txc->onreadable); + finishers[n]->queue(txc->onreadable); txc->onreadable = NULL; } if (txc->oncommit) { - finisher.queue(txc->oncommit); + finishers[n]->queue(txc->oncommit); txc->oncommit = NULL; } while (!txc->oncommits.empty()) { - finisher.queue(txc->oncommits.front()); + auto f = txc->oncommits.front(); + finishers[n]->queue(f); txc->oncommits.pop_front(); } diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h index 7949152a4e88..173cf9b82c12 100644 --- a/src/os/bluestore/BlueStore.h +++ b/src/os/bluestore/BlueStore.h @@ -1140,7 +1140,8 @@ private: ThreadPool wal_tp; WALWQ wal_wq; - Finisher finisher; + int m_finisher_num; + vector finishers; KVSyncThread kv_sync_thread; std::mutex kv_lock; diff --git a/src/osd/PG.cc b/src/osd/PG.cc index 79a5da866d39..8b80e7e6d44c 100644 --- a/src/osd/PG.cc +++ b/src/osd/PG.cc @@ -249,6 +249,7 @@ PG::PG(OSDService *o, OSDMapRef curmap, #ifdef PG_DEBUG_REFS osd->add_pgid(p, this); #endif + osr->shard_hint = p; } PG::~PG() -- 2.47.3