os/bluestore: add multiple finishers to bluestore

- The single finisher of a bluestore can be a bottleneck
when using an SSD as a backend device. If too much load
is given to the single finisher, client-side IO latency
increases. So we add multiple finishers to the
bluestore, which shows better performance.

- 'bluestore_shard_finishers' option is added to
be able to configure wheather finsihers is multiple or
not.

- a finisher is selected according to the shard id of a
sequencer.

- the number of finishers is decided by
osd_op_num_shards.

Signed-off-by: Ilsoo Byun <ilsoo.byun@sk.com>

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index f2a50165fcb93b1f662c7fad83d7f85ae2bc12eb..65e5878c8fd8aacbef12df32927e5e78cbbd0201 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -1011,6 +1011,7 @@ OPTION(bluestore_debug_freelist, OPT_BOOL, false)
 OPTION(bluestore_debug_prefill, OPT_FLOAT, 0)
 OPTION(bluestore_debug_prefragment_max, OPT_INT, 1048576)
 OPTION(bluestore_inject_wal_apply_delay, OPT_FLOAT, 0)
+OPTION(bluestore_shard_finishers, OPT_BOOL, false)
 
 OPTION(kstore_max_ops, OPT_U64, 512)
 OPTION(kstore_max_bytes, OPT_U64, 64*1024*1024)

diff --git a/src/os/ObjectStore.h b/src/os/ObjectStore.h
index 98260880f6ecbd18a21984f45b2951a38f058b73..ab200ea1b2cd50f8470b047b39220606f07ca935 100644 (file)
--- a/src/os/ObjectStore.h
+++ b/src/os/ObjectStore.h
@@ -179,10 +179,11 @@ public:
    */
   struct Sequencer {
     string name;
+    spg_t shard_hint;
     Sequencer_implRef p;
 
     explicit Sequencer(string n)
-      : name(n), p(NULL) {}
+      : name(n), shard_hint(spg_t()), p(NULL) {}
     ~Sequencer() {
     }
 

diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index eac416d0e8e63778825a4142c68bba9af74d31b2..e3de7b96f2a6bbe785f9563a1c604348390d3323 100644 (file)
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -1400,7 +1400,7 @@ BlueStore::BlueStore(CephContext *cct, const string& path)
             cct->_conf->bluestore_wal_thread_timeout,
             cct->_conf->bluestore_wal_thread_suicide_timeout,
             &wal_tp),
-    finisher(cct),
+    m_finisher_num(1),
     kv_sync_thread(this),
     kv_stop(false),
     logger(NULL),
@@ -1410,10 +1410,26 @@ BlueStore::BlueStore(CephContext *cct, const string& path)
   _init_logger();
   g_ceph_context->_conf->add_observer(this);
   set_cache_shards(1);
+
+  if (cct->_conf->bluestore_shard_finishers) {
+    m_finisher_num = cct->_conf->osd_op_num_shards;
+  }
+
+  for (int i = 0; i < m_finisher_num; ++i) {
+    ostringstream oss;
+    oss << "finisher-" << i;
+    Finisher *f = new Finisher(cct, oss.str(), "finisher");
+    finishers.push_back(f);
+  }
 }
 
 BlueStore::~BlueStore()
 {
+  for (auto f : finishers) {
+    delete f;
+    f = NULL;
+  }
+
   g_ceph_context->_conf->remove_observer(this);
   _shutdown_logger();
   assert(!mounted);
@@ -2824,7 +2840,9 @@ int BlueStore::mount()
       goto out_coll;
   }
 
-  finisher.start();
+  for (auto f : finishers) {
+    f->start();
+  }
   wal_tp.start();
   kv_sync_thread.create("bstore_kv_sync");
 
@@ -2842,8 +2860,10 @@ int BlueStore::mount()
   _kv_stop();
   wal_wq.drain();
   wal_tp.stop();
-  finisher.wait_for_empty();
-  finisher.stop();
+  for (auto f : finishers) {
+    f->wait_for_empty();
+    f->stop();
+  }
  out_coll:
   coll_map.clear();
  out_alloc:
@@ -2876,10 +2896,12 @@ int BlueStore::umount()
   wal_wq.drain();
   dout(20) << __func__ << " stopping wal_tp" << dendl;
   wal_tp.stop();
-  dout(20) << __func__ << " draining finisher" << dendl;
-  finisher.wait_for_empty();
-  dout(20) << __func__ << " stopping finisher" << dendl;
-  finisher.stop();
+  for (auto f : finishers) {
+    dout(20) << __func__ << " draining finisher" << dendl;
+    f->wait_for_empty();
+    dout(20) << __func__ << " stopping finisher" << dendl;
+    f->stop();
+  }
   dout(20) << __func__ << " closing" << dendl;
 
   mounted = false;
@@ -4944,16 +4966,18 @@ void BlueStore::_txc_finish_kv(TransContext *txc)
     txc->onreadable_sync->complete(0);
     txc->onreadable_sync = NULL;
   }
+  unsigned n = txc->osr->parent->shard_hint.hash_to_shard(m_finisher_num);
   if (txc->onreadable) {
-    finisher.queue(txc->onreadable);
+    finishers[n]->queue(txc->onreadable);
     txc->onreadable = NULL;
   }
   if (txc->oncommit) {
-    finisher.queue(txc->oncommit);
+    finishers[n]->queue(txc->oncommit);
     txc->oncommit = NULL;
   }
   while (!txc->oncommits.empty()) {
-    finisher.queue(txc->oncommits.front());
+    auto f = txc->oncommits.front();
+    finishers[n]->queue(f);
     txc->oncommits.pop_front();
   }
 

diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h
index 7949152a4e88b2fc6093ee1b80758621652e390a..173cf9b82c1263076094caab353689bd01ef6ef3 100644 (file)
--- a/src/os/bluestore/BlueStore.h
+++ b/src/os/bluestore/BlueStore.h
@@ -1140,7 +1140,8 @@ private:
   ThreadPool wal_tp;
   WALWQ wal_wq;
 
-  Finisher finisher;
+  int m_finisher_num;
+  vector<Finisher*> finishers;
 
   KVSyncThread kv_sync_thread;
   std::mutex kv_lock;

diff --git a/src/osd/PG.cc b/src/osd/PG.cc
index 79a5da866d39f17adf5f4cdc9d7bea3318c5d7e8..8b80e7e6d44c772b99d6bad3ca758f613261886d 100644 (file)
--- a/src/osd/PG.cc
+++ b/src/osd/PG.cc
@@ -249,6 +249,7 @@ PG::PG(OSDService *o, OSDMapRef curmap,
 #ifdef PG_DEBUG_REFS
   osd->add_pgid(p, this);
 #endif
+  osr->shard_hint = p;
 }
 
 PG::~PG()