From dde64b335e251b630b3c0f960060883ab94c430b Mon Sep 17 00:00:00 2001 From: Ning Yao Date: Wed, 11 Feb 2015 10:47:11 +0000 Subject: [PATCH] FileStore:: queue_transactions, fine-grain submitManager lock there is no need to lock the code, which is dealing with encoding tls to tbl. The submitManager lock is used to make sure the sequencial submission for op. So encoding transaction data out of lock. Signed-off-by: Ning Yao --- src/os/FileStore.cc | 13 +++++++++--- src/os/JournalingObjectStore.cc | 37 +++++++++++++++++++-------------- src/os/JournalingObjectStore.h | 4 +++- 3 files changed, 34 insertions(+), 20 deletions(-) diff --git a/src/os/FileStore.cc b/src/os/FileStore.cc index f6c3bb872a2e..825ce5d2883a 100644 --- a/src/os/FileStore.cc +++ b/src/os/FileStore.cc @@ -1886,6 +1886,9 @@ int FileStore::queue_transactions(Sequencer *posr, list &tls, Op *o = build_op(tls, onreadable, onreadable_sync, osd_op); op_queue_reserve_throttle(o, handle); journal->throttle(); + //prepare and encode transactions data out of lock + bufferlist tbl; + int data_align = _op_journal_transactions_prepare(o->tls, tbl); uint64_t op_num = submit_manager.op_submit_start(); o->op = op_num; @@ -1895,7 +1898,7 @@ int FileStore::queue_transactions(Sequencer *posr, list &tls, if (m_filestore_journal_parallel) { dout(5) << "queue_transactions (parallel) " << o->op << " " << o->tls << dendl; - _op_journal_transactions(o->tls, o->op, ondisk, osd_op); + _op_journal_transactions(tbl, data_align, o->op, ondisk, osd_op); // queue inside submit_manager op submission lock queue_op(osr, o); @@ -1904,7 +1907,7 @@ int FileStore::queue_transactions(Sequencer *posr, list &tls, osr->queue_journal(o->op); - _op_journal_transactions(o->tls, o->op, + _op_journal_transactions(tbl, data_align, o->op, new C_JournaledAhead(this, osr, o, ondisk), osd_op); } else { @@ -1934,6 +1937,10 @@ int FileStore::queue_transactions(Sequencer *posr, list &tls, return 0; } + + //prepare and encode transactions data out of lock + bufferlist tbl; + int data_align = _op_journal_transactions_prepare(tls, tbl); uint64_t op = submit_manager.op_submit_start(); dout(5) << "queue_transactions (trailing journal) " << op << " " << tls << dendl; @@ -1944,7 +1951,7 @@ int FileStore::queue_transactions(Sequencer *posr, list &tls, int r = do_transactions(tls, op); if (r >= 0) { - _op_journal_transactions(tls, op, ondisk, osd_op); + _op_journal_transactions(tbl, data_align, op, ondisk, osd_op); } else { delete ondisk; } diff --git a/src/os/JournalingObjectStore.cc b/src/os/JournalingObjectStore.cc index 518d54edf662..15a7c7303c43 100644 --- a/src/os/JournalingObjectStore.cc +++ b/src/os/JournalingObjectStore.cc @@ -251,27 +251,32 @@ void JournalingObjectStore::ApplyManager::commit_finish() } void JournalingObjectStore::_op_journal_transactions( - list& tls, uint64_t op, + bufferlist& tbl, int data_align, uint64_t op, Context *onjournal, TrackedOpRef osd_op) { - dout(10) << "op_journal_transactions " << op << " " << tls << dendl; - + dout(10) << "op_journal_transactions " << op << dendl; if (journal && journal->is_writeable()) { - bufferlist tbl; - unsigned data_len = 0; - int data_align = -1; // -1 indicates that we don't care about the alignment - for (list::iterator p = tls.begin(); - p != tls.end(); ++p) { - ObjectStore::Transaction *t = *p; - if (t->get_data_length() > data_len && - (int)t->get_data_length() >= g_conf->journal_align_min_size) { - data_len = t->get_data_length(); - data_align = (t->get_data_alignment() - tbl.length()) & ~CEPH_PAGE_MASK; - } - ::encode(*t, tbl); - } journal->submit_entry(op, tbl, data_align, onjournal, osd_op); } else if (onjournal) { apply_manager.add_waiter(op, onjournal); } } + +int JournalingObjectStore::_op_journal_transactions_prepare( + list& tls, bufferlist& tbl) +{ + dout(10) << "_op_journal_transactions_prepare " << tls << dendl; + unsigned data_len = 0; + int data_align = -1; // -1 indicates that we don't care about the alignment + for (list::iterator p = tls.begin(); + p != tls.end(); ++p) { + ObjectStore::Transaction *t = *p; + if (t->get_data_length() > data_len && + (int)t->get_data_length() >= g_conf->journal_align_min_size) { + data_len = t->get_data_length(); + data_align = (t->get_data_alignment() - tbl.length()) & ~CEPH_PAGE_MASK; + } + ::encode(*t, tbl); + } + return data_align; +} diff --git a/src/os/JournalingObjectStore.h b/src/os/JournalingObjectStore.h index 6a09c6133de9..fbfa20ce0f1f 100644 --- a/src/os/JournalingObjectStore.h +++ b/src/os/JournalingObjectStore.h @@ -114,7 +114,9 @@ protected: void journal_write_close(); int journal_replay(uint64_t fs_op_seq); - void _op_journal_transactions(list& tls, uint64_t op, + int _op_journal_transactions_prepare( + list& tls, bufferlist& tbl); + void _op_journal_transactions(bufferlist& tls, int data_align, uint64_t op, Context *onjournal, TrackedOpRef osd_op); virtual int do_transactions(list& tls, uint64_t op_seq) = 0; -- 2.47.3