FileStore:: queue_transactions, fine-grain submitManager lock

there is no need to lock the code, which is dealing with encoding tls to tbl. The submitManager lock is used to make sure the sequencial submission for op. So encoding transaction data out of lock.

Signed-off-by: Ning Yao <yaoning@ruijie.com.cn>

diff --git a/src/os/FileStore.cc b/src/os/FileStore.cc
index f6c3bb872a2ea0ad2be85c72a3a9b25f24a670a2..825ce5d2883a51fa320c44bda01e033010252c79 100644 (file)
--- a/src/os/FileStore.cc
+++ b/src/os/FileStore.cc
@@ -1886,6 +1886,9 @@ int FileStore::queue_transactions(Sequencer *posr, list<Transaction*> &tls,
     Op *o = build_op(tls, onreadable, onreadable_sync, osd_op);
     op_queue_reserve_throttle(o, handle);
     journal->throttle();
+    //prepare and encode transactions data out of lock
+    bufferlist tbl;
+    int data_align = _op_journal_transactions_prepare(o->tls, tbl);
     uint64_t op_num = submit_manager.op_submit_start();
     o->op = op_num;
 
@@ -1895,7 +1898,7 @@ int FileStore::queue_transactions(Sequencer *posr, list<Transaction*> &tls,
     if (m_filestore_journal_parallel) {
       dout(5) << "queue_transactions (parallel) " << o->op << " " << o->tls << dendl;
       
-      _op_journal_transactions(o->tls, o->op, ondisk, osd_op);
+      _op_journal_transactions(tbl, data_align, o->op, ondisk, osd_op);
       
       // queue inside submit_manager op submission lock
       queue_op(osr, o);
@@ -1904,7 +1907,7 @@ int FileStore::queue_transactions(Sequencer *posr, list<Transaction*> &tls,
       
       osr->queue_journal(o->op);
 
-      _op_journal_transactions(o->tls, o->op,
+      _op_journal_transactions(tbl, data_align, o->op,
                               new C_JournaledAhead(this, osr, o, ondisk),
                               osd_op);
     } else {
@@ -1934,6 +1937,10 @@ int FileStore::queue_transactions(Sequencer *posr, list<Transaction*> &tls,
     return 0;
   }
 
+
+  //prepare and encode transactions data out of lock
+  bufferlist tbl;
+  int data_align = _op_journal_transactions_prepare(tls, tbl);
   uint64_t op = submit_manager.op_submit_start();
   dout(5) << "queue_transactions (trailing journal) " << op << " " << tls << dendl;
 
@@ -1944,7 +1951,7 @@ int FileStore::queue_transactions(Sequencer *posr, list<Transaction*> &tls,
   int r = do_transactions(tls, op);
     
   if (r >= 0) {
-    _op_journal_transactions(tls, op, ondisk, osd_op);
+    _op_journal_transactions(tbl, data_align, op, ondisk, osd_op);
   } else {
     delete ondisk;
   }

diff --git a/src/os/JournalingObjectStore.cc b/src/os/JournalingObjectStore.cc
index 518d54edf6621ee2ef274499f664f3c865dbff7b..15a7c7303c43caf24a0f22d9074855dbe389a689 100644 (file)
--- a/src/os/JournalingObjectStore.cc
+++ b/src/os/JournalingObjectStore.cc
@@ -251,27 +251,32 @@ void JournalingObjectStore::ApplyManager::commit_finish()
 }
 
 void JournalingObjectStore::_op_journal_transactions(
-  list<ObjectStore::Transaction*>& tls, uint64_t op,
+  bufferlist& tbl, int data_align,  uint64_t op,
   Context *onjournal, TrackedOpRef osd_op)
 {
-  dout(10) << "op_journal_transactions " << op << " " << tls << dendl;
-
+  dout(10) << "op_journal_transactions " << op << dendl;
   if (journal && journal->is_writeable()) {
-    bufferlist tbl;
-    unsigned data_len = 0;
-    int data_align = -1; // -1 indicates that we don't care about the alignment
-    for (list<ObjectStore::Transaction*>::iterator p = tls.begin();
-        p != tls.end(); ++p) {
-      ObjectStore::Transaction *t = *p;
-      if (t->get_data_length() > data_len &&
-       (int)t->get_data_length() >= g_conf->journal_align_min_size) {
-       data_len = t->get_data_length();
-       data_align = (t->get_data_alignment() - tbl.length()) & ~CEPH_PAGE_MASK;
-      }
-      ::encode(*t, tbl);
-    }
     journal->submit_entry(op, tbl, data_align, onjournal, osd_op);
   } else if (onjournal) {
     apply_manager.add_waiter(op, onjournal);
   }
 }
+
+int JournalingObjectStore::_op_journal_transactions_prepare(
+  list<ObjectStore::Transaction*>& tls, bufferlist& tbl)
+{
+  dout(10) << "_op_journal_transactions_prepare " << tls << dendl;
+  unsigned data_len = 0;
+  int data_align = -1; // -1 indicates that we don't care about the alignment
+  for (list<ObjectStore::Transaction*>::iterator p = tls.begin();
+      p != tls.end(); ++p) {
+    ObjectStore::Transaction *t = *p;
+    if (t->get_data_length() > data_len &&
+     (int)t->get_data_length() >= g_conf->journal_align_min_size) {
+     data_len = t->get_data_length();
+     data_align = (t->get_data_alignment() - tbl.length()) & ~CEPH_PAGE_MASK;
+    }
+    ::encode(*t, tbl);
+  }
+  return data_align;
+}

diff --git a/src/os/JournalingObjectStore.h b/src/os/JournalingObjectStore.h
index 6a09c6133de90eb9d8b6dc11b930cac21d3e993e..fbfa20ce0f1f53207940c2b602726af74c869e8b 100644 (file)
--- a/src/os/JournalingObjectStore.h
+++ b/src/os/JournalingObjectStore.h
@@ -114,7 +114,9 @@ protected:
   void journal_write_close();
   int journal_replay(uint64_t fs_op_seq);
 
-  void _op_journal_transactions(list<ObjectStore::Transaction*>& tls, uint64_t op,
+  int _op_journal_transactions_prepare(
+    list<ObjectStore::Transaction*>& tls, bufferlist& tbl);
+  void _op_journal_transactions(bufferlist& tls, int data_align, uint64_t op,
                                Context *onjournal, TrackedOpRef osd_op);
 
   virtual int do_transactions(list<ObjectStore::Transaction*>& tls, uint64_t op_seq) = 0;