filestore: assert op_submit_finish is called in order

Verify/assert that we aren't screwing up the submission pipeline ordering.
Namely, we want to make sure that if op_apply_start() blocks, we wake up
in the proper order and don't screw up the journaling.

Signed-off-by: Sage Weil <sage@newdream.net>

diff --git a/src/os/FileStore.cc b/src/os/FileStore.cc
index 2d90a8cbd09c2a638a2f7ac8b388a56d9ec1ea8f..840564d2a9bb89bed69e4ecd9fa11eba2d55868d 100644 (file)
--- a/src/os/FileStore.cc
+++ b/src/os/FileStore.cc
@@ -1401,7 +1401,7 @@ int FileStore::queue_transactions(Sequencer *osr, list<Transaction*> &tls,
       // queue inside journal lock, to preserve ordering
       queue_op(osr, op, tls, onreadable, onreadable_sync);
       
-      op_submit_finish();
+      op_submit_finish(op);
       return 0;
     }
     else if (g_conf.filestore_journal_writeahead) {
@@ -1413,18 +1413,19 @@ int FileStore::queue_transactions(Sequencer *osr, list<Transaction*> &tls,
       dout(10) << "queue_transactions (writeahead) " << op << " " << tls << dendl;
       _op_journal_transactions(tls, op,
                               new C_JournaledAhead(this, osr, op, tls, onreadable, ondisk, onreadable_sync));
-      op_submit_finish();
+      op_submit_finish(op);
       return 0;
     }
   }
 
-  uint64_t op_seq = op_apply_start(0);
-  dout(10) << "queue_transactions (trailing journal) " << op_seq << " " << tls << dendl;
-  int r = do_transactions(tls, op_seq);
-  op_apply_finish(op_seq);
+  uint64_t op = op_submit_start();
+  dout(10) << "queue_transactions (trailing journal) " << op << " " << tls << dendl;
+
+  _op_apply_start(op);
+  int r = do_transactions(tls, op);
     
   if (r >= 0) {
-    op_journal_transactions(tls, op_seq, ondisk);
+    op_journal_transactions(tls, op, ondisk);
   } else {
     delete ondisk;
   }
@@ -1437,6 +1438,9 @@ int FileStore::queue_transactions(Sequencer *osr, list<Transaction*> &tls,
   }
   op_finisher.queue(onreadable, r);
 
+  op_submit_finish(op);
+  op_apply_finish(op);
+
   return r;
 }
 

diff --git a/src/os/JournalingObjectStore.cc b/src/os/JournalingObjectStore.cc
index 747e9d711a146326ad79c74b09b0350c72a55a97..2b48529fb87c436c4c8f25d04bd34736ee5605f3 100644 (file)
--- a/src/os/JournalingObjectStore.cc
+++ b/src/os/JournalingObjectStore.cc
@@ -132,11 +132,20 @@ void JournalingObjectStore::op_apply_finish(uint64_t op)
 uint64_t JournalingObjectStore::op_submit_start()
 {
   journal_lock.Lock();
-  return ++op_seq;
+  uint64_t op = ++op_seq;
+  dout(10) << "op_submit_start " << op << dendl;
+  ops_submitting.push_back(op);
+  return op;
 }
 
-void JournalingObjectStore::op_submit_finish()
+void JournalingObjectStore::op_submit_finish(uint64_t op)
 {
+  dout(10) << "op_submit_finish " << op << dendl;
+  if (op != ops_submitting.front()) {
+    dout(0) << "op_submit_finish " << op << " expected " << ops_submitting.front()
+           << ", OUT OF ORDER" << dendl;
+  }
+  ops_submitting.pop_front();
   journal_lock.Unlock();
 }
 

diff --git a/src/os/JournalingObjectStore.h b/src/os/JournalingObjectStore.h
index bca6d48eb59e9b5705845b3e0d1bd42281df9111..a074943a973ecac651041e2821247e08eb486b9b 100644 (file)
--- a/src/os/JournalingObjectStore.h
+++ b/src/os/JournalingObjectStore.h
@@ -34,6 +34,8 @@ protected:
   Cond cond;
   Mutex journal_lock;
 
+  list<uint64_t> ops_submitting;
+
 protected:
   void journal_start();
   void journal_stop();
@@ -41,7 +43,7 @@ protected:
 
   // --
   uint64_t op_submit_start();
-  void op_submit_finish();
+  void op_submit_finish(uint64_t op_seq);
 
   uint64_t op_apply_start(uint64_t op);
   uint64_t _op_apply_start(uint64_t op);