mon/Paxos: make backend write async

Move into the WRITING state and do the write to leveldb (or whatever the
backend is) asynchronously.

A few tricks here:
 - we can't do the is_updating() state check because we will always be in
   REFRESH.  Instead, make commit_proposal() tolerate the case where it is
   called but the top proposal isn't the one we just did (or the list is
   empty).  This makes the callers simpler.
 - do_refresh() may call bootstrap.  If we do bootstrap while in REFRESH,
   don't do a sync/flush on the backend store because *we* are async
   completion thread and we'll deadlock.  All other callers need to wait
   for this, though!

Signed-off-by: Sage Weil <sage@redhat.com>

diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc
index 4b357a8af756f5129732eff4566a10026b386c5f..571a8c0ee219437c3f600e2fc5bed4062e45f812 100644 (file)
--- a/src/mon/Monitor.cc
+++ b/src/mon/Monitor.cc
@@ -781,6 +781,12 @@ void Monitor::shutdown()
 
 void Monitor::bootstrap()
 {
+  if (paxos->is_writing()) {
+    dout(10) << "bootstrap flushing pending write" << dendl;
+    lock.Unlock();
+    store->flush();
+    lock.Lock();
+  }
   dout(10) << "bootstrap" << dendl;
 
   sync_reset_requester();

diff --git a/src/mon/Paxos.cc b/src/mon/Paxos.cc
index 4ffbb48812413e8c23e79665cf9b6c4b76ec5690..03058781b6f685161f8f45da657356db2e4c7b03 100644 (file)
--- a/src/mon/Paxos.cc
+++ b/src/mon/Paxos.cc
@@ -801,6 +801,16 @@ void Paxos::accept_timeout()
   mon->bootstrap();
 }
 
+struct C_Committed : public Context {
+  Paxos *paxos;
+  C_Committed(Paxos *p) : paxos(p) {}
+  void finish(int r) {
+    assert(r >= 0);
+    Mutex::Locker l(paxos->mon->lock);
+    paxos->commit_finish();
+  }
+};
+
 void Paxos::commit_start()
 {
   dout(10) << __func__ << " " << (last_committed+1) << dendl;
@@ -827,9 +837,9 @@ void Paxos::commit_start()
   logger->inc(l_paxos_commit_bytes, t->get_bytes());
   commit_start_stamp = ceph_clock_now(NULL);
 
-  get_store()->apply_transaction(t);
+  get_store()->queue_transaction(t, new C_Committed(this));
 
-  commit_finish();
+  state = STATE_WRITING;
 }
 
 void Paxos::commit_finish()
@@ -876,9 +886,14 @@ void Paxos::commit_finish()
 
   remove_legacy_versions();
 
+  // WRITING -> REFRESH
+  // among other things, this lets do_refresh() -> mon->bootstrap() know
+  // it doesn't need to flush the store queue
+  assert(state == STATE_WRITING);
+  state = STATE_REFRESH;
+
   if (do_refresh()) {
-    if (is_updating())
-      commit_proposal();
+    commit_proposal();
 
     finish_contexts(g_ceph_context, waiting_for_commit);
 
@@ -1010,16 +1025,22 @@ void Paxos::commit_proposal()
 {
   dout(10) << __func__ << dendl;
   assert(mon->is_leader());
-  assert(!proposals.empty());
-  assert(is_updating());
+  assert(is_refresh());
+
+  if (proposals.empty()) {
+    return;  // must have been updating previous
+  }
 
   C_Proposal *proposal = static_cast<C_Proposal*>(proposals.front());
-  assert(proposal->proposed);
-  dout(10) << __func__ << " proposal " << proposal << " took "
-          << (ceph_clock_now(NULL) - proposal->proposal_time)
-          << " to finish" << dendl;
-  proposals.pop_front();
-  proposal->complete(0);
+  if (proposal->proposed) {
+    dout(10) << __func__ << " proposal " << proposal << " took "
+            << (ceph_clock_now(NULL) - proposal->proposal_time)
+            << " to finish" << dendl;
+    proposals.pop_front();
+    proposal->complete(0);
+  } else {
+    // must have been updating previous.
+  }
 }
 
 void Paxos::finish_round()

diff --git a/src/mon/Paxos.h b/src/mon/Paxos.h
index 5a3fdbf13a3453d0e6ba444f79d3f7e05afe1bd0..aa591de7658e0e5dd35d98f0b4361dce5b2e38eb 100644 (file)
--- a/src/mon/Paxos.h
+++ b/src/mon/Paxos.h
@@ -889,6 +889,7 @@ private:
 
 
   utime_t commit_start_stamp;
+  friend struct C_Committed;
 
   /**
    * Commit a value throughout the system.