mon/Paxos: assert that the store gives us back what we just wrote

In bug #5424 I observed leveldb failing internally and then returning
bad info.  We then hit a random/confusing assert.  Try to detect this
earlier by verifying that a get of a just-written last_committed gives
us back the right thing.

Signed-off-by: Sage Weil <sage@inktank.com>
Reviewed-by: Joao Eduardo Luis <joao.luis@inktank.com>

diff --git a/src/mon/Paxos.cc b/src/mon/Paxos.cc
index 2a324c97236d05af0fd2873fa7dd1a75b4d4032c..4ccf52635ad1a6cefdf52547329a7ee2a341b442 100644 (file)
--- a/src/mon/Paxos.cc
+++ b/src/mon/Paxos.cc
@@ -327,6 +327,7 @@ void Paxos::store_state(MMonPaxos *m)
 
     get_store()->apply_transaction(t);
 
+    _sanity_check_store();
   }
 
   if (get_store()->exists(get_name(), "conversion_first")) {
@@ -342,6 +343,13 @@ void Paxos::store_state(MMonPaxos *m)
 
 }
 
+void Paxos::_sanity_check_store()
+{
+  version_t lc = get_store()->get(get_name(), "last_committed");
+  assert(lc == last_committed);
+}
+
+
 // leader
 void Paxos::handle_last(MMonPaxos *last)
 {
@@ -671,6 +679,8 @@ void Paxos::commit()
 
   get_store()->apply_transaction(t);
 
+  _sanity_check_store();
+
   // tell everyone
   for (set<int>::const_iterator p = mon->get_quorum().begin();
        p != mon->get_quorum().end();

diff --git a/src/mon/Paxos.h b/src/mon/Paxos.h
index 4f1af82836e5f82a2f98ddc5a9e25a010f18c48b..fcc0660402419f63db3893aec4197f29bb9499ef 100644 (file)
--- a/src/mon/Paxos.h
+++ b/src/mon/Paxos.h
@@ -1086,6 +1086,7 @@ public:
    * @param m A message
    */
   void store_state(MMonPaxos *m);
+  void _sanity_check_store();
   /**
    * Helper function to decode a bufferlist into a transaction and append it
    * to another transaction.