mon/Paxos: add failure injection points

Signed-off-by: Sage Weil <sage@inktank.com>

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index defb71ee514c8d03eb36cf56d08d5301fb57c38a..fabb8ec689da624d3b00b78b2b9291eed56c98d8 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -207,6 +207,7 @@ OPTION(paxos_trim_min, OPT_INT, 250)  // number of extra proposals tolerated bef
 OPTION(paxos_trim_max, OPT_INT, 500) // max number of extra proposals to trim at a time
 OPTION(paxos_service_trim_min, OPT_INT, 250) // minimum amount of versions to trigger a trim (0 disables it)
 OPTION(paxos_service_trim_max, OPT_INT, 500) // maximum amount of versions to trim during a single proposal (0 disables it)
+OPTION(paxos_kill_at, OPT_INT, 0)
 OPTION(clock_offset, OPT_DOUBLE, 0) // how much to offset the system clock in Clock.cc
 OPTION(auth_cluster_required, OPT_STR, "cephx")   // required of mon, mds, osd daemons
 OPTION(auth_service_required, OPT_STR, "cephx")   // required by daemons of clients

diff --git a/src/mon/Paxos.cc b/src/mon/Paxos.cc
index 2a9f547ee478b07a6dfc412d148f8b3d9b5ed4c3..508669deef542ef2c852ab104eb876613a01ef1a 100644 (file)
--- a/src/mon/Paxos.cc
+++ b/src/mon/Paxos.cc
@@ -394,9 +394,13 @@ void Paxos::handle_last(MMonPaxos *last)
     return;
   }
 
+  assert(g_conf->paxos_kill_at != 1);
+
   // store any committed values if any are specified in the message
   store_state(last);
 
+  assert(g_conf->paxos_kill_at != 2);
+
   // do they accept your pn?
   if (last->pn > accepted_pn) {
     // no, try again.
@@ -552,6 +556,8 @@ void Paxos::begin(bufferlist& v)
 
   get_store()->apply_transaction(t);
 
+  assert(g_conf->paxos_kill_at != 3);
+
   if (mon->get_quorum().size() == 1) {
     // we're alone, take it easy
     commit();
@@ -602,6 +608,8 @@ void Paxos::handle_begin(MMonPaxos *begin)
   assert(begin->pn == accepted_pn);
   assert(begin->last_committed == last_committed);
   
+  assert(g_conf->paxos_kill_at != 4);
+
   // set state.
   state = STATE_UPDATING;
   lease_expire = utime_t();  // cancel lease
@@ -626,6 +634,8 @@ void Paxos::handle_begin(MMonPaxos *begin)
 
   get_store()->apply_transaction(t);
 
+  assert(g_conf->paxos_kill_at != 5);
+
   // reply
   MMonPaxos *accept = new MMonPaxos(mon->get_epoch(), MMonPaxos::OP_ACCEPT,
                                    ceph_clock_now(g_ceph_context));
@@ -660,6 +670,8 @@ void Paxos::handle_accept(MMonPaxos *accept)
   accepted.insert(from);
   dout(10) << " now " << accepted << " have accepted" << dendl;
 
+  assert(g_conf->paxos_kill_at != 6);
+
   // new majority?
   if (accepted.size() == (unsigned)mon->monmap->size()/2+1) {
     // yay, commit!
@@ -683,6 +695,8 @@ void Paxos::handle_accept(MMonPaxos *accept)
     // yay!
     extend_lease();
 
+    assert(g_conf->paxos_kill_at != 10);
+
     finish_round();
 
     // wake people up
@@ -713,6 +727,8 @@ void Paxos::commit()
   //   leader still got a majority and committed with out us.)
   lease_expire = utime_t();  // cancel lease
 
+  assert(g_conf->paxos_kill_at != 7);
+
   MonitorDBStore::Transaction t;
 
   // commit locally
@@ -732,6 +748,8 @@ void Paxos::commit()
 
   get_store()->apply_transaction(t);
 
+  assert(g_conf->paxos_kill_at != 8);
+
   // refresh first_committed; this txn may have trimmed.
   first_committed = get_store()->get(get_name(), "first_committed");
 
@@ -753,6 +771,8 @@ void Paxos::commit()
     mon->messenger->send_message(commit, mon->monmap->get_inst(*p));
   }
 
+  assert(g_conf->paxos_kill_at != 9);
+
   // get ready for a new round.
   new_value.clear();