mds: Add kill points for backtrace testing

To test the mds journal and replay behavior, and the
functionality for storing backtraces on inodes, we
add kill points to the MDS in the openc, journal replay,
and journal expire paths.

Signed-off-by: Sam Lang <sam.lang@inktank.com>
Reviewed-by: Greg Farnum <greg@inktank.com>

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index 39f059570e05a75df3819bdeff59dfceb14e890c..8e87f6f07ae27bed7d48ef7d94492a662230a2c6 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -304,6 +304,10 @@ OPTION(mds_kill_export_at, OPT_INT, 0)
 OPTION(mds_kill_import_at, OPT_INT, 0)
 OPTION(mds_kill_link_at, OPT_INT, 0)
 OPTION(mds_kill_rename_at, OPT_INT, 0)
+OPTION(mds_kill_openc_at, OPT_INT, 0)
+OPTION(mds_kill_journal_at, OPT_INT, 0)
+OPTION(mds_kill_journal_expire_at, OPT_INT, 0)
+OPTION(mds_kill_journal_replay_at, OPT_INT, 0)
 OPTION(mds_inject_traceless_reply_probability, OPT_DOUBLE, 0) /* percentage
                                of MDS modify replies to skip sending the
                                client a trace on [0-1]*/

diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index 4880f9cae3050bb7820bdcee4161e4955408711b..1b9def4b2711b005a8d7cee22f235d4418884208 100644 (file)
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -5905,6 +5905,7 @@ void Server::_rename_finish(MDRequest *mdr, CDentry *srcdn, CDentry *destdn, CDe
 
   // did we import srci?  if so, explicitly ack that import that, before we unlock and reply.
 
+  assert(g_conf->mds_kill_rename_at != 7);
   // backtrace
   if (destdnl->inode->is_dir()) {
     // replace previous backtrace on this inode with myself
@@ -5918,6 +5919,7 @@ void Server::_rename_finish(MDRequest *mdr, CDentry *srcdn, CDentry *destdn, CDe
     // queue an updated backtrace
     mdr->ls->queue_backtrace_update(destdnl->inode, destdnl->inode->inode.layout.fl_pg_pool);
   }
+  assert(g_conf->mds_kill_rename_at != 8);
 
   // reply
   MClientReply *reply = new MClientReply(mdr->client_request, 0);

diff --git a/src/mds/journal.cc b/src/mds/journal.cc
index cdf11a609352422102213e92d7617c2935b6d755..90ea0f28917f8800ae38bc7c083d4ec25b87da55 100644 (file)
--- a/src/mds/journal.cc
+++ b/src/mds/journal.cc
@@ -67,6 +67,8 @@ void LogSegment::try_to_expire(MDS *mds, C_GatherBuilder &gather_bld)
 
   dout(6) << "LogSegment(" << offset << ").try_to_expire" << dendl;
 
+  assert(g_conf->mds_kill_journal_expire_at != 1);
+
   // commit dirs
   for (elist<CDir*>::iterator p = new_dirfrags.begin(); !p.end(); ++p) {
     dout(20) << " new_dirfrag " << **p << dendl;
@@ -133,6 +135,8 @@ void LogSegment::try_to_expire(MDS *mds, C_GatherBuilder &gather_bld)
     mds->locker->scatter_nudge(&in->nestlock, gather_bld.new_sub());
   }
 
+  assert(g_conf->mds_kill_journal_expire_at != 2);
+
   // open files
   if (!open_files.empty()) {
     assert(!mds->mdlog->is_capped()); // hmm FIXME
@@ -178,12 +182,16 @@ void LogSegment::try_to_expire(MDS *mds, C_GatherBuilder &gather_bld)
     }
   }
 
+  assert(g_conf->mds_kill_journal_expire_at != 3);
+
   // backtraces to be stored/updated
   for (elist<BacktraceInfo*>::iterator p = update_backtraces.begin(); !p.end(); ++p) {
     BacktraceInfo *btinfo = *p;
     store_backtrace_update(mds, btinfo, gather_bld.new_sub());
   }
 
+  assert(g_conf->mds_kill_journal_expire_at != 4);
+
   // slave updates
   for (elist<MDSlaveUpdate*>::iterator p = slave_updates.begin(member_offset(MDSlaveUpdate,
                                                                             item));
@@ -254,6 +262,7 @@ void LogSegment::try_to_expire(MDS *mds, C_GatherBuilder &gather_bld)
     dout(6) << "LogSegment(" << offset << ").try_to_expire waiting" << dendl;
     mds->mdlog->flush();
   } else {
+    assert(g_conf->mds_kill_journal_expire_at != 5);
     dout(6) << "LogSegment(" << offset << ").try_to_expire success" << dendl;
   }
 }
@@ -984,6 +993,8 @@ void EMetaBlob::replay(MDS *mds, LogSegment *logseg, MDSlaveUpdate *slaveup)
 
   assert(logseg);
 
+  assert(g_conf->mds_kill_journal_replay_at != 1);
+
   for (list<std::tr1::shared_ptr<fullbit> >::iterator p = roots.begin(); p != roots.end(); p++) {
     CInode *in = mds->mdcache->get_inode((*p)->inode.ino);
     bool isnew = in ? false:true;
@@ -1159,6 +1170,8 @@ void EMetaBlob::replay(MDS *mds, LogSegment *logseg, MDSlaveUpdate *slaveup)
               (in->is_multiversion() && in->first > p->dnfirst));
       }
 
+      assert(g_conf->mds_kill_journal_replay_at != 2);
+
       // store backtrace for allocated inos (create, mkdir, symlink, mknod)
       if (allocated_ino || used_preallocated_ino) {
        if (in->inode.is_dir()) {
@@ -1247,6 +1260,8 @@ void EMetaBlob::replay(MDS *mds, LogSegment *logseg, MDSlaveUpdate *slaveup)
     }
   }
 
+  assert(g_conf->mds_kill_journal_replay_at != 3);
+
   if (renamed_dirino) {
     if (renamed_diri) {
       assert(unlinked.count(renamed_diri));
@@ -1451,6 +1466,8 @@ void EMetaBlob::replay(MDS *mds, LogSegment *logseg, MDSlaveUpdate *slaveup)
 
   // update segment
   update_segment(logseg);
+
+  assert(g_conf->mds_kill_journal_replay_at != 4);
 }
 
 // -----------------------