mds: provide test configs for creating first corruption

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>

diff --git a/src/common/options/mds.yaml.in b/src/common/options/mds.yaml.in
index bb468a5a939619ea2b6b255b12e670195e2209ba..cbbc00e41ed78999a3acdfb1cfe46812c4fbea86 100644 (file)
--- a/src/common/options/mds.yaml.in
+++ b/src/common/options/mds.yaml.in
@@ -959,6 +959,24 @@ options:
   services:
   - mds
   fmt_desc: MDS will abort if dentry is detected newly corrupted.
+- name: mds_inject_rename_corrupt_dentry_first
+  type: float
+  level: dev
+  default: 0.0
+  services:
+  - mds
+  fmt_desc: probabilistically inject corrupt CDentry::first at rename
+  flags:
+  - runtime
+- name: mds_inject_journal_corrupt_dentry_first
+  type: float
+  level: dev
+  default: 0.0
+  services:
+  - mds
+  fmt_desc: probabilistically inject corrupt CDentry::first at journal load
+  flags:
+  - runtime
 - name: mds_kill_mdstable_at
   type: int
   level: dev

diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index 888a0062e2043e3bc737260f1dd87181fe1045f8..0cef68b5e98e6b486d355c6e3077618305f0ebd9 100644 (file)
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -514,6 +514,7 @@ MDSRank::MDSRank(
     messenger(msgr), monc(monc_), mgrc(mgrc),
     respawn_hook(respawn_hook_),
     suicide_hook(suicide_hook_),
+    inject_journal_corrupt_dentry_first(g_conf().get_val<double>("mds_inject_journal_corrupt_dentry_first")),
     starttime(mono_clock::now()),
     ioc(ioc)
 {
@@ -3812,6 +3813,8 @@ const char** MDSRankDispatcher::get_tracked_conf_keys() const
     "mds_dir_max_entries",
     "mds_symlink_recovery",
     "mds_extraordinary_events_dump_interval",
+    "mds_inject_rename_corrupt_dentry_first",
+    "mds_inject_journal_corrupt_dentry_first",
     NULL
   };
   return KEYS;
@@ -3876,6 +3879,9 @@ void MDSRankDispatcher::handle_conf_change(const ConfigProxy& conf, const std::s
       changed.count("fsid")) {
     update_log_config();
   }
+  if (changed.count("mds_inject_journal_corrupt_dentry_first")) {
+    inject_journal_corrupt_dentry_first = g_conf().get_val<double>("mds_inject_journal_corrupt_dentry_first");
+  }
 
   finisher->queue(new LambdaContext([this, changed](int) {
     std::scoped_lock lock(mds_lock);

diff --git a/src/mds/MDSRank.h b/src/mds/MDSRank.h
index d28933c6aaf12b9782d2bd5c7ea99d8337d3fe51..c00af5fe084187aebda03b40f59ab9ea2d83311a 100644 (file)
--- a/src/mds/MDSRank.h
+++ b/src/mds/MDSRank.h
@@ -375,6 +375,10 @@ class MDSRank {
                      std::ostream& ss);
     void schedule_inmemory_logger();
 
+    double get_inject_journal_corrupt_dentry_first() const {
+      return inject_journal_corrupt_dentry_first;
+    }
+
     // Reference to global MDS::mds_lock, so that users of MDSRank don't
     // carry around references to the outer MDS, and we can substitute
     // a separate lock here in future potentially.
@@ -619,6 +623,7 @@ class MDSRank {
 
     bool standby_replaying = false;  // true if current replay pass is in standby-replay mode
     uint64_t extraordinary_events_dump_interval = 0;
+    double inject_journal_corrupt_dentry_first = 0.0;
 private:
     bool send_status = true;
 

diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index 55d2932fcfebf855cd7244d0ffa12e13212aa3f6..786e05620bca8437307d4818cad2c00d9824def4 100644 (file)
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -255,6 +255,7 @@ void Server::create_logger()
 Server::Server(MDSRank *m, MetricsHandler *metrics_handler) :
   mds(m), 
   mdcache(mds->mdcache), mdlog(mds->mdlog),
+  inject_rename_corrupt_dentry_first(g_conf().get_val<double>("mds_inject_rename_corrupt_dentry_first")),
   recall_throttle(g_conf().get_val<double>("mds_recall_max_decay_rate")),
   metrics_handler(metrics_handler)
 {
@@ -1318,6 +1319,9 @@ void Server::handle_conf_change(const std::set<std::string>& changed) {
     dout(20) << __func__ << " max fragment size changed to "
             << bal_fragment_size_max << dendl;
   }
+  if (changed.count("mds_inject_rename_corrupt_dentry_first")) {
+    inject_rename_corrupt_dentry_first = g_conf().get_val<double>("mds_inject_rename_corrupt_dentry_first");
+  }
 }
 
 /*
@@ -9486,6 +9490,16 @@ void Server::_rename_prepare(MDRequestRef& mdr,
       mdcache->journal_cow_dentry(mdr.get(), metablob, destdn, CEPH_NOSNAP, 0, destdnl);
 
     destdn->first = mdcache->get_global_snaprealm()->get_newest_seq() + 1;
+    {
+      auto do_corruption = inject_rename_corrupt_dentry_first;
+      if (unlikely(do_corruption > 0.0)) {
+        auto r = ceph::util::generate_random_number(0.0, 1.0);
+        if (r < do_corruption) {
+          dout(0) << "corrupting dn: " << *destdn << dendl;
+          destdn->first = -10;
+        }
+      }
+    }
 
     if (destdn->is_auth())
       metablob->add_primary_dentry(destdn, srci, true, true);

diff --git a/src/mds/Server.h b/src/mds/Server.h
index f955e5f942b93c89431348e1a0928225ce602f6f..437a3517e0c32e1d09d72c2763bbef3c58527629 100644 (file)
--- a/src/mds/Server.h
+++ b/src/mds/Server.h
@@ -507,6 +507,8 @@ private:
   uint64_t dir_max_entries = 0;
   int64_t bal_fragment_size_max = 0;
 
+  double inject_rename_corrupt_dentry_first = 0.0;
+
   DecayCounter recall_throttle;
   time last_recall_state;
 

diff --git a/src/mds/journal.cc b/src/mds/journal.cc
index f8e3d802d4112a37b383a3a03edf16fdcf76946a..b586d2fa98da2c1045c13869536137a3497532b7 100644 (file)
--- a/src/mds/journal.cc
+++ b/src/mds/journal.cc
@@ -1362,6 +1362,17 @@ void EMetaBlob::replay(MDSRank *mds, LogSegment *logseg, MDPeerUpdate *peerup)
        in->state_clear(CInode::STATE_AUTH);
       ceph_assert(g_conf()->mds_kill_journal_replay_at != 2);
 
+      {
+        auto do_corruption = mds->get_inject_journal_corrupt_dentry_first();
+        if (unlikely(do_corruption > 0.0)) {
+          auto r = ceph::util::generate_random_number(0.0, 1.0);
+          if (r < do_corruption) {
+            dout(0) << "corrupting dn: " << *dn << dendl;
+            dn->first = -10;
+          }
+        }
+      }
+
       if (!(++count % mds->heartbeat_reset_grace()))
         mds->heartbeat_reset();
     }