mds: provide test configs for creating first corruption

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
(cherry picked from commit fe258dc54bbeee5913289cad08634b5ff1d737ba)

diff --git a/src/common/options/mds.yaml.in b/src/common/options/mds.yaml.in
index 49c6acaf0291850eef8e54db23657847bb853562..cf52bed0eb77de9789eb3fdeba85a88d52e66e8d 100644 (file)
--- a/src/common/options/mds.yaml.in
+++ b/src/common/options/mds.yaml.in
@@ -968,6 +968,24 @@ options:
   services:
   - mds
   fmt_desc: MDS will abort if dentry is detected newly corrupted.
+- name: mds_inject_rename_corrupt_dentry_first
+  type: float
+  level: dev
+  default: 0.0
+  services:
+  - mds
+  fmt_desc: probabilistically inject corrupt CDentry::first at rename
+  flags:
+  - runtime
+- name: mds_inject_journal_corrupt_dentry_first
+  type: float
+  level: dev
+  default: 0.0
+  services:
+  - mds
+  fmt_desc: probabilistically inject corrupt CDentry::first at journal load
+  flags:
+  - runtime
 - name: mds_kill_mdstable_at
   type: int
   level: dev

diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index d1294c8ce97103412f92d96552b91e14271525bc..e9026bf0ee28f0038cc605587cb207958a81db78 100644 (file)
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -514,6 +514,7 @@ MDSRank::MDSRank(
     messenger(msgr), monc(monc_), mgrc(mgrc),
     respawn_hook(respawn_hook_),
     suicide_hook(suicide_hook_),
+    inject_journal_corrupt_dentry_first(g_conf().get_val<double>("mds_inject_journal_corrupt_dentry_first")),
     starttime(mono_clock::now()),
     ioc(ioc)
 {
@@ -3780,6 +3781,8 @@ const char** MDSRankDispatcher::get_tracked_conf_keys() const
     "mds_dir_max_entries",
     "mds_symlink_recovery",
     "mds_extraordinary_events_dump_interval",
+    "mds_inject_rename_corrupt_dentry_first",
+    "mds_inject_journal_corrupt_dentry_first",
     NULL
   };
   return KEYS;
@@ -3844,6 +3847,9 @@ void MDSRankDispatcher::handle_conf_change(const ConfigProxy& conf, const std::s
       changed.count("fsid")) {
     update_log_config();
   }
+  if (changed.count("mds_inject_journal_corrupt_dentry_first")) {
+    inject_journal_corrupt_dentry_first = g_conf().get_val<double>("mds_inject_journal_corrupt_dentry_first");
+  }
 
   finisher->queue(new LambdaContext([this, changed](int) {
     std::scoped_lock lock(mds_lock);

diff --git a/src/mds/MDSRank.h b/src/mds/MDSRank.h
index fb2061eb548ab06168715920bc83fa121de35305..7e147c94c96e04c57231b1672bf695f8ce1803c7 100644 (file)
--- a/src/mds/MDSRank.h
+++ b/src/mds/MDSRank.h
@@ -375,6 +375,10 @@ class MDSRank {
                      std::ostream& ss);
     void schedule_inmemory_logger();
 
+    double get_inject_journal_corrupt_dentry_first() const {
+      return inject_journal_corrupt_dentry_first;
+    }
+
     // Reference to global MDS::mds_lock, so that users of MDSRank don't
     // carry around references to the outer MDS, and we can substitute
     // a separate lock here in future potentially.
@@ -618,6 +622,7 @@ class MDSRank {
 
     bool standby_replaying = false;  // true if current replay pass is in standby-replay mode
     uint64_t extraordinary_events_dump_interval = 0;
+    double inject_journal_corrupt_dentry_first = 0.0;
 private:
     bool send_status = true;
 

diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index fbac61e0870514881fca4b3d76ecf13eae290cb1..a53a51647462dd7c447f0cbc8b80c40f03021f60 100644 (file)
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -255,6 +255,7 @@ void Server::create_logger()
 Server::Server(MDSRank *m, MetricsHandler *metrics_handler) :
   mds(m), 
   mdcache(mds->mdcache), mdlog(mds->mdlog),
+  inject_rename_corrupt_dentry_first(g_conf().get_val<double>("mds_inject_rename_corrupt_dentry_first")),
   recall_throttle(g_conf().get_val<double>("mds_recall_max_decay_rate")),
   metrics_handler(metrics_handler)
 {
@@ -1318,6 +1319,9 @@ void Server::handle_conf_change(const std::set<std::string>& changed) {
     dout(20) << __func__ << " max fragment size changed to "
             << bal_fragment_size_max << dendl;
   }
+  if (changed.count("mds_inject_rename_corrupt_dentry_first")) {
+    inject_rename_corrupt_dentry_first = g_conf().get_val<double>("mds_inject_rename_corrupt_dentry_first");
+  }
 }
 
 /*
@@ -9482,6 +9486,16 @@ void Server::_rename_prepare(MDRequestRef& mdr,
       mdcache->journal_cow_dentry(mdr.get(), metablob, destdn, CEPH_NOSNAP, 0, destdnl);
 
     destdn->first = mdcache->get_global_snaprealm()->get_newest_seq() + 1;
+    {
+      auto do_corruption = inject_rename_corrupt_dentry_first;
+      if (unlikely(do_corruption > 0.0)) {
+        auto r = ceph::util::generate_random_number(0.0, 1.0);
+        if (r < do_corruption) {
+          dout(0) << "corrupting dn: " << *destdn << dendl;
+          destdn->first = -10;
+        }
+      }
+    }
 
     if (destdn->is_auth())
       metablob->add_primary_dentry(destdn, srci, true, true);

diff --git a/src/mds/Server.h b/src/mds/Server.h
index a2d8f69670871d043fb824fa10d406124aa6bc68..b1b1bcf1c92762fa0e839a086eb1998c0b6926d3 100644 (file)
--- a/src/mds/Server.h
+++ b/src/mds/Server.h
@@ -504,6 +504,8 @@ private:
   uint64_t dir_max_entries = 0;
   int64_t bal_fragment_size_max = 0;
 
+  double inject_rename_corrupt_dentry_first = 0.0;
+
   DecayCounter recall_throttle;
   time last_recall_state;
 

diff --git a/src/mds/journal.cc b/src/mds/journal.cc
index 0059c07c8ea7fdf53636f6bca5e3f10c8cff68a1..c24051f3727f74c5f3d718e5a8a2f62bbc64538c 100644 (file)
--- a/src/mds/journal.cc
+++ b/src/mds/journal.cc
@@ -1362,6 +1362,17 @@ void EMetaBlob::replay(MDSRank *mds, LogSegment *logseg, MDPeerUpdate *peerup)
        in->state_clear(CInode::STATE_AUTH);
       ceph_assert(g_conf()->mds_kill_journal_replay_at != 2);
 
+      {
+        auto do_corruption = mds->get_inject_journal_corrupt_dentry_first();
+        if (unlikely(do_corruption > 0.0)) {
+          auto r = ceph::util::generate_random_number(0.0, 1.0);
+          if (r < do_corruption) {
+            dout(0) << "corrupting dn: " << *dn << dendl;
+            dn->first = -10;
+          }
+        }
+      }
+
       if (!(++count % mds->heartbeat_reset_grace()))
         mds->heartbeat_reset();
     }