mds: provide test configs for creating first corruption

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
(cherry picked from commit fe258dc54bbeee5913289cad08634b5ff1d737ba)

Conflicts:
	src/mds/MDSRank.cc
	src/mds/MDSRank.h
	src/mds/Server.cc

diff --git a/src/common/options.cc b/src/common/options.cc
index d3ced762ea86bf61d8f491363417373e6cb165c4..8b2ff98722815d6b2e1018d8901614cb3ca2e958 100644 (file)
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -8688,6 +8688,16 @@ std::vector<Option> get_mds_options() {
     .set_default(true)
     .set_description("MDS will abort if dentry is detected newly corrupted."),
 
+    Option("mds_inject_rename_corrupt_dentry_first", Option::TYPE_FLOAT, Option::LEVEL_DEV)
+    .set_default(0.0)
+    .set_flag(Option::FLAG_RUNTIME)
+    .set_description("probabilistically inject corrupt CDentry::first at rename"),
+
+    Option("mds_inject_journal_corrupt_dentry_first", Option::TYPE_FLOAT, Option::LEVEL_DEV)
+    .set_default(0.0)
+    .set_flag(Option::FLAG_RUNTIME)
+    .set_description("probabilistically inject corrupt CDentry::first at journal load"),
+
     Option("mds_kill_mdstable_at", Option::TYPE_INT, Option::LEVEL_DEV)
     .set_default(0)
     .set_description(""),

diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index bae7706f6ad2f3c9062dcb78d7d6cdd8fe51f564..4f286af7030f84e39bd0cd7e80b27ddb35a0d297 100644 (file)
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -509,6 +509,7 @@ MDSRank::MDSRank(
     messenger(msgr), monc(monc_), mgrc(mgrc),
     respawn_hook(respawn_hook_),
     suicide_hook(suicide_hook_),
+    inject_journal_corrupt_dentry_first(g_conf().get_val<double>("mds_inject_journal_corrupt_dentry_first")),
     starttime(mono_clock::now()),
     ioc(ioc)
 {
@@ -3769,6 +3770,8 @@ const char** MDSRankDispatcher::get_tracked_conf_keys() const
     "mds_cap_acquisition_throttle_retry_request_time",
     "mds_alternate_name_max",
     "mds_dir_max_entries",
+    "mds_inject_rename_corrupt_dentry_first",
+    "mds_inject_journal_corrupt_dentry_first",
     NULL
   };
   return KEYS;
@@ -3804,6 +3807,9 @@ void MDSRankDispatcher::handle_conf_change(const ConfigProxy& conf, const std::s
       changed.count("fsid")) {
     update_log_config();
   }
+  if (changed.count("mds_inject_journal_corrupt_dentry_first")) {
+    inject_journal_corrupt_dentry_first = g_conf().get_val<double>("mds_inject_journal_corrupt_dentry_first");
+  }
 
   finisher->queue(new LambdaContext([this, changed](int) {
     std::scoped_lock lock(mds_lock);

diff --git a/src/mds/MDSRank.h b/src/mds/MDSRank.h
index 31d9992bed7120add07ae0a039b1fe8ef2aa1432..07f69721f8021bcb8fbc549b5b8f8ddadd5c8c71 100644 (file)
--- a/src/mds/MDSRank.h
+++ b/src/mds/MDSRank.h
@@ -375,6 +375,10 @@ class MDSRank {
                      const std::string& option, const std::string& value,
                      std::ostream& ss);
 
+    double get_inject_journal_corrupt_dentry_first() const {
+      return inject_journal_corrupt_dentry_first;
+    }
+
     // Reference to global MDS::mds_lock, so that users of MDSRank don't
     // carry around references to the outer MDS, and we can substitute
     // a separate lock here in future potentially.
@@ -616,6 +620,7 @@ class MDSRank {
     Context *suicide_hook;
 
     bool standby_replaying = false;  // true if current replay pass is in standby-replay mode
+    double inject_journal_corrupt_dentry_first = 0.0;
 private:
     bool send_status = true;
 

diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index 55e7028e0557bdf6273e4fba09d860071ed34daf..270e17ae063b0acd8a82d808832ef215b70a76e9 100644 (file)
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -252,6 +252,7 @@ void Server::create_logger()
 Server::Server(MDSRank *m, MetricsHandler *metrics_handler) :
   mds(m), 
   mdcache(mds->mdcache), mdlog(mds->mdlog),
+  inject_rename_corrupt_dentry_first(g_conf().get_val<double>("mds_inject_rename_corrupt_dentry_first")),
   recall_throttle(g_conf().get_val<double>("mds_recall_max_decay_rate")),
   metrics_handler(metrics_handler)
 {
@@ -1296,6 +1297,9 @@ void Server::handle_conf_change(const std::set<std::string>& changed) {
     dout(20) << __func__ << " max entries per directory changed to "
             << dir_max_entries << dendl;
   }
+  if (changed.count("mds_inject_rename_corrupt_dentry_first")) {
+    inject_rename_corrupt_dentry_first = g_conf().get_val<double>("mds_inject_rename_corrupt_dentry_first");
+  }
 }
 
 /*
@@ -9375,6 +9379,16 @@ void Server::_rename_prepare(MDRequestRef& mdr,
       mdcache->journal_cow_dentry(mdr.get(), metablob, destdn, CEPH_NOSNAP, 0, destdnl);
 
     destdn->first = mdcache->get_global_snaprealm()->get_newest_seq() + 1;
+    {
+      auto do_corruption = inject_rename_corrupt_dentry_first;
+      if (unlikely(do_corruption > 0.0)) {
+        auto r = ceph::util::generate_random_number(0.0, 1.0);
+        if (r < do_corruption) {
+          dout(0) << "corrupting dn: " << *destdn << dendl;
+          destdn->first = -10;
+        }
+      }
+    }
 
     if (destdn->is_auth())
       metablob->add_primary_dentry(destdn, srci, true, true);

diff --git a/src/mds/Server.h b/src/mds/Server.h
index e4791dc6b721f8fc669611ef705fd0431515a31d..91a01a19ee83a4af8b567a55030d14730f983af7 100644 (file)
--- a/src/mds/Server.h
+++ b/src/mds/Server.h
@@ -501,6 +501,8 @@ private:
   unsigned delegate_inos_pct = 0;
   uint64_t dir_max_entries = 0;
 
+  double inject_rename_corrupt_dentry_first = 0.0;
+
   DecayCounter recall_throttle;
   time last_recall_state;
 

diff --git a/src/mds/journal.cc b/src/mds/journal.cc
index 09c39d15d81173f64c557d09094bd03202f2c442..956c0e08ea97abe1970f3ef72aa532ad216d7019 100644 (file)
--- a/src/mds/journal.cc
+++ b/src/mds/journal.cc
@@ -1355,6 +1355,17 @@ void EMetaBlob::replay(MDSRank *mds, LogSegment *logseg, MDPeerUpdate *peerup)
        in->state_clear(CInode::STATE_AUTH);
       ceph_assert(g_conf()->mds_kill_journal_replay_at != 2);
 
+      {
+        auto do_corruption = mds->get_inject_journal_corrupt_dentry_first();
+        if (unlikely(do_corruption > 0.0)) {
+          auto r = ceph::util::generate_random_number(0.0, 1.0);
+          if (r < do_corruption) {
+            dout(0) << "corrupting dn: " << *dn << dendl;
+            dn->first = -10;
+          }
+        }
+      }
+
       if (!(++count % mds->heartbeat_reset_grace()))
         mds->heartbeat_reset();
     }