From: Patrick Donnelly Date: Wed, 18 Jan 2023 02:30:04 +0000 (-0500) Subject: mds: provide test configs for creating first corruption X-Git-Tag: v18.1.0~33^2~5 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=fc1c23f836a2f006a92d701968773abbee47f80a;p=ceph.git mds: provide test configs for creating first corruption Signed-off-by: Patrick Donnelly (cherry picked from commit fe258dc54bbeee5913289cad08634b5ff1d737ba) --- diff --git a/src/common/options/mds.yaml.in b/src/common/options/mds.yaml.in index 49c6acaf029..cf52bed0eb7 100644 --- a/src/common/options/mds.yaml.in +++ b/src/common/options/mds.yaml.in @@ -968,6 +968,24 @@ options: services: - mds fmt_desc: MDS will abort if dentry is detected newly corrupted. +- name: mds_inject_rename_corrupt_dentry_first + type: float + level: dev + default: 0.0 + services: + - mds + fmt_desc: probabilistically inject corrupt CDentry::first at rename + flags: + - runtime +- name: mds_inject_journal_corrupt_dentry_first + type: float + level: dev + default: 0.0 + services: + - mds + fmt_desc: probabilistically inject corrupt CDentry::first at journal load + flags: + - runtime - name: mds_kill_mdstable_at type: int level: dev diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc index d1294c8ce97..e9026bf0ee2 100644 --- a/src/mds/MDSRank.cc +++ b/src/mds/MDSRank.cc @@ -514,6 +514,7 @@ MDSRank::MDSRank( messenger(msgr), monc(monc_), mgrc(mgrc), respawn_hook(respawn_hook_), suicide_hook(suicide_hook_), + inject_journal_corrupt_dentry_first(g_conf().get_val("mds_inject_journal_corrupt_dentry_first")), starttime(mono_clock::now()), ioc(ioc) { @@ -3780,6 +3781,8 @@ const char** MDSRankDispatcher::get_tracked_conf_keys() const "mds_dir_max_entries", "mds_symlink_recovery", "mds_extraordinary_events_dump_interval", + "mds_inject_rename_corrupt_dentry_first", + "mds_inject_journal_corrupt_dentry_first", NULL }; return KEYS; @@ -3844,6 +3847,9 @@ void MDSRankDispatcher::handle_conf_change(const ConfigProxy& conf, const std::s changed.count("fsid")) { update_log_config(); } + if (changed.count("mds_inject_journal_corrupt_dentry_first")) { + inject_journal_corrupt_dentry_first = g_conf().get_val("mds_inject_journal_corrupt_dentry_first"); + } finisher->queue(new LambdaContext([this, changed](int) { std::scoped_lock lock(mds_lock); diff --git a/src/mds/MDSRank.h b/src/mds/MDSRank.h index fb2061eb548..7e147c94c96 100644 --- a/src/mds/MDSRank.h +++ b/src/mds/MDSRank.h @@ -375,6 +375,10 @@ class MDSRank { std::ostream& ss); void schedule_inmemory_logger(); + double get_inject_journal_corrupt_dentry_first() const { + return inject_journal_corrupt_dentry_first; + } + // Reference to global MDS::mds_lock, so that users of MDSRank don't // carry around references to the outer MDS, and we can substitute // a separate lock here in future potentially. @@ -618,6 +622,7 @@ class MDSRank { bool standby_replaying = false; // true if current replay pass is in standby-replay mode uint64_t extraordinary_events_dump_interval = 0; + double inject_journal_corrupt_dentry_first = 0.0; private: bool send_status = true; diff --git a/src/mds/Server.cc b/src/mds/Server.cc index fbac61e0870..a53a5164746 100644 --- a/src/mds/Server.cc +++ b/src/mds/Server.cc @@ -255,6 +255,7 @@ void Server::create_logger() Server::Server(MDSRank *m, MetricsHandler *metrics_handler) : mds(m), mdcache(mds->mdcache), mdlog(mds->mdlog), + inject_rename_corrupt_dentry_first(g_conf().get_val("mds_inject_rename_corrupt_dentry_first")), recall_throttle(g_conf().get_val("mds_recall_max_decay_rate")), metrics_handler(metrics_handler) { @@ -1318,6 +1319,9 @@ void Server::handle_conf_change(const std::set& changed) { dout(20) << __func__ << " max fragment size changed to " << bal_fragment_size_max << dendl; } + if (changed.count("mds_inject_rename_corrupt_dentry_first")) { + inject_rename_corrupt_dentry_first = g_conf().get_val("mds_inject_rename_corrupt_dentry_first"); + } } /* @@ -9482,6 +9486,16 @@ void Server::_rename_prepare(MDRequestRef& mdr, mdcache->journal_cow_dentry(mdr.get(), metablob, destdn, CEPH_NOSNAP, 0, destdnl); destdn->first = mdcache->get_global_snaprealm()->get_newest_seq() + 1; + { + auto do_corruption = inject_rename_corrupt_dentry_first; + if (unlikely(do_corruption > 0.0)) { + auto r = ceph::util::generate_random_number(0.0, 1.0); + if (r < do_corruption) { + dout(0) << "corrupting dn: " << *destdn << dendl; + destdn->first = -10; + } + } + } if (destdn->is_auth()) metablob->add_primary_dentry(destdn, srci, true, true); diff --git a/src/mds/Server.h b/src/mds/Server.h index a2d8f696708..b1b1bcf1c92 100644 --- a/src/mds/Server.h +++ b/src/mds/Server.h @@ -504,6 +504,8 @@ private: uint64_t dir_max_entries = 0; int64_t bal_fragment_size_max = 0; + double inject_rename_corrupt_dentry_first = 0.0; + DecayCounter recall_throttle; time last_recall_state; diff --git a/src/mds/journal.cc b/src/mds/journal.cc index 0059c07c8ea..c24051f3727 100644 --- a/src/mds/journal.cc +++ b/src/mds/journal.cc @@ -1362,6 +1362,17 @@ void EMetaBlob::replay(MDSRank *mds, LogSegment *logseg, MDPeerUpdate *peerup) in->state_clear(CInode::STATE_AUTH); ceph_assert(g_conf()->mds_kill_journal_replay_at != 2); + { + auto do_corruption = mds->get_inject_journal_corrupt_dentry_first(); + if (unlikely(do_corruption > 0.0)) { + auto r = ceph::util::generate_random_number(0.0, 1.0); + if (r < do_corruption) { + dout(0) << "corrupting dn: " << *dn << dendl; + dn->first = -10; + } + } + } + if (!(++count % mds->heartbeat_reset_grace())) mds->heartbeat_reset(); }