From: Patrick Donnelly Date: Wed, 18 Jan 2023 02:30:04 +0000 (-0500) Subject: mds: provide test configs for creating first corruption X-Git-Tag: v19.0.0~1492^2~5 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=fe258dc54bbeee5913289cad08634b5ff1d737ba;p=ceph.git mds: provide test configs for creating first corruption Signed-off-by: Patrick Donnelly --- diff --git a/src/common/options/mds.yaml.in b/src/common/options/mds.yaml.in index bb468a5a9396..cbbc00e41ed7 100644 --- a/src/common/options/mds.yaml.in +++ b/src/common/options/mds.yaml.in @@ -959,6 +959,24 @@ options: services: - mds fmt_desc: MDS will abort if dentry is detected newly corrupted. +- name: mds_inject_rename_corrupt_dentry_first + type: float + level: dev + default: 0.0 + services: + - mds + fmt_desc: probabilistically inject corrupt CDentry::first at rename + flags: + - runtime +- name: mds_inject_journal_corrupt_dentry_first + type: float + level: dev + default: 0.0 + services: + - mds + fmt_desc: probabilistically inject corrupt CDentry::first at journal load + flags: + - runtime - name: mds_kill_mdstable_at type: int level: dev diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc index 888a0062e204..0cef68b5e98e 100644 --- a/src/mds/MDSRank.cc +++ b/src/mds/MDSRank.cc @@ -514,6 +514,7 @@ MDSRank::MDSRank( messenger(msgr), monc(monc_), mgrc(mgrc), respawn_hook(respawn_hook_), suicide_hook(suicide_hook_), + inject_journal_corrupt_dentry_first(g_conf().get_val("mds_inject_journal_corrupt_dentry_first")), starttime(mono_clock::now()), ioc(ioc) { @@ -3812,6 +3813,8 @@ const char** MDSRankDispatcher::get_tracked_conf_keys() const "mds_dir_max_entries", "mds_symlink_recovery", "mds_extraordinary_events_dump_interval", + "mds_inject_rename_corrupt_dentry_first", + "mds_inject_journal_corrupt_dentry_first", NULL }; return KEYS; @@ -3876,6 +3879,9 @@ void MDSRankDispatcher::handle_conf_change(const ConfigProxy& conf, const std::s changed.count("fsid")) { update_log_config(); } + if (changed.count("mds_inject_journal_corrupt_dentry_first")) { + inject_journal_corrupt_dentry_first = g_conf().get_val("mds_inject_journal_corrupt_dentry_first"); + } finisher->queue(new LambdaContext([this, changed](int) { std::scoped_lock lock(mds_lock); diff --git a/src/mds/MDSRank.h b/src/mds/MDSRank.h index d28933c6aaf1..c00af5fe0841 100644 --- a/src/mds/MDSRank.h +++ b/src/mds/MDSRank.h @@ -375,6 +375,10 @@ class MDSRank { std::ostream& ss); void schedule_inmemory_logger(); + double get_inject_journal_corrupt_dentry_first() const { + return inject_journal_corrupt_dentry_first; + } + // Reference to global MDS::mds_lock, so that users of MDSRank don't // carry around references to the outer MDS, and we can substitute // a separate lock here in future potentially. @@ -619,6 +623,7 @@ class MDSRank { bool standby_replaying = false; // true if current replay pass is in standby-replay mode uint64_t extraordinary_events_dump_interval = 0; + double inject_journal_corrupt_dentry_first = 0.0; private: bool send_status = true; diff --git a/src/mds/Server.cc b/src/mds/Server.cc index 55d2932fcfeb..786e05620bca 100644 --- a/src/mds/Server.cc +++ b/src/mds/Server.cc @@ -255,6 +255,7 @@ void Server::create_logger() Server::Server(MDSRank *m, MetricsHandler *metrics_handler) : mds(m), mdcache(mds->mdcache), mdlog(mds->mdlog), + inject_rename_corrupt_dentry_first(g_conf().get_val("mds_inject_rename_corrupt_dentry_first")), recall_throttle(g_conf().get_val("mds_recall_max_decay_rate")), metrics_handler(metrics_handler) { @@ -1318,6 +1319,9 @@ void Server::handle_conf_change(const std::set& changed) { dout(20) << __func__ << " max fragment size changed to " << bal_fragment_size_max << dendl; } + if (changed.count("mds_inject_rename_corrupt_dentry_first")) { + inject_rename_corrupt_dentry_first = g_conf().get_val("mds_inject_rename_corrupt_dentry_first"); + } } /* @@ -9486,6 +9490,16 @@ void Server::_rename_prepare(MDRequestRef& mdr, mdcache->journal_cow_dentry(mdr.get(), metablob, destdn, CEPH_NOSNAP, 0, destdnl); destdn->first = mdcache->get_global_snaprealm()->get_newest_seq() + 1; + { + auto do_corruption = inject_rename_corrupt_dentry_first; + if (unlikely(do_corruption > 0.0)) { + auto r = ceph::util::generate_random_number(0.0, 1.0); + if (r < do_corruption) { + dout(0) << "corrupting dn: " << *destdn << dendl; + destdn->first = -10; + } + } + } if (destdn->is_auth()) metablob->add_primary_dentry(destdn, srci, true, true); diff --git a/src/mds/Server.h b/src/mds/Server.h index f955e5f942b9..437a3517e0c3 100644 --- a/src/mds/Server.h +++ b/src/mds/Server.h @@ -507,6 +507,8 @@ private: uint64_t dir_max_entries = 0; int64_t bal_fragment_size_max = 0; + double inject_rename_corrupt_dentry_first = 0.0; + DecayCounter recall_throttle; time last_recall_state; diff --git a/src/mds/journal.cc b/src/mds/journal.cc index f8e3d802d411..b586d2fa98da 100644 --- a/src/mds/journal.cc +++ b/src/mds/journal.cc @@ -1362,6 +1362,17 @@ void EMetaBlob::replay(MDSRank *mds, LogSegment *logseg, MDPeerUpdate *peerup) in->state_clear(CInode::STATE_AUTH); ceph_assert(g_conf()->mds_kill_journal_replay_at != 2); + { + auto do_corruption = mds->get_inject_journal_corrupt_dentry_first(); + if (unlikely(do_corruption > 0.0)) { + auto r = ceph::util::generate_random_number(0.0, 1.0); + if (r < do_corruption) { + dout(0) << "corrupting dn: " << *dn << dendl; + dn->first = -10; + } + } + } + if (!(++count % mds->heartbeat_reset_grace())) mds->heartbeat_reset(); }