From: Patrick Donnelly Date: Tue, 7 Mar 2023 18:20:19 +0000 (-0500) Subject: mds: add killpoints for MDS shutdown X-Git-Tag: v19.0.0~760^2~15 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=c32555347661ddd941b684bfbda73c6917092d63;p=ceph.git mds: add killpoints for MDS shutdown Signed-off-by: Patrick Donnelly --- diff --git a/src/common/options/mds.yaml.in b/src/common/options/mds.yaml.in index 3214970ce02f..87f216433b58 100644 --- a/src/common/options/mds.yaml.in +++ b/src/common/options/mds.yaml.in @@ -999,6 +999,12 @@ options: services: - mds fmt_desc: probabilistically inject corrupt CDentry::first at journal load +- name: mds_kill_shutdown_at + type: uint + level: dev + default: 0 + services: + - mds flags: - runtime - name: mds_kill_mdstable_at diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc index f4ff185056a8..2cc3b177e344 100644 --- a/src/mds/MDCache.cc +++ b/src/mds/MDCache.cc @@ -139,6 +139,8 @@ MDCache::MDCache(MDSRank *m, PurgeQueue &purge_queue_) : symlink_recovery = g_conf().get_val("mds_symlink_recovery"); + kill_shutdown_at = g_conf().get_val("mds_kill_shutdown_at"); + lru.lru_set_midpoint(g_conf().get_val("mds_cache_mid")); bottom_lru.lru_set_midpoint(0); @@ -199,6 +201,9 @@ void MDCache::handle_conf_change(const std::set& changed, const MDS symlink_recovery = g_conf().get_val("mds_symlink_recovery"); dout(10) << "Storing symlink targets on file object's head " << symlink_recovery << dendl; } + if (changed.count("mds_kill_shutdown_at")) { + kill_shutdown_at = g_conf().get_val("mds_kill_shutdown_at"); + } migrator->handle_conf_change(changed, mdsmap); mds->balancer->handle_conf_change(changed, mdsmap); @@ -7878,6 +7883,7 @@ void MDCache::shutdown_start() bool MDCache::shutdown_pass() { dout(7) << "shutdown_pass" << dendl; + ceph_assert(kill_shutdown_at != KILL_SHUTDOWN_AT::SHUTDOWN_START); if (mds->is_stopped()) { dout(7) << " already shut down" << dendl; @@ -7892,6 +7898,7 @@ bool MDCache::shutdown_pass() // trim cache trim(UINT64_MAX); dout(5) << "lru size now " << lru.lru_get_size() << "/" << bottom_lru.lru_get_size() << dendl; + ceph_assert(kill_shutdown_at != KILL_SHUTDOWN_AT::SHUTDOWN_POSTTRIM); // Export all subtrees to another active (usually rank 0) if not rank 0 int num_auth_subtree = 0; @@ -7922,9 +7929,12 @@ bool MDCache::shutdown_pass() dest = 0; dout(7) << "sending " << *dir << " back to mds." << dest << dendl; migrator->export_dir_nicely(dir, dest); + ceph_assert(kill_shutdown_at != KILL_SHUTDOWN_AT::SHUTDOWN_POSTONEEXPORT); } } + ceph_assert(kill_shutdown_at != KILL_SHUTDOWN_AT::SHUTDOWN_POSTALLEXPORTS); + if (!strays_all_exported) { dout(7) << "waiting for strays to migrate" << dendl; return false; @@ -7943,6 +7953,7 @@ bool MDCache::shutdown_pass() mds->server->terminate_sessions(); return false; } + ceph_assert(kill_shutdown_at != KILL_SHUTDOWN_AT::SHUTDOWN_SESSIONTERMINATE); // Fully trim the log so that all objects in cache are clean and may be // trimmed by a future MDCache::trim. Note that MDSRank::tick does not @@ -7955,6 +7966,7 @@ bool MDCache::shutdown_pass() auto sle = create_subtree_map(); mds->mdlog->submit_entry(sle); mds->mdlog->flush(); + ceph_assert(kill_shutdown_at != KILL_SHUTDOWN_AT::SHUTDOWN_SUBTREEMAP); } } mds->mdlog->trim_all(); @@ -7962,6 +7974,7 @@ bool MDCache::shutdown_pass() dout(7) << "still >1 segments, waiting for log to trim" << dendl; return false; } + ceph_assert(kill_shutdown_at != KILL_SHUTDOWN_AT::SHUTDOWN_TRIMALL); // drop our reference to our stray dir inode for (int i = 0; i < NUM_STRAY; ++i) { @@ -7972,6 +7985,7 @@ bool MDCache::shutdown_pass() strays[i]->put_stickydirs(); } } + ceph_assert(kill_shutdown_at != KILL_SHUTDOWN_AT::SHUTDOWN_STRAYPUT); CDir *mydir = myin ? myin->get_dirfrag(frag_t()) : NULL; if (mydir && !mydir->is_subtree_root()) @@ -8007,6 +8021,7 @@ bool MDCache::shutdown_pass() dout(7) << "capping the mdlog" << dendl; mds->mdlog->cap(); } + ceph_assert(kill_shutdown_at != KILL_SHUTDOWN_AT::SHUTDOWN_LOGCAP); if (!mds->mdlog->empty()) mds->mdlog->trim(0); @@ -8054,16 +8069,19 @@ bool MDCache::shutdown_pass() myin->close_dirfrag(mydir->get_frag()); } ceph_assert(subtrees.empty()); + ceph_assert(kill_shutdown_at != KILL_SHUTDOWN_AT::SHUTDOWN_EMPTYSUBTREES); if (myin) { remove_inode(myin); ceph_assert(!myin); } + ceph_assert(kill_shutdown_at != KILL_SHUTDOWN_AT::SHUTDOWN_MYINREMOVAL); if (global_snaprealm) { remove_inode(global_snaprealm->inode); global_snaprealm = nullptr; } + ceph_assert(kill_shutdown_at != KILL_SHUTDOWN_AT::SHUTDOWN_GLOBALSNAPREALMREMOVAL); // done! dout(5) << "shutdown done." << dendl; diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h index 1280fa1d01e4..fabd847cf9d1 100644 --- a/src/mds/MDCache.h +++ b/src/mds/MDCache.h @@ -1290,6 +1290,23 @@ class MDCache { int num_remote_waiters = 0; // number of remote authpin waiters }; + enum KILL_SHUTDOWN_AT { + SHUTDOWN_NULL, + SHUTDOWN_START, + SHUTDOWN_POSTTRIM, + SHUTDOWN_POSTONEEXPORT, + SHUTDOWN_POSTALLEXPORTS, + SHUTDOWN_SESSIONTERMINATE, + SHUTDOWN_SUBTREEMAP, + SHUTDOWN_TRIMALL, + SHUTDOWN_STRAYPUT, + SHUTDOWN_LOGCAP, + SHUTDOWN_EMPTYSUBTREES, + SHUTDOWN_MYINREMOVAL, + SHUTDOWN_GLOBALSNAPREALMREMOVAL, + SHUTDOWN_UNUSED + }; + typedef std::map::iterator fragment_info_iterator; friend class EFragment; @@ -1389,6 +1406,8 @@ class MDCache { time upkeep_last_trim = time::min(); time upkeep_last_release = time::min(); std::atomic upkeep_trim_shutdown{false}; + + uint64_t kill_shutdown_at = 0; }; class C_MDS_RetryRequest : public MDSInternalContext { diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc index da681e735a3f..6e6e518870d2 100644 --- a/src/mds/MDSRank.cc +++ b/src/mds/MDSRank.cc @@ -3820,6 +3820,7 @@ const char** MDSRankDispatcher::get_tracked_conf_keys() const "mds_inject_journal_corrupt_dentry_first", "mds_inject_migrator_session_race", "mds_inject_rename_corrupt_dentry_first", + "mds_kill_shutdown_at", "mds_log_event_large_threshold", "mds_log_events_per_segment", "mds_log_major_segment_event_ratio",