From 5f5bf82e5719e92e019e00cf1d6094c9e8adf570 Mon Sep 17 00:00:00 2001 From: Kotresh HR Date: Thu, 15 May 2025 21:45:29 +0530 Subject: [PATCH] mds: Mark the scrub passed if dirfrag is dirty The in-memory and on-disk stats might not match on a directory inode if any of the local or remote dirfrag is dirty. So don't fail the scrub and mark it as passed if the local or remote dirfrag is dirty. Signed-off-by: Kotresh HR Fixes: https://tracker.ceph.com/issues/65020 --- src/mds/CInode.cc | 15 ++++++++++++++- src/mds/CInode.h | 11 +++++++++++ src/mds/ScrubStack.cc | 10 +++++++++- src/messages/MMDSScrub.h | 13 ++++++++++--- 4 files changed, 44 insertions(+), 5 deletions(-) diff --git a/src/mds/CInode.cc b/src/mds/CInode.cc index e538a759f952..c0209e9cab8a 100644 --- a/src/mds/CInode.cc +++ b/src/mds/CInode.cc @@ -5086,6 +5086,7 @@ next: results->raw_stats.memory_value.rstat = in->get_inode()->rstat; frag_info_t& dir_info = results->raw_stats.ondisk_value.dirstat; nest_info_t& nest_info = results->raw_stats.ondisk_value.rstat; + bool has_local_dirty_dirfrag = false; if (rval != 0) { results->raw_stats.error_str << "Failed to read dirfrags off disk"; @@ -5103,6 +5104,8 @@ next: ceph_assert(dir->get_version() > 0); nest_info.add(dir->get_fnode()->accounted_rstat); dir_info.add(dir->get_fnode()->accounted_fragstat); + if (dir->is_auth() && dir->is_dirty()) + has_local_dirty_dirfrag = true; } } nest_info.rsubdirs++; // it gets one to account for self @@ -5128,7 +5131,17 @@ next: "please rerun scrub when system is stable; " "assuming passed for now;" << dendl; results->raw_stats.passed = true; - } + } else if (has_local_dirty_dirfrag || in->has_dirty_remote_dirfrag_scrubbed()) { + MDCache *mdcache = in->mdcache; // for dout() + auto ino = [this]() { return in->ino(); }; // for dout() + dout(20) << (has_local_dirty_dirfrag ? "local": "remote") << " dirfrag : " + "raw stats most likely wont match since it's a directory " + "inode and a dirfrag is dirty; please rerun scrub when " + "system is stable; assuming passed for now;" << dendl; + results->raw_stats.passed = true; + if (in->has_dirty_remote_dirfrag_scrubbed()) + in->clear_dirty_remote_dirfrag_scrubbed(); + } goto next; } diff --git a/src/mds/CInode.h b/src/mds/CInode.h index bddba4c7f199..1337cbfc2795 100644 --- a/src/mds/CInode.h +++ b/src/mds/CInode.h @@ -467,6 +467,16 @@ class CInode : public MDSCacheObject, public InodeStoreBase, public Counterqueued_frags; } + bool has_dirty_remote_dirfrag_scrubbed() { + return dirty_remote_dirfrag_scrubbed; + } + void mark_dirty_remote_dirfrag_scrubbed() { + dirty_remote_dirfrag_scrubbed = true; + } + void clear_dirty_remote_dirfrag_scrubbed() { + dirty_remote_dirfrag_scrubbed = false; + } + bool is_multiversion() const { return snaprealm || // other snaprealms will link to me get_inode()->is_dir() || // links to me in other snaps @@ -1255,6 +1265,7 @@ private: int stickydir_ref = 0; std::unique_ptr scrub_infop; + bool dirty_remote_dirfrag_scrubbed = false; /** @} Scrubbing and fsck */ }; diff --git a/src/mds/ScrubStack.cc b/src/mds/ScrubStack.cc index 785bd3c34d80..9c401f14aab7 100644 --- a/src/mds/ScrubStack.cc +++ b/src/mds/ScrubStack.cc @@ -1001,6 +1001,7 @@ void ScrubStack::handle_scrub(const cref_t &m) ceph_assert(diri); std::vector dfs; + bool has_dirty_dirfrag = false; MDSGatherBuilder gather(g_ceph_context); frag_vec_t frags; diri->dirfragtree.get_leaves(frags); @@ -1024,6 +1025,10 @@ void ScrubStack::handle_scrub(const cref_t &m) dir->add_waiter(CDir::WAIT_UNFREEZE, gather.new_sub()); continue; } + if (dir->is_dirty()) { + dout(10) << __func__ << " found dirty remote dirfrag " << *dir << dendl; + has_dirty_dirfrag = true; + } dfs.push_back(dir); } } @@ -1056,7 +1061,8 @@ void ScrubStack::handle_scrub(const cref_t &m) } auto r = make_message(MMDSScrub::OP_QUEUEDIR_ACK, m->get_ino(), - std::move(queued), m->get_tag()); + std::move(queued), m->get_tag(), + inodeno_t(), false, false, false, false, has_dirty_dirfrag); mdcache->mds->send_message_mds(r, from); } break; @@ -1078,6 +1084,8 @@ void ScrubStack::handle_scrub(const cref_t &m) const auto& header = diri->get_scrub_header(); header->set_epoch_last_forwarded(scrub_epoch); + if (m->is_remote_dirfrag_dirty()) + diri->mark_dirty_remote_dirfrag_scrubbed(); remove_from_waiting(diri); } } diff --git a/src/messages/MMDSScrub.h b/src/messages/MMDSScrub.h index 7714b377608e..41fc8788aefd 100644 --- a/src/messages/MMDSScrub.h +++ b/src/messages/MMDSScrub.h @@ -51,6 +51,7 @@ public: if (is_force()) out << " force"; if (is_recursive()) out << " recursive"; if (is_repair()) out << " repair"; + if (is_remote_dirfrag_dirty()) out << " dirty_remote_dirfrag"; out << ")"; } void encode_payload(uint64_t features) override { @@ -99,23 +100,28 @@ public: bool is_repair() const { return flags & FLAG_REPAIR; } + bool is_remote_dirfrag_dirty() const { + return flags & FLAG_DIRTY_REMOTE_DIRFRAG; + } protected: - static constexpr int HEAD_VERSION = 1; - static constexpr int COMPAT_VERSION = 1; + static constexpr int HEAD_VERSION = 2; + static constexpr int COMPAT_VERSION = 2; MMDSScrub() : MMDSOp(MSG_MDS_SCRUB, HEAD_VERSION, COMPAT_VERSION) {} MMDSScrub(int o) : MMDSOp(MSG_MDS_SCRUB, HEAD_VERSION, COMPAT_VERSION), op(o) {} MMDSScrub(int o, inodeno_t i, fragset_t&& _frags, std::string_view _tag, inodeno_t _origin=inodeno_t(), bool internal_tag=false, - bool force=false, bool recursive=false, bool repair=false) + bool force=false, bool recursive=false, bool repair=false, + bool remote_dirfrag_dirty=false) : MMDSOp(MSG_MDS_SCRUB, HEAD_VERSION, COMPAT_VERSION), op(o), ino(i), frags(std::move(_frags)), tag(_tag), origin(_origin) { if (internal_tag) flags |= FLAG_INTERNAL_TAG; if (force) flags |= FLAG_FORCE; if (recursive) flags |= FLAG_RECURSIVE; if (repair) flags |= FLAG_REPAIR; + if (remote_dirfrag_dirty) flags |= FLAG_DIRTY_REMOTE_DIRFRAG; } ~MMDSScrub() override {} @@ -129,6 +135,7 @@ private: static constexpr unsigned FLAG_FORCE = 1<<1; static constexpr unsigned FLAG_RECURSIVE = 1<<2; static constexpr unsigned FLAG_REPAIR = 1<<3; + static constexpr unsigned FLAG_DIRTY_REMOTE_DIRFRAG = 1<<4; int32_t op; inodeno_t ino; -- 2.47.3