From: Xuehan Xu Date: Sun, 21 Jun 2026 12:53:48 +0000 (+0800) Subject: osd: speed up upmap underfull fallback X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=cfd916b1b4190c1d961efcc700feba9164e9dd87;p=ceph.git osd: speed up upmap underfull fallback According to our tests, this commit can speed up pg upmap calculations by orders of magnitude Performance analysis: Suppose: R = number of pg_upmap_items (the full candidate pool, after to_skip/only_pools filtering) M = how many OSDs the deviation_osd loop visits before it breaks (less or equal to underfull count) U = the number of distinct from OSDs in the upmap items. U is less or equal to number of OSDs, and should be less than R x k k = mapping pairs per candidate (usually 1, small) Time complexity of OSDMap::calc_pg_upmaps(): Before the commit: O(M * R) After the commit: O(R * k + M * log(U)) ~= O(R + M *log(U)) Performant evaluation: Existing upmap items Underfull OSDs Old path New path Speedup 50000 1000 6.51s 0.047s 137x 100000 1000 12.05s 0.086s 140x 200000 2000 63.93s 0.281s 227x Fixes: https://tracker.ceph.com/issues/77563 Signed-off-by: Xuehan Xu Assisted-by: Codex:GPT-5.5 --- diff --git a/src/osd/OSDMap.cc b/src/osd/OSDMap.cc index 59daa0a156f..7579e054c8e 100644 --- a/src/osd/OSDMap.cc +++ b/src/osd/OSDMap.cc @@ -5981,28 +5981,32 @@ int OSDMap::calc_pg_upmaps( ceph_assert(!(to_unmap.size() || to_upmap.size())); ldout(cct, 10) << " failed to find any changes for overfull osds" << dendl; - for (auto& [deviation, osd] : deviation_osd) { - if (std::find(underfull.begin(), underfull.end(), osd) == - underfull.end()) - break; - float target = osd_weight[osd] * pgs_per_weight; - ceph_assert(target > 0); - if (fabsf(deviation) < max_deviation) { - // respect max_deviation too - ldout(cct, 10) << " osd." << osd - << " target " << target - << " deviation " << deviation - << " -> absolute " << fabsf(deviation) - << " < max " << max_deviation - << dendl; - break; - } - // look for remaps we can un-remap - candidates_t candidates = build_candidates(cct, tmp_osd_map, to_skip, - only_pools, aggressive, p_seed); - if (try_drop_remap_underfull(cct, candidates, osd, temp_pgs_by_osd, - to_unmap, to_upmap)) { - goto test_change; + { + const auto candidates_by_osd = build_candidates_by_osd( + cct, tmp_osd_map, to_skip, only_pools, aggressive, p_seed); + for (auto& [deviation, osd] : deviation_osd) { + if (std::find(underfull.begin(), underfull.end(), osd) == + underfull.end()) + break; + float target = osd_weight[osd] * pgs_per_weight; + ceph_assert(target > 0); + if (fabsf(deviation) < max_deviation) { + // respect max_deviation too + ldout(cct, 10) << " osd." << osd + << " target " << target + << " deviation " << deviation + << " -> absolute " << fabsf(deviation) + << " < max " << max_deviation + << dendl; + break; + } + // look for remaps we can un-remap + auto candidates = candidates_by_osd.find(osd); + if (candidates != candidates_by_osd.end() && + try_drop_remap_underfull(cct, candidates->second, osd, + temp_pgs_by_osd, to_unmap, to_upmap)) { + goto test_change; + } } } @@ -6527,7 +6531,7 @@ int OSDMap::find_best_remap ( return best_pos; } -OSDMap::candidates_t OSDMap::build_candidates( +OSDMap::candidates_by_osd_t OSDMap::build_candidates_by_osd( CephContext *cct, const OSDMap& tmp_osd_map, const set to_skip, @@ -6551,7 +6555,13 @@ OSDMap::candidates_t OSDMap::build_candidates( // shuffle candidates so they all get equal (in)attention std::shuffle(candidates.begin(), candidates.end(), get_random_engine(cct, p_seed)); } - return candidates; + candidates_by_osd_t candidates_by_osd; + for (auto& candidate : candidates) { + for (auto& mapping : candidate.second) { + candidates_by_osd[mapping.first].push_back(candidate); + } + } + return candidates_by_osd; } // return -1 if all PGs are OK, else the first PG which includes only zero PA OSDs diff --git a/src/osd/OSDMap.h b/src/osd/OSDMap.h index 202475ab16e..0faaad96d38 100644 --- a/src/osd/OSDMap.h +++ b/src/osd/OSDMap.h @@ -1638,6 +1638,7 @@ private: // Bunch of internal functions used only by calc_pg_upmaps (result of c typedef std::vector>>> candidates_t; +typedef std::map candidates_by_osd_t; bool try_drop_remap_underfull( CephContext *cct, @@ -1669,7 +1670,7 @@ bool try_drop_remap_underfull( const std::map osd_deviation ); - candidates_t build_candidates( + candidates_by_osd_t build_candidates_by_osd( CephContext *cct, const OSDMap& tmp_osd_map, const std::set to_skip,