osd: speed up upmap underfull fallback

According to our tests, this commit can speed up pg upmap calculations
by orders of magnitude

Performance analysis:
Suppose:
R = number of pg_upmap_items (the full candidate pool, after to_skip/only_pools filtering)
M = how many OSDs the deviation_osd loop visits before it breaks (less or equal to underfull count)
U = the number of distinct from OSDs in the upmap items. U is less or equal to number of OSDs, and should be less than R x k
k = mapping pairs per candidate (usually 1, small)

Time complexity of OSDMap::calc_pg_upmaps():
Before the commit: O(M * R)
After the commit: O(R * k + M * log(U)) ~= O(R + M *log(U))

Performant evaluation:
Existing upmap items    Underfull OSDs  Old path        New path        Speedup
50000                   1000            6.51s           0.047s          137x
100000                  1000            12.05s          0.086s          140x
200000                  2000            63.93s          0.281s          227x

Fixes: https://tracker.ceph.com/issues/77563
Signed-off-by: Xuehan Xu <xuxuehan@qianxin.com>
Assisted-by: Codex:GPT-5.5

diff --git a/src/osd/OSDMap.cc b/src/osd/OSDMap.cc
index 59daa0a156f9a926176667535b14cfae6f70dc02..7579e054c8e98487f49e06f68d11d9ee3ebda1d7 100644 (file)
--- a/src/osd/OSDMap.cc
+++ b/src/osd/OSDMap.cc
@@ -5981,28 +5981,32 @@ int OSDMap::calc_pg_upmaps(
     ceph_assert(!(to_unmap.size() || to_upmap.size()));
     ldout(cct, 10) << " failed to find any changes for overfull osds"
                    << dendl;
-    for (auto& [deviation, osd] : deviation_osd) {
-      if (std::find(underfull.begin(), underfull.end(), osd) ==
-                    underfull.end())
-        break;
-      float target = osd_weight[osd] * pgs_per_weight;
-      ceph_assert(target > 0);
-      if (fabsf(deviation) < max_deviation) {
-        // respect max_deviation too
-        ldout(cct, 10) << " osd." << osd
-                       << " target " << target
-                       << " deviation " << deviation
-                       << " -> absolute " << fabsf(deviation)
-                       << " < max " << max_deviation
-                       << dendl;
-        break;
-      }
-      // look for remaps we can un-remap
-      candidates_t candidates = build_candidates(cct, tmp_osd_map, to_skip,
-                                                only_pools, aggressive, p_seed);
-      if (try_drop_remap_underfull(cct, candidates, osd, temp_pgs_by_osd,
-          to_unmap, to_upmap)) {
-       goto test_change;
+    {
+      const auto candidates_by_osd = build_candidates_by_osd(
+        cct, tmp_osd_map, to_skip, only_pools, aggressive, p_seed);
+      for (auto& [deviation, osd] : deviation_osd) {
+        if (std::find(underfull.begin(), underfull.end(), osd) ==
+                      underfull.end())
+          break;
+        float target = osd_weight[osd] * pgs_per_weight;
+        ceph_assert(target > 0);
+        if (fabsf(deviation) < max_deviation) {
+          // respect max_deviation too
+          ldout(cct, 10) << " osd." << osd
+                         << " target " << target
+                         << " deviation " << deviation
+                         << " -> absolute " << fabsf(deviation)
+                         << " < max " << max_deviation
+                         << dendl;
+          break;
+        }
+        // look for remaps we can un-remap
+        auto candidates = candidates_by_osd.find(osd);
+        if (candidates != candidates_by_osd.end() &&
+            try_drop_remap_underfull(cct, candidates->second, osd,
+                                     temp_pgs_by_osd, to_unmap, to_upmap)) {
+         goto test_change;
+        }
       }
     }
 
@@ -6527,7 +6531,7 @@ int OSDMap::find_best_remap (
   return best_pos;
 }
 
-OSDMap::candidates_t OSDMap::build_candidates(
+OSDMap::candidates_by_osd_t OSDMap::build_candidates_by_osd(
   CephContext *cct,
   const OSDMap& tmp_osd_map,
   const set<pg_t> to_skip,
@@ -6551,7 +6555,13 @@ OSDMap::candidates_t OSDMap::build_candidates(
     // shuffle candidates so they all get equal (in)attention
     std::shuffle(candidates.begin(), candidates.end(), get_random_engine(cct, p_seed));
   }
-  return candidates;
+  candidates_by_osd_t candidates_by_osd;
+  for (auto& candidate : candidates) {
+    for (auto& mapping : candidate.second) {
+      candidates_by_osd[mapping.first].push_back(candidate);
+    }
+  }
+  return candidates_by_osd;
 }
 
 // return -1 if all PGs are OK, else the first PG which includes only zero PA OSDs

diff --git a/src/osd/OSDMap.h b/src/osd/OSDMap.h
index 202475ab16ee96457d5a2fa57d873ecb30a2d8dc..0faaad96d384f6d3ec24861dfbc75ac7c20a4d4a 100644 (file)
--- a/src/osd/OSDMap.h
+++ b/src/osd/OSDMap.h
@@ -1638,6 +1638,7 @@ private: // Bunch of internal functions used only by calc_pg_upmaps (result of c
 
 typedef std::vector<std::pair<pg_t, mempool::osdmap::vector<std::pair<int, int>>>>
   candidates_t;
+typedef std::map<int, candidates_t> candidates_by_osd_t;
 
 bool try_drop_remap_underfull(
     CephContext *cct,
@@ -1669,7 +1670,7 @@ bool try_drop_remap_underfull(
     const std::map<int,float> osd_deviation
   );
 
-  candidates_t build_candidates(
+  candidates_by_osd_t build_candidates_by_osd(
     CephContext *cct,
     const OSDMap& tmp_osd_map,
     const std::set<pg_t> to_skip,