From bd828fc033222e60e9b6671ccb826cec29ec5f36 Mon Sep 17 00:00:00 2001 From: xie xingguo Date: Mon, 5 Mar 2018 09:30:45 +0800 Subject: [PATCH] mon/OSDMonitor: kill pending upmap changes too if pool is gone There are quite a lot mgr/balancer related tests that I can observe the following logged errors: ``` 2018-03-03 08:14:00.359946 7fdda18cc700 -1 maybe_remove_pg_upmaps unable to load crush-rule of pg 5.b ``` which turns out to be a *pool-deletion* vs *balancer-auto-injected-upmap-changes* race issue. The root cause is that we don't clean up those *pending* pg_upmap/pg_upmap_items injected by the mgr/balancer properly simultaneously when the corresponding pool is gone, and hence the above problem can be fixed by: 1. clean up any pending upmap changes too if the corresponding pool is gone 2. re-check pending pool removal queue before we can safely apply any new upmap changes Signed-off-by: xie xingguo --- src/mon/OSDMonitor.cc | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc index 2f9a29cb70d6b..dc9c93cf45977 100644 --- a/src/mon/OSDMonitor.cc +++ b/src/mon/OSDMonitor.cc @@ -9540,6 +9540,14 @@ bool OSDMonitor::prepare_command_impl(MonOpRequestRef op, err = -ENOENT; goto reply; } + if (pending_inc.old_pools.count(pgid.pool())) { + ss << "pool of " << pgid << " is pending removal"; + err = -ENOENT; + getline(ss, rs); + wait_for_finished_proposal(op, + new Monitor::C_Command(mon, op, err, rs, get_last_committed() + 1)); + return true; + } enum { OP_PG_UPMAP, @@ -11489,6 +11497,20 @@ int OSDMonitor::_prepare_remove_pool( pending_inc.old_pg_upmap.insert(p.first); } } + // remove any pending pg_upmap mappings for this pool + { + auto it = pending_inc.new_pg_upmap.begin(); + while (it != pending_inc.new_pg_upmap.end()) { + if (it->first.pool() == (uint64_t)pool) { + dout(10) << __func__ << " " << pool + << " removing pending pg_upmap " + << it->first << dendl; + it = pending_inc.new_pg_upmap.erase(it); + } else { + it++; + } + } + } // remove any pg_upmap_items mappings for this pool for (auto& p : osdmap.pg_upmap_items) { if (p.first.pool() == (uint64_t)pool) { @@ -11498,6 +11520,20 @@ int OSDMonitor::_prepare_remove_pool( pending_inc.old_pg_upmap_items.insert(p.first); } } + // remove any pending pg_upmap mappings for this pool + { + auto it = pending_inc.new_pg_upmap_items.begin(); + while (it != pending_inc.new_pg_upmap_items.end()) { + if (it->first.pool() == (uint64_t)pool) { + dout(10) << __func__ << " " << pool + << " removing pending pg_upmap_items " + << it->first << dendl; + it = pending_inc.new_pg_upmap_items.erase(it); + } else { + it++; + } + } + } // remove any choose_args for this pool CrushWrapper newcrush; -- 2.39.5