From 617a82c45e75fcea775450a051166f5cfcacfe45 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Tue, 9 Jan 2018 15:18:58 -0600 Subject: [PATCH] osd: track deleted pools' pg_nums for calculating split This is needed to determine (quickly) whether PGs have split. Calling get_pg_num() on the latest map does not work when the pool has been deleted from that map. Signed-off-by: Sage Weil --- src/osd/OSD.cc | 49 +++++++++++++++++++++++++++++++++++++------------ src/osd/OSD.h | 20 ++++++++++++++++++++ 2 files changed, 57 insertions(+), 12 deletions(-) diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index 13898b2b315..751b17bf639 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -399,16 +399,18 @@ void OSDService::_maybe_split_pgid(OSDMapRef old_map, OSDMapRef new_map, spg_t pgid) { - assert(old_map->have_pg_pool(pgid.pool())); + if (!old_map->have_pg_pool(pgid.pool())) { + return; + } int old_pgnum = old_map->get_pg_num(pgid.pool()); + int new_pgnum = get_possibly_deleted_pool_pg_num(new_map, pgid.pool()); if (pgid.ps() < static_cast(old_pgnum)) { set children; - if (pgid.is_split(old_pgnum, - new_map->get_pg_num(pgid.pool()), &children)) { + if (pgid.is_split(old_pgnum, new_pgnum, &children)) { _start_split(pgid, children); } } else { - assert(pgid.ps() < static_cast(new_map->get_pg_num(pgid.pool()))); + assert(pgid.ps() < static_cast(new_pgnum)); } } @@ -417,13 +419,14 @@ void OSDService::init_splits_between(spg_t pgid, OSDMapRef tomap) { // First, check whether we can avoid this potentially expensive check - if (frommap->have_pg_pool(pgid.pool()) && - (!tomap->have_pg_pool(pgid.pool()) || // pool is deleted, must check for splits - pgid.is_split( - frommap->get_pg_num(pgid.pool()), - tomap->get_pg_num(pgid.pool()), - NULL))) { - dout(20) << __func__ << " " << pgid << " from " << frommap->get_epoch() << " -> " + if (!frommap->have_pg_pool(pgid.pool())) { + return; + } + int old_pgnum = frommap->get_pg_num(pgid.pool()); + int new_pgnum = get_possibly_deleted_pool_pg_num(tomap, pgid.pool()); + if (pgid.is_split(old_pgnum, new_pgnum, NULL)) { + dout(20) << __func__ << " " << pgid << " from " << frommap->get_epoch() + << " -> " << tomap->get_epoch() << " enumerating split children" << dendl; // Ok, a split happened, so we need to walk the osdmaps set new_pgs; // pgs to scan on each map @@ -438,7 +441,8 @@ void OSDService::init_splits_between(spg_t pgid, continue; } if (!nextmap->have_pg_pool(pgid.pool())) { - dout(20) << __func__ << " " << pgid << " pool deleted in " << nextmap->get_epoch() + dout(20) << __func__ << " " << pgid << " pool deleted in " + << nextmap->get_epoch() << dendl; break; } @@ -1566,6 +1570,26 @@ void OSDService::check_map_bl_cache_pins() clear_map_bl_cache_pins(0); } +int OSDService::get_deleted_pool_pg_num(int64_t pool) +{ + Mutex::Locker l(map_cache_lock); + auto p = deleted_pool_pg_nums.find(pool); + if (p != deleted_pool_pg_nums.end()) { + return p->second; + } + dout(20) << __func__ << " " << pool << " loading" << dendl; + ghobject_t oid = OSD::make_final_pool_info_oid(pool); + bufferlist bl; + int r = store->read(coll_t::meta(), oid, 0, 0, bl); + ceph_assert(r >= 0); + auto blp = bl.begin(); + pg_pool_t pi; + ::decode(pi, blp); + deleted_pool_pg_nums[pool] = pi.get_pg_num(); + dout(20) << __func__ << " " << pool << " got " << pi.get_pg_num() << dendl; + return pi.get_pg_num(); +} + OSDMapRef OSDService::_add_map(OSDMap *o) { epoch_t e = o->get_epoch(); @@ -7348,6 +7372,7 @@ void OSD::handle_osd_map(MOSDMap *m) string name = lastmap->get_pool_name(j.first); encode(name, bl); t.write(coll_t::meta(), obj, 0, bl.length(), bl); + service.store_deleted_pool_pg_num(j.first, j.second.get_pg_num()); } } lastmap = i.second; diff --git a/src/osd/OSD.h b/src/osd/OSD.h index 1efb051b50e..ee94a503207 100644 --- a/src/osd/OSD.h +++ b/src/osd/OSD.h @@ -867,6 +867,9 @@ public: /// true if pg consumption affected our unpinning std::atomic map_cache_pinned_low = {false}; + /// final pg_num values for recently deleted pools + map deleted_pool_pg_nums; + OSDMapRef try_get_map(epoch_t e); OSDMapRef get_map(epoch_t e) { OSDMapRef ret(try_get_map(e)); @@ -902,6 +905,23 @@ public: void clear_map_bl_cache_pins(epoch_t e); void check_map_bl_cache_pins(); + /// get last pg_num before a pool was deleted (if any) + int get_deleted_pool_pg_num(int64_t pool); + + void store_deleted_pool_pg_num(int64_t pool, int pg_num) { + Mutex::Locker l(map_cache_lock); + deleted_pool_pg_nums[pool] = pg_num; + } + + /// get pgnum from newmap or, if pool was deleted, last map pool existed in + int get_possibly_deleted_pool_pg_num(OSDMapRef newmap, + int64_t pool) { + if (newmap->have_pg_pool(pool)) { + return newmap->get_pg_num(pool); + } + return get_deleted_pool_pg_num(pool); + } + void need_heartbeat_peer_update(); void init(); -- 2.39.5