From 16eb42a1d8cef5cf008b04b27d51e13dbd6ec495 Mon Sep 17 00:00:00 2001 From: Kamoltat Date: Fri, 25 Jun 2021 22:40:43 +0000 Subject: [PATCH] pybind/mgr/autoscaler: don't scale pools with overlapping roots In the previous version of get_subtree_resource_status() in src/pybind/mgr/pg_autoscaler/module.py we ignore overlapping pools which in some cases if combined with the new `scale-down` algorithm in https://github.com/ceph/ceph/pull/38805 can cause some pools to scale up/down to inapproriate amount of pgs. Therefore, the PR identifies the overlapping roots and prevent the pools with such roots from scaling. This only happens with `scale-down` profile as we see no problem with the default `scale-up` profile. Removed the variable `pool_root` since it is not used anywhere in the code, it only gets assigned and reassigned Also included a unit test test_overlapping_roots.py that tests the function identify_subtrees_and_overlaps() as well as edited test_cal_final_pg_target.py to account for pools that contain overlapping roots, therefore, those pools are expected not to scale. Signed-off-by: Kamoltat --- src/pybind/mgr/pg_autoscaler/module.py | 121 ++- .../tests/test_cal_final_pg_target.py | 880 ++++++++++-------- .../mgr/pg_autoscaler/tests/test_cal_ratio.py | 3 + .../tests/test_overlapping_roots.py | 479 ++++++++++ 4 files changed, 1056 insertions(+), 427 deletions(-) create mode 100644 src/pybind/mgr/pg_autoscaler/tests/test_overlapping_roots.py diff --git a/src/pybind/mgr/pg_autoscaler/module.py b/src/pybind/mgr/pg_autoscaler/module.py index 9d0c41cca21f6..d06a0077b4170 100644 --- a/src/pybind/mgr/pg_autoscaler/module.py +++ b/src/pybind/mgr/pg_autoscaler/module.py @@ -76,6 +76,7 @@ class PgAdjustmentProgress(object): """ Keeps the initial and target pg_num values """ + def __init__(self, pool_id: int, pg_num: int, pg_num_target: int) -> None: self.ev_id = str(uuid.uuid4()) self.pool_id = pool_id @@ -89,7 +90,7 @@ class PgAdjustmentProgress(object): desc = 'increasing' if self.pg_num < self.pg_num_target else 'decreasing' module.remote('progress', 'update', self.ev_id, ev_msg="PG autoscaler %s pool %d PGs from %d to %d" % - (desc, self.pool_id, self.pg_num, self.pg_num_target), + (desc, self.pool_id, self.pg_num, self.pg_num_target), ev_progress=progress, refs=[("pool", self.pool_id)]) @@ -108,7 +109,7 @@ class CrushSubtreeResourceStatus: self.pool_count: Optional[int] = None self.pool_used = 0 self.total_target_ratio = 0.0 - self.total_target_bytes = 0 # including replication / EC overhead + self.total_target_bytes = 0 # including replication / EC overhead class PgAutoscaler(MgrModule): @@ -176,7 +177,7 @@ class PgAutoscaler(MgrModule): osdmap = self.get_osdmap() pools = osdmap.get_pools_by_name() profile = self.autoscale_profile - ps, root_map, pool_root = self._get_pool_status(osdmap, pools, profile) + ps, root_map = self._get_pool_status(osdmap, pools, profile) if format in ('json', 'json-pretty'): return 0, json.dumps(ps, indent=4, sort_keys=True), '' @@ -273,35 +274,34 @@ class PgAutoscaler(MgrModule): self.log.info('Stopping pg_autoscaler') self._shutdown.set() - def get_subtree_resource_status(self, - osdmap: OSDMap, - crush: CRUSHMap) -> Tuple[Dict[int, CrushSubtreeResourceStatus], - Dict[int, int]]: - """ - For each CRUSH subtree of interest (i.e. the roots under which - we have pools), calculate the current resource usages and targets, - such as how many PGs there are, vs. how many PGs we would - like there to be. - """ - result: Dict[int, CrushSubtreeResourceStatus] = {} - pool_root = {} - roots = [] + def identify_subtrees_and_overlaps(self, + osdmap: OSDMap, + crush: CRUSHMap, + result: Dict[int, CrushSubtreeResourceStatus], + overlapped_roots: Set[int], + roots: List[CrushSubtreeResourceStatus]) -> \ + Tuple[List[CrushSubtreeResourceStatus], + Set[int]]: - # identify subtrees (note that they may overlap!) + # We identify subtrees and overlapping roots from osdmap for pool_id, pool in osdmap.get_pools().items(): crush_rule = crush.get_rule_by_id(pool['crush_rule']) assert crush_rule is not None cr_name = crush_rule['rule_name'] root_id = crush.get_rule_root(cr_name) assert root_id is not None - pool_root[pool_id] = root_id osds = set(crush.get_osds_under(root_id)) - # do we intersect an existing root? + # Are there overlapping roots? s = None - for prev in result.values(): + for prev_root_id, prev in result.items(): if osds & prev.osds: s = prev + if prev_root_id != root_id: + overlapped_roots.add(prev_root_id) + overlapped_roots.add(root_id) + self.log.error('pool %d has overlapping roots: %s', + pool_id, overlapped_roots) break if not s: s = CrushSubtreeResourceStatus() @@ -319,7 +319,24 @@ class PgAutoscaler(MgrModule): target_bytes = pool['options'].get('target_size_bytes', 0) if target_bytes: s.total_target_bytes += target_bytes * osdmap.pool_raw_used_rate(pool_id) + return roots, overlapped_roots + def get_subtree_resource_status(self, + osdmap: OSDMap, + crush: CRUSHMap) -> Tuple[Dict[int, CrushSubtreeResourceStatus], + Set[int]]: + """ + For each CRUSH subtree of interest (i.e. the roots under which + we have pools), calculate the current resource usages and targets, + such as how many PGs there are, vs. how many PGs we would + like there to be. + """ + result: Dict[int, CrushSubtreeResourceStatus] = {} + roots: List[CrushSubtreeResourceStatus] = [] + overlapped_roots: Set[int] = set() + # identify subtrees and overlapping roots + roots, overlapped_roots = self.identify_subtrees_and_overlaps(osdmap, + crush, result, overlapped_roots, roots) # finish subtrees all_stats = self.get('osd_stats') for s in roots: @@ -338,14 +355,13 @@ class PgAutoscaler(MgrModule): capacity += osd_stats['kb'] * 1024 s.capacity = capacity - self.log.debug('root_ids %s pools %s with %d osds, pg_target %d', s.root_ids, s.pool_ids, s.osd_count, s.pg_target) - return result, pool_root + return result, overlapped_roots def _calc_final_pg_target( self, @@ -362,9 +378,9 @@ class PgAutoscaler(MgrModule): """ `profile` determines behaviour of the autoscaler. `is_used` flag used to determine if this is the first - pass where the caller tries to calculate/adjust pools that has - used_ratio > even_ratio else this is the second pass, - we calculate final_ratio by giving it 1 / pool_count + pass where the caller tries to calculate/adjust pools that has + used_ratio > even_ratio else this is the second pass, + we calculate final_ratio by giving it 1 / pool_count of the root we are currently looking at. """ if profile == "scale-up": @@ -374,7 +390,7 @@ class PgAutoscaler(MgrModule): assert pg_target is not None pool_pg_target = (final_ratio * pg_target) / p['size'] * bias final_pg_target = max(p.get('options', {}).get('pg_num_min', PG_NUM_MIN), - nearest_power_of_two(pool_pg_target)) + nearest_power_of_two(pool_pg_target)) else: if is_used: @@ -405,7 +421,7 @@ class PgAutoscaler(MgrModule): pool_pg_target = (final_ratio * root_map[root_id].pg_left) / p['size'] * bias final_pg_target = max(p.get('options', {}).get('pg_num_min', PG_NUM_MIN), - nearest_power_of_two(pool_pg_target)) + nearest_power_of_two(pool_pg_target)) self.log.info("Pool '{0}' root_id {1} using {2} of space, bias {3}, " "pg target {4} quantized to {5} (current {6})".format( @@ -426,17 +442,17 @@ class PgAutoscaler(MgrModule): pools: Dict[str, Dict[str, Any]], crush_map: CRUSHMap, root_map: Dict[int, CrushSubtreeResourceStatus], - pool_root: Dict[int, int], pool_stats: Dict[int, Dict[str, int]], ret: List[Dict[str, Any]], threshold: float, is_used: bool, profile: 'ScaleModeT', + overlapped_roots: Set[int], ) -> Tuple[List[Dict[str, Any]], Dict[str, Dict[str, Any]]]: """ Calculates final_pg_target of each pools and determine if it needs scaling, this depends on the profile of the autoscaler. For scale-down, - we start out with a full complement of pgs and only descrease it when other + we start out with a full complement of pgs and only descrease it when other pools needs more pgs due to increased usage. For scale-up, we start out with the minimal amount of pgs and only scale when there is increase in usage. """ @@ -454,7 +470,12 @@ class PgAutoscaler(MgrModule): cr_name = crush_rule['rule_name'] root_id = crush_map.get_rule_root(cr_name) assert root_id is not None - + if root_id in overlapped_roots and profile == "scale-down": + # for scale-down profile skip pools + # with overlapping roots + self.log.warn("pool %d contains an overlapping root %d" + "... skipping scaling", pool_id, root_id) + continue capacity = root_map[root_id].capacity assert capacity is not None if capacity == 0: @@ -489,19 +510,17 @@ class PgAutoscaler(MgrModule): capacity) capacity_ratio = max(capacity_ratio, target_ratio) - final_ratio, pool_pg_target, final_pg_target = self._calc_final_pg_target(p, - pool_name, root_map, root_id, capacity_ratio, even_pools, bias, is_used, - profile, - ) + final_ratio, pool_pg_target, final_pg_target = self._calc_final_pg_target( + p, pool_name, root_map, root_id, capacity_ratio, even_pools, bias, is_used, profile) if final_ratio is None: continue adjust = False - if (final_pg_target > p['pg_num_target'] * threshold or \ - final_pg_target < p['pg_num_target'] / threshold) and \ - final_ratio >= 0.0 and \ - final_ratio <= 1.0: + if (final_pg_target > p['pg_num_target'] * threshold or + final_pg_target < p['pg_num_target'] / threshold) and \ + final_ratio >= 0.0 and \ + final_ratio <= 1.0: adjust = True assert pool_pg_target is not None @@ -525,7 +544,7 @@ class PgAutoscaler(MgrModule): 'pg_num_final': final_pg_target, 'would_adjust': adjust, 'bias': p.get('options', {}).get('pg_autoscale_bias', 1.0), - }); + }) return ret, even_pools @@ -536,12 +555,11 @@ class PgAutoscaler(MgrModule): profile: 'ScaleModeT', threshold: float = 3.0, ) -> Tuple[List[Dict[str, Any]], - Dict[int, CrushSubtreeResourceStatus], - Dict[int, int]]: + Dict[int, CrushSubtreeResourceStatus]]: assert threshold >= 2.0 crush_map = osdmap.get_crush() - root_map, pool_root = self.get_subtree_resource_status(osdmap, crush_map) + root_map, overlapped_roots = self.get_subtree_resource_status(osdmap, crush_map) df = self.get('df') pool_stats = dict([(p['id'], p['stats']) for p in df['pools']]) @@ -551,18 +569,18 @@ class PgAutoscaler(MgrModule): # First call of _calc_pool_targets() is to find/adjust pools that uses more capacaity than # the even_ratio of other pools and we adjust those first. # Second call make use of the even_pools we keep track of in the first call. - # All we need to do is iterate over those and give them 1/pool_count of the + # All we need to do is iterate over those and give them 1/pool_count of the # total pgs. - ret, even_pools = self._calc_pool_targets(osdmap, pools, crush_map, root_map, pool_root, - pool_stats, ret, threshold, True, profile) + ret, even_pools = self._calc_pool_targets(osdmap, pools, crush_map, root_map, + pool_stats, ret, threshold, True, profile, overlapped_roots) if profile == "scale-down": - # We only have adjust even_pools when we use scale-down profile - ret, _ = self._calc_pool_targets(osdmap, even_pools, crush_map, root_map, pool_root, - pool_stats, ret, threshold, False, profile) + # We only have adjust even_pools when we use scale-down profile + ret, _ = self._calc_pool_targets(osdmap, even_pools, crush_map, root_map, + pool_stats, ret, threshold, False, profile, overlapped_roots) - return (ret, root_map, pool_root) + return (ret, root_map) def _update_progress_events(self) -> None: osdmap = self.get_osdmap() @@ -584,7 +602,7 @@ class PgAutoscaler(MgrModule): return pools = osdmap.get_pools_by_name() profile = self.autoscale_profile - ps, root_map, pool_root = self._get_pool_status(osdmap, pools, profile) + ps, root_map = self._get_pool_status(osdmap, pools, profile) # Anyone in 'warn', set the health message for them and then # drop them from consideration. @@ -601,7 +619,8 @@ class PgAutoscaler(MgrModule): pool_id = p['pool_id'] pool_opts = pools[p['pool_name']]['options'] if pool_opts.get('target_size_ratio', 0) > 0 and pool_opts.get('target_size_bytes', 0) > 0: - bytes_and_ratio.append('Pool %s has target_size_bytes and target_size_ratio set' % p['pool_name']) + bytes_and_ratio.append( + 'Pool %s has target_size_bytes and target_size_ratio set' % p['pool_name']) total_bytes[p['crush_root_id']] += max( p['actual_raw_used'], p['target_bytes'] * p['raw_used_rate']) diff --git a/src/pybind/mgr/pg_autoscaler/tests/test_cal_final_pg_target.py b/src/pybind/mgr/pg_autoscaler/tests/test_cal_final_pg_target.py index 35269d5971759..9a30114b9d17a 100644 --- a/src/pybind/mgr/pg_autoscaler/tests/test_cal_final_pg_target.py +++ b/src/pybind/mgr/pg_autoscaler/tests/test_cal_final_pg_target.py @@ -1,10 +1,11 @@ -#python unit test +# python unit test import unittest from tests import mock import pytest import json from pg_autoscaler import module + class RootMapItem: def __init__(self, pool_count, pg_target, pg_left): @@ -14,23 +15,31 @@ class RootMapItem: self.pg_left = pg_left self.pool_used = 0 + class TestPgAutoscaler(object): def setup(self): # a bunch of attributes for testing. self.autoscaler = module.PgAutoscaler('module_name', 0, 0) - def helper_test(self, pools, root_map, bias, profile): + def helper_test(self, pools, root_map, bias, profile, overlapped_roots): # Here we simulate how _calc_pool_target() works. even_pools = {} for pool_name, p in pools.items(): + root_id = p['root_id'] + if root_id in overlapped_roots and profile == "scale-down": + # for scale-down profile skip pools + # with overlapping roots + assert p['no_scale'] + continue + final_ratio, pool_pg_target, final_pg_target = self.autoscaler._calc_final_pg_target(p, pool_name, root_map, - p['root_id'], p['capacity_ratio'], even_pools, bias, True, profile) + p['root_id'], p['capacity_ratio'], even_pools, bias, True, profile) if final_ratio == None: # no final_ratio means current pool is an even pool # and we do not have to do any assertion on it. - # You will never hit this case with a scale up profile. + # You will never hit this case with a scale up profile. continue assert p['expected_final_pg_target'] == final_pg_target @@ -38,12 +47,12 @@ class TestPgAutoscaler(object): if profile == "scale-down": # We only care about even_pools when profile is a scale-down - assert not p['even_pools'] and pool_name not in even_pools + assert not p['even_pools'] and pool_name not in even_pools if profile == "scale-down": for pool_name, p in even_pools.items(): final_ratio, pool_pg_target, final_pg_target = self.autoscaler._calc_final_pg_target(p, pool_name, root_map, - p['root_id'], p['capacity_ratio'], even_pools, bias, False, profile) + p['root_id'], p['capacity_ratio'], even_pools, bias, False, profile) assert p['expected_final_pg_target'] == final_pg_target assert p['expected_final_ratio'] == final_ratio @@ -52,433 +61,552 @@ class TestPgAutoscaler(object): def test_all_even_pools_scale_up(self): pools = { - "test0":{ - - "pool": 0, - "pool_name": "test0", - "pg_num_target": 32, - "capacity_ratio": 0.2, - "root_id":"0", - "expected_final_pg_target": 64, - "expected_final_ratio": 0.2, - "even_pools": True, - "size": 1, - }, - - "test1":{ - - "pool": 1, - "pool_name": "test1", - "pg_num_target": 32, - "capacity_ratio": 0.2, - "root_id":"0", - "expected_final_pg_target": 64, - "expected_final_ratio": 0.2, - "even_pools": True, - "size": 1, - }, - - "test2":{ - - "pool": 2, - "pool_name": "test2", - "pg_num_target": 32, - "capacity_ratio": 0.2, - "root_id":"0", - "expected_final_pg_target": 64, - "expected_final_ratio": 0.2, - "even_pools": True, - "size": 1, - }, - - "test3":{ - - "pool": 3, - "pool_name": "test3", - "pg_num_target": 32, - "capacity_ratio": 0.1, - "root_id": "0", - "expected_final_pg_target": 32, - "expected_final_ratio": 0.1, - "even_pools": True, - "size": 1, - }, - - } + "test0": { + + "pool": 0, + "pool_name": "test0", + "pg_num_target": 32, + "capacity_ratio": 0.2, + "root_id": 0, + "expected_final_pg_target": 64, + "expected_final_ratio": 0.2, + "even_pools": True, + "size": 1, + "no_scale": False, + }, + + "test1": { + + "pool": 1, + "pool_name": "test1", + "pg_num_target": 32, + "capacity_ratio": 0.2, + "root_id": 0, + "expected_final_pg_target": 64, + "expected_final_ratio": 0.2, + "even_pools": True, + "size": 1, + "no_scale": False, + }, + + "test2": { + + "pool": 2, + "pool_name": "test2", + "pg_num_target": 32, + "capacity_ratio": 0.2, + "root_id": 0, + "expected_final_pg_target": 64, + "expected_final_ratio": 0.2, + "even_pools": True, + "size": 1, + "no_scale": False, + }, + + "test3": { + + "pool": 3, + "pool_name": "test3", + "pg_num_target": 32, + "capacity_ratio": 0.1, + "root_id": 0, + "expected_final_pg_target": 32, + "expected_final_ratio": 0.1, + "even_pools": True, + "size": 1, + "no_scale": False, + }, + + } root_map = { - "0": RootMapItem(4, 400, 400), - "1": RootMapItem(4, 400, 400), + 0: RootMapItem(4, 400, 400), + 1: RootMapItem(4, 400, 400), - } + } profile = "scale-up" bias = 1 - self.helper_test(pools, root_map, bias, profile) + overlapped_roots = set() + self.helper_test(pools, root_map, bias, profile, overlapped_roots) def test_all_even_pools_scale_down(self): pools = { - "test0":{ - - "pool": 0, - "pool_name": "test0", - "pg_num_target": 32, - "capacity_ratio": 0.2, - "root_id":"0", - "expected_final_pg_target": 128, - "expected_final_ratio": 0.25, - "even_pools": True, - "size": 1, - }, - - "test1":{ - - "pool": 1, - "pool_name": "test1", - "pg_num_target": 32, - "capacity_ratio": 0.2, - "root_id":"0", - "expected_final_pg_target": 128, - "expected_final_ratio": 0.25, - "even_pools": True, - "size": 1, - }, - - "test2":{ - - "pool": 2, - "pool_name": "test2", - "pg_num_target": 32, - "capacity_ratio": 0.2, - "root_id":"0", - "expected_final_pg_target": 128, - "expected_final_ratio": 0.25, - "even_pools": True, - "size": 1, - }, - - "test3":{ - - "pool": 3, - "pool_name": "test3", - "pg_num_target": 32, - "capacity_ratio": 0.1, - "root_id": "0", - "expected_final_pg_target": 128, - "expected_final_ratio": 0.25, - "even_pools": True, - "size": 1, - }, - - } + "test0": { + + "pool": 0, + "pool_name": "test0", + "pg_num_target": 32, + "capacity_ratio": 0.2, + "root_id": 0, + "expected_final_pg_target": 128, + "expected_final_ratio": 0.25, + "even_pools": True, + "size": 1, + "no_scale": False, + }, + + "test1": { + + "pool": 1, + "pool_name": "test1", + "pg_num_target": 32, + "capacity_ratio": 0.2, + "root_id": 0, + "expected_final_pg_target": 128, + "expected_final_ratio": 0.25, + "even_pools": True, + "size": 1, + "no_scale": False, + }, + + "test2": { + + "pool": 2, + "pool_name": "test2", + "pg_num_target": 32, + "capacity_ratio": 0.2, + "root_id": 0, + "expected_final_pg_target": 128, + "expected_final_ratio": 0.25, + "even_pools": True, + "size": 1, + "no_scale": False, + }, + + "test3": { + + "pool": 3, + "pool_name": "test3", + "pg_num_target": 32, + "capacity_ratio": 0.1, + "root_id": 0, + "expected_final_pg_target": 128, + "expected_final_ratio": 0.25, + "even_pools": True, + "size": 1, + "no_scale": False, + }, + + } root_map = { - "0": RootMapItem(4, 400, 400), - "1": RootMapItem(4, 400, 400), + 0: RootMapItem(4, 400, 400), + 1: RootMapItem(4, 400, 400), - } + } - profile = "scale-down" + profile = "scale-down" bias = 1 - self.helper_test(pools, root_map, bias, profile) + overlapped_roots = set() + self.helper_test(pools, root_map, bias, profile, overlapped_roots) def test_uneven_pools_scale_up(self): pools = { - "test0":{ - - "pool": 0, - "pool_name": "test0", - "pg_num_target": 32, - "capacity_ratio": 0.1, - "root_id":"0", - "expected_final_pg_target": 32, - "expected_final_ratio": 0.1, - "even_pools": True, - "size": 1, - }, - - "test1":{ - - "pool": 1, - "pool_name": "test1", - "pg_num_target": 32, - "capacity_ratio": 0.5, - "root_id":"0", - "expected_final_pg_target": 256, - "expected_final_ratio": 0.5, - "even_pools": False, - "size": 1, - }, - - "test2":{ - - "pool": 2, - "pool_name": "test2", - "pg_num_target": 32, - "capacity_ratio": 0.1, - "root_id":"0", - "expected_final_pg_target": 32, - "expected_final_ratio": 0.1, - "even_pools": True, - "size": 1, - }, - - "test3":{ - - "pool": 3, - "pool_name": "test3", - "pg_num_target": 32, - "capacity_ratio": 0.1, - "root_id": "0", - "expected_final_pg_target": 32, - "expected_final_ratio": 0.1, - "even_pools": True, - "size": 1, - }, - - } + "test0": { + + "pool": 0, + "pool_name": "test0", + "pg_num_target": 32, + "capacity_ratio": 0.1, + "root_id": 0, + "expected_final_pg_target": 32, + "expected_final_ratio": 0.1, + "even_pools": True, + "size": 1, + "no_scale": False, + }, + + "test1": { + + "pool": 1, + "pool_name": "test1", + "pg_num_target": 32, + "capacity_ratio": 0.5, + "root_id": 0, + "expected_final_pg_target": 256, + "expected_final_ratio": 0.5, + "even_pools": False, + "size": 1, + "no_scale": False, + }, + + "test2": { + + "pool": 2, + "pool_name": "test2", + "pg_num_target": 32, + "capacity_ratio": 0.1, + "root_id": 0, + "expected_final_pg_target": 32, + "expected_final_ratio": 0.1, + "even_pools": True, + "size": 1, + "no_scale": False, + }, + + "test3": { + + "pool": 3, + "pool_name": "test3", + "pg_num_target": 32, + "capacity_ratio": 0.1, + "root_id": 0, + "expected_final_pg_target": 32, + "expected_final_ratio": 0.1, + "even_pools": True, + "size": 1, + "no_scale": False, + }, + + } root_map = { - "0": RootMapItem(4, 400, 400), - "1": RootMapItem(4, 400, 400), + 0: RootMapItem(4, 400, 400), + 1: RootMapItem(4, 400, 400), - } + } profile = "scale-up" bias = 1 - self.helper_test(pools, root_map, bias, profile) + overlapped_roots = set() + self.helper_test(pools, root_map, bias, profile, overlapped_roots) def test_uneven_pools_scale_down(self): pools = { - "test0":{ - - "pool": 0, - "pool_name": "test0", - "pg_num_target": 32, - "capacity_ratio": 0.1, - "root_id":"0", - "expected_final_pg_target": 64, - "expected_final_ratio": 1/3, - "even_pools": True, - "size": 1, - }, - - "test1":{ - - "pool": 1, - "pool_name": "test1", - "pg_num_target": 32, - "capacity_ratio": 0.5, - "root_id":"0", - "expected_final_pg_target": 256, - "expected_final_ratio": 0.5, - "even_pools": False, - "size": 1, - }, - - "test2":{ - - "pool": 2, - "pool_name": "test2", - "pg_num_target": 32, - "capacity_ratio": 0.1, - "root_id":"0", - "expected_final_pg_target": 64, - "expected_final_ratio": 1/3, - "even_pools": True, - "size": 1, - }, - - "test3":{ - - "pool": 3, - "pool_name": "test3", - "pg_num_target": 32, - "capacity_ratio": 0.1, - "root_id": "0", - "expected_final_pg_target": 64, - "expected_final_ratio": 1/3, - "even_pools": True, - "size": 1, - }, - - } + "test0": { + + "pool": 0, + "pool_name": "test0", + "pg_num_target": 32, + "capacity_ratio": 0.1, + "root_id": 0, + "expected_final_pg_target": 64, + "expected_final_ratio": 1/3, + "even_pools": True, + "size": 1, + "no_scale": False, + }, + + "test1": { + + "pool": 1, + "pool_name": "test1", + "pg_num_target": 32, + "capacity_ratio": 0.5, + "root_id": 0, + "expected_final_pg_target": 256, + "expected_final_ratio": 0.5, + "even_pools": False, + "size": 1, + "no_scale": False, + }, + + "test2": { + + "pool": 2, + "pool_name": "test2", + "pg_num_target": 32, + "capacity_ratio": 0.1, + "root_id": 0, + "expected_final_pg_target": 64, + "expected_final_ratio": 1/3, + "even_pools": True, + "size": 1, + "no_scale": False, + }, + + "test3": { + + "pool": 3, + "pool_name": "test3", + "pg_num_target": 32, + "capacity_ratio": 0.1, + "root_id": 0, + "expected_final_pg_target": 64, + "expected_final_ratio": 1/3, + "even_pools": True, + "size": 1, + "no_scale": False, + }, + + } root_map = { - "0": RootMapItem(4, 400, 400), - "1": RootMapItem(4, 400, 400), + 0: RootMapItem(4, 400, 400), + 1: RootMapItem(4, 400, 400), - } + } - profile = "scale-down" + profile = "scale-down" bias = 1 - self.helper_test(pools, root_map, bias, profile) + overlapped_roots = set() + self.helper_test(pools, root_map, bias, profile, overlapped_roots) def test_uneven_pools_with_diff_roots_scale_up(self): pools = { - "test0":{ - - "pool": 0, - "pool_name": "test0", - "pg_num_target": 32, - "capacity_ratio": 0.4, - "root_id":"0", - "expected_final_pg_target": 2048, - "expected_final_ratio": 0.4, - "even_pools": False, - "size": 1, - }, - - "test1":{ - - "pool": 1, - "pool_name": "test1", - "pg_num_target": 32, - "capacity_ratio": 0.6, - "root_id":"1", - "expected_final_pg_target": 2048, - "expected_final_ratio": 0.6, - "even_pools": False, - "size": 1, - }, - - "test2":{ - - "pool": 2, - "pool_name": "test2", - "pg_num_target": 32, - "capacity_ratio": 0.5, - "root_id":"0", - "expected_final_pg_target": 2048, - "expected_final_ratio": 0.5, - "even_pools": False, - "size": 1, - }, - - "test3":{ - - "pool": 3, - "pool_name": "test3", - "pg_num_target": 32, - "capacity_ratio": 0.1, - "root_id": "0", - "expected_final_pg_target": 512, - "expected_final_ratio": 0.1, - "even_pools": True, - "size": 1, - }, - - "test4":{ - - "pool": 4, - "pool_name": "test4", - "pg_num_target": 32, - "capacity_ratio": 0.4, - "root_id": "1", - "expected_final_pg_target": 2048, - "expected_final_ratio": 0.4, - "even_pools": True, - "size": 1, - }, - - } + "test0": { + + "pool": 0, + "pool_name": "test0", + "pg_num_target": 32, + "capacity_ratio": 0.4, + "root_id": 0, + "expected_final_pg_target": 2048, + "expected_final_ratio": 0.4, + "even_pools": False, + "size": 1, + "no_scale": False, + }, + + "test1": { + + "pool": 1, + "pool_name": "test1", + "pg_num_target": 32, + "capacity_ratio": 0.6, + "root_id": 1, + "expected_final_pg_target": 2048, + "expected_final_ratio": 0.6, + "even_pools": False, + "size": 1, + "no_scale": False, + }, + + "test2": { + + "pool": 2, + "pool_name": "test2", + "pg_num_target": 32, + "capacity_ratio": 0.5, + "root_id": 0, + "expected_final_pg_target": 2048, + "expected_final_ratio": 0.5, + "even_pools": False, + "size": 1, + "no_scale": False, + }, + + "test3": { + + "pool": 3, + "pool_name": "test3", + "pg_num_target": 32, + "capacity_ratio": 0.1, + "root_id": 0, + "expected_final_pg_target": 512, + "expected_final_ratio": 0.1, + "even_pools": True, + "size": 1, + "no_scale": False, + }, + + "test4": { + + "pool": 4, + "pool_name": "test4", + "pg_num_target": 32, + "capacity_ratio": 0.4, + "root_id": 1, + "expected_final_pg_target": 2048, + "expected_final_ratio": 0.4, + "even_pools": True, + "size": 1, + "no_scale": False, + }, + + } root_map = { - "0": RootMapItem(3, 5000, 5000), - "1": RootMapItem(2, 5000, 5000), + 0: RootMapItem(3, 5000, 5000), + 1: RootMapItem(2, 5000, 5000), - } + } profile = "scale-up" bias = 1 - self.helper_test(pools, root_map, bias, profile) + overlapped_roots = set() + self.helper_test(pools, root_map, bias, profile, overlapped_roots) def test_uneven_pools_with_diff_roots_scale_down(self): pools = { - "test0":{ - - "pool": 0, - "pool_name": "test0", - "pg_num_target": 32, - "capacity_ratio": 0.4, - "root_id":"0", - "expected_final_pg_target": 2048, - "expected_final_ratio": 0.4, - "even_pools": False, - "size": 1, - }, - - "test1":{ - - "pool": 1, - "pool_name": "test1", - "pg_num_target": 32, - "capacity_ratio": 0.6, - "root_id":"1", - "expected_final_pg_target": 2048, - "expected_final_ratio": 0.6, - "even_pools": False, - "size": 1, - }, - - "test2":{ - - "pool": 2, - "pool_name": "test2", - "pg_num_target": 32, - "capacity_ratio": 0.5, - "root_id":"0", - "expected_final_pg_target": 2048, - "expected_final_ratio": 0.5, - "even_pools": False, - "size": 1, - }, - - "test3":{ - - "pool": 3, - "pool_name": "test3", - "pg_num_target": 32, - "capacity_ratio": 0.1, - "root_id": "0", - "expected_final_pg_target": 512, - "expected_final_ratio": 1, - "even_pools": True, - "size": 1, - }, - - "test4":{ - - "pool": 4, - "pool_name": "test4", - "pg_num_target": 32, - "capacity_ratio": 0.4, - "root_id": "1", - "expected_final_pg_target": 2048, - "expected_final_ratio": 1, - "even_pools": True, - "size": 1, - }, - - } + "test0": { + + "pool": 0, + "pool_name": "test0", + "pg_num_target": 32, + "capacity_ratio": 0.4, + "root_id": 0, + "expected_final_pg_target": 2048, + "expected_final_ratio": 0.4, + "even_pools": False, + "size": 1, + "no_scale": False, + }, + + "test1": { + + "pool": 1, + "pool_name": "test1", + "pg_num_target": 32, + "capacity_ratio": 0.6, + "root_id": 1, + "expected_final_pg_target": 2048, + "expected_final_ratio": 0.6, + "even_pools": False, + "size": 1, + "no_scale": False, + }, + + "test2": { + + "pool": 2, + "pool_name": "test2", + "pg_num_target": 32, + "capacity_ratio": 0.5, + "root_id": 0, + "expected_final_pg_target": 2048, + "expected_final_ratio": 0.5, + "even_pools": False, + "size": 1, + "no_scale": False, + }, + + "test3": { + + "pool": 3, + "pool_name": "test3", + "pg_num_target": 32, + "capacity_ratio": 0.1, + "root_id": 0, + "expected_final_pg_target": 512, + "expected_final_ratio": 1, + "even_pools": True, + "size": 1, + "no_scale": False, + }, + + "test4": { + + "pool": 4, + "pool_name": "test4", + "pg_num_target": 32, + "capacity_ratio": 0.4, + "root_id": 1, + "expected_final_pg_target": 2048, + "expected_final_ratio": 1, + "even_pools": True, + "size": 1, + "no_scale": False, + }, + + } + + root_map = { + + 0: RootMapItem(3, 5000, 5000), + 1: RootMapItem(2, 5000, 5000), + + } + + profile = "scale-down" + bias = 1 + overlapped_roots = set() + self.helper_test(pools, root_map, bias, profile, overlapped_roots) + + def test_uneven_pools_with_overllaped_roots_scale_down(self): + pools = { + + "test0": { + + "pool": 0, + "pool_name": "test0", + "pg_num_target": 32, + "capacity_ratio": 0.4, + "root_id": 0, + "expected_final_pg_target": 2048, + "expected_final_ratio": 0.4, + "even_pools": False, + "size": 1, + "no_scale": True, + }, + + "test1": { + + "pool": 1, + "pool_name": "test1", + "pg_num_target": 32, + "capacity_ratio": 0.6, + "root_id": 1, + "expected_final_pg_target": 2048, + "expected_final_ratio": 0.6, + "even_pools": False, + "size": 1, + "no_scale": True, + }, + + "test2": { + + "pool": 2, + "pool_name": "test2", + "pg_num_target": 32, + "capacity_ratio": 0.5, + "root_id": 0, + "expected_final_pg_target": 2048, + "expected_final_ratio": 0.5, + "even_pools": False, + "size": 1, + "no_scale": True, + }, + + "test3": { + + "pool": 3, + "pool_name": "test3", + "pg_num_target": 32, + "capacity_ratio": 0.1, + "root_id": 0, + "expected_final_pg_target": 512, + "expected_final_ratio": 1, + "even_pools": True, + "size": 1, + "no_scale": True, + }, + + "test4": { + + "pool": 4, + "pool_name": "test4", + "pg_num_target": 32, + "capacity_ratio": 0.4, + "root_id": 1, + "expected_final_pg_target": 2048, + "expected_final_ratio": 1, + "even_pools": True, + "size": 1, + "no_scale": True, + }, + + } root_map = { - "0": RootMapItem(3, 5000, 5000), - "1": RootMapItem(2, 5000, 5000), + 0: RootMapItem(3, 5000, 5000), + 1: RootMapItem(2, 5000, 5000), - } + } profile = "scale-down" bias = 1 - self.helper_test(pools, root_map, bias, profile) + overlapped_roots = {0, 1} + self.helper_test(pools, root_map, bias, profile, overlapped_roots) diff --git a/src/pybind/mgr/pg_autoscaler/tests/test_cal_ratio.py b/src/pybind/mgr/pg_autoscaler/tests/test_cal_ratio.py index 122d95274d071..d966713605359 100644 --- a/src/pybind/mgr/pg_autoscaler/tests/test_cal_ratio.py +++ b/src/pybind/mgr/pg_autoscaler/tests/test_cal_ratio.py @@ -1,11 +1,13 @@ from pg_autoscaler import effective_target_ratio from pytest import approx + def check_simple_ratio(target_ratio, tot_ratio): etr = effective_target_ratio(target_ratio, tot_ratio, 0, 0) assert (target_ratio / tot_ratio) == approx(etr) return etr + def test_simple(): etr1 = check_simple_ratio(0.2, 0.9) etr2 = check_simple_ratio(2, 9) @@ -19,6 +21,7 @@ def test_simple(): etr2 = check_simple_ratio(0.5, 1.0) assert etr1 == approx(etr2) + def test_total_bytes(): etr = effective_target_ratio(1, 10, 5, 10) assert etr == approx(0.05) diff --git a/src/pybind/mgr/pg_autoscaler/tests/test_overlapping_roots.py b/src/pybind/mgr/pg_autoscaler/tests/test_overlapping_roots.py new file mode 100644 index 0000000000000..bf4ddfb62020d --- /dev/null +++ b/src/pybind/mgr/pg_autoscaler/tests/test_overlapping_roots.py @@ -0,0 +1,479 @@ +# python unit test +import unittest +from tests import mock +import pytest +import json +from pg_autoscaler import module + + +class OSDMAP: + def __init__(self, pools): + self.pools = pools + + def get_pools(self): + return self.pools + + def pool_raw_used_rate(pool_id): + return 1 + + +class CRUSH: + def __init__(self, rules, osd_dic): + self.rules = rules + self.osd_dic = osd_dic + + def get_rule_by_id(self, rule_id): + for rule in self.rules: + if rule['rule_id'] == rule_id: + return rule + + return None + + def get_rule_root(self, rule_name): + for rule in self.rules: + if rule['rule_name'] == rule_name: + return rule['root_id'] + + return None + + def get_osds_under(self, root_id): + return self.osd_dic[root_id] + + +class TestPgAutoscaler(object): + + def setup(self): + # a bunch of attributes for testing. + self.autoscaler = module.PgAutoscaler('module_name', 0, 0) + + def helper_test(self, osd_dic, rules, pools, expected_overlapped_roots): + result = {} + roots = [] + overlapped_roots = set() + osdmap = OSDMAP(pools) + crush = CRUSH(rules, osd_dic) + roots, overlapped_roots = self.autoscaler.identify_subtrees_and_overlaps(osdmap, + crush, result, overlapped_roots, roots) + assert overlapped_roots == expected_overlapped_roots + + def test_subtrees_and_overlaps(self): + osd_dic = { + -1: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10], + -40: [11, 12, 13, 14, 15], + -5: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], + } + + rules = [ + { + "rule_id": 0, + "rule_name": "data", + "ruleset": 0, + "type": 1, + "min_size": 1, + "max_size": 10, + "root_id": -1, + }, + { + "rule_id": 1, + "rule_name": "teuthology-data-ec", + "ruleset": 1, + "type": 3, + "min_size": 3, + "max_size": 6, + "root_id": -5, + }, + { + "rule_id": 4, + "rule_name": "rep-ssd", + "ruleset": 4, + "type": 1, + "min_size": 1, + "max_size": 10, + "root_id": -40, + }, + ] + pools = { + 0: { + "pool_name": "data", + "pg_num_target": 1024, + "size": 3, + "crush_rule": 0, + "capacity_ratio": 0.1624, + "options": { + "pg_num_min": 1024, + }, + "expected_final_pg_target": 1024, + }, + 1: { + "pool_name": "metadata", + "pg_num_target": 64, + "size": 3, + "crush_rule": 4, + "capacity_ratio": 0.0144, + "options": { + "pg_num_min": 64, + }, + "expected_final_pg_target": 64, + }, + 4: { + "pool_name": "libvirt-pool", + "pg_num_target": 32, + "size": 3, + "crush_rule": 0, + "capacity_ratio": 0.0001, + "options": {}, + "expected_final_pg_target": 128, + }, + 93: { + "pool_name": ".rgw.root", + "pg_num_target": 32, + "size": 3, + "crush_rule": 4, + "capacity_ratio": 0, + "options": {}, + "expected_final_pg_target": 32, + }, + 94: { + "pool_name": "default.rgw.control", + "pg_num_target": 32, + "size": 3, + "crush_rule": 4, + "capacity_ratio": 0, + "options": {}, + "expected_final_pg_target": 32, + }, + 95: { + "pool_name": "default.rgw.meta", + "pg_num_target": 32, + "size": 3, + "crush_rule": 4, + "capacity_ratio": 0, + "options": {}, + "expected_final_pg_target": 32, + }, + 96: { + "pool_name": "default.rgw.log", + "pg_num_target": 32, + "size": 3, + "crush_rule": 4, + "capacity_ratio": 0, + "options": {}, + "expected_final_pg_target": 32, + }, + 97: { + "pool_name": "default.rgw.buckets.index", + "pg_num_target": 32, + "size": 3, + "crush_rule": 4, + "capacity_ratio": 0.0002, + "options": {}, + "expected_final_pg_target": 32, + }, + 98: { + "pool_name": "default.rgw.buckets.data", + "pg_num_target": 32, + "size": 3, + "crush_rule": 0, + "capacity_ratio": 0.0457, + "options": {}, + "expected_final_pg_target": 128, + }, + 99: { + "pool_name": "default.rgw.buckets.non-ec", + "pg_num_target": 32, + "size": 3, + "crush_rule": 0, + "capacity_ratio": 0, + "options": {}, + "expected_final_pg_target": 32, + }, + 100: { + "pool_name": "device_health_metrics", + "pg_num_target": 1, + "size": 3, + "crush_rule": 0, + "capacity_ratio": 0, + "options": { + "pg_num_min": 1 + }, + "expected_final_pg_target": 1, + }, + 113: { + "pool_name": "cephfs.teuthology.meta", + "pg_num_target": 64, + "size": 3, + "crush_rule": 4, + "capacity_ratio": 0.1389, + "options": { + "pg_autoscale_bias": 4, + "pg_num_min": 64, + }, + "expected_final_pg_target": 512, + }, + 114: { + "pool_name": "cephfs.teuthology.data", + "pg_num_target": 256, + "size": 3, + "crush_rule": 0, + "capacity_ratio": 0.0006, + "options": { + "pg_num_min": 128, + }, + "expected_final_pg_target": 1024, + "expected_final_pg_target": 256, + }, + 117: { + "pool_name": "cephfs.scratch.meta", + "pg_num_target": 32, + "size": 3, + "crush_rule": 4, + "capacity_ratio": 0.0027, + "options": { + "pg_autoscale_bias": 4, + "pg_num_min": 16, + }, + "expected_final_pg_target": 64, + }, + 118: { + "pool_name": "cephfs.scratch.data", + "pg_num_target": 32, + "size": 3, + "crush_rule": 0, + "capacity_ratio": 0.0027, + "options": {}, + "expected_final_pg_target": 128, + }, + 119: { + "pool_name": "cephfs.teuthology.data-ec", + "pg_num_target": 1024, + "size": 6, + "crush_rule": 1, + "capacity_ratio": 0.8490, + "options": { + "pg_num_min": 1024 + }, + "expected_final_pg_target": 1024, + }, + 121: { + "pool_name": "cephsqlite", + "pg_num_target": 32, + "size": 3, + "crush_rule": 4, + "capacity_ratio": 0, + "options": {}, + "expected_final_pg_target": 128, + }, + } + expected_overlapped_roots = {-40, -1, -5} + self.helper_test(osd_dic, rules, pools, expected_overlapped_roots) + + def test_no_overlaps(self): + osd_dic = { + -1: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10], + -40: [11, 12, 13, 14, 15], + -5: [16, 17, 18], + } + + rules = [ + { + "rule_id": 0, + "rule_name": "data", + "ruleset": 0, + "type": 1, + "min_size": 1, + "max_size": 10, + "root_id": -1, + }, + { + "rule_id": 1, + "rule_name": "teuthology-data-ec", + "ruleset": 1, + "type": 3, + "min_size": 3, + "max_size": 6, + "root_id": -5, + }, + { + "rule_id": 4, + "rule_name": "rep-ssd", + "ruleset": 4, + "type": 1, + "min_size": 1, + "max_size": 10, + "root_id": -40, + }, + ] + pools = { + 0: { + "pool_name": "data", + "pg_num_target": 1024, + "size": 3, + "crush_rule": 0, + "capacity_ratio": 0.1624, + "options": { + "pg_num_min": 1024, + }, + "expected_final_pg_target": 1024, + }, + 1: { + "pool_name": "metadata", + "pg_num_target": 64, + "size": 3, + "crush_rule": 4, + "capacity_ratio": 0.0144, + "options": { + "pg_num_min": 64, + }, + "expected_final_pg_target": 64, + }, + 4: { + "pool_name": "libvirt-pool", + "pg_num_target": 32, + "size": 3, + "crush_rule": 0, + "capacity_ratio": 0.0001, + "options": {}, + "expected_final_pg_target": 128, + }, + 93: { + "pool_name": ".rgw.root", + "pg_num_target": 32, + "size": 3, + "crush_rule": 4, + "capacity_ratio": 0, + "options": {}, + "expected_final_pg_target": 32, + }, + 94: { + "pool_name": "default.rgw.control", + "pg_num_target": 32, + "size": 3, + "crush_rule": 4, + "capacity_ratio": 0, + "options": {}, + "expected_final_pg_target": 32, + }, + 95: { + "pool_name": "default.rgw.meta", + "pg_num_target": 32, + "size": 3, + "crush_rule": 4, + "capacity_ratio": 0, + "options": {}, + "expected_final_pg_target": 32, + }, + 96: { + "pool_name": "default.rgw.log", + "pg_num_target": 32, + "size": 3, + "crush_rule": 4, + "capacity_ratio": 0, + "options": {}, + "expected_final_pg_target": 32, + }, + 97: { + "pool_name": "default.rgw.buckets.index", + "pg_num_target": 32, + "size": 3, + "crush_rule": 4, + "capacity_ratio": 0.0002, + "options": {}, + "expected_final_pg_target": 32, + }, + 98: { + "pool_name": "default.rgw.buckets.data", + "pg_num_target": 32, + "size": 3, + "crush_rule": 0, + "capacity_ratio": 0.0457, + "options": {}, + "expected_final_pg_target": 128, + }, + 99: { + "pool_name": "default.rgw.buckets.non-ec", + "pg_num_target": 32, + "size": 3, + "crush_rule": 0, + "capacity_ratio": 0, + "options": {}, + "expected_final_pg_target": 32, + }, + 100: { + "pool_name": "device_health_metrics", + "pg_num_target": 1, + "size": 3, + "crush_rule": 0, + "capacity_ratio": 0, + "options": { + "pg_num_min": 1 + }, + "expected_final_pg_target": 1, + }, + 113: { + "pool_name": "cephfs.teuthology.meta", + "pg_num_target": 64, + "size": 3, + "crush_rule": 4, + "capacity_ratio": 0.1389, + "options": { + "pg_autoscale_bias": 4, + "pg_num_min": 64, + }, + "expected_final_pg_target": 512, + }, + 114: { + "pool_name": "cephfs.teuthology.data", + "pg_num_target": 256, + "size": 3, + "crush_rule": 0, + "capacity_ratio": 0.0006, + "options": { + "pg_num_min": 128, + }, + "expected_final_pg_target": 1024, + "expected_final_pg_target": 256, + }, + 117: { + "pool_name": "cephfs.scratch.meta", + "pg_num_target": 32, + "size": 3, + "crush_rule": 4, + "capacity_ratio": 0.0027, + "options": { + "pg_autoscale_bias": 4, + "pg_num_min": 16, + }, + "expected_final_pg_target": 64, + }, + 118: { + "pool_name": "cephfs.scratch.data", + "pg_num_target": 32, + "size": 3, + "crush_rule": 0, + "capacity_ratio": 0.0027, + "options": {}, + "expected_final_pg_target": 128, + }, + 119: { + "pool_name": "cephfs.teuthology.data-ec", + "pg_num_target": 1024, + "size": 6, + "crush_rule": 1, + "capacity_ratio": 0.8490, + "options": { + "pg_num_min": 1024 + }, + "expected_final_pg_target": 1024, + }, + 121: { + "pool_name": "cephsqlite", + "pg_num_target": 32, + "size": 3, + "crush_rule": 4, + "capacity_ratio": 0, + "options": {}, + "expected_final_pg_target": 128, + }, + } + expected_overlapped_roots = set() + self.helper_test(osd_dic, rules, pools, expected_overlapped_roots) -- 2.39.5