From: Kamoltat Date: Thu, 7 Jan 2021 15:39:19 +0000 (+0000) Subject: mgr/pg_autoscaler: avoid scale-down until there is pressure X-Git-Tag: v17.1.0~3100^2 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=daeb6f6ac0c8f77ae07147f9d1e2ed18d6d8e4cc;p=ceph.git mgr/pg_autoscaler: avoid scale-down until there is pressure The autoscaler will start out with scaling each pools to have a full complements of pgs from the start and will only decrease it when pools need more due to increased usage. Introduced a unit test that tests only the function get_final_pg_target_and_ratio() which deals with the distrubtion of pgs amongst the pools Edited workunit script to reflect the change of how pgs are calculated and distrubted. Signed-off-by: Kamoltat --- diff --git a/qa/workunits/mon/pg_autoscaler.sh b/qa/workunits/mon/pg_autoscaler.sh index 706f87d0e1bda..3d24b1a6c50c2 100755 --- a/qa/workunits/mon/pg_autoscaler.sh +++ b/qa/workunits/mon/pg_autoscaler.sh @@ -30,6 +30,8 @@ function wait_for() { return 0 } +function power2() { echo "x=l($1)/l(2); scale=0; 2^((x+0.5)/1)" | bc -l;} + # enable ceph config set mgr mgr/pg_autoscaler/sleep_interval 5 ceph mgr module enable pg_autoscaler @@ -40,8 +42,20 @@ ceph osd pool create b 16 --pg-num-min 2 ceph osd pool set a pg_autoscale_mode on ceph osd pool set b pg_autoscale_mode on -wait_for 120 "ceph osd pool get a pg_num | grep 4" -wait_for 120 "ceph osd pool get b pg_num | grep 2" +# get num pools again since we created more pools +NUM_POOLS=$(ceph osd pool ls | wc -l) + +# get pool size +POOL_SIZE_A=$(ceph osd pool get a size| grep -Eo '[0-9]{1,4}') +POOL_SIZE_B=$(ceph osd pool get b size| grep -Eo '[0-9]{1,4}') + +# calculate target pg of each pools +TARGET_PG_A=$(power2 $((($NUM_OSDS * 100)/($NUM_POOLS)/($POOL_SIZE_A)))) +TARGET_PG_B=$(power2 $((($NUM_OSDS * 100)/($NUM_POOLS)/($POOL_SIZE_B)))) + +# evaluate target_pg against pg num of each pools +wait_for 120 "ceph osd pool get a pg_num | grep $TARGET_PG_A" +wait_for 120 "ceph osd pool get b pg_num | grep $TARGET_PG_B" # target ratio ceph osd pool set a target_size_ratio 5 diff --git a/src/pybind/mgr/pg_autoscaler/__init__.py b/src/pybind/mgr/pg_autoscaler/__init__.py index f0bffcdcd6293..2394d37e5692e 100644 --- a/src/pybind/mgr/pg_autoscaler/__init__.py +++ b/src/pybind/mgr/pg_autoscaler/__init__.py @@ -1 +1,6 @@ +import os + +if 'UNITTEST' in os.environ: + import tests + from .module import PgAutoscaler, effective_target_ratio diff --git a/src/pybind/mgr/pg_autoscaler/module.py b/src/pybind/mgr/pg_autoscaler/module.py index 24a8e4a584c59..0c5bfd2523363 100644 --- a/src/pybind/mgr/pg_autoscaler/module.py +++ b/src/pybind/mgr/pg_autoscaler/module.py @@ -228,9 +228,12 @@ class PgAutoscaler(MgrModule): self.osd_count = None # Number of OSDs self.pg_target = None # Ideal full-capacity PG count? self.pg_current = 0 # How many PGs already? + self.pg_left = 0 self.capacity = None # Total capacity of OSDs in subtree self.pool_ids = [] self.pool_names = [] + self.pool_count = None + self.pool_used = 0 self.total_target_ratio = 0.0 self.total_target_bytes = 0 # including replication / EC overhead @@ -269,7 +272,8 @@ class PgAutoscaler(MgrModule): for s in roots: s.osd_count = len(s.osds) s.pg_target = s.osd_count * self.mon_target_pg_per_osd - + s.pg_left = s.pg_target + s.pool_count = len(s.pool_ids) capacity = 0.0 for osd_stats in all_stats['osd_stats']: if osd_stats['osd'] in s.osds: @@ -289,24 +293,79 @@ class PgAutoscaler(MgrModule): return result, pool_root - def _get_pool_status( - self, - osdmap, - pools, - threshold=3.0, + def _calc_final_pg_target( + self, + p, + pool_name, + root_map, + root_id, + capacity_ratio, + even_pools, + bias, + is_used, ): - assert threshold >= 2.0 - - crush_map = osdmap.get_crush() - - root_map, pool_root = self.get_subtree_resource_status(osdmap, crush_map) + """ + is_used flag used to determine if this is the first + pass where the caller tries to calculate/adjust pools that has + used_ratio > even_ratio else this is the second pass, + we calculate final_ratio by giving it 1 / pool_count + of the root we are looking. + """ + if is_used: + even_ratio = 1 / root_map[root_id].pool_count + used_ratio = capacity_ratio - df = self.get('df') - pool_stats = dict([(p['id'], p['stats']) for p in df['pools']]) + if used_ratio > even_ratio: + root_map[root_id].pool_used += 1 + else: + # keep track of even_pools to be used in second pass + # of the caller function + even_pools[pool_name] = p + return None, None, None - ret = [] + final_ratio = max(used_ratio, even_ratio) + used_pg = final_ratio * root_map[root_id].pg_target + root_map[root_id].pg_left -= used_pg + pool_pg_target = used_pg / p['size'] * bias - # iterate over all pools to determine how they should be sized + else: + final_ratio = 1 / (root_map[root_id].pool_count - root_map[root_id].pool_used) + pool_pg_target = (final_ratio * root_map[root_id].pg_left) / p['size'] * bias + + final_pg_target = max(p.get('options', {}).get('pg_num_min', PG_NUM_MIN), + nearest_power_of_two(pool_pg_target)) + + self.log.info("Pool '{0}' root_id {1} using {2} of space, bias {3}, " + "pg target {4} quantized to {5} (current {6})".format( + p['pool_name'], + root_id, + capacity_ratio, + bias, + pool_pg_target, + final_pg_target, + p['pg_num_target'] + )) + + return final_ratio, pool_pg_target, final_pg_target + + def _calc_pool_targets( + self, + osdmap, + pools, + crush_map, + root_map, + pool_root, + pool_stats, + ret, + threshold, + is_used, + ): + """ + Calculates final_pg_target of each pools and determine if it needs + scaling by starting out with a full complement of pgs and only + descreasing it when other pools need more due to increased usage. + """ + even_pools = {} for pool_name, p in pools.items(): pool_id = p['pool'] if pool_id not in pool_stats: @@ -317,6 +376,7 @@ class PgAutoscaler(MgrModule): # may not be true. cr_name = crush_map.get_rule_by_id(p['crush_rule'])['rule_name'] root_id = int(crush_map.get_rule_root(cr_name)) + pool_root[pool_name] = root_id capacity = root_map[root_id].capacity @@ -345,29 +405,18 @@ class PgAutoscaler(MgrModule): root_map[root_id].total_target_ratio, root_map[root_id].total_target_bytes, capacity)) + target_ratio = effective_target_ratio(p['options'].get('target_size_ratio', 0.0), root_map[root_id].total_target_ratio, root_map[root_id].total_target_bytes, capacity) - final_ratio = max(capacity_ratio, target_ratio) + capacity_ratio = max(capacity_ratio, target_ratio) + final_ratio, pool_pg_target, final_pg_target = self._calc_final_pg_target(p, + pool_name, root_map, root_id, capacity_ratio, even_pools, bias, is_used) - # So what proportion of pg allowance should we be using? - pool_pg_target = (final_ratio * root_map[root_id].pg_target) / p['size'] * bias - - final_pg_target = max(p['options'].get('pg_num_min', PG_NUM_MIN), - nearest_power_of_two(pool_pg_target)) - - self.log.info("Pool '{0}' root_id {1} using {2} of space, bias {3}, " - "pg target {4} quantized to {5} (current {6})".format( - p['pool_name'], - root_id, - final_ratio, - bias, - pool_pg_target, - final_pg_target, - p['pg_num_target'] - )) + if final_ratio == None: + continue adjust = False if (final_pg_target > p['pg_num_target'] * threshold or \ @@ -398,6 +447,35 @@ class PgAutoscaler(MgrModule): 'bias': p.get('options', {}).get('pg_autoscale_bias', 1.0), }); + return ret, even_pools + + + def _get_pool_status( + self, + osdmap, + pools, + threshold=3.0, + ): + assert threshold >= 2.0 + + crush_map = osdmap.get_crush() + root_map, pool_root = self.get_subtree_resource_status(osdmap, crush_map) + df = self.get('df') + pool_stats = dict([(p['id'], p['stats']) for p in df['pools']]) + + ret = [] + # Iterate over all pools to determine how they should be sized. + # First call is to find/adjust pools that uses more capacaity than + # the even_ratio of other pools and we adjust those first. + # Second call make use of the even_pools we keep track of in the first call. + # All we need to do is iterate over those and give them 1/pool_count of the + # total pgs. + ret, even_pools = self._calc_pool_targets(osdmap, pools, crush_map, root_map, pool_root, + pool_stats, ret, threshold, True) + + ret, _ = self._calc_pool_targets(osdmap, even_pools, crush_map, root_map, pool_root, + pool_stats, ret, threshold, False) + return (ret, root_map, pool_root) def _update_progress_events(self): diff --git a/src/pybind/mgr/pg_autoscaler/tests/__init__.py b/src/pybind/mgr/pg_autoscaler/tests/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/src/pybind/mgr/pg_autoscaler/tests/test_cal_final_pg_target.py b/src/pybind/mgr/pg_autoscaler/tests/test_cal_final_pg_target.py new file mode 100644 index 0000000000000..78960eca652c2 --- /dev/null +++ b/src/pybind/mgr/pg_autoscaler/tests/test_cal_final_pg_target.py @@ -0,0 +1,255 @@ +#python unit test +import unittest +from tests import mock +import pytest +import json +from pg_autoscaler import module + +class RootMapItem: + + def __init__(self, pool_count, pg_target, pg_left): + + self.pool_count = pool_count + self.pg_target = pg_target + self.pg_left = pg_left + self.pool_used = 0 + +class TestPgAutoscaler(object): + + def setup(self): + # a bunch of attributes for testing + self.autoscaler = module.PgAutoscaler('module_name', 0, 0) + + def helper_test(self, pools, root_map, bias): + + even_pools = {} + for pool_name, p in pools.items(): + final_ratio, pool_pg_target, final_pg_target = self.autoscaler._calc_final_pg_target(p, pool_name, root_map, p['root_id'], p['capacity_ratio'], even_pools, bias, True) + + if final_ratio == None: + continue + + assert p['expected_final_pg_target'] == final_pg_target + assert p['expected_final_ratio'] == final_ratio + assert not p['even_pools'] and pool_name not in even_pools + + for pool_name, p in even_pools.items(): + final_ratio, pool_pg_target, final_pg_target = self.autoscaler._calc_final_pg_target(p, pool_name, root_map, p['root_id'], p['capacity_ratio'], even_pools, bias, False) + + assert p['expected_final_pg_target'] == final_pg_target + assert p['expected_final_ratio'] == final_ratio + assert p['even_pools'] and pool_name in even_pools + + def test_all_even_pools(self): + pools = { + + "test0":{ + + "pool": 0, + "pool_name": "test0", + "pg_num_target": 32, + "capacity_ratio": 0.2, + "root_id":"0", + "expected_final_pg_target": 128, + "expected_final_ratio": 0.25, + "even_pools": True, + "size": 1, + }, + + "test1":{ + + "pool": 1, + "pool_name": "test1", + "pg_num_target": 32, + "capacity_ratio": 0.2, + "root_id":"0", + "expected_final_pg_target": 128, + "expected_final_ratio": 0.25, + "even_pools": True, + "size": 1, + }, + + "test2":{ + + "pool": 2, + "pool_name": "test2", + "pg_num_target": 32, + "capacity_ratio": 0.2, + "root_id":"0", + "expected_final_pg_target": 128, + "expected_final_ratio": 0.25, + "even_pools": True, + "size": 1, + }, + + "test3":{ + + "pool": 3, + "pool_name": "test3", + "pg_num_target": 32, + "capacity_ratio": 0.1, + "root_id": "0", + "expected_final_pg_target": 128, + "expected_final_ratio": 0.25, + "even_pools": True, + "size": 1, + }, + + } + + root_map = { + + "0": RootMapItem(4, 400, 400), + "1": RootMapItem(4, 400, 400), + + } + + bias = 1 + self.helper_test(pools, root_map, bias) + + def test_uneven_pools(self): + pools = { + + "test0":{ + + "pool": 0, + "pool_name": "test0", + "pg_num_target": 32, + "capacity_ratio": 0.1, + "root_id":"0", + "expected_final_pg_target": 64, + "expected_final_ratio": 1/3, + "even_pools": True, + "size": 1, + }, + + "test1":{ + + "pool": 1, + "pool_name": "test1", + "pg_num_target": 32, + "capacity_ratio": 0.5, + "root_id":"0", + "expected_final_pg_target": 256, + "expected_final_ratio": 0.5, + "even_pools": False, + "size": 1, + }, + + "test2":{ + + "pool": 2, + "pool_name": "test2", + "pg_num_target": 32, + "capacity_ratio": 0.1, + "root_id":"0", + "expected_final_pg_target": 64, + "expected_final_ratio": 1/3, + "even_pools": True, + "size": 1, + }, + + "test3":{ + + "pool": 3, + "pool_name": "test3", + "pg_num_target": 32, + "capacity_ratio": 0.1, + "root_id": "0", + "expected_final_pg_target": 64, + "expected_final_ratio": 1/3, + "even_pools": True, + "size": 1, + }, + + } + + root_map = { + + "0": RootMapItem(4, 400, 400), + "1": RootMapItem(4, 400, 400), + + } + + bias = 1 + self.helper_test(pools, root_map, bias) + + def test_uneven_pools_with_diff_roots(self): + pools = { + + "test0":{ + + "pool": 0, + "pool_name": "test0", + "pg_num_target": 32, + "capacity_ratio": 0.4, + "root_id":"0", + "expected_final_pg_target": 2048, + "expected_final_ratio": 0.4, + "even_pools": False, + "size": 1, + }, + + "test1":{ + + "pool": 1, + "pool_name": "test1", + "pg_num_target": 32, + "capacity_ratio": 0.6, + "root_id":"1", + "expected_final_pg_target": 2048, + "expected_final_ratio": 0.6, + "even_pools": False, + "size": 1, + }, + + "test2":{ + + "pool": 2, + "pool_name": "test2", + "pg_num_target": 32, + "capacity_ratio": 0.5, + "root_id":"0", + "expected_final_pg_target": 2048, + "expected_final_ratio": 0.5, + "even_pools": False, + "size": 1, + }, + + "test3":{ + + "pool": 3, + "pool_name": "test3", + "pg_num_target": 32, + "capacity_ratio": 0.1, + "root_id": "0", + "expected_final_pg_target": 512, + "expected_final_ratio": 1, + "even_pools": True, + "size": 1, + }, + + "test4":{ + + "pool": 4, + "pool_name": "test4", + "pg_num_target": 32, + "capacity_ratio": 0.4, + "root_id": "1", + "expected_final_pg_target": 2048, + "expected_final_ratio": 1, + "even_pools": True, + "size": 1, + }, + + } + + root_map = { + + "0": RootMapItem(3, 5000, 5000), + "1": RootMapItem(2, 5000, 5000), + + } + + bias = 1 + self.helper_test(pools, root_map, bias)