From: Neha Ojha Date: Thu, 18 Feb 2021 19:46:00 +0000 (+0000) Subject: Revert "mgr/pg_autoscaler: avoid scale-down until there is pressure" X-Git-Tag: v15.2.9~4^2~1 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=7d446d954adf065d1d59a16c4f08bfcd507cf64f;p=ceph.git Revert "mgr/pg_autoscaler: avoid scale-down until there is pressure" This reverts commit 49dba6edaa807f2c1047c40ce96a76fc5f409b82. Reason: Needs to bake more given that it may cause pg budget issues when multiple pools are created at the same time. More details in https://tracker.ceph.com/issues/49364. Signed-off-by: Neha Ojha --- diff --git a/qa/workunits/mon/pg_autoscaler.sh b/qa/workunits/mon/pg_autoscaler.sh index 3d24b1a6c50c..706f87d0e1bd 100755 --- a/qa/workunits/mon/pg_autoscaler.sh +++ b/qa/workunits/mon/pg_autoscaler.sh @@ -30,8 +30,6 @@ function wait_for() { return 0 } -function power2() { echo "x=l($1)/l(2); scale=0; 2^((x+0.5)/1)" | bc -l;} - # enable ceph config set mgr mgr/pg_autoscaler/sleep_interval 5 ceph mgr module enable pg_autoscaler @@ -42,20 +40,8 @@ ceph osd pool create b 16 --pg-num-min 2 ceph osd pool set a pg_autoscale_mode on ceph osd pool set b pg_autoscale_mode on -# get num pools again since we created more pools -NUM_POOLS=$(ceph osd pool ls | wc -l) - -# get pool size -POOL_SIZE_A=$(ceph osd pool get a size| grep -Eo '[0-9]{1,4}') -POOL_SIZE_B=$(ceph osd pool get b size| grep -Eo '[0-9]{1,4}') - -# calculate target pg of each pools -TARGET_PG_A=$(power2 $((($NUM_OSDS * 100)/($NUM_POOLS)/($POOL_SIZE_A)))) -TARGET_PG_B=$(power2 $((($NUM_OSDS * 100)/($NUM_POOLS)/($POOL_SIZE_B)))) - -# evaluate target_pg against pg num of each pools -wait_for 120 "ceph osd pool get a pg_num | grep $TARGET_PG_A" -wait_for 120 "ceph osd pool get b pg_num | grep $TARGET_PG_B" +wait_for 120 "ceph osd pool get a pg_num | grep 4" +wait_for 120 "ceph osd pool get b pg_num | grep 2" # target ratio ceph osd pool set a target_size_ratio 5 diff --git a/src/pybind/mgr/pg_autoscaler/__init__.py b/src/pybind/mgr/pg_autoscaler/__init__.py index 2394d37e5692..f0bffcdcd629 100644 --- a/src/pybind/mgr/pg_autoscaler/__init__.py +++ b/src/pybind/mgr/pg_autoscaler/__init__.py @@ -1,6 +1 @@ -import os - -if 'UNITTEST' in os.environ: - import tests - from .module import PgAutoscaler, effective_target_ratio diff --git a/src/pybind/mgr/pg_autoscaler/module.py b/src/pybind/mgr/pg_autoscaler/module.py index eebd4b091d95..64e2a7465831 100644 --- a/src/pybind/mgr/pg_autoscaler/module.py +++ b/src/pybind/mgr/pg_autoscaler/module.py @@ -229,12 +229,9 @@ class PgAutoscaler(MgrModule): self.osd_count = None # Number of OSDs self.pg_target = None # Ideal full-capacity PG count? self.pg_current = 0 # How many PGs already? - self.pg_left = 0 self.capacity = None # Total capacity of OSDs in subtree self.pool_ids = [] self.pool_names = [] - self.pool_count = None - self.pool_used = 0 self.total_target_ratio = 0.0 self.total_target_bytes = 0 # including replication / EC overhead @@ -273,8 +270,7 @@ class PgAutoscaler(MgrModule): for s in roots: s.osd_count = len(s.osds) s.pg_target = s.osd_count * self.mon_target_pg_per_osd - s.pg_left = s.pg_target - s.pool_count = len(s.pool_ids) + capacity = 0.0 for osd_stats in all_stats['osd_stats']: if osd_stats['osd'] in s.osds: @@ -294,80 +290,25 @@ class PgAutoscaler(MgrModule): return result, pool_root - def _calc_final_pg_target( - self, - p, - pool_name, - root_map, - root_id, - capacity_ratio, - even_pools, - bias, - is_used, + def _get_pool_status( + self, + osdmap, + pools, + threshold=3.0, ): - """ - is_used flag used to determine if this is the first - pass where the caller tries to calculate/adjust pools that has - used_ratio > even_ratio else this is the second pass, - we calculate final_ratio by giving it 1 / pool_count - of the root we are looking. - """ - if is_used: - even_ratio = 1 / root_map[root_id].pool_count - used_ratio = capacity_ratio + assert threshold >= 2.0 - if used_ratio > even_ratio: - root_map[root_id].pool_used += 1 - else: - # keep track of even_pools to be used in second pass - # of the caller function - even_pools[pool_name] = p - return None, None, None + crush_map = osdmap.get_crush() - final_ratio = max(used_ratio, even_ratio) - used_pg = final_ratio * root_map[root_id].pg_target - root_map[root_id].pg_left -= used_pg - pool_pg_target = used_pg / p['size'] * bias + root_map, pool_root = self.get_subtree_resource_status(osdmap, crush_map) - else: - final_ratio = 1 / (root_map[root_id].pool_count - root_map[root_id].pool_used) - pool_pg_target = (final_ratio * root_map[root_id].pg_left) / p['size'] * bias - - final_pg_target = max(p.get('options', {}).get('pg_num_min', PG_NUM_MIN), - nearest_power_of_two(pool_pg_target)) - - self.log.info("Pool '{0}' root_id {1} using {2} of space, bias {3}, " - "pg target {4} quantized to {5} (current {6})".format( - p['pool_name'], - root_id, - capacity_ratio, - bias, - pool_pg_target, - final_pg_target, - p['pg_num_target'] - )) - - return final_ratio, pool_pg_target, final_pg_target - - def _calc_pool_targets( - self, - osdmap, - pools, - crush_map, - root_map, - pool_root, - pool_stats, - ret, - threshold, - is_used, - ): - """ - Calculates final_pg_target of each pools and determine if it needs - scaling by starting out with a full complement of pgs and only - descreasing it when other pools need more due to increased usage. - """ - even_pools = {} - for pool_name, p in pools.items(): + df = self.get('df') + pool_stats = dict([(p['id'], p['stats']) for p in df['pools']]) + + ret = [] + + # iterate over all pools to determine how they should be sized + for pool_name, p in iteritems(pools): pool_id = p['pool'] if pool_id not in pool_stats: # race with pool deletion; skip @@ -377,7 +318,6 @@ class PgAutoscaler(MgrModule): # may not be true. cr_name = crush_map.get_rule_by_id(p['crush_rule'])['rule_name'] root_id = int(crush_map.get_rule_root(cr_name)) - pool_root[pool_name] = root_id capacity = root_map[root_id].capacity @@ -406,18 +346,29 @@ class PgAutoscaler(MgrModule): root_map[root_id].total_target_ratio, root_map[root_id].total_target_bytes, capacity)) - target_ratio = effective_target_ratio(p['options'].get('target_size_ratio', 0.0), root_map[root_id].total_target_ratio, root_map[root_id].total_target_bytes, capacity) - capacity_ratio = max(capacity_ratio, target_ratio) - final_ratio, pool_pg_target, final_pg_target = self._calc_final_pg_target(p, - pool_name, root_map, root_id, capacity_ratio, even_pools, bias, is_used) + final_ratio = max(capacity_ratio, target_ratio) - if final_ratio == None: - continue + # So what proportion of pg allowance should we be using? + pool_pg_target = (final_ratio * root_map[root_id].pg_target) / p['size'] * bias + + final_pg_target = max(p['options'].get('pg_num_min', PG_NUM_MIN), + nearest_power_of_two(pool_pg_target)) + + self.log.info("Pool '{0}' root_id {1} using {2} of space, bias {3}, " + "pg target {4} quantized to {5} (current {6})".format( + p['pool_name'], + root_id, + final_ratio, + bias, + pool_pg_target, + final_pg_target, + p['pg_num_target'] + )) adjust = False if (final_pg_target > p['pg_num_target'] * threshold or \ @@ -448,35 +399,6 @@ class PgAutoscaler(MgrModule): 'bias': p.get('options', {}).get('pg_autoscale_bias', 1.0), }); - return ret, even_pools - - - def _get_pool_status( - self, - osdmap, - pools, - threshold=3.0, - ): - assert threshold >= 2.0 - - crush_map = osdmap.get_crush() - root_map, pool_root = self.get_subtree_resource_status(osdmap, crush_map) - df = self.get('df') - pool_stats = dict([(p['id'], p['stats']) for p in df['pools']]) - - ret = [] - # Iterate over all pools to determine how they should be sized. - # First call is to find/adjust pools that uses more capacaity than - # the even_ratio of other pools and we adjust those first. - # Second call make use of the even_pools we keep track of in the first call. - # All we need to do is iterate over those and give them 1/pool_count of the - # total pgs. - ret, even_pools = self._calc_pool_targets(osdmap, pools, crush_map, root_map, pool_root, - pool_stats, ret, threshold, True) - - ret, _ = self._calc_pool_targets(osdmap, even_pools, crush_map, root_map, pool_root, - pool_stats, ret, threshold, False) - return (ret, root_map, pool_root) def _update_progress_events(self): diff --git a/src/pybind/mgr/pg_autoscaler/tests/__init__.py b/src/pybind/mgr/pg_autoscaler/tests/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/src/pybind/mgr/pg_autoscaler/tests/test_cal_final_pg_target.py b/src/pybind/mgr/pg_autoscaler/tests/test_cal_final_pg_target.py deleted file mode 100644 index 78960eca652c..000000000000 --- a/src/pybind/mgr/pg_autoscaler/tests/test_cal_final_pg_target.py +++ /dev/null @@ -1,255 +0,0 @@ -#python unit test -import unittest -from tests import mock -import pytest -import json -from pg_autoscaler import module - -class RootMapItem: - - def __init__(self, pool_count, pg_target, pg_left): - - self.pool_count = pool_count - self.pg_target = pg_target - self.pg_left = pg_left - self.pool_used = 0 - -class TestPgAutoscaler(object): - - def setup(self): - # a bunch of attributes for testing - self.autoscaler = module.PgAutoscaler('module_name', 0, 0) - - def helper_test(self, pools, root_map, bias): - - even_pools = {} - for pool_name, p in pools.items(): - final_ratio, pool_pg_target, final_pg_target = self.autoscaler._calc_final_pg_target(p, pool_name, root_map, p['root_id'], p['capacity_ratio'], even_pools, bias, True) - - if final_ratio == None: - continue - - assert p['expected_final_pg_target'] == final_pg_target - assert p['expected_final_ratio'] == final_ratio - assert not p['even_pools'] and pool_name not in even_pools - - for pool_name, p in even_pools.items(): - final_ratio, pool_pg_target, final_pg_target = self.autoscaler._calc_final_pg_target(p, pool_name, root_map, p['root_id'], p['capacity_ratio'], even_pools, bias, False) - - assert p['expected_final_pg_target'] == final_pg_target - assert p['expected_final_ratio'] == final_ratio - assert p['even_pools'] and pool_name in even_pools - - def test_all_even_pools(self): - pools = { - - "test0":{ - - "pool": 0, - "pool_name": "test0", - "pg_num_target": 32, - "capacity_ratio": 0.2, - "root_id":"0", - "expected_final_pg_target": 128, - "expected_final_ratio": 0.25, - "even_pools": True, - "size": 1, - }, - - "test1":{ - - "pool": 1, - "pool_name": "test1", - "pg_num_target": 32, - "capacity_ratio": 0.2, - "root_id":"0", - "expected_final_pg_target": 128, - "expected_final_ratio": 0.25, - "even_pools": True, - "size": 1, - }, - - "test2":{ - - "pool": 2, - "pool_name": "test2", - "pg_num_target": 32, - "capacity_ratio": 0.2, - "root_id":"0", - "expected_final_pg_target": 128, - "expected_final_ratio": 0.25, - "even_pools": True, - "size": 1, - }, - - "test3":{ - - "pool": 3, - "pool_name": "test3", - "pg_num_target": 32, - "capacity_ratio": 0.1, - "root_id": "0", - "expected_final_pg_target": 128, - "expected_final_ratio": 0.25, - "even_pools": True, - "size": 1, - }, - - } - - root_map = { - - "0": RootMapItem(4, 400, 400), - "1": RootMapItem(4, 400, 400), - - } - - bias = 1 - self.helper_test(pools, root_map, bias) - - def test_uneven_pools(self): - pools = { - - "test0":{ - - "pool": 0, - "pool_name": "test0", - "pg_num_target": 32, - "capacity_ratio": 0.1, - "root_id":"0", - "expected_final_pg_target": 64, - "expected_final_ratio": 1/3, - "even_pools": True, - "size": 1, - }, - - "test1":{ - - "pool": 1, - "pool_name": "test1", - "pg_num_target": 32, - "capacity_ratio": 0.5, - "root_id":"0", - "expected_final_pg_target": 256, - "expected_final_ratio": 0.5, - "even_pools": False, - "size": 1, - }, - - "test2":{ - - "pool": 2, - "pool_name": "test2", - "pg_num_target": 32, - "capacity_ratio": 0.1, - "root_id":"0", - "expected_final_pg_target": 64, - "expected_final_ratio": 1/3, - "even_pools": True, - "size": 1, - }, - - "test3":{ - - "pool": 3, - "pool_name": "test3", - "pg_num_target": 32, - "capacity_ratio": 0.1, - "root_id": "0", - "expected_final_pg_target": 64, - "expected_final_ratio": 1/3, - "even_pools": True, - "size": 1, - }, - - } - - root_map = { - - "0": RootMapItem(4, 400, 400), - "1": RootMapItem(4, 400, 400), - - } - - bias = 1 - self.helper_test(pools, root_map, bias) - - def test_uneven_pools_with_diff_roots(self): - pools = { - - "test0":{ - - "pool": 0, - "pool_name": "test0", - "pg_num_target": 32, - "capacity_ratio": 0.4, - "root_id":"0", - "expected_final_pg_target": 2048, - "expected_final_ratio": 0.4, - "even_pools": False, - "size": 1, - }, - - "test1":{ - - "pool": 1, - "pool_name": "test1", - "pg_num_target": 32, - "capacity_ratio": 0.6, - "root_id":"1", - "expected_final_pg_target": 2048, - "expected_final_ratio": 0.6, - "even_pools": False, - "size": 1, - }, - - "test2":{ - - "pool": 2, - "pool_name": "test2", - "pg_num_target": 32, - "capacity_ratio": 0.5, - "root_id":"0", - "expected_final_pg_target": 2048, - "expected_final_ratio": 0.5, - "even_pools": False, - "size": 1, - }, - - "test3":{ - - "pool": 3, - "pool_name": "test3", - "pg_num_target": 32, - "capacity_ratio": 0.1, - "root_id": "0", - "expected_final_pg_target": 512, - "expected_final_ratio": 1, - "even_pools": True, - "size": 1, - }, - - "test4":{ - - "pool": 4, - "pool_name": "test4", - "pg_num_target": 32, - "capacity_ratio": 0.4, - "root_id": "1", - "expected_final_pg_target": 2048, - "expected_final_ratio": 1, - "even_pools": True, - "size": 1, - }, - - } - - root_map = { - - "0": RootMapItem(3, 5000, 5000), - "1": RootMapItem(2, 5000, 5000), - - } - - bias = 1 - self.helper_test(pools, root_map, bias)