From d90a3746deb3e6fa94ee1a9f060a6d8043abc15a Mon Sep 17 00:00:00 2001 From: Kamoltat Date: Mon, 13 Dec 2021 15:30:17 +0000 Subject: [PATCH] pg_autoscaler/test: Modified unit-test for bulk flag Modified the unit-test cases to account for bulk flag and remove any `profile` related things. Signed-off-by: Kamoltat --- .../tests/test_cal_final_pg_target.py | 326 +++++++++++------- 1 file changed, 195 insertions(+), 131 deletions(-) diff --git a/src/pybind/mgr/pg_autoscaler/tests/test_cal_final_pg_target.py b/src/pybind/mgr/pg_autoscaler/tests/test_cal_final_pg_target.py index 9a30114b9d17a..d8ba83d111af4 100644 --- a/src/pybind/mgr/pg_autoscaler/tests/test_cal_final_pg_target.py +++ b/src/pybind/mgr/pg_autoscaler/tests/test_cal_final_pg_target.py @@ -22,99 +22,126 @@ class TestPgAutoscaler(object): # a bunch of attributes for testing. self.autoscaler = module.PgAutoscaler('module_name', 0, 0) - def helper_test(self, pools, root_map, bias, profile, overlapped_roots): - # Here we simulate how _calc_pool_target() works. + def helper_test(self, pools, root_map, bias, overlapped_roots): + # Here we simulate how _get_pool_pg_target() works. + + bulk_pools = {} even_pools = {} + + # first pass for pool_name, p in pools.items(): root_id = p['root_id'] - if root_id in overlapped_roots and profile == "scale-down": - # for scale-down profile skip pools - # with overlapping roots + if root_id in overlapped_roots: + # skip pools with overlapping roots assert p['no_scale'] continue - final_ratio, pool_pg_target, final_pg_target = self.autoscaler._calc_final_pg_target(p, pool_name, root_map, - p['root_id'], p['capacity_ratio'], even_pools, bias, True, profile) + final_ratio, pool_pg_target, final_pg_target = self.autoscaler._calc_final_pg_target( + p, pool_name, root_map, + p['root_id'], p['capacity_ratio'], + bias, even_pools, bulk_pools, 'first', p['bulk']) if final_ratio == None: # no final_ratio means current pool is an even pool # and we do not have to do any assertion on it. - # You will never hit this case with a scale up profile. continue assert p['expected_final_pg_target'] == final_pg_target assert p['expected_final_ratio'] == final_ratio + assert not p['expected_bulk_pool'] and pool_name not in bulk_pools - if profile == "scale-down": - # We only care about even_pools when profile is a scale-down - assert not p['even_pools'] and pool_name not in even_pools + # second pass + for pool_name, p in bulk_pools.items(): + final_ratio, pool_pg_target, final_pg_target = self.autoscaler._calc_final_pg_target( + p, pool_name, root_map, + p['root_id'], p['capacity_ratio'], + bias, even_pools, bulk_pools, 'second', p['bulk']) - if profile == "scale-down": - for pool_name, p in even_pools.items(): - final_ratio, pool_pg_target, final_pg_target = self.autoscaler._calc_final_pg_target(p, pool_name, root_map, - p['root_id'], p['capacity_ratio'], even_pools, bias, False, profile) + if final_ratio == None: + # no final_ratio means current pool is an even pool + # and we do not have to do any assertion on it. + continue + + assert p['expected_final_pg_target'] == final_pg_target + assert p['expected_final_ratio'] == final_ratio + assert not p['even_pools'] and pool_name not in even_pools - assert p['expected_final_pg_target'] == final_pg_target - assert p['expected_final_ratio'] == final_ratio - assert p['even_pools'] and pool_name in even_pools + #third pass + for pool_name, p in even_pools.items(): + final_ratio, pool_pg_target, final_pg_target = self.autoscaler._calc_final_pg_target( + p, pool_name, root_map, + p['root_id'], p['capacity_ratio'], + bias, even_pools, bulk_pools, 'third', p['bulk']) - def test_all_even_pools_scale_up(self): + assert p['expected_final_pg_target'] == final_pg_target + assert p['expected_final_ratio'] == final_ratio + assert p['even_pools'] and pool_name in even_pools + + def test_even_pools_one_meta_three_bulk(self): pools = { - "test0": { + "meta_0": { "pool": 0, - "pool_name": "test0", + "pool_name": "meta_0", "pg_num_target": 32, "capacity_ratio": 0.2, "root_id": 0, "expected_final_pg_target": 64, "expected_final_ratio": 0.2, - "even_pools": True, + "expected_bulk_pool": False, + "even_pools": False, "size": 1, "no_scale": False, + "bulk": False, }, - "test1": { + "bulk_0": { "pool": 1, - "pool_name": "test1", + "pool_name": "bulk_0", "pg_num_target": 32, "capacity_ratio": 0.2, "root_id": 0, - "expected_final_pg_target": 64, - "expected_final_ratio": 0.2, + "expected_final_pg_target": 128, + "expected_final_ratio": 1/3, + "expected_bulk_pool": True, "even_pools": True, "size": 1, "no_scale": False, + "bulk": True, }, - "test2": { + "bulk_1": { "pool": 2, - "pool_name": "test2", + "pool_name": "bulk_1", "pg_num_target": 32, "capacity_ratio": 0.2, "root_id": 0, - "expected_final_pg_target": 64, - "expected_final_ratio": 0.2, + "expected_final_pg_target": 128, + "expected_final_ratio": 1/3, + "expected_bulk_pool": True, "even_pools": True, "size": 1, "no_scale": False, + "bulk": True, }, - "test3": { + "bulk_2": { "pool": 3, - "pool_name": "test3", + "pool_name": "bulk_2", "pg_num_target": 32, "capacity_ratio": 0.1, "root_id": 0, - "expected_final_pg_target": 32, - "expected_final_ratio": 0.1, + "expected_final_pg_target": 128, + "expected_final_ratio": 1/3, + "expected_bulk_pool": True, "even_pools": True, "size": 1, "no_scale": False, + "bulk": True, }, } @@ -126,57 +153,62 @@ class TestPgAutoscaler(object): } - profile = "scale-up" bias = 1 overlapped_roots = set() - self.helper_test(pools, root_map, bias, profile, overlapped_roots) + self.helper_test(pools, root_map, bias, overlapped_roots) - def test_all_even_pools_scale_down(self): + def test_even_pools_two_meta_two_bulk(self): pools = { - "test0": { + "meta0": { "pool": 0, - "pool_name": "test0", + "pool_name": "meta0", "pg_num_target": 32, "capacity_ratio": 0.2, "root_id": 0, - "expected_final_pg_target": 128, - "expected_final_ratio": 0.25, + "expected_final_pg_target": 64, + "expected_final_ratio": 0.2, + "expected_bulk_pool": False, "even_pools": True, "size": 1, "no_scale": False, + "bulk": False, }, - "test1": { + "meta1": { "pool": 1, - "pool_name": "test1", + "pool_name": "meta1", "pg_num_target": 32, "capacity_ratio": 0.2, "root_id": 0, - "expected_final_pg_target": 128, - "expected_final_ratio": 0.25, + "expected_final_pg_target": 64, + "expected_final_ratio": 0.2, + "expected_bulk_pool": False, "even_pools": True, "size": 1, "no_scale": False, + "bulk": False, }, - "test2": { + "bulk0": { "pool": 2, - "pool_name": "test2", + "pool_name": "bulk0", "pg_num_target": 32, "capacity_ratio": 0.2, "root_id": 0, "expected_final_pg_target": 128, - "expected_final_ratio": 0.25, + "expected_final_ratio": 0.5, + "expected_bulk_pool": True, "even_pools": True, "size": 1, "no_scale": False, + "bulk": True, }, - "test3": { + "bulk1": { "pool": 3, "pool_name": "test3", @@ -184,10 +216,12 @@ class TestPgAutoscaler(object): "capacity_ratio": 0.1, "root_id": 0, "expected_final_pg_target": 128, - "expected_final_ratio": 0.25, + "expected_final_ratio": 0.5, + "expected_bulk_pool": True, "even_pools": True, "size": 1, "no_scale": False, + "bulk": True, }, } @@ -199,68 +233,75 @@ class TestPgAutoscaler(object): } - profile = "scale-down" bias = 1 overlapped_roots = set() - self.helper_test(pools, root_map, bias, profile, overlapped_roots) + self.helper_test(pools, root_map, bias, overlapped_roots) - def test_uneven_pools_scale_up(self): + def test_uneven_pools_one_meta_three_bulk(self): pools = { - "test0": { + "meta0": { "pool": 0, - "pool_name": "test0", + "pool_name": "meta0", "pg_num_target": 32, "capacity_ratio": 0.1, "root_id": 0, "expected_final_pg_target": 32, "expected_final_ratio": 0.1, + "expected_bulk_pool": False, "even_pools": True, "size": 1, "no_scale": False, + "bulk": False, }, - "test1": { + "bulk0": { "pool": 1, - "pool_name": "test1", + "pool_name": "bulk0", "pg_num_target": 32, "capacity_ratio": 0.5, "root_id": 0, - "expected_final_pg_target": 256, + "expected_final_pg_target": 128, "expected_final_ratio": 0.5, + "expected_bulk_pool": True, "even_pools": False, "size": 1, "no_scale": False, + "bulk": True, }, - "test2": { + "bulk1": { "pool": 2, - "pool_name": "test2", + "pool_name": "bulk1", "pg_num_target": 32, "capacity_ratio": 0.1, "root_id": 0, - "expected_final_pg_target": 32, - "expected_final_ratio": 0.1, + "expected_final_pg_target": 64, + "expected_final_ratio": 0.5, + "expected_bulk_pool": True, "even_pools": True, "size": 1, "no_scale": False, + "bulk": True, }, - "test3": { + "bulk2": { "pool": 3, - "pool_name": "test3", + "pool_name": "bulk2", "pg_num_target": 32, "capacity_ratio": 0.1, "root_id": 0, - "expected_final_pg_target": 32, - "expected_final_ratio": 0.1, + "expected_final_pg_target": 64, + "expected_final_ratio": 0.5, + "expected_bulk_pool": True, "even_pools": True, "size": 1, "no_scale": False, + "bulk": True, }, } @@ -272,68 +313,75 @@ class TestPgAutoscaler(object): } - profile = "scale-up" bias = 1 overlapped_roots = set() - self.helper_test(pools, root_map, bias, profile, overlapped_roots) + self.helper_test(pools, root_map, bias, overlapped_roots) - def test_uneven_pools_scale_down(self): + def test_uneven_pools_two_meta_two_bulk(self): pools = { - "test0": { + "meta0": { "pool": 0, - "pool_name": "test0", + "pool_name": "meta0", "pg_num_target": 32, "capacity_ratio": 0.1, "root_id": 0, - "expected_final_pg_target": 64, - "expected_final_ratio": 1/3, + "expected_final_pg_target": 32, + "expected_final_ratio": 0.1, + "expected_bulk_pool": False, "even_pools": True, "size": 1, "no_scale": False, + "bulk": False, }, - "test1": { + "meta1": { "pool": 1, - "pool_name": "test1", + "pool_name": "meta1", "pg_num_target": 32, - "capacity_ratio": 0.5, + "capacity_ratio": 0.1, "root_id": 0, - "expected_final_pg_target": 256, - "expected_final_ratio": 0.5, + "expected_final_pg_target": 32, + "expected_final_ratio": 0.1, + "expected_bulk_pool": False, "even_pools": False, "size": 1, "no_scale": False, + "bulk": False, }, - "test2": { + "bulk0": { "pool": 2, - "pool_name": "test2", + "pool_name": "bulk0", "pg_num_target": 32, - "capacity_ratio": 0.1, + "capacity_ratio": 0.5, "root_id": 0, - "expected_final_pg_target": 64, - "expected_final_ratio": 1/3, + "expected_final_pg_target": 128, + "expected_final_ratio": 0.5, + "expected_bulk_pool": True, "even_pools": True, "size": 1, "no_scale": False, + "bulk": True, }, - "test3": { + "bulk1": { "pool": 3, - "pool_name": "test3", + "pool_name": "bulk1", "pg_num_target": 32, "capacity_ratio": 0.1, "root_id": 0, - "expected_final_pg_target": 64, - "expected_final_ratio": 1/3, + "expected_final_pg_target": 128, + "expected_final_ratio": 0.5, + "expected_bulk_pool": True, "even_pools": True, "size": 1, "no_scale": False, + "bulk": True, }, } @@ -345,82 +393,91 @@ class TestPgAutoscaler(object): } - profile = "scale-down" bias = 1 overlapped_roots = set() - self.helper_test(pools, root_map, bias, profile, overlapped_roots) + self.helper_test(pools, root_map, bias, overlapped_roots) - def test_uneven_pools_with_diff_roots_scale_up(self): + def test_uneven_pools_with_diff_roots(self): pools = { - "test0": { + "meta0": { "pool": 0, - "pool_name": "test0", + "pool_name": "meta0", "pg_num_target": 32, - "capacity_ratio": 0.4, + "capacity_ratio": 0.3, "root_id": 0, - "expected_final_pg_target": 2048, - "expected_final_ratio": 0.4, + "expected_final_pg_target": 1024, + "expected_final_ratio": 0.3, + "expected_bulk_pool": False, "even_pools": False, "size": 1, "no_scale": False, + "bulk": False, }, - "test1": { + "meta1": { "pool": 1, - "pool_name": "test1", + "pool_name": "meta1", "pg_num_target": 32, "capacity_ratio": 0.6, "root_id": 1, "expected_final_pg_target": 2048, "expected_final_ratio": 0.6, + "expected_bulk_pool": False, "even_pools": False, "size": 1, "no_scale": False, + "bulk": False, }, - "test2": { + "bulk2": { "pool": 2, - "pool_name": "test2", + "pool_name": "bulk2", "pg_num_target": 32, - "capacity_ratio": 0.5, + "capacity_ratio": 0.6, "root_id": 0, "expected_final_pg_target": 2048, - "expected_final_ratio": 0.5, + "expected_final_ratio": 0.6, + "expected_bulk_pool": True, "even_pools": False, "size": 1, "no_scale": False, + "bulk": True, }, - "test3": { + "bulk3": { "pool": 3, "pool_name": "test3", "pg_num_target": 32, "capacity_ratio": 0.1, "root_id": 0, - "expected_final_pg_target": 512, - "expected_final_ratio": 0.1, + "expected_final_pg_target": 1024, + "expected_final_ratio": 1, + "expected_bulk_pool": True, "even_pools": True, "size": 1, "no_scale": False, + "bulk": True, }, - "test4": { + "bulk4": { "pool": 4, - "pool_name": "test4", + "pool_name": "bulk4", "pg_num_target": 32, "capacity_ratio": 0.4, "root_id": 1, "expected_final_pg_target": 2048, - "expected_final_ratio": 0.4, + "expected_final_ratio": 1, + "expected_bulk_pool": True, "even_pools": True, "size": 1, "no_scale": False, + "bulk": True, }, } @@ -432,82 +489,91 @@ class TestPgAutoscaler(object): } - profile = "scale-up" bias = 1 overlapped_roots = set() - self.helper_test(pools, root_map, bias, profile, overlapped_roots) + self.helper_test(pools, root_map, bias, overlapped_roots) - def test_uneven_pools_with_diff_roots_scale_down(self): + def test_even_pools_with_diff_roots(self): pools = { - "test0": { + "meta0": { "pool": 0, - "pool_name": "test0", + "pool_name": "meta0", "pg_num_target": 32, "capacity_ratio": 0.4, "root_id": 0, "expected_final_pg_target": 2048, "expected_final_ratio": 0.4, + "expected_bulk_pool": False, "even_pools": False, "size": 1, "no_scale": False, + "bulk": False, }, - "test1": { + "meta1": { "pool": 1, - "pool_name": "test1", + "pool_name": "meta1", "pg_num_target": 32, "capacity_ratio": 0.6, "root_id": 1, "expected_final_pg_target": 2048, "expected_final_ratio": 0.6, + "expected_bulk_pool": False, "even_pools": False, "size": 1, "no_scale": False, + "bulk": False, }, - "test2": { + "bulk1": { "pool": 2, - "pool_name": "test2", + "pool_name": "bulk1", "pg_num_target": 32, - "capacity_ratio": 0.5, + "capacity_ratio": 0.2, "root_id": 0, - "expected_final_pg_target": 2048, + "expected_final_pg_target": 1024, "expected_final_ratio": 0.5, - "even_pools": False, + "expected_bulk_pool": True, + "even_pools": True, "size": 1, "no_scale": False, + "bulk": True, }, - "test3": { + "bulk2": { "pool": 3, - "pool_name": "test3", + "pool_name": "bulk2", "pg_num_target": 32, "capacity_ratio": 0.1, "root_id": 0, - "expected_final_pg_target": 512, - "expected_final_ratio": 1, + "expected_final_pg_target": 1024, + "expected_final_ratio": 0.5, + "expected_bulk_pool": True, "even_pools": True, "size": 1, "no_scale": False, + "bulk": True, }, - "test4": { + "bulk3": { "pool": 4, - "pool_name": "test4", + "pool_name": "bulk4", "pg_num_target": 32, - "capacity_ratio": 0.4, + "capacity_ratio": 0.25, "root_id": 1, "expected_final_pg_target": 2048, "expected_final_ratio": 1, + "expected_bulk_pool": True, "even_pools": True, "size": 1, "no_scale": False, + "bulk": True, }, } @@ -519,12 +585,11 @@ class TestPgAutoscaler(object): } - profile = "scale-down" bias = 1 overlapped_roots = set() - self.helper_test(pools, root_map, bias, profile, overlapped_roots) + self.helper_test(pools, root_map, bias, overlapped_roots) - def test_uneven_pools_with_overllaped_roots_scale_down(self): + def test_uneven_pools_with_overlapped_roots(self): pools = { "test0": { @@ -606,7 +671,6 @@ class TestPgAutoscaler(object): } - profile = "scale-down" bias = 1 overlapped_roots = {0, 1} - self.helper_test(pools, root_map, bias, profile, overlapped_roots) + self.helper_test(pools, root_map, bias, overlapped_roots) -- 2.39.5