From 6bc9770b2b8ba5accd36b0e5d4f8a210669d03cd Mon Sep 17 00:00:00 2001 From: Kyr Shatskyy Date: Wed, 13 May 2020 09:54:06 +0200 Subject: [PATCH] teuthology-describe-tests: add --summary command Add --summary argument to teuthology-describe-tests for estimating run capacity, printing descriptions, dumping matrix, yaml fragments. Refactor get_combinations code to use the same code that is used by teuthology-suite for filtering. Signed-off-by: Kyr Shatskyy --- scripts/describe_tests.py | 26 ++++- teuthology/describe_tests.py | 146 ++++++++++++++++++++----- teuthology/test/test_describe_tests.py | 72 ++++++++++-- 3 files changed, 199 insertions(+), 45 deletions(-) diff --git a/scripts/describe_tests.py b/scripts/describe_tests.py index 2adeb5973e..1c17a095fa 100644 --- a/scripts/describe_tests.py +++ b/scripts/describe_tests.py @@ -37,12 +37,23 @@ optional arguments: options only for describing combinations represented by a suite: -c, --combinations Describe test combinations rather than individual yaml fragments - --filter Only list jobs whose filenames contain at - least one of the keywords in the comma - separated keyword string specified. - --filter-out Do not list jobs whose filenames contain + -s, --summary Print summary + --filter Only list tests whose description contains + at least one of the keywords in the comma + separated keyword string specified + --filter-out Do not list tests whose description contains any of the keywords in the comma separated - keyword string specified. + keyword string specified + --filter-all Only list tests whose description contains + each of the keywords in the comma separated + keyword string specified + -F, --filter-fragments Check fragments additionaly to descriptions + using keywords specified with 'filter', + 'filter-out' and 'filter-all' options. + -D, --print-description Print job descriptions for the suite, + used only in combination with 'summary' + -L, --print-fragments Print file list inovolved for each facet, + used only in combination with 'summary' -l , --limit List at most this many jobs [default: 0] --subset Instead of listing the entire @@ -54,6 +65,11 @@ options only for describing combinations represented by a suite: 2/ ... -1/ will list all jobs in the suite (many more than once). + -S , --seed Used for pseudo-random tests generation + involving facet whose path ends with '$' + operator, where negative value used for + a random seed + [default: -1] """ diff --git a/teuthology/describe_tests.py b/teuthology/describe_tests.py index 76c7297141..c65b55088c 100644 --- a/teuthology/describe_tests.py +++ b/teuthology/describe_tests.py @@ -7,9 +7,13 @@ import os import sys import yaml -from teuthology.exceptions import ParseError -from teuthology.suite.build_matrix import build_matrix, combine_path +import random +from distutils.util import strtobool +from teuthology.exceptions import ParseError +from teuthology.suite.build_matrix import \ + build_matrix, generate_combinations, _get_matrix +from teuthology.suite import util def main(args): try: @@ -20,27 +24,60 @@ def main(args): def describe_tests(args): suite_dir = os.path.abspath(args[""]) - fields = args["--fields"].split(',') - include_facet = args['--show-facet'] == 'yes' output_format = args['--format'] + conf=dict() + rename_args = { + 'filter': 'filter_in', + } + for (key, value) in args.items(): + key = key.lstrip('--').replace('-', '_') + key = rename_args.get(key) or key + if key in ('filter_all', 'filter_in', 'filter_out', 'fields'): + if not value: + value = [] + else: + value = [_ for _ in + (x.strip() for x in value.split(',')) if _] + elif key in ('limit'): + value = int(value) + elif key in ('seed'): + value = int(value) + if value < 0: + value = None + elif key == 'subset' and value is not None: + # take input string '2/3' and turn into (2, 3) + value = tuple(map(int, value.split('/'))) + elif key in ('show_facet'): + value = strtobool(value) + conf[key] = value + if args['--combinations']: - limit = int(args['--limit']) - filter_in = None - if args['--filter']: - filter_in = [f.strip() for f in args['--filter'].split(',')] - filter_out = None - if args['--filter-out']: - filter_out = [f.strip() for f in args['--filter-out'].split(',')] - subset = None - if args['--subset']: - subset = map(int, args['--subset'].split('/')) - headers, rows = get_combinations(suite_dir, fields, subset, - limit, filter_in, - filter_out, include_facet) + headers, rows = get_combinations(suite_dir, + limit=conf['limit'], + seed=conf['seed'], + subset=conf['subset'], + fields=conf['fields'], + filter_in=conf['filter_in'], + filter_out=conf['filter_out'], + filter_all=conf['filter_all'], + filter_fragments=conf['filter_fragments'], + include_facet=conf['show_facet']) hrule = ALL + elif args['--summary']: + output_summary(suite_dir, + limit=conf['limit'], + seed=conf['seed'], + subset=conf['subset'], + show_desc=conf['print_description'], + show_frag=conf['print_fragments'], + filter_in=conf['filter_in'], + filter_out=conf['filter_out'], + filter_all=conf['filter_all'], + filter_fragments=conf['filter_fragments']) + exit(0) else: - headers, rows = describe_suite(suite_dir, fields, include_facet, + headers, rows = describe_suite(suite_dir, conf['fields'], conf['show_facet'], output_format) hrule = FRAME @@ -69,9 +106,60 @@ def output_results(headers, rows, output_format, hrule): print(table) -def get_combinations(suite_dir, fields, subset, - limit, filter_in, filter_out, - include_facet): +def output_summary(path, limit=0, + seed=None, + subset=None, + show_desc=True, + show_frag=False, + show_matrix=False, + filter_in=None, + filter_out=None, + filter_all=None, + filter_fragments=True): + """ + Prints number of all facets for a given suite for inspection, + taking into accout such options like --subset, --filter, + --filter-out and --filter-all. Optionally dumps matrix objects, + yaml files which is used for generating combinations. + """ + + random.seed(seed) + mat, first, matlimit = _get_matrix(path, subset) + configs = generate_combinations(path, mat, first, matlimit) + print("# {} (not filtered) {}".format(len(configs), path)) + count = 0 + suite = os.path.basename(path) + config_list = util.filter_configs(configs, + suite_name=suite, + filter_in=filter_in, + filter_out=filter_out, + filter_all=filter_all, + filter_fragments=filter_fragments) + if show_desc or show_frag: + for c in config_list: + if limit and count >= limit: + break + count += 1 + print(" {}".format(c[0])) + if show_frag: + for path in c[1]: + print(" {}".format(util.strip_fragment_path(path))) + else: + count=sum(1 for _ in config_list) + if show_matrix: + print(mat.tostr(1)) + print(" {} (total filtered)".format(count)) + +def get_combinations(suite_dir, + limit=0, + seed=None, + subset=None, + fields=[], + filter_in=None, + filter_out=None, + filter_all=None, + filter_fragments=False, + include_facet=True): """ Describes the combinations of a suite, optionally limiting or filtering output based on the given parameters. Includes @@ -80,8 +168,8 @@ def get_combinations(suite_dir, fields, subset, Returns a tuple of (headers, rows) where both elements are lists of strings. """ - configs = [(combine_path(suite_dir, item[0]), item[1]) for item in - build_matrix(suite_dir, subset)] + suite = os.path.basename(suite_dir) + configs = build_matrix(suite_dir, subset, seed) num_listed = 0 rows = [] @@ -90,15 +178,15 @@ def get_combinations(suite_dir, fields, subset, dirs = {} max_dir_depth = 0 + configs = util.filter_configs(configs, + suite_name=suite, + filter_in=filter_in, + filter_out=filter_out, + filter_all=filter_all, + filter_fragments=filter_fragments) for _, fragment_paths in configs: if limit > 0 and num_listed >= limit: break - if filter_in and not any([f in path for f in filter_in - for path in fragment_paths]): - continue - if filter_out and any([f in path for f in filter_out - for path in fragment_paths]): - continue fragment_fields = [extract_info(path, fields) for path in fragment_paths] diff --git a/teuthology/test/test_describe_tests.py b/teuthology/test/test_describe_tests.py index d1b4ee3f1e..0110cc7bf1 100644 --- a/teuthology/test/test_describe_tests.py +++ b/teuthology/test/test_describe_tests.py @@ -23,6 +23,21 @@ install: - desc: single node cluster roles: - [osd.0, osd.1, osd.2, mon.a, mon.b, mon.c, client.0] +""", + 'fixed-2.yaml': + """meta: +- desc: couple node cluster +roles: +- [osd.0, osd.1, osd.2, mon.a, mon.b, mon.c] +- [client.0] +""", + 'fixed-3.yaml': + """meta: +- desc: triple node cluster +roles: +- [osd.0, osd.1, osd.2, mon.a, mon.b, mon.c] +- [client.0] +- [client.1] """ }, 'workloads': { @@ -62,7 +77,9 @@ expected_tree = """├── % ├── base │ └── install.yaml ├── clusters -│ └── fixed-1.yaml +│ ├── fixed-1.yaml +│ ├── fixed-2.yaml +│ └── fixed-3.yaml └── workloads ├── rbd_api_tests.yaml └── rbd_api_tests_old_format.yaml""".split('\n') @@ -74,6 +91,8 @@ expected_facets = [ 'base', '', 'clusters', + 'clusters', + 'clusters', '', 'workloads', 'workloads', @@ -86,6 +105,8 @@ expected_desc = [ 'install ceph', '', 'single node cluster', + 'couple node cluster', + 'triple node cluster', '', 'c/c++ librbd api tests with default settings', 'c/c++ librbd api tests with format 1 images', @@ -99,6 +120,8 @@ expected_rbd_features = [ '', '', '', + '', + '', 'default', 'none', ] @@ -190,13 +213,18 @@ class TestDescribeTests(object): expected_desc)] def test_combinations_only_facets(self): - headers, rows = get_combinations('basic', [], None, 1, None, None, True) + headers, rows = get_combinations('basic', + fields=[], subset=None, limit=1, + filter_in=None, filter_out=None, filter_all=None, + include_facet=True) self.assert_expected_combo_headers(headers) assert rows == [['basic', 'install', 'fixed-1', 'rbd_api_tests']] def test_combinations_desc_features(self): - headers, rows = get_combinations('basic', ['desc', 'rbd_features'], - None, 1, None, None, False) + headers, rows = get_combinations('basic', + fields=['desc', 'rbd_features'], subset=None, limit=1, + filter_in=None, filter_out=None, filter_all=None, + include_facet=False) assert headers == ['desc', 'rbd_features'] descriptions = '\n'.join([ 'install ceph', @@ -206,17 +234,39 @@ class TestDescribeTests(object): assert rows == [[descriptions, 'default']] def test_combinations_filter_in(self): - headers, rows = get_combinations('basic', [], None, 0, ['old_format'], - None, True) + headers, rows = get_combinations('basic', + fields=[], subset=None, limit=0, + filter_in=['old_format'], filter_out=None, filter_all=None, + include_facet=True) self.assert_expected_combo_headers(headers) - assert rows == [['basic', 'install', 'fixed-1', - 'rbd_api_tests_old_format']] + assert rows == [ + ['basic', 'install', 'fixed-1', 'rbd_api_tests_old_format'], + ['basic', 'install', 'fixed-2', 'rbd_api_tests_old_format'], + ['basic', 'install', 'fixed-3', 'rbd_api_tests_old_format'], + ] def test_combinations_filter_out(self): - headers, rows = get_combinations('basic', [], None, 0, None, - ['old_format'], True) + headers, rows = get_combinations('basic', + fields=[], subset=None, limit=0, + filter_in=None, filter_out=['old_format'], filter_all=None, + include_facet=True) self.assert_expected_combo_headers(headers) - assert rows == [['basic', 'install', 'fixed-1', 'rbd_api_tests']] + assert rows == [ + ['basic', 'install', 'fixed-1', 'rbd_api_tests'], + ['basic', 'install', 'fixed-2', 'rbd_api_tests'], + ['basic', 'install', 'fixed-3', 'rbd_api_tests'], + ] + + def test_combinations_filter_all(self): + headers, rows = get_combinations('basic', + fields=[], subset=None, limit=0, + filter_in=None, filter_out=None, + filter_all=['fixed-2', 'old_format'], + include_facet=True) + self.assert_expected_combo_headers(headers) + assert rows == [ + ['basic', 'install', 'fixed-2', 'rbd_api_tests_old_format'] + ] @patch('teuthology.describe_tests.open') -- 2.39.5