From: Josh Durgin Date: Mon, 30 Nov 2015 23:03:06 +0000 (-0800) Subject: teuthology-describe-tests: rename from teuthology-tree X-Git-Tag: 1.1.0~727^2~12 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=2a049da8a65324853a6aea92d005b32407c6e18d;p=teuthology.git teuthology-describe-tests: rename from teuthology-tree Signed-off-by: Josh Durgin --- diff --git a/scripts/describe_tests.py b/scripts/describe_tests.py new file mode 100644 index 000000000..c6648abcc --- /dev/null +++ b/scripts/describe_tests.py @@ -0,0 +1,31 @@ +import docopt + +import teuthology.config +import teuthology.describe_tests + +doc = """ +usage: + teuthology-describe-tests -h + teuthology-describe-tests [options] [--] + +Describe the contents of a qa suite by extracting comments +starting with particular prefixes from files in the suite. + +By default, the remainder of a line starting with '# desc:' will +be included from each file in the specified suite directory. + +positional arguments: + qa suite path to traverse and describe + +optional arguments: + -h, --help Show this help message and exit + -p , --prefix Comma-separated list of prefixes + [default: desc] + --show-facet [yes|no] List the facet of each file + [default: yes] +""" + + +def main(): + args = docopt.docopt(doc) + teuthology.describe_tests.main(args) diff --git a/scripts/tree.py b/scripts/tree.py deleted file mode 100644 index 5a1de47f3..000000000 --- a/scripts/tree.py +++ /dev/null @@ -1,31 +0,0 @@ -import docopt - -import teuthology.config -import teuthology.tree - -doc = """ -usage: - teuthology-tree -h - teuthology-tree [options] [--] - -Describe the contents of a qa suite by extracting comments -starting with particular prefixes from files in the suite. - -By default, the remainder of a line starting with '# desc:' will -be included from each file in the specified suite directory. - -positional arguments: - qa suite path to traverse and describe - -optional arguments: - -h, --help Show this help message and exit - -p , --prefix Comma-separated list of prefixes - [default: desc] - --show-facet [yes|no] List the facet of each file - [default: yes] -""" - - -def main(): - args = docopt.docopt(doc) - teuthology.tree.main(args) diff --git a/setup.py b/setup.py index b0cd0d928..22b8a7545 100644 --- a/setup.py +++ b/setup.py @@ -85,7 +85,7 @@ setup( 'teuthology-kill = scripts.kill:main', 'teuthology-queue = scripts.queue:main', 'teuthology-prune-logs = scripts.prune_logs:main', - 'teuthology-tree = scripts.tree:main', + 'teuthology-describe-tests = scripts.describe_tests:main', ], }, diff --git a/teuthology/describe_tests.py b/teuthology/describe_tests.py new file mode 100644 index 000000000..13f192eca --- /dev/null +++ b/teuthology/describe_tests.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- +from prettytable import PrettyTable, FRAME, ALL +import os + +def main(args): + suite_dir = os.path.abspath(args[""]) + filters = args["--prefix"].split(',') + include_facet = args['--show-facet'] == 'yes' + + print(suite_dir) + rows = tree_with_info(suite_dir, filters, include_facet, '', []) + + headers = ['path'] + if include_facet: + headers.append('facet') + + table = PrettyTable(headers + filters) + table.align = 'l' + table.vrules = ALL + table.hrules = FRAME + + for row in rows: + table.add_row(row) + print(table) + +def extract_info(file_name, filters, _isdir=os.path.isdir, _open=open): + result = {f: '' for f in filters} + if _isdir(file_name): + return result + with _open(file_name, 'r') as f: + for line in f: + for filt in filters: + prefix = '# ' + filt + ':' + if line.startswith(prefix): + if result[filt]: + result[filt] += '\n' + result[filt] += line[len(prefix):].rstrip('\n').strip() + return result + +def tree_with_info(cur_dir, filters, include_facet, prefix, rows, + _listdir=os.listdir, _isdir=os.path.isdir, + _open=open): + files = sorted(_listdir(cur_dir)) + has_yamls = any([x.endswith('.yaml') for x in files]) + facet = os.path.basename(cur_dir) if has_yamls else '' + for i, f in enumerate(files): + path = os.path.join(cur_dir, f) + if i == len(files) - 1: + file_pad = '└── ' + dir_pad = ' ' + else: + file_pad = '├── ' + dir_pad = '│ ' + info = extract_info(path, filters, _isdir, _open) + tree_node = prefix + file_pad + f + meta = [info[f] for f in filters] + row = [tree_node] + if include_facet: + row.append(facet) + rows.append(row + meta) + if _isdir(path): + tree_with_info(path, filters, include_facet, + prefix + dir_pad, rows, + _listdir, _isdir, _open) + return rows diff --git a/teuthology/test/test_describe_tests.py b/teuthology/test/test_describe_tests.py new file mode 100644 index 000000000..5a8ce32db --- /dev/null +++ b/teuthology/test/test_describe_tests.py @@ -0,0 +1,191 @@ +# -*- coding: utf-8 -*- +from fake_fs import make_fake_fstools +from teuthology.describe_tests import tree_with_info, extract_info + +realistic_fs = { + 'basic': { + '%': None, + 'base': { + 'install.yaml': + """# desc: install ceph +install: +""" + }, + 'clusters': { + 'fixed-1.yaml': + """# desc: single node cluster +roles: +- [osd.0, osd.1, osd.2, mon.a, mon.b, mon.c, client.0] +""" + }, + 'workloads': { + 'rbd_api_tests_old_format.yaml': + """# desc: c/c++ librbd api tests with format 1 images +# rbd_features: none +overrides: + ceph: + conf: + client: + rbd default format: 1 +tasks: +- workunit: + env: + RBD_FEATURES: 0 + clients: + client.0: + - rbd/test_librbd.sh +""", + 'rbd_api_tests.yaml': + """# desc: c/c++ librbd api tests with default settings +# rbd_features: default +tasks: +- workunit: + clients: + client.0: + - rbd/test_librbd.sh +""", + }, + }, +} + + +expected_tree = """├── % +├── base +│ └── install.yaml +├── clusters +│ └── fixed-1.yaml +└── workloads + ├── rbd_api_tests.yaml + └── rbd_api_tests_old_format.yaml""".split('\n') + + +expected_facets = [ + '', + '', + 'base', + '', + 'clusters', + '', + 'workloads', + 'workloads', +] + + +expected_desc = [ + '', + '', + 'install ceph', + '', + 'single node cluster', + '', + 'c/c++ librbd api tests with default settings', + 'c/c++ librbd api tests with format 1 images', +] + + +expected_rbd_features = [ + '', + '', + '', + '', + '', + '', + 'default', + 'none', +] + + +class TestDescribeTests(object): + + def setup(self): + self.fake_listdir, _, self.fake_isdir, self.fake_open = \ + make_fake_fstools(realistic_fs) + + def test_no_filters(self): + rows = tree_with_info('basic', [], False, '', [], + self.fake_listdir, self.fake_isdir, + self.fake_open) + assert rows == [ [x] for x in expected_tree] + + def test_single_filter(self): + rows = tree_with_info('basic', ['desc'], False, '', [], + self.fake_listdir, self.fake_isdir, + self.fake_open) + assert rows == map(list, zip(expected_tree, expected_desc)) + + rows = tree_with_info('basic', ['rbd_features'], False, '', [], + self.fake_listdir, self.fake_isdir, + self.fake_open) + assert rows == map(list, zip(expected_tree, expected_rbd_features)) + + def test_single_filter_with_facets(self): + rows = tree_with_info('basic', ['desc'], True, '', [], + self.fake_listdir, self.fake_isdir, + self.fake_open) + assert rows == map(list, zip(expected_tree, expected_facets, + expected_desc)) + + rows = tree_with_info('basic', ['rbd_features'], True, '', [], + self.fake_listdir, self.fake_isdir, + self.fake_open) + assert rows == map(list, zip(expected_tree, expected_facets, + expected_rbd_features)) + + def test_no_matching(self): + fake_listdir, _, fake_isdir, fake_open = \ + make_fake_fstools(realistic_fs) + rows = tree_with_info('basic', ['extra'], False, '', [], + self.fake_listdir, self.fake_isdir, + self.fake_open) + assert rows == map(list, zip(expected_tree, [''] * len(expected_tree))) + + rows = tree_with_info('basic', ['extra'], True, '', [], + self.fake_listdir, self.fake_isdir, + self.fake_open) + assert rows == map(list, zip(expected_tree, expected_facets, + [''] * len(expected_tree))) + + def test_multiple_filters(self): + fake_listdir, _, fake_isdir, fake_open = \ + make_fake_fstools(realistic_fs) + rows = tree_with_info('basic', ['desc', 'rbd_features'], False, + '', [], self.fake_listdir, + self.fake_isdir, self.fake_open) + assert rows == map(list, zip(expected_tree, + expected_desc, + expected_rbd_features)) + + rows = tree_with_info('basic', ['rbd_features', 'desc'], False, + '', [], self.fake_listdir, + self.fake_isdir, self.fake_open) + assert rows == map(list, zip(expected_tree, + expected_rbd_features, + expected_desc)) + + + def test_multiple_filters_with_facets(self): + rows = tree_with_info('basic', ['desc', 'rbd_features'], True, + '', [], self.fake_listdir, + self.fake_isdir, self.fake_open) + assert rows == map(list, zip(expected_tree, + expected_facets, + expected_desc, + expected_rbd_features)) + + rows = tree_with_info('basic', ['rbd_features', 'desc'], True, + '', [], self.fake_listdir, + self.fake_isdir, self.fake_open) + assert rows == map(list, zip(expected_tree, + expected_facets, + expected_rbd_features, + expected_desc)) + + +def test_extract_info_dir(): + simple_fs = {'a': {'b': '# foo:'}} + _, _, fake_isdir, fake_open = make_fake_fstools(simple_fs) + info = extract_info('a', [], fake_isdir, fake_open) + assert info == {} + + info = extract_info('a', ['foo', 'bar'], fake_isdir, fake_open) + assert info == {'foo': '', 'bar': ''} diff --git a/teuthology/test/test_tree.py b/teuthology/test/test_tree.py deleted file mode 100644 index bc4815679..000000000 --- a/teuthology/test/test_tree.py +++ /dev/null @@ -1,191 +0,0 @@ -# -*- coding: utf-8 -*- -from fake_fs import make_fake_fstools -from teuthology.tree import tree_with_info, extract_info - -realistic_fs = { - 'basic': { - '%': None, - 'base': { - 'install.yaml': - """# desc: install ceph -install: -""" - }, - 'clusters': { - 'fixed-1.yaml': - """# desc: single node cluster -roles: -- [osd.0, osd.1, osd.2, mon.a, mon.b, mon.c, client.0] -""" - }, - 'workloads': { - 'rbd_api_tests_old_format.yaml': - """# desc: c/c++ librbd api tests with format 1 images -# rbd_features: none -overrides: - ceph: - conf: - client: - rbd default format: 1 -tasks: -- workunit: - env: - RBD_FEATURES: 0 - clients: - client.0: - - rbd/test_librbd.sh -""", - 'rbd_api_tests.yaml': - """# desc: c/c++ librbd api tests with default settings -# rbd_features: default -tasks: -- workunit: - clients: - client.0: - - rbd/test_librbd.sh -""", - }, - }, -} - - -expected_tree = """├── % -├── base -│ └── install.yaml -├── clusters -│ └── fixed-1.yaml -└── workloads - ├── rbd_api_tests.yaml - └── rbd_api_tests_old_format.yaml""".split('\n') - - -expected_facets = [ - '', - '', - 'base', - '', - 'clusters', - '', - 'workloads', - 'workloads', -] - - -expected_desc = [ - '', - '', - 'install ceph', - '', - 'single node cluster', - '', - 'c/c++ librbd api tests with default settings', - 'c/c++ librbd api tests with format 1 images', -] - - -expected_rbd_features = [ - '', - '', - '', - '', - '', - '', - 'default', - 'none', -] - - -class TestTree(object): - - def setup(self): - self.fake_listdir, _, self.fake_isdir, self.fake_open = \ - make_fake_fstools(realistic_fs) - - def test_no_filters(self): - rows = tree_with_info('basic', [], False, '', [], - self.fake_listdir, self.fake_isdir, - self.fake_open) - assert rows == [ [x] for x in expected_tree] - - def test_single_filter(self): - rows = tree_with_info('basic', ['desc'], False, '', [], - self.fake_listdir, self.fake_isdir, - self.fake_open) - assert rows == map(list, zip(expected_tree, expected_desc)) - - rows = tree_with_info('basic', ['rbd_features'], False, '', [], - self.fake_listdir, self.fake_isdir, - self.fake_open) - assert rows == map(list, zip(expected_tree, expected_rbd_features)) - - def test_single_filter_with_facets(self): - rows = tree_with_info('basic', ['desc'], True, '', [], - self.fake_listdir, self.fake_isdir, - self.fake_open) - assert rows == map(list, zip(expected_tree, expected_facets, - expected_desc)) - - rows = tree_with_info('basic', ['rbd_features'], True, '', [], - self.fake_listdir, self.fake_isdir, - self.fake_open) - assert rows == map(list, zip(expected_tree, expected_facets, - expected_rbd_features)) - - def test_no_matching(self): - fake_listdir, _, fake_isdir, fake_open = \ - make_fake_fstools(realistic_fs) - rows = tree_with_info('basic', ['extra'], False, '', [], - self.fake_listdir, self.fake_isdir, - self.fake_open) - assert rows == map(list, zip(expected_tree, [''] * len(expected_tree))) - - rows = tree_with_info('basic', ['extra'], True, '', [], - self.fake_listdir, self.fake_isdir, - self.fake_open) - assert rows == map(list, zip(expected_tree, expected_facets, - [''] * len(expected_tree))) - - def test_multiple_filters(self): - fake_listdir, _, fake_isdir, fake_open = \ - make_fake_fstools(realistic_fs) - rows = tree_with_info('basic', ['desc', 'rbd_features'], False, - '', [], self.fake_listdir, - self.fake_isdir, self.fake_open) - assert rows == map(list, zip(expected_tree, - expected_desc, - expected_rbd_features)) - - rows = tree_with_info('basic', ['rbd_features', 'desc'], False, - '', [], self.fake_listdir, - self.fake_isdir, self.fake_open) - assert rows == map(list, zip(expected_tree, - expected_rbd_features, - expected_desc)) - - - def test_multiple_filters_with_facets(self): - rows = tree_with_info('basic', ['desc', 'rbd_features'], True, - '', [], self.fake_listdir, - self.fake_isdir, self.fake_open) - assert rows == map(list, zip(expected_tree, - expected_facets, - expected_desc, - expected_rbd_features)) - - rows = tree_with_info('basic', ['rbd_features', 'desc'], True, - '', [], self.fake_listdir, - self.fake_isdir, self.fake_open) - assert rows == map(list, zip(expected_tree, - expected_facets, - expected_rbd_features, - expected_desc)) - - -def test_extract_info_dir(): - simple_fs = {'a': {'b': '# foo:'}} - _, _, fake_isdir, fake_open = make_fake_fstools(simple_fs) - info = extract_info('a', [], fake_isdir, fake_open) - assert info == {} - - info = extract_info('a', ['foo', 'bar'], fake_isdir, fake_open) - assert info == {'foo': '', 'bar': ''} diff --git a/teuthology/tree.py b/teuthology/tree.py deleted file mode 100644 index 13f192eca..000000000 --- a/teuthology/tree.py +++ /dev/null @@ -1,65 +0,0 @@ -# -*- coding: utf-8 -*- -from prettytable import PrettyTable, FRAME, ALL -import os - -def main(args): - suite_dir = os.path.abspath(args[""]) - filters = args["--prefix"].split(',') - include_facet = args['--show-facet'] == 'yes' - - print(suite_dir) - rows = tree_with_info(suite_dir, filters, include_facet, '', []) - - headers = ['path'] - if include_facet: - headers.append('facet') - - table = PrettyTable(headers + filters) - table.align = 'l' - table.vrules = ALL - table.hrules = FRAME - - for row in rows: - table.add_row(row) - print(table) - -def extract_info(file_name, filters, _isdir=os.path.isdir, _open=open): - result = {f: '' for f in filters} - if _isdir(file_name): - return result - with _open(file_name, 'r') as f: - for line in f: - for filt in filters: - prefix = '# ' + filt + ':' - if line.startswith(prefix): - if result[filt]: - result[filt] += '\n' - result[filt] += line[len(prefix):].rstrip('\n').strip() - return result - -def tree_with_info(cur_dir, filters, include_facet, prefix, rows, - _listdir=os.listdir, _isdir=os.path.isdir, - _open=open): - files = sorted(_listdir(cur_dir)) - has_yamls = any([x.endswith('.yaml') for x in files]) - facet = os.path.basename(cur_dir) if has_yamls else '' - for i, f in enumerate(files): - path = os.path.join(cur_dir, f) - if i == len(files) - 1: - file_pad = '└── ' - dir_pad = ' ' - else: - file_pad = '├── ' - dir_pad = '│ ' - info = extract_info(path, filters, _isdir, _open) - tree_node = prefix + file_pad + f - meta = [info[f] for f in filters] - row = [tree_node] - if include_facet: - row.append(facet) - rows.append(row + meta) - if _isdir(path): - tree_with_info(path, filters, include_facet, - prefix + dir_pad, rows, - _listdir, _isdir, _open) - return rows