From 3c00922430b1ebdbb42d85c81e9e42e35efce1e0 Mon Sep 17 00:00:00 2001 From: rakeshgm Date: Thu, 23 Apr 2020 19:10:26 +0530 Subject: [PATCH] generate polarion frag-ids Signed-off-by: rakeshgm --- scripts/polarion.py | 26 ++++++++++++++ setup.py | 3 ++ teuthology/polarion/__init__.py | 34 ++++++++++++++++++ teuthology/polarion/frag_id.py | 61 +++++++++++++++++++++++++++++++++ teuthology/polarion/util.py | 22 ++++++++++++ teuthology/polarion/write.py | 47 +++++++++++++++++++++++++ 6 files changed, 193 insertions(+) create mode 100644 scripts/polarion.py create mode 100644 teuthology/polarion/__init__.py create mode 100644 teuthology/polarion/frag_id.py create mode 100644 teuthology/polarion/util.py create mode 100644 teuthology/polarion/write.py diff --git a/scripts/polarion.py b/scripts/polarion.py new file mode 100644 index 0000000000..50e05761a6 --- /dev/null +++ b/scripts/polarion.py @@ -0,0 +1,26 @@ +import docopt +import sys + +import teuthology.polarion + +doc = """ +usage: teuthology-polarion --help + teuthology-polarion --suite --suite-dir [--output ] + teuthology-polarion --suite --suite-repo --suite-branch [--output ] + +generate unique polarion_id/desc based on frag_id in suite yaml + +Standard arguments: + -s , --suite The suite to generate polarion_ids + --suite-repo Use tasks and suite definition in this repository + + --suite-branch Use this suite branch instead of the ceph branch + --suite-dir Use this alternative directory if you have suite + directory present in local disk + --output write the frag_ids to a file, supported types are .csv +""" + + +def main(argv=sys.argv[1:]): + args = docopt.docopt(doc, argv=argv) + teuthology.polarion.main(args) diff --git a/setup.py b/setup.py index 7b08ee8313..1c75d8cfc6 100644 --- a/setup.py +++ b/setup.py @@ -129,6 +129,9 @@ setup( 'teuthology-prune-logs = scripts.prune_logs:main', 'teuthology-describe = scripts.describe:main', 'teuthology-reimage = scripts.reimage:main' + 'teuthology-describe-tests = scripts.describe_tests:main', + 'teuthology-gencov = scripts.gencov:main', + 'teuthology-polarion = scripts.polarion:main', ], }, diff --git a/teuthology/polarion/__init__.py b/teuthology/polarion/__init__.py new file mode 100644 index 0000000000..d934dc7084 --- /dev/null +++ b/teuthology/polarion/__init__.py @@ -0,0 +1,34 @@ +import logging + +import teuthology +from teuthology.config import YamlConfig +from teuthology.polarion.frag_id import BuildPolarionFragmentIds +from teuthology.polarion.write import Writer + +log = logging.getLogger(__name__) + + +def process_args(args): + conf = YamlConfig() + for (key, value) in args.items(): + key = key.lstrip('--').replace('-', '_') + conf[key] = value or '' + return conf + + +def main(args): + conf = process_args(args) + frag_ids = BuildPolarionFragmentIds(conf) + frag_ids.build() + + log.info('polarion_frag_ids for {}'.format(conf.suite.replace('/', ':'))) + for id, frags in frag_ids.ids: + log.info('polarion_frag_id: {}'.format(id)) + log.info('polaion_frag_ids count is: {}'.format(len(frag_ids.ids))) + + if conf.output: + frag_ids.write_to_file() + + + + diff --git a/teuthology/polarion/frag_id.py b/teuthology/polarion/frag_id.py new file mode 100644 index 0000000000..07e23d9ad4 --- /dev/null +++ b/teuthology/polarion/frag_id.py @@ -0,0 +1,61 @@ +import os +import logging +import yaml + +from teuthology.config import config +from teuthology.polarion.write import Writer +from teuthology.suite.build_matrix import build_matrix +from teuthology.polarion.util import clone_qa_suite + +log = logging.getLogger(__name__) + + +class BuildPolarionFragmentIds: + """ + Generate full polarion_frag_id from fragments. + if 'frag_id' is present in fragments, then make id with it. + """ + + def __init__(self, conf): + self._args = conf + self.ids = set() + + @property + def suite_path(self): + if self._args.suite_dir: + suite_repo_path = self._args.suite_dir + else: + suite_repo_path = clone_qa_suite(self._args) + suite_rel_path = 'qa' + + return os.path.normpath(os.path.join( + suite_repo_path, + suite_rel_path, + 'suites', + self._args.suite, + )) + + @property + def fragments(self): + return build_matrix(self.suite_path) + + def build(self): + for _, fragment_paths in self.fragments: + parts = [] + frags = [] # storing frag to add as automated_script in polarion + for fragment in fragment_paths: + with open(fragment, 'r') as fp: + yml = yaml.safe_load(fp) or {} + if 'frag_id' in yml: + parts.append(yml.get('frag_id')) + frags.append(fp.name) + parts = sorted([i for i in parts if i]) + id = '-'.join(parts) + frags = ','.join(frags) + self.ids.add((id, frags)) + + def write_to_file(self): + config.frag_ids_path = self._args.output + config.frag_ids = self.ids + writer = Writer() + writer.write() diff --git a/teuthology/polarion/util.py b/teuthology/polarion/util.py new file mode 100644 index 0000000000..81537362c6 --- /dev/null +++ b/teuthology/polarion/util.py @@ -0,0 +1,22 @@ +import logging + +from teuthology.repo_utils import fetch_qa_suite +from teuthology.config import config +from teuthology.exceptions import BranchNotFoundError, GitError + +log = logging.getLogger(__name__) + + +def clone_qa_suite(conf): + + """ + clone qa suit to the local disk to ~/src dir + :param conf: + :return: suit_repo_path after cloning for valid suite_repo and suite_branch + """ + try: + config.ceph_qa_suite_git_url = conf.suite_repo + suite_repo_path = fetch_qa_suite(conf.suite_branch) + return suite_repo_path + except BranchNotFoundError as exc: + raise GitError('{}'.format(str(exc))) diff --git a/teuthology/polarion/write.py b/teuthology/polarion/write.py new file mode 100644 index 0000000000..0cc78e2c21 --- /dev/null +++ b/teuthology/polarion/write.py @@ -0,0 +1,47 @@ +import logging +import csv +import os + +from teuthology.config import config +from teuthology.exceptions import ConfigError + +log = logging.getLogger(__name__) + + +class CSVWriter: + + def __init__(self): + self.field_names = ['Description', 'Scripts'] + self._contents = config.frag_ids + self._path = os.path.normpath(config.frag_ids_path) + + def __rows(self): + rows = [] + if self._contents: + for each in self._contents: + rows.append(dict(zip(self.field_names, each))) + return rows + + def write(self): + with open(self._path, 'w') as fp: + writer = csv.DictWriter(fp, fieldnames=self.field_names) + writer.writeheader() + writer.writerows(self.__rows()) + + log.info('csv file will be at {}'.format(self._path)) + + +class Writer: + def __init__(self,): + supported = ['.csv'] + self._path = config.frag_ids_path + self._filename, self._extension = os.path.splitext(self._path) + if self._extension not in supported: + raise ConfigError('unsupported file type, supported options are: {}'.format(','.join(supported))) + + def write(self): + if self._extension == '.csv': + csv_writer = CSVWriter() + csv_writer.write() + + -- 2.39.5