From d9680c187beccda452f5e5a5853502b3c6e93b63 Mon Sep 17 00:00:00 2001 From: Kyr Shatskyy Date: Mon, 21 Jan 2019 16:35:06 +0100 Subject: [PATCH] suite/run: isolate description filter code Refactor collect_jobs in order to isolate descriptions filter code in order to make it possible to reuse it and test better. Introduce --filter-fragment option defaults to False, to align code logic with --filter help and documentation. Signed-off-by: Kyr Shatskyy --- scripts/suite.py | 5 +++++ teuthology/suite/__init__.py | 2 +- teuthology/suite/run.py | 42 +++++++++++------------------------- teuthology/suite/util.py | 40 ++++++++++++++++++++++++++++++++++ 4 files changed, 59 insertions(+), 30 deletions(-) diff --git a/scripts/suite.py b/scripts/suite.py index 5d62859c57..da01427386 100644 --- a/scripts/suite.py +++ b/scripts/suite.py @@ -122,6 +122,11 @@ Scheduler arguments: --filter-all KEYWORDS Only run jobs whose description contains each one of the keywords in the comma separated keyword string specified. + -F, --filter-fragments + Check yaml fragments too if job description + does not match the filters provided with + options --filter, --filter-out, and --filter-all. + [default: false] --archive-upload RSYNC_DEST Rsync destination to upload archives. --archive-upload-url URL Public facing URL where archives are uploaded. --throttle SLEEP When scheduling, wait SLEEP seconds between jobs. diff --git a/teuthology/suite/__init__.py b/teuthology/suite/__init__.py index c35456d545..46578fc9ce 100644 --- a/teuthology/suite/__init__.py +++ b/teuthology/suite/__init__.py @@ -78,7 +78,7 @@ def process_args(args): value = expand_short_repo_name( value, config.get_ceph_qa_suite_git_url()) - elif key in ('validate_sha1'): + elif key in ('validate_sha1', 'filter_fragments'): value = strtobool(value) conf[key] = value return conf diff --git a/teuthology/suite/run.py b/teuthology/suite/run.py index 249d289b21..e3c38aa7eb 100644 --- a/teuthology/suite/run.py +++ b/teuthology/suite/run.py @@ -21,7 +21,7 @@ from teuthology.orchestra.opsys import OS from teuthology.repo_utils import build_git_url from teuthology.suite import util -from teuthology.suite.build_matrix import combine_path, build_matrix +from teuthology.suite.build_matrix import build_matrix from teuthology.suite.placeholder import substitute_placeholders, dict_templ log = logging.getLogger(__name__) @@ -363,34 +363,11 @@ class Run(object): jobs_to_schedule = [] jobs_missing_packages = [] for description, fragment_paths in configs: - base_frag_paths = [ - util.strip_fragment_path(x) for x in fragment_paths - ] if limit > 0 and len(jobs_to_schedule) >= limit: log.info( 'Stopped after {limit} jobs due to --limit={limit}'.format( limit=limit)) break - # Break apart the filter parameter (one string) into comma - # separated components to be used in searches. - def matches(f): - if f in description: - return True - if any(f in path for path in base_frag_paths): - return True - return False - filter_all = self.args.filter_all - if filter_all: - if not all(matches(f) for f in filter_all): - continue - filter_in = self.args.filter_in - if filter_in: - if not any(matches(f) for f in filter_in): - continue - filter_out = self.args.filter_out - if filter_out: - if any(matches(f) for f in filter_out): - continue raw_yaml = '\n'.join([open(a, 'r').read() for a in fragment_paths]) @@ -487,6 +464,7 @@ class Run(object): log.info("pause between jobs : --throttle " + str(throttle)) time.sleep(int(throttle)) + def schedule_suite(self): """ Schedule the suite-run. Returns the number of jobs scheduled. @@ -501,10 +479,9 @@ class Run(object): self.base_config.suite.replace(':', '/'), )) log.debug('Suite %s in %s' % (suite_name, suite_path)) - configs = [ - (combine_path(suite_name, item[0]), item[1]) for item in - build_matrix(suite_path, subset=self.args.subset, seed=self.args.seed) - ] + configs = build_matrix(suite_path, + subset=self.args.subset, + seed=self.args.seed) log.info('Suite %s in %s generated %d jobs (not yet filtered)' % ( suite_name, suite_path, len(configs))) @@ -558,7 +535,14 @@ class Run(object): limit = self.args.newest while backtrack <= limit: jobs_missing_packages, jobs_to_schedule = \ - self.collect_jobs(arch, configs, self.args.newest, job_limit) + self.collect_jobs(arch, + util.filter_configs(configs, + filter_in=self.args.filter_in, + filter_out=self.args.filter_out, + filter_all=self.args.filter_all, + filter_fragments=self.args.filter_fragments, + suite_name=suite_name), + self.args.newest, job_limit) if jobs_missing_packages and self.args.newest: new_sha1 = \ util.find_git_parent('ceph', self.base_config.sha1) diff --git a/teuthology/suite/util.py b/teuthology/suite/util.py index b82c4f79a4..9083138f83 100644 --- a/teuthology/suite/util.py +++ b/teuthology/suite/util.py @@ -21,6 +21,7 @@ from teuthology.repo_utils import fetch_qa_suite, fetch_teuthology from teuthology.orchestra.opsys import OS from teuthology.packaging import get_builder_project from teuthology.repo_utils import build_git_url +from teuthology.suite.build_matrix import combine_path from teuthology.task.install import get_flavor log = logging.getLogger(__name__) @@ -483,3 +484,42 @@ def find_git_parent(project, sha1): return sha1s[1] else: return None + + +def filter_configs(configs, suite_name=None, + filter_in=None, + filter_out=None, + filter_all=None, + filter_fragments=True): + """ + Returns a generator for pairs of description and fragment paths. + + Usage: + + configs = build_matrix(path, subset, seed) + for description, fragments in filter_configs(configs): + pass + """ + for item in configs: + fragment_paths = item[1] + description = combine_path(suite_name, item[0]) \ + if suite_name else item[0] + base_frag_paths = [strip_fragment_path(x) + for x in fragment_paths] + def matches(f): + if f in description: + return True + if filter_fragments and \ + any(f in path for path in base_frag_paths): + return True + return False + if filter_all: + if not all(matches(f) for f in filter_all): + continue + if filter_in: + if not any(matches(f) for f in filter_in): + continue + if filter_out: + if any(matches(f) for f in filter_out): + continue + yield([description, fragment_paths]) -- 2.39.5