suite/run: isolate description filter code

Refactor collect_jobs in order to isolate descriptions filter code
in order to make it possible to reuse it and test better.

Introduce --filter-fragment option defaults to False, to align code
logic with --filter help and documentation.

Signed-off-by: Kyr Shatskyy <kyrylo.shatskyy@suse.de>

diff --git a/scripts/suite.py b/scripts/suite.py
index 5d62859c5727cdd0d44e37ac1cfd9df5b4f774a1..da01427386cb00792cd6ff62916cdb2bceb66095 100644 (file)
--- a/scripts/suite.py
+++ b/scripts/suite.py
@@ -122,6 +122,11 @@ Scheduler arguments:
   --filter-all KEYWORDS       Only run jobs whose description contains each one
                               of the keywords in the comma separated keyword
                               string specified.
+  -F, --filter-fragments <bool>
+                              Check yaml fragments too if job description
+                              does not match the filters provided with
+                              options --filter, --filter-out, and --filter-all.
+                              [default: false]
   --archive-upload RSYNC_DEST Rsync destination to upload archives.
   --archive-upload-url URL    Public facing URL where archives are uploaded.
   --throttle SLEEP            When scheduling, wait SLEEP seconds between jobs.

diff --git a/teuthology/suite/__init__.py b/teuthology/suite/__init__.py
index c35456d5455a868773565b1c114ee720c48197af..46578fc9ce1182095995932c9e1a8db36b80aa0f 100644 (file)
--- a/teuthology/suite/__init__.py
+++ b/teuthology/suite/__init__.py
@@ -78,7 +78,7 @@ def process_args(args):
             value = expand_short_repo_name(
                 value,
                 config.get_ceph_qa_suite_git_url())
-        elif key in ('validate_sha1'):
+        elif key in ('validate_sha1', 'filter_fragments'):
             value = strtobool(value)
         conf[key] = value
     return conf

diff --git a/teuthology/suite/run.py b/teuthology/suite/run.py
index 249d289b21a77d1c9c06680d8a9b3cf45702b48b..e3c38aa7eb6875ac7d04bc97d056cf4ad3018096 100644 (file)
--- a/teuthology/suite/run.py
+++ b/teuthology/suite/run.py
@@ -21,7 +21,7 @@ from teuthology.orchestra.opsys import OS
 from teuthology.repo_utils import build_git_url
 
 from teuthology.suite import util
-from teuthology.suite.build_matrix import combine_path, build_matrix
+from teuthology.suite.build_matrix import build_matrix
 from teuthology.suite.placeholder import substitute_placeholders, dict_templ
 
 log = logging.getLogger(__name__)
@@ -363,34 +363,11 @@ class Run(object):
         jobs_to_schedule = []
         jobs_missing_packages = []
         for description, fragment_paths in configs:
-            base_frag_paths = [
-                util.strip_fragment_path(x) for x in fragment_paths
-            ]
             if limit > 0 and len(jobs_to_schedule) >= limit:
                 log.info(
                     'Stopped after {limit} jobs due to --limit={limit}'.format(
                         limit=limit))
                 break
-            # Break apart the filter parameter (one string) into comma
-            # separated components to be used in searches.
-            def matches(f):
-                if f in description:
-                    return True
-                if any(f in path for path in base_frag_paths):
-                    return True
-                return False
-            filter_all = self.args.filter_all
-            if filter_all:
-                if not all(matches(f) for f in filter_all):
-                    continue
-            filter_in = self.args.filter_in
-            if filter_in:
-                if not any(matches(f) for f in filter_in):
-                    continue
-            filter_out = self.args.filter_out
-            if filter_out:
-                if any(matches(f) for f in filter_out):
-                    continue
 
             raw_yaml = '\n'.join([open(a, 'r').read() for a in fragment_paths])
 
@@ -487,6 +464,7 @@ class Run(object):
                 log.info("pause between jobs : --throttle " + str(throttle))
                 time.sleep(int(throttle))
 
+
     def schedule_suite(self):
         """
         Schedule the suite-run. Returns the number of jobs scheduled.
@@ -501,10 +479,9 @@ class Run(object):
             self.base_config.suite.replace(':', '/'),
         ))
         log.debug('Suite %s in %s' % (suite_name, suite_path))
-        configs = [
-            (combine_path(suite_name, item[0]), item[1]) for item in
-            build_matrix(suite_path, subset=self.args.subset, seed=self.args.seed)
-        ]
+        configs = build_matrix(suite_path,
+                               subset=self.args.subset,
+                               seed=self.args.seed)
         log.info('Suite %s in %s generated %d jobs (not yet filtered)' % (
             suite_name, suite_path, len(configs)))
 
@@ -558,7 +535,14 @@ class Run(object):
         limit = self.args.newest
         while backtrack <= limit:
             jobs_missing_packages, jobs_to_schedule = \
-                self.collect_jobs(arch, configs, self.args.newest, job_limit)
+                self.collect_jobs(arch,
+                    util.filter_configs(configs,
+                        filter_in=self.args.filter_in,
+                        filter_out=self.args.filter_out,
+                        filter_all=self.args.filter_all,
+                        filter_fragments=self.args.filter_fragments,
+                        suite_name=suite_name),
+                                  self.args.newest, job_limit)
             if jobs_missing_packages and self.args.newest:
                 new_sha1 = \
                     util.find_git_parent('ceph', self.base_config.sha1)

diff --git a/teuthology/suite/util.py b/teuthology/suite/util.py
index b82c4f79a4394932601df092ce76ceb6454e0733..9083138f83ba18087230922f193c55aca86cadbb 100644 (file)
--- a/teuthology/suite/util.py
+++ b/teuthology/suite/util.py
@@ -21,6 +21,7 @@ from teuthology.repo_utils import fetch_qa_suite, fetch_teuthology
 from teuthology.orchestra.opsys import OS
 from teuthology.packaging import get_builder_project
 from teuthology.repo_utils import build_git_url
+from teuthology.suite.build_matrix import combine_path
 from teuthology.task.install import get_flavor
 
 log = logging.getLogger(__name__)
@@ -483,3 +484,42 @@ def find_git_parent(project, sha1):
         return sha1s[1]
     else:
         return None
+
+
+def filter_configs(configs, suite_name=None,
+                            filter_in=None,
+                            filter_out=None,
+                            filter_all=None,
+                            filter_fragments=True):
+    """
+    Returns a generator for pairs of description and fragment paths.
+
+    Usage:
+
+        configs = build_matrix(path, subset, seed)
+        for description, fragments in filter_configs(configs):
+            pass
+    """
+    for item in configs:
+        fragment_paths = item[1]
+        description = combine_path(suite_name, item[0]) \
+                                        if suite_name else item[0]
+        base_frag_paths = [strip_fragment_path(x)
+                                        for x in fragment_paths]
+        def matches(f):
+            if f in description:
+                return True
+            if filter_fragments and \
+                    any(f in path for path in base_frag_paths):
+                return True
+            return False
+        if filter_all:
+            if not all(matches(f) for f in filter_all):
+                continue
+        if filter_in:
+            if not any(matches(f) for f in filter_in):
+                continue
+        if filter_out:
+            if any(matches(f) for f in filter_out):
+                continue
+        yield([description, fragment_paths])