teuthology-suite: add --seed option for repeatable random test

currently --rerun does not match tests of
'supported-random-distro$/ubuntu_latest.yaml' with
'supported-random-distro$/centos_latest.yaml'. the former could be part
of description of a failed test, the latter is a a part of job
description generated by build_matrix(). because the '$' operator
instructs theuthology to choose a random file under the directory ending
with '$', and we expand the '$' to a randomly picked file *before*
filtering the generated job list with the filter collected from the
failed tests, there is good chance that the job descriptions of the
failed jobs in self.args.filter_in cannot match with the randomly
generated ones.

so, we introduce an argument '--seed' for teuthology-suite for the
repeatable random test. this argument allows user to specify a seed for
tne RNG used by build_matrix().

Signed-off-by: Kefu Chai <tchaikov@gmail.com>

diff --git a/scripts/suite.py b/scripts/suite.py
index 89b11902042e5d30c441caca036b15dcc2b3ea40..60b70cf0e398e93865b471a4479cbaba5254397d 100644 (file)
--- a/scripts/suite.py
+++ b/scripts/suite.py
@@ -117,12 +117,17 @@ Scheduler arguments:
                               the original run, the resulting one will only
                               inherit the suite value. Any others must be
                               passed as normal while scheduling with this
-                              feature.
+                              feature. For random tests involving facet whose
+                              path ends with '$' operator, you might want to
+                              use --seed argument to repeat them.
  -R, --rerun-statuses <statuses>
                               A comma-separated list of statuses to be used
                               with --rerun. Supported statuses are: 'dead',
                               'fail', 'pass', 'queued', 'running', 'waiting'
                               [default: fail,dead]
+ --seed SEED                  An random number mostly useful when used along
+                              with --rerun argument. This number can be found
+                              in the output of teuthology-suite command.
 
 """.format(
     default_machine_type=config.default_machine_type,

diff --git a/teuthology/suite/__init__.py b/teuthology/suite/__init__.py
index e89a4fc39975e749666e295c2a0f3de9fe0c3f1f..299f5e0747d2d38309117debf8d11b5578b2b615 100644 (file)
--- a/teuthology/suite/__init__.py
+++ b/teuthology/suite/__init__.py
@@ -4,6 +4,7 @@
 
 import logging
 import os
+import random
 import time
 
 import teuthology
@@ -82,6 +83,9 @@ def main(args):
             return
         conf.filter_in.extend(rerun_filters['descriptions'])
         conf.suite = normalize_suite_name(rerun_filters['suite'])
+    if conf.seed is None:
+        conf.seed = random.randint(0, 9999)
+        log.info('Using random seed=%s', conf.seed)
 
     run = Run(conf)
     name = run.name

diff --git a/teuthology/suite/build_matrix.py b/teuthology/suite/build_matrix.py
index 196a31d4bd58512c767244485eea7c28dd2e88bf..e9692a8667bce474680c06189672d75e93667006 100644 (file)
--- a/teuthology/suite/build_matrix.py
+++ b/teuthology/suite/build_matrix.py
@@ -1,12 +1,13 @@
 import logging
 import os
+import random
 
 from . import matrix
 
 log = logging.getLogger(__name__)
 
 
-def build_matrix(path, subset=None):
+def build_matrix(path, subset=None, seed=None):
     """
     Return a list of items descibed by path such that if the list of
     items is chunked into mincyclicity pieces, each piece is still a
@@ -45,12 +46,14 @@ def build_matrix(path, subset=None):
 
     :param path:        The path to search for yaml fragments
     :param subset:     (index, outof)
+    :param seed:        The seed for repeatable random test
     """
     if subset:
         log.info(
             'Subset=%s/%s' %
             (str(subset[0]), str(subset[1]))
         )
+    random.seed(seed)
     mat, first, matlimit = _get_matrix(path, subset)
     return generate_combinations(path, mat, first, matlimit)
 

diff --git a/teuthology/suite/run.py b/teuthology/suite/run.py
index a56f43e8c1d6784e4e6a839020b14b92d3cda01c..1388fcbd2647010a0537a83cea5ab5d7fe21199f 100644 (file)
--- a/teuthology/suite/run.py
+++ b/teuthology/suite/run.py
@@ -470,7 +470,7 @@ class Run(object):
         log.debug('Suite %s in %s' % (suite_name, suite_path))
         configs = [
             (combine_path(suite_name, item[0]), item[1]) for item in
-            build_matrix(suite_path, subset=self.args.subset)
+            build_matrix(suite_path, subset=self.args.subset, seed=self.args.seed)
         ]
         log.info('Suite %s in %s generated %d jobs (not yet filtered)' % (
             suite_name, suite_path, len(configs)))