results,schedule,woker: persist --seed and --subset in results.log

to create a repeatable test suite, in addition to `--seed <SEED>`, we also
need to pass the same `--subset <SUBSET>` to teuthology-suite when
rerunning the failed tests. but it would be handy if teuthology-suite
could remember these settings and recall them when `--rerun <RUN>`.

in this change, we repurpose the last job sending the email to report
the test result to note down the subset and seed used for scheduling the
test suite. these variables are stored in results.log at this moment.

Signed-off-by: Kefu Chai <kchai@redhat.com>

diff --git a/scripts/results.py b/scripts/results.py
index 646a7220a80fb0063ceb30c1135f25d66cfe7343..e0e995bef9bdbb51776d66846cfcf7af47d99931 100644 (file)
--- a/scripts/results.py
+++ b/scripts/results.py
@@ -1,5 +1,5 @@
 """
-usage: teuthology-results [-h] [-v] [--dry-run] [--email EMAIL] [--timeout TIMEOUT] --archive-dir DIR --name NAME
+usage: teuthology-results [-h] [-v] [--dry-run] [--email EMAIL] [--timeout TIMEOUT] --archive-dir DIR --name NAME --subset SUBSET --seed SEED
 
 Email teuthology suite results
 
@@ -12,6 +12,8 @@ optional arguments:
                      [default: 0]
   --archive-dir DIR  path under which results for the suite are stored
   --name NAME        name of the suite
+  --subset SUBSET    subset passed to teuthology-suite
+  --seed SEED        random seed used in teuthology-suite
 """
 import docopt
 import teuthology.results

diff --git a/scripts/schedule.py b/scripts/schedule.py
index c6c73dde7a014b1f04d436067006220316284814..35a4f70ae9b1df83661c30b6992a21bc089c6723 100644 (file)
--- a/scripts/schedule.py
+++ b/scripts/schedule.py
@@ -34,6 +34,10 @@ optional arguments:
   --timeout <timeout>                  How many seconds to wait for jobs to
                                        finish before emailing results. Only
                                        applies to the last job in a suite.
+  --seed <seed>                        The random seed for rerunning the suite.
+                                       Only applies to the last job in a suite.
+  --subset <subset>                    The subset option passed to teuthology-suite.
+                                       Only applies to the last job in a suite.
   --dry-run                            Instead of scheduling, just output the
                                        job config.
 

diff --git a/scripts/suite.py b/scripts/suite.py
index 60b70cf0e398e93865b471a4479cbaba5254397d..44b4fac74d697d19910b9979384dc3204250eae5 100644 (file)
--- a/scripts/suite.py
+++ b/scripts/suite.py
@@ -127,7 +127,8 @@ Scheduler arguments:
                               [default: fail,dead]
  --seed SEED                  An random number mostly useful when used along
                               with --rerun argument. This number can be found
-                              in the output of teuthology-suite command.
+                              in the output of teuthology-suite command. -1
+                              for a random seed [default: -1].
 
 """.format(
     default_machine_type=config.default_machine_type,

diff --git a/teuthology/report.py b/teuthology/report.py
index 11d22ed441883390c081103fe70a326243151f52..2ed930631bdb1b02f865ebfe25fc6c5e78351b88 100644 (file)
--- a/teuthology/report.py
+++ b/teuthology/report.py
@@ -381,6 +381,35 @@ class ResultsReporter(object):
         response.raise_for_status()
         return response.json()
 
+    def _parse_log_line(self, line, prefix):
+        msg = line.split(' ', 1)[1].split(':', 2)[-1]
+        if not msg.startswith(prefix):
+            return None
+        else:
+            return msg[len(prefix):]
+
+    def get_rerun_conf(self, run_name):
+        log_path = os.path.join(self.archive_base, run_name, 'results.log')
+        # parse the log file generated by teuthology.results.results()
+        subset = None
+        seed = -1
+        with file(log_path) as results_log:
+            for line in results_log:
+                if ':' not in line:
+                    # stop if this does not look line a log line
+                    break
+                if subset is None:
+                    subset = self._parse_log_line(line, 'subset:')
+                elif seed is None:
+                    seed = self._parse_log_line(line, 'seed:')
+                else:
+                    break
+        if subset is not None:
+            subset = tuple(int(i) for i in subset.split('/'))
+        if seed is not None:
+            seed = int(seed)
+        return subset, seed
+
     def delete_job(self, run_name, job_id):
         """
         Delete a job from the results server.

diff --git a/teuthology/results.py b/teuthology/results.py
index 422aaab3803f881152f84267271ae7e931c1c384..f652a2d9b009924cb05c139d4dd38c0e1e9919e7 100644 (file)
--- a/teuthology/results.py
+++ b/teuthology/results.py
@@ -29,15 +29,20 @@ def main(args):
 
     try:
         results(args['--archive-dir'], args['--name'], args['--email'],
-                int(args['--timeout']), args['--dry-run'])
+                int(args['--timeout']), args['--dry-run'],
+                args['--subset'], args['--seed'])
     except Exception:
         log.exception('error generating results')
         raise
 
 
-def results(archive_dir, name, email, timeout, dry_run):
+def results(archive_dir, name, email, timeout, dry_run, subset, seed):
     starttime = time.time()
 
+    if subset:
+        log.info('subset: %r', subset)
+    if seed:
+        log.info('seed: %r', seed)
     if timeout:
         log.info('Waiting up to %d seconds for tests to finish...', timeout)
 

diff --git a/teuthology/schedule.py b/teuthology/schedule.py
index 35dae1a44f92f72689130d65d385fd6a3aedc0ab..d7762c268c0ab2a98ff9c8d75eba790ca48e6d72 100644 (file)
--- a/teuthology/schedule.py
+++ b/teuthology/schedule.py
@@ -8,12 +8,12 @@ from teuthology import report
 
 def main(args):
     if not args['--last-in-suite']:
-        if args['--email']:
-            raise ValueError(
-                '--email is only applicable to the last job in a suite')
-        if args['--timeout']:
-            raise ValueError(
-                '--timeout is only applicable to the last job in a suite')
+        last_job_args = ['email', 'timeout', 'subset', 'seed']
+        for arg in last_job_args:
+            opt = '--{arg}'.format(arg=arg)
+            msg_fmt = '{opt} is only applicable to the last job in a suite'
+            if args[opt]:
+                raise ValueError(msg_fmt.format(opt=opt))
 
     name = args['--name']
     if not name or name.isdigit():
@@ -56,8 +56,13 @@ def build_config(args):
     # settings in the yaml override what's passed on the command line. This is
     # primarily to accommodate jobs with multiple machine types.
     job_config.update(conf_dict)
-    if args['--timeout'] is not None:
-        job_config['results_timeout'] = args['--timeout']
+    for arg,conf in {'--timeout':'results_timeout',
+                     '--seed': 'seed',
+                     '--subset': 'subset'}.items():
+        val = args.get(arg, None)
+        if val is not None:
+            job_config[conf] = val
+
     return job_config
 
 

diff --git a/teuthology/suite/__init__.py b/teuthology/suite/__init__.py
index 299f5e0747d2d38309117debf8d11b5578b2b615..363c2f123ec3a0aa3c2c4b3df43ef1ff5ec1b36d 100644 (file)
--- a/teuthology/suite/__init__.py
+++ b/teuthology/suite/__init__.py
@@ -38,7 +38,7 @@ def process_args(args):
             value = normalize_suite_name(value)
         if key == 'suite_relpath' and value is None:
             value = ''
-        elif key in ('limit', 'priority', 'num', 'newest'):
+        elif key in ('limit', 'priority', 'num', 'newest', 'seed'):
             value = int(value)
         elif key == 'subset' and value is not None:
             # take input string '2/3' and turn into (2, 3)
@@ -83,7 +83,8 @@ def main(args):
             return
         conf.filter_in.extend(rerun_filters['descriptions'])
         conf.suite = normalize_suite_name(rerun_filters['suite'])
-    if conf.seed is None:
+        conf.subset, conf.seed = get_rerun_conf(conf)
+    if conf.seed < 0:
         conf.seed = random.randint(0, 9999)
         log.info('Using random seed=%s', conf.seed)
 
@@ -108,6 +109,28 @@ def get_rerun_filters(name, statuses):
     return filters
 
 
+def get_rerun_conf(conf):
+    reporter = ResultsReporter()
+    subset, seed = reporter.get_rerun_conf(conf.rerun)
+    if seed < 0:
+        return conf.subset, conf.seed
+    if conf.seed < 0:
+        log.info('Using stored seed=%s', seed)
+    elif conf.seed != seed:
+        log.error('--seed {conf_seed} does not match with ' +
+                  'stored seed: {stored_seed}',
+                  conf_seed=conf.seed,
+                  stored_seed=seed)
+    if conf.subset is None:
+        log.info('Using stored subset=%s', subset)
+    elif conf.subset != subset:
+        log.error('--subset {conf_subset} does not match with ' +
+                  'stored subset: {stored_subset}',
+                  conf_subset=conf.subset,
+                  stored_subset=subset)
+    return subset, seed
+
+
 class WaitException(Exception):
     pass
 

diff --git a/teuthology/suite/run.py b/teuthology/suite/run.py
index 1388fcbd2647010a0537a83cea5ab5d7fe21199f..49d653cacbcb1619d3e4a3fd9e27d8cb975fbcdf 100644 (file)
--- a/teuthology/suite/run.py
+++ b/teuthology/suite/run.py
@@ -308,17 +308,22 @@ class Run(object):
 
         num_jobs = self.schedule_suite()
 
-        if self.base_config.email and num_jobs:
+        if num_jobs:
             arg = copy.deepcopy(self.base_args)
             arg.append('--last-in-suite')
-            arg.extend(['--email', self.base_config.email])
+            if self.base_config.email:
+                arg.extend(['--email', self.base_config.email])
+            if self.args.subset:
+                subset = '/'.join(str(i) for i in self.args.subset)
+                arg.extend(['--subset', subset])
+            arg.extend(['--seed', str(self.args.seed)])
             if self.args.timeout:
                 arg.extend(['--timeout', self.args.timeout])
             util.teuthology_schedule(
                 args=arg,
                 dry_run=self.args.dry_run,
                 verbose=self.args.verbose,
-                log_prefix="Results email: ",
+                log_prefix="Results: ",
             )
             results_url = get_results_url(self.base_config.name)
             if results_url:

diff --git a/teuthology/worker.py b/teuthology/worker.py
index 00bcd23f07204b64fc01d7fb1d369357dcbb013a..4b76de96056f545d2c58ca420680cd9bf135c560 100644 (file)
--- a/teuthology/worker.py
+++ b/teuthology/worker.py
@@ -193,19 +193,23 @@ def run_job(job_config, teuth_bin_path, archive_dir, verbose):
     if job_config.get('last_in_suite'):
         if teuth_config.results_server:
             report.try_delete_jobs(job_config['name'], job_config['job_id'])
-        log.info('Generating results email for %s', job_config['name'])
+        log.info('Generating results for %s', job_config['name'])
         args = [
             os.path.join(teuth_bin_path, 'teuthology-results'),
             '--timeout',
             str(job_config.get('results_timeout',
                                teuth_config.results_timeout)),
-            '--email',
-            job_config['email'],
             '--archive-dir',
             os.path.join(archive_dir, safe_archive),
             '--name',
             job_config['name'],
+            '--seed',
+            job_config['seed'],
         ]
+        if job_config.get('email'):
+            args.extend(['--email', job_config['email']])
+        if job_config.get('subset'):
+            args.extend(['--subset', job_config['subset']])
         # Execute teuthology-results, passing 'preexec_fn=os.setpgrp' to
         # make sure that it will continue to run if this worker process
         # dies (e.g. because of a restart)