From 4f4227a44d70b54ca4f58ba8a0c342fffc7b8ba5 Mon Sep 17 00:00:00 2001 From: Josh Durgin Date: Thu, 25 Aug 2011 17:11:33 -0700 Subject: [PATCH] Generate coverage at the end of a suite run, and optionally email failures and ongoing jobs. --- setup.py | 1 + teuthology/queue.py | 29 ++++++-- teuthology/run.py | 24 ++++++- teuthology/suite.py | 166 ++++++++++++++++++++++++++++++++++++++++---- 4 files changed, 197 insertions(+), 23 deletions(-) diff --git a/setup.py b/setup.py index 348edd483ae0b..aa975b2498e22 100644 --- a/setup.py +++ b/setup.py @@ -33,6 +33,7 @@ setup( 'teuthology-schedule = teuthology.run:schedule', 'teuthology-updatekeys = teuthology.lock:update_hostkeys', 'teuthology-coverage = teuthology.coverage:analyze', + 'teuthology-results = teuthology.suite:results', ], }, diff --git a/teuthology/queue.py b/teuthology/queue.py index 7614190a0520d..5857c4ca8866a 100644 --- a/teuthology/queue.py +++ b/teuthology/queue.py @@ -72,17 +72,32 @@ describe. One job is run at a time. # bury the job so it won't be re-run if it fails job.bury() + log.debug('Reserved job %d', job.jid) log.debug('Config is: %s', job.body) job_config = yaml.safe_load(job.body) - - log.debug('Creating archive dir...') safe_archive = safepath.munge(job_config['name']) - safepath.makedirs(ctx.archive_dir, safe_archive) - archive_path = os.path.join(ctx.archive_dir, safe_archive, str(job.jid)) - log.info('Running job %d', job.jid) - run_job(job_config, archive_path) - job.delete() + if job_config.get('last_in_suite', False): + log.debug('Generating coverage for %s', job_config['name']) + subprocess.Popen( + args=[ + os.path.join(os.path.dirname(sys.argv[0]), 'teuthology-results'), + '--timeout', + job_config.get('results_timeout', '21600'), + '--email', + job_config['email'], + '--archive-dir', + os.path.join(ctx.archive_dir, safe_archive), + '--name', + job_config['name'], + ]) + else: + log.debug('Creating archive dir...') + safepath.makedirs(ctx.archive_dir, safe_archive) + archive_path = os.path.join(ctx.archive_dir, safe_archive, str(job.jid)) + log.info('Running job %d', job.jid) + run_job(job_config, archive_path) + job.delete() def run_job(job_config, archive_path): arg = [ diff --git a/teuthology/run.py b/teuthology/run.py index f162335d0ce83..b4e53f8fdcab0 100644 --- a/teuthology/run.py +++ b/teuthology/run.py @@ -164,7 +164,7 @@ def schedule(): parser.add_argument( 'config', metavar='CONFFILE', - nargs='+', + nargs='*', type=config_file, action=MergeConfig, default={}, @@ -173,7 +173,22 @@ def schedule(): parser.add_argument( '--name', required=True, - help='job name', + help='name of suite run the job is part of', + ) + parser.add_argument( + '--last-in-suite', + action='store_true', + default=False, + help='mark the last job in a suite so suite post-processing can be run', + ) + parser.add_argument( + '--email', + help='where to send the results of a suite (only applies to the last job in a suite)', + ) + parser.add_argument( + '--timeout', + help='how many seconds to wait for jobs to finish before emailing results (only applies to the last job in a suite', + type=int, ) parser.add_argument( '--description', @@ -191,6 +206,9 @@ def schedule(): ) ctx = parser.parse_args() + if not ctx.last_in_suite: + assert not ctx.email, '--email is only applicable to the last job in a suite' + assert not ctx.timeout, '--timeout is only applicable to the last job in a suite' from teuthology.misc import read_config, get_user if ctx.owner is None: @@ -204,6 +222,8 @@ def schedule(): job = yaml.safe_dump(dict( config=ctx.config, name=ctx.name, + last_in_suite=ctx.last_in_suite, + email=ctx.email, description=ctx.description, owner=ctx.owner, verbose=ctx.verbose, diff --git a/teuthology/suite.py b/teuthology/suite.py index 26953eee93f2f..3372508ccaf6e 100644 --- a/teuthology/suite.py +++ b/teuthology/suite.py @@ -1,10 +1,15 @@ import argparse +import copy import errno import itertools import logging import os import subprocess import sys +import time +import yaml + +from teuthology import misc as teuthology log = logging.getLogger(__name__) @@ -47,6 +52,14 @@ combination, and will override anything in the suite. help='name for this suite', required=True, ) + parser.add_argument( + '--email', + help='address to email test failures to', + ) + parser.add_argument( + '--timeout', + help='how many seconds to wait for jobs to finish before emailing results', + ) parser.add_argument( 'config', metavar='CONFFILE', @@ -78,6 +91,15 @@ combination, and will override anything in the suite. # degenerate case; 'suite' is actually a single collection collections = [(args.suite, 'none')] + base_arg = [ + os.path.join(os.path.dirname(sys.argv[0]), 'teuthology-schedule'), + '--name', args.name, + ] + if args.verbose: + base_arg.append('-v') + if args.owner: + base_arg.extend(['--owner', args.owner]) + for collection, collection_name in sorted(collections): log.info('Collection %s in %s' % (collection_name, collection)) facets = [ @@ -101,22 +123,11 @@ combination, and will override anything in the suite. log.info( 'Running teuthology-schedule with facets %s', description ) - arg = [ - os.path.join(os.path.dirname(sys.argv[0]), 'teuthology-schedule'), - ] - - if args.verbose: - arg.append('-v') - - if args.owner: - arg.extend(['--owner', args.owner]) - + arg = copy.deepcopy(base_arg) arg.extend([ - '--name', args.name, '--description', description, '--', ]) - arg.extend(path for facet, name, path in configs) arg.extend(args.config) print arg @@ -124,6 +135,16 @@ combination, and will override anything in the suite. args=arg, ) + arg = copy.deepcopy(base_arg) + arg.append('--last-in-suite') + if args.email: + arg.extend(['--email', args.email]) + if args.timeout: + arg.extend(['--timeout', args.timeout]) + subprocess.check_call( + args=arg + ) + def ls(): parser = argparse.ArgumentParser(description='List teuthology job results') parser.add_argument( @@ -134,8 +155,6 @@ def ls(): ) args = parser.parse_args() - import yaml - for j in sorted(os.listdir(args.archive_dir)): if j.startswith('.'): continue @@ -159,3 +178,122 @@ def ls(): desc=summary.get('description', '-'), success='pass' if summary['success'] else 'FAIL', ) + +def results(): + parser = argparse.ArgumentParser(description='Email teuthology suite results') + parser.add_argument( + '--email', + help='address to email test failures to', + ) + parser.add_argument( + '--timeout', + help='how many seconds to wait for all tests to finish (default no wait)', + type=int, + default=0, + ) + parser.add_argument( + '--archive-dir', + metavar='DIR', + help='path under which results for the suite are stored', + required=True, + ) + parser.add_argument( + '--name', + help='name of the suite', + required=True, + ) + parser.add_argument( + '-v', '--verbose', + action='store_true', default=False, + help='be more verbose', + ) + args = parser.parse_args() + + loglevel = logging.INFO + if args.verbose: + loglevel = logging.DEBUG + + logging.basicConfig( + level=loglevel, + ) + + teuthology.read_config(args) + + running_tests = [ + f for f in sorted(os.listdir(args.archive_dir)) + if not f.startswith('.') + and not os.path.exists(os.path.join(args.archive_dir, f, 'summary.yaml')) + ] + starttime = time.time() + while running_tests and args.timeout > 0: + if os.path.exists(os.path.join( + args.archive_dir, + running_tests[-1], 'summary.yaml')): + running_tests.pop() + else: + if time.time() - starttime > args.timeout: + log.warn('test(s) did not finish before timeout of %d seconds', + args.timeout) + break + time.sleep(10) + + subprocess.Popen( + args=[ + os.path.join(os.path.dirname(sys.argv[0]), 'teuthology-coverage'), + '-v', + '-o', + os.path.join(args.teuthology_config['coverage_output_dir'], args.name), + '--html-output', + os.path.join(args.teuthology_config['coverage_html_dir'], args.name), + '--cov-tools-dir', + args.teuthology_config['coverage_tools_dir'], + args.archive_dir, + ], + ) + + failures = [] + unfinished = [] + for j in sorted(os.listdir(args.archive_dir)): + if j.startswith('.'): + continue + summary_fn = os.path.join(args.archive_dir, j, 'summary.yaml') + if not os.path.exists(summary_fn): + unfinished.append(j) + continue + summary = {} + with file(summary_fn) as f: + g = yaml.safe_load_all(f) + for new in g: + summary.update(new) + if not summary['success']: + failures.append('{test}: {desc}'.format( + desc=summary['description'], + test=j, + )) + if (not failures and not unfinished) or not args.email: + return + + import smtplib + from email.mime.text import MIMEText + msg = MIMEText(""" +The following tests failed: + +{failures} + +These tests may be hung (did not finish in {timeout} seconds after the last test in the suite): +{unfinished}""".format( + failures='\n'.join(failures), + unfinished='\n'.join(unfinished), + timeout=args.timeout, + )) + msg['Subject'] = '{num_failed} failed and {num_hung} possibly hung tests in {suite}'.format( + num_failed=len(failures), + num_hung=len(unfinished), + suite=args.name, + ) + msg['From'] = args.teuthology_config['results_sending_email'] + msg['To'] = args.email + log.debug('sending email %s', msg.as_string()) + smtp = smtplib.SMTP('localhost') + smtp.sendmail(msg['From'], [msg['To']], msg.as_string()) + smtp.quit() -- 2.39.5