Generate coverage at the end of a suite run,

and optionally email failures and ongoing jobs.

diff --git a/setup.py b/setup.py
index 348edd483ae0b6c89f45bb1b27fc29b6f1b15695..aa975b2498e2226bf86098e7ac5cbf6f63cc25f0 100644 (file)
--- a/setup.py
+++ b/setup.py
@@ -33,6 +33,7 @@ setup(
             'teuthology-schedule = teuthology.run:schedule',
             'teuthology-updatekeys = teuthology.lock:update_hostkeys',
             'teuthology-coverage = teuthology.coverage:analyze',
+            'teuthology-results = teuthology.suite:results',
             ],
         },
 

diff --git a/teuthology/queue.py b/teuthology/queue.py
index 7614190a0520da339e3e38e3832302d181aa3bd0..5857c4ca8866a8e1a007d65559510a14a99befa8 100644 (file)
--- a/teuthology/queue.py
+++ b/teuthology/queue.py
@@ -72,17 +72,32 @@ describe. One job is run at a time.
 
         # bury the job so it won't be re-run if it fails
         job.bury()
+        log.debug('Reserved job %d', job.jid)
         log.debug('Config is: %s', job.body)
         job_config = yaml.safe_load(job.body)
-
-        log.debug('Creating archive dir...')
         safe_archive = safepath.munge(job_config['name'])
-        safepath.makedirs(ctx.archive_dir, safe_archive)
-        archive_path = os.path.join(ctx.archive_dir, safe_archive, str(job.jid))
 
-        log.info('Running job %d', job.jid)
-        run_job(job_config, archive_path)
-        job.delete()
+        if job_config.get('last_in_suite', False):
+            log.debug('Generating coverage for %s', job_config['name'])
+            subprocess.Popen(
+                args=[
+                    os.path.join(os.path.dirname(sys.argv[0]), 'teuthology-results'),
+                    '--timeout',
+                    job_config.get('results_timeout', '21600'),
+                    '--email',
+                    job_config['email'],
+                    '--archive-dir',
+                    os.path.join(ctx.archive_dir, safe_archive),
+                    '--name',
+                    job_config['name'],
+                    ])
+        else:
+            log.debug('Creating archive dir...')
+            safepath.makedirs(ctx.archive_dir, safe_archive)
+            archive_path = os.path.join(ctx.archive_dir, safe_archive, str(job.jid))
+            log.info('Running job %d', job.jid)
+            run_job(job_config, archive_path)
+            job.delete()
 
 def run_job(job_config, archive_path):
     arg = [

diff --git a/teuthology/run.py b/teuthology/run.py
index f162335d0ce835769eb48b9cbb9c498d92c06417..b4e53f8fdcab0825264307cbfe13047f963d028d 100644 (file)
--- a/teuthology/run.py
+++ b/teuthology/run.py
@@ -164,7 +164,7 @@ def schedule():
     parser.add_argument(
         'config',
         metavar='CONFFILE',
-        nargs='+',
+        nargs='*',
         type=config_file,
         action=MergeConfig,
         default={},
@@ -173,7 +173,22 @@ def schedule():
     parser.add_argument(
         '--name',
         required=True,
-        help='job name',
+        help='name of suite run the job is part of',
+        )
+    parser.add_argument(
+        '--last-in-suite',
+        action='store_true',
+        default=False,
+        help='mark the last job in a suite so suite post-processing can be run',
+        )
+    parser.add_argument(
+        '--email',
+        help='where to send the results of a suite (only applies to the last job in a suite)',
+        )
+    parser.add_argument(
+        '--timeout',
+        help='how many seconds to wait for jobs to finish before emailing results (only applies to the last job in a suite',
+        type=int,
         )
     parser.add_argument(
         '--description',
@@ -191,6 +206,9 @@ def schedule():
         )
 
     ctx = parser.parse_args()
+    if not ctx.last_in_suite:
+        assert not ctx.email, '--email is only applicable to the last job in a suite'
+        assert not ctx.timeout, '--timeout is only applicable to the last job in a suite'
 
     from teuthology.misc import read_config, get_user
     if ctx.owner is None:
@@ -204,6 +222,8 @@ def schedule():
     job = yaml.safe_dump(dict(
             config=ctx.config,
             name=ctx.name,
+            last_in_suite=ctx.last_in_suite,
+            email=ctx.email,
             description=ctx.description,
             owner=ctx.owner,
             verbose=ctx.verbose,

diff --git a/teuthology/suite.py b/teuthology/suite.py
index 26953eee93f2fe225246b1d63175cc2fc98c1517..3372508ccaf6eb55813f1cc7c79706ef84c17f30 100644 (file)
--- a/teuthology/suite.py
+++ b/teuthology/suite.py
@@ -1,10 +1,15 @@
 import argparse
+import copy
 import errno
 import itertools
 import logging
 import os
 import subprocess
 import sys
+import time
+import yaml
+
+from teuthology import misc as teuthology
 
 log = logging.getLogger(__name__)
 
@@ -47,6 +52,14 @@ combination, and will override anything in the suite.
         help='name for this suite',
         required=True,
         )
+    parser.add_argument(
+        '--email',
+        help='address to email test failures to',
+        )
+    parser.add_argument(
+        '--timeout',
+        help='how many seconds to wait for jobs to finish before emailing results',
+        )
     parser.add_argument(
         'config',
         metavar='CONFFILE',
@@ -78,6 +91,15 @@ combination, and will override anything in the suite.
         # degenerate case; 'suite' is actually a single collection
         collections = [(args.suite, 'none')]
 
+    base_arg = [
+        os.path.join(os.path.dirname(sys.argv[0]), 'teuthology-schedule'),
+        '--name', args.name,
+        ]
+    if args.verbose:
+        base_arg.append('-v')
+    if args.owner:
+        base_arg.extend(['--owner', args.owner])
+
     for collection, collection_name in sorted(collections):
         log.info('Collection %s in %s' % (collection_name, collection))
         facets = [
@@ -101,22 +123,11 @@ combination, and will override anything in the suite.
             log.info(
                 'Running teuthology-schedule with facets %s', description
                 )
-            arg = [
-                os.path.join(os.path.dirname(sys.argv[0]), 'teuthology-schedule'),
-                ]
-
-            if args.verbose:
-                arg.append('-v')
-
-            if args.owner:
-                arg.extend(['--owner', args.owner])
-
+            arg = copy.deepcopy(base_arg)
             arg.extend([
-                    '--name', args.name,
                     '--description', description,
                     '--',
                     ])
-
             arg.extend(path for facet, name, path in configs)
             arg.extend(args.config)
             print arg
@@ -124,6 +135,16 @@ combination, and will override anything in the suite.
                 args=arg,
                 )
 
+    arg = copy.deepcopy(base_arg)
+    arg.append('--last-in-suite')
+    if args.email:
+        arg.extend(['--email', args.email])
+    if args.timeout:
+        arg.extend(['--timeout', args.timeout])
+    subprocess.check_call(
+        args=arg
+        )
+
 def ls():
     parser = argparse.ArgumentParser(description='List teuthology job results')
     parser.add_argument(
@@ -134,8 +155,6 @@ def ls():
         )
     args = parser.parse_args()
 
-    import yaml
-
     for j in sorted(os.listdir(args.archive_dir)):
         if j.startswith('.'):
             continue
@@ -159,3 +178,122 @@ def ls():
             desc=summary.get('description', '-'),
             success='pass' if summary['success'] else 'FAIL',
             )
+
+def results():
+    parser = argparse.ArgumentParser(description='Email teuthology suite results')
+    parser.add_argument(
+        '--email',
+        help='address to email test failures to',
+        )
+    parser.add_argument(
+        '--timeout',
+        help='how many seconds to wait for all tests to finish (default no wait)',
+        type=int,
+        default=0,
+        )
+    parser.add_argument(
+        '--archive-dir',
+        metavar='DIR',
+        help='path under which results for the suite are stored',
+        required=True,
+        )
+    parser.add_argument(
+        '--name',
+        help='name of the suite',
+        required=True,
+        )
+    parser.add_argument(
+        '-v', '--verbose',
+        action='store_true', default=False,
+        help='be more verbose',
+        )
+    args = parser.parse_args()
+
+    loglevel = logging.INFO
+    if args.verbose:
+        loglevel = logging.DEBUG
+
+    logging.basicConfig(
+        level=loglevel,
+        )
+
+    teuthology.read_config(args)
+
+    running_tests = [
+        f for f in sorted(os.listdir(args.archive_dir))
+        if not f.startswith('.')
+        and not os.path.exists(os.path.join(args.archive_dir, f, 'summary.yaml'))
+        ]
+    starttime = time.time()
+    while running_tests and args.timeout > 0:
+        if os.path.exists(os.path.join(
+                args.archive_dir,
+                running_tests[-1], 'summary.yaml')):
+            running_tests.pop()
+        else:
+            if time.time() - starttime > args.timeout:
+                log.warn('test(s) did not finish before timeout of %d seconds',
+                         args.timeout)
+                break
+            time.sleep(10)
+
+    subprocess.Popen(
+        args=[
+            os.path.join(os.path.dirname(sys.argv[0]), 'teuthology-coverage'),
+            '-v',
+            '-o',
+            os.path.join(args.teuthology_config['coverage_output_dir'], args.name),
+            '--html-output',
+            os.path.join(args.teuthology_config['coverage_html_dir'], args.name),
+            '--cov-tools-dir',
+            args.teuthology_config['coverage_tools_dir'],
+            args.archive_dir,
+            ],
+        )
+
+    failures = []
+    unfinished = []
+    for j in sorted(os.listdir(args.archive_dir)):
+        if j.startswith('.'):
+            continue
+        summary_fn = os.path.join(args.archive_dir, j, 'summary.yaml')
+        if not os.path.exists(summary_fn):
+            unfinished.append(j)
+            continue
+        summary = {}
+        with file(summary_fn) as f:
+            g = yaml.safe_load_all(f)
+            for new in g:
+                summary.update(new)
+        if not summary['success']:
+            failures.append('{test}: {desc}'.format(
+                    desc=summary['description'],
+                    test=j,
+                    ))
+    if (not failures and not unfinished) or not args.email:
+        return
+
+    import smtplib
+    from email.mime.text import MIMEText
+    msg = MIMEText("""
+The following tests failed:
+
+{failures}
+
+These tests may be hung (did not finish in {timeout} seconds after the last test in the suite):
+{unfinished}""".format(
+            failures='\n'.join(failures),
+            unfinished='\n'.join(unfinished),
+            timeout=args.timeout,
+            ))
+    msg['Subject'] = '{num_failed} failed and {num_hung} possibly hung tests in {suite}'.format(
+        num_failed=len(failures),
+        num_hung=len(unfinished),
+        suite=args.name,
+        )
+    msg['From'] = args.teuthology_config['results_sending_email']
+    msg['To'] = args.email
+    log.debug('sending email %s', msg.as_string())
+    smtp = smtplib.SMTP('localhost')
+    smtp.sendmail(msg['From'], [msg['To']], msg.as_string())
+    smtp.quit()