From: Kyr Shatskyy Date: Wed, 20 May 2020 16:06:01 +0000 (+0200) Subject: run_tasks: add email notification for sleep-before-teardown X-Git-Tag: 1.1.0~101^2 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=37d89cb3ccf4ec286986ddc2b1652b2a6b7e2727;p=teuthology.git run_tasks: add email notification for sleep-before-teardown Signed-off-by: Kyr Shatskyy --- diff --git a/teuthology/run_tasks.py b/teuthology/run_tasks.py index 527fd8b2dd..099ed31908 100644 --- a/teuthology/run_tasks.py +++ b/teuthology/run_tasks.py @@ -1,15 +1,18 @@ +import jinja2 import logging import os +import six import sys import time import types from copy import deepcopy +from humanfriendly import format_timespan from teuthology.config import config as teuth_config from teuthology.exceptions import ConnectionLostError -from teuthology.job_status import set_status -from teuthology.misc import get_http_log_path +from teuthology.job_status import set_status, get_status +from teuthology.misc import get_http_log_path, get_results_url from teuthology.sentry import get_client as get_sentry_client from teuthology.timer import Timer @@ -159,6 +162,7 @@ def run_tasks(tasks, ctx): 'Sleeping for {} seconds before unwinding because' ' --sleep-before-teardown was given...' .format(sleep_before_teardown)) + notify_sleep_before_teardown(ctx, stack, sleep_before_teardown) time.sleep(sleep_before_teardown) while stack: taskname, manager = stack.pop() @@ -199,3 +203,58 @@ def run_tasks(tasks, ctx): # be careful about cyclic references del exc_info timer.mark("tasks complete") + +def build_email_body(ctx, stack, sleep_time_sec): + email_template_path = os.path.dirname(__file__) + \ + '/templates/email-sleep-before-teardown.jinja2' + + with open(email_template_path, 'r') as f: + template_text = six.ensure_str(f.read()) + + email_template = jinja2.Template(template_text) + archive_path = ctx.config.get('archive_path') + job_id = ctx.config.get('job_id') + status = get_status(ctx.summary) + stack_path = '/'.join(task for task, _ in stack) + suite_name=ctx.config.get('suite') + sleep_date=time.time() + sleep_date_str=time.strftime('%Y-%m-%d %H:%M:%S', + time.gmtime(sleep_date)) + + body = email_template.render( + sleep_time=format_timespan(sleep_time_sec), + sleep_time_sec=sleep_time_sec, + sleep_date=sleep_date_str, + owner=ctx.owner, + run_name=ctx.name, + job_id=ctx.config.get('job_id'), + job_info=get_results_url(ctx.name), + job_logs=get_http_log_path(archive_path, job_id), + suite_name=suite_name, + status=status, + task_stack=stack_path, + ) + subject = ( + 'teuthology job {run}/{job} has fallen asleep at {date}' + .format(run=ctx.name, job=job_id, date=sleep_date_str) + ) + return (subject.strip(), body.strip()) + +def notify_sleep_before_teardown(ctx, stack, sleep_time): + email = ctx.config.get('email', None) + if not email: + # we have no email configured, return silently + return + (subject, body) = build_email_body(ctx, stack, sleep_time) + log.info('Sending no to {to}: {body}'.format(to=email, body=body)) + import smtplib + from email.mime.text import MIMEText + msg = MIMEText(body) + msg['Subject'] = subject + msg['From'] = teuth_config.results_sending_email or 'teuthology' + msg['To'] = email + log.debug('sending email %s', msg.as_string()) + smtp = smtplib.SMTP('localhost') + smtp.sendmail(msg['From'], [msg['To']], msg.as_string()) + smtp.quit() + diff --git a/teuthology/templates/email-sleep-before-teardown.jinja2 b/teuthology/templates/email-sleep-before-teardown.jinja2 new file mode 100644 index 0000000000..9cc054f408 --- /dev/null +++ b/teuthology/templates/email-sleep-before-teardown.jinja2 @@ -0,0 +1,10 @@ +Teuthology job {{ run_name }}/{{ job_id }} has fallen asleep at {{ sleep_date }} for {{ sleep_time }} + +Owner: {{ owner }} +Suite Name: {{ suite_name }} +Sleep Date: {{ sleep_date }} +Sleep Time: {{ sleep_time_sec }} seconds ({{ sleep_time }}) +Job Info: {{ job_info }} +Job Logs: {{ job_logs }} +Task Stack: {{ task_stack }} +Current Status: {{ status }} diff --git a/teuthology/test/test_email_sleep_before_teardown.py b/teuthology/test/test_email_sleep_before_teardown.py new file mode 100644 index 0000000000..fcc83b697d --- /dev/null +++ b/teuthology/test/test_email_sleep_before_teardown.py @@ -0,0 +1,81 @@ +from humanfriendly import format_timespan +from mock import Mock, patch +from pytest import mark +from teuthology.config import config +from teuthology.run_tasks import build_email_body as email_body +from textwrap import dedent + +class TestSleepBeforeTeardownEmail(object): + def setup(self): + config.results_ui_server = "http://example.com/" + config.archive_server = "http://qa-proxy.ceph.com/teuthology/" + + @mark.parametrize( + ['status', 'owner', 'suite_name', 'run_name', 'job_id', 'dura'], + [ + [ + 'pass', + 'noreply@host', + 'dummy', + 'run-name', + 123, + 3600, + ], + [ + 'fail', + 'noname', + 'yummy', + 'next-run', + 1000, + 99999, + ], + ] + ) + @patch("teuthology.run_tasks.time.time") + def test_sleep_before_teardown_email_body(self, m_time, status, owner, + suite_name, run_name, job_id, dura): + ctx = Mock() + archive_path='archive/path' + archive_dir='/archive/dir' + date_sec=3661 + date_str='1970-01-01 01:01:01' + m_time.return_value=float(date_sec) + duration_sec=dura + duration_str=format_timespan(duration_sec) + ref_body=dedent(""" + Teuthology job {run}/{job} has fallen asleep at {date} for {duration_str} + + Owner: {owner} + Suite Name: {suite} + Sleep Date: {date} + Sleep Time: {duration_sec} seconds ({duration_str}) + Job Info: http://example.com/{run}/ + Job Logs: http://qa-proxy.ceph.com/teuthology/path/{job}/ + Task Stack: a/b/c + Current Status: {status}""" + .format(duration_sec=duration_sec, duration_str=duration_str, + owner=owner, suite=suite_name, run=run_name, + job=job_id, status=status, date=date_str)) + print(ref_body) + ctx.config = dict( + archive_path=archive_path, + job_id=job_id, + suite=suite_name, + ) + if status == 'pass': + ctx.summary = dict( + success=True, + ) + elif status == 'fail': + ctx.summary = dict( + success=False, + ) + else: + ctx.summary = dict() + + ctx.owner = owner + ctx.name = run_name + ctx.archive_dir = archive_dir + tasks = [('a', None), ('b', None), ('c', None)] + (subj, body) = email_body(ctx, tasks, dura) + assert body == ref_body.lstrip('\n')