From: Josh Durgin Date: Thu, 4 Feb 2021 22:56:53 +0000 (-0500) Subject: dispatcher/supervisor: always unlock machines and save status X-Git-Tag: 1.1.0~14^2 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=1312857b4e1415c8e2749a0d06c489d406fcf30d;p=teuthology.git dispatcher/supervisor: always unlock machines and save status If we can't connect to the machines anymore, we still need to clean up. Signed-off-by: Josh Durgin --- diff --git a/teuthology/dispatcher/supervisor.py b/teuthology/dispatcher/supervisor.py index 1e8533efab..4a58b2f186 100644 --- a/teuthology/dispatcher/supervisor.py +++ b/teuthology/dispatcher/supervisor.py @@ -215,10 +215,17 @@ def run_with_watchdog(process, job_config): if total_seconds > teuth_config.max_job_time: log.warning("Job ran longer than {max}s. Killing...".format( max=teuth_config.max_job_time)) - transfer_archives(job_info['name'], job_info['job_id'], - teuth_config.archive_base, job_config) - kill_job(job_info['name'], job_info['job_id'], - teuth_config.archive_base, job_config['owner']) + try: + transfer_archives(job_info['name'], job_info['job_id'], + teuth_config.archive_base, job_config) + except Exception: + log.exception('Could not save logs') + + try: + kill_job(job_info['name'], job_info['job_id'], + teuth_config.archive_base, job_config['owner']) + except Exception: + log.exception('Failed to kill job') # calling this without a status just updates the jobs updated time report.try_push_job_info(job_info)