From 1312857b4e1415c8e2749a0d06c489d406fcf30d Mon Sep 17 00:00:00 2001 From: Josh Durgin Date: Thu, 4 Feb 2021 17:56:53 -0500 Subject: [PATCH] dispatcher/supervisor: always unlock machines and save status If we can't connect to the machines anymore, we still need to clean up. Signed-off-by: Josh Durgin --- teuthology/dispatcher/supervisor.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/teuthology/dispatcher/supervisor.py b/teuthology/dispatcher/supervisor.py index 1e8533efab..4a58b2f186 100644 --- a/teuthology/dispatcher/supervisor.py +++ b/teuthology/dispatcher/supervisor.py @@ -215,10 +215,17 @@ def run_with_watchdog(process, job_config): if total_seconds > teuth_config.max_job_time: log.warning("Job ran longer than {max}s. Killing...".format( max=teuth_config.max_job_time)) - transfer_archives(job_info['name'], job_info['job_id'], - teuth_config.archive_base, job_config) - kill_job(job_info['name'], job_info['job_id'], - teuth_config.archive_base, job_config['owner']) + try: + transfer_archives(job_info['name'], job_info['job_id'], + teuth_config.archive_base, job_config) + except Exception: + log.exception('Could not save logs') + + try: + kill_job(job_info['name'], job_info['job_id'], + teuth_config.archive_base, job_config['owner']) + except Exception: + log.exception('Failed to kill job') # calling this without a status just updates the jobs updated time report.try_push_job_info(job_info) -- 2.39.5