dispatcher/supervisor: always unlock machines and save status

If we can't connect to the machines anymore, we still need to clean
up.

Signed-off-by: Josh Durgin <jdurgin@redhat.com>

diff --git a/teuthology/dispatcher/supervisor.py b/teuthology/dispatcher/supervisor.py
index 1e8533efab8c688df70eccd44d04779126de9f62..4a58b2f186707a0f79991b4e19de42bd6d1a6852 100644 (file)
--- a/teuthology/dispatcher/supervisor.py
+++ b/teuthology/dispatcher/supervisor.py
@@ -215,10 +215,17 @@ def run_with_watchdog(process, job_config):
         if total_seconds > teuth_config.max_job_time:
             log.warning("Job ran longer than {max}s. Killing...".format(
                 max=teuth_config.max_job_time))
-            transfer_archives(job_info['name'], job_info['job_id'],
-                              teuth_config.archive_base, job_config)
-            kill_job(job_info['name'], job_info['job_id'],
-                     teuth_config.archive_base, job_config['owner'])
+            try:
+                transfer_archives(job_info['name'], job_info['job_id'],
+                                  teuth_config.archive_base, job_config)
+            except Exception:
+                log.exception('Could not save logs')
+
+            try:
+                kill_job(job_info['name'], job_info['job_id'],
+                         teuth_config.archive_base, job_config['owner'])
+            except Exception:
+                log.exception('Failed to kill job')
 
         # calling this without a status just updates the jobs updated time
         report.try_push_job_info(job_info)