From: Zack Cerza <zack@cerza.org>
Date: Mon, 17 Mar 2014 16:19:02 +0000 (-0500)
Subject: Fix max_job_time timeout
X-Git-Tag: 1.1.0~1588
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=98e27d7b137228991e84d2d60cd955e0bd9885d7;p=teuthology.git

Fix max_job_time timeout

Signed-off-by: Zack Cerza <zack.cerza@inktank.com>
---

diff --git a/teuthology/queue.py b/teuthology/queue.py
index aad21e6c7..3ce3c0c40 100644
--- a/teuthology/queue.py
+++ b/teuthology/queue.py
@@ -226,13 +226,13 @@ def run_with_watchdog(process, job_config):
     symlink_worker_log(job_config['worker_log'], job_config['archive_path'])
     while process.poll() is None:
         # Kill jobs that have been running longer than the global max
-        job_run_time = datetime.utcnow() - job_start_time
-        if job_run_time.seconds > teuth_config.max_job_time:
+        run_time = datetime.utcnow() - job_start_time
+        total_seconds = run_time.days * 60 * 60 * 24 + run_time.seconds
+        if total_seconds > teuth_config.max_job_time:
             log.warning("Job ran longer than {max}s. Killing...".format(
                 max=teuth_config.max_job_time))
             kill_job(job_info['name'], job_info['job_id'],
                      teuth_config.archive_base)
-            break
 
         report.try_push_job_info(job_info, dict(status='running'))
         time.sleep(teuth_config.watchdog_interval)