From: Zack Cerza Date: Tue, 28 Feb 2023 00:47:33 +0000 (-0700) Subject: supervisor: Ignore some reporting failures X-Git-Tag: 1.2.0~125^2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=refs%2Fpull%2F1816%2Fhead;p=teuthology.git supervisor: Ignore some reporting failures Specifically, the pings coming from the watchdog loop Signed-off-by: Zack Cerza --- diff --git a/teuthology/dispatcher/supervisor.py b/teuthology/dispatcher/supervisor.py index d7a695475..bf1c02528 100644 --- a/teuthology/dispatcher/supervisor.py +++ b/teuthology/dispatcher/supervisor.py @@ -12,7 +12,7 @@ import teuthology from teuthology import report from teuthology import safepath from teuthology.config import config as teuth_config -from teuthology.exceptions import SkipJob +from teuthology.exceptions import SkipJob, MaxWhileTries from teuthology import setup_log_file, install_except_hook from teuthology.lock.ops import reimage_machines from teuthology.misc import get_user, archive_logs, compress_logs @@ -299,7 +299,10 @@ def run_with_watchdog(process, job_config): log.exception('Failed to kill job and unlock machines') # calling this without a status just updates the jobs updated time - report.try_push_job_info(job_info) + try: + report.try_push_job_info(job_info) + except MaxWhileTries: + log.exception("Failed to report job status; ignoring") time.sleep(teuth_config.watchdog_interval) # we no longer support testing theses old branches