From 548bde99e68b24e87bedd718cc28ad6ddb062f3f Mon Sep 17 00:00:00 2001 From: Adam King Date: Wed, 7 Jun 2023 10:33:13 -0400 Subject: [PATCH] mgr/cephadm: Also catch concurrent.futures.TimeoutError for timeouts On python 3.6 which Ceph currently uses for its container builds (which are based on centos 8 stream builds hence the python version) the exception raised by a timeout from a concurrent.futures.Future is successfully caught by looking for asyncio.TimeoutError. However, in builds with later python versions, e.g. 3.9.16, the timeout is no longer caught. This results in situations like Traceback (most recent call last): File "/usr/share/ceph/mgr/cephadm/utils.py", line 79, in do_work return f(*arg) File "/usr/share/ceph/mgr/cephadm/serve.py", line 241, in refresh r = self._refresh_host_devices(host) File "/usr/share/ceph/mgr/cephadm/serve.py", line 352, in _refresh_host_devices devices = self.mgr.wait_async(self._run_cephadm_json( File "/usr/share/ceph/mgr/cephadm/module.py", line 635, in wait_async return self.event_loop.get_result(coro, timeout) File "/usr/share/ceph/mgr/cephadm/ssh.py", line 63, in get_result return future.result(timeout) File "/lib64/python3.9/concurrent/futures/_base.py", line 448, in result raise TimeoutError() concurrent.futures._base.TimeoutError which causes the cephadm module to crash whenever one of these command timeouts happen. This patch is to also catch the newer exception type so it works on later python versions as well Signed-off-by: Adam King (cherry picked from commit 4df348c17ba0c72d2ecf08d3607b70b3ae253176) --- src/pybind/mgr/cephadm/module.py | 3 ++- src/pybind/mgr/cephadm/ssh.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py index b59cf6687f9..5b80862f750 100644 --- a/src/pybind/mgr/cephadm/module.py +++ b/src/pybind/mgr/cephadm/module.py @@ -1,4 +1,5 @@ import asyncio +import concurrent import json import errno import ipaddress @@ -732,7 +733,7 @@ class CephadmOrchestrator(orchestrator.Orchestrator, MgrModule, # are provided, that will be included in the OrchestratorError's message try: yield - except asyncio.TimeoutError: + except (asyncio.TimeoutError, concurrent.futures.TimeoutError): err_str: str = '' if cmd: err_str = f'Command "{cmd}" timed out ' diff --git a/src/pybind/mgr/cephadm/ssh.py b/src/pybind/mgr/cephadm/ssh.py index d17cc0fcc19..7460fc159d7 100644 --- a/src/pybind/mgr/cephadm/ssh.py +++ b/src/pybind/mgr/cephadm/ssh.py @@ -1,6 +1,7 @@ import logging import os import asyncio +import concurrent from tempfile import NamedTemporaryFile from threading import Thread from contextlib import contextmanager @@ -61,7 +62,7 @@ class EventLoopThread(Thread): future = asyncio.run_coroutine_threadsafe(coro, self._loop) try: return future.result(timeout) - except asyncio.TimeoutError: + except (asyncio.TimeoutError, concurrent.futures.TimeoutError): # try to cancel the task before raising the exception further up future.cancel() raise -- 2.39.5