mgr/cephadm: Also catch concurrent.futures.TimeoutError for timeouts

On python 3.6 which Ceph currently uses for its
container builds (which are based on centos 8 stream builds
hence the python version) the exception raised by a timeout
from a concurrent.futures.Future is successfully caught by
looking for asyncio.TimeoutError. However, in builds with
later python versions, e.g. 3.9.16, the timeout is no
longer caught. This results in situations like

Traceback (most recent call last):
  File "/usr/share/ceph/mgr/cephadm/utils.py", line 79, in do_work
    return f(*arg)
  File "/usr/share/ceph/mgr/cephadm/serve.py", line 241, in refresh
    r = self._refresh_host_devices(host)
  File "/usr/share/ceph/mgr/cephadm/serve.py", line 352, in _refresh_host_devices
    devices = self.mgr.wait_async(self._run_cephadm_json(
  File "/usr/share/ceph/mgr/cephadm/module.py", line 635, in wait_async
    return self.event_loop.get_result(coro, timeout)
  File "/usr/share/ceph/mgr/cephadm/ssh.py", line 63, in get_result
    return future.result(timeout)
  File "/lib64/python3.9/concurrent/futures/_base.py", line 448, in result
    raise TimeoutError()
concurrent.futures._base.TimeoutError

which causes the cephadm module to crash whenever one of these
command timeouts happen. This patch is to also catch the
newer exception type so it works on later python versions as well

Signed-off-by: Adam King <adking@redhat.com>
(cherry picked from commit 4df348c17ba0c72d2ecf08d3607b70b3ae253176)

diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py
index b59cf6687f9f46df331ebdb051deef25c7b459d6..5b80862f7502562f5a652be81a964ccbf28df032 100644 (file)
--- a/src/pybind/mgr/cephadm/module.py
+++ b/src/pybind/mgr/cephadm/module.py
@@ -1,4 +1,5 @@
 import asyncio
+import concurrent
 import json
 import errno
 import ipaddress
@@ -732,7 +733,7 @@ class CephadmOrchestrator(orchestrator.Orchestrator, MgrModule,
         # are provided, that will be included in the OrchestratorError's message
         try:
             yield
-        except asyncio.TimeoutError:
+        except (asyncio.TimeoutError, concurrent.futures.TimeoutError):
             err_str: str = ''
             if cmd:
                 err_str = f'Command "{cmd}" timed out '

diff --git a/src/pybind/mgr/cephadm/ssh.py b/src/pybind/mgr/cephadm/ssh.py
index d17cc0fcc1985b7f364e1d3ba6e92630e5a651ca..7460fc159d7b29427da1ca6b98cbaa09bb0af678 100644 (file)
--- a/src/pybind/mgr/cephadm/ssh.py
+++ b/src/pybind/mgr/cephadm/ssh.py
@@ -1,6 +1,7 @@
 import logging
 import os
 import asyncio
+import concurrent
 from tempfile import NamedTemporaryFile
 from threading import Thread
 from contextlib import contextmanager
@@ -61,7 +62,7 @@ class EventLoopThread(Thread):
         future = asyncio.run_coroutine_threadsafe(coro, self._loop)
         try:
             return future.result(timeout)
-        except asyncio.TimeoutError:
+        except (asyncio.TimeoutError, concurrent.futures.TimeoutError):
             # try to cancel the task before raising the exception further up
             future.cancel()
             raise